Apply preempt_rt patch-4.9-rt1.patch.xz

[zynq/linux.git] / net / core / skbuff.c

diff --git a/net/core/skbuff.c b/net/core/skbuff.c
index 1e3e0087245b713de61f51687fc4e06ea1025b50..1077b39db7174d23e5c655e5771d85c49a229ab8 100644 (file)
--- a/net/core/skbuff.c
+++ b/net/core/skbuff.c
@@ -64,6 +64,7 @@
 #include <linux/errqueue.h>
 #include <linux/prefetch.h>
 #include <linux/if_vlan.h>
+#include <linux/locallock.h>
 
 #include <net/protocol.h>
 #include <net/dst.h>
@@ -360,6 +361,8 @@ struct napi_alloc_cache {
 
 static DEFINE_PER_CPU(struct page_frag_cache, netdev_alloc_cache);
 static DEFINE_PER_CPU(struct napi_alloc_cache, napi_alloc_cache);
+static DEFINE_LOCAL_IRQ_LOCK(netdev_alloc_lock);
+static DEFINE_LOCAL_IRQ_LOCK(napi_alloc_cache_lock);
 
 static void *__netdev_alloc_frag(unsigned int fragsz, gfp_t gfp_mask)
 {
@@ -367,10 +370,10 @@ static void *__netdev_alloc_frag(unsigned int fragsz, gfp_t gfp_mask)
        unsigned long flags;
        void *data;
 
-       local_irq_save(flags);
+       local_lock_irqsave(netdev_alloc_lock, flags);
        nc = this_cpu_ptr(&netdev_alloc_cache);
        data = __alloc_page_frag(nc, fragsz, gfp_mask);
-       local_irq_restore(flags);
+       local_unlock_irqrestore(netdev_alloc_lock, flags);
        return data;
 }
 
@@ -389,9 +392,13 @@ EXPORT_SYMBOL(netdev_alloc_frag);
 
 static void *__napi_alloc_frag(unsigned int fragsz, gfp_t gfp_mask)
 {
-       struct napi_alloc_cache *nc = this_cpu_ptr(&napi_alloc_cache);
+       struct napi_alloc_cache *nc;
+       void *data;
 
-       return __alloc_page_frag(&nc->page, fragsz, gfp_mask);
+       nc = &get_locked_var(napi_alloc_cache_lock, napi_alloc_cache);
+       data = __alloc_page_frag(&nc->page, fragsz, gfp_mask);
+       put_locked_var(napi_alloc_cache_lock, napi_alloc_cache);
+       return data;
 }
 
 void *napi_alloc_frag(unsigned int fragsz)
@@ -438,13 +445,13 @@ struct sk_buff *__netdev_alloc_skb(struct net_device *dev, unsigned int len,
        if (sk_memalloc_socks())
                gfp_mask |= __GFP_MEMALLOC;
 
-       local_irq_save(flags);
+       local_lock_irqsave(netdev_alloc_lock, flags);
 
        nc = this_cpu_ptr(&netdev_alloc_cache);
        data = __alloc_page_frag(nc, len, gfp_mask);
        pfmemalloc = nc->pfmemalloc;
 
-       local_irq_restore(flags);
+       local_unlock_irqrestore(netdev_alloc_lock, flags);
 
        if (unlikely(!data))
                return NULL;
@@ -485,9 +492,10 @@ EXPORT_SYMBOL(__netdev_alloc_skb);
 struct sk_buff *__napi_alloc_skb(struct napi_struct *napi, unsigned int len,
                                 gfp_t gfp_mask)
 {
-       struct napi_alloc_cache *nc = this_cpu_ptr(&napi_alloc_cache);
+       struct napi_alloc_cache *nc;
        struct sk_buff *skb;
        void *data;
+       bool pfmemalloc;
 
        len += NET_SKB_PAD + NET_IP_ALIGN;
 
@@ -505,7 +513,10 @@ struct sk_buff *__napi_alloc_skb(struct napi_struct *napi, unsigned int len,
        if (sk_memalloc_socks())
                gfp_mask |= __GFP_MEMALLOC;
 
+       nc = &get_locked_var(napi_alloc_cache_lock, napi_alloc_cache);
        data = __alloc_page_frag(&nc->page, len, gfp_mask);
+       pfmemalloc = nc->page.pfmemalloc;
+       put_locked_var(napi_alloc_cache_lock, napi_alloc_cache);
        if (unlikely(!data))
                return NULL;
 
@@ -516,7 +527,7 @@ struct sk_buff *__napi_alloc_skb(struct napi_struct *napi, unsigned int len,
        }
 
        /* use OR instead of assignment to avoid clearing of bits in mask */
-       if (nc->page.pfmemalloc)
+       if (pfmemalloc)
                skb->pfmemalloc = 1;
        skb->head_frag = 1;
 
@@ -760,23 +771,26 @@ EXPORT_SYMBOL(consume_skb);
 
 void __kfree_skb_flush(void)
 {
-       struct napi_alloc_cache *nc = this_cpu_ptr(&napi_alloc_cache);
+       struct napi_alloc_cache *nc;
 
+       nc = &get_locked_var(napi_alloc_cache_lock, napi_alloc_cache);
        /* flush skb_cache if containing objects */
        if (nc->skb_count) {
                kmem_cache_free_bulk(skbuff_head_cache, nc->skb_count,
                                     nc->skb_cache);
                nc->skb_count = 0;
        }
+       put_locked_var(napi_alloc_cache_lock, napi_alloc_cache);
 }
 
 static inline void _kfree_skb_defer(struct sk_buff *skb)
 {
-       struct napi_alloc_cache *nc = this_cpu_ptr(&napi_alloc_cache);
+       struct napi_alloc_cache *nc;
 
        /* drop skb->head and call any destructors for packet */
        skb_release_all(skb);
 
+       nc = &get_locked_var(napi_alloc_cache_lock, napi_alloc_cache);
        /* record skb to CPU local list */
        nc->skb_cache[nc->skb_count++] = skb;
 
@@ -791,6 +805,7 @@ static inline void _kfree_skb_defer(struct sk_buff *skb)
                                     nc->skb_cache);
                nc->skb_count = 0;
        }
+       put_locked_var(napi_alloc_cache_lock, napi_alloc_cache);
 }
 void __kfree_skb_defer(struct sk_buff *skb)
 {