mm/slub: move slab initialization into irq enabled region

[zynq/linux.git] / mm / slub.c

diff --git a/mm/slub.c b/mm/slub.c
index 6d7f5165a6345426241278ea7c75979ce57bfe4b..935ad58acde49109bf62aa719d7664f0472b8d38 100644 (file)
--- a/mm/slub.c
+++ b/mm/slub.c
@@ -1314,6 +1314,17 @@ static inline void slab_free_hook(struct kmem_cache *s, void *x)
        kasan_slab_free(s, x);
 }
 
+static void setup_object(struct kmem_cache *s, struct page *page,
+                               void *object)
+{
+       setup_object_debug(s, page, object);
+       if (unlikely(s->ctor)) {
+               kasan_unpoison_object_data(s, object);
+               s->ctor(object);
+               kasan_poison_object_data(s, object);
+       }
+}
+
 /*
  * Slab allocation and freeing
  */
@@ -1344,6 +1355,8 @@ static struct page *allocate_slab(struct kmem_cache *s, gfp_t flags, int node)
        struct page *page;
        struct kmem_cache_order_objects oo = s->oo;
        gfp_t alloc_gfp;
+       void *start, *p;
+       int idx, order;
        bool enableirqs;
 
        flags &= gfp_allowed_mask;
@@ -1372,13 +1385,13 @@ static struct page *allocate_slab(struct kmem_cache *s, gfp_t flags, int node)
                 * Try a lower order alloc if possible
                 */
                page = alloc_slab_page(s, alloc_gfp, node, oo);
-
-               if (page)
-                       stat(s, ORDER_FALLBACK);
+               if (!unlikely(page))
+                       goto out;
+               stat(s, ORDER_FALLBACK);
        }
 
-       if (kmemcheck_enabled && page
-               && !(s->flags & (SLAB_NOTRACK | DEBUG_DEFAULT_FLAGS))) {
+       if (kmemcheck_enabled &&
+               !(s->flags & (SLAB_NOTRACK | DEBUG_DEFAULT_FLAGS))) {
                int pages = 1 << oo_order(oo);
 
                kmemcheck_alloc_shadow(page, oo_order(oo), alloc_gfp, node);
@@ -1393,51 +1406,9 @@ static struct page *allocate_slab(struct kmem_cache *s, gfp_t flags, int node)
                        kmemcheck_mark_unallocated_pages(page, pages);
        }
 
-       if (enableirqs)
-               local_irq_disable();
-       if (!page)
-               return NULL;
-
        page->objects = oo_objects(oo);
-       mod_zone_page_state(page_zone(page),
-               (s->flags & SLAB_RECLAIM_ACCOUNT) ?
-               NR_SLAB_RECLAIMABLE : NR_SLAB_UNRECLAIMABLE,
-               1 << oo_order(oo));
-
-       return page;
-}
-
-static void setup_object(struct kmem_cache *s, struct page *page,
-                               void *object)
-{
-       setup_object_debug(s, page, object);
-       if (unlikely(s->ctor)) {
-               kasan_unpoison_object_data(s, object);
-               s->ctor(object);
-               kasan_poison_object_data(s, object);
-       }
-}
-
-static struct page *new_slab(struct kmem_cache *s, gfp_t flags, int node)
-{
-       struct page *page;
-       void *start;
-       void *p;
-       int order;
-       int idx;
-
-       if (unlikely(flags & GFP_SLAB_BUG_MASK)) {
-               pr_emerg("gfp: %u\n", flags & GFP_SLAB_BUG_MASK);
-               BUG();
-       }
-
-       page = allocate_slab(s,
-               flags & (GFP_RECLAIM_MASK | GFP_CONSTRAINT_MASK), node);
-       if (!page)
-               goto out;
 
        order = compound_order(page);
-       inc_slabs_node(s, page_to_nid(page), page->objects);
        page->slab_cache = s;
        __SetPageSlab(page);
        if (page->pfmemalloc)
@@ -1461,10 +1432,34 @@ static struct page *new_slab(struct kmem_cache *s, gfp_t flags, int node)
        page->freelist = start;
        page->inuse = page->objects;
        page->frozen = 1;
+
 out:
+       if (enableirqs)
+               local_irq_disable();
+       if (!page)
+               return NULL;
+
+       mod_zone_page_state(page_zone(page),
+                           (s->flags & SLAB_RECLAIM_ACCOUNT) ?
+                           NR_SLAB_RECLAIMABLE : NR_SLAB_UNRECLAIMABLE,
+                           1 << oo_order(oo));
+
+       inc_slabs_node(s, page_to_nid(page), page->objects);
+
        return page;
 }
 
+static struct page *new_slab(struct kmem_cache *s, gfp_t flags, int node)
+{
+       if (unlikely(flags & GFP_SLAB_BUG_MASK)) {
+               pr_emerg("gfp: %u\n", flags & GFP_SLAB_BUG_MASK);
+               BUG();
+       }
+
+       return allocate_slab(s,
+                    flags & (GFP_RECLAIM_MASK | GFP_CONSTRAINT_MASK), node);
+}
+
 static void __free_slab(struct kmem_cache *s, struct page *page)
 {
        int order = compound_order(page);