rt-patches/0066-mm-Enable-SLUB-for-RT.patch

   1 From e59760df865009e750972664f06ea565109a58b2 Mon Sep 17 00:00:00 2001
   2 From: Thomas Gleixner <tglx@linutronix.de>
   3 Date: Thu, 25 Oct 2012 10:32:35 +0100
   4 Subject: [PATCH 066/366] mm: Enable SLUB for RT
   5
   6 Make SLUB RT aware by converting locks to raw and using free lists to
   7 move the freeing out of the lock held region.
   8
   9 Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
  10 ---
  11  mm/slab.h |   4 ++
  12  mm/slub.c | 125 ++++++++++++++++++++++++++++++++++++++++++++++++--------------
  13  2 files changed, 102 insertions(+), 27 deletions(-)
  14
  15 diff --git a/mm/slab.h b/mm/slab.h
  16 index 7b60871..afdc579 100644
  17 --- a/mm/slab.h
  18 +++ b/mm/slab.h
  19 @@ -324,7 +324,11 @@ static inline struct kmem_cache *cache_from_obj(struct kmem_cache *s, void *x)
  20   * The slab lists for all objects.
  21   */
  22  struct kmem_cache_node {
  23 +#ifdef CONFIG_SLUB
  24 +       raw_spinlock_t list_lock;
  25 +#else
  26         spinlock_t list_lock;
  27 +#endif
  28
  29  #ifdef CONFIG_SLAB
  30         struct list_head slabs_partial; /* partial list first, better asm code */
  31 diff --git a/mm/slub.c b/mm/slub.c
  32 index 65d5f92..492dbfd 100644
  33 --- a/mm/slub.c
  34 +++ b/mm/slub.c
  35 @@ -1075,7 +1075,7 @@ static noinline struct kmem_cache_node *free_debug_processing(
  36         void *object = head;
  37         int cnt = 0;
  38
  39 -       spin_lock_irqsave(&n->list_lock, *flags);
  40 +       raw_spin_lock_irqsave(&n->list_lock, *flags);
  41         slab_lock(page);
  42
  43         if (!check_slab(s, page))
  44 @@ -1136,7 +1136,7 @@ out:
  45
  46  fail:
  47         slab_unlock(page);
  48 -       spin_unlock_irqrestore(&n->list_lock, *flags);
  49 +       raw_spin_unlock_irqrestore(&n->list_lock, *flags);
  50         slab_fix(s, "Object at 0x%p not freed", object);
  51         return NULL;
  52  }
  53 @@ -1263,6 +1263,12 @@ static inline void dec_slabs_node(struct kmem_cache *s, int node,
  54
  55  #endif /* CONFIG_SLUB_DEBUG */
  56
  57 +struct slub_free_list {
  58 +       raw_spinlock_t          lock;
  59 +       struct list_head        list;
  60 +};
  61 +static DEFINE_PER_CPU(struct slub_free_list, slub_free_list);
  62 +
  63  /*
  64   * Hooks for other subsystems that check memory allocations. In a typical
  65   * production configuration these hooks all should produce no code at all.
  66 @@ -1402,7 +1408,11 @@ static struct page *allocate_slab(struct kmem_cache *s, gfp_t flags, int node)
  67
  68         flags &= gfp_allowed_mask;
  69
  70 +#ifdef CONFIG_PREEMPT_RT_FULL
  71 +       if (system_state == SYSTEM_RUNNING)
  72 +#else
  73         if (gfpflags_allow_blocking(flags))
  74 +#endif
  75                 local_irq_enable();
  76
  77         flags |= s->allocflags;
  78 @@ -1473,7 +1483,11 @@ static struct page *allocate_slab(struct kmem_cache *s, gfp_t flags, int node)
  79         page->frozen = 1;
  80
  81  out:
  82 +#ifdef CONFIG_PREEMPT_RT_FULL
  83 +       if (system_state == SYSTEM_RUNNING)
  84 +#else
  85         if (gfpflags_allow_blocking(flags))
  86 +#endif
  87                 local_irq_disable();
  88         if (!page)
  89                 return NULL;
  90 @@ -1529,6 +1543,16 @@ static void __free_slab(struct kmem_cache *s, struct page *page)
  91         __free_kmem_pages(page, order);
  92  }
  93
  94 +static void free_delayed(struct list_head *h)
  95 +{
  96 +       while(!list_empty(h)) {
  97 +               struct page *page = list_first_entry(h, struct page, lru);
  98 +
  99 +               list_del(&page->lru);
 100 +               __free_slab(page->slab_cache, page);
 101 +       }
 102 +}
 103 +
 104  #define need_reserve_slab_rcu                                          \
 105         (sizeof(((struct page *)NULL)->lru) < sizeof(struct rcu_head))
 106
 107 @@ -1560,6 +1584,12 @@ static void free_slab(struct kmem_cache *s, struct page *page)
 108                 }
 109
 110                 call_rcu(head, rcu_free_slab);
 111 +       } else if (irqs_disabled()) {
 112 +               struct slub_free_list *f = this_cpu_ptr(&slub_free_list);
 113 +
 114 +               raw_spin_lock(&f->lock);
 115 +               list_add(&page->lru, &f->list);
 116 +               raw_spin_unlock(&f->lock);
 117         } else
 118                 __free_slab(s, page);
 119  }
 120 @@ -1673,7 +1703,7 @@ static void *get_partial_node(struct kmem_cache *s, struct kmem_cache_node *n,
 121         if (!n || !n->nr_partial)
 122                 return NULL;
 123
 124 -       spin_lock(&n->list_lock);
 125 +       raw_spin_lock(&n->list_lock);
 126         list_for_each_entry_safe(page, page2, &n->partial, lru) {
 127                 void *t;
 128
 129 @@ -1698,7 +1728,7 @@ static void *get_partial_node(struct kmem_cache *s, struct kmem_cache_node *n,
 130                         break;
 131
 132         }
 133 -       spin_unlock(&n->list_lock);
 134 +       raw_spin_unlock(&n->list_lock);
 135         return object;
 136  }
 137
 138 @@ -1944,7 +1974,7 @@ redo:
 139                          * that acquire_slab() will see a slab page that
 140                          * is frozen
 141                          */
 142 -                       spin_lock(&n->list_lock);
 143 +                       raw_spin_lock(&n->list_lock);
 144                 }
 145         } else {
 146                 m = M_FULL;
 147 @@ -1955,7 +1985,7 @@ redo:
 148                          * slabs from diagnostic functions will not see
 149                          * any frozen slabs.
 150                          */
 151 -                       spin_lock(&n->list_lock);
 152 +                       raw_spin_lock(&n->list_lock);
 153                 }
 154         }
 155
 156 @@ -1990,7 +2020,7 @@ redo:
 157                 goto redo;
 158
 159         if (lock)
 160 -               spin_unlock(&n->list_lock);
 161 +               raw_spin_unlock(&n->list_lock);
 162
 163         if (m == M_FREE) {
 164                 stat(s, DEACTIVATE_EMPTY);
 165 @@ -2022,10 +2052,10 @@ static void unfreeze_partials(struct kmem_cache *s,
 166                 n2 = get_node(s, page_to_nid(page));
 167                 if (n != n2) {
 168                         if (n)
 169 -                               spin_unlock(&n->list_lock);
 170 +                               raw_spin_unlock(&n->list_lock);
 171
 172                         n = n2;
 173 -                       spin_lock(&n->list_lock);
 174 +                       raw_spin_lock(&n->list_lock);
 175                 }
 176
 177                 do {
 178 @@ -2054,7 +2084,7 @@ static void unfreeze_partials(struct kmem_cache *s,
 179         }
 180
 181         if (n)
 182 -               spin_unlock(&n->list_lock);
 183 +               raw_spin_unlock(&n->list_lock);
 184
 185         while (discard_page) {
 186                 page = discard_page;
 187 @@ -2093,14 +2123,21 @@ static void put_cpu_partial(struct kmem_cache *s, struct page *page, int drain)
 188                         pobjects = oldpage->pobjects;
 189                         pages = oldpage->pages;
 190                         if (drain && pobjects > s->cpu_partial) {
 191 +                               struct slub_free_list *f;
 192                                 unsigned long flags;
 193 +                               LIST_HEAD(tofree);
 194                                 /*
 195                                  * partial array is full. Move the existing
 196                                  * set to the per node partial list.
 197                                  */
 198                                 local_irq_save(flags);
 199                                 unfreeze_partials(s, this_cpu_ptr(s->cpu_slab));
 200 +                               f = this_cpu_ptr(&slub_free_list);
 201 +                               raw_spin_lock(&f->lock);
 202 +                               list_splice_init(&f->list, &tofree);
 203 +                               raw_spin_unlock(&f->lock);
 204                                 local_irq_restore(flags);
 205 +                               free_delayed(&tofree);
 206                                 oldpage = NULL;
 207                                 pobjects = 0;
 208                                 pages = 0;
 209 @@ -2172,7 +2209,22 @@ static bool has_cpu_slab(int cpu, void *info)
 210
 211  static void flush_all(struct kmem_cache *s)
 212  {
 213 +       LIST_HEAD(tofree);
 214 +       int cpu;
 215 +
 216         on_each_cpu_cond(has_cpu_slab, flush_cpu_slab, s, 1, GFP_ATOMIC);
 217 +       for_each_online_cpu(cpu) {
 218 +               struct slub_free_list *f;
 219 +
 220 +               if (!has_cpu_slab(cpu, s))
 221 +                       continue;
 222 +
 223 +               f = &per_cpu(slub_free_list, cpu);
 224 +               raw_spin_lock_irq(&f->lock);
 225 +               list_splice_init(&f->list, &tofree);
 226 +               raw_spin_unlock_irq(&f->lock);
 227 +               free_delayed(&tofree);
 228 +       }
 229  }
 230
 231  /*
 232 @@ -2208,10 +2260,10 @@ static unsigned long count_partial(struct kmem_cache_node *n,
 233         unsigned long x = 0;
 234         struct page *page;
 235
 236 -       spin_lock_irqsave(&n->list_lock, flags);
 237 +       raw_spin_lock_irqsave(&n->list_lock, flags);
 238         list_for_each_entry(page, &n->partial, lru)
 239                 x += get_count(page);
 240 -       spin_unlock_irqrestore(&n->list_lock, flags);
 241 +       raw_spin_unlock_irqrestore(&n->list_lock, flags);
 242         return x;
 243  }
 244  #endif /* CONFIG_SLUB_DEBUG || CONFIG_SYSFS */
 245 @@ -2349,8 +2401,10 @@ static inline void *get_freelist(struct kmem_cache *s, struct page *page)
 246   * already disabled (which is the case for bulk allocation).
 247   */
 248  static void *___slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node,
 249 -                         unsigned long addr, struct kmem_cache_cpu *c)
 250 +                         unsigned long addr, struct kmem_cache_cpu *c,
 251 +                         struct list_head *to_free)
 252  {
 253 +       struct slub_free_list *f;
 254         void *freelist;
 255         struct page *page;
 256
 257 @@ -2410,6 +2464,13 @@ load_freelist:
 258         VM_BUG_ON(!c->page->frozen);
 259         c->freelist = get_freepointer(s, freelist);
 260         c->tid = next_tid(c->tid);
 261 +
 262 +out:
 263 +       f = this_cpu_ptr(&slub_free_list);
 264 +       raw_spin_lock(&f->lock);
 265 +       list_splice_init(&f->list, to_free);
 266 +       raw_spin_unlock(&f->lock);
 267 +
 268         return freelist;
 269
 270  new_slab:
 271 @@ -2441,7 +2502,7 @@ new_slab:
 272         deactivate_slab(s, page, get_freepointer(s, freelist));
 273         c->page = NULL;
 274         c->freelist = NULL;
 275 -       return freelist;
 276 +       goto out;
 277  }
 278
 279  /*
 280 @@ -2453,6 +2514,7 @@ static void *__slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node,
 281  {
 282         void *p;
 283         unsigned long flags;
 284 +       LIST_HEAD(tofree);
 285
 286         local_irq_save(flags);
 287  #ifdef CONFIG_PREEMPT
 288 @@ -2464,8 +2526,9 @@ static void *__slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node,
 289         c = this_cpu_ptr(s->cpu_slab);
 290  #endif
 291
 292 -       p = ___slab_alloc(s, gfpflags, node, addr, c);
 293 +       p = ___slab_alloc(s, gfpflags, node, addr, c, &tofree);
 294         local_irq_restore(flags);
 295 +       free_delayed(&tofree);
 296         return p;
 297  }
 298
 299 @@ -2652,7 +2715,7 @@ static void __slab_free(struct kmem_cache *s, struct page *page,
 300
 301         do {
 302                 if (unlikely(n)) {
 303 -                       spin_unlock_irqrestore(&n->list_lock, flags);
 304 +                       raw_spin_unlock_irqrestore(&n->list_lock, flags);
 305                         n = NULL;
 306                 }
 307                 prior = page->freelist;
 308 @@ -2684,7 +2747,7 @@ static void __slab_free(struct kmem_cache *s, struct page *page,
 309                                  * Otherwise the list_lock will synchronize with
 310                                  * other processors updating the list of slabs.
 311                                  */
 312 -                               spin_lock_irqsave(&n->list_lock, flags);
 313 +                               raw_spin_lock_irqsave(&n->list_lock, flags);
 314
 315                         }
 316                 }
 317 @@ -2726,7 +2789,7 @@ static void __slab_free(struct kmem_cache *s, struct page *page,
 318                 add_partial(n, page, DEACTIVATE_TO_TAIL);
 319                 stat(s, FREE_ADD_PARTIAL);
 320         }
 321 -       spin_unlock_irqrestore(&n->list_lock, flags);
 322 +       raw_spin_unlock_irqrestore(&n->list_lock, flags);
 323         return;
 324
 325  slab_empty:
 326 @@ -2741,7 +2804,7 @@ slab_empty:
 327                 remove_full(s, n, page);
 328         }
 329
 330 -       spin_unlock_irqrestore(&n->list_lock, flags);
 331 +       raw_spin_unlock_irqrestore(&n->list_lock, flags);
 332         stat(s, FREE_SLAB);
 333         discard_slab(s, page);
 334  }
 335 @@ -2913,6 +2976,7 @@ int kmem_cache_alloc_bulk(struct kmem_cache *s, gfp_t flags, size_t size,
 336                           void **p)
 337  {
 338         struct kmem_cache_cpu *c;
 339 +       LIST_HEAD(to_free);
 340         int i;
 341
 342         /* memcg and kmem_cache debug support */
 343 @@ -2936,7 +3000,7 @@ int kmem_cache_alloc_bulk(struct kmem_cache *s, gfp_t flags, size_t size,
 344                          * of re-populating per CPU c->freelist
 345                          */
 346                         p[i] = ___slab_alloc(s, flags, NUMA_NO_NODE,
 347 -                                           _RET_IP_, c);
 348 +                                           _RET_IP_, c, &to_free);
 349                         if (unlikely(!p[i]))
 350                                 goto error;
 351
 352 @@ -2948,6 +3012,7 @@ int kmem_cache_alloc_bulk(struct kmem_cache *s, gfp_t flags, size_t size,
 353         }
 354         c->tid = next_tid(c->tid);
 355         local_irq_enable();
 356 +       free_delayed(&to_free);
 357
 358         /* Clear memory outside IRQ disabled fastpath loop */
 359         if (unlikely(flags & __GFP_ZERO)) {
 360 @@ -3095,7 +3160,7 @@ static void
 361  init_kmem_cache_node(struct kmem_cache_node *n)
 362  {
 363         n->nr_partial = 0;
 364 -       spin_lock_init(&n->list_lock);
 365 +       raw_spin_lock_init(&n->list_lock);
 366         INIT_LIST_HEAD(&n->partial);
 367  #ifdef CONFIG_SLUB_DEBUG
 368         atomic_long_set(&n->nr_slabs, 0);
 369 @@ -3677,7 +3742,7 @@ int __kmem_cache_shrink(struct kmem_cache *s, bool deactivate)
 370                 for (i = 0; i < SHRINK_PROMOTE_MAX; i++)
 371                         INIT_LIST_HEAD(promote + i);
 372
 373 -               spin_lock_irqsave(&n->list_lock, flags);
 374 +               raw_spin_lock_irqsave(&n->list_lock, flags);
 375
 376                 /*
 377                  * Build lists of slabs to discard or promote.
 378 @@ -3708,7 +3773,7 @@ int __kmem_cache_shrink(struct kmem_cache *s, bool deactivate)
 379                 for (i = SHRINK_PROMOTE_MAX - 1; i >= 0; i--)
 380                         list_splice(promote + i, &n->partial);
 381
 382 -               spin_unlock_irqrestore(&n->list_lock, flags);
 383 +               raw_spin_unlock_irqrestore(&n->list_lock, flags);
 384
 385                 /* Release empty slabs */
 386                 list_for_each_entry_safe(page, t, &discard, lru)
 387 @@ -3884,6 +3949,12 @@ void __init kmem_cache_init(void)
 388  {
 389         static __initdata struct kmem_cache boot_kmem_cache,
 390                 boot_kmem_cache_node;
 391 +       int cpu;
 392 +
 393 +       for_each_possible_cpu(cpu) {
 394 +               raw_spin_lock_init(&per_cpu(slub_free_list, cpu).lock);
 395 +               INIT_LIST_HEAD(&per_cpu(slub_free_list, cpu).list);
 396 +       }
 397
 398         if (debug_guardpage_minorder())
 399                 slub_max_order = 0;
 400 @@ -4127,7 +4198,7 @@ static int validate_slab_node(struct kmem_cache *s,
 401         struct page *page;
 402         unsigned long flags;
 403
 404 -       spin_lock_irqsave(&n->list_lock, flags);
 405 +       raw_spin_lock_irqsave(&n->list_lock, flags);
 406
 407         list_for_each_entry(page, &n->partial, lru) {
 408                 validate_slab_slab(s, page, map);
 409 @@ -4149,7 +4220,7 @@ static int validate_slab_node(struct kmem_cache *s,
 410                        s->name, count, atomic_long_read(&n->nr_slabs));
 411
 412  out:
 413 -       spin_unlock_irqrestore(&n->list_lock, flags);
 414 +       raw_spin_unlock_irqrestore(&n->list_lock, flags);
 415         return count;
 416  }
 417
 418 @@ -4337,12 +4408,12 @@ static int list_locations(struct kmem_cache *s, char *buf,
 419                 if (!atomic_long_read(&n->nr_slabs))
 420                         continue;
 421
 422 -               spin_lock_irqsave(&n->list_lock, flags);
 423 +               raw_spin_lock_irqsave(&n->list_lock, flags);
 424                 list_for_each_entry(page, &n->partial, lru)
 425                         process_slab(&t, s, page, alloc, map);
 426                 list_for_each_entry(page, &n->full, lru)
 427                         process_slab(&t, s, page, alloc, map);
 428 -               spin_unlock_irqrestore(&n->list_lock, flags);
 429 +               raw_spin_unlock_irqrestore(&n->list_lock, flags);
 430         }
 431
 432         for (i = 0; i < t.count; i++) {
 433 --
 434 1.9.1
 435