drivers/video/tegra/nvmap/nvmap_pp.c

   1 /*
   2  * drivers/video/tegra/nvmap/nvmap_pp.c
   3  *
   4  * Manage page pools to speed up page allocation.
   5  *
   6  * Copyright (c) 2009-2014, NVIDIA CORPORATION. All rights reserved.
   7  *
   8  * This program is free software; you can redistribute it and/or modify
   9  * it under the terms of the GNU General Public License as published by
  10  * the Free Software Foundation; either version 2 of the License, or
  11  * (at your option) any later version.
  12  *
  13  * This program is distributed in the hope that it will be useful, but WITHOUT
  14  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  15  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  16  * more details.
  17  *
  18  * You should have received a copy of the GNU General Public License along
  19  * with this program; if not, write to the Free Software Foundation, Inc.,
  20  * 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
  21  */
  22
  23 #define pr_fmt(fmt) "%s: " fmt, __func__
  24
  25 #include <linux/kernel.h>
  26 #include <linux/vmalloc.h>
  27 #include <linux/moduleparam.h>
  28 #include <linux/shrinker.h>
  29 #include <linux/kthread.h>
  30 #include <linux/debugfs.h>
  31 #include <linux/freezer.h>
  32 #include <linux/highmem.h>
  33
  34 #include "nvmap_priv.h"
  35
  36 #define NVMAP_TEST_PAGE_POOL_SHRINKER     1
  37 #define PENDING_PAGES_SIZE                (SZ_1M / PAGE_SIZE)
  38
  39 static bool enable_pp = 1;
  40 static int pool_size;
  41 static u32 fill_thresh = NVMAP_PP_DEF_FILL_THRESH;
  42 static u32 zero_mem_fill_min = NVMAP_PP_DEF_ZERO_MEM_FILL_MIN;
  43 static u32 min_available_mb = NVMAP_PP_DEF_MIN_AVAILABLE_MB;
  44
  45 module_param(fill_thresh, uint, 0644);
  46 module_param(zero_mem_fill_min, uint, 0644);
  47 module_param(min_available_mb, uint, 0644);
  48
  49 static struct task_struct *background_allocator;
  50 static DECLARE_WAIT_QUEUE_HEAD(nvmap_bg_wait);
  51
  52 #ifdef CONFIG_NVMAP_PAGE_POOL_DEBUG
  53 static inline void __pp_dbg_var_add(u64 *dbg_var, u32 nr)
  54 {
  55         *dbg_var += nr;
  56 }
  57 #else
  58 #define __pp_dbg_var_add(dbg_var, nr)
  59 #endif
  60
  61 #define pp_alloc_add(pool, nr) __pp_dbg_var_add(&(pool)->allocs, nr)
  62 #define pp_fill_add(pool, nr)  __pp_dbg_var_add(&(pool)->fills, nr)
  63 #define pp_hit_add(pool, nr)   __pp_dbg_var_add(&(pool)->hits, nr)
  64 #define pp_miss_add(pool, nr)  __pp_dbg_var_add(&(pool)->misses, nr)
  65
  66 static int __nvmap_page_pool_fill_lots_locked(struct nvmap_page_pool *pool,
  67                                        struct page **pages, u32 nr);
  68
  69 /*
  70  * Make sure any data in the caches is cleaned out before
  71  * passing these pages to userspace. otherwise, It can lead to
  72  * corruption in pages that get mapped as something
  73  * other than WB in userspace and leaked kernel data.
  74  *
  75  * Must be called with pool->lock held.
  76  */
  77 static void pp_clean_cache(struct nvmap_page_pool *pool)
  78 {
  79         struct page *page;
  80         u32 dirty_pages = pool->dirty_pages;
  81
  82         if (!dirty_pages)
  83                 return;
  84         if (dirty_pages >= (cache_maint_inner_threshold >> PAGE_SHIFT)) {
  85                 inner_clean_cache_all();
  86                 outer_clean_all();
  87         } else {
  88                 list_for_each_entry_reverse(page, &pool->page_list, lru) {
  89                         nvmap_clean_cache_page(page);
  90                         dirty_pages--;
  91                         if (!dirty_pages)
  92                                 break;
  93                 }
  94                 BUG_ON(dirty_pages);
  95         }
  96         pool->dirty_pages = 0;
  97 }
  98
  99 static inline struct page *get_zero_list_page(struct nvmap_page_pool *pool)
 100 {
 101         struct page *page;
 102
 103         if (list_empty(&pool->zero_list))
 104                 return NULL;
 105
 106         page = list_first_entry(&pool->zero_list, struct page, lru);
 107         list_del(&page->lru);
 108
 109         pool->to_zero--;
 110
 111         return page;
 112 }
 113
 114 static inline struct page *get_page_list_page(struct nvmap_page_pool *pool)
 115 {
 116         struct page *page;
 117
 118         if (list_empty(&pool->page_list))
 119                 return NULL;
 120
 121         page = list_first_entry(&pool->page_list, struct page, lru);
 122         list_del(&page->lru);
 123
 124         pool->count--;
 125
 126         return page;
 127 }
 128
 129 static inline bool nvmap_bg_should_run(struct nvmap_page_pool *pool)
 130 {
 131         bool ret;
 132
 133         mutex_lock(&pool->lock);
 134         ret = (pool->to_zero > 0);
 135         mutex_unlock(&pool->lock);
 136
 137         return ret;
 138 }
 139
 140 static int nvmap_pp_zero_pages(struct page **pages, int nr)
 141 {
 142         int i;
 143
 144         for (i = 0; i < nr; i++)
 145                 clear_highpage(pages[i]);
 146
 147         return 0;
 148 }
 149
 150 static void nvmap_pp_do_background_zero_pages(struct nvmap_page_pool *pool)
 151 {
 152         int i;
 153         struct page *page;
 154         int ret;
 155
 156         /*
 157          * Statically declared array of pages to be zeroed in a batch,
 158          * local to this thread but too big for the stack.
 159          */
 160         static struct page *pending_zero_pages[PENDING_PAGES_SIZE];
 161
 162         mutex_lock(&pool->lock);
 163         for (i = 0; i < PENDING_PAGES_SIZE; i++) {
 164                 page = get_zero_list_page(pool);
 165                 if (page == NULL)
 166                         break;
 167                 pending_zero_pages[i] = page;
 168         }
 169         mutex_unlock(&pool->lock);
 170
 171         ret = nvmap_pp_zero_pages(pending_zero_pages, i);
 172         if (ret < 0) {
 173                 ret = 0;
 174                 goto out;
 175         }
 176
 177         mutex_lock(&pool->lock);
 178         ret = __nvmap_page_pool_fill_lots_locked(pool, pending_zero_pages, i);
 179         mutex_unlock(&pool->lock);
 180
 181 out:
 182         for (; ret < i; ret++)
 183                 __free_page(pending_zero_pages[ret]);
 184 }
 185
 186 /*
 187  * This thread fills the page pools with zeroed pages. We avoid releasing the
 188  * pages directly back into the page pools since we would then have to zero
 189  * them ourselves. Instead it is easier to just reallocate zeroed pages. This
 190  * happens in the background so that the overhead of allocating zeroed pages is
 191  * not directly seen by userspace. Of course if the page pools are empty user
 192  * space will suffer.
 193  */
 194 static int nvmap_background_zero_thread(void *arg)
 195 {
 196         struct nvmap_page_pool *pool = &nvmap_dev->pool;
 197         struct sched_param param = { .sched_priority = 0 };
 198
 199         pr_info("PP zeroing thread starting.\n");
 200
 201         set_freezable();
 202         sched_setscheduler(current, SCHED_IDLE, &param);
 203
 204         while (!kthread_should_stop()) {
 205                 while (nvmap_bg_should_run(pool))
 206                         nvmap_pp_do_background_zero_pages(pool);
 207
 208                 /* clean cache in the background so that allocations immediately
 209                  * after fill don't suffer the cache clean overhead.
 210                  */
 211                 if (pool->dirty_pages >=
 212                     (cache_maint_inner_threshold >> PAGE_SHIFT)) {
 213                         mutex_lock(&pool->lock);
 214                         pp_clean_cache(pool);
 215                         mutex_unlock(&pool->lock);
 216                 }
 217                 wait_event_freezable(nvmap_bg_wait,
 218                                 nvmap_bg_should_run(pool) ||
 219                                 kthread_should_stop());
 220         }
 221
 222         return 0;
 223 }
 224
 225 /*
 226  * This removes a page from the page pool. If ignore_disable is set, then
 227  * the enable_pp flag is ignored.
 228  */
 229 static struct page *nvmap_page_pool_alloc_locked(struct nvmap_page_pool *pool,
 230                                                  int force_alloc)
 231 {
 232         struct page *page;
 233
 234         if (!force_alloc && !enable_pp)
 235                 return NULL;
 236
 237         if (list_empty(&pool->page_list)) {
 238                 pp_miss_add(pool, 1);
 239                 return NULL;
 240         }
 241
 242         if (IS_ENABLED(CONFIG_NVMAP_PAGE_POOL_DEBUG))
 243                 BUG_ON(pool->count == 0);
 244
 245         pp_clean_cache(pool);
 246         page = get_page_list_page(pool);
 247         if (!page)
 248                 return NULL;
 249
 250         /* Sanity check. */
 251         if (IS_ENABLED(CONFIG_NVMAP_PAGE_POOL_DEBUG)) {
 252                 atomic_dec(&page->_count);
 253                 BUG_ON(atomic_read(&page->_count) != 1);
 254         }
 255
 256         pp_alloc_add(pool, 1);
 257         pp_hit_add(pool, 1);
 258
 259         return page;
 260 }
 261
 262 /*
 263  * Alloc a bunch of pages from the page pool. This will alloc as many as it can
 264  * and return the number of pages allocated. Pages are placed into the passed
 265  * array in a linear fashion starting from index 0.
 266  */
 267 int nvmap_page_pool_alloc_lots(struct nvmap_page_pool *pool,
 268                                 struct page **pages, u32 nr)
 269 {
 270         u32 real_nr;
 271         u32 ind = 0;
 272
 273         if (!enable_pp)
 274                 return 0;
 275
 276         mutex_lock(&pool->lock);
 277         pp_clean_cache(pool);
 278
 279         real_nr = min_t(u32, nr, pool->count);
 280
 281         while (real_nr--) {
 282                 struct page *page;
 283                 if (IS_ENABLED(CONFIG_NVMAP_PAGE_POOL_DEBUG))
 284                         BUG_ON(list_empty(&pool->page_list));
 285                 page = get_page_list_page(pool);
 286                 pages[ind++] = page;
 287                 if (IS_ENABLED(CONFIG_NVMAP_PAGE_POOL_DEBUG)) {
 288                         atomic_dec(&page->_count);
 289                         BUG_ON(atomic_read(&page->_count) != 1);
 290                 }
 291         }
 292         mutex_unlock(&pool->lock);
 293
 294         pp_alloc_add(pool, ind);
 295         pp_hit_add(pool, ind);
 296         pp_miss_add(pool, nr - ind);
 297
 298         return ind;
 299 }
 300
 301 /*
 302  * Fill a bunch of pages into the page pool. This will fill as many as it can
 303  * and return the number of pages filled. Pages are used from the start of the
 304  * passed page pointer array in a linear fashion.
 305  *
 306  * You must lock the page pool before using this.
 307  */
 308 static int __nvmap_page_pool_fill_lots_locked(struct nvmap_page_pool *pool,
 309                                        struct page **pages, u32 nr)
 310 {
 311         u32 real_nr;
 312         u32 ind = 0;
 313
 314         if (!enable_pp)
 315                 return 0;
 316
 317         real_nr = min_t(u32, pool->max - pool->count, nr);
 318         if (real_nr == 0)
 319                 return 0;
 320
 321         while (real_nr--) {
 322                 if (IS_ENABLED(CONFIG_NVMAP_PAGE_POOL_DEBUG)) {
 323                         atomic_inc(&pages[ind]->_count);
 324                         BUG_ON(atomic_read(&pages[ind]->_count) != 2);
 325                 }
 326                 list_add_tail(&pages[ind++]->lru, &pool->page_list);
 327         }
 328
 329         pool->dirty_pages += ind;
 330         pool->count += ind;
 331         pp_fill_add(pool, ind);
 332
 333         return ind;
 334 }
 335
 336 int nvmap_page_pool_fill_lots(struct nvmap_page_pool *pool,
 337                                        struct page **pages, u32 nr)
 338 {
 339         int ret;
 340
 341         mutex_lock(&pool->lock);
 342         if (zero_memory) {
 343                 int i;
 344
 345                 nr = min(nr, pool->max - pool->count - pool->to_zero);
 346
 347                 for (i = 0; i < nr; i++) {
 348                         list_add_tail(&pages[i]->lru, &pool->zero_list);
 349                         pool->to_zero++;
 350                 }
 351
 352                 wake_up_interruptible(&nvmap_bg_wait);
 353
 354                 ret = i;
 355         } else {
 356                 ret = __nvmap_page_pool_fill_lots_locked(pool, pages, nr);
 357         }
 358         mutex_unlock(&pool->lock);
 359
 360         return ret;
 361 }
 362
 363 /*
 364  * Free the passed number of pages from the page pool. This happen irregardless
 365  * of whether ther page pools are enabled. This lets one disable the page pools
 366  * and then free all the memory therein.
 367  */
 368 static int nvmap_page_pool_free(struct nvmap_page_pool *pool, int nr_free)
 369 {
 370         int i = nr_free;
 371         struct page *page;
 372
 373         if (!nr_free)
 374                 return nr_free;
 375
 376         mutex_lock(&pool->lock);
 377         while (i) {
 378                 page = get_zero_list_page(pool);
 379                 if (!page)
 380                         page = nvmap_page_pool_alloc_locked(pool, 1);
 381                 if (!page)
 382                         break;
 383                 __free_page(page);
 384                 i--;
 385         }
 386         mutex_unlock(&pool->lock);
 387
 388         return i;
 389 }
 390
 391 ulong nvmap_page_pool_get_unused_pages(void)
 392 {
 393         int total = 0;
 394
 395         if (!nvmap_dev)
 396                 return 0;
 397
 398         total = nvmap_dev->pool.count + nvmap_dev->pool.to_zero;
 399
 400         return total;
 401 }
 402
 403 /*
 404  * Remove and free to the system all the pages currently in the page
 405  * pool. This operation will happen even if the page pools are disabled.
 406  */
 407 int nvmap_page_pool_clear(void)
 408 {
 409         struct page *page;
 410         struct nvmap_page_pool *pool = &nvmap_dev->pool;
 411
 412         mutex_lock(&pool->lock);
 413
 414         while ((page = nvmap_page_pool_alloc_locked(pool, 1)) != NULL)
 415                 __free_page(page);
 416
 417         while (!list_empty(&pool->zero_list)) {
 418                 page = get_zero_list_page(pool);
 419                 __free_page(page);
 420         }
 421
 422         /* For some reason, if an error occured... */
 423         if (!list_empty(&pool->page_list) || !list_empty(&pool->zero_list)) {
 424                 mutex_unlock(&pool->lock);
 425                 return -ENOMEM;
 426         }
 427
 428         mutex_unlock(&pool->lock);
 429
 430         return 0;
 431 }
 432
 433 /*
 434  * Resizes the page pool to the passed size. If the passed size is 0 then
 435  * all associated resources are released back to the system. This operation
 436  * will only occur if the page pools are enabled.
 437  */
 438 static void nvmap_page_pool_resize(struct nvmap_page_pool *pool, int size)
 439 {
 440         if (!enable_pp || size == pool->max || size < 0)
 441                 return;
 442
 443         mutex_lock(&pool->lock);
 444
 445         while (pool->count > size)
 446                 __free_page(nvmap_page_pool_alloc_locked(pool, 0));
 447
 448         pool->max = size;
 449
 450         pr_debug("page pool resized to %d from %d pages\n", size, pool->max);
 451         mutex_unlock(&pool->lock);
 452 }
 453
 454 static int nvmap_page_pool_shrink(struct shrinker *shrinker,
 455                                   struct shrink_control *sc)
 456 {
 457         int shrink_pages = sc->nr_to_scan;
 458
 459         if (!shrink_pages)
 460                 goto out;
 461
 462         pr_debug("sh_pages=%d", shrink_pages);
 463
 464         shrink_pages = nvmap_page_pool_free(&nvmap_dev->pool, shrink_pages);
 465 out:
 466         return nvmap_page_pool_get_unused_pages();
 467 }
 468
 469 static struct shrinker nvmap_page_pool_shrinker = {
 470         .shrink = nvmap_page_pool_shrink,
 471         .seeks = 1,
 472 };
 473
 474 static void shrink_page_pools(int *total_pages, int *available_pages)
 475 {
 476         struct shrink_control sc;
 477
 478         if (*total_pages == 0) {
 479                 sc.gfp_mask = GFP_KERNEL;
 480                 sc.nr_to_scan = 0;
 481                 *total_pages = nvmap_page_pool_shrink(NULL, &sc);
 482         }
 483         sc.nr_to_scan = *total_pages;
 484         *available_pages = nvmap_page_pool_shrink(NULL, &sc);
 485 }
 486
 487 #if NVMAP_TEST_PAGE_POOL_SHRINKER
 488 static int shrink_pp;
 489 static int shrink_set(const char *arg, const struct kernel_param *kp)
 490 {
 491         unsigned long long t1, t2;
 492         int total_pages, available_pages;
 493
 494         param_set_int(arg, kp);
 495
 496         if (shrink_pp) {
 497                 total_pages = shrink_pp;
 498                 t1 = local_clock();
 499                 shrink_page_pools(&total_pages, &available_pages);
 500                 t2 = local_clock();
 501                 pr_debug("shrink page pools: time=%lldns, "
 502                         "total_pages_released=%d, free_pages_available=%d",
 503                         t2-t1, total_pages, available_pages);
 504         }
 505         return 0;
 506 }
 507
 508 static int shrink_get(char *buff, const struct kernel_param *kp)
 509 {
 510         return param_get_int(buff, kp);
 511 }
 512
 513 static struct kernel_param_ops shrink_ops = {
 514         .get = shrink_get,
 515         .set = shrink_set,
 516 };
 517
 518 module_param_cb(shrink_page_pools, &shrink_ops, &shrink_pp, 0644);
 519 #endif
 520
 521 static int enable_pp_set(const char *arg, const struct kernel_param *kp)
 522 {
 523         int ret;
 524
 525         ret = param_set_bool(arg, kp);
 526         if (ret)
 527                 return ret;
 528
 529         if (!enable_pp)
 530                 nvmap_page_pool_clear();
 531
 532         return 0;
 533 }
 534
 535 static int enable_pp_get(char *buff, const struct kernel_param *kp)
 536 {
 537         return param_get_int(buff, kp);
 538 }
 539
 540 static struct kernel_param_ops enable_pp_ops = {
 541         .get = enable_pp_get,
 542         .set = enable_pp_set,
 543 };
 544
 545 module_param_cb(enable_page_pools, &enable_pp_ops, &enable_pp, 0644);
 546
 547 static int pool_size_set(const char *arg, const struct kernel_param *kp)
 548 {
 549         param_set_int(arg, kp);
 550         nvmap_page_pool_resize(&nvmap_dev->pool, pool_size);
 551         return 0;
 552 }
 553
 554 static int pool_size_get(char *buff, const struct kernel_param *kp)
 555 {
 556         return param_get_int(buff, kp);
 557 }
 558
 559 static struct kernel_param_ops pool_size_ops = {
 560         .get = pool_size_get,
 561         .set = pool_size_set,
 562 };
 563
 564 module_param_cb(pool_size, &pool_size_ops, &pool_size, 0644);
 565
 566 int nvmap_page_pool_debugfs_init(struct dentry *nvmap_root)
 567 {
 568         struct dentry *pp_root;
 569
 570         if (!nvmap_root)
 571                 return -ENODEV;
 572
 573         pp_root = debugfs_create_dir("pagepool", nvmap_root);
 574         if (!pp_root)
 575                 return -ENODEV;
 576
 577         debugfs_create_u32("page_pool_available_pages",
 578                            S_IRUGO, pp_root,
 579                            &nvmap_dev->pool.count);
 580         debugfs_create_u32("page_pool_pages_to_zero",
 581                            S_IRUGO, pp_root,
 582                            &nvmap_dev->pool.to_zero);
 583 #ifdef CONFIG_NVMAP_PAGE_POOL_DEBUG
 584         debugfs_create_u64("page_pool_allocs",
 585                            S_IRUGO, pp_root,
 586                            &nvmap_dev->pool.allocs);
 587         debugfs_create_u64("page_pool_fills",
 588                            S_IRUGO, pp_root,
 589                            &nvmap_dev->pool.fills);
 590         debugfs_create_u64("page_pool_hits",
 591                            S_IRUGO, pp_root,
 592                            &nvmap_dev->pool.hits);
 593         debugfs_create_u64("page_pool_misses",
 594                            S_IRUGO, pp_root,
 595                            &nvmap_dev->pool.misses);
 596 #endif
 597
 598         return 0;
 599 }
 600
 601 int nvmap_page_pool_init(struct nvmap_device *dev)
 602 {
 603         struct sysinfo info;
 604         struct nvmap_page_pool *pool = &dev->pool;
 605
 606         memset(pool, 0x0, sizeof(*pool));
 607         mutex_init(&pool->lock);
 608         INIT_LIST_HEAD(&pool->page_list);
 609         INIT_LIST_HEAD(&pool->zero_list);
 610
 611         si_meminfo(&info);
 612         pr_info("Total RAM pages: %lu\n", info.totalram);
 613
 614         if (!CONFIG_NVMAP_PAGE_POOL_SIZE)
 615                 /* The ratio is pool pages per 1K ram pages.
 616                  * So, the >> 10 */
 617                 pool->max = (info.totalram * NVMAP_PP_POOL_SIZE) >> 10;
 618         else
 619                 pool->max = CONFIG_NVMAP_PAGE_POOL_SIZE;
 620
 621         if (pool->max >= info.totalram)
 622                 goto fail;
 623         pool_size = pool->max;
 624
 625         pr_info("nvmap page pool size: %u pages (%u MB)\n", pool->max,
 626                 (pool->max * info.mem_unit) >> 20);
 627
 628         background_allocator = kthread_run(nvmap_background_zero_thread,
 629                                             NULL, "nvmap-bz");
 630         if (IS_ERR(background_allocator))
 631                 goto fail;
 632
 633         register_shrinker(&nvmap_page_pool_shrinker);
 634
 635         return 0;
 636 fail:
 637         nvmap_page_pool_fini(dev);
 638         return -ENOMEM;
 639 }
 640
 641 int nvmap_page_pool_fini(struct nvmap_device *dev)
 642 {
 643         struct nvmap_page_pool *pool = &dev->pool;
 644
 645         /*
 646          * if background allocator is not initialzed or not
 647          * properly initialized, then shrinker is also not
 648          * registered
 649          */
 650         if (!IS_ERR_OR_NULL(background_allocator)) {
 651                 unregister_shrinker(&nvmap_page_pool_shrinker);
 652                 kthread_stop(background_allocator);
 653         }
 654
 655         WARN_ON(!list_empty(&pool->page_list));
 656
 657         return 0;
 658 }