drivers/video/tegra/nvmap/nvmap_pp.c

   1 /*
   2  * drivers/video/tegra/nvmap/nvmap_pp.c
   3  *
   4  * Manage page pools to speed up page allocation.
   5  *
   6  * Copyright (c) 2009-2014, NVIDIA CORPORATION. All rights reserved.
   7  *
   8  * This program is free software; you can redistribute it and/or modify
   9  * it under the terms of the GNU General Public License as published by
  10  * the Free Software Foundation; either version 2 of the License, or
  11  * (at your option) any later version.
  12  *
  13  * This program is distributed in the hope that it will be useful, but WITHOUT
  14  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  15  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  16  * more details.
  17  *
  18  * You should have received a copy of the GNU General Public License along
  19  * with this program; if not, write to the Free Software Foundation, Inc.,
  20  * 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
  21  */
  22
  23 #define pr_fmt(fmt) "%s: " fmt, __func__
  24
  25 #include <linux/kernel.h>
  26 #include <linux/vmalloc.h>
  27 #include <linux/moduleparam.h>
  28 #include <linux/shrinker.h>
  29 #include <linux/kthread.h>
  30 #include <linux/debugfs.h>
  31 #include <linux/freezer.h>
  32 #include <linux/highmem.h>
  33
  34 #include "nvmap_priv.h"
  35
  36 #define NVMAP_TEST_PAGE_POOL_SHRINKER     1
  37 #define PENDING_PAGES_SIZE                (SZ_1M / PAGE_SIZE)
  38
  39 static bool enable_pp = 1;
  40 static int pool_size;
  41
  42 static struct task_struct *background_allocator;
  43 static DECLARE_WAIT_QUEUE_HEAD(nvmap_bg_wait);
  44
  45 #ifdef CONFIG_NVMAP_PAGE_POOL_DEBUG
  46 static inline void __pp_dbg_var_add(u64 *dbg_var, u32 nr)
  47 {
  48         *dbg_var += nr;
  49 }
  50 #else
  51 #define __pp_dbg_var_add(dbg_var, nr)
  52 #endif
  53
  54 #define pp_alloc_add(pool, nr) __pp_dbg_var_add(&(pool)->allocs, nr)
  55 #define pp_fill_add(pool, nr)  __pp_dbg_var_add(&(pool)->fills, nr)
  56 #define pp_hit_add(pool, nr)   __pp_dbg_var_add(&(pool)->hits, nr)
  57 #define pp_miss_add(pool, nr)  __pp_dbg_var_add(&(pool)->misses, nr)
  58
  59 static int __nvmap_page_pool_fill_lots_locked(struct nvmap_page_pool *pool,
  60                                        struct page **pages, u32 nr);
  61
  62 /*
  63  * Make sure any data in the caches is cleaned out before
  64  * passing these pages to userspace. otherwise, It can lead to
  65  * corruption in pages that get mapped as something
  66  * other than WB in userspace and leaked kernel data.
  67  *
  68  * Must be called with pool->lock held.
  69  */
  70 static void pp_clean_cache(struct nvmap_page_pool *pool)
  71 {
  72         if (pool->contains_dirty_pages) {
  73                 inner_clean_cache_all();
  74                 outer_clean_all();
  75                 pool->contains_dirty_pages = false;
  76         }
  77 }
  78
  79 static inline struct page *get_zero_list_page(struct nvmap_page_pool *pool)
  80 {
  81         struct page *page;
  82
  83         if (list_empty(&pool->zero_list))
  84                 return NULL;
  85
  86         page = list_first_entry(&pool->zero_list, struct page, lru);
  87         list_del(&page->lru);
  88
  89         pool->to_zero--;
  90
  91         return page;
  92 }
  93
  94 static inline struct page *get_page_list_page(struct nvmap_page_pool *pool)
  95 {
  96         struct page *page;
  97
  98         if (list_empty(&pool->page_list))
  99                 return NULL;
 100
 101         page = list_first_entry(&pool->page_list, struct page, lru);
 102         list_del(&page->lru);
 103
 104         pool->count--;
 105
 106         return page;
 107 }
 108
 109 static inline bool nvmap_bg_should_run(struct nvmap_page_pool *pool)
 110 {
 111         bool ret;
 112
 113         mutex_lock(&pool->lock);
 114         ret = (pool->to_zero > 0);
 115         mutex_unlock(&pool->lock);
 116
 117         return ret;
 118 }
 119
 120 static int nvmap_pp_zero_pages(struct page **pages, int nr)
 121 {
 122         int i;
 123
 124         for (i = 0; i < nr; i++)
 125                 clear_highpage(pages[i]);
 126
 127         return 0;
 128 }
 129
 130 static void nvmap_pp_do_background_zero_pages(struct nvmap_page_pool *pool)
 131 {
 132         int i;
 133         struct page *page;
 134         int ret;
 135
 136         /*
 137          * Statically declared array of pages to be zeroed in a batch,
 138          * local to this thread but too big for the stack.
 139          */
 140         static struct page *pending_zero_pages[PENDING_PAGES_SIZE];
 141
 142         mutex_lock(&pool->lock);
 143         for (i = 0; i < PENDING_PAGES_SIZE; i++) {
 144                 page = get_zero_list_page(pool);
 145                 if (page == NULL)
 146                         break;
 147                 pending_zero_pages[i] = page;
 148         }
 149         mutex_unlock(&pool->lock);
 150
 151         ret = nvmap_pp_zero_pages(pending_zero_pages, i);
 152         if (ret < 0) {
 153                 ret = 0;
 154                 goto out;
 155         }
 156
 157         mutex_lock(&pool->lock);
 158         ret = __nvmap_page_pool_fill_lots_locked(pool, pending_zero_pages, i);
 159         mutex_unlock(&pool->lock);
 160
 161 out:
 162         for (; ret < i; ret++)
 163                 __free_page(pending_zero_pages[ret]);
 164
 165         /* clean cache in the background so that allocations immediately
 166          * after fill don't suffer the cache clean overhead.
 167          */
 168         mutex_lock(&pool->lock);
 169         pp_clean_cache(pool);
 170         mutex_unlock(&pool->lock);
 171 }
 172
 173 /*
 174  * This thread fills the page pools with zeroed pages. We avoid releasing the
 175  * pages directly back into the page pools since we would then have to zero
 176  * them ourselves. Instead it is easier to just reallocate zeroed pages. This
 177  * happens in the background so that the overhead of allocating zeroed pages is
 178  * not directly seen by userspace. Of course if the page pools are empty user
 179  * space will suffer.
 180  */
 181 static int nvmap_background_zero_thread(void *arg)
 182 {
 183         struct nvmap_page_pool *pool = &nvmap_dev->pool;
 184         struct sched_param param = { .sched_priority = 0 };
 185
 186         pr_info("PP zeroing thread starting.\n");
 187
 188         set_freezable();
 189         sched_setscheduler(current, SCHED_IDLE, &param);
 190
 191         while (!kthread_should_stop()) {
 192                 while (nvmap_bg_should_run(pool)) {
 193                         nvmap_pp_do_background_zero_pages(pool);
 194                 }
 195
 196                 wait_event_freezable(nvmap_bg_wait,
 197                                 nvmap_bg_should_run(pool) ||
 198                                 kthread_should_stop());
 199         }
 200
 201         return 0;
 202 }
 203
 204 /*
 205  * This removes a page from the page pool. If ignore_disable is set, then
 206  * the enable_pp flag is ignored.
 207  */
 208 static struct page *nvmap_page_pool_alloc_locked(struct nvmap_page_pool *pool,
 209                                                  int force_alloc)
 210 {
 211         struct page *page;
 212
 213         if (!force_alloc && !enable_pp)
 214                 return NULL;
 215
 216         if (list_empty(&pool->page_list)) {
 217                 pp_miss_add(pool, 1);
 218                 return NULL;
 219         }
 220
 221         if (IS_ENABLED(CONFIG_NVMAP_PAGE_POOL_DEBUG))
 222                 BUG_ON(pool->count == 0);
 223
 224         pp_clean_cache(pool);
 225         page = get_page_list_page(pool);
 226         if (!page)
 227                 return NULL;
 228
 229         /* Sanity check. */
 230         if (IS_ENABLED(CONFIG_NVMAP_PAGE_POOL_DEBUG)) {
 231                 atomic_dec(&page->_count);
 232                 BUG_ON(atomic_read(&page->_count) != 1);
 233         }
 234
 235         pp_alloc_add(pool, 1);
 236         pp_hit_add(pool, 1);
 237
 238         return page;
 239 }
 240
 241 /*
 242  * Alloc a bunch of pages from the page pool. This will alloc as many as it can
 243  * and return the number of pages allocated. Pages are placed into the passed
 244  * array in a linear fashion starting from index 0.
 245  */
 246 int nvmap_page_pool_alloc_lots(struct nvmap_page_pool *pool,
 247                                 struct page **pages, u32 nr)
 248 {
 249         u32 real_nr;
 250         u32 ind = 0;
 251
 252         if (!enable_pp)
 253                 return 0;
 254
 255         mutex_lock(&pool->lock);
 256         pp_clean_cache(pool);
 257
 258         real_nr = min_t(u32, nr, pool->count);
 259
 260         while (real_nr--) {
 261                 struct page *page;
 262                 if (IS_ENABLED(CONFIG_NVMAP_PAGE_POOL_DEBUG))
 263                         BUG_ON(list_empty(&pool->page_list));
 264                 page = get_page_list_page(pool);
 265                 pages[ind++] = page;
 266                 if (IS_ENABLED(CONFIG_NVMAP_PAGE_POOL_DEBUG)) {
 267                         atomic_dec(&page->_count);
 268                         BUG_ON(atomic_read(&page->_count) != 1);
 269                 }
 270         }
 271         mutex_unlock(&pool->lock);
 272
 273         pp_alloc_add(pool, ind);
 274         pp_hit_add(pool, ind);
 275         pp_miss_add(pool, nr - ind);
 276
 277         return ind;
 278 }
 279
 280 /*
 281  * Fill a bunch of pages into the page pool. This will fill as many as it can
 282  * and return the number of pages filled. Pages are used from the start of the
 283  * passed page pointer array in a linear fashion.
 284  *
 285  * You must lock the page pool before using this.
 286  */
 287 static int __nvmap_page_pool_fill_lots_locked(struct nvmap_page_pool *pool,
 288                                        struct page **pages, u32 nr)
 289 {
 290         u32 real_nr;
 291         u32 ind = 0;
 292
 293         if (!enable_pp)
 294                 return 0;
 295
 296         pool->contains_dirty_pages = true;
 297
 298         real_nr = min_t(u32, pool->max - pool->count, nr);
 299         if (real_nr == 0)
 300                 return 0;
 301
 302         while (real_nr--) {
 303                 if (IS_ENABLED(CONFIG_NVMAP_PAGE_POOL_DEBUG)) {
 304                         atomic_inc(&pages[ind]->_count);
 305                         BUG_ON(atomic_read(&pages[ind]->_count) != 2);
 306                 }
 307                 list_add_tail(&pages[ind++]->lru, &pool->page_list);
 308         }
 309
 310         pool->count += ind;
 311         pp_fill_add(pool, ind);
 312
 313         return ind;
 314 }
 315
 316 int nvmap_page_pool_fill_lots(struct nvmap_page_pool *pool,
 317                                        struct page **pages, u32 nr)
 318 {
 319         int ret;
 320
 321         mutex_lock(&pool->lock);
 322         if (zero_memory) {
 323                 int i;
 324
 325                 nr = min(nr, pool->max - pool->count - pool->to_zero);
 326
 327                 for (i = 0; i < nr; i++) {
 328                         list_add_tail(&pages[i]->lru, &pool->zero_list);
 329                         pool->to_zero++;
 330                 }
 331
 332                 wake_up_interruptible(&nvmap_bg_wait);
 333
 334                 ret = i;
 335         } else {
 336                 ret = __nvmap_page_pool_fill_lots_locked(pool, pages, nr);
 337         }
 338         mutex_unlock(&pool->lock);
 339
 340         return ret;
 341 }
 342
 343 /*
 344  * Free the passed number of pages from the page pool. This happen irregardless
 345  * of whether ther page pools are enabled. This lets one disable the page pools
 346  * and then free all the memory therein.
 347  */
 348 static int nvmap_page_pool_free(struct nvmap_page_pool *pool, int nr_free)
 349 {
 350         int i = nr_free;
 351         struct page *page;
 352
 353         if (!nr_free)
 354                 return nr_free;
 355
 356         mutex_lock(&pool->lock);
 357         while (i) {
 358                 page = get_zero_list_page(pool);
 359                 if (!page)
 360                         page = nvmap_page_pool_alloc_locked(pool, 1);
 361                 if (!page)
 362                         break;
 363                 __free_page(page);
 364                 i--;
 365         }
 366         mutex_unlock(&pool->lock);
 367
 368         return i;
 369 }
 370
 371 ulong nvmap_page_pool_get_unused_pages(void)
 372 {
 373         int total = 0;
 374
 375         if (!nvmap_dev)
 376                 return 0;
 377
 378         total = nvmap_dev->pool.count + nvmap_dev->pool.to_zero;
 379
 380         return total;
 381 }
 382
 383 /*
 384  * Remove and free to the system all the pages currently in the page
 385  * pool. This operation will happen even if the page pools are disabled.
 386  */
 387 int nvmap_page_pool_clear(void)
 388 {
 389         struct page *page;
 390         struct nvmap_page_pool *pool = &nvmap_dev->pool;
 391
 392         mutex_lock(&pool->lock);
 393
 394         while ((page = nvmap_page_pool_alloc_locked(pool, 1)) != NULL)
 395                 __free_page(page);
 396
 397         while (!list_empty(&pool->zero_list)) {
 398                 page = get_zero_list_page(pool);
 399                 __free_page(page);
 400         }
 401
 402         /* For some reason, if an error occured... */
 403         if (!list_empty(&pool->page_list) || !list_empty(&pool->zero_list)) {
 404                 mutex_unlock(&pool->lock);
 405                 return -ENOMEM;
 406         }
 407
 408         mutex_unlock(&pool->lock);
 409
 410         return 0;
 411 }
 412
 413 /*
 414  * Resizes the page pool to the passed size. If the passed size is 0 then
 415  * all associated resources are released back to the system. This operation
 416  * will only occur if the page pools are enabled.
 417  */
 418 static void nvmap_page_pool_resize(struct nvmap_page_pool *pool, int size)
 419 {
 420         if (!enable_pp || size == pool->max || size < 0)
 421                 return;
 422
 423         mutex_lock(&pool->lock);
 424
 425         while (pool->count > size)
 426                 __free_page(nvmap_page_pool_alloc_locked(pool, 0));
 427
 428         pool->max = size;
 429
 430         pr_debug("page pool resized to %d from %d pages\n", size, pool->max);
 431         mutex_unlock(&pool->lock);
 432 }
 433
 434 static int nvmap_page_pool_shrink(struct shrinker *shrinker,
 435                                   struct shrink_control *sc)
 436 {
 437         int shrink_pages = sc->nr_to_scan;
 438
 439         if (!shrink_pages)
 440                 goto out;
 441
 442         pr_debug("sh_pages=%d", shrink_pages);
 443
 444         shrink_pages = nvmap_page_pool_free(&nvmap_dev->pool, shrink_pages);
 445 out:
 446         return nvmap_page_pool_get_unused_pages();
 447 }
 448
 449 static struct shrinker nvmap_page_pool_shrinker = {
 450         .shrink = nvmap_page_pool_shrink,
 451         .seeks = 1,
 452 };
 453
 454 static void shrink_page_pools(int *total_pages, int *available_pages)
 455 {
 456         struct shrink_control sc;
 457
 458         if (*total_pages == 0) {
 459                 sc.gfp_mask = GFP_KERNEL;
 460                 sc.nr_to_scan = 0;
 461                 *total_pages = nvmap_page_pool_shrink(NULL, &sc);
 462         }
 463         sc.nr_to_scan = *total_pages;
 464         *available_pages = nvmap_page_pool_shrink(NULL, &sc);
 465 }
 466
 467 #if NVMAP_TEST_PAGE_POOL_SHRINKER
 468 static int shrink_pp;
 469 static int shrink_set(const char *arg, const struct kernel_param *kp)
 470 {
 471         int cpu = smp_processor_id();
 472         unsigned long long t1, t2;
 473         int total_pages, available_pages;
 474
 475         param_set_int(arg, kp);
 476
 477         if (shrink_pp) {
 478                 total_pages = shrink_pp;
 479                 t1 = cpu_clock(cpu);
 480                 shrink_page_pools(&total_pages, &available_pages);
 481                 t2 = cpu_clock(cpu);
 482                 pr_debug("shrink page pools: time=%lldns, "
 483                         "total_pages_released=%d, free_pages_available=%d",
 484                         t2-t1, total_pages, available_pages);
 485         }
 486         return 0;
 487 }
 488
 489 static int shrink_get(char *buff, const struct kernel_param *kp)
 490 {
 491         return param_get_int(buff, kp);
 492 }
 493
 494 static struct kernel_param_ops shrink_ops = {
 495         .get = shrink_get,
 496         .set = shrink_set,
 497 };
 498
 499 module_param_cb(shrink_page_pools, &shrink_ops, &shrink_pp, 0644);
 500 #endif
 501
 502 static int enable_pp_set(const char *arg, const struct kernel_param *kp)
 503 {
 504         int ret;
 505
 506         ret = param_set_bool(arg, kp);
 507         if (ret)
 508                 return ret;
 509
 510         if (!enable_pp)
 511                 nvmap_page_pool_clear();
 512
 513         return 0;
 514 }
 515
 516 static int enable_pp_get(char *buff, const struct kernel_param *kp)
 517 {
 518         return param_get_int(buff, kp);
 519 }
 520
 521 static struct kernel_param_ops enable_pp_ops = {
 522         .get = enable_pp_get,
 523         .set = enable_pp_set,
 524 };
 525
 526 module_param_cb(enable_page_pools, &enable_pp_ops, &enable_pp, 0644);
 527
 528 static int pool_size_set(const char *arg, const struct kernel_param *kp)
 529 {
 530         param_set_int(arg, kp);
 531         nvmap_page_pool_resize(&nvmap_dev->pool, pool_size);
 532         return 0;
 533 }
 534
 535 static int pool_size_get(char *buff, const struct kernel_param *kp)
 536 {
 537         return param_get_int(buff, kp);
 538 }
 539
 540 static struct kernel_param_ops pool_size_ops = {
 541         .get = pool_size_get,
 542         .set = pool_size_set,
 543 };
 544
 545 module_param_cb(pool_size, &pool_size_ops, &pool_size, 0644);
 546
 547 int nvmap_page_pool_debugfs_init(struct dentry *nvmap_root)
 548 {
 549         struct dentry *pp_root;
 550
 551         if (!nvmap_root)
 552                 return -ENODEV;
 553
 554         pp_root = debugfs_create_dir("pagepool", nvmap_root);
 555         if (!pp_root)
 556                 return -ENODEV;
 557
 558         debugfs_create_u32("page_pool_available_pages",
 559                            S_IRUGO, pp_root,
 560                            &nvmap_dev->pool.count);
 561         debugfs_create_u32("page_pool_pages_to_zero",
 562                            S_IRUGO, pp_root,
 563                            &nvmap_dev->pool.to_zero);
 564 #ifdef CONFIG_NVMAP_PAGE_POOL_DEBUG
 565         debugfs_create_u64("page_pool_allocs",
 566                            S_IRUGO, pp_root,
 567                            &nvmap_dev->pool.allocs);
 568         debugfs_create_u64("page_pool_fills",
 569                            S_IRUGO, pp_root,
 570                            &nvmap_dev->pool.fills);
 571         debugfs_create_u64("page_pool_hits",
 572                            S_IRUGO, pp_root,
 573                            &nvmap_dev->pool.hits);
 574         debugfs_create_u64("page_pool_misses",
 575                            S_IRUGO, pp_root,
 576                            &nvmap_dev->pool.misses);
 577 #endif
 578
 579         return 0;
 580 }
 581
 582 int nvmap_page_pool_init(struct nvmap_device *dev)
 583 {
 584         unsigned long totalram_mb;
 585         struct sysinfo info;
 586         struct nvmap_page_pool *pool = &dev->pool;
 587
 588         memset(pool, 0x0, sizeof(*pool));
 589         mutex_init(&pool->lock);
 590         INIT_LIST_HEAD(&pool->page_list);
 591         INIT_LIST_HEAD(&pool->zero_list);
 592
 593         si_meminfo(&info);
 594         totalram_mb = (info.totalram * info.mem_unit) >> 20;
 595         pr_info("Total MB RAM: %lu\n", totalram_mb);
 596
 597         if (!CONFIG_NVMAP_PAGE_POOL_SIZE)
 598                 /* The ratio is pool pages per 1K ram pages. So, the >> 10. */
 599                 pool->max = (info.totalram * NVMAP_PP_POOL_SIZE) >> 10;
 600         else
 601                 pool->max = CONFIG_NVMAP_PAGE_POOL_SIZE;
 602
 603         if (pool->max >= info.totalram)
 604                 goto fail;
 605         pool_size = pool->max;
 606
 607         pr_info("nvmap page pool size: %u pages (%u MB)\n", pool->max,
 608                 (pool->max * info.mem_unit) >> 20);
 609
 610         background_allocator = kthread_run(nvmap_background_zero_thread,
 611                                             NULL, "nvmap-bz");
 612         if (IS_ERR_OR_NULL(background_allocator))
 613                 goto fail;
 614
 615         register_shrinker(&nvmap_page_pool_shrinker);
 616
 617         return 0;
 618 fail:
 619         nvmap_page_pool_fini(dev);
 620         return -ENOMEM;
 621 }
 622
 623 int nvmap_page_pool_fini(struct nvmap_device *dev)
 624 {
 625         struct nvmap_page_pool *pool = &dev->pool;
 626
 627         if (!IS_ERR_OR_NULL(background_allocator))
 628                 kthread_stop(background_allocator);
 629
 630         WARN_ON(!list_empty(&pool->page_list));
 631
 632         return 0;
 633 }