drivers/video/tegra/nvmap/nvmap_handle.c

   1 /*
   2  * drivers/video/tegra/nvmap/nvmap_handle.c
   3  *
   4  * Handle allocation and freeing routines for nvmap
   5  *
   6  * Copyright (c) 2009-2014, NVIDIA CORPORATION. All rights reserved.
   7  *
   8  * This program is free software; you can redistribute it and/or modify
   9  * it under the terms of the GNU General Public License as published by
  10  * the Free Software Foundation; either version 2 of the License, or
  11  * (at your option) any later version.
  12  *
  13  * This program is distributed in the hope that it will be useful, but WITHOUT
  14  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  15  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  16  * more details.
  17  *
  18  * You should have received a copy of the GNU General Public License along
  19  * with this program; if not, write to the Free Software Foundation, Inc.,
  20  * 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
  21  */
  22
  23 #define pr_fmt(fmt)     "%s: " fmt, __func__
  24
  25 #include <linux/err.h>
  26 #include <linux/io.h>
  27 #include <linux/kernel.h>
  28 #include <linux/list.h>
  29 #include <linux/mm.h>
  30 #include <linux/rbtree.h>
  31 #include <linux/dma-buf.h>
  32 #include <linux/moduleparam.h>
  33 #include <linux/nvmap.h>
  34 #include <linux/tegra-soc.h>
  35
  36 #include <asm/pgtable.h>
  37
  38 #include <trace/events/nvmap.h>
  39
  40 #include "nvmap_priv.h"
  41 #include "nvmap_ioctl.h"
  42
  43 bool zero_memory;
  44
  45 static int zero_memory_set(const char *arg, const struct kernel_param *kp)
  46 {
  47         param_set_bool(arg, kp);
  48         nvmap_page_pool_clear();
  49         return 0;
  50 }
  51
  52 static struct kernel_param_ops zero_memory_ops = {
  53         .get = param_get_bool,
  54         .set = zero_memory_set,
  55 };
  56
  57 module_param_cb(zero_memory, &zero_memory_ops, &zero_memory, 0644);
  58
  59 u32 nvmap_max_handle_count;
  60
  61 /* handles may be arbitrarily large (16+MiB), and any handle allocated from
  62  * the kernel (i.e., not a carveout handle) includes its array of pages. to
  63  * preserve kmalloc space, if the array of pages exceeds PAGELIST_VMALLOC_MIN,
  64  * the array is allocated using vmalloc. */
  65 #define PAGELIST_VMALLOC_MIN    (PAGE_SIZE)
  66
  67 void *nvmap_altalloc(size_t len)
  68 {
  69         if (len > PAGELIST_VMALLOC_MIN)
  70                 return vmalloc(len);
  71         else
  72                 return kmalloc(len, GFP_KERNEL);
  73 }
  74
  75 void nvmap_altfree(void *ptr, size_t len)
  76 {
  77         if (!ptr)
  78                 return;
  79
  80         if (len > PAGELIST_VMALLOC_MIN)
  81                 vfree(ptr);
  82         else
  83                 kfree(ptr);
  84 }
  85
  86 void _nvmap_handle_free(struct nvmap_handle *h)
  87 {
  88         unsigned int i, nr_page, page_index = 0;
  89 #if defined(CONFIG_NVMAP_PAGE_POOLS) && \
  90         !defined(CONFIG_NVMAP_FORCE_ZEROED_USER_PAGES)
  91         struct nvmap_page_pool *pool;
  92 #endif
  93
  94         if (h->nvhost_priv)
  95                 h->nvhost_priv_delete(h->nvhost_priv);
  96
  97         if (nvmap_handle_remove(nvmap_dev, h) != 0)
  98                 return;
  99
 100         if (!h->alloc)
 101                 goto out;
 102
 103         nvmap_stats_inc(NS_RELEASE, h->size);
 104         nvmap_stats_dec(NS_TOTAL, PAGE_ALIGN(h->orig_size));
 105         if (!h->heap_pgalloc) {
 106                 nvmap_heap_free(h->carveout);
 107                 goto out;
 108         }
 109
 110         nr_page = DIV_ROUND_UP(h->size, PAGE_SIZE);
 111
 112         BUG_ON(h->size & ~PAGE_MASK);
 113         BUG_ON(!h->pgalloc.pages);
 114
 115 #ifdef NVMAP_LAZY_VFREE
 116         if (h->vaddr)
 117                 vm_unmap_ram(h->vaddr, h->size >> PAGE_SHIFT);
 118 #endif
 119
 120         for (i = 0; i < nr_page; i++)
 121                 h->pgalloc.pages[i] = nvmap_to_page(h->pgalloc.pages[i]);
 122
 123 #if defined(CONFIG_NVMAP_PAGE_POOLS) && \
 124         !defined(CONFIG_NVMAP_FORCE_ZEROED_USER_PAGES)
 125         if (!zero_memory) {
 126                 pool = &nvmap_dev->pool;
 127
 128                 nvmap_page_pool_lock(pool);
 129                 page_index = __nvmap_page_pool_fill_lots_locked(pool,
 130                                                 h->pgalloc.pages, nr_page);
 131                 nvmap_page_pool_unlock(pool);
 132         }
 133 #endif
 134
 135         for (i = page_index; i < nr_page; i++)
 136                 __free_page(h->pgalloc.pages[i]);
 137
 138         nvmap_altfree(h->pgalloc.pages, nr_page * sizeof(struct page *));
 139
 140 out:
 141         kfree(h);
 142 }
 143
 144 static struct page *nvmap_alloc_pages_exact(gfp_t gfp, size_t size)
 145 {
 146         struct page *page, *p, *e;
 147         unsigned int order;
 148
 149         size = PAGE_ALIGN(size);
 150         order = get_order(size);
 151         page = alloc_pages(gfp, order);
 152
 153         if (!page)
 154                 return NULL;
 155
 156         split_page(page, order);
 157         e = page + (1 << order);
 158         for (p = page + (size >> PAGE_SHIFT); p < e; p++)
 159                 __free_page(p);
 160
 161         return page;
 162 }
 163
 164 static int handle_page_alloc(struct nvmap_client *client,
 165                              struct nvmap_handle *h, bool contiguous)
 166 {
 167         size_t size = PAGE_ALIGN(h->size);
 168         unsigned int nr_page = size >> PAGE_SHIFT;
 169         pgprot_t prot;
 170         unsigned int i = 0, page_index = 0;
 171         struct page **pages;
 172 #ifdef CONFIG_NVMAP_PAGE_POOLS
 173         struct nvmap_page_pool *pool = NULL;
 174 #endif
 175         gfp_t gfp = GFP_NVMAP;
 176
 177         if (zero_memory)
 178                 gfp |= __GFP_ZERO;
 179
 180         pages = nvmap_altalloc(nr_page * sizeof(*pages));
 181         if (!pages)
 182                 return -ENOMEM;
 183
 184         prot = nvmap_pgprot(h, PG_PROT_KERNEL);
 185
 186         if (contiguous) {
 187                 struct page *page;
 188                 page = nvmap_alloc_pages_exact(gfp, size);
 189                 if (!page)
 190                         goto fail;
 191
 192                 for (i = 0; i < nr_page; i++)
 193                         pages[i] = nth_page(page, i);
 194
 195         } else {
 196 #ifdef CONFIG_NVMAP_PAGE_POOLS
 197                 pool = &nvmap_dev->pool;
 198
 199                 /*
 200                  * Get as many pages from the pools as possible.
 201                  */
 202                 nvmap_page_pool_lock(pool);
 203                 page_index = __nvmap_page_pool_alloc_lots_locked(pool, pages,
 204                                                                  nr_page);
 205                 nvmap_page_pool_unlock(pool);
 206 #endif
 207                 for (i = page_index; i < nr_page; i++) {
 208                         pages[i] = nvmap_alloc_pages_exact(gfp, PAGE_SIZE);
 209                         if (!pages[i])
 210                                 goto fail;
 211                 }
 212         }
 213
 214         /*
 215          * Make sure any data in the caches is cleaned out before
 216          * passing these pages to userspace. otherwise, It can lead to
 217          * corruption in pages that get mapped as something other than WB in
 218          * userspace and leaked kernel data structures.
 219          *
 220          * FIXME: For ARMv7 we don't have __clean_dcache_page() so we continue
 221          * to use the flush cache version.
 222          */
 223         if (page_index < nr_page)
 224 #ifdef ARM64
 225                 nvmap_clean_cache(&pages[page_index], nr_page - page_index);
 226 #else
 227                 nvmap_flush_cache(&pages[page_index], nr_page - page_index);
 228 #endif
 229
 230         h->size = size;
 231         h->pgalloc.pages = pages;
 232         h->pgalloc.contig = contiguous;
 233         atomic_set(&h->pgalloc.ndirty, 0);
 234         return 0;
 235
 236 fail:
 237         while (i--)
 238                 __free_page(pages[i]);
 239         nvmap_altfree(pages, nr_page * sizeof(*pages));
 240         wmb();
 241         return -ENOMEM;
 242 }
 243
 244 static void alloc_handle(struct nvmap_client *client,
 245                          struct nvmap_handle *h, unsigned int type)
 246 {
 247         unsigned int carveout_mask = NVMAP_HEAP_CARVEOUT_MASK;
 248         unsigned int iovmm_mask = NVMAP_HEAP_IOVMM;
 249
 250         BUG_ON(type & (type - 1));
 251
 252 #ifdef CONFIG_NVMAP_CONVERT_CARVEOUT_TO_IOVMM
 253         /* Convert generic carveout requests to iovmm requests. */
 254         carveout_mask &= ~NVMAP_HEAP_CARVEOUT_GENERIC;
 255         iovmm_mask |= NVMAP_HEAP_CARVEOUT_GENERIC;
 256 #endif
 257
 258         if (type & carveout_mask) {
 259                 struct nvmap_heap_block *b;
 260
 261                 b = nvmap_carveout_alloc(client, h, type);
 262                 if (b) {
 263                         h->heap_type = type;
 264                         h->heap_pgalloc = false;
 265                         /* barrier to ensure all handle alloc data
 266                          * is visible before alloc is seen by other
 267                          * processors.
 268                          */
 269                         mb();
 270                         h->alloc = true;
 271                 }
 272         } else if (type & iovmm_mask) {
 273                 int ret;
 274
 275                 ret = handle_page_alloc(client, h,
 276                         h->userflags & NVMAP_HANDLE_PHYS_CONTIG);
 277                 if (ret)
 278                         return;
 279                 h->heap_type = NVMAP_HEAP_IOVMM;
 280                 h->heap_pgalloc = true;
 281                 mb();
 282                 h->alloc = true;
 283         }
 284 }
 285
 286 /* small allocations will try to allocate from generic OS memory before
 287  * any of the limited heaps, to increase the effective memory for graphics
 288  * allocations, and to reduce fragmentation of the graphics heaps with
 289  * sub-page splinters */
 290 static const unsigned int heap_policy_small[] = {
 291         NVMAP_HEAP_CARVEOUT_VPR,
 292         NVMAP_HEAP_CARVEOUT_IRAM,
 293         NVMAP_HEAP_CARVEOUT_MASK,
 294         NVMAP_HEAP_IOVMM,
 295         0,
 296 };
 297
 298 static const unsigned int heap_policy_large[] = {
 299         NVMAP_HEAP_CARVEOUT_VPR,
 300         NVMAP_HEAP_CARVEOUT_IRAM,
 301         NVMAP_HEAP_IOVMM,
 302         NVMAP_HEAP_CARVEOUT_MASK,
 303         0,
 304 };
 305
 306 int nvmap_alloc_handle(struct nvmap_client *client,
 307                        struct nvmap_handle *h, unsigned int heap_mask,
 308                        size_t align,
 309                        u8 kind,
 310                        unsigned int flags)
 311 {
 312         const unsigned int *alloc_policy;
 313         int nr_page;
 314         int err = -ENOMEM;
 315
 316         h = nvmap_handle_get(h);
 317
 318         if (!h)
 319                 return -EINVAL;
 320
 321         if (h->alloc) {
 322                 nvmap_handle_put(h);
 323                 return -EEXIST;
 324         }
 325
 326         nvmap_stats_inc(NS_TOTAL, PAGE_ALIGN(h->orig_size));
 327         nvmap_stats_inc(NS_ALLOC, PAGE_ALIGN(h->size));
 328         trace_nvmap_alloc_handle(client, h,
 329                 h->size, heap_mask, align, flags,
 330                 nvmap_stats_read(NS_TOTAL),
 331                 nvmap_stats_read(NS_ALLOC));
 332         h->userflags = flags;
 333         nr_page = ((h->size + PAGE_SIZE - 1) >> PAGE_SHIFT);
 334         h->flags = (flags & NVMAP_HANDLE_CACHE_FLAG);
 335         h->align = max_t(size_t, align, L1_CACHE_BYTES);
 336         h->kind = kind;
 337
 338         /* convert iovmm requests to generic carveout. */
 339         if (heap_mask & NVMAP_HEAP_IOVMM) {
 340                 heap_mask = (heap_mask & ~NVMAP_HEAP_IOVMM) |
 341                             NVMAP_HEAP_CARVEOUT_GENERIC;
 342         }
 343
 344         if (!heap_mask) {
 345                 err = -EINVAL;
 346                 goto out;
 347         }
 348
 349         alloc_policy = (nr_page == 1) ? heap_policy_small : heap_policy_large;
 350
 351         while (!h->alloc && *alloc_policy) {
 352                 unsigned int heap_type;
 353
 354                 heap_type = *alloc_policy++;
 355                 heap_type &= heap_mask;
 356
 357                 if (!heap_type)
 358                         continue;
 359
 360                 heap_mask &= ~heap_type;
 361
 362                 while (heap_type && !h->alloc) {
 363                         unsigned int heap;
 364
 365                         /* iterate possible heaps MSB-to-LSB, since higher-
 366                          * priority carveouts will have higher usage masks */
 367                         heap = 1 << __fls(heap_type);
 368                         alloc_handle(client, h, heap);
 369                         heap_type &= ~heap;
 370                 }
 371         }
 372
 373 out:
 374         if (h->alloc) {
 375                 if (client->kernel_client)
 376                         nvmap_stats_inc(NS_KALLOC, h->size);
 377                 else
 378                         nvmap_stats_inc(NS_UALLOC, h->size);
 379         } else {
 380                 nvmap_stats_dec(NS_TOTAL, PAGE_ALIGN(h->orig_size));
 381                 nvmap_stats_dec(NS_ALLOC, PAGE_ALIGN(h->orig_size));
 382         }
 383
 384         err = (h->alloc) ? 0 : err;
 385         nvmap_handle_put(h);
 386         return err;
 387 }
 388
 389 void nvmap_free_handle(struct nvmap_client *client,
 390                        struct nvmap_handle *handle)
 391 {
 392         struct nvmap_handle_ref *ref;
 393         struct nvmap_handle *h;
 394         int pins;
 395
 396         nvmap_ref_lock(client);
 397
 398         ref = __nvmap_validate_locked(client, handle);
 399         if (!ref) {
 400                 nvmap_ref_unlock(client);
 401                 return;
 402         }
 403
 404         trace_nvmap_free_handle(client, handle);
 405         BUG_ON(!ref->handle);
 406         h = ref->handle;
 407
 408         if (atomic_dec_return(&ref->dupes)) {
 409                 nvmap_ref_unlock(client);
 410                 goto out;
 411         }
 412
 413         smp_rmb();
 414         pins = atomic_read(&ref->pin);
 415         rb_erase(&ref->node, &client->handle_refs);
 416         client->handle_count--;
 417         atomic_dec(&ref->handle->share_count);
 418
 419         nvmap_ref_unlock(client);
 420
 421         if (pins)
 422                 pr_debug("%s freeing pinned handle %p\n",
 423                             current->group_leader->comm, h);
 424
 425         while (atomic_read(&ref->pin))
 426                 __nvmap_unpin(ref);
 427
 428         if (h->owner == client)
 429                 h->owner = NULL;
 430
 431         dma_buf_put(ref->handle->dmabuf);
 432         kfree(ref);
 433
 434 out:
 435         BUG_ON(!atomic_read(&h->ref));
 436         nvmap_handle_put(h);
 437 }
 438 EXPORT_SYMBOL(nvmap_free_handle);
 439
 440 void nvmap_free_handle_user_id(struct nvmap_client *client,
 441                                unsigned long user_id)
 442 {
 443         nvmap_free_handle(client, unmarshal_user_id(user_id));
 444 }
 445
 446 static void add_handle_ref(struct nvmap_client *client,
 447                            struct nvmap_handle_ref *ref)
 448 {
 449         struct rb_node **p, *parent = NULL;
 450
 451         nvmap_ref_lock(client);
 452         p = &client->handle_refs.rb_node;
 453         while (*p) {
 454                 struct nvmap_handle_ref *node;
 455                 parent = *p;
 456                 node = rb_entry(parent, struct nvmap_handle_ref, node);
 457                 if (ref->handle > node->handle)
 458                         p = &parent->rb_right;
 459                 else
 460                         p = &parent->rb_left;
 461         }
 462         rb_link_node(&ref->node, parent, p);
 463         rb_insert_color(&ref->node, &client->handle_refs);
 464         client->handle_count++;
 465         if (client->handle_count > nvmap_max_handle_count)
 466                 nvmap_max_handle_count = client->handle_count;
 467         atomic_inc(&ref->handle->share_count);
 468         nvmap_ref_unlock(client);
 469 }
 470
 471 struct nvmap_handle_ref *nvmap_create_handle(struct nvmap_client *client,
 472                                              size_t size)
 473 {
 474         void *err = ERR_PTR(-ENOMEM);
 475         struct nvmap_handle *h;
 476         struct nvmap_handle_ref *ref = NULL;
 477
 478         if (!client)
 479                 return ERR_PTR(-EINVAL);
 480
 481         if (!size)
 482                 return ERR_PTR(-EINVAL);
 483
 484         h = kzalloc(sizeof(*h), GFP_KERNEL);
 485         if (!h)
 486                 return ERR_PTR(-ENOMEM);
 487
 488         ref = kzalloc(sizeof(*ref), GFP_KERNEL);
 489         if (!ref)
 490                 goto ref_alloc_fail;
 491
 492         atomic_set(&h->ref, 1);
 493         atomic_set(&h->pin, 0);
 494         h->owner = client;
 495         BUG_ON(!h->owner);
 496         h->size = h->orig_size = size;
 497         h->flags = NVMAP_HANDLE_WRITE_COMBINE;
 498         mutex_init(&h->lock);
 499         INIT_LIST_HEAD(&h->vmas);
 500
 501         /*
 502          * This takes out 1 ref on the dambuf. This corresponds to the
 503          * handle_ref that gets automatically made by nvmap_create_handle().
 504          */
 505         h->dmabuf = __nvmap_make_dmabuf(client, h);
 506         if (IS_ERR(h->dmabuf)) {
 507                 err = h->dmabuf;
 508                 goto make_dmabuf_fail;
 509         }
 510
 511         /*
 512          * Pre-attach nvmap to this new dmabuf. This gets unattached during the
 513          * dma_buf_release() operation.
 514          */
 515         h->attachment = dma_buf_attach(h->dmabuf, nvmap_dev->dev_user.parent);
 516         if (IS_ERR(h->attachment)) {
 517                 err = h->attachment;
 518                 goto dma_buf_attach_fail;
 519         }
 520
 521         nvmap_handle_add(nvmap_dev, h);
 522
 523         /*
 524          * Major assumption here: the dma_buf object that the handle contains
 525          * is created with a ref count of 1.
 526          */
 527         atomic_set(&ref->dupes, 1);
 528         ref->handle = h;
 529         atomic_set(&ref->pin, 0);
 530         add_handle_ref(client, ref);
 531         trace_nvmap_create_handle(client, client->name, h, size, ref);
 532         return ref;
 533
 534 dma_buf_attach_fail:
 535         dma_buf_put(h->dmabuf);
 536 make_dmabuf_fail:
 537         kfree(ref);
 538 ref_alloc_fail:
 539         kfree(h);
 540         return err;
 541 }
 542
 543 struct nvmap_handle_ref *nvmap_duplicate_handle(struct nvmap_client *client,
 544                                         struct nvmap_handle *h, bool skip_val)
 545 {
 546         struct nvmap_handle_ref *ref = NULL;
 547
 548         BUG_ON(!client);
 549         /* on success, the reference count for the handle should be
 550          * incremented, so the success paths will not call nvmap_handle_put */
 551         h = nvmap_validate_get(h);
 552
 553         if (!h) {
 554                 pr_debug("%s duplicate handle failed\n",
 555                             current->group_leader->comm);
 556                 return ERR_PTR(-EPERM);
 557         }
 558
 559         if (!h->alloc) {
 560                 pr_err("%s duplicating unallocated handle\n",
 561                         current->group_leader->comm);
 562                 nvmap_handle_put(h);
 563                 return ERR_PTR(-EINVAL);
 564         }
 565
 566         nvmap_ref_lock(client);
 567         ref = __nvmap_validate_locked(client, h);
 568
 569         if (ref) {
 570                 /* handle already duplicated in client; just increment
 571                  * the reference count rather than re-duplicating it */
 572                 atomic_inc(&ref->dupes);
 573                 nvmap_ref_unlock(client);
 574                 return ref;
 575         }
 576
 577         nvmap_ref_unlock(client);
 578
 579         ref = kzalloc(sizeof(*ref), GFP_KERNEL);
 580         if (!ref) {
 581                 nvmap_handle_put(h);
 582                 return ERR_PTR(-ENOMEM);
 583         }
 584
 585         atomic_set(&ref->dupes, 1);
 586         ref->handle = h;
 587         atomic_set(&ref->pin, 0);
 588         add_handle_ref(client, ref);
 589
 590         /*
 591          * Ref counting on the dma_bufs follows the creation and destruction of
 592          * nvmap_handle_refs. That is every time a handle_ref is made the
 593          * dma_buf ref count goes up and everytime a handle_ref is destroyed
 594          * the dma_buf ref count goes down.
 595          */
 596         get_dma_buf(h->dmabuf);
 597
 598         trace_nvmap_duplicate_handle(client, h, ref);
 599         return ref;
 600 }
 601
 602 struct nvmap_handle_ref *nvmap_create_handle_from_fd(
 603                         struct nvmap_client *client, int fd)
 604 {
 605         struct nvmap_handle *handle;
 606         struct nvmap_handle_ref *ref;
 607
 608         BUG_ON(!client);
 609
 610         handle = nvmap_get_id_from_dmabuf_fd(client, fd);
 611         if (IS_ERR(handle))
 612                 return ERR_CAST(handle);
 613         ref = nvmap_duplicate_handle(client, handle, 1);
 614         return ref;
 615 }
 616
 617 struct nvmap_handle *nvmap_duplicate_handle_id_ex(struct nvmap_client *client,
 618                                                         struct nvmap_handle *h)
 619 {
 620         struct nvmap_handle_ref *ref = nvmap_duplicate_handle(client, h, 0);
 621
 622         if (IS_ERR(ref))
 623                 return 0;
 624
 625         return __nvmap_ref_to_id(ref);
 626 }
 627 EXPORT_SYMBOL(nvmap_duplicate_handle_id_ex);
 628
 629 int nvmap_get_page_list_info(struct nvmap_client *client,
 630                                 struct nvmap_handle *handle, u32 *size,
 631                                 u32 *flags, u32 *nr_page, bool *contig)
 632 {
 633         struct nvmap_handle *h;
 634
 635         BUG_ON(!size || !flags || !nr_page || !contig);
 636         BUG_ON(!client);
 637
 638         *size = 0;
 639         *flags = 0;
 640         *nr_page = 0;
 641
 642         h = nvmap_handle_get(handle);
 643
 644         if (!h) {
 645                 pr_err("%s query invalid handle %p\n",
 646                         current->group_leader->comm, handle);
 647                 return -EINVAL;
 648         }
 649
 650         if (!h->alloc || !h->heap_pgalloc) {
 651                 pr_err("%s query unallocated handle %p\n",
 652                         current->group_leader->comm, handle);
 653                 nvmap_handle_put(h);
 654                 return -EINVAL;
 655         }
 656
 657         *flags = h->flags;
 658         *size = h->orig_size;
 659         *nr_page = PAGE_ALIGN(h->size) >> PAGE_SHIFT;
 660         *contig = h->pgalloc.contig;
 661
 662         nvmap_handle_put(h);
 663         return 0;
 664 }
 665 EXPORT_SYMBOL(nvmap_get_page_list_info);
 666
 667 int nvmap_acquire_page_list(struct nvmap_client *client,
 668                         struct nvmap_handle *handle, struct page **pages,
 669                         u32 nr_page)
 670 {
 671         struct nvmap_handle *h;
 672         struct nvmap_handle_ref *ref;
 673         int idx;
 674         phys_addr_t dummy;
 675
 676         BUG_ON(!client);
 677
 678         h = nvmap_handle_get(handle);
 679
 680         if (!h) {
 681                 pr_err("%s query invalid handle %p\n",
 682                           current->group_leader->comm, handle);
 683                 return -EINVAL;
 684         }
 685
 686         if (!h->alloc || !h->heap_pgalloc) {
 687                 pr_err("%s query unallocated handle %p\n",
 688                           current->group_leader->comm, handle);
 689                 nvmap_handle_put(h);
 690                 return -EINVAL;
 691         }
 692
 693         BUG_ON(nr_page != PAGE_ALIGN(h->size) >> PAGE_SHIFT);
 694
 695         for (idx = 0; idx < nr_page; idx++)
 696                 pages[idx] = h->pgalloc.pages[idx];
 697
 698         nvmap_ref_lock(client);
 699         ref = __nvmap_validate_locked(client, h);
 700         if (ref)
 701                 __nvmap_pin(ref, &dummy);
 702         nvmap_ref_unlock(client);
 703
 704         return 0;
 705 }
 706 EXPORT_SYMBOL(nvmap_acquire_page_list);
 707
 708 int nvmap_release_page_list(struct nvmap_client *client,
 709                                 struct nvmap_handle *handle)
 710 {
 711         struct nvmap_handle_ref *ref;
 712         struct nvmap_handle *h = NULL;
 713
 714         BUG_ON(!client);
 715
 716         nvmap_ref_lock(client);
 717
 718         ref = __nvmap_validate_locked(client, handle);
 719         if (ref)
 720                 __nvmap_unpin(ref);
 721
 722         nvmap_ref_unlock(client);
 723
 724         if (ref)
 725                 h = ref->handle;
 726         if (h)
 727                 nvmap_handle_put(h);
 728
 729         return 0;
 730 }
 731 EXPORT_SYMBOL(nvmap_release_page_list);
 732
 733 int __nvmap_get_handle_param(struct nvmap_client *client,
 734                              struct nvmap_handle *h, u32 param, u64 *result)
 735 {
 736         int err = 0;
 737
 738         if (WARN_ON(!virt_addr_valid(h)))
 739                 return -EINVAL;
 740
 741         switch (param) {
 742         case NVMAP_HANDLE_PARAM_SIZE:
 743                 *result = h->orig_size;
 744                 break;
 745         case NVMAP_HANDLE_PARAM_ALIGNMENT:
 746                 *result = h->align;
 747                 break;
 748         case NVMAP_HANDLE_PARAM_BASE:
 749                 if (!h->alloc || !atomic_read(&h->pin))
 750                         *result = -EINVAL;
 751                 else if (!h->heap_pgalloc) {
 752                         mutex_lock(&h->lock);
 753                         *result = h->carveout->base;
 754                         mutex_unlock(&h->lock);
 755                 } else if (h->attachment->priv)
 756                         *result = sg_dma_address(
 757                                 ((struct sg_table *)h->attachment->priv)->sgl);
 758                 else
 759                         *result = -EINVAL;
 760                 break;
 761         case NVMAP_HANDLE_PARAM_HEAP:
 762                 if (!h->alloc)
 763                         *result = 0;
 764                 else if (!h->heap_pgalloc) {
 765                         mutex_lock(&h->lock);
 766                         *result = nvmap_carveout_usage(client, h->carveout);
 767                         mutex_unlock(&h->lock);
 768                 } else
 769                         *result = NVMAP_HEAP_IOVMM;
 770                 break;
 771         case NVMAP_HANDLE_PARAM_KIND:
 772                 *result = h->kind;
 773                 break;
 774         case NVMAP_HANDLE_PARAM_COMPR:
 775                 /* ignored, to be removed */
 776                 break;
 777         default:
 778                 err = -EINVAL;
 779                 break;
 780         }
 781         return err;
 782 }
 783
 784 int nvmap_get_handle_param(struct nvmap_client *client,
 785                            struct nvmap_handle_ref *ref, u32 param, u64 *result)
 786 {
 787         if (WARN_ON(!virt_addr_valid(ref)) ||
 788             WARN_ON(!virt_addr_valid(client)) ||
 789             WARN_ON(!result))
 790                 return -EINVAL;
 791
 792         return __nvmap_get_handle_param(client, ref->handle, param, result);
 793 }