dma-buf: defer unmapping for dma shared buffer till free

Implement dma-buf "Deferred unmapping", which stores all attachments and
mappings until its dmabuf is freed. This can be
modified to allocate dynamically after some statistics is
implemented.

This patch allows the exporter specify if the dmabuf it exports can
use the deferred unmapping. This patch also adds refcounting to the
dma_buf_attachment to find out any incorrect usage by dmabuf importers.

Also, the exporter can specify if it needs the dmabuf framework to do
cache sync operations while dma map/unmap occur.

Similarly, devices need to be able to opt for the deferred unmapping.
This is taken care in a different patch.

For the further optimization, if we want to clean caches under some
memory pressure(ex: lack of IOVA space), we need to implement the
following features:

  (1) Statistics: Used by Shrinker.
  (2) Shrinker: Reduce cached mappings upon demand

Bug 1546690
Bug 200229632

Change-Id: I564ebce99e5a7d90b75f040cc38f3e5fa4428447
Signed-off-by: Hiroshi Doyu <hdoyu@nvidia.com>
Reviewed-on: http://git-master/r/488096
(cherry picked from commit 48880108cd08966039f2db76c6054fb737d180a1)
Signed-off-by: Sri Krishna chowdary <schowdary@nvidia.com>
Reviewed-on: http://git-master/r/1209020
GVS: Gerrit_Virtual_Submit
Reviewed-by: svccoveritychecker <svccoveritychecker@nvidia.com>
Reviewed-by: Sachin Nikam <snikam@nvidia.com>

diff --git a/drivers/base/Kconfig b/drivers/base/Kconfig
index 4a6be6d3652898d4c251ed6a17feed6d6a44f7b3..421b4c76005d663d39bc8001832a8d695004e84d 100644 (file)
--- a/drivers/base/Kconfig
+++ b/drivers/base/Kconfig
@@ -238,6 +238,15 @@ config DMA_SHARED_BUFFER
          APIs extension; the file's descriptor can then be passed on to other
          driver.
 
+config DMABUF_DEFERRED_UNMAPPING
+       bool "Defer unmapping till shared DMA buffer is free"
+       default y
+       depends on DMA_SHARED_BUFFER
+       help
+         This option enables the buffer-sharing framework to use deferred unmapping.
+         Buffer mapping would be reused and kept repeatedly until dmabuf is freed.
+         Dmabuf exporter can specify if it is compatible with this option.
+
 config FENCE_TRACE
        bool "Enable verbose FENCE_TRACE messages"
        depends on DMA_SHARED_BUFFER

diff --git a/drivers/dma-buf/dma-buf.c b/drivers/dma-buf/dma-buf.c
index b1e82c9535727d7eb5b71157c13548282474e60b..ad638ddd05d809da82aebc561a834dbac9857f53 100644 (file)
--- a/drivers/dma-buf/dma-buf.c
+++ b/drivers/dma-buf/dma-buf.c
@@ -92,10 +92,22 @@ void *dma_buf_get_drvdata(struct dma_buf *dmabuf, struct device *device)
 }
 EXPORT_SYMBOL(dma_buf_get_drvdata);
 
+static bool dmabuf_can_defer_unmap(struct dma_buf *dmabuf,
+               struct device *device)
+{
+       if (!config_enabled(CONFIG_DMABUF_DEFERRED_UNMAPPING))
+               return false;
+
+       if (!(dmabuf->flags & DMABUF_CAN_DEFER_UNMAP))
+               return false;
+
+       return true;
+}
 
 static int dma_buf_release(struct inode *inode, struct file *file)
 {
        struct dma_buf *dmabuf;
+       struct dma_buf_attachment *attach, *next;
 
        if (!is_dma_buf_file(file))
                return -EINVAL;
@@ -103,6 +115,22 @@ static int dma_buf_release(struct inode *inode, struct file *file)
        dmabuf = file->private_data;
 
        BUG_ON(dmabuf->vmapping_counter);
+       mutex_lock(&dmabuf->lock);
+       list_for_each_entry_safe(attach, next, &dmabuf->attachments, node) {
+               BUG_ON(atomic_read(&attach->ref) != 1);
+               BUG_ON(atomic_read(&attach->maps));
+
+               list_del(&attach->node);
+               if (dmabuf_can_defer_unmap(dmabuf, attach->dev)) {
+                       attach->dmabuf->ops->unmap_dma_buf(attach, attach->sg_table,
+                                                               DMA_BIDIRECTIONAL);
+                       if (dmabuf->ops->detach)
+                               dmabuf->ops->detach(dmabuf, attach);
+                       kzfree(attach);
+               }
+
+       }
+       mutex_unlock(&dmabuf->lock);
 
        /*
         * Any fences that a dma-buf poll can wait on should be signaled
@@ -124,7 +152,7 @@ static int dma_buf_release(struct inode *inode, struct file *file)
                reservation_object_fini(dmabuf->resv);
 
        module_put(dmabuf->owner);
-       kfree(dmabuf);
+       kzfree(dmabuf);
        return 0;
 }
 
@@ -368,6 +396,7 @@ struct dma_buf *dma_buf_export(const struct dma_buf_export_info *exp_info)
        dmabuf->ops = exp_info->ops;
        dmabuf->size = exp_info->size;
        dmabuf->exp_name = exp_info->exp_name;
+       dmabuf->flags = exp_info->exp_flags;
        dmabuf->owner = exp_info->owner;
        init_waitqueue_head(&dmabuf->poll);
        dmabuf->cb_excl.poll = dmabuf->cb_shared.poll = &dmabuf->poll;
@@ -483,14 +512,43 @@ struct dma_buf_attachment *dma_buf_attach(struct dma_buf *dmabuf,
        if (WARN_ON(!dmabuf || !dev))
                return ERR_PTR(-EINVAL);
 
+       mutex_lock(&dmabuf->lock);
+       if (dmabuf_can_defer_unmap(dmabuf, dev)) {
+               /* Don't allow multiple attachments for a device */
+               list_for_each_entry(attach, &dmabuf->attachments, node) {
+                       int ref;
+
+                       if (attach->dev != dev)
+                               continue;
+
+                       /* attach is ready for free. Do not use it. */
+                       ref = atomic_inc_not_zero(&attach->ref);
+                       BUG_ON(ref < 0);
+                       if (ref == 0)
+                               continue;
+
+                       mutex_unlock(&dmabuf->lock);
+                       return attach;
+               }
+       }
+
        attach = kzalloc(sizeof(struct dma_buf_attachment), GFP_KERNEL);
-       if (attach == NULL)
+       if (attach == NULL) {
+               mutex_unlock(&dmabuf->lock);
                return ERR_PTR(-ENOMEM);
+       }
 
        attach->dev = dev;
        attach->dmabuf = dmabuf;
-
-       mutex_lock(&dmabuf->lock);
+       /*
+        * 2 because it is possible that a dmabuf has matching
+        * number of attach/detach in many intermediate states
+        * till the buffer is freed. This extra ref count will
+        * prevent multiple mappings for a given device in such
+        * scenarios.
+        */
+       atomic_set(&attach->ref, 2);
+       atomic_set(&attach->maps, 0);
 
        if (dmabuf->ops->attach) {
                ret = dmabuf->ops->attach(dmabuf, dev, attach);
@@ -521,13 +579,21 @@ void dma_buf_detach(struct dma_buf *dmabuf, struct dma_buf_attachment *attach)
        if (WARN_ON(!dmabuf || !attach))
                return;
 
+       if (atomic_dec_return(&attach->ref) > 0)
+               return;
+
+       BUG_ON(atomic_read(&attach->maps));
+
+       if (dmabuf_can_defer_unmap(dmabuf, attach->dev))
+               return;
+
        mutex_lock(&dmabuf->lock);
        list_del(&attach->node);
        if (dmabuf->ops->detach)
                dmabuf->ops->detach(dmabuf, attach);
 
        mutex_unlock(&dmabuf->lock);
-       kfree(attach);
+       kzfree(attach);
 }
 EXPORT_SYMBOL_GPL(dma_buf_detach);
 
@@ -548,13 +614,28 @@ struct sg_table *dma_buf_map_attachment(struct dma_buf_attachment *attach,
 
        might_sleep();
 
-       if (WARN_ON(!attach || !attach->dmabuf))
+       if (WARN_ON(!attach || !attach->dmabuf ||
+                   !atomic_inc_not_zero(&attach->ref)))
                return ERR_PTR(-EINVAL);
 
+       sg_table = attach->sg_table;
+       if (dmabuf_can_defer_unmap(attach->dmabuf, attach->dev) && sg_table) {
+               if (!(attach->dmabuf->flags & DMABUF_SKIP_CACHE_SYNC))
+                       dma_sync_sg_for_device(attach->dev, sg_table->sgl,
+                                       sg_table->nents, direction);
+               goto finish;
+       }
+
        sg_table = attach->dmabuf->ops->map_dma_buf(attach, direction);
        if (!sg_table)
                sg_table = ERR_PTR(-ENOMEM);
 
+       attach->sg_table = sg_table;
+finish:
+       if (!IS_ERR(sg_table))
+               atomic_inc(&attach->maps);
+       else
+               atomic_dec(&attach->ref);
        return sg_table;
 }
 EXPORT_SYMBOL_GPL(dma_buf_map_attachment);
@@ -577,8 +658,18 @@ void dma_buf_unmap_attachment(struct dma_buf_attachment *attach,
        if (WARN_ON(!attach || !attach->dmabuf || !sg_table))
                return;
 
+       if (dmabuf_can_defer_unmap(attach->dmabuf, attach->dev)) {
+               if (!(attach->dmabuf->flags & DMABUF_SKIP_CACHE_SYNC))
+                       dma_sync_sg_for_cpu(attach->dev, sg_table->sgl,
+                                       sg_table->nents, direction);
+               goto finish;
+       }
+
        attach->dmabuf->ops->unmap_dma_buf(attach, sg_table,
                                                direction);
+finish:
+       atomic_dec(&attach->maps);
+       atomic_dec(&attach->ref);
 }
 EXPORT_SYMBOL_GPL(dma_buf_unmap_attachment);
 

diff --git a/include/linux/dma-buf.h b/include/linux/dma-buf.h
index e9346755a0bd8659dbeaff3beac9fdd19ca0f01c..50c8f2e36e58fd33e196717f1e2ab141febff11e 100644 (file)
--- a/include/linux/dma-buf.h
+++ b/include/linux/dma-buf.h
@@ -37,6 +37,9 @@ struct device;
 struct dma_buf;
 struct dma_buf_attachment;
 
+#define DMABUF_CAN_DEFER_UNMAP         BIT(0)
+#define DMABUF_SKIP_CACHE_SYNC         BIT(1)
+
 /**
  * struct dma_buf_ops - operations possible on struct dma_buf
  * @attach: [optional] allows different devices to 'attach' themselves to the
@@ -135,6 +138,7 @@ struct dma_buf {
        unsigned vmapping_counter;
        void *vmap_ptr;
        const char *exp_name;
+       unsigned long flags;
        struct module *owner;
        struct list_head list_node;
        void *priv;
@@ -167,6 +171,9 @@ struct dma_buf_attachment {
        struct device *dev;
        struct list_head node;
        void *priv;
+       struct sg_table *sg_table;
+       atomic_t ref;
+       atomic_t maps;
 };
 
 /**
@@ -188,6 +195,7 @@ struct dma_buf_export_info {
        const struct dma_buf_ops *ops;
        size_t size;
        int flags;
+       int exp_flags;
        struct reservation_object *resv;
        void *priv;
 };