mm: cma: migrate cma pages during __get_user_pages

This is necessary to avoid CMA allocation failures.
serialize page following to migration failures resulting
from additonal refcount on page.
Also, Avoid refcount on page during __migration_entry_wait to
ensure that migration don't cause a livelock between
migration retries and user process continuous access to
same page.

Bug 1550455

Change-Id: I007d0e0c262cc0c6776e185654a770f5ebbee112
Signed-off-by: Krishna Reddy <vdumpa@nvidia.com>
Reviewed-on: http://git-master/r/552943

diff --git a/drivers/base/dma-coherent.c b/drivers/base/dma-coherent.c
index 9ad58f818782f2ccd51dda2efa093efc35f79ea1..7836183fd964fad2ac7b0e1844ce6e36463a4d81 100644 (file)
--- a/drivers/base/dma-coherent.c
+++ b/drivers/base/dma-coherent.c
@@ -425,11 +425,11 @@ alloc_success:
 
                if (PageHighMem(page)) {
                        void *ptr = kmap_atomic(page);
-                       dmac_flush_range(ptr, ptr + PAGE_SIZE);
+//                     dmac_flush_range(ptr, ptr + PAGE_SIZE);
                        kunmap_atomic(ptr);
                } else {
                        void *ptr = page_address(page);
-                       dmac_flush_range(ptr, ptr + PAGE_SIZE);
+//                     dmac_flush_range(ptr, ptr + PAGE_SIZE);
                }
        }
 

diff --git a/include/linux/pagemap.h b/include/linux/pagemap.h
index 26eabf5ec718a457eb9cef5635f5c475cca0c48d..c22b2350d262d41f793e40da55435d4c5814fff1 100644 (file)
--- a/include/linux/pagemap.h
+++ b/include/linux/pagemap.h
@@ -491,6 +491,8 @@ static inline int lock_page_or_retry(struct page *page, struct mm_struct *mm,
  */
 extern void wait_on_page_bit(struct page *page, int bit_nr);
 
+extern void wait_on_page_bit_timeout(struct page *page, int bit_nr);
+
 extern int wait_on_page_bit_killable(struct page *page, int bit_nr);
 extern int wait_on_page_bit_killable_timeout(struct page *page,
                                             int bit_nr, unsigned long timeout);
@@ -521,6 +523,12 @@ static inline void wait_on_page_locked(struct page *page)
                wait_on_page_bit(page, PG_locked);
 }
 
+static inline void wait_on_page_locked_timeout(struct page *page)
+{
+       if (PageLocked(page))
+               wait_on_page_bit_timeout(page, PG_locked);
+}
+
 /* 
  * Wait for a page to complete writeback
  */

diff --git a/mm/filemap.c b/mm/filemap.c
index 1bb007624b53e1cc086ae26d91238d6c3683d6b2..8049f4b8529d286657bd75fa4464cec229b47dfd 100644 (file)
--- a/mm/filemap.c
+++ b/mm/filemap.c
@@ -251,6 +251,11 @@ void delete_from_page_cache(struct page *page)
 }
 EXPORT_SYMBOL(delete_from_page_cache);
 
+static int sleep_on_page_timeout(struct wait_bit_key *word)
+{
+       return io_schedule_timeout(2) ? 0 : -EAGAIN;
+}
+
 static int filemap_check_errors(struct address_space *mapping)
 {
        int ret = 0;
@@ -753,6 +758,16 @@ void wait_on_page_bit(struct page *page, int bit_nr)
 }
 EXPORT_SYMBOL(wait_on_page_bit);
 
+void wait_on_page_bit_timeout(struct page *page, int bit_nr)
+{
+       DEFINE_WAIT_BIT(wait, &page->flags, bit_nr);
+
+       if (test_bit(bit_nr, &page->flags))
+               __wait_on_bit(page_waitqueue(page), &wait,
+                               sleep_on_page_timeout, TASK_UNINTERRUPTIBLE);
+}
+EXPORT_SYMBOL(wait_on_page_bit_timeout);
+
 int wait_on_page_bit_killable(struct page *page, int bit_nr)
 {
        DEFINE_WAIT_BIT(wait, &page->flags, bit_nr);

diff --git a/mm/gup.c b/mm/gup.c
index e4befd7a790a24a468f669d8d665578dd4a64f58..805b675762956e33e80957bfa302cd60c143ecee 100644 (file)
--- a/mm/gup.c
+++ b/mm/gup.c
@@ -424,8 +424,10 @@ static inline struct page *migrate_replace_cma_page(struct page *page)
         */
        get_page_foll(newpage);
 
-       if (migrate_replace_page(page, newpage) == 0)
+       if (migrate_replace_page(page, newpage) == 0) {
+               put_page(newpage);
                return newpage;
+       }
 
        put_page(newpage);
        __free_page(newpage);
@@ -519,6 +521,7 @@ long __get_user_pages(struct task_struct *tsk, struct mm_struct *mm,
                struct page *page;
                unsigned int foll_flags = gup_flags;
                unsigned int page_increm;
+               static DEFINE_MUTEX(s_follow_page_lock);
 
                /* first iteration or cross vma bound */
                if (!vma || start >= vma->vm_end) {
@@ -551,11 +554,13 @@ retry:
                if (unlikely(fatal_signal_pending(current)))
                        return i ? i : -ERESTARTSYS;
                cond_resched();
+               mutex_lock(&s_follow_page_lock);
                page = follow_page_mask(vma, start, foll_flags, &page_mask);
                if (!page) {
                        int ret;
                        ret = faultin_page(tsk, vma, start, &foll_flags,
                                        nonblocking);
+                       mutex_unlock(&s_follow_page_lock);
                        switch (ret) {
                        case 0:
                                goto retry;
@@ -579,8 +584,46 @@ retry:
                        return i ? i : PTR_ERR(page);
                }
 
-               if ((gup_flags & FOLL_DURABLE) && is_cma_page(page))
+               if (is_cma_page(page) && (foll_flags & FOLL_GET)) {
+                       struct page *old_page = page;
+                       unsigned int fault_flags = 0;
+
+                       put_page(page);
+                       wait_on_page_locked_timeout(page);
                        page = migrate_replace_cma_page(page);
+                       /* migration might be successful. vma mapping
+                        * might have changed if there had been a write
+                        * fault from other accesses before migration
+                        * code locked the page. Follow the page again
+                        * to get the latest mapping. If migration was
+                        * successful, follow again would get
+                        * non-CMA page. If there had been a write
+                        * page fault, follow page and CMA page
+                        * replacement(if necessary) would restart with
+                        * new page.
+                        */
+                       if (page == old_page)
+                               wait_on_page_locked_timeout(page);
+                       if (foll_flags & FOLL_WRITE) {
+                               /* page would be marked as old during
+                                * migration. To make it young, call
+                                * handle_mm_fault.
+                                * This to avoid the sanity check
+                                * failures in the calling code, which
+                                * check for pte write permission
+                                * bits.
+                                */
+                               fault_flags |= FAULT_FLAG_WRITE;
+                               handle_mm_fault(mm, vma,
+                                       start, fault_flags);
+                       }
+                       foll_flags = gup_flags;
+                       mutex_unlock(&s_follow_page_lock);
+                       goto retry;
+               }
+
+               mutex_unlock(&s_follow_page_lock);
+               BUG_ON(is_cma_page(page) && (foll_flags & FOLL_GET));
 
                if (pages) {
                        pages[i] = page;

diff --git a/mm/migrate.c b/mm/migrate.c
index 8a88d916c7abd276c34382351d2061ba4630c512..f12e096bb1104767a24d970dfb6a392baceb6288 100644 (file)
--- a/mm/migrate.c
+++ b/mm/migrate.c
@@ -219,18 +219,31 @@ void __migration_entry_wait(struct mm_struct *mm, pte_t *ptep,
 
        page = migration_entry_to_page(entry);
 
-       /*
-        * Once radix-tree replacement of page migration started, page_count
-        * *must* be zero. And, we don't want to call wait_on_page_locked()
-        * against a page without get_page().
-        * So, we use get_page_unless_zero(), here. Even failed, page fault
-        * will occur again.
-        */
-       if (!get_page_unless_zero(page))
-               goto out;
-       pte_unmap_unlock(ptep, ptl);
-       wait_on_page_locked(page);
-       put_page(page);
+       if (is_cma_page(page)) {
+               pte_unmap_unlock(ptep, ptl);
+               /* don't take ref on page, as it causes
+                * migration to get aborted in between.
+                * migration goes ahead after locking the page.
+                * Wait on page to be unlocked. In case page get
+                * unlocked, allocated and locked again forever,
+                * before this function call, it would timeout in
+                * next tick and exit.
+                */
+               wait_on_page_locked_timeout(page);
+       } else {
+               /*
+                * Once radix-tree replacement of page migration started, page_count
+                * *must* be zero. And, we don't want to call wait_on_page_locked()
+                * against a page without get_page().
+                * So, we use get_page_unless_zero(), here. Even failed, page fault
+                * will occur again.
+                */
+               if (!get_page_unless_zero(page))
+                       goto out;
+               pte_unmap_unlock(ptep, ptl);
+               wait_on_page_locked(page);
+               put_page(page);
+       }
        return;
 out:
        pte_unmap_unlock(ptep, ptl);
@@ -1219,9 +1232,6 @@ int migrate_replace_page(struct page *page, struct page *newpage)
                return -EAGAIN;
        }
 
-       /* page is now isolated, so release additional reference */
-       put_page(page);
-
        for (pass = 0; pass < 10 && ret != 0; pass++) {
                cond_resched();