arm64: guarantee correct tlb flushes with preeption on

We need to guarantee that our thread hasn't switched
cores between being asked to flush the local core's
tlb and having actually performed the task. If it
has, we need to perform a global tlbi.

Change-Id: I4b1bc5fbe53a7d35a2442753d8fe3f0ae86415ac
Signed-off-by: Rich Wiley <rwiley@nvidia.com>
Reviewed-on: http://git-master/r/433805
GVS: Gerrit_Virtual_Submit
Reviewed-by: Peng Du <pdu@nvidia.com>
Reviewed-by: Alexander Van Brunt <avanbrunt@nvidia.com>
Reviewed-by: Mitch Luban <mluban@nvidia.com>

diff --git a/arch/arm64/include/asm/tlbflush.h b/arch/arm64/include/asm/tlbflush.h
index 875f7440305f5c674ef78956f7d69d8c35e45e97..7d2fe5be486d5c407094e27ad4ecfd07bc6192d0 100644 (file)
--- a/arch/arm64/include/asm/tlbflush.h
+++ b/arch/arm64/include/asm/tlbflush.h
@@ -108,7 +108,8 @@ static inline void flush_tlb_mm(struct mm_struct *mm)
        dsb();
        if (cpumask_ran_on_only_one(mm_cpumask(mm)))
                asm("tlbi       aside1, %0" : : "r" (asid));
-       else
+       barrier();
+       if (!cpumask_ran_on_only_one(mm_cpumask(mm)))
                asm("tlbi       aside1is, %0" : : "r" (asid));
        dsb();
 }
@@ -122,7 +123,8 @@ static inline void flush_tlb_page(struct vm_area_struct *vma,
        dsb();
        if (cpumask_ran_on_only_one(mm_cpumask(vma->vm_mm)))
                asm("tlbi       vae1, %0" : : "r" (addr));
-       else
+       barrier();
+       if (!cpumask_ran_on_only_one(mm_cpumask(vma->vm_mm)))
                asm("tlbi       vae1is, %0" : : "r" (addr));
        dsb();
 }
@@ -130,16 +132,19 @@ static inline void flush_tlb_page(struct vm_area_struct *vma,
 static inline void flush_tlb_range(struct vm_area_struct *vma,
                                        unsigned long start, unsigned long end)
 {
-       if (cpumask_ran_on_only_one(mm_cpumask(vma->vm_mm)))
+       if (cpumask_ran_on_only_one(mm_cpumask(vma->vm_mm))){
                if (end - start > FLUSH_TLB_ALL_THRESHOLD)
                        local_flush_tlb_all();
                else
                        __local_cpu_flush_user_tlb_range(start,end,vma);
-       else
+       }
+       barrier();
+       if (!cpumask_ran_on_only_one(mm_cpumask(vma->vm_mm))){
                if (end - start > FLUSH_TLB_ALL_THRESHOLD)
                        flush_tlb_all();
                else
                        __cpu_flush_user_tlb_range(start,end,vma);
+       }
 }
 
 static inline void flush_tlb_kernel_range(unsigned long start,