arm: Complete paging invalidations

This patch is based on the original version by Jean-Philippe Brucker. It
fills different paging stubs:
- the arch_flush_cell_vcpu_caches stub, which is used by the core via
  config_commit each time the memory is remapped. It allows to
  invalidate the TLBs on all affected CPUs of the cell.
- the arch_paging_flush_cpu_caches function is used to flush the
  hypervisor page table entries when using the PAGE_MAP_COHERENT flag
  (useful for IOMMU, not currently in use on the arm side.)
- the arch_paging_flush_page_tlbs function is used to invalidate a TLB
  entry after modifying the hypervisor paging structures. It must ignore
  accesses done from the initial setup code at EL1, which are committed
  once at EL2 with a TLBIALLH, just before enabling the MMU.

arch_config_commit has nothing to do so far. Will change when an IOMMU is
supported.

Signed-off-by: Jan Kiszka <jan.kiszka@siemens.com>

diff --git a/hypervisor/arch/arm/control.c b/hypervisor/arch/arm/control.c
index 4d7546ec425c867049fd272f8e1d09f615de9efe..27af483059413d5d7718f47256909a6c58032298 100644 (file)
--- a/hypervisor/arch/arm/control.c
+++ b/hypervisor/arch/arm/control.c
@@ -134,6 +134,9 @@ static void arch_reset_self(struct per_cpu *cpu_data)
 static void arch_suspend_self(struct per_cpu *cpu_data)
 {
        psci_suspend(cpu_data);
+
+       if (cpu_data->flush_vcpu_caches)
+               arch_cpu_tlb_flush(cpu_data);
 }
 
 struct registers* arch_handle_exit(struct per_cpu *cpu_data,
@@ -241,3 +244,19 @@ void arch_cell_destroy(struct cell *cell)
        for_each_cpu(cpu, cell->cpu_set)
                arch_reset_cpu(cpu);
 }
+
+/* Note: only supports synchronous flushing as triggered by config_commit! */
+void arch_flush_cell_vcpu_caches(struct cell *cell)
+{
+       unsigned int cpu;
+
+       for_each_cpu(cpu, cell->cpu_set)
+               if (cpu == this_cpu_id())
+                       arch_cpu_tlb_flush(per_cpu(cpu));
+               else
+                       per_cpu(cpu)->flush_vcpu_caches = true;
+}
+
+void arch_config_commit(struct cell *cell_added_removed)
+{
+}

diff --git a/hypervisor/arch/arm/include/asm/control.h b/hypervisor/arch/arm/include/asm/control.h
index 592ee29f76a33159e9223df033bdc607af4229d3..bb97ff3ace5bb76713b4c99f3e3643b67a790f53 100644 (file)
--- a/hypervisor/arch/arm/include/asm/control.h
+++ b/hypervisor/arch/arm/include/asm/control.h
@@ -26,6 +26,7 @@
 
 void arch_cpu_dcaches_flush(unsigned int action);
 void arch_cpu_icache_flush(void);
+void arch_cpu_tlb_flush(struct per_cpu *cpu_data);
 void arch_cell_caches_flush(struct cell *cell);
 int arch_mmu_cell_init(struct cell *cell);
 void arch_mmu_cell_destroy(struct cell *cell);

diff --git a/hypervisor/arch/arm/include/asm/paging.h b/hypervisor/arch/arm/include/asm/paging.h
index 3e32aae10b7fd0810b8773438e9ffefd31847a23..872bdb8615a9ae2a34476a8cf3d203e4151173da 100644 (file)
--- a/hypervisor/arch/arm/include/asm/paging.h
+++ b/hypervisor/arch/arm/include/asm/paging.h
@@ -16,6 +16,7 @@
 #include <jailhouse/types.h>
 #include <jailhouse/utils.h>
 #include <asm/processor.h>
+#include <asm/sysregs.h>
 
 #define PAGE_SIZE              4096
 #define PAGE_MASK              ~(PAGE_SIZE - 1)
@@ -176,12 +177,28 @@
 
 typedef u64 *pt_entry_t;
 
+/* Only executed on hypervisor paging struct changes */
 static inline void arch_paging_flush_page_tlbs(unsigned long page_addr)
 {
+       /*
+        * This instruction is UNDEF at EL1, but the whole TLB is invalidated
+        * before enabling the EL2 stage 1 MMU anyway.
+        */
+       if (is_el2())
+               arm_write_sysreg(TLBIMVAH, page_addr & PAGE_MASK);
 }
 
+extern unsigned int cache_line_size;
+
+/* Used to clean the PAGE_MAP_COHERENT page table changes */
 static inline void arch_paging_flush_cpu_caches(void *addr, long size)
 {
+       do {
+               /* Clean & invalidate by MVA to PoC */
+               arm_write_sysreg(DCCIMVAC, addr);
+               size -= cache_line_size;
+               addr += cache_line_size;
+       } while (size > 0);
 }
 
 #endif /* !__ASSEMBLY__ */

diff --git a/hypervisor/arch/arm/include/asm/percpu.h b/hypervisor/arch/arm/include/asm/percpu.h
index 510316569391eb6508426a4733adc9001f3f64be..9d0c06fd627d055d137e2c793a9310abf2d77a77 100644 (file)
--- a/hypervisor/arch/arm/include/asm/percpu.h
+++ b/hypervisor/arch/arm/include/asm/percpu.h
@@ -58,7 +58,7 @@ struct per_cpu {
        /* The mbox will be accessed with a ldrd, which requires alignment */
        __attribute__((aligned(8))) struct psci_mbox psci_mbox;
 
-       bool flush_caches;
+       bool flush_vcpu_caches;
        int shutdown_state;
        bool failed;
 } __attribute__((aligned(PAGE_SIZE)));

diff --git a/hypervisor/arch/arm/include/asm/processor.h b/hypervisor/arch/arm/include/asm/processor.h
index 451f55c2d71329052f9c437eb7d79485630b3e81..0dbd4dfaa42a8c2984065333d16dd477f6200471 100644 (file)
--- a/hypervisor/arch/arm/include/asm/processor.h
+++ b/hypervisor/arch/arm/include/asm/processor.h
@@ -13,6 +13,7 @@
 #ifndef _JAILHOUSE_ASM_PROCESSOR_H
 #define _JAILHOUSE_ASM_PROCESSOR_H
 
+#include <jailhouse/types.h>
 #include <jailhouse/utils.h>
 
 #define PSR_MODE_MASK  0xf
@@ -176,6 +177,15 @@ static inline void memory_load_barrier(void)
 {
 }
 
+static inline bool is_el2(void)
+{
+       u32 psr;
+
+       asm volatile ("mrs      %0, cpsr" : "=r" (psr));
+
+       return (psr & PSR_MODE_MASK) == PSR_HYP_MODE;
+}
+
 #endif /* !__ASSEMBLY__ */
 
 #endif /* !_JAILHOUSE_ASM_PROCESSOR_H */

diff --git a/hypervisor/arch/arm/include/asm/sysregs.h b/hypervisor/arch/arm/include/asm/sysregs.h
index 347ad047e0626074b4ab9db1d67a4cdb943a3f2a..1f9abeb4881771360c7028724d9976fe1079182c 100644 (file)
--- a/hypervisor/arch/arm/include/asm/sysregs.h
+++ b/hypervisor/arch/arm/include/asm/sysregs.h
@@ -29,6 +29,7 @@
  * 32bit sysregs definitions
  * (Use the AArch64 names to ease the compatibility work)
  */
+#define CTR_EL0                SYSREG_32(0, c0, c0, 1)
 #define MPIDR_EL1      SYSREG_32(0, c0, c0, 5)
 #define ID_PFR0_EL1    SYSREG_32(0, c0, c1, 0)
 #define ID_PFR1_EL1    SYSREG_32(0, c0, c1, 1)
@@ -94,6 +95,7 @@
 
 #define ICIALLUIS      SYSREG_32(0, c7, c1, 0)
 #define ICIALLU                SYSREG_32(0, c7, c5, 0)
+#define DCCIMVAC       SYSREG_32(0, c7, c10, 1)
 #define DCCSW          SYSREG_32(0, c7, c10, 2)
 #define DCCISW         SYSREG_32(0, c7, c14, 2)
 

diff --git a/hypervisor/arch/arm/mmu_cell.c b/hypervisor/arch/arm/mmu_cell.c
index 90e06543e6971535520bf34f301ec28d4b6071de..e69fb04c428c5c11990860db1aaf7a534474d8dd 100644 (file)
--- a/hypervisor/arch/arm/mmu_cell.c
+++ b/hypervisor/arch/arm/mmu_cell.c
@@ -95,14 +95,27 @@ int arch_mmu_cpu_cell_init(struct per_cpu *cpu_data)
        arm_write_sysreg(VTTBR_EL2, vttbr);
        arm_write_sysreg(VTCR_EL2, vtcr);
 
+       /* Ensure that the new VMID is present before flushing the caches */
        isb();
+       /*
+        * At initialisation, arch_config_commit does not act on other CPUs,
+        * since they register themselves to the root cpu_set afterwards. It
+        * means that this unconditionnal flush is redundant on master CPU.
+        */
+       arch_cpu_tlb_flush(cpu_data);
+
+       return 0;
+}
+
+void arch_cpu_tlb_flush(struct per_cpu *cpu_data)
+{
        /*
         * Invalidate all stage-1 and 2 TLB entries for the current VMID
         * ERET will ensure completion of these ops
         */
        arm_write_sysreg(TLBIALL, 1);
-
-       return 0;
+       dsb(nsh);
+       cpu_data->flush_vcpu_caches = false;
 }
 
 void arch_cell_caches_flush(struct cell *cell)

diff --git a/hypervisor/arch/arm/setup.c b/hypervisor/arch/arm/setup.c
index ae17a09c27ebbda79ca29b6a0e70105dda49fde6..ef408a8afbd9a5642a472dbf04a33260f1714e70 100644 (file)
--- a/hypervisor/arch/arm/setup.c
+++ b/hypervisor/arch/arm/setup.c
@@ -21,14 +21,22 @@
 #include <jailhouse/paging.h>
 #include <jailhouse/string.h>
 
+unsigned int cache_line_size;
+
 static int arch_check_features(void)
 {
        u32 pfr1;
+       u32 ctr;
+
        arm_read_sysreg(ID_PFR1_EL1, pfr1);
 
        if (!PFR1_VIRT(pfr1))
                return -ENODEV;
 
+       arm_read_sysreg(CTR_EL0, ctr);
+       /* Extract the minimal cache line size */
+       cache_line_size = 4 << (ctr >> 16 & 0xf);
+
        return 0;
 }
 
@@ -112,8 +120,6 @@ void arch_cpu_restore(struct per_cpu *cpu_data)
 
 // catch missing symbols
 void arch_shutdown_cpu(unsigned int cpu_id) {}
-void arch_flush_cell_vcpu_caches(struct cell *cell) {}
-void arch_config_commit(struct cell *cell_added_removed) {}
 void arch_shutdown(void) {}
 void arch_panic_stop(void) {__builtin_unreachable();}
 void arch_panic_park(void) {}