core: Add support for per-CPU statistics

Record VM exits on a per-CPU basis and export this information via the
"CPU Get Info" hypercall.

Signed-off-by: Jan Kiszka <jan.kiszka@siemens.com>

diff --git a/Documentation/hypervisor-interfaces.txt b/Documentation/hypervisor-interfaces.txt
index a9fa43a8fe164620ac32c5c3ad883a060a6bef66..82cd6b59939ad57935406443469891eb249d6239 100644 (file)
--- a/Documentation/hypervisor-interfaces.txt
+++ b/Documentation/hypervisor-interfaces.txt
@@ -186,6 +186,16 @@ Obtain information about a specific CPU.
 Arguments: 1. Logical ID of CPU to be queried
            2. Information type:
                   0 - CPU state
+               1000 - Total number of VM exits
+               1001 - VM exits due to MMIO access
+               1002 - VM exits due to PIO access
+               1003 - VM exits due to IPI submissions
+               1004 - VM exits due to management events
+               1005 - VM exits due to hypercalls
+
+Statistic counters are reset when a CPU is assigned to a different cell. The
+total number of VM exits may be different from the sum of all specific VM exit
+counters.
 
 Return code: Requested value (>=0) or negative error code
 

diff --git a/hypervisor/arch/arm/include/asm/jailhouse_hypercall.h b/hypervisor/arch/arm/include/asm/jailhouse_hypercall.h
index ab70376135f39d0078b5a1b1df41f567d94543d0..7a07f2749f6a9ff9c953e0591bd4978b7331511d 100644 (file)
--- a/hypervisor/arch/arm/include/asm/jailhouse_hypercall.h
+++ b/hypervisor/arch/arm/include/asm/jailhouse_hypercall.h
@@ -18,6 +18,9 @@
 #define JAILHOUSE_CALL_ARG1            "r1"
 #define JAILHOUSE_CALL_ARG2            "r2"
 
+/* CPU statistics */
+#define JAILHOUSE_NUM_CPU_STATS                        JAILHOUSE_GENERIC_CPU_STATS
+
 #ifndef __asmeq
 #define __asmeq(x, y)  ".ifnc " x "," y " ; .err ; .endif\n\t"
 #endif

diff --git a/hypervisor/arch/arm/include/asm/percpu.h b/hypervisor/arch/arm/include/asm/percpu.h
index 103337da316eb17218587f62b65416ea7a474900..c8aaf09407d116a4b3442f25b316ebaab3ba61f0 100644 (file)
--- a/hypervisor/arch/arm/include/asm/percpu.h
+++ b/hypervisor/arch/arm/include/asm/percpu.h
@@ -36,6 +36,8 @@ struct per_cpu {
 //     u32 apic_id;
        struct cell *cell;
 
+       u32 stats[JAILHOUSE_NUM_CPU_STATS];
+
        unsigned long linux_reg[NUM_ENTRY_REGS];
 //     unsigned long linux_ip;
        bool initialized;

diff --git a/hypervisor/arch/x86/include/asm/jailhouse_hypercall.h b/hypervisor/arch/x86/include/asm/jailhouse_hypercall.h
index 5887c8721d2a2a5d24bf7f34c5d58f0165fe9f53..9145e6e778da68c42a6801c0fff6dc558cc7fe02 100644 (file)
--- a/hypervisor/arch/x86/include/asm/jailhouse_hypercall.h
+++ b/hypervisor/arch/x86/include/asm/jailhouse_hypercall.h
@@ -22,6 +22,15 @@
 #define JAILHOUSE_CALL_ARG1    "D" (arg1)
 #define JAILHOUSE_CALL_ARG2    "S" (arg2)
 
+/* CPU statistics */
+#define JAILHOUSE_CPU_STAT_VMEXITS_PIO         JAILHOUSE_GENERIC_CPU_STATS
+#define JAILHOUSE_CPU_STAT_VMEXITS_XAPIC       JAILHOUSE_GENERIC_CPU_STATS + 1
+#define JAILHOUSE_CPU_STAT_VMEXITS_CR          JAILHOUSE_GENERIC_CPU_STATS + 2
+#define JAILHOUSE_CPU_STAT_VMEXITS_MSR         JAILHOUSE_GENERIC_CPU_STATS + 3
+#define JAILHOUSE_CPU_STAT_VMEXITS_CPUID       JAILHOUSE_GENERIC_CPU_STATS + 4
+#define JAILHOUSE_CPU_STAT_VMEXITS_XSETBV      JAILHOUSE_GENERIC_CPU_STATS + 5
+#define JAILHOUSE_NUM_CPU_STATS                        JAILHOUSE_GENERIC_CPU_STATS + 6
+
 #ifndef __ASSEMBLY__
 
 static inline __u32 jailhouse_call(__u32 num)

diff --git a/hypervisor/arch/x86/include/asm/percpu.h b/hypervisor/arch/x86/include/asm/percpu.h
index 1f0cccd31d2645330dcdd15832a3cfa03eaea2af..dbddbca810aa22974be0a31da0d9340052fbcc75 100644 (file)
--- a/hypervisor/arch/x86/include/asm/percpu.h
+++ b/hypervisor/arch/x86/include/asm/percpu.h
@@ -17,6 +17,8 @@
 #include <asm/paging.h>
 #include <asm/processor.h>
 
+#include <jailhouse/hypercall.h>
+
 #define NUM_ENTRY_REGS                 6
 
 /* Keep in sync with struct per_cpu! */
@@ -45,6 +47,8 @@ struct per_cpu {
        u32 apic_id;
        struct cell *cell;
 
+       u32 stats[JAILHOUSE_NUM_CPU_STATS];
+
        struct desc_table_reg linux_gdtr;
        struct desc_table_reg linux_idtr;
        unsigned long linux_reg[NUM_ENTRY_REGS];

diff --git a/hypervisor/arch/x86/vmx.c b/hypervisor/arch/x86/vmx.c
index dd724e2ccef5c5a39467ef944bd8af0b8f3a45fc..33c9336926e58e10e7982507fc9ce12a631edb36 100644 (file)
--- a/hypervisor/arch/x86/vmx.c
+++ b/hypervisor/arch/x86/vmx.c
@@ -1072,11 +1072,14 @@ void vmx_handle_exit(struct registers *guest_regs, struct per_cpu *cpu_data)
        u32 reason = vmcs_read32(VM_EXIT_REASON);
        int sipi_vector;
 
+       cpu_data->stats[JAILHOUSE_CPU_STAT_VMEXITS_TOTAL]++;
+
        switch (reason) {
        case EXIT_REASON_EXCEPTION_NMI:
                asm volatile("int %0" : : "i" (NMI_VECTOR));
                /* fall through */
        case EXIT_REASON_PREEMPTION_TIMER:
+               cpu_data->stats[JAILHOUSE_CPU_STAT_VMEXITS_MANAGEMENT]++;
                vmx_disable_preemption_timer();
                sipi_vector = x86_handle_events(cpu_data);
                if (sipi_vector >= 0) {
@@ -1100,10 +1103,12 @@ void vmx_handle_exit(struct registers *guest_regs, struct per_cpu *cpu_data)
                vmx_handle_hypercall(guest_regs, cpu_data);
                return;
        case EXIT_REASON_CR_ACCESS:
+               cpu_data->stats[JAILHOUSE_CPU_STAT_VMEXITS_CR]++;
                if (vmx_handle_cr(guest_regs, cpu_data))
                        return;
                break;
        case EXIT_REASON_MSR_READ:
+               cpu_data->stats[JAILHOUSE_CPU_STAT_VMEXITS_MSR]++;
                if (guest_regs->rcx >= MSR_X2APIC_BASE &&
                    guest_regs->rcx <= MSR_X2APIC_END) {
                        vmx_skip_emulated_instruction(X86_INST_LEN_RDMSR);
@@ -1114,6 +1119,7 @@ void vmx_handle_exit(struct registers *guest_regs, struct per_cpu *cpu_data)
                             guest_regs->rcx);
                break;
        case EXIT_REASON_MSR_WRITE:
+               cpu_data->stats[JAILHOUSE_CPU_STAT_VMEXITS_MSR]++;
                if (guest_regs->rcx == MSR_X2APIC_ICR) {
                        if (!apic_handle_icr_write(cpu_data, guest_regs->rax,
                                                   guest_regs->rdx))
@@ -1131,10 +1137,12 @@ void vmx_handle_exit(struct registers *guest_regs, struct per_cpu *cpu_data)
                             guest_regs->rcx);
                break;
        case EXIT_REASON_APIC_ACCESS:
+               cpu_data->stats[JAILHOUSE_CPU_STAT_VMEXITS_XAPIC]++;
                if (vmx_handle_apic_access(guest_regs, cpu_data))
                        return;
                break;
        case EXIT_REASON_XSETBV:
+               cpu_data->stats[JAILHOUSE_CPU_STAT_VMEXITS_XSETBV]++;
                if (guest_regs->rax & X86_XCR0_FP &&
                    (guest_regs->rax & ~cpuid_eax(0x0d)) == 0 &&
                    guest_regs->rcx == 0 && guest_regs->rdx == 0) {
@@ -1150,10 +1158,12 @@ void vmx_handle_exit(struct registers *guest_regs, struct per_cpu *cpu_data)
                             guest_regs->rdx, guest_regs->rax);
                break;
        case EXIT_REASON_IO_INSTRUCTION:
+               cpu_data->stats[JAILHOUSE_CPU_STAT_VMEXITS_PIO]++;
                if (vmx_handle_io_access(guest_regs, cpu_data))
                        return;
                break;
        case EXIT_REASON_EPT_VIOLATION:
+               cpu_data->stats[JAILHOUSE_CPU_STAT_VMEXITS_MMIO]++;
                if (vmx_handle_ept_violation(guest_regs, cpu_data))
                        return;
                break;

diff --git a/hypervisor/control.c b/hypervisor/control.c
index 12676be2f64415d2ea88999a3d2399193e089637..274a0d75e3bf8f23efe6752c673c5f6521e33283 100644 (file)
--- a/hypervisor/control.c
+++ b/hypervisor/control.c
@@ -16,6 +16,7 @@
 #include <jailhouse/paging.h>
 #include <jailhouse/processor.h>
 #include <jailhouse/string.h>
+#include <jailhouse/utils.h>
 #include <asm/bitops.h>
 #include <asm/spinlock.h>
 
@@ -271,6 +272,7 @@ static void cell_destroy_internal(struct per_cpu *cpu_data, struct cell *cell)
                set_bit(cpu, root_cell.cpu_set->bitmap);
                per_cpu(cpu)->cell = &root_cell;
                per_cpu(cpu)->failed = false;
+               memset(per_cpu(cpu)->stats, 0, sizeof(per_cpu(cpu)->stats));
        }
 
        for (n = 0; n < cell->config->num_memory_regions; n++, mem++) {
@@ -384,6 +386,7 @@ static int cell_create(struct per_cpu *cpu_data, unsigned long config_address)
 
                clear_bit(cpu, root_cell.cpu_set->bitmap);
                per_cpu(cpu)->cell = cell;
+               memset(per_cpu(cpu)->stats, 0, sizeof(per_cpu(cpu)->stats));
        }
 
        /*
@@ -716,6 +719,10 @@ static int cpu_get_info(struct per_cpu *cpu_data, unsigned long cpu_id,
        if (type == JAILHOUSE_CPU_INFO_STATE) {
                return per_cpu(cpu_id)->failed ? JAILHOUSE_CPU_FAILED :
                        JAILHOUSE_CPU_RUNNING;
+       } else if (type >= JAILHOUSE_CPU_INFO_STAT_BASE &&
+               type - JAILHOUSE_CPU_INFO_STAT_BASE < JAILHOUSE_NUM_CPU_STATS) {
+               type -= JAILHOUSE_CPU_INFO_STAT_BASE;
+               return per_cpu(cpu_id)->stats[type] & BIT_MASK(30, 0);
        } else
                return -EINVAL;
 }
@@ -723,6 +730,8 @@ static int cpu_get_info(struct per_cpu *cpu_data, unsigned long cpu_id,
 long hypercall(struct per_cpu *cpu_data, unsigned long code,
               unsigned long arg1, unsigned long arg2)
 {
+       cpu_data->stats[JAILHOUSE_CPU_STAT_VMEXITS_HYPERCALL]++;
+
        switch (code) {
        case JAILHOUSE_HC_DISABLE:
                return shutdown(cpu_data);

diff --git a/hypervisor/include/jailhouse/hypercall.h b/hypervisor/include/jailhouse/hypercall.h
index 140e7fe65fdc8ed5d15801f67ebb50c946a2f56b..9cbc807b1f296c507ff7ee218b659ffbab80caf3 100644 (file)
--- a/hypervisor/include/jailhouse/hypercall.h
+++ b/hypervisor/include/jailhouse/hypercall.h
@@ -31,11 +31,19 @@
 
 /* Hypervisor information type */
 #define JAILHOUSE_CPU_INFO_STATE               0
+#define JAILHOUSE_CPU_INFO_STAT_BASE           1000
 
 /* CPU state */
 #define JAILHOUSE_CPU_RUNNING                  0
 #define JAILHOUSE_CPU_FAILED                   2 /* terminal state */
 
+/* CPU statistics */
+#define JAILHOUSE_CPU_STAT_VMEXITS_TOTAL       0
+#define JAILHOUSE_CPU_STAT_VMEXITS_MMIO                1
+#define JAILHOUSE_CPU_STAT_VMEXITS_MANAGEMENT  2
+#define JAILHOUSE_CPU_STAT_VMEXITS_HYPERCALL   3
+#define JAILHOUSE_GENERIC_CPU_STATS            4
+
 #define JAILHOUSE_MSG_NONE                     0
 
 /* messages to cell */
@@ -54,6 +62,8 @@
 #define JAILHOUSE_CELL_SHUT_DOWN               2 /* terminal state */
 #define JAILHOUSE_CELL_FAILED                  3 /* terminal state */
 
+#ifndef __ASSEMBLY__
+
 struct jailhouse_comm_region {
        volatile __u32 msg_to_cell;
        volatile __u32 reply_from_cell;
@@ -63,6 +73,8 @@ struct jailhouse_comm_region {
        /* errors etc. */
 };
 
+#endif /* !__ASSEMBLY__ */
+
 #include <asm/jailhouse_hypercall.h>
 
 #endif /* !_JAILHOUSE_HYPERCALL_H */