From d93a78c545e009049ca2e67616cdda2891ca82a2 Mon Sep 17 00:00:00 2001 From: Jan Kiszka Date: Sat, 14 Jun 2014 08:52:39 +0200 Subject: [PATCH] core: Add support for per-CPU statistics Record VM exits on a per-CPU basis and export this information via the "CPU Get Info" hypercall. Signed-off-by: Jan Kiszka --- Documentation/hypervisor-interfaces.txt | 10 ++++++++++ .../arch/arm/include/asm/jailhouse_hypercall.h | 3 +++ hypervisor/arch/arm/include/asm/percpu.h | 2 ++ .../arch/x86/include/asm/jailhouse_hypercall.h | 9 +++++++++ hypervisor/arch/x86/include/asm/percpu.h | 4 ++++ hypervisor/arch/x86/vmx.c | 10 ++++++++++ hypervisor/control.c | 9 +++++++++ hypervisor/include/jailhouse/hypercall.h | 12 ++++++++++++ 8 files changed, 59 insertions(+) diff --git a/Documentation/hypervisor-interfaces.txt b/Documentation/hypervisor-interfaces.txt index a9fa43a..82cd6b5 100644 --- a/Documentation/hypervisor-interfaces.txt +++ b/Documentation/hypervisor-interfaces.txt @@ -186,6 +186,16 @@ Obtain information about a specific CPU. Arguments: 1. Logical ID of CPU to be queried 2. Information type: 0 - CPU state + 1000 - Total number of VM exits + 1001 - VM exits due to MMIO access + 1002 - VM exits due to PIO access + 1003 - VM exits due to IPI submissions + 1004 - VM exits due to management events + 1005 - VM exits due to hypercalls + +Statistic counters are reset when a CPU is assigned to a different cell. The +total number of VM exits may be different from the sum of all specific VM exit +counters. Return code: Requested value (>=0) or negative error code diff --git a/hypervisor/arch/arm/include/asm/jailhouse_hypercall.h b/hypervisor/arch/arm/include/asm/jailhouse_hypercall.h index ab70376..7a07f27 100644 --- a/hypervisor/arch/arm/include/asm/jailhouse_hypercall.h +++ b/hypervisor/arch/arm/include/asm/jailhouse_hypercall.h @@ -18,6 +18,9 @@ #define JAILHOUSE_CALL_ARG1 "r1" #define JAILHOUSE_CALL_ARG2 "r2" +/* CPU statistics */ +#define JAILHOUSE_NUM_CPU_STATS JAILHOUSE_GENERIC_CPU_STATS + #ifndef __asmeq #define __asmeq(x, y) ".ifnc " x "," y " ; .err ; .endif\n\t" #endif diff --git a/hypervisor/arch/arm/include/asm/percpu.h b/hypervisor/arch/arm/include/asm/percpu.h index 103337d..c8aaf09 100644 --- a/hypervisor/arch/arm/include/asm/percpu.h +++ b/hypervisor/arch/arm/include/asm/percpu.h @@ -36,6 +36,8 @@ struct per_cpu { // u32 apic_id; struct cell *cell; + u32 stats[JAILHOUSE_NUM_CPU_STATS]; + unsigned long linux_reg[NUM_ENTRY_REGS]; // unsigned long linux_ip; bool initialized; diff --git a/hypervisor/arch/x86/include/asm/jailhouse_hypercall.h b/hypervisor/arch/x86/include/asm/jailhouse_hypercall.h index 5887c87..9145e6e 100644 --- a/hypervisor/arch/x86/include/asm/jailhouse_hypercall.h +++ b/hypervisor/arch/x86/include/asm/jailhouse_hypercall.h @@ -22,6 +22,15 @@ #define JAILHOUSE_CALL_ARG1 "D" (arg1) #define JAILHOUSE_CALL_ARG2 "S" (arg2) +/* CPU statistics */ +#define JAILHOUSE_CPU_STAT_VMEXITS_PIO JAILHOUSE_GENERIC_CPU_STATS +#define JAILHOUSE_CPU_STAT_VMEXITS_XAPIC JAILHOUSE_GENERIC_CPU_STATS + 1 +#define JAILHOUSE_CPU_STAT_VMEXITS_CR JAILHOUSE_GENERIC_CPU_STATS + 2 +#define JAILHOUSE_CPU_STAT_VMEXITS_MSR JAILHOUSE_GENERIC_CPU_STATS + 3 +#define JAILHOUSE_CPU_STAT_VMEXITS_CPUID JAILHOUSE_GENERIC_CPU_STATS + 4 +#define JAILHOUSE_CPU_STAT_VMEXITS_XSETBV JAILHOUSE_GENERIC_CPU_STATS + 5 +#define JAILHOUSE_NUM_CPU_STATS JAILHOUSE_GENERIC_CPU_STATS + 6 + #ifndef __ASSEMBLY__ static inline __u32 jailhouse_call(__u32 num) diff --git a/hypervisor/arch/x86/include/asm/percpu.h b/hypervisor/arch/x86/include/asm/percpu.h index 1f0cccd..dbddbca 100644 --- a/hypervisor/arch/x86/include/asm/percpu.h +++ b/hypervisor/arch/x86/include/asm/percpu.h @@ -17,6 +17,8 @@ #include #include +#include + #define NUM_ENTRY_REGS 6 /* Keep in sync with struct per_cpu! */ @@ -45,6 +47,8 @@ struct per_cpu { u32 apic_id; struct cell *cell; + u32 stats[JAILHOUSE_NUM_CPU_STATS]; + struct desc_table_reg linux_gdtr; struct desc_table_reg linux_idtr; unsigned long linux_reg[NUM_ENTRY_REGS]; diff --git a/hypervisor/arch/x86/vmx.c b/hypervisor/arch/x86/vmx.c index dd724e2..33c9336 100644 --- a/hypervisor/arch/x86/vmx.c +++ b/hypervisor/arch/x86/vmx.c @@ -1072,11 +1072,14 @@ void vmx_handle_exit(struct registers *guest_regs, struct per_cpu *cpu_data) u32 reason = vmcs_read32(VM_EXIT_REASON); int sipi_vector; + cpu_data->stats[JAILHOUSE_CPU_STAT_VMEXITS_TOTAL]++; + switch (reason) { case EXIT_REASON_EXCEPTION_NMI: asm volatile("int %0" : : "i" (NMI_VECTOR)); /* fall through */ case EXIT_REASON_PREEMPTION_TIMER: + cpu_data->stats[JAILHOUSE_CPU_STAT_VMEXITS_MANAGEMENT]++; vmx_disable_preemption_timer(); sipi_vector = x86_handle_events(cpu_data); if (sipi_vector >= 0) { @@ -1100,10 +1103,12 @@ void vmx_handle_exit(struct registers *guest_regs, struct per_cpu *cpu_data) vmx_handle_hypercall(guest_regs, cpu_data); return; case EXIT_REASON_CR_ACCESS: + cpu_data->stats[JAILHOUSE_CPU_STAT_VMEXITS_CR]++; if (vmx_handle_cr(guest_regs, cpu_data)) return; break; case EXIT_REASON_MSR_READ: + cpu_data->stats[JAILHOUSE_CPU_STAT_VMEXITS_MSR]++; if (guest_regs->rcx >= MSR_X2APIC_BASE && guest_regs->rcx <= MSR_X2APIC_END) { vmx_skip_emulated_instruction(X86_INST_LEN_RDMSR); @@ -1114,6 +1119,7 @@ void vmx_handle_exit(struct registers *guest_regs, struct per_cpu *cpu_data) guest_regs->rcx); break; case EXIT_REASON_MSR_WRITE: + cpu_data->stats[JAILHOUSE_CPU_STAT_VMEXITS_MSR]++; if (guest_regs->rcx == MSR_X2APIC_ICR) { if (!apic_handle_icr_write(cpu_data, guest_regs->rax, guest_regs->rdx)) @@ -1131,10 +1137,12 @@ void vmx_handle_exit(struct registers *guest_regs, struct per_cpu *cpu_data) guest_regs->rcx); break; case EXIT_REASON_APIC_ACCESS: + cpu_data->stats[JAILHOUSE_CPU_STAT_VMEXITS_XAPIC]++; if (vmx_handle_apic_access(guest_regs, cpu_data)) return; break; case EXIT_REASON_XSETBV: + cpu_data->stats[JAILHOUSE_CPU_STAT_VMEXITS_XSETBV]++; if (guest_regs->rax & X86_XCR0_FP && (guest_regs->rax & ~cpuid_eax(0x0d)) == 0 && guest_regs->rcx == 0 && guest_regs->rdx == 0) { @@ -1150,10 +1158,12 @@ void vmx_handle_exit(struct registers *guest_regs, struct per_cpu *cpu_data) guest_regs->rdx, guest_regs->rax); break; case EXIT_REASON_IO_INSTRUCTION: + cpu_data->stats[JAILHOUSE_CPU_STAT_VMEXITS_PIO]++; if (vmx_handle_io_access(guest_regs, cpu_data)) return; break; case EXIT_REASON_EPT_VIOLATION: + cpu_data->stats[JAILHOUSE_CPU_STAT_VMEXITS_MMIO]++; if (vmx_handle_ept_violation(guest_regs, cpu_data)) return; break; diff --git a/hypervisor/control.c b/hypervisor/control.c index 12676be..274a0d7 100644 --- a/hypervisor/control.c +++ b/hypervisor/control.c @@ -16,6 +16,7 @@ #include #include #include +#include #include #include @@ -271,6 +272,7 @@ static void cell_destroy_internal(struct per_cpu *cpu_data, struct cell *cell) set_bit(cpu, root_cell.cpu_set->bitmap); per_cpu(cpu)->cell = &root_cell; per_cpu(cpu)->failed = false; + memset(per_cpu(cpu)->stats, 0, sizeof(per_cpu(cpu)->stats)); } for (n = 0; n < cell->config->num_memory_regions; n++, mem++) { @@ -384,6 +386,7 @@ static int cell_create(struct per_cpu *cpu_data, unsigned long config_address) clear_bit(cpu, root_cell.cpu_set->bitmap); per_cpu(cpu)->cell = cell; + memset(per_cpu(cpu)->stats, 0, sizeof(per_cpu(cpu)->stats)); } /* @@ -716,6 +719,10 @@ static int cpu_get_info(struct per_cpu *cpu_data, unsigned long cpu_id, if (type == JAILHOUSE_CPU_INFO_STATE) { return per_cpu(cpu_id)->failed ? JAILHOUSE_CPU_FAILED : JAILHOUSE_CPU_RUNNING; + } else if (type >= JAILHOUSE_CPU_INFO_STAT_BASE && + type - JAILHOUSE_CPU_INFO_STAT_BASE < JAILHOUSE_NUM_CPU_STATS) { + type -= JAILHOUSE_CPU_INFO_STAT_BASE; + return per_cpu(cpu_id)->stats[type] & BIT_MASK(30, 0); } else return -EINVAL; } @@ -723,6 +730,8 @@ static int cpu_get_info(struct per_cpu *cpu_data, unsigned long cpu_id, long hypercall(struct per_cpu *cpu_data, unsigned long code, unsigned long arg1, unsigned long arg2) { + cpu_data->stats[JAILHOUSE_CPU_STAT_VMEXITS_HYPERCALL]++; + switch (code) { case JAILHOUSE_HC_DISABLE: return shutdown(cpu_data); diff --git a/hypervisor/include/jailhouse/hypercall.h b/hypervisor/include/jailhouse/hypercall.h index 140e7fe..9cbc807 100644 --- a/hypervisor/include/jailhouse/hypercall.h +++ b/hypervisor/include/jailhouse/hypercall.h @@ -31,11 +31,19 @@ /* Hypervisor information type */ #define JAILHOUSE_CPU_INFO_STATE 0 +#define JAILHOUSE_CPU_INFO_STAT_BASE 1000 /* CPU state */ #define JAILHOUSE_CPU_RUNNING 0 #define JAILHOUSE_CPU_FAILED 2 /* terminal state */ +/* CPU statistics */ +#define JAILHOUSE_CPU_STAT_VMEXITS_TOTAL 0 +#define JAILHOUSE_CPU_STAT_VMEXITS_MMIO 1 +#define JAILHOUSE_CPU_STAT_VMEXITS_MANAGEMENT 2 +#define JAILHOUSE_CPU_STAT_VMEXITS_HYPERCALL 3 +#define JAILHOUSE_GENERIC_CPU_STATS 4 + #define JAILHOUSE_MSG_NONE 0 /* messages to cell */ @@ -54,6 +62,8 @@ #define JAILHOUSE_CELL_SHUT_DOWN 2 /* terminal state */ #define JAILHOUSE_CELL_FAILED 3 /* terminal state */ +#ifndef __ASSEMBLY__ + struct jailhouse_comm_region { volatile __u32 msg_to_cell; volatile __u32 reply_from_cell; @@ -63,6 +73,8 @@ struct jailhouse_comm_region { /* errors etc. */ }; +#endif /* !__ASSEMBLY__ */ + #include #endif /* !_JAILHOUSE_HYPERCALL_H */ -- 2.39.2