x86: Park CPU in guest mode while waiting for SIPI

Instead of spinning inside the hypervisor while in wait-for-SIPI state,
introduce and use vmx_park_cpu. This functions stops the guest CPU in
halted state, interrupts disabled, so that only external events that
trigger a guest exit will wake it up again. This feature will also allow
us to keep the CPU for a longer period in the parked state without
burning lots of energy.

Signed-off-by: Jan Kiszka <jan.kiszka@siemens.com>

diff --git a/README b/README
index 9d7bf65c47e0f07d522a7bdfdafa267dac51e0f8..6e8f77bdd166c8cfd90a4effd402ebac270b4889 100644 (file)
--- a/README
+++ b/README
@@ -82,12 +82,12 @@ Demonstration in QEMU/KVM
 
 The included system configuration qemu-vm.c can be used to run Jailhouse in
 QEMU/KVM virtual machine on Intel x86 hosts. Currently it requires kvm.git,
-next branch on the host (in order to get support for nested unrestricted guest
-mode). 3.13 is expected to include all necessary feature for this test. You
-also need a Linux guest image with a recent kernel (tested with >= 3.9) and
-the ability to build a module for this kernel. Make sure the kvm-intel module
-was loaded with nested=1 to enable nested VMX support. Start the virtual
-machine as follows:
+next branch on the host (in order to get support for guest activity state HLT).
+3.14 is expected to include all necessary features for this setup. You also
+need a Linux guest image with a recent kernel (tested with >= 3.9) and the
+ability to build a module for this kernel. Make sure the kvm-intel module was
+loaded with nested=1 to enable nested VMX support. Start the virtual machine as
+follows:
 
     qemu-system-x86_64 LinuxInstallation.img -m 1G -enable-kvm -serial stdio \
         -cpu kvm64,-kvm_pv_eoi,-kvm_steal_time,-kvm_asyncpf,-kvmclock,+vmx,+x2apic \

diff --git a/hypervisor/arch/x86/apic.c b/hypervisor/arch/x86/apic.c
index 60f98067a862b732651bcdaa0939dbd299b1c457..04b47830cd587d2e3d17190ac681a4c4d9261938 100644 (file)
--- a/hypervisor/arch/x86/apic.c
+++ b/hypervisor/arch/x86/apic.c
@@ -181,6 +181,7 @@ void arch_resume_cpu(unsigned int cpu_id)
 /* target cpu has to be stopped */
 void arch_reset_cpu(unsigned int cpu_id)
 {
+       per_cpu(cpu_id)->wait_for_sipi = true;
        per_cpu(cpu_id)->sipi_vector = APIC_BSP_PSEUDO_SIPI;
 
        arch_resume_cpu(cpu_id);
@@ -206,20 +207,24 @@ void apic_nmi_handler(struct per_cpu *cpu_data)
 
 int apic_handle_events(struct per_cpu *cpu_data)
 {
+       int sipi_vector = -1;
+
        spin_lock(&wait_lock);
 
        do {
                if (cpu_data->init_signaled) {
                        cpu_data->init_signaled = false;
                        cpu_data->wait_for_sipi = true;
-               } else
-                       cpu_data->sipi_vector = -1;
+                       sipi_vector = -1;
+                       vmx_cpu_park();
+                       break;
+               }
 
                cpu_data->cpu_stopped = true;
 
                spin_unlock(&wait_lock);
 
-               while (cpu_data->wait_for_sipi || cpu_data->stop_cpu)
+               while (cpu_data->stop_cpu)
                        cpu_relax();
 
                if (cpu_data->shutdown_cpu) {
@@ -232,6 +237,11 @@ int apic_handle_events(struct per_cpu *cpu_data)
                spin_lock(&wait_lock);
 
                cpu_data->cpu_stopped = false;
+
+               if (cpu_data->wait_for_sipi) {
+                       cpu_data->wait_for_sipi = false;
+                       sipi_vector = cpu_data->sipi_vector;
+               }
        } while (cpu_data->init_signaled);
 
        if (cpu_data->flush_caches) {
@@ -242,7 +252,7 @@ int apic_handle_events(struct per_cpu *cpu_data)
 
        spin_unlock(&wait_lock);
 
-       return cpu_data->sipi_vector;
+       return sipi_vector;
 }
 
 static void apic_validate_ipi_mode(struct per_cpu *cpu_data, u32 lo_val)
@@ -276,6 +286,7 @@ static void apic_deliver_ipi(struct per_cpu *cpu_data,
                             u32 orig_icr_hi, u32 icr_lo)
 {
        struct per_cpu *target_data;
+       bool send_nmi;
 
        if (target_cpu_id == APIC_INVALID_ID ||
            !test_bit(target_cpu_id, cpu_data->cell->cpu_set->bitmap)) {
@@ -293,27 +304,26 @@ static void apic_deliver_ipi(struct per_cpu *cpu_data,
                printk("Ignoring NMI IPI\n");
                return;
        case APIC_ICR_DLVR_INIT:
-               spin_lock(&wait_lock);
-
-               if (!target_data->wait_for_sipi)
-                       target_data->init_signaled = true;
-
-               spin_unlock(&wait_lock);
-
-               apic_send_nmi_ipi(target_data);
-               return;
        case APIC_ICR_DLVR_SIPI:
-               target_data = per_cpu(target_cpu_id);
+               send_nmi = false;
 
                spin_lock(&wait_lock);
 
-               if (target_data->wait_for_sipi) {
-                       target_data->wait_for_sipi = false;
+               if ((icr_lo & APIC_ICR_DLVR_MASK) == APIC_ICR_DLVR_INIT) {
+                       if (!target_data->wait_for_sipi) {
+                               target_data->init_signaled = true;
+                               send_nmi = true;
+                       }
+               } else if (target_data->wait_for_sipi) {
                        target_data->sipi_vector =
                                icr_lo & APIC_ICR_VECTOR_MASK;
+                       send_nmi = true;
                }
 
                spin_unlock(&wait_lock);
+
+               if (send_nmi)
+                       apic_send_nmi_ipi(target_data);
                return;
        }
 

diff --git a/hypervisor/arch/x86/include/asm/processor.h b/hypervisor/arch/x86/include/asm/processor.h
index f25bb33596f050cc3851c43e2cb7d0d731e36862..61f734a3e9615ac84bdcd19e4702e30507575e16 100644 (file)
--- a/hypervisor/arch/x86/include/asm/processor.h
+++ b/hypervisor/arch/x86/include/asm/processor.h
@@ -38,6 +38,7 @@
 #define MSR_IA32_VMX_PROCBASED_CTLS                    0x00000482
 #define MSR_IA32_VMX_EXIT_CTLS                         0x00000483
 #define MSR_IA32_VMX_ENTRY_CTLS                                0x00000484
+#define MSR_IA32_VMX_MISC                              0x00000485
 #define MSR_IA32_VMX_CR0_FIXED0                                0x00000486
 #define MSR_IA32_VMX_CR0_FIXED1                                0x00000487
 #define MSR_IA32_VMX_CR4_FIXED0                                0x00000488

diff --git a/hypervisor/arch/x86/include/asm/vmx.h b/hypervisor/arch/x86/include/asm/vmx.h
index 74305914f1d96ae10d5c3b11ceb1266a3fbb09ee..4b9607140e5970f2dd98672a8d198bd5421659e0 100644 (file)
--- a/hypervisor/arch/x86/include/asm/vmx.h
+++ b/hypervisor/arch/x86/include/asm/vmx.h
@@ -178,6 +178,7 @@ enum vmcs_field {
 #define GUEST_SEG_BASE                 (GUEST_ES_BASE - GUEST_ES_SELECTOR)
 
 #define GUEST_ACTIVITY_ACTIVE                  0
+#define GUEST_ACTIVITY_HLT                     1
 
 #define VMX_MSR_BITMAP_0000_READ               0
 #define VMX_MSR_BITMAP_C000_READ               1
@@ -202,6 +203,8 @@ enum vmcs_field {
 #define VM_ENTRY_IA32E_MODE                    0x00000200
 #define VM_ENTRY_LOAD_IA32_EFER                        0x00008000
 
+#define VMX_MISC_ACTIVITY_HLT                  0x00000040
+
 #define INTR_INFO_UNBLOCK_NMI                  0x1000
 
 #define EXIT_REASONS_FAILED_VMENTRY            0x80000000
@@ -292,3 +295,4 @@ void vmx_entry_failure(struct per_cpu *cpu_data);
 void vmx_invept(void);
 
 void vmx_schedule_vmexit(struct per_cpu *cpu_data);
+void vmx_cpu_park(void);

diff --git a/hypervisor/arch/x86/vmx.c b/hypervisor/arch/x86/vmx.c
index b575c821666338d2ebe4f940be5b15faa6617a2e..10cdb27eeda0c4511f7f46e0508ad3e3755ee8ba 100644 (file)
--- a/hypervisor/arch/x86/vmx.c
+++ b/hypervisor/arch/x86/vmx.c
@@ -519,6 +519,10 @@ int vmx_cpu_init(struct per_cpu *cpu_data)
            !(vmx_proc_ctrl2 & SECONDARY_EXEC_UNRESTRICTED_GUEST))
                return -EIO;
 
+       /* require activity state HLT */
+       if (!(read_msr(MSR_IA32_VMX_MISC) & VMX_MISC_ACTIVITY_HLT))
+               return -EIO;
+
        revision_id = (u32)vmx_basic;
        cpu_data->vmxon_region.revision_id = revision_id;
        cpu_data->vmxon_region.shadow_indicator = 0;
@@ -756,6 +760,12 @@ void vmx_schedule_vmexit(struct per_cpu *cpu_data)
        vmcs_write32(PIN_BASED_VM_EXEC_CONTROL, pin_based_ctrl);
 }
 
+void vmx_cpu_park(void)
+{
+       vmcs_write64(GUEST_RFLAGS, 0x02);
+       vmcs_write32(GUEST_ACTIVITY_STATE, GUEST_ACTIVITY_HLT);
+}
+
 static void vmx_disable_preemption_timer(void)
 {
        u32 pin_based_ctrl = vmcs_read32(PIN_BASED_VM_EXEC_CONTROL);