Changeset 42156 in vbox

Timestamp:

Jul 16, 2012 6:59:45 AM (13 years ago)

Author:

vboxsync

Message:

VMM/VMMR0: HWVMXR0: Use MSR auto load/store areas in the VMCS. Added IA32_TSC_AUX for auto load/store. Cleanups.

Location:

trunk/src/VBox/VMM

Files:

• VMMR0/HWACCMR0Mixed.mac (modified) (8 diffs)
• VMMR0/HWVMXR0.cpp (modified) (18 diffs)
• include/HWACCMInternal.h (modified) (2 diffs)
• include/HWACCMInternal.mac (modified) (1 diff)

trunk/src/VBox/VMM/VMMR0/HWACCMR0Mixed.mac

@@ -404 +404 @@
 %endif
 
-    ; Save the host LSTAR, CSTAR, SFMASK & KERNEL_GSBASE MSRs and restore the guest MSRs
-    ;; @todo use the automatic load feature for MSRs
+%ifndef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
+    ; Save the host MSRs and load the guest MSRs
     LOADGUESTMSR MSR_K8_LSTAR, CPUMCTX.msrLSTAR
 %if 0  ; not supported on Intel CPUs
@@ -412 +412 @@
     LOADGUESTMSR MSR_K6_STAR, CPUMCTX.msrSTAR
     LOADGUESTMSR MSR_K8_SF_MASK, CPUMCTX.msrSFMASK
+%endif
+    ; Kernel GS Base is special, we need to manually load/store it, see @bugref{6208}
     LOADGUESTMSR MSR_K8_KERNEL_GS_BASE, CPUMCTX.msrKERNELGSBASE
 
@@ -538 +540 @@
     pop     xSI         ; pCtx (needed in rsi by the macros below)
 
-    ; Restore the host LSTAR, CSTAR, SFMASK & KERNEL_GSBASE MSRs
-    ;; @todo use the automatic load feature for MSRs
+    ; Kernel GS Base is special, we need to manually load/store it, see @bugref{6208}.
     LOADHOSTMSREX MSR_K8_KERNEL_GS_BASE, CPUMCTX.msrKERNELGSBASE
+%ifndef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
+    ; Save the guest MSRs and load the host MSRs
     LOADHOSTMSREX MSR_K8_SF_MASK, CPUMCTX.msrSFMASK
     LOADHOSTMSREX MSR_K6_STAR, CPUMCTX.msrSTAR
@@ -547 +550 @@
 %endif
     LOADHOSTMSREX MSR_K8_LSTAR, CPUMCTX.msrLSTAR
+%endif
 
 %ifdef VMX_USE_CACHED_VMCS_ACCESSES
@@ -609 +613 @@
     pop     xSI         ; pCtx (needed in rsi by the macros below)
 
-    ; Restore the host LSTAR, CSTAR, SFMASK & KERNEL_GSBASE MSRs
-    ;; @todo use the automatic load feature for MSRs
+    ; Kernel GS base is special, we need to manually load/store it See @bugref{6208}.
     LOADHOSTMSREX MSR_K8_KERNEL_GS_BASE, CPUMCTX.msrKERNELGSBASE
+%ifndef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
+    ; Load the host MSRs
     LOADHOSTMSR MSR_K8_SF_MASK
     LOADHOSTMSR MSR_K6_STAR
@@ -618 +623 @@
 %endif
     LOADHOSTMSR MSR_K8_LSTAR
+%endif
 
 %ifdef VMX_USE_CACHED_VMCS_ACCESSES
@@ -656 +662 @@
     pop     xSI         ; pCtx (needed in rsi by the macros below)
 
-    ; Restore the host LSTAR, CSTAR, SFMASK & KERNEL_GSBASE MSRs
-    ;; @todo use the automatic load feature for MSRs
+    ; Kernel GS base is special, load it manually. See @bugref{6208}.
     LOADHOSTMSREX MSR_K8_KERNEL_GS_BASE, CPUMCTX.msrKERNELGSBASE
+%ifndef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
+    ; Load the host MSRs
     LOADHOSTMSR MSR_K8_SF_MASK
     LOADHOSTMSR MSR_K6_STAR
@@ -665 +672 @@
 %endif
     LOADHOSTMSR MSR_K8_LSTAR
+%endif
 
 %ifdef VMX_USE_CACHED_VMCS_ACCESSES

trunk/src/VBox/VMM/VMMR0/HWVMXR0.cpp

@@ -210 +210 @@
 
 #ifdef LOG_ENABLED
-    SUPR0Printf("VMXR0InitVM %x\n", pVM);
+    SUPR0Printf("VMXR0InitVM %p\n", pVM);
 #endif
 
@@ -626 +626 @@
             AssertRC(rc);
 
-            /* Allow the guest to directly modify these MSRs; they are restored and saved automatically. */
+            /*
+             * Allow the guest to directly modify these MSRs; they are loaded/stored automatically
+             * using MSR-load/store areas in the VMCS.
+             */
             hmR0VmxSetMSRPermission(pVCpu, MSR_IA32_SYSENTER_CS, true, true);
             hmR0VmxSetMSRPermission(pVCpu, MSR_IA32_SYSENTER_ESP, true, true);
@@ -636 +639 @@
             hmR0VmxSetMSRPermission(pVCpu, MSR_K8_GS_BASE, true, true);
             hmR0VmxSetMSRPermission(pVCpu, MSR_K8_FS_BASE, true, true);
+            if (pVCpu->hwaccm.s.vmx.proc_ctls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
+                hmR0VmxSetMSRPermission(pVCpu, MSR_K8_TSC_AUX, true, true);
         }
 
@@ -647 +652 @@
         rc = VMXWriteVMCS64(VMX_VMCS_CTRL_VMEXIT_MSR_STORE_FULL, pVCpu->hwaccm.s.vmx.pGuestMSRPhys);
         AssertRC(rc);
-
         Assert(pVCpu->hwaccm.s.vmx.pHostMSRPhys);
         rc = VMXWriteVMCS64(VMX_VMCS_CTRL_VMEXIT_MSR_LOAD_FULL,  pVCpu->hwaccm.s.vmx.pHostMSRPhys);
@@ -656 +660 @@
         AssertRC(rc);
         rc = VMXWriteVMCS(VMX_VMCS_CTRL_EXIT_MSR_STORE_COUNT, 0);
+        AssertRC(rc);
+        rc = VMXWriteVMCS(VMX_VMCS_CTRL_EXIT_MSR_LOAD_COUNT, 0);
         AssertRC(rc);
 
@@ -1319 +1325 @@
          * Check if EFER MSR present.
          */
-        if (ASMCpuId_EDX(0x80000001) & (X86_CPUID_EXT_FEATURE_EDX_NX | X86_CPUID_EXT_FEATURE_EDX_LONG_MODE))
-        {
-            if (ASMCpuId_EDX(0x80000001) & X86_CPUID_EXT_FEATURE_EDX_SYSCALL)
+        uint32_t u32HostExtFeatures = ASMCpuId_EDX(0x80000001);
+        if (u32HostExtFeatures & (X86_CPUID_EXT_FEATURE_EDX_NX | X86_CPUID_EXT_FEATURE_EDX_LONG_MODE))
+        {
+            if (u32HostExtFeatures & X86_CPUID_EXT_FEATURE_EDX_SYSCALL)
             {
                 pMsr->u32IndexMSR = MSR_K6_STAR;
@@ -1354 +1361 @@
             pMsr->u64Value    = ASMRdMsr(MSR_K8_SF_MASK);           /* syscall flag mask */
             pMsr++; idxMsr++;
+
+            /* The KERNEL_GS_BASE MSR doesn't work reliably with auto load/store. See @bugref{6208}  */
+#if 0
             pMsr->u32IndexMSR = MSR_K8_KERNEL_GS_BASE;
             pMsr->u32Reserved = 0;
             pMsr->u64Value    = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);    /* swapgs exchange value */
             pMsr++; idxMsr++;
+#endif
         }
 # endif
+
+        if (pVCpu->hwaccm.s.vmx.proc_ctls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
+        {
+            Assert(u32HostExtFeatures & X86_CPUID_EXT_FEATURE_EDX_RDTSCP);
+            pMsr->u32IndexMSR = MSR_K8_TSC_AUX;
+            pMsr->u32Reserved = 0;
+            pMsr->u64Value    = ASMRdMsr(MSR_K8_TSC_AUX);
+            pMsr++; idxMsr++;
+        }
+
+        /** @todo r=ramshankar: check IA32_VMX_MISC bits 27:25 for valid idxMsr
+         *        range. */
         rc = VMXWriteVMCS(VMX_VMCS_CTRL_EXIT_MSR_LOAD_COUNT, idxMsr);
         AssertRC(rc);
+
+        pVCpu->hwaccm.s.vmx.cCachedMSRs = idxMsr;
 #endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
 
@@ -2093 +2118 @@
 
 #ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
-    /* Store all guest MSRs in the VM-Entry load area, so they will be loaded during the world switch. */
+    /*
+     * Store all guest MSRs in the VM-entry load area, so they will be loaded
+     * during the world switch.
+     */
     PVMXMSR pMsr = (PVMXMSR)pVCpu->hwaccm.s.vmx.pGuestMSR;
     unsigned idxMsr = 0;
 
-    uint32_t ulEdx;
-    uint32_t ulTemp;
-    CPUMGetGuestCpuId(pVCpu, 0x80000001, &ulTemp, &ulTemp, &ulTemp, &ulEdx);
-    /* EFER MSR present? */
-    if (ulEdx & (X86_CPUID_EXT_FEATURE_EDX_NX | X86_CPUID_EXT_FEATURE_EDX_LONG_MODE))
+    uint32_t u32GstExtFeatures;
+    uint32_t u32Temp;
+    CPUMGetGuestCpuId(pVCpu, 0x80000001, &u32Temp, &u32Temp, &u32Temp, &u32GstExtFeatures);
+
+    /*
+     * Check if EFER MSR present.
+     */
+    if (u32GstExtFeatures & (X86_CPUID_EXT_FEATURE_EDX_NX | X86_CPUID_EXT_FEATURE_EDX_LONG_MODE))
     {
         pMsr->u32IndexMSR = MSR_K6_EFER;
@@ -2111 +2142 @@
         pMsr++; idxMsr++;
 
-        if (ulEdx & X86_CPUID_EXT_FEATURE_EDX_LONG_MODE)
+        if (u32GstExtFeatures & X86_CPUID_EXT_FEATURE_EDX_LONG_MODE)
         {
             pMsr->u32IndexMSR = MSR_K8_LSTAR;
@@ -2125 +2156 @@
             pMsr->u64Value    = pCtx->msrSFMASK;          /* syscall flag mask */
             pMsr++; idxMsr++;
+
+            /* The KERNEL_GS_BASE MSR doesn't work reliably with auto load/store. See @bugref{6208}  */
+#if 0
             pMsr->u32IndexMSR = MSR_K8_KERNEL_GS_BASE;
             pMsr->u32Reserved = 0;
             pMsr->u64Value    = pCtx->msrKERNELGSBASE;    /* swapgs exchange value */
             pMsr++; idxMsr++;
-        }
-    }
-    pVCpu->hwaccm.s.vmx.cCachedMSRs = idxMsr;
+#endif
+        }
+    }
+
+    if (pVCpu->hwaccm.s.vmx.proc_ctls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
+    {
+        Assert(u32GstExtFeatures & X86_CPUID_EXT_FEATURE_EDX_RDTSCP);
+        pMsr->u32IndexMSR = MSR_K8_TSC_AUX;
+        pMsr->u32Reserved = 0;
+        rc = CPUMQueryGuestMsr(pVCpu, MSR_K8_TSC_AUX, &pMsr->u64Value);
+        AssertRC(rc);
+        pMsr++; idxMsr++;
+    }
+
+    /*
+     * The number of host MSRs saved must be identical to the number of guest MSRs loaded.
+     * It's not a VT-x requirement but how it's practically used here.
+     */
+    Assert(pVCpu->hwaccm.s.vmx.cCachedMSRs == idxMsr);
 
     rc = VMXWriteVMCS(VMX_VMCS_CTRL_ENTRY_MSR_LOAD_COUNT, idxMsr);
@@ -2327 +2377 @@
 
 #ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
-    /* Save the possibly changed MSRs that we automatically restore and save during a world switch. */
+    /*
+     * Save the possibly changed MSRs that we automatically restore and save during a world switch.
+     */
     for (unsigned i = 0; i < pVCpu->hwaccm.s.vmx.cCachedMSRs; i++)
     {
@@ -2344 +2396 @@
                 pCtx->msrSFMASK = pMsr->u64Value;
                 break;
+
+            /* The KERNEL_GS_BASE MSR doesn't work reliably with auto load/store. See @bugref{6208}  */
+#if 0
             case MSR_K8_KERNEL_GS_BASE:
                 pCtx->msrKERNELGSBASE = pMsr->u64Value;
                 break;
+#endif
+            case MSR_K8_TSC_AUX:
+                CPUMSetGuestMsr(pVCpu, MSR_K8_TSC_AUX, pMsr->u64Value);
+                break;
+
             case MSR_K6_EFER:
                 /* EFER can't be changed without causing a VM-exit. */
@@ -2967 +3027 @@
             pCtx->msrLSTAR = u8LastTPR;
 
+            /** @todo r=ramshankar: we should check for MSR-bitmap support here. */
             if (fPending)
             {
@@ -3099 +3160 @@
     TMNotifyStartOfExecution(pVCpu);
 
+#ifndef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
     /*
      * Save the current Host TSC_AUX and write the guest TSC_AUX to the host, so that
@@ -3112 +3174 @@
         ASMWrMsr(MSR_K8_TSC_AUX, u64GuestTSCAux);
     }
+#endif
 
 #ifdef VBOX_WITH_KERNEL_USING_XMM
@@ -3121 +3184 @@
     ASMAtomicIncU32(&pVCpu->hwaccm.s.cWorldSwitchExits);
 
+#ifndef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
     /*
      * Restore host's TSC_AUX.
@@ -3129 +3193 @@
         ASMWrMsr(MSR_K8_TSC_AUX, pVCpu->hwaccm.s.vmx.u64HostTSCAux);
     }
+#endif
 
     /* Possibly the last TSC value seen by the guest (too high) (only when we're in TSC offset mode). */
@@ -5172 +5237 @@
                 if (VMX_IS_64BIT_HOST_MODE())
                 {
-                    Log(("MSR_K6_EFER       = %RX64\n", ASMRdMsr(MSR_K6_EFER)));
-                    Log(("MSR_K6_STAR       = %RX64\n", ASMRdMsr(MSR_K6_STAR)));
-                    Log(("MSR_K8_LSTAR      = %RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
-                    Log(("MSR_K8_CSTAR      = %RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
-                    Log(("MSR_K8_SF_MASK    = %RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
+                    Log(("MSR_K6_EFER            = %RX64\n", ASMRdMsr(MSR_K6_EFER)));
+                    Log(("MSR_K6_STAR            = %RX64\n", ASMRdMsr(MSR_K6_STAR)));
+                    Log(("MSR_K8_LSTAR           = %RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
+                    Log(("MSR_K8_CSTAR           = %RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
+                    Log(("MSR_K8_SF_MASK         = %RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
+                    Log(("MSR_K8_KERNEL_GS_BASE  = %RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
                 }
 # endif

trunk/src/VBox/VMM/include/HWACCMInternal.h

@@ -42 +42 @@
 
 
-#if 0
-/* Seeing somewhat random behaviour on my Nehalem system with auto-save of guest MSRs;
- * for some strange reason the CPU doesn't save the MSRs during the VM-exit.
- * Clearly visible with a dual VCPU configured OpenSolaris 200906 live cd VM.
+/* The MSR auto load/store does not work for KERNEL_GS_BASE MSR, thus we
+ * handle this MSR manually. See @bugref{6208}. This is clearly visible while
+ * booting Solaris 11 (11.1 b19) VMs with 2 Cpus.
  *
- * Note: change the assembly files when enabling this! (remove the manual auto load/save)
+ * Note: don't forget to update the assembly files while modifying this!
  */
 #define VBOX_WITH_AUTO_MSR_LOAD_RESTORE
-#endif
 
 RT_C_DECLS_BEGIN
@@ -626 +624 @@
         /** Virtual address of the MSR load area (1 page). */
         R0PTRTYPE(uint8_t *)        pHostMSR;
-#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
 
         /* Number of automatically loaded/restored MSRs. */
         uint32_t                    cCachedMSRs;
         uint32_t                    uAlignement;
+#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
 
         /* Host's IA32_TSC_AUX MSR (for RDTSCP in VMX non-root). */

trunk/src/VBox/VMM/include/HWACCMInternal.mac

@@ -16 +16 @@
 
 %define VMX_USE_CACHED_VMCS_ACCESSES
+%define VBOX_WITH_AUTO_MSR_LOAD_RESTORE
 
 ;Maximum number of cached entries.