VirtualBox

source: vbox/trunk/src/VBox/VMM/VMMR0/HWVMXR0.cpp@ 42173

Last change on this file since 42173 was 42173, checked in by vboxsync, 12 years ago

VMM/HWVMXR0: Fixed assertion in saved states due to differing CpuIds.
  • Property svn:eol-style set to native
  • Property svn:keywords set to Id Revision
File size: 220.6 KB
Line 
1/* $Id: HWVMXR0.cpp 42173 2012-07-17 10:15:18Z vboxsync $ */
2/** @file
3 * HM VMX (VT-x) - Host Context Ring-0.
4 */
5
6/*
7 * Copyright (C) 2006-2012 Oracle Corporation
8 *
9 * This file is part of VirtualBox Open Source Edition (OSE), as
10 * available from http://www.virtualbox.org. This file is free software;
11 * you can redistribute it and/or modify it under the terms of the GNU
12 * General Public License (GPL) as published by the Free Software
13 * Foundation, in version 2 as it comes in the "COPYING" file of the
14 * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15 * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16 */
17
18
19/*******************************************************************************
20* Header Files *
21*******************************************************************************/
22#define LOG_GROUP LOG_GROUP_HWACCM
23#include <iprt/asm-amd64-x86.h>
24#include <VBox/vmm/hwaccm.h>
25#include <VBox/vmm/pgm.h>
26#include <VBox/vmm/dbgf.h>
27#include <VBox/vmm/dbgftrace.h>
28#include <VBox/vmm/selm.h>
29#include <VBox/vmm/iom.h>
30#ifdef VBOX_WITH_REM
31# include <VBox/vmm/rem.h>
32#endif
33#include <VBox/vmm/tm.h>
34#include "HWACCMInternal.h"
35#include <VBox/vmm/vm.h>
36#include <VBox/vmm/pdmapi.h>
37#include <VBox/err.h>
38#include <VBox/log.h>
39#include <iprt/assert.h>
40#include <iprt/param.h>
41#include <iprt/string.h>
42#include <iprt/time.h>
43#ifdef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
44# include <iprt/thread.h>
45#endif
46#include <iprt/x86.h>
47#include "HWVMXR0.h"
48
49#include "dtrace/VBoxVMM.h"
50
51
52/*******************************************************************************
53* Defined Constants And Macros *
54*******************************************************************************/
55#if defined(RT_ARCH_AMD64)
56# define VMX_IS_64BIT_HOST_MODE() (true)
57#elif defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
58# define VMX_IS_64BIT_HOST_MODE() (g_fVMXIs64bitHost != 0)
59#else
60# define VMX_IS_64BIT_HOST_MODE() (false)
61#endif
62
63
64/*******************************************************************************
65* Global Variables *
66*******************************************************************************/
67/* IO operation lookup arrays. */
68static uint32_t const g_aIOSize[4] = {1, 2, 0, 4};
69static uint32_t const g_aIOOpAnd[4] = {0xff, 0xffff, 0, 0xffffffff};
70
71#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
72/** See HWACCMR0A.asm. */
73extern "C" uint32_t g_fVMXIs64bitHost;
74#endif
75
76
77/*******************************************************************************
78* Local Functions *
79*******************************************************************************/
80static DECLCALLBACK(void) hmR0VmxSetupTLBEPT(PVM pVM, PVMCPU pVCpu);
81static DECLCALLBACK(void) hmR0VmxSetupTLBVPID(PVM pVM, PVMCPU pVCpu);
82static DECLCALLBACK(void) hmR0VmxSetupTLBBoth(PVM pVM, PVMCPU pVCpu);
83static DECLCALLBACK(void) hmR0VmxSetupTLBDummy(PVM pVM, PVMCPU pVCpu);
84static void hmR0VmxFlushEPT(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_EPT enmFlush);
85static void hmR0VmxFlushVPID(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_VPID enmFlush, RTGCPTR GCPtr);
86static void hmR0VmxUpdateExceptionBitmap(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx);
87static void hmR0VmxSetMSRPermission(PVMCPU pVCpu, unsigned ulMSR, bool fRead, bool fWrite);
88static void hmR0VmxReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, VBOXSTRICTRC rc, PCPUMCTX pCtx);
89
90
91/**
92 * Updates error from VMCS to HWACCMCPU's lasterror record.
93 *
94 * @param pVM Pointer to the VM.
95 * @param pVCpu Pointer to the VMCPU.
96 * @param rc The error code.
97 */
98static void hmR0VmxCheckError(PVM pVM, PVMCPU pVCpu, int rc)
99{
100 if (rc == VERR_VMX_GENERIC)
101 {
102 RTCCUINTREG instrError;
103
104 VMXReadVMCS(VMX_VMCS32_RO_VM_INSTR_ERROR, &instrError);
105 pVCpu->hwaccm.s.vmx.lasterror.ulInstrError = instrError;
106 }
107 pVM->hwaccm.s.lLastError = rc;
108}
109
110
111/**
112 * Sets up and activates VT-x on the current CPU.
113 *
114 * @returns VBox status code.
115 * @param pCpu Pointer to the CPU info struct.
116 * @param pVM Pointer to the VM. (can be NULL after a resume!!)
117 * @param pvCpuPage Pointer to the global CPU page.
118 * @param HCPhysCpuPage Physical address of the global CPU page.
119 */
120VMMR0DECL(int) VMXR0EnableCpu(PHMGLOBLCPUINFO pCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
121{
122 AssertReturn(HCPhysCpuPage != 0 && HCPhysCpuPage != NIL_RTHCPHYS, VERR_INVALID_PARAMETER);
123 AssertReturn(pvCpuPage, VERR_INVALID_PARAMETER);
124
125 if (pVM)
126 {
127 /* Set revision dword at the beginning of the VMXON structure. */
128 *(uint32_t *)pvCpuPage = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hwaccm.s.vmx.msr.vmx_basic_info);
129 }
130
131 /** @todo we should unmap the two pages from the virtual address space in order to prevent accidental corruption.
132 * (which can have very bad consequences!!!)
133 */
134
135 if (ASMGetCR4() & X86_CR4_VMXE)
136 return VERR_VMX_IN_VMX_ROOT_MODE;
137
138 ASMSetCR4(ASMGetCR4() | X86_CR4_VMXE); /* Make sure the VMX instructions don't cause #UD faults. */
139
140 /*
141 * Enter VM root mode.
142 */
143 int rc = VMXEnable(HCPhysCpuPage);
144 if (RT_FAILURE(rc))
145 {
146 ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
147 return VERR_VMX_VMXON_FAILED;
148 }
149
150 /*
151 * Flush all VPIDs (in case we or any other hypervisor have been using VPIDs) so that
152 * we can avoid an explicit flush while using new VPIDs. We would still need to flush
153 * each time while reusing a VPID after hitting the MaxASID limit once.
154 */
155 if ( pVM
156 && pVM->hwaccm.s.vmx.fVPID
157 && (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVVPID_CAPS_ALL_CONTEXTS))
158 {
159 hmR0VmxFlushVPID(pVM, NULL /* pvCpu */, VMX_FLUSH_VPID_ALL_CONTEXTS, 0 /* GCPtr */);
160 pCpu->fFlushASIDBeforeUse = false;
161 }
162 else
163 pCpu->fFlushASIDBeforeUse = true;
164
165 /*
166 * Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}.
167 */
168 ++pCpu->cTLBFlushes;
169
170 return VINF_SUCCESS;
171}
172
173
174/**
175 * Deactivates VT-x on the current CPU.
176 *
177 * @returns VBox status code.
178 * @param pCpu Pointer to the CPU info struct.
179 * @param pvCpuPage Pointer to the global CPU page.
180 * @param HCPhysCpuPage Physical address of the global CPU page.
181 */
182VMMR0DECL(int) VMXR0DisableCpu(PHMGLOBLCPUINFO pCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
183{
184 AssertReturn(HCPhysCpuPage != 0 && HCPhysCpuPage != NIL_RTHCPHYS, VERR_INVALID_PARAMETER);
185 AssertReturn(pvCpuPage, VERR_INVALID_PARAMETER);
186 NOREF(pCpu);
187
188 /* If we're somehow not in VMX root mode, then we shouldn't dare leaving it. */
189 if (!(ASMGetCR4() & X86_CR4_VMXE))
190 return VERR_VMX_NOT_IN_VMX_ROOT_MODE;
191
192 /* Leave VMX Root Mode. */
193 VMXDisable();
194
195 /* And clear the X86_CR4_VMXE bit. */
196 ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
197 return VINF_SUCCESS;
198}
199
200
201/**
202 * Does Ring-0 per VM VT-x initialization.
203 *
204 * @returns VBox status code.
205 * @param pVM Pointer to the VM.
206 */
207VMMR0DECL(int) VMXR0InitVM(PVM pVM)
208{
209 int rc;
210
211#ifdef LOG_ENABLED
212 SUPR0Printf("VMXR0InitVM %p\n", pVM);
213#endif
214
215 pVM->hwaccm.s.vmx.pMemObjAPIC = NIL_RTR0MEMOBJ;
216
217 if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
218 {
219 /* Allocate one page for the APIC physical page (serves for filtering accesses). */
220 rc = RTR0MemObjAllocCont(&pVM->hwaccm.s.vmx.pMemObjAPIC, PAGE_SIZE, true /* executable R0 mapping */);
221 AssertRC(rc);
222 if (RT_FAILURE(rc))
223 return rc;
224
225 pVM->hwaccm.s.vmx.pAPIC = (uint8_t *)RTR0MemObjAddress(pVM->hwaccm.s.vmx.pMemObjAPIC);
226 pVM->hwaccm.s.vmx.pAPICPhys = RTR0MemObjGetPagePhysAddr(pVM->hwaccm.s.vmx.pMemObjAPIC, 0);
227 ASMMemZero32(pVM->hwaccm.s.vmx.pAPIC, PAGE_SIZE);
228 }
229 else
230 {
231 pVM->hwaccm.s.vmx.pMemObjAPIC = 0;
232 pVM->hwaccm.s.vmx.pAPIC = 0;
233 pVM->hwaccm.s.vmx.pAPICPhys = 0;
234 }
235
236#ifdef VBOX_WITH_CRASHDUMP_MAGIC
237 {
238 rc = RTR0MemObjAllocCont(&pVM->hwaccm.s.vmx.pMemObjScratch, PAGE_SIZE, true /* executable R0 mapping */);
239 AssertRC(rc);
240 if (RT_FAILURE(rc))
241 return rc;
242
243 pVM->hwaccm.s.vmx.pScratch = (uint8_t *)RTR0MemObjAddress(pVM->hwaccm.s.vmx.pMemObjScratch);
244 pVM->hwaccm.s.vmx.pScratchPhys = RTR0MemObjGetPagePhysAddr(pVM->hwaccm.s.vmx.pMemObjScratch, 0);
245
246 ASMMemZero32(pVM->hwaccm.s.vmx.pScratch, PAGE_SIZE);
247 strcpy((char *)pVM->hwaccm.s.vmx.pScratch, "SCRATCH Magic");
248 *(uint64_t *)(pVM->hwaccm.s.vmx.pScratch + 16) = UINT64_C(0xDEADBEEFDEADBEEF);
249 }
250#endif
251
252 /* Allocate VMCSs for all guest CPUs. */
253 for (VMCPUID i = 0; i < pVM->cCpus; i++)
254 {
255 PVMCPU pVCpu = &pVM->aCpus[i];
256
257 pVCpu->hwaccm.s.vmx.hMemObjVMCS = NIL_RTR0MEMOBJ;
258
259 /* Allocate one page for the VM control structure (VMCS). */
260 rc = RTR0MemObjAllocCont(&pVCpu->hwaccm.s.vmx.hMemObjVMCS, PAGE_SIZE, true /* executable R0 mapping */);
261 AssertRC(rc);
262 if (RT_FAILURE(rc))
263 return rc;
264
265 pVCpu->hwaccm.s.vmx.pvVMCS = RTR0MemObjAddress(pVCpu->hwaccm.s.vmx.hMemObjVMCS);
266 pVCpu->hwaccm.s.vmx.HCPhysVMCS = RTR0MemObjGetPagePhysAddr(pVCpu->hwaccm.s.vmx.hMemObjVMCS, 0);
267 ASMMemZeroPage(pVCpu->hwaccm.s.vmx.pvVMCS);
268
269 pVCpu->hwaccm.s.vmx.cr0_mask = 0;
270 pVCpu->hwaccm.s.vmx.cr4_mask = 0;
271
272 /* Allocate one page for the virtual APIC page for TPR caching. */
273 rc = RTR0MemObjAllocCont(&pVCpu->hwaccm.s.vmx.hMemObjVAPIC, PAGE_SIZE, true /* executable R0 mapping */);
274 AssertRC(rc);
275 if (RT_FAILURE(rc))
276 return rc;
277
278 pVCpu->hwaccm.s.vmx.pbVAPIC = (uint8_t *)RTR0MemObjAddress(pVCpu->hwaccm.s.vmx.hMemObjVAPIC);
279 pVCpu->hwaccm.s.vmx.HCPhysVAPIC = RTR0MemObjGetPagePhysAddr(pVCpu->hwaccm.s.vmx.hMemObjVAPIC, 0);
280 ASMMemZeroPage(pVCpu->hwaccm.s.vmx.pbVAPIC);
281
282 /* Allocate the MSR bitmap if this feature is supported. */
283 if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
284 {
285 rc = RTR0MemObjAllocCont(&pVCpu->hwaccm.s.vmx.pMemObjMSRBitmap, PAGE_SIZE, true /* executable R0 mapping */);
286 AssertRC(rc);
287 if (RT_FAILURE(rc))
288 return rc;
289
290 pVCpu->hwaccm.s.vmx.pMSRBitmap = (uint8_t *)RTR0MemObjAddress(pVCpu->hwaccm.s.vmx.pMemObjMSRBitmap);
291 pVCpu->hwaccm.s.vmx.pMSRBitmapPhys = RTR0MemObjGetPagePhysAddr(pVCpu->hwaccm.s.vmx.pMemObjMSRBitmap, 0);
292 memset(pVCpu->hwaccm.s.vmx.pMSRBitmap, 0xff, PAGE_SIZE);
293 }
294
295#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
296 /* Allocate one page for the guest MSR load area (for preloading guest MSRs during the world switch). */
297 rc = RTR0MemObjAllocCont(&pVCpu->hwaccm.s.vmx.pMemObjGuestMSR, PAGE_SIZE, true /* executable R0 mapping */);
298 AssertRC(rc);
299 if (RT_FAILURE(rc))
300 return rc;
301
302 pVCpu->hwaccm.s.vmx.pGuestMSR = (uint8_t *)RTR0MemObjAddress(pVCpu->hwaccm.s.vmx.pMemObjGuestMSR);
303 pVCpu->hwaccm.s.vmx.pGuestMSRPhys = RTR0MemObjGetPagePhysAddr(pVCpu->hwaccm.s.vmx.pMemObjGuestMSR, 0);
304 memset(pVCpu->hwaccm.s.vmx.pGuestMSR, 0, PAGE_SIZE);
305
306 /* Allocate one page for the host MSR load area (for restoring host MSRs after the world switch back). */
307 rc = RTR0MemObjAllocCont(&pVCpu->hwaccm.s.vmx.pMemObjHostMSR, PAGE_SIZE, true /* executable R0 mapping */);
308 AssertRC(rc);
309 if (RT_FAILURE(rc))
310 return rc;
311
312 pVCpu->hwaccm.s.vmx.pHostMSR = (uint8_t *)RTR0MemObjAddress(pVCpu->hwaccm.s.vmx.pMemObjHostMSR);
313 pVCpu->hwaccm.s.vmx.pHostMSRPhys = RTR0MemObjGetPagePhysAddr(pVCpu->hwaccm.s.vmx.pMemObjHostMSR, 0);
314 memset(pVCpu->hwaccm.s.vmx.pHostMSR, 0, PAGE_SIZE);
315#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
316
317 /* Current guest paging mode. */
318 pVCpu->hwaccm.s.vmx.enmLastSeenGuestMode = PGMMODE_REAL;
319
320#ifdef LOG_ENABLED
321 SUPR0Printf("VMXR0InitVM %x VMCS=%x (%x)\n", pVM, pVCpu->hwaccm.s.vmx.pvVMCS, (uint32_t)pVCpu->hwaccm.s.vmx.HCPhysVMCS);
322#endif
323 }
324
325 return VINF_SUCCESS;
326}
327
328
329/**
330 * Does Ring-0 per VM VT-x termination.
331 *
332 * @returns VBox status code.
333 * @param pVM Pointer to the VM.
334 */
335VMMR0DECL(int) VMXR0TermVM(PVM pVM)
336{
337 for (VMCPUID i = 0; i < pVM->cCpus; i++)
338 {
339 PVMCPU pVCpu = &pVM->aCpus[i];
340
341 if (pVCpu->hwaccm.s.vmx.hMemObjVMCS != NIL_RTR0MEMOBJ)
342 {
343 RTR0MemObjFree(pVCpu->hwaccm.s.vmx.hMemObjVMCS, false);
344 pVCpu->hwaccm.s.vmx.hMemObjVMCS = NIL_RTR0MEMOBJ;
345 pVCpu->hwaccm.s.vmx.pvVMCS = 0;
346 pVCpu->hwaccm.s.vmx.HCPhysVMCS = 0;
347 }
348 if (pVCpu->hwaccm.s.vmx.hMemObjVAPIC != NIL_RTR0MEMOBJ)
349 {
350 RTR0MemObjFree(pVCpu->hwaccm.s.vmx.hMemObjVAPIC, false);
351 pVCpu->hwaccm.s.vmx.hMemObjVAPIC = NIL_RTR0MEMOBJ;
352 pVCpu->hwaccm.s.vmx.pbVAPIC = 0;
353 pVCpu->hwaccm.s.vmx.HCPhysVAPIC = 0;
354 }
355 if (pVCpu->hwaccm.s.vmx.pMemObjMSRBitmap != NIL_RTR0MEMOBJ)
356 {
357 RTR0MemObjFree(pVCpu->hwaccm.s.vmx.pMemObjMSRBitmap, false);
358 pVCpu->hwaccm.s.vmx.pMemObjMSRBitmap = NIL_RTR0MEMOBJ;
359 pVCpu->hwaccm.s.vmx.pMSRBitmap = 0;
360 pVCpu->hwaccm.s.vmx.pMSRBitmapPhys = 0;
361 }
362#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
363 if (pVCpu->hwaccm.s.vmx.pMemObjHostMSR != NIL_RTR0MEMOBJ)
364 {
365 RTR0MemObjFree(pVCpu->hwaccm.s.vmx.pMemObjHostMSR, false);
366 pVCpu->hwaccm.s.vmx.pMemObjHostMSR = NIL_RTR0MEMOBJ;
367 pVCpu->hwaccm.s.vmx.pHostMSR = 0;
368 pVCpu->hwaccm.s.vmx.pHostMSRPhys = 0;
369 }
370 if (pVCpu->hwaccm.s.vmx.pMemObjGuestMSR != NIL_RTR0MEMOBJ)
371 {
372 RTR0MemObjFree(pVCpu->hwaccm.s.vmx.pMemObjGuestMSR, false);
373 pVCpu->hwaccm.s.vmx.pMemObjGuestMSR = NIL_RTR0MEMOBJ;
374 pVCpu->hwaccm.s.vmx.pGuestMSR = 0;
375 pVCpu->hwaccm.s.vmx.pGuestMSRPhys = 0;
376 }
377#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
378 }
379 if (pVM->hwaccm.s.vmx.pMemObjAPIC != NIL_RTR0MEMOBJ)
380 {
381 RTR0MemObjFree(pVM->hwaccm.s.vmx.pMemObjAPIC, false);
382 pVM->hwaccm.s.vmx.pMemObjAPIC = NIL_RTR0MEMOBJ;
383 pVM->hwaccm.s.vmx.pAPIC = 0;
384 pVM->hwaccm.s.vmx.pAPICPhys = 0;
385 }
386#ifdef VBOX_WITH_CRASHDUMP_MAGIC
387 if (pVM->hwaccm.s.vmx.pMemObjScratch != NIL_RTR0MEMOBJ)
388 {
389 ASMMemZero32(pVM->hwaccm.s.vmx.pScratch, PAGE_SIZE);
390 RTR0MemObjFree(pVM->hwaccm.s.vmx.pMemObjScratch, false);
391 pVM->hwaccm.s.vmx.pMemObjScratch = NIL_RTR0MEMOBJ;
392 pVM->hwaccm.s.vmx.pScratch = 0;
393 pVM->hwaccm.s.vmx.pScratchPhys = 0;
394 }
395#endif
396 return VINF_SUCCESS;
397}
398
399
400/**
401 * Sets up VT-x for the specified VM.
402 *
403 * @returns VBox status code.
404 * @param pVM Pointer to the VM.
405 */
406VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
407{
408 int rc = VINF_SUCCESS;
409 uint32_t val;
410
411 AssertReturn(pVM, VERR_INVALID_PARAMETER);
412
413 /* Initialize these always, see hwaccmR3InitFinalizeR0().*/
414 pVM->hwaccm.s.vmx.enmFlushEPT = VMX_FLUSH_EPT_NONE;
415 pVM->hwaccm.s.vmx.enmFlushVPID = VMX_FLUSH_VPID_NONE;
416
417 /* Determine optimal flush type for EPT. */
418 if (pVM->hwaccm.s.fNestedPaging)
419 {
420 if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVEPT_CAPS_SINGLE_CONTEXT)
421 pVM->hwaccm.s.vmx.enmFlushEPT = VMX_FLUSH_EPT_SINGLE_CONTEXT;
422 else if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVEPT_CAPS_ALL_CONTEXTS)
423 pVM->hwaccm.s.vmx.enmFlushEPT = VMX_FLUSH_EPT_ALL_CONTEXTS;
424 else
425 {
426 /*
427 * Should never really happen. EPT is supported but no suitable flush types supported.
428 * We cannot ignore EPT at this point as we've already setup Unrestricted Guest execution.
429 */
430 pVM->hwaccm.s.vmx.enmFlushEPT = VMX_FLUSH_EPT_NOT_SUPPORTED;
431 return VERR_VMX_GENERIC;
432 }
433 }
434
435 /* Determine optimal flush type for VPID. */
436 if (pVM->hwaccm.s.vmx.fVPID)
437 {
438 if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVVPID_CAPS_SINGLE_CONTEXT)
439 pVM->hwaccm.s.vmx.enmFlushVPID = VMX_FLUSH_VPID_SINGLE_CONTEXT;
440 else if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVVPID_CAPS_ALL_CONTEXTS)
441 pVM->hwaccm.s.vmx.enmFlushVPID = VMX_FLUSH_VPID_ALL_CONTEXTS;
442 else
443 {
444 /*
445 * Neither SINGLE nor ALL context flush types for VPID supported by the CPU.
446 * We do not handle other flush type combinations, ignore VPID capabilities.
447 */
448 if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVVPID_CAPS_INDIV_ADDR)
449 Log(("VMXR0SetupVM: Only VMX_FLUSH_VPID_INDIV_ADDR supported. Ignoring VPID.\n"));
450 if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVVPID_CAPS_SINGLE_CONTEXT_RETAIN_GLOBALS)
451 Log(("VMXR0SetupVM: Only VMX_FLUSH_VPID_SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
452 pVM->hwaccm.s.vmx.enmFlushVPID = VMX_FLUSH_VPID_NOT_SUPPORTED;
453 pVM->hwaccm.s.vmx.fVPID = false;
454 }
455 }
456
457 for (VMCPUID i = 0; i < pVM->cCpus; i++)
458 {
459 PVMCPU pVCpu = &pVM->aCpus[i];
460
461 AssertPtr(pVCpu->hwaccm.s.vmx.pvVMCS);
462
463 /* Set revision dword at the beginning of the VMCS structure. */
464 *(uint32_t *)pVCpu->hwaccm.s.vmx.pvVMCS = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hwaccm.s.vmx.msr.vmx_basic_info);
465
466 /*
467 * Clear and activate the VMCS.
468 */
469 Log(("HCPhysVMCS = %RHp\n", pVCpu->hwaccm.s.vmx.HCPhysVMCS));
470 rc = VMXClearVMCS(pVCpu->hwaccm.s.vmx.HCPhysVMCS);
471 if (RT_FAILURE(rc))
472 goto vmx_end;
473
474 rc = VMXActivateVMCS(pVCpu->hwaccm.s.vmx.HCPhysVMCS);
475 if (RT_FAILURE(rc))
476 goto vmx_end;
477
478 /*
479 * VMX_VMCS_CTRL_PIN_EXEC_CONTROLS
480 * Set required bits to one and zero according to the MSR capabilities.
481 */
482 val = pVM->hwaccm.s.vmx.msr.vmx_pin_ctls.n.disallowed0;
483 val |= VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_EXT_INT_EXIT /* External interrupts */
484 | VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_NMI_EXIT; /* Non-maskable interrupts */
485
486 /*
487 * Enable the VMX preemption timer.
488 */
489 if (pVM->hwaccm.s.vmx.fUsePreemptTimer)
490 val |= VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_PREEMPT_TIMER;
491 val &= pVM->hwaccm.s.vmx.msr.vmx_pin_ctls.n.allowed1;
492
493 rc = VMXWriteVMCS(VMX_VMCS_CTRL_PIN_EXEC_CONTROLS, val);
494 AssertRC(rc);
495
496 /*
497 * VMX_VMCS_CTRL_PROC_EXEC_CONTROLS
498 * Set required bits to one and zero according to the MSR capabilities.
499 */
500 val = pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.disallowed0;
501 /* Program which event cause VM-exits and which features we want to use. */
502 val |= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_HLT_EXIT
503 | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_TSC_OFFSET
504 | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT
505 | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_UNCOND_IO_EXIT
506 | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDPMC_EXIT
507 | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_EXIT
508 | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MWAIT_EXIT; /* don't execute mwait or else we'll idle inside
509 the guest (host thinks the cpu load is high) */
510
511 /* Without nested paging we should intercept invlpg and cr3 mov instructions. */
512 if (!pVM->hwaccm.s.fNestedPaging)
513 {
514 val |= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INVLPG_EXIT
515 | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
516 | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT;
517 }
518
519 /*
520 * VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MWAIT_EXIT might cause a vmlaunch
521 * failure with an invalid control fields error. (combined with some other exit reasons)
522 */
523 if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
524 {
525 /* CR8 reads from the APIC shadow page; writes cause an exit is they lower the TPR below the threshold */
526 val |= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW;
527 Assert(pVM->hwaccm.s.vmx.pAPIC);
528 }
529 else
530 /* Exit on CR8 reads & writes in case the TPR shadow feature isn't present. */
531 val |= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR8_STORE_EXIT | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR8_LOAD_EXIT;
532
533 if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
534 {
535 Assert(pVCpu->hwaccm.s.vmx.pMSRBitmapPhys);
536 val |= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS;
537 }
538
539 /* We will use the secondary control if it's present. */
540 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
541
542 /* Mask away the bits that the CPU doesn't support */
543 /** @todo make sure they don't conflict with the above requirements. */
544 val &= pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1;
545 pVCpu->hwaccm.s.vmx.proc_ctls = val;
546
547 rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, val);
548 AssertRC(rc);
549
550 if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
551 {
552 /*
553 * VMX_VMCS_CTRL_PROC_EXEC_CONTROLS2
554 * Set required bits to one and zero according to the MSR capabilities.
555 */
556 val = pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2.n.disallowed0;
557 val |= VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT;
558
559 if (pVM->hwaccm.s.fNestedPaging)
560 val |= VMX_VMCS_CTRL_PROC_EXEC2_EPT;
561
562 if (pVM->hwaccm.s.vmx.fVPID)
563 val |= VMX_VMCS_CTRL_PROC_EXEC2_VPID;
564
565 if (pVM->hwaccm.s.fHasIoApic)
566 val |= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC;
567
568 if (pVM->hwaccm.s.vmx.fUnrestrictedGuest)
569 val |= VMX_VMCS_CTRL_PROC_EXEC2_REAL_MODE;
570
571 if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
572 val |= VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP;
573
574 /* Mask away the bits that the CPU doesn't support */
575 /** @todo make sure they don't conflict with the above requirements. */
576 val &= pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2.n.allowed1;
577 pVCpu->hwaccm.s.vmx.proc_ctls2 = val;
578 rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS2, val);
579 AssertRC(rc);
580 }
581
582 /*
583 * VMX_VMCS_CTRL_CR3_TARGET_COUNT
584 * Set required bits to one and zero according to the MSR capabilities.
585 */
586 rc = VMXWriteVMCS(VMX_VMCS_CTRL_CR3_TARGET_COUNT, 0);
587 AssertRC(rc);
588
589 /*
590 * Forward all exception except #NM & #PF to the guest.
591 * We always need to check pagefaults since our shadow page table can be out of sync.
592 * And we always lazily sync the FPU & XMM state. .
593 */
594
595 /** @todo Possible optimization:
596 * Keep the FPU and XMM state current in the EM thread. That way there's no need to
597 * lazily sync anything, but the downside is that we can't use the FPU stack or XMM
598 * registers ourselves of course.
599 *
600 * Note: only possible if the current state is actually ours (X86_CR0_TS flag)
601 */
602
603 /*
604 * Don't filter page faults, all of them should cause a world switch.
605 */
606 rc = VMXWriteVMCS(VMX_VMCS_CTRL_PAGEFAULT_ERROR_MASK, 0);
607 AssertRC(rc);
608 rc = VMXWriteVMCS(VMX_VMCS_CTRL_PAGEFAULT_ERROR_MATCH, 0);
609 AssertRC(rc);
610
611 rc = VMXWriteVMCS64(VMX_VMCS_CTRL_TSC_OFFSET_FULL, 0);
612 AssertRC(rc);
613 rc = VMXWriteVMCS64(VMX_VMCS_CTRL_IO_BITMAP_A_FULL, 0);
614 AssertRC(rc);
615 rc = VMXWriteVMCS64(VMX_VMCS_CTRL_IO_BITMAP_B_FULL, 0);
616 AssertRC(rc);
617
618 /*
619 * Set the MSR bitmap address.
620 */
621 if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
622 {
623 Assert(pVCpu->hwaccm.s.vmx.pMSRBitmapPhys);
624
625 rc = VMXWriteVMCS64(VMX_VMCS_CTRL_MSR_BITMAP_FULL, pVCpu->hwaccm.s.vmx.pMSRBitmapPhys);
626 AssertRC(rc);
627
628 /*
629 * Allow the guest to directly modify these MSRs; they are loaded/stored automatically
630 * using MSR-load/store areas in the VMCS.
631 */
632 hmR0VmxSetMSRPermission(pVCpu, MSR_IA32_SYSENTER_CS, true, true);
633 hmR0VmxSetMSRPermission(pVCpu, MSR_IA32_SYSENTER_ESP, true, true);
634 hmR0VmxSetMSRPermission(pVCpu, MSR_IA32_SYSENTER_EIP, true, true);
635 hmR0VmxSetMSRPermission(pVCpu, MSR_K8_LSTAR, true, true);
636 hmR0VmxSetMSRPermission(pVCpu, MSR_K6_STAR, true, true);
637 hmR0VmxSetMSRPermission(pVCpu, MSR_K8_SF_MASK, true, true);
638 hmR0VmxSetMSRPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, true, true);
639 hmR0VmxSetMSRPermission(pVCpu, MSR_K8_GS_BASE, true, true);
640 hmR0VmxSetMSRPermission(pVCpu, MSR_K8_FS_BASE, true, true);
641 if (pVCpu->hwaccm.s.vmx.proc_ctls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
642 hmR0VmxSetMSRPermission(pVCpu, MSR_K8_TSC_AUX, true, true);
643 }
644
645#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
646 /*
647 * Set the guest & host MSR load/store physical addresses.
648 */
649 Assert(pVCpu->hwaccm.s.vmx.pGuestMSRPhys);
650 rc = VMXWriteVMCS64(VMX_VMCS_CTRL_VMENTRY_MSR_LOAD_FULL, pVCpu->hwaccm.s.vmx.pGuestMSRPhys);
651 AssertRC(rc);
652 rc = VMXWriteVMCS64(VMX_VMCS_CTRL_VMEXIT_MSR_STORE_FULL, pVCpu->hwaccm.s.vmx.pGuestMSRPhys);
653 AssertRC(rc);
654 Assert(pVCpu->hwaccm.s.vmx.pHostMSRPhys);
655 rc = VMXWriteVMCS64(VMX_VMCS_CTRL_VMEXIT_MSR_LOAD_FULL, pVCpu->hwaccm.s.vmx.pHostMSRPhys);
656 AssertRC(rc);
657#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
658
659 rc = VMXWriteVMCS(VMX_VMCS_CTRL_ENTRY_MSR_LOAD_COUNT, 0);
660 AssertRC(rc);
661 rc = VMXWriteVMCS(VMX_VMCS_CTRL_EXIT_MSR_STORE_COUNT, 0);
662 AssertRC(rc);
663 rc = VMXWriteVMCS(VMX_VMCS_CTRL_EXIT_MSR_LOAD_COUNT, 0);
664 AssertRC(rc);
665
666 if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
667 {
668 Assert(pVM->hwaccm.s.vmx.pMemObjAPIC);
669 /* Optional */
670 rc = VMXWriteVMCS(VMX_VMCS_CTRL_TPR_THRESHOLD, 0);
671 rc |= VMXWriteVMCS64(VMX_VMCS_CTRL_VAPIC_PAGEADDR_FULL, pVCpu->hwaccm.s.vmx.HCPhysVAPIC);
672
673 if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
674 rc |= VMXWriteVMCS64(VMX_VMCS_CTRL_APIC_ACCESSADDR_FULL, pVM->hwaccm.s.vmx.pAPICPhys);
675
676 AssertRC(rc);
677 }
678
679 /* Set link pointer to -1. Not currently used. */
680 rc = VMXWriteVMCS64(VMX_VMCS_GUEST_LINK_PTR_FULL, 0xFFFFFFFFFFFFFFFFULL);
681 AssertRC(rc);
682
683 /*
684 * Clear VMCS, marking it inactive. Clear implementation specific data and writing back
685 * VMCS data back to memory.
686 */
687 rc = VMXClearVMCS(pVCpu->hwaccm.s.vmx.HCPhysVMCS);
688 AssertRC(rc);
689
690 /*
691 * Configure the VMCS read cache.
692 */
693 PVMCSCACHE pCache = &pVCpu->hwaccm.s.vmx.VMCSCache;
694
695 VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_RIP);
696 VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_RSP);
697 VMXSetupCachedReadVMCS(pCache, VMX_VMCS_GUEST_RFLAGS);
698 VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE);
699 VMXSetupCachedReadVMCS(pCache, VMX_VMCS_CTRL_CR0_READ_SHADOW);
700 VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_CR0);
701 VMXSetupCachedReadVMCS(pCache, VMX_VMCS_CTRL_CR4_READ_SHADOW);
702 VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_CR4);
703 VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_DR7);
704 VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_GUEST_SYSENTER_CS);
705 VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_SYSENTER_EIP);
706 VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_SYSENTER_ESP);
707 VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_GUEST_GDTR_LIMIT);
708 VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_GDTR_BASE);
709 VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_GUEST_IDTR_LIMIT);
710 VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_IDTR_BASE);
711
712 VMX_SETUP_SELREG(ES, pCache);
713 VMX_SETUP_SELREG(SS, pCache);
714 VMX_SETUP_SELREG(CS, pCache);
715 VMX_SETUP_SELREG(DS, pCache);
716 VMX_SETUP_SELREG(FS, pCache);
717 VMX_SETUP_SELREG(GS, pCache);
718 VMX_SETUP_SELREG(LDTR, pCache);
719 VMX_SETUP_SELREG(TR, pCache);
720
721 /*
722 * Status code VMCS reads.
723 */
724 VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_EXIT_REASON);
725 VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_VM_INSTR_ERROR);
726 VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_EXIT_INSTR_LENGTH);
727 VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_EXIT_INTERRUPTION_ERRCODE);
728 VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO);
729 VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_EXIT_INSTR_INFO);
730 VMXSetupCachedReadVMCS(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
731 VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_IDT_INFO);
732 VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_IDT_ERRCODE);
733
734 if (pVM->hwaccm.s.fNestedPaging)
735 {
736 VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_CR3);
737 VMXSetupCachedReadVMCS(pCache, VMX_VMCS_EXIT_PHYS_ADDR_FULL);
738 pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
739 }
740 else
741 pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
742 } /* for each VMCPU */
743
744 /*
745 * Setup the right TLB function based on CPU capabilities.
746 */
747 if (pVM->hwaccm.s.fNestedPaging && pVM->hwaccm.s.vmx.fVPID)
748 pVM->hwaccm.s.vmx.pfnSetupTaggedTLB = hmR0VmxSetupTLBBoth;
749 else if (pVM->hwaccm.s.fNestedPaging)
750 pVM->hwaccm.s.vmx.pfnSetupTaggedTLB = hmR0VmxSetupTLBEPT;
751 else if (pVM->hwaccm.s.vmx.fVPID)
752 pVM->hwaccm.s.vmx.pfnSetupTaggedTLB = hmR0VmxSetupTLBVPID;
753 else
754 pVM->hwaccm.s.vmx.pfnSetupTaggedTLB = hmR0VmxSetupTLBDummy;
755
756vmx_end:
757 hmR0VmxCheckError(pVM, &pVM->aCpus[0], rc);
758 return rc;
759}
760
761
762/**
763 * Sets the permission bits for the specified MSR.
764 *
765 * @param pVCpu Pointer to the VMCPU.
766 * @param ulMSR The MSR value.
767 * @param fRead Whether reading is allowed.
768 * @param fWrite Whether writing is allowed.
769 */
770static void hmR0VmxSetMSRPermission(PVMCPU pVCpu, unsigned ulMSR, bool fRead, bool fWrite)
771{
772 unsigned ulBit;
773 uint8_t *pMSRBitmap = (uint8_t *)pVCpu->hwaccm.s.vmx.pMSRBitmap;
774
775 /*
776 * Layout:
777 * 0x000 - 0x3ff - Low MSR read bits
778 * 0x400 - 0x7ff - High MSR read bits
779 * 0x800 - 0xbff - Low MSR write bits
780 * 0xc00 - 0xfff - High MSR write bits
781 */
782 if (ulMSR <= 0x00001FFF)
783 {
784 /* Pentium-compatible MSRs */
785 ulBit = ulMSR;
786 }
787 else if ( ulMSR >= 0xC0000000
788 && ulMSR <= 0xC0001FFF)
789 {
790 /* AMD Sixth Generation x86 Processor MSRs */
791 ulBit = (ulMSR - 0xC0000000);
792 pMSRBitmap += 0x400;
793 }
794 else
795 {
796 AssertFailed();
797 return;
798 }
799
800 Assert(ulBit <= 0x1fff);
801 if (fRead)
802 ASMBitClear(pMSRBitmap, ulBit);
803 else
804 ASMBitSet(pMSRBitmap, ulBit);
805
806 if (fWrite)
807 ASMBitClear(pMSRBitmap + 0x800, ulBit);
808 else
809 ASMBitSet(pMSRBitmap + 0x800, ulBit);
810}
811
812
813/**
814 * Injects an event (trap or external interrupt).
815 *
816 * @returns VBox status code. Note that it may return VINF_EM_RESET to
817 * indicate a triple fault when injecting X86_XCPT_DF.
818 *
819 * @param pVM Pointer to the VM.
820 * @param pVCpu Pointer to the VMCPU.
821 * @param pCtx Pointer to the guest CPU Context.
822 * @param intInfo VMX interrupt info.
823 * @param cbInstr Opcode length of faulting instruction.
824 * @param errCode Error code (optional).
825 */
826static int hmR0VmxInjectEvent(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, uint32_t intInfo, uint32_t cbInstr, uint32_t errCode)
827{
828 int rc;
829 uint32_t iGate = VMX_EXIT_INTERRUPTION_INFO_VECTOR(intInfo);
830
831#ifdef VBOX_WITH_STATISTICS
832 STAM_COUNTER_INC(&pVCpu->hwaccm.s.paStatInjectedIrqsR0[iGate & MASK_INJECT_IRQ_STAT]);
833#endif
834
835#ifdef VBOX_STRICT
836 if (iGate == 0xE)
837 {
838 LogFlow(("hmR0VmxInjectEvent: Injecting interrupt %d at %RGv error code=%08x CR2=%RGv intInfo=%08x\n", iGate,
839 (RTGCPTR)pCtx->rip, errCode, pCtx->cr2, intInfo));
840 }
841 else if (iGate < 0x20)
842 {
843 LogFlow(("hmR0VmxInjectEvent: Injecting interrupt %d at %RGv error code=%08x\n", iGate, (RTGCPTR)pCtx->rip,
844 errCode));
845 }
846 else
847 {
848 LogFlow(("INJ-EI: %x at %RGv\n", iGate, (RTGCPTR)pCtx->rip));
849 Assert( VMX_EXIT_INTERRUPTION_INFO_TYPE(intInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW
850 || !VMCPU_FF_ISSET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
851 Assert( VMX_EXIT_INTERRUPTION_INFO_TYPE(intInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW
852 || pCtx->eflags.u32 & X86_EFL_IF);
853 }
854#endif
855
856 if ( CPUMIsGuestInRealModeEx(pCtx)
857 && pVM->hwaccm.s.vmx.pRealModeTSS)
858 {
859 RTGCPHYS GCPhysHandler;
860 uint16_t offset, ip;
861 RTSEL sel;
862
863 /*
864 * Injecting events doesn't work right with real mode emulation.
865 * (#GP if we try to inject external hardware interrupts)
866 * Inject the interrupt or trap directly instead.
867 *
868 * ASSUMES no access handlers for the bits we read or write below (should be safe).
869 */
870 Log(("Manual interrupt/trap '%x' inject (real mode)\n", iGate));
871
872 /*
873 * Check if the interrupt handler is present.
874 */
875 if (iGate * 4 + 3 > pCtx->idtr.cbIdt)
876 {
877 Log(("IDT cbIdt violation\n"));
878 if (iGate != X86_XCPT_DF)
879 {
880 uint32_t intInfo2;
881
882 intInfo2 = (iGate == X86_XCPT_GP) ? (uint32_t)X86_XCPT_DF : iGate;
883 intInfo2 |= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
884 intInfo2 |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
885 intInfo2 |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HWEXCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
886
887 return hmR0VmxInjectEvent(pVM, pVCpu, pCtx, intInfo2, 0, 0 /* no error code according to the Intel docs */);
888 }
889 Log(("Triple fault -> reset the VM!\n"));
890 return VINF_EM_RESET;
891 }
892 if ( VMX_EXIT_INTERRUPTION_INFO_TYPE(intInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW
893 || iGate == 3 /* Both #BP and #OF point to the instruction after. */
894 || iGate == 4)
895 {
896 ip = pCtx->ip + cbInstr;
897 }
898 else
899 ip = pCtx->ip;
900
901 /*
902 * Read the selector:offset pair of the interrupt handler.
903 */
904 GCPhysHandler = (RTGCPHYS)pCtx->idtr.pIdt + iGate * 4;
905 rc = PGMPhysSimpleReadGCPhys(pVM, &offset, GCPhysHandler, sizeof(offset)); AssertRC(rc);
906 rc = PGMPhysSimpleReadGCPhys(pVM, &sel, GCPhysHandler + 2, sizeof(sel)); AssertRC(rc);
907
908 LogFlow(("IDT handler %04X:%04X\n", sel, offset));
909
910 /*
911 * Construct the stack frame.
912 */
913 /** @todo Check stack limit. */
914 pCtx->sp -= 2;
915 LogFlow(("ss:sp %04X:%04X eflags=%x\n", pCtx->ss.Sel, pCtx->sp, pCtx->eflags.u));
916 rc = PGMPhysSimpleWriteGCPhys(pVM, pCtx->ss.u64Base + pCtx->sp, &pCtx->eflags, sizeof(uint16_t)); AssertRC(rc);
917 pCtx->sp -= 2;
918 LogFlow(("ss:sp %04X:%04X cs=%x\n", pCtx->ss.Sel, pCtx->sp, pCtx->cs.Sel));
919 rc = PGMPhysSimpleWriteGCPhys(pVM, pCtx->ss.u64Base + pCtx->sp, &pCtx->cs, sizeof(uint16_t)); AssertRC(rc);
920 pCtx->sp -= 2;
921 LogFlow(("ss:sp %04X:%04X ip=%x\n", pCtx->ss.Sel, pCtx->sp, ip));
922 rc = PGMPhysSimpleWriteGCPhys(pVM, pCtx->ss.u64Base + pCtx->sp, &ip, sizeof(ip)); AssertRC(rc);
923
924 /*
925 * Update the CPU state for executing the handler.
926 */
927 pCtx->rip = offset;
928 pCtx->cs.Sel = sel;
929 pCtx->cs.u64Base = sel << 4;
930 pCtx->eflags.u &= ~(X86_EFL_IF | X86_EFL_TF | X86_EFL_RF | X86_EFL_AC);
931
932 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_GUEST_SEGMENT_REGS;
933 return VINF_SUCCESS;
934 }
935
936 /*
937 * Set event injection state.
938 */
939 rc = VMXWriteVMCS(VMX_VMCS_CTRL_ENTRY_IRQ_INFO, intInfo | (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT));
940 rc |= VMXWriteVMCS(VMX_VMCS_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
941 rc |= VMXWriteVMCS(VMX_VMCS_CTRL_ENTRY_EXCEPTION_ERRCODE, errCode);
942
943 AssertRC(rc);
944 return rc;
945}
946
947
948/**
949 * Checks for pending guest interrupts and injects them.
950 *
951 * @returns VBox status code.
952 * @param pVM Pointer to the VM.
953 * @param pVCpu Pointer to the VMCPU.
954 * @param pCtx Pointer to the guest CPU context.
955 */
956static int hmR0VmxCheckPendingInterrupt(PVM pVM, PVMCPU pVCpu, CPUMCTX *pCtx)
957{
958 int rc;
959
960 /*
961 * Dispatch any pending interrupts (injected before, but a VM exit occurred prematurely).
962 */
963 if (pVCpu->hwaccm.s.Event.fPending)
964 {
965 Log(("CPU%d: Reinjecting event %RX64 %08x at %RGv cr2=%RX64\n", pVCpu->idCpu, pVCpu->hwaccm.s.Event.intInfo,
966 pVCpu->hwaccm.s.Event.errCode, (RTGCPTR)pCtx->rip, pCtx->cr2));
967 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatIntReinject);
968 rc = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, pVCpu->hwaccm.s.Event.intInfo, 0, pVCpu->hwaccm.s.Event.errCode);
969 AssertRC(rc);
970
971 pVCpu->hwaccm.s.Event.fPending = false;
972 return VINF_SUCCESS;
973 }
974
975 /*
976 * If an active trap is already pending, we must forward it first!
977 */
978 if (!TRPMHasTrap(pVCpu))
979 {
980 if (VMCPU_FF_TESTANDCLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI))
981 {
982 RTGCUINTPTR intInfo;
983
984 Log(("CPU%d: injecting #NMI\n", pVCpu->idCpu));
985
986 intInfo = X86_XCPT_NMI;
987 intInfo |= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
988 intInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
989
990 rc = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, intInfo, 0, 0);
991 AssertRC(rc);
992
993 return VINF_SUCCESS;
994 }
995
996 /** @todo SMI interrupts. */
997
998 /*
999 * When external interrupts are pending, we should exit the VM when IF is set.
1000 */
1001 if (VMCPU_FF_ISPENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC|VMCPU_FF_INTERRUPT_PIC)))
1002 {
1003 if (!(pCtx->eflags.u32 & X86_EFL_IF))
1004 {
1005 if (!(pVCpu->hwaccm.s.vmx.proc_ctls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_IRQ_WINDOW_EXIT))
1006 {
1007 LogFlow(("Enable irq window exit!\n"));
1008 pVCpu->hwaccm.s.vmx.proc_ctls |= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_IRQ_WINDOW_EXIT;
1009 rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
1010 AssertRC(rc);
1011 }
1012 /* else nothing to do but wait */
1013 }
1014 else if (!VMCPU_FF_ISSET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
1015 {
1016 uint8_t u8Interrupt;
1017
1018 rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
1019 Log(("CPU%d: Dispatch interrupt: u8Interrupt=%x (%d) rc=%Rrc cs:rip=%04X:%RGv\n", pVCpu->idCpu,
1020 u8Interrupt, u8Interrupt, rc, pCtx->cs.Sel, (RTGCPTR)pCtx->rip));
1021 if (RT_SUCCESS(rc))
1022 {
1023 rc = TRPMAssertTrap(pVCpu, u8Interrupt, TRPM_HARDWARE_INT);
1024 AssertRC(rc);
1025 }
1026 else
1027 {
1028 /* Can only happen in rare cases where a pending interrupt is cleared behind our back */
1029 Assert(!VMCPU_FF_ISPENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC|VMCPU_FF_INTERRUPT_PIC)));
1030 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatSwitchGuestIrq);
1031 /* Just continue */
1032 }
1033 }
1034 else
1035 Log(("Pending interrupt blocked at %RGv by VM_FF_INHIBIT_INTERRUPTS!!\n", (RTGCPTR)pCtx->rip));
1036 }
1037 }
1038
1039#ifdef VBOX_STRICT
1040 if (TRPMHasTrap(pVCpu))
1041 {
1042 uint8_t u8Vector;
1043 rc = TRPMQueryTrapAll(pVCpu, &u8Vector, 0, 0, 0);
1044 AssertRC(rc);
1045 }
1046#endif
1047
1048 if ( (pCtx->eflags.u32 & X86_EFL_IF)
1049 && (!VMCPU_FF_ISSET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
1050 && TRPMHasTrap(pVCpu)
1051 )
1052 {
1053 uint8_t u8Vector;
1054 TRPMEVENT enmType;
1055 RTGCUINTPTR intInfo;
1056 RTGCUINT errCode;
1057
1058 /*
1059 * If a new event is pending, dispatch it now.
1060 */
1061 rc = TRPMQueryTrapAll(pVCpu, &u8Vector, &enmType, &errCode, 0);
1062 AssertRC(rc);
1063 Assert(pCtx->eflags.Bits.u1IF == 1 || enmType == TRPM_TRAP);
1064 Assert(enmType != TRPM_SOFTWARE_INT);
1065
1066 /*
1067 * Clear the pending trap.
1068 */
1069 rc = TRPMResetTrap(pVCpu);
1070 AssertRC(rc);
1071
1072 intInfo = u8Vector;
1073 intInfo |= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
1074
1075 if (enmType == TRPM_TRAP)
1076 {
1077 switch (u8Vector)
1078 {
1079 case X86_XCPT_DF:
1080 case X86_XCPT_TS:
1081 case X86_XCPT_NP:
1082 case X86_XCPT_SS:
1083 case X86_XCPT_GP:
1084 case X86_XCPT_PF:
1085 case X86_XCPT_AC:
1086 {
1087 /* Valid error codes. */
1088 intInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
1089 break;
1090 }
1091
1092 default:
1093 break;
1094 }
1095
1096 if ( u8Vector == X86_XCPT_BP
1097 || u8Vector == X86_XCPT_OF)
1098 {
1099 intInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SWEXCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
1100 }
1101 else
1102 intInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HWEXCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
1103 }
1104 else
1105 intInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
1106
1107 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatIntInject);
1108 rc = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, intInfo, 0, errCode);
1109 AssertRC(rc);
1110 } /* if (interrupts can be dispatched) */
1111
1112 return VINF_SUCCESS;
1113}
1114
1115
1116/**
1117 * Save the host state into the VMCS.
1118 *
1119 * @returns VBox status code.
1120 * @param pVM Pointer to the VM.
1121 * @param pVCpu Pointer to the VMCPU.
1122 */
1123VMMR0DECL(int) VMXR0SaveHostState(PVM pVM, PVMCPU pVCpu)
1124{
1125 int rc = VINF_SUCCESS;
1126 NOREF(pVM);
1127
1128 /*
1129 * Host CPU Context.
1130 */
1131 if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_HOST_CONTEXT)
1132 {
1133 RTIDTR idtr;
1134 RTGDTR gdtr;
1135 RTSEL SelTR;
1136 PCX86DESCHC pDesc;
1137 uintptr_t trBase;
1138 RTSEL cs;
1139 RTSEL ss;
1140 uint64_t cr3;
1141
1142 /*
1143 * Control registers.
1144 */
1145 rc = VMXWriteVMCS(VMX_VMCS_HOST_CR0, ASMGetCR0());
1146 Log2(("VMX_VMCS_HOST_CR0 %08x\n", ASMGetCR0()));
1147#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
1148 if (VMX_IS_64BIT_HOST_MODE())
1149 {
1150 cr3 = hwaccmR0Get64bitCR3();
1151 rc |= VMXWriteVMCS64(VMX_VMCS_HOST_CR3, cr3);
1152 }
1153 else
1154#endif
1155 {
1156 cr3 = ASMGetCR3();
1157 rc |= VMXWriteVMCS(VMX_VMCS_HOST_CR3, cr3);
1158 }
1159 Log2(("VMX_VMCS_HOST_CR3 %08RX64\n", cr3));
1160 rc |= VMXWriteVMCS(VMX_VMCS_HOST_CR4, ASMGetCR4());
1161 Log2(("VMX_VMCS_HOST_CR4 %08x\n", ASMGetCR4()));
1162 AssertRC(rc);
1163
1164 /*
1165 * Selector registers.
1166 */
1167#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
1168 if (VMX_IS_64BIT_HOST_MODE())
1169 {
1170 cs = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelCS;
1171 ss = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelSS;
1172 }
1173 else
1174 {
1175 /* sysenter loads LDT cs & ss, VMX doesn't like this. Load the GDT ones (safe). */
1176 cs = (RTSEL)(uintptr_t)&SUPR0AbsKernelCS;
1177 ss = (RTSEL)(uintptr_t)&SUPR0AbsKernelSS;
1178 }
1179#else
1180 cs = ASMGetCS();
1181 ss = ASMGetSS();
1182#endif
1183 Assert(!(cs & X86_SEL_LDT)); Assert((cs & X86_SEL_RPL) == 0);
1184 Assert(!(ss & X86_SEL_LDT)); Assert((ss & X86_SEL_RPL) == 0);
1185 rc = VMXWriteVMCS(VMX_VMCS16_HOST_FIELD_CS, cs);
1186 /* Note: VMX is (again) very picky about the RPL of the selectors here; we'll restore them manually. */
1187 rc |= VMXWriteVMCS(VMX_VMCS16_HOST_FIELD_DS, 0);
1188 rc |= VMXWriteVMCS(VMX_VMCS16_HOST_FIELD_ES, 0);
1189#if HC_ARCH_BITS == 32
1190 if (!VMX_IS_64BIT_HOST_MODE())
1191 {
1192 rc |= VMXWriteVMCS(VMX_VMCS16_HOST_FIELD_FS, 0);
1193 rc |= VMXWriteVMCS(VMX_VMCS16_HOST_FIELD_GS, 0);
1194 }
1195#endif
1196 rc |= VMXWriteVMCS(VMX_VMCS16_HOST_FIELD_SS, ss);
1197 SelTR = ASMGetTR();
1198 rc |= VMXWriteVMCS(VMX_VMCS16_HOST_FIELD_TR, SelTR);
1199 AssertRC(rc);
1200 Log2(("VMX_VMCS_HOST_FIELD_CS %08x (%08x)\n", cs, ASMGetSS()));
1201 Log2(("VMX_VMCS_HOST_FIELD_DS 00000000 (%08x)\n", ASMGetDS()));
1202 Log2(("VMX_VMCS_HOST_FIELD_ES 00000000 (%08x)\n", ASMGetES()));
1203 Log2(("VMX_VMCS_HOST_FIELD_FS 00000000 (%08x)\n", ASMGetFS()));
1204 Log2(("VMX_VMCS_HOST_FIELD_GS 00000000 (%08x)\n", ASMGetGS()));
1205 Log2(("VMX_VMCS_HOST_FIELD_SS %08x (%08x)\n", ss, ASMGetSS()));
1206 Log2(("VMX_VMCS_HOST_FIELD_TR %08x\n", ASMGetTR()));
1207
1208 /*
1209 * GDTR & IDTR.
1210 */
1211#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
1212 if (VMX_IS_64BIT_HOST_MODE())
1213 {
1214 X86XDTR64 gdtr64, idtr64;
1215 hwaccmR0Get64bitGDTRandIDTR(&gdtr64, &idtr64);
1216 rc = VMXWriteVMCS64(VMX_VMCS_HOST_GDTR_BASE, gdtr64.uAddr);
1217 rc |= VMXWriteVMCS64(VMX_VMCS_HOST_IDTR_BASE, gdtr64.uAddr);
1218 AssertRC(rc);
1219 Log2(("VMX_VMCS_HOST_GDTR_BASE %RX64\n", gdtr64.uAddr));
1220 Log2(("VMX_VMCS_HOST_IDTR_BASE %RX64\n", idtr64.uAddr));
1221 gdtr.cbGdt = gdtr64.cb;
1222 gdtr.pGdt = (uintptr_t)gdtr64.uAddr;
1223 }
1224 else
1225#endif
1226 {
1227 ASMGetGDTR(&gdtr);
1228 rc = VMXWriteVMCS(VMX_VMCS_HOST_GDTR_BASE, gdtr.pGdt);
1229 ASMGetIDTR(&idtr);
1230 rc |= VMXWriteVMCS(VMX_VMCS_HOST_IDTR_BASE, idtr.pIdt);
1231 AssertRC(rc);
1232 Log2(("VMX_VMCS_HOST_GDTR_BASE %RHv\n", gdtr.pGdt));
1233 Log2(("VMX_VMCS_HOST_IDTR_BASE %RHv\n", idtr.pIdt));
1234 }
1235
1236 /*
1237 * Save the base address of the TR selector.
1238 */
1239 if (SelTR > gdtr.cbGdt)
1240 {
1241 AssertMsgFailed(("Invalid TR selector %x. GDTR.cbGdt=%x\n", SelTR, gdtr.cbGdt));
1242 return VERR_VMX_INVALID_HOST_STATE;
1243 }
1244
1245 pDesc = (PCX86DESCHC)(gdtr.pGdt + (SelTR & X86_SEL_MASK));
1246#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
1247 if (VMX_IS_64BIT_HOST_MODE())
1248 {
1249 uint64_t trBase64 = X86DESC64_BASE(*(PX86DESC64)pDesc);
1250 rc = VMXWriteVMCS64(VMX_VMCS_HOST_TR_BASE, trBase64);
1251 Log2(("VMX_VMCS_HOST_TR_BASE %RX64\n", trBase64));
1252 AssertRC(rc);
1253 }
1254 else
1255#endif
1256 {
1257#if HC_ARCH_BITS == 64
1258 trBase = X86DESC64_BASE(*pDesc);
1259#else
1260 trBase = X86DESC_BASE(*pDesc);
1261#endif
1262 rc = VMXWriteVMCS(VMX_VMCS_HOST_TR_BASE, trBase);
1263 AssertRC(rc);
1264 Log2(("VMX_VMCS_HOST_TR_BASE %RHv\n", trBase));
1265 }
1266
1267 /*
1268 * FS base and GS base.
1269 */
1270#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
1271 if (VMX_IS_64BIT_HOST_MODE())
1272 {
1273 Log2(("MSR_K8_FS_BASE = %RX64\n", ASMRdMsr(MSR_K8_FS_BASE)));
1274 Log2(("MSR_K8_GS_BASE = %RX64\n", ASMRdMsr(MSR_K8_GS_BASE)));
1275 rc = VMXWriteVMCS64(VMX_VMCS_HOST_FS_BASE, ASMRdMsr(MSR_K8_FS_BASE));
1276 rc |= VMXWriteVMCS64(VMX_VMCS_HOST_GS_BASE, ASMRdMsr(MSR_K8_GS_BASE));
1277 }
1278#endif
1279 AssertRC(rc);
1280
1281 /*
1282 * Sysenter MSRs.
1283 */
1284 /** @todo expensive!! */
1285 rc = VMXWriteVMCS(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
1286 Log2(("VMX_VMCS_HOST_SYSENTER_CS %08x\n", ASMRdMsr_Low(MSR_IA32_SYSENTER_CS)));
1287#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
1288 if (VMX_IS_64BIT_HOST_MODE())
1289 {
1290 Log2(("VMX_VMCS_HOST_SYSENTER_EIP %RX64\n", ASMRdMsr(MSR_IA32_SYSENTER_EIP)));
1291 Log2(("VMX_VMCS_HOST_SYSENTER_ESP %RX64\n", ASMRdMsr(MSR_IA32_SYSENTER_ESP)));
1292 rc |= VMXWriteVMCS64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
1293 rc |= VMXWriteVMCS64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
1294 }
1295 else
1296 {
1297 rc |= VMXWriteVMCS(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
1298 rc |= VMXWriteVMCS(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
1299 Log2(("VMX_VMCS_HOST_SYSENTER_EIP %RX32\n", ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP)));
1300 Log2(("VMX_VMCS_HOST_SYSENTER_ESP %RX32\n", ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP)));
1301 }
1302#elif HC_ARCH_BITS == 32
1303 rc |= VMXWriteVMCS(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
1304 rc |= VMXWriteVMCS(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
1305 Log2(("VMX_VMCS_HOST_SYSENTER_EIP %RX32\n", ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP)));
1306 Log2(("VMX_VMCS_HOST_SYSENTER_ESP %RX32\n", ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP)));
1307#else
1308 Log2(("VMX_VMCS_HOST_SYSENTER_EIP %RX64\n", ASMRdMsr(MSR_IA32_SYSENTER_EIP)));
1309 Log2(("VMX_VMCS_HOST_SYSENTER_ESP %RX64\n", ASMRdMsr(MSR_IA32_SYSENTER_ESP)));
1310 rc |= VMXWriteVMCS64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
1311 rc |= VMXWriteVMCS64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
1312#endif
1313 AssertRC(rc);
1314
1315
1316#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
1317 /*
1318 * Store all host MSRs in the VM-Exit load area, so they will be reloaded after
1319 * the world switch back to the host.
1320 */
1321 PVMXMSR pMsr = (PVMXMSR)pVCpu->hwaccm.s.vmx.pHostMSR;
1322 unsigned idxMsr = 0;
1323
1324 /*
1325 * Check if EFER MSR present.
1326 */
1327 uint32_t u32HostExtFeatures = ASMCpuId_EDX(0x80000001);
1328 if (u32HostExtFeatures & (X86_CPUID_EXT_FEATURE_EDX_NX | X86_CPUID_EXT_FEATURE_EDX_LONG_MODE))
1329 {
1330 if (u32HostExtFeatures & X86_CPUID_EXT_FEATURE_EDX_SYSCALL)
1331 {
1332 pMsr->u32IndexMSR = MSR_K6_STAR;
1333 pMsr->u32Reserved = 0;
1334 pMsr->u64Value = ASMRdMsr(MSR_K6_STAR); /* legacy syscall eip, cs & ss */
1335 pMsr++; idxMsr++;
1336 }
1337
1338 pMsr->u32IndexMSR = MSR_K6_EFER;
1339 pMsr->u32Reserved = 0;
1340# if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
1341 if (CPUMIsGuestInLongMode(pVCpu))
1342 {
1343 /* Must match the EFER value in our 64 bits switcher. */
1344 pMsr->u64Value = ASMRdMsr(MSR_K6_EFER) | MSR_K6_EFER_LME | MSR_K6_EFER_SCE | MSR_K6_EFER_NXE;
1345 }
1346 else
1347# endif
1348 pMsr->u64Value = ASMRdMsr(MSR_K6_EFER);
1349 pMsr++; idxMsr++;
1350 }
1351
1352# if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
1353 if (VMX_IS_64BIT_HOST_MODE())
1354 {
1355 pMsr->u32IndexMSR = MSR_K8_LSTAR;
1356 pMsr->u32Reserved = 0;
1357 pMsr->u64Value = ASMRdMsr(MSR_K8_LSTAR); /* 64 bits mode syscall rip */
1358 pMsr++; idxMsr++;
1359 pMsr->u32IndexMSR = MSR_K8_SF_MASK;
1360 pMsr->u32Reserved = 0;
1361 pMsr->u64Value = ASMRdMsr(MSR_K8_SF_MASK); /* syscall flag mask */
1362 pMsr++; idxMsr++;
1363
1364 /* The KERNEL_GS_BASE MSR doesn't work reliably with auto load/store. See @bugref{6208} */
1365#if 0
1366 pMsr->u32IndexMSR = MSR_K8_KERNEL_GS_BASE;
1367 pMsr->u32Reserved = 0;
1368 pMsr->u64Value = ASMRdMsr(MSR_K8_KERNEL_GS_BASE); /* swapgs exchange value */
1369 pMsr++; idxMsr++;
1370#endif
1371 }
1372# endif
1373
1374 if ( (u32HostExtFeatures & X86_CPUID_EXT_FEATURE_EDX_RDTSCP)
1375 && pVCpu->hwaccm.s.vmx.proc_ctls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
1376 {
1377 pMsr->u32IndexMSR = MSR_K8_TSC_AUX;
1378 pMsr->u32Reserved = 0;
1379 pMsr->u64Value = ASMRdMsr(MSR_K8_TSC_AUX);
1380 pMsr++; idxMsr++;
1381 }
1382
1383 /** @todo r=ramshankar: check IA32_VMX_MISC bits 27:25 for valid idxMsr
1384 * range. */
1385 rc = VMXWriteVMCS(VMX_VMCS_CTRL_EXIT_MSR_LOAD_COUNT, idxMsr);
1386 AssertRC(rc);
1387
1388 pVCpu->hwaccm.s.vmx.cCachedMSRs = idxMsr;
1389#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
1390
1391 pVCpu->hwaccm.s.fContextUseFlags &= ~HWACCM_CHANGED_HOST_CONTEXT;
1392 }
1393 return rc;
1394}
1395
1396
1397/**
1398 * Loads the 4 PDPEs into the guest state when nested paging is used and the
1399 * guest operates in PAE mode.
1400 *
1401 * @returns VBox status code.
1402 * @param pVCpu Pointer to the VMCPU.
1403 * @param pCtx Pointer to the guest CPU context.
1404 */
1405static int hmR0VmxLoadPaePdpes(PVMCPU pVCpu, PCPUMCTX pCtx)
1406{
1407 if (CPUMIsGuestInPAEModeEx(pCtx))
1408 {
1409 X86PDPE aPdpes[4];
1410 int rc = PGMGstGetPaePdpes(pVCpu, &aPdpes[0]);
1411 AssertRCReturn(rc, rc);
1412
1413 rc = VMXWriteVMCS64(VMX_VMCS_GUEST_PDPTR0_FULL, aPdpes[0].u); AssertRCReturn(rc, rc);
1414 rc = VMXWriteVMCS64(VMX_VMCS_GUEST_PDPTR1_FULL, aPdpes[1].u); AssertRCReturn(rc, rc);
1415 rc = VMXWriteVMCS64(VMX_VMCS_GUEST_PDPTR2_FULL, aPdpes[2].u); AssertRCReturn(rc, rc);
1416 rc = VMXWriteVMCS64(VMX_VMCS_GUEST_PDPTR3_FULL, aPdpes[3].u); AssertRCReturn(rc, rc);
1417 }
1418 return VINF_SUCCESS;
1419}
1420
1421
1422/**
1423 * Saves the 4 PDPEs into the guest state when nested paging is used and the
1424 * guest operates in PAE mode.
1425 *
1426 * @returns VBox status code.
1427 * @param pVCpu Pointer to the VM CPU.
1428 * @param pCtx Pointer to the guest CPU context.
1429 *
1430 * @remarks Tell PGM about CR3 changes before calling this helper.
1431 */
1432static int hmR0VmxSavePaePdpes(PVMCPU pVCpu, PCPUMCTX pCtx)
1433{
1434 if (CPUMIsGuestInPAEModeEx(pCtx))
1435 {
1436 int rc;
1437 X86PDPE aPdpes[4];
1438 rc = VMXReadVMCS64(VMX_VMCS_GUEST_PDPTR0_FULL, &aPdpes[0].u); AssertRCReturn(rc, rc);
1439 rc = VMXReadVMCS64(VMX_VMCS_GUEST_PDPTR1_FULL, &aPdpes[1].u); AssertRCReturn(rc, rc);
1440 rc = VMXReadVMCS64(VMX_VMCS_GUEST_PDPTR2_FULL, &aPdpes[2].u); AssertRCReturn(rc, rc);
1441 rc = VMXReadVMCS64(VMX_VMCS_GUEST_PDPTR3_FULL, &aPdpes[3].u); AssertRCReturn(rc, rc);
1442
1443 rc = PGMGstUpdatePaePdpes(pVCpu, &aPdpes[0]);
1444 AssertRCReturn(rc, rc);
1445 }
1446 return VINF_SUCCESS;
1447}
1448
1449
1450/**
1451 * Update the exception bitmap according to the current CPU state.
1452 *
1453 * @param pVM Pointer to the VM.
1454 * @param pVCpu Pointer to the VMCPU.
1455 * @param pCtx Pointer to the guest CPU context.
1456 */
1457static void hmR0VmxUpdateExceptionBitmap(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
1458{
1459 uint32_t u32TrapMask;
1460 Assert(pCtx);
1461
1462 /*
1463 * Set up a mask for intercepting traps.
1464 */
1465 /** @todo Do we really need to always intercept #DB? */
1466 u32TrapMask = RT_BIT(X86_XCPT_DB)
1467 | RT_BIT(X86_XCPT_NM)
1468#ifdef VBOX_ALWAYS_TRAP_PF
1469 | RT_BIT(X86_XCPT_PF)
1470#endif
1471#ifdef VBOX_STRICT
1472 | RT_BIT(X86_XCPT_BP)
1473 | RT_BIT(X86_XCPT_DB)
1474 | RT_BIT(X86_XCPT_DE)
1475 | RT_BIT(X86_XCPT_NM)
1476 | RT_BIT(X86_XCPT_UD)
1477 | RT_BIT(X86_XCPT_NP)
1478 | RT_BIT(X86_XCPT_SS)
1479 | RT_BIT(X86_XCPT_GP)
1480 | RT_BIT(X86_XCPT_MF)
1481#endif
1482 ;
1483
1484 /*
1485 * Without nested paging, #PF must be intercepted to implement shadow paging.
1486 */
1487 /** @todo NP state won't change so maybe we should build the initial trap mask up front? */
1488 if (!pVM->hwaccm.s.fNestedPaging)
1489 u32TrapMask |= RT_BIT(X86_XCPT_PF);
1490
1491 /* Catch floating point exceptions if we need to report them to the guest in a different way. */
1492 if (!(pCtx->cr0 & X86_CR0_NE))
1493 u32TrapMask |= RT_BIT(X86_XCPT_MF);
1494
1495#ifdef VBOX_STRICT
1496 Assert(u32TrapMask & RT_BIT(X86_XCPT_GP));
1497#endif
1498
1499 /*
1500 * Intercept all exceptions in real mode as none of them can be injected directly (#GP otherwise).
1501 */
1502 /** @todo Despite the claim to intercept everything, with NP we do not intercept #PF. Should we? */
1503 if ( CPUMIsGuestInRealModeEx(pCtx)
1504 && pVM->hwaccm.s.vmx.pRealModeTSS)
1505 {
1506 u32TrapMask |= RT_BIT(X86_XCPT_DE)
1507 | RT_BIT(X86_XCPT_DB)
1508 | RT_BIT(X86_XCPT_NMI)
1509 | RT_BIT(X86_XCPT_BP)
1510 | RT_BIT(X86_XCPT_OF)
1511 | RT_BIT(X86_XCPT_BR)
1512 | RT_BIT(X86_XCPT_UD)
1513 | RT_BIT(X86_XCPT_DF)
1514 | RT_BIT(X86_XCPT_CO_SEG_OVERRUN)
1515 | RT_BIT(X86_XCPT_TS)
1516 | RT_BIT(X86_XCPT_NP)
1517 | RT_BIT(X86_XCPT_SS)
1518 | RT_BIT(X86_XCPT_GP)
1519 | RT_BIT(X86_XCPT_MF)
1520 | RT_BIT(X86_XCPT_AC)
1521 | RT_BIT(X86_XCPT_MC)
1522 | RT_BIT(X86_XCPT_XF)
1523 ;
1524 }
1525
1526 int rc = VMXWriteVMCS(VMX_VMCS_CTRL_EXCEPTION_BITMAP, u32TrapMask);
1527 AssertRC(rc);
1528}
1529
1530
1531/**
1532 * Loads a minimal guest state.
1533 *
1534 * NOTE: Don't do anything here that can cause a jump back to ring 3!!!!!
1535 *
1536 * @param pVM Pointer to the VM.
1537 * @param pVCpu Pointer to the VMCPU.
1538 * @param pCtx Pointer to the guest CPU context.
1539 */
1540VMMR0DECL(void) VMXR0LoadMinimalGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
1541{
1542 int rc;
1543 X86EFLAGS eflags;
1544
1545 Assert(!(pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_ALL_GUEST));
1546
1547 /*
1548 * Load EIP, ESP and EFLAGS.
1549 */
1550 rc = VMXWriteVMCS64(VMX_VMCS64_GUEST_RIP, pCtx->rip);
1551 rc |= VMXWriteVMCS64(VMX_VMCS64_GUEST_RSP, pCtx->rsp);
1552 AssertRC(rc);
1553
1554 /*
1555 * Bits 22-31, 15, 5 & 3 must be zero. Bit 1 must be 1.
1556 */
1557 eflags = pCtx->eflags;
1558 eflags.u32 &= VMX_EFLAGS_RESERVED_0;
1559 eflags.u32 |= VMX_EFLAGS_RESERVED_1;
1560
1561 /*
1562 * Check if real mode emulation using v86 mode.
1563 */
1564 if ( CPUMIsGuestInRealModeEx(pCtx)
1565 && pVM->hwaccm.s.vmx.pRealModeTSS)
1566 {
1567 pVCpu->hwaccm.s.vmx.RealMode.eflags = eflags;
1568
1569 eflags.Bits.u1VM = 1;
1570 eflags.Bits.u2IOPL = 0; /* must always be 0 or else certain instructions won't cause faults. */
1571 }
1572 rc = VMXWriteVMCS(VMX_VMCS_GUEST_RFLAGS, eflags.u32);
1573 AssertRC(rc);
1574}
1575
1576
1577/**
1578 * Loads the guest state.
1579 *
1580 * NOTE: Don't do anything here that can cause a jump back to ring 3!!!!!
1581 *
1582 * @returns VBox status code.
1583 * @param pVM Pointer to the VM.
1584 * @param pVCpu Pointer to the VMCPU.
1585 * @param pCtx Pointer to the guest CPU context.
1586 */
1587VMMR0DECL(int) VMXR0LoadGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
1588{
1589 int rc = VINF_SUCCESS;
1590 RTGCUINTPTR val;
1591
1592 /*
1593 * VMX_VMCS_CTRL_ENTRY_CONTROLS
1594 * Set required bits to one and zero according to the MSR capabilities.
1595 */
1596 val = pVM->hwaccm.s.vmx.msr.vmx_entry.n.disallowed0;
1597
1598 /*
1599 * Load guest debug controls (DR7 & IA32_DEBUGCTL_MSR).
1600 * Forced to 1 on the 'first' VT-x capable CPUs; this actually includes the newest Nehalem CPUs
1601 */
1602 val |= VMX_VMCS_CTRL_ENTRY_CONTROLS_LOAD_DEBUG;
1603
1604 if (CPUMIsGuestInLongModeEx(pCtx))
1605 val |= VMX_VMCS_CTRL_ENTRY_CONTROLS_IA64_MODE;
1606 /* else Must be zero when AMD64 is not available. */
1607
1608 /*
1609 * Mask away the bits that the CPU doesn't support.
1610 */
1611 val &= pVM->hwaccm.s.vmx.msr.vmx_entry.n.allowed1;
1612 rc = VMXWriteVMCS(VMX_VMCS_CTRL_ENTRY_CONTROLS, val);
1613 AssertRC(rc);
1614
1615 /*
1616 * VMX_VMCS_CTRL_EXIT_CONTROLS
1617 * Set required bits to one and zero according to the MSR capabilities.
1618 */
1619 val = pVM->hwaccm.s.vmx.msr.vmx_exit.n.disallowed0;
1620
1621 /*
1622 * Save debug controls (DR7 & IA32_DEBUGCTL_MSR)
1623 * Forced to 1 on the 'first' VT-x capable CPUs; this actually includes the newest Nehalem CPUs
1624 */
1625 val |= VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_DEBUG;
1626
1627#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
1628 if (VMX_IS_64BIT_HOST_MODE())
1629 val |= VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_AMD64;
1630 /* else Must be zero when AMD64 is not available. */
1631#elif HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
1632 if (CPUMIsGuestInLongModeEx(pCtx))
1633 val |= VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_AMD64; /* our switcher goes to long mode */
1634 else
1635 Assert(!(val & VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_AMD64));
1636#endif
1637 val &= pVM->hwaccm.s.vmx.msr.vmx_exit.n.allowed1;
1638
1639 /*
1640 * Don't acknowledge external interrupts on VM-exit.
1641 */
1642 rc = VMXWriteVMCS(VMX_VMCS_CTRL_EXIT_CONTROLS, val);
1643 AssertRC(rc);
1644
1645 /*
1646 * Guest CPU context: ES, CS, SS, DS, FS, GS.
1647 */
1648 if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_SEGMENT_REGS)
1649 {
1650 if (pVM->hwaccm.s.vmx.pRealModeTSS)
1651 {
1652 PGMMODE enmGuestMode = PGMGetGuestMode(pVCpu);
1653 if (pVCpu->hwaccm.s.vmx.enmLastSeenGuestMode != enmGuestMode)
1654 {
1655 /*
1656 * Correct weird requirements for switching to protected mode.
1657 */
1658 if ( pVCpu->hwaccm.s.vmx.enmLastSeenGuestMode == PGMMODE_REAL
1659 && enmGuestMode >= PGMMODE_PROTECTED)
1660 {
1661#ifdef VBOX_WITH_REM
1662 /*
1663 * Flush the recompiler code cache as it's not unlikely the guest will rewrite code
1664 * it will later execute in real mode (OpenBSD 4.0 is one such example)
1665 */
1666 REMFlushTBs(pVM);
1667#endif
1668
1669 /*
1670 * DPL of all hidden selector registers must match the current CPL (0).
1671 */
1672 pCtx->cs.Attr.n.u2Dpl = 0;
1673 pCtx->cs.Attr.n.u4Type = X86_SEL_TYPE_CODE | X86_SEL_TYPE_RW_ACC;
1674
1675 pCtx->ds.Attr.n.u2Dpl = 0;
1676 pCtx->es.Attr.n.u2Dpl = 0;
1677 pCtx->fs.Attr.n.u2Dpl = 0;
1678 pCtx->gs.Attr.n.u2Dpl = 0;
1679 pCtx->ss.Attr.n.u2Dpl = 0;
1680 }
1681 pVCpu->hwaccm.s.vmx.enmLastSeenGuestMode = enmGuestMode;
1682 }
1683 else if ( CPUMIsGuestInRealModeEx(pCtx)
1684 && pCtx->cs.u64Base == 0xffff0000)
1685 {
1686 /* VT-x will fail with a guest invalid state otherwise... (CPU state after a reset) */
1687 pCtx->cs.u64Base = 0xf0000;
1688 pCtx->cs.Sel = 0xf000;
1689 }
1690 }
1691
1692 VMX_WRITE_SELREG(ES, es);
1693 AssertRC(rc);
1694
1695 VMX_WRITE_SELREG(CS, cs);
1696 AssertRC(rc);
1697
1698 VMX_WRITE_SELREG(SS, ss);
1699 AssertRC(rc);
1700
1701 VMX_WRITE_SELREG(DS, ds);
1702 AssertRC(rc);
1703
1704 VMX_WRITE_SELREG(FS, fs);
1705 AssertRC(rc);
1706
1707 VMX_WRITE_SELREG(GS, gs);
1708 AssertRC(rc);
1709 }
1710
1711 /*
1712 * Guest CPU context: LDTR.
1713 */
1714 if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_LDTR)
1715 {
1716 if (pCtx->ldtr.Sel == 0)
1717 {
1718 rc = VMXWriteVMCS(VMX_VMCS16_GUEST_FIELD_LDTR, 0);
1719 rc |= VMXWriteVMCS(VMX_VMCS32_GUEST_LDTR_LIMIT, 0);
1720 rc |= VMXWriteVMCS64(VMX_VMCS64_GUEST_LDTR_BASE, 0);
1721 /* Note: vmlaunch will fail with 0 or just 0x02. No idea why. */
1722 rc |= VMXWriteVMCS(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, 0x82 /* present, LDT */);
1723 }
1724 else
1725 {
1726 rc = VMXWriteVMCS(VMX_VMCS16_GUEST_FIELD_LDTR, pCtx->ldtr.Sel);
1727 rc |= VMXWriteVMCS(VMX_VMCS32_GUEST_LDTR_LIMIT, pCtx->ldtr.u32Limit);
1728 rc |= VMXWriteVMCS64(VMX_VMCS64_GUEST_LDTR_BASE, pCtx->ldtr.u64Base);
1729 rc |= VMXWriteVMCS(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, pCtx->ldtr.Attr.u);
1730 }
1731 AssertRC(rc);
1732 }
1733
1734 /*
1735 * Guest CPU context: TR.
1736 */
1737 if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_TR)
1738 {
1739 /*
1740 * Real mode emulation using v86 mode with CR4.VME (interrupt redirection
1741 * using the int bitmap in the TSS).
1742 */
1743 if ( CPUMIsGuestInRealModeEx(pCtx)
1744 && pVM->hwaccm.s.vmx.pRealModeTSS)
1745 {
1746 RTGCPHYS GCPhys;
1747
1748 /* We convert it here every time as PCI regions could be reconfigured. */
1749 rc = PDMVMMDevHeapR3ToGCPhys(pVM, pVM->hwaccm.s.vmx.pRealModeTSS, &GCPhys);
1750 AssertRC(rc);
1751
1752 rc = VMXWriteVMCS(VMX_VMCS16_GUEST_FIELD_TR, 0);
1753 rc |= VMXWriteVMCS(VMX_VMCS32_GUEST_TR_LIMIT, HWACCM_VTX_TSS_SIZE);
1754 rc |= VMXWriteVMCS64(VMX_VMCS64_GUEST_TR_BASE, GCPhys /* phys = virt in this mode */);
1755
1756 X86DESCATTR attr;
1757
1758 attr.u = 0;
1759 attr.n.u1Present = 1;
1760 attr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
1761 val = attr.u;
1762 }
1763 else
1764 {
1765 rc = VMXWriteVMCS(VMX_VMCS16_GUEST_FIELD_TR, pCtx->tr.Sel);
1766 rc |= VMXWriteVMCS(VMX_VMCS32_GUEST_TR_LIMIT, pCtx->tr.u32Limit);
1767 rc |= VMXWriteVMCS64(VMX_VMCS64_GUEST_TR_BASE, pCtx->tr.u64Base);
1768
1769 val = pCtx->tr.Attr.u;
1770
1771 /* The TSS selector must be busy (REM bugs? see defect #XXXX). */
1772 if (!(val & X86_SEL_TYPE_SYS_TSS_BUSY_MASK))
1773 {
1774 if (val & 0xf)
1775 val |= X86_SEL_TYPE_SYS_TSS_BUSY_MASK;
1776 else
1777 /* Default if no TR selector has been set (otherwise vmlaunch will fail!) */
1778 val = (val & ~0xF) | X86_SEL_TYPE_SYS_386_TSS_BUSY;
1779 }
1780 AssertMsg((val & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY || (val & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY,
1781 ("%#x\n", val));
1782 }
1783 rc |= VMXWriteVMCS(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, val);
1784 AssertRC(rc);
1785 }
1786
1787 /*
1788 * Guest CPU context: GDTR.
1789 */
1790 if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_GDTR)
1791 {
1792 rc = VMXWriteVMCS(VMX_VMCS32_GUEST_GDTR_LIMIT, pCtx->gdtr.cbGdt);
1793 rc |= VMXWriteVMCS64(VMX_VMCS64_GUEST_GDTR_BASE, pCtx->gdtr.pGdt);
1794 AssertRC(rc);
1795 }
1796
1797 /*
1798 * Guest CPU context: IDTR.
1799 */
1800 if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_IDTR)
1801 {
1802 rc = VMXWriteVMCS(VMX_VMCS32_GUEST_IDTR_LIMIT, pCtx->idtr.cbIdt);
1803 rc |= VMXWriteVMCS64(VMX_VMCS64_GUEST_IDTR_BASE, pCtx->idtr.pIdt);
1804 AssertRC(rc);
1805 }
1806
1807 /*
1808 * Sysenter MSRs.
1809 */
1810 if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_MSR)
1811 {
1812 rc = VMXWriteVMCS(VMX_VMCS32_GUEST_SYSENTER_CS, pCtx->SysEnter.cs);
1813 rc |= VMXWriteVMCS64(VMX_VMCS64_GUEST_SYSENTER_EIP, pCtx->SysEnter.eip);
1814 rc |= VMXWriteVMCS64(VMX_VMCS64_GUEST_SYSENTER_ESP, pCtx->SysEnter.esp);
1815 AssertRC(rc);
1816 }
1817
1818 /*
1819 * Guest CPU context: Control registers.
1820 */
1821 if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_CR0)
1822 {
1823 val = pCtx->cr0;
1824 rc = VMXWriteVMCS(VMX_VMCS_CTRL_CR0_READ_SHADOW, val);
1825 Log2(("Guest CR0-shadow %08x\n", val));
1826 if (CPUMIsGuestFPUStateActive(pVCpu) == false)
1827 {
1828 /* Always use #NM exceptions to load the FPU/XMM state on demand. */
1829 val |= X86_CR0_TS | X86_CR0_ET | X86_CR0_NE | X86_CR0_MP;
1830 }
1831 else
1832 {
1833 /** @todo check if we support the old style mess correctly. */
1834 if (!(val & X86_CR0_NE))
1835 Log(("Forcing X86_CR0_NE!!!\n"));
1836
1837 val |= X86_CR0_NE; /* always turn on the native mechanism to report FPU errors (old style uses interrupts) */
1838 }
1839 /* Protected mode & paging are always enabled; we use them for emulating real and protected mode without paging too. */
1840 if (!pVM->hwaccm.s.vmx.fUnrestrictedGuest)
1841 val |= X86_CR0_PE | X86_CR0_PG;
1842
1843 if (pVM->hwaccm.s.fNestedPaging)
1844 {
1845 if (CPUMIsGuestInPagedProtectedModeEx(pCtx))
1846 {
1847 /* Disable CR3 read/write monitoring as we don't need it for EPT. */
1848 pVCpu->hwaccm.s.vmx.proc_ctls &= ~( VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
1849 | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT);
1850 }
1851 else
1852 {
1853 /* Reenable CR3 read/write monitoring as our identity mapped page table is active. */
1854 pVCpu->hwaccm.s.vmx.proc_ctls |= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
1855 | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT;
1856 }
1857 rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
1858 AssertRC(rc);
1859 }
1860 else
1861 {
1862 /* Note: We must also set this as we rely on protecting various pages for which supervisor writes must be caught. */
1863 val |= X86_CR0_WP;
1864 }
1865
1866 /* Always enable caching. */
1867 val &= ~(X86_CR0_CD|X86_CR0_NW);
1868
1869 rc |= VMXWriteVMCS64(VMX_VMCS64_GUEST_CR0, val);
1870 Log2(("Guest CR0 %08x\n", val));
1871
1872 /*
1873 * CR0 flags owned by the host; if the guests attempts to change them, then the VM will exit.
1874 */
1875 val = X86_CR0_PE /* Must monitor this bit (assumptions are made for real mode emulation) */
1876 | X86_CR0_WP /* Must monitor this bit (it must always be enabled). */
1877 | X86_CR0_PG /* Must monitor this bit (assumptions are made for real mode & protected mode without paging emulation) */
1878 | X86_CR0_CD /* Bit not restored during VM-exit! */
1879 | X86_CR0_NW /* Bit not restored during VM-exit! */
1880 | X86_CR0_NE;
1881
1882 /*
1883 * When the guest's FPU state is active, then we no longer care about the FPU related bits.
1884 */
1885 if (CPUMIsGuestFPUStateActive(pVCpu) == false)
1886 val |= X86_CR0_TS | X86_CR0_ET | X86_CR0_MP;
1887
1888 pVCpu->hwaccm.s.vmx.cr0_mask = val;
1889
1890 rc |= VMXWriteVMCS(VMX_VMCS_CTRL_CR0_MASK, val);
1891 Log2(("Guest CR0-mask %08x\n", val));
1892 AssertRC(rc);
1893 }
1894
1895 if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_CR4)
1896 {
1897 rc = VMXWriteVMCS(VMX_VMCS_CTRL_CR4_READ_SHADOW, pCtx->cr4);
1898 Log2(("Guest CR4-shadow %08x\n", pCtx->cr4));
1899 /* Set the required bits in cr4 too (currently X86_CR4_VMXE). */
1900 val = pCtx->cr4 | (uint32_t)pVM->hwaccm.s.vmx.msr.vmx_cr4_fixed0;
1901
1902 if (!pVM->hwaccm.s.fNestedPaging)
1903 {
1904 switch(pVCpu->hwaccm.s.enmShadowMode)
1905 {
1906 case PGMMODE_REAL: /* Real mode -> emulated using v86 mode */
1907 case PGMMODE_PROTECTED: /* Protected mode, no paging -> emulated using identity mapping. */
1908 case PGMMODE_32_BIT: /* 32-bit paging. */
1909 val &= ~X86_CR4_PAE;
1910 break;
1911
1912 case PGMMODE_PAE: /* PAE paging. */
1913 case PGMMODE_PAE_NX: /* PAE paging with NX enabled. */
1914 /** Must use PAE paging as we could use physical memory > 4 GB */
1915 val |= X86_CR4_PAE;
1916 break;
1917
1918 case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
1919 case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
1920#ifdef VBOX_ENABLE_64_BITS_GUESTS
1921 break;
1922#else
1923 AssertFailed();
1924 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
1925#endif
1926 default: /* shut up gcc */
1927 AssertFailed();
1928 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
1929 }
1930 }
1931 else if ( !CPUMIsGuestInPagedProtectedModeEx(pCtx)
1932 && !pVM->hwaccm.s.vmx.fUnrestrictedGuest)
1933 {
1934 /* We use 4 MB pages in our identity mapping page table for real and protected mode without paging. */
1935 val |= X86_CR4_PSE;
1936 /* Our identity mapping is a 32 bits page directory. */
1937 val &= ~X86_CR4_PAE;
1938 }
1939
1940 /*
1941 * Turn off VME if we're in emulated real mode.
1942 */
1943 if ( CPUMIsGuestInRealModeEx(pCtx)
1944 && pVM->hwaccm.s.vmx.pRealModeTSS)
1945 {
1946 val &= ~X86_CR4_VME;
1947 }
1948
1949 rc |= VMXWriteVMCS64(VMX_VMCS64_GUEST_CR4, val);
1950 Log2(("Guest CR4 %08x\n", val));
1951
1952 /*
1953 * CR4 flags owned by the host; if the guests attempts to change them, then the VM will exit.
1954 */
1955 val = 0
1956 | X86_CR4_VME
1957 | X86_CR4_PAE
1958 | X86_CR4_PGE
1959 | X86_CR4_PSE
1960 | X86_CR4_VMXE;
1961 pVCpu->hwaccm.s.vmx.cr4_mask = val;
1962
1963 rc |= VMXWriteVMCS(VMX_VMCS_CTRL_CR4_MASK, val);
1964 Log2(("Guest CR4-mask %08x\n", val));
1965 AssertRC(rc);
1966 }
1967
1968#if 0
1969 /* Enable single stepping if requested and CPU supports it. */
1970 if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_TRAP_FLAG)
1971 if (DBGFIsStepping(pVCpu))
1972 {
1973 pVCpu->hwaccm.s.vmx.proc_ctls |= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_TRAP_FLAG;
1974 rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
1975 AssertRC(rc);
1976 }
1977#endif
1978
1979 if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_CR3)
1980 {
1981 if (pVM->hwaccm.s.fNestedPaging)
1982 {
1983 Assert(PGMGetHyperCR3(pVCpu));
1984 pVCpu->hwaccm.s.vmx.GCPhysEPTP = PGMGetHyperCR3(pVCpu);
1985
1986 Assert(!(pVCpu->hwaccm.s.vmx.GCPhysEPTP & 0xfff));
1987 /** @todo Check the IA32_VMX_EPT_VPID_CAP MSR for other supported memory types. */
1988 pVCpu->hwaccm.s.vmx.GCPhysEPTP |= VMX_EPT_MEMTYPE_WB
1989 | (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
1990
1991 rc = VMXWriteVMCS64(VMX_VMCS_CTRL_EPTP_FULL, pVCpu->hwaccm.s.vmx.GCPhysEPTP);
1992 AssertRC(rc);
1993
1994 if ( !CPUMIsGuestInPagedProtectedModeEx(pCtx)
1995 && !pVM->hwaccm.s.vmx.fUnrestrictedGuest)
1996 {
1997 RTGCPHYS GCPhys;
1998
1999 /* We convert it here every time as PCI regions could be reconfigured. */
2000 rc = PDMVMMDevHeapR3ToGCPhys(pVM, pVM->hwaccm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
2001 AssertMsgRC(rc, ("pNonPagingModeEPTPageTable = %RGv\n", pVM->hwaccm.s.vmx.pNonPagingModeEPTPageTable));
2002
2003 /*
2004 * We use our identity mapping page table here as we need to map guest virtual to
2005 * guest physical addresses; EPT will take care of the translation to host physical addresses.
2006 */
2007 val = GCPhys;
2008 }
2009 else
2010 {
2011 /* Save the real guest CR3 in VMX_VMCS_GUEST_CR3 */
2012 val = pCtx->cr3;
2013 rc = hmR0VmxLoadPaePdpes(pVCpu, pCtx);
2014 AssertRCReturn(rc, rc);
2015 }
2016 }
2017 else
2018 {
2019 val = PGMGetHyperCR3(pVCpu);
2020 Assert(val || VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL));
2021 }
2022
2023 /* Save our shadow CR3 register. */
2024 rc = VMXWriteVMCS64(VMX_VMCS64_GUEST_CR3, val);
2025 AssertRC(rc);
2026 }
2027
2028 /*
2029 * Guest CPU context: Debug registers.
2030 */
2031 if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_DEBUG)
2032 {
2033 pCtx->dr[6] |= X86_DR6_INIT_VAL; /* set all reserved bits to 1. */
2034 pCtx->dr[6] &= ~RT_BIT(12); /* must be zero. */
2035
2036 pCtx->dr[7] &= 0xffffffff; /* upper 32 bits reserved */
2037 pCtx->dr[7] &= ~(RT_BIT(11) | RT_BIT(12) | RT_BIT(14) | RT_BIT(15)); /* must be zero */
2038 pCtx->dr[7] |= 0x400; /* must be one */
2039
2040 /* Resync DR7 */
2041 rc = VMXWriteVMCS64(VMX_VMCS64_GUEST_DR7, pCtx->dr[7]);
2042 AssertRC(rc);
2043
2044#ifdef DEBUG
2045 /* Sync the hypervisor debug state now if any breakpoint is armed. */
2046 if ( CPUMGetHyperDR7(pVCpu) & (X86_DR7_ENABLED_MASK|X86_DR7_GD)
2047 && !CPUMIsHyperDebugStateActive(pVCpu)
2048 && !DBGFIsStepping(pVCpu))
2049 {
2050 /* Save the host and load the hypervisor debug state. */
2051 rc = CPUMR0LoadHyperDebugState(pVM, pVCpu, pCtx, true /* include DR6 */);
2052 AssertRC(rc);
2053
2054 /* DRx intercepts remain enabled. */
2055
2056 /* Override dr7 with the hypervisor value. */
2057 rc = VMXWriteVMCS64(VMX_VMCS64_GUEST_DR7, CPUMGetHyperDR7(pVCpu));
2058 AssertRC(rc);
2059 }
2060 else
2061#endif
2062 /* Sync the debug state now if any breakpoint is armed. */
2063 if ( (pCtx->dr[7] & (X86_DR7_ENABLED_MASK|X86_DR7_GD))
2064 && !CPUMIsGuestDebugStateActive(pVCpu)
2065 && !DBGFIsStepping(pVCpu))
2066 {
2067 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatDRxArmed);
2068
2069 /* Disable DRx move intercepts. */
2070 pVCpu->hwaccm.s.vmx.proc_ctls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT;
2071 rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
2072 AssertRC(rc);
2073
2074 /* Save the host and load the guest debug state. */
2075 rc = CPUMR0LoadGuestDebugState(pVM, pVCpu, pCtx, true /* include DR6 */);
2076 AssertRC(rc);
2077 }
2078
2079 /* IA32_DEBUGCTL MSR. */
2080 rc = VMXWriteVMCS64(VMX_VMCS_GUEST_DEBUGCTL_FULL, 0);
2081 AssertRC(rc);
2082
2083 /** @todo do we really ever need this? */
2084 rc |= VMXWriteVMCS(VMX_VMCS_GUEST_DEBUG_EXCEPTIONS, 0);
2085 AssertRC(rc);
2086 }
2087
2088 /*
2089 * 64-bit guest mode.
2090 */
2091 if (CPUMIsGuestInLongModeEx(pCtx))
2092 {
2093#if !defined(VBOX_ENABLE_64_BITS_GUESTS)
2094 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
2095#elif HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2096 pVCpu->hwaccm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
2097#else
2098# ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2099 if (!pVM->hwaccm.s.fAllow64BitGuests)
2100 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
2101# endif
2102 pVCpu->hwaccm.s.vmx.pfnStartVM = VMXR0StartVM64;
2103#endif
2104 if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_MSR)
2105 {
2106 /* Update these as wrmsr might have changed them. */
2107 rc = VMXWriteVMCS64(VMX_VMCS64_GUEST_FS_BASE, pCtx->fs.u64Base);
2108 AssertRC(rc);
2109 rc = VMXWriteVMCS64(VMX_VMCS64_GUEST_GS_BASE, pCtx->gs.u64Base);
2110 AssertRC(rc);
2111 }
2112 }
2113 else
2114 {
2115 pVCpu->hwaccm.s.vmx.pfnStartVM = VMXR0StartVM32;
2116 }
2117
2118 hmR0VmxUpdateExceptionBitmap(pVM, pVCpu, pCtx);
2119
2120#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
2121 /*
2122 * Store all guest MSRs in the VM-entry load area, so they will be loaded
2123 * during the world switch.
2124 */
2125 PVMXMSR pMsr = (PVMXMSR)pVCpu->hwaccm.s.vmx.pGuestMSR;
2126 unsigned idxMsr = 0;
2127
2128 uint32_t u32GstExtFeatures;
2129 uint32_t u32Temp;
2130 CPUMGetGuestCpuId(pVCpu, 0x80000001, &u32Temp, &u32Temp, &u32Temp, &u32GstExtFeatures);
2131
2132 /*
2133 * Check if EFER MSR present.
2134 */
2135 if (u32GstExtFeatures & (X86_CPUID_EXT_FEATURE_EDX_NX | X86_CPUID_EXT_FEATURE_EDX_LONG_MODE))
2136 {
2137 pMsr->u32IndexMSR = MSR_K6_EFER;
2138 pMsr->u32Reserved = 0;
2139 pMsr->u64Value = pCtx->msrEFER;
2140 /* VT-x will complain if only MSR_K6_EFER_LME is set. */
2141 if (!CPUMIsGuestInLongModeEx(pCtx))
2142 pMsr->u64Value &= ~(MSR_K6_EFER_LMA | MSR_K6_EFER_LME);
2143 pMsr++; idxMsr++;
2144
2145 if (u32GstExtFeatures & X86_CPUID_EXT_FEATURE_EDX_LONG_MODE)
2146 {
2147 pMsr->u32IndexMSR = MSR_K8_LSTAR;
2148 pMsr->u32Reserved = 0;
2149 pMsr->u64Value = pCtx->msrLSTAR; /* 64 bits mode syscall rip */
2150 pMsr++; idxMsr++;
2151 pMsr->u32IndexMSR = MSR_K6_STAR;
2152 pMsr->u32Reserved = 0;
2153 pMsr->u64Value = pCtx->msrSTAR; /* legacy syscall eip, cs & ss */
2154 pMsr++; idxMsr++;
2155 pMsr->u32IndexMSR = MSR_K8_SF_MASK;
2156 pMsr->u32Reserved = 0;
2157 pMsr->u64Value = pCtx->msrSFMASK; /* syscall flag mask */
2158 pMsr++; idxMsr++;
2159
2160 /* The KERNEL_GS_BASE MSR doesn't work reliably with auto load/store. See @bugref{6208} */
2161#if 0
2162 pMsr->u32IndexMSR = MSR_K8_KERNEL_GS_BASE;
2163 pMsr->u32Reserved = 0;
2164 pMsr->u64Value = pCtx->msrKERNELGSBASE; /* swapgs exchange value */
2165 pMsr++; idxMsr++;
2166#endif
2167 }
2168 }
2169
2170 if ( pVCpu->hwaccm.s.vmx.proc_ctls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP
2171 && (u32GstExtFeatures & X86_CPUID_EXT_FEATURE_EDX_RDTSCP))
2172 {
2173 pMsr->u32IndexMSR = MSR_K8_TSC_AUX;
2174 pMsr->u32Reserved = 0;
2175 rc = CPUMQueryGuestMsr(pVCpu, MSR_K8_TSC_AUX, &pMsr->u64Value);
2176 AssertRC(rc);
2177 pMsr++; idxMsr++;
2178 }
2179
2180 /*
2181 * The number of host MSRs saved must be identical to the number of guest MSRs loaded.
2182 * It's not a VT-x requirement but how it's practically used here.
2183 */
2184 Assert(pVCpu->hwaccm.s.vmx.cCachedMSRs == idxMsr);
2185
2186 rc = VMXWriteVMCS(VMX_VMCS_CTRL_ENTRY_MSR_LOAD_COUNT, idxMsr);
2187 AssertRC(rc);
2188
2189 rc = VMXWriteVMCS(VMX_VMCS_CTRL_EXIT_MSR_STORE_COUNT, idxMsr);
2190 AssertRC(rc);
2191#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
2192
2193 bool fOffsettedTsc;
2194 if (pVM->hwaccm.s.vmx.fUsePreemptTimer)
2195 {
2196 uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVCpu, &fOffsettedTsc, &pVCpu->hwaccm.s.vmx.u64TSCOffset);
2197
2198 /* Make sure the returned values have sane upper and lower boundaries. */
2199 uint64_t u64CpuHz = SUPGetCpuHzFromGIP(g_pSUPGlobalInfoPage);
2200
2201 cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64 of a second */
2202 cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
2203
2204 cTicksToDeadline >>= pVM->hwaccm.s.vmx.cPreemptTimerShift;
2205 uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
2206 rc = VMXWriteVMCS(VMX_VMCS32_GUEST_PREEMPTION_TIMER_VALUE, cPreemptionTickCount);
2207 AssertRC(rc);
2208 }
2209 else
2210 fOffsettedTsc = TMCpuTickCanUseRealTSC(pVCpu, &pVCpu->hwaccm.s.vmx.u64TSCOffset);
2211
2212 if (fOffsettedTsc)
2213 {
2214 uint64_t u64CurTSC = ASMReadTSC();
2215 if (u64CurTSC + pVCpu->hwaccm.s.vmx.u64TSCOffset >= TMCpuTickGetLastSeen(pVCpu))
2216 {
2217 /* Note: VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT takes precedence over TSC_OFFSET, applies to RDTSCP too. */
2218 rc = VMXWriteVMCS64(VMX_VMCS_CTRL_TSC_OFFSET_FULL, pVCpu->hwaccm.s.vmx.u64TSCOffset);
2219 AssertRC(rc);
2220
2221 pVCpu->hwaccm.s.vmx.proc_ctls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT;
2222 rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
2223 AssertRC(rc);
2224 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatTSCOffset);
2225 }
2226 else
2227 {
2228 /* Fall back to rdtsc, rdtscp emulation as we would otherwise pass decreasing tsc values to the guest. */
2229 LogFlow(("TSC %RX64 offset %RX64 time=%RX64 last=%RX64 (diff=%RX64, virt_tsc=%RX64)\n", u64CurTSC,
2230 pVCpu->hwaccm.s.vmx.u64TSCOffset, u64CurTSC + pVCpu->hwaccm.s.vmx.u64TSCOffset,
2231 TMCpuTickGetLastSeen(pVCpu), TMCpuTickGetLastSeen(pVCpu) - u64CurTSC - pVCpu->hwaccm.s.vmx.u64TSCOffset,
2232 TMCpuTickGet(pVCpu)));
2233 pVCpu->hwaccm.s.vmx.proc_ctls |= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT;
2234 rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
2235 AssertRC(rc);
2236 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatTSCInterceptOverFlow);
2237 }
2238 }
2239 else
2240 {
2241 pVCpu->hwaccm.s.vmx.proc_ctls |= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT;
2242 rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
2243 AssertRC(rc);
2244 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatTSCIntercept);
2245 }
2246
2247 /* Done with the major changes */
2248 pVCpu->hwaccm.s.fContextUseFlags &= ~HWACCM_CHANGED_ALL_GUEST;
2249
2250 /* Minimal guest state update (ESP, EIP, EFLAGS mostly) */
2251 VMXR0LoadMinimalGuestState(pVM, pVCpu, pCtx);
2252 return rc;
2253}
2254
2255
2256/**
2257 * Syncs back the guest state from VMCS.
2258 *
2259 * @returns VBox status code.
2260 * @param pVM Pointer to the VM.
2261 * @param pVCpu Pointer to the VMCPU.
2262 * @param pCtx Pointer to the guest CPU context.
2263 */
2264DECLINLINE(int) VMXR0SaveGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
2265{
2266 RTGCUINTREG val, valShadow;
2267 RTGCUINTPTR uInterruptState;
2268 int rc;
2269
2270 /* First sync back EIP, ESP, and EFLAGS. */
2271 rc = VMXReadCachedVMCS(VMX_VMCS64_GUEST_RIP, &val);
2272 AssertRC(rc);
2273 pCtx->rip = val;
2274 rc = VMXReadCachedVMCS(VMX_VMCS64_GUEST_RSP, &val);
2275 AssertRC(rc);
2276 pCtx->rsp = val;
2277 rc = VMXReadCachedVMCS(VMX_VMCS_GUEST_RFLAGS, &val);
2278 AssertRC(rc);
2279 pCtx->eflags.u32 = val;
2280
2281 /* Take care of instruction fusing (sti, mov ss) */
2282 rc |= VMXReadCachedVMCS(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &val);
2283 uInterruptState = val;
2284 if (uInterruptState != 0)
2285 {
2286 Assert(uInterruptState <= 2); /* only sti & mov ss */
2287 Log(("uInterruptState %x eip=%RGv\n", (uint32_t)uInterruptState, pCtx->rip));
2288 EMSetInhibitInterruptsPC(pVCpu, pCtx->rip);
2289 }
2290 else
2291 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
2292
2293 /* Control registers. */
2294 VMXReadCachedVMCS(VMX_VMCS_CTRL_CR0_READ_SHADOW, &valShadow);
2295 VMXReadCachedVMCS(VMX_VMCS64_GUEST_CR0, &val);
2296 val = (valShadow & pVCpu->hwaccm.s.vmx.cr0_mask) | (val & ~pVCpu->hwaccm.s.vmx.cr0_mask);
2297 CPUMSetGuestCR0(pVCpu, val);
2298
2299 VMXReadCachedVMCS(VMX_VMCS_CTRL_CR4_READ_SHADOW, &valShadow);
2300 VMXReadCachedVMCS(VMX_VMCS64_GUEST_CR4, &val);
2301 val = (valShadow & pVCpu->hwaccm.s.vmx.cr4_mask) | (val & ~pVCpu->hwaccm.s.vmx.cr4_mask);
2302 CPUMSetGuestCR4(pVCpu, val);
2303
2304 /*
2305 * No reason to sync back the CRx registers. They can't be changed by the guest unless in
2306 * the nested paging case where CR3 & CR4 can be changed by the guest.
2307 */
2308 if ( pVM->hwaccm.s.fNestedPaging
2309 && CPUMIsGuestInPagedProtectedModeEx(pCtx)) /** @todo check if we will always catch mode switches and such... */
2310 {
2311 PVMCSCACHE pCache = &pVCpu->hwaccm.s.vmx.VMCSCache;
2312
2313 /* Can be updated behind our back in the nested paging case. */
2314 CPUMSetGuestCR2(pVCpu, pCache->cr2);
2315
2316 VMXReadCachedVMCS(VMX_VMCS64_GUEST_CR3, &val);
2317
2318 if (val != pCtx->cr3)
2319 {
2320 CPUMSetGuestCR3(pVCpu, val);
2321 PGMUpdateCR3(pVCpu, val);
2322 }
2323 rc = hmR0VmxSavePaePdpes(pVCpu, pCtx);
2324 AssertRCReturn(rc, rc);
2325 }
2326
2327 /* Sync back DR7. */
2328 VMXReadCachedVMCS(VMX_VMCS64_GUEST_DR7, &val);
2329 pCtx->dr[7] = val;
2330
2331 /* Guest CPU context: ES, CS, SS, DS, FS, GS. */
2332 VMX_READ_SELREG(ES, es);
2333 VMX_READ_SELREG(SS, ss);
2334 VMX_READ_SELREG(CS, cs);
2335 VMX_READ_SELREG(DS, ds);
2336 VMX_READ_SELREG(FS, fs);
2337 VMX_READ_SELREG(GS, gs);
2338
2339 /* System MSRs */
2340 VMXReadCachedVMCS(VMX_VMCS32_GUEST_SYSENTER_CS, &val);
2341 pCtx->SysEnter.cs = val;
2342 VMXReadCachedVMCS(VMX_VMCS64_GUEST_SYSENTER_EIP, &val);
2343 pCtx->SysEnter.eip = val;
2344 VMXReadCachedVMCS(VMX_VMCS64_GUEST_SYSENTER_ESP, &val);
2345 pCtx->SysEnter.esp = val;
2346
2347 /* Misc. registers; must sync everything otherwise we can get out of sync when jumping to ring 3. */
2348 VMX_READ_SELREG(LDTR, ldtr);
2349
2350 VMXReadCachedVMCS(VMX_VMCS32_GUEST_GDTR_LIMIT, &val);
2351 pCtx->gdtr.cbGdt = val;
2352 VMXReadCachedVMCS(VMX_VMCS64_GUEST_GDTR_BASE, &val);
2353 pCtx->gdtr.pGdt = val;
2354
2355 VMXReadCachedVMCS(VMX_VMCS32_GUEST_IDTR_LIMIT, &val);
2356 pCtx->idtr.cbIdt = val;
2357 VMXReadCachedVMCS(VMX_VMCS64_GUEST_IDTR_BASE, &val);
2358 pCtx->idtr.pIdt = val;
2359
2360 /* Real mode emulation using v86 mode. */
2361 if ( CPUMIsGuestInRealModeEx(pCtx)
2362 && pVM->hwaccm.s.vmx.pRealModeTSS)
2363 {
2364 /* Hide our emulation flags */
2365 pCtx->eflags.Bits.u1VM = 0;
2366
2367 /* Restore original IOPL setting as we always use 0. */
2368 pCtx->eflags.Bits.u2IOPL = pVCpu->hwaccm.s.vmx.RealMode.eflags.Bits.u2IOPL;
2369
2370 /* Force a TR resync every time in case we switch modes. */
2371 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_GUEST_TR;
2372 }
2373 else
2374 {
2375 /* In real mode we have a fake TSS, so only sync it back when it's supposed to be valid. */
2376 VMX_READ_SELREG(TR, tr);
2377 }
2378
2379#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
2380 /*
2381 * Save the possibly changed MSRs that we automatically restore and save during a world switch.
2382 */
2383 for (unsigned i = 0; i < pVCpu->hwaccm.s.vmx.cCachedMSRs; i++)
2384 {
2385 PVMXMSR pMsr = (PVMXMSR)pVCpu->hwaccm.s.vmx.pGuestMSR;
2386 pMsr += i;
2387
2388 switch (pMsr->u32IndexMSR)
2389 {
2390 case MSR_K8_LSTAR:
2391 pCtx->msrLSTAR = pMsr->u64Value;
2392 break;
2393 case MSR_K6_STAR:
2394 pCtx->msrSTAR = pMsr->u64Value;
2395 break;
2396 case MSR_K8_SF_MASK:
2397 pCtx->msrSFMASK = pMsr->u64Value;
2398 break;
2399
2400 /* The KERNEL_GS_BASE MSR doesn't work reliably with auto load/store. See @bugref{6208} */
2401#if 0
2402 case MSR_K8_KERNEL_GS_BASE:
2403 pCtx->msrKERNELGSBASE = pMsr->u64Value;
2404 break;
2405#endif
2406 case MSR_K8_TSC_AUX:
2407 CPUMSetGuestMsr(pVCpu, MSR_K8_TSC_AUX, pMsr->u64Value);
2408 break;
2409
2410 case MSR_K6_EFER:
2411 /* EFER can't be changed without causing a VM-exit. */
2412 /* Assert(pCtx->msrEFER == pMsr->u64Value); */
2413 break;
2414 default:
2415 AssertFailed();
2416 return VERR_HM_UNEXPECTED_LD_ST_MSR;
2417 }
2418 }
2419#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
2420 return VINF_SUCCESS;
2421}
2422
2423
2424/**
2425 * Dummy placeholder for TLB flush handling before VM-entry. Used in the case
2426 * where neither EPT nor VPID is supported by the CPU.
2427 *
2428 * @param pVM Pointer to the VM.
2429 * @param pVCpu Pointer to the VMCPU.
2430 */
2431static DECLCALLBACK(void) hmR0VmxSetupTLBDummy(PVM pVM, PVMCPU pVCpu)
2432{
2433 NOREF(pVM);
2434 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
2435 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
2436 pVCpu->hwaccm.s.TlbShootdown.cPages = 0;
2437 return;
2438}
2439
2440
2441/**
2442 * Setup the tagged TLB for EPT+VPID.
2443 *
2444 * @param pVM Pointer to the VM.
2445 * @param pVCpu Pointer to the VMCPU.
2446 */
2447static DECLCALLBACK(void) hmR0VmxSetupTLBBoth(PVM pVM, PVMCPU pVCpu)
2448{
2449 PHMGLOBLCPUINFO pCpu;
2450
2451 Assert(pVM->hwaccm.s.fNestedPaging && pVM->hwaccm.s.vmx.fVPID);
2452
2453 pCpu = HWACCMR0GetCurrentCpu();
2454
2455 /*
2456 * Force a TLB flush for the first world switch if the current CPU differs from the one we ran on last
2457 * This can happen both for start & resume due to long jumps back to ring-3.
2458 * If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
2459 * or the host Cpu is online after a suspend/resume, so we cannot reuse the current ASID anymore.
2460 */
2461 bool fNewASID = false;
2462 if ( pVCpu->hwaccm.s.idLastCpu != pCpu->idCpu
2463 || pVCpu->hwaccm.s.cTLBFlushes != pCpu->cTLBFlushes)
2464 {
2465 pVCpu->hwaccm.s.fForceTLBFlush = true;
2466 fNewASID = true;
2467 }
2468
2469 /*
2470 * Check for explicit TLB shootdowns.
2471 */
2472 if (VMCPU_FF_TESTANDCLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2473 pVCpu->hwaccm.s.fForceTLBFlush = true;
2474
2475 pVCpu->hwaccm.s.idLastCpu = pCpu->idCpu;
2476
2477 if (pVCpu->hwaccm.s.fForceTLBFlush)
2478 {
2479 if (fNewASID)
2480 {
2481 ++pCpu->uCurrentASID;
2482 if (pCpu->uCurrentASID >= pVM->hwaccm.s.uMaxASID)
2483 {
2484 pCpu->uCurrentASID = 1; /* start at 1; host uses 0 */
2485 pCpu->cTLBFlushes++;
2486 pCpu->fFlushASIDBeforeUse = true;
2487 }
2488
2489 pVCpu->hwaccm.s.uCurrentASID = pCpu->uCurrentASID;
2490 if (pCpu->fFlushASIDBeforeUse)
2491 {
2492 hmR0VmxFlushVPID(pVM, pVCpu, pVM->hwaccm.s.vmx.enmFlushVPID, 0 /* GCPtr */);
2493#ifdef VBOX_WITH_STATISTICS
2494 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatFlushASID);
2495#endif
2496 }
2497 }
2498 else
2499 {
2500 if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVVPID_CAPS_SINGLE_CONTEXT)
2501 hmR0VmxFlushVPID(pVM, pVCpu, VMX_FLUSH_VPID_SINGLE_CONTEXT, 0 /* GCPtr */);
2502 else
2503 hmR0VmxFlushEPT(pVM, pVCpu, pVM->hwaccm.s.vmx.enmFlushEPT);
2504
2505#ifdef VBOX_WITH_STATISTICS
2506 /*
2507 * This is not terribly accurate (i.e. we don't have any StatFlushEPT counter). We currently count these
2508 * as ASID flushes too, better than including them under StatFlushTLBWorldSwitch.
2509 */
2510 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatFlushASID);
2511#endif
2512 }
2513
2514 pVCpu->hwaccm.s.cTLBFlushes = pCpu->cTLBFlushes;
2515 pVCpu->hwaccm.s.fForceTLBFlush = false;
2516 }
2517 else
2518 {
2519 AssertMsg(pVCpu->hwaccm.s.uCurrentASID && pCpu->uCurrentASID,
2520 ("hwaccm->uCurrentASID=%lu hwaccm->cTLBFlushes=%lu cpu->uCurrentASID=%lu cpu->cTLBFlushes=%lu\n",
2521 pVCpu->hwaccm.s.uCurrentASID, pVCpu->hwaccm.s.cTLBFlushes,
2522 pCpu->uCurrentASID, pCpu->cTLBFlushes));
2523
2524 /** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
2525 * not be executed. See hwaccmQueueInvlPage() where it is commented
2526 * out. Support individual entry flushing someday. */
2527 if (VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
2528 {
2529 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatTlbShootdown);
2530
2531 /*
2532 * Flush individual guest entries using VPID from the TLB or as little as possible with EPT
2533 * as supported by the CPU.
2534 */
2535 if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVVPID_CAPS_INDIV_ADDR)
2536 {
2537 for (unsigned i = 0; i < pVCpu->hwaccm.s.TlbShootdown.cPages; i++)
2538 hmR0VmxFlushVPID(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, pVCpu->hwaccm.s.TlbShootdown.aPages[i]);
2539 }
2540 else
2541 hmR0VmxFlushEPT(pVM, pVCpu, pVM->hwaccm.s.vmx.enmFlushEPT);
2542 }
2543 else
2544 {
2545#ifdef VBOX_WITH_STATISTICS
2546 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatNoFlushTLBWorldSwitch);
2547#endif
2548 }
2549 }
2550 pVCpu->hwaccm.s.TlbShootdown.cPages = 0;
2551 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
2552
2553 AssertMsg(pVCpu->hwaccm.s.cTLBFlushes == pCpu->cTLBFlushes,
2554 ("Flush count mismatch for cpu %d (%x vs %x)\n", pCpu->idCpu, pVCpu->hwaccm.s.cTLBFlushes, pCpu->cTLBFlushes));
2555 AssertMsg(pCpu->uCurrentASID >= 1 && pCpu->uCurrentASID < pVM->hwaccm.s.uMaxASID,
2556 ("cpu%d uCurrentASID = %x\n", pCpu->idCpu, pCpu->uCurrentASID));
2557 AssertMsg(pVCpu->hwaccm.s.uCurrentASID >= 1 && pVCpu->hwaccm.s.uCurrentASID < pVM->hwaccm.s.uMaxASID,
2558 ("cpu%d VM uCurrentASID = %x\n", pCpu->idCpu, pVCpu->hwaccm.s.uCurrentASID));
2559
2560 /* Update VMCS with the VPID. */
2561 int rc = VMXWriteVMCS(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hwaccm.s.uCurrentASID);
2562 AssertRC(rc);
2563}
2564
2565
2566/**
2567 * Setup the tagged TLB for EPT only.
2568 *
2569 * @returns VBox status code.
2570 * @param pVM Pointer to the VM.
2571 * @param pVCpu Pointer to the VMCPU.
2572 */
2573static DECLCALLBACK(void) hmR0VmxSetupTLBEPT(PVM pVM, PVMCPU pVCpu)
2574{
2575 PHMGLOBLCPUINFO pCpu;
2576
2577 Assert(pVM->hwaccm.s.fNestedPaging);
2578 Assert(!pVM->hwaccm.s.vmx.fVPID);
2579
2580 pCpu = HWACCMR0GetCurrentCpu();
2581
2582 /*
2583 * Force a TLB flush for the first world switch if the current CPU differs from the one we ran on last
2584 * This can happen both for start & resume due to long jumps back to ring-3.
2585 * A change in the TLB flush count implies the host Cpu is online after a suspend/resume.
2586 */
2587 if ( pVCpu->hwaccm.s.idLastCpu != pCpu->idCpu
2588 || pVCpu->hwaccm.s.cTLBFlushes != pCpu->cTLBFlushes)
2589 {
2590 pVCpu->hwaccm.s.fForceTLBFlush = true;
2591 }
2592
2593 /*
2594 * Check for explicit TLB shootdown flushes.
2595 */
2596 if (VMCPU_FF_TESTANDCLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2597 pVCpu->hwaccm.s.fForceTLBFlush = true;
2598
2599 pVCpu->hwaccm.s.idLastCpu = pCpu->idCpu;
2600 pVCpu->hwaccm.s.cTLBFlushes = pCpu->cTLBFlushes;
2601
2602 if (pVCpu->hwaccm.s.fForceTLBFlush)
2603 hmR0VmxFlushEPT(pVM, pVCpu, pVM->hwaccm.s.vmx.enmFlushEPT);
2604 else
2605 {
2606 /** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
2607 * not be executed. See hwaccmQueueInvlPage() where it is commented
2608 * out. Support individual entry flushing someday. */
2609 if (VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
2610 {
2611 /*
2612 * We cannot flush individual entries without VPID support. Flush using EPT.
2613 */
2614 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatTlbShootdown);
2615 hmR0VmxFlushEPT(pVM, pVCpu, pVM->hwaccm.s.vmx.enmFlushEPT);
2616 }
2617 }
2618 pVCpu->hwaccm.s.TlbShootdown.cPages= 0;
2619 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
2620
2621#ifdef VBOX_WITH_STATISTICS
2622 if (pVCpu->hwaccm.s.fForceTLBFlush)
2623 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatFlushTLBWorldSwitch);
2624 else
2625 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatNoFlushTLBWorldSwitch);
2626#endif
2627}
2628
2629
2630/**
2631 * Setup the tagged TLB for VPID.
2632 *
2633 * @returns VBox status code.
2634 * @param pVM Pointer to the VM.
2635 * @param pVCpu Pointer to the VMCPU.
2636 */
2637static DECLCALLBACK(void) hmR0VmxSetupTLBVPID(PVM pVM, PVMCPU pVCpu)
2638{
2639 PHMGLOBLCPUINFO pCpu;
2640
2641 Assert(pVM->hwaccm.s.vmx.fVPID);
2642 Assert(!pVM->hwaccm.s.fNestedPaging);
2643
2644 pCpu = HWACCMR0GetCurrentCpu();
2645
2646 /*
2647 * Force a TLB flush for the first world switch if the current CPU differs from the one we ran on last
2648 * This can happen both for start & resume due to long jumps back to ring-3.
2649 * If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
2650 * or the host Cpu is online after a suspend/resume, so we cannot reuse the current ASID anymore.
2651 */
2652 if ( pVCpu->hwaccm.s.idLastCpu != pCpu->idCpu
2653 || pVCpu->hwaccm.s.cTLBFlushes != pCpu->cTLBFlushes)
2654 {
2655 /* Force a TLB flush on VM entry. */
2656 pVCpu->hwaccm.s.fForceTLBFlush = true;
2657 }
2658
2659 /*
2660 * Check for explicit TLB shootdown flushes.
2661 */
2662 if (VMCPU_FF_TESTANDCLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2663 pVCpu->hwaccm.s.fForceTLBFlush = true;
2664
2665 pVCpu->hwaccm.s.idLastCpu = pCpu->idCpu;
2666
2667 if (pVCpu->hwaccm.s.fForceTLBFlush)
2668 {
2669 ++pCpu->uCurrentASID;
2670 if (pCpu->uCurrentASID >= pVM->hwaccm.s.uMaxASID)
2671 {
2672 pCpu->uCurrentASID = 1; /* start at 1; host uses 0 */
2673 pCpu->cTLBFlushes++;
2674 pCpu->fFlushASIDBeforeUse = true;
2675 }
2676 else
2677 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatFlushASID);
2678
2679 pVCpu->hwaccm.s.fForceTLBFlush = false;
2680 pVCpu->hwaccm.s.cTLBFlushes = pCpu->cTLBFlushes;
2681 pVCpu->hwaccm.s.uCurrentASID = pCpu->uCurrentASID;
2682 if (pCpu->fFlushASIDBeforeUse)
2683 hmR0VmxFlushVPID(pVM, pVCpu, pVM->hwaccm.s.vmx.enmFlushVPID, 0 /* GCPtr */);
2684 }
2685 else
2686 {
2687 AssertMsg(pVCpu->hwaccm.s.uCurrentASID && pCpu->uCurrentASID,
2688 ("hwaccm->uCurrentASID=%lu hwaccm->cTLBFlushes=%lu cpu->uCurrentASID=%lu cpu->cTLBFlushes=%lu\n",
2689 pVCpu->hwaccm.s.uCurrentASID, pVCpu->hwaccm.s.cTLBFlushes,
2690 pCpu->uCurrentASID, pCpu->cTLBFlushes));
2691
2692 /** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
2693 * not be executed. See hwaccmQueueInvlPage() where it is commented
2694 * out. Support individual entry flushing someday. */
2695 if (VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
2696 {
2697 /*
2698 * Flush individual guest entries using VPID from the TLB or as little as possible with EPT
2699 * as supported by the CPU.
2700 */
2701 if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVVPID_CAPS_INDIV_ADDR)
2702 {
2703 for (unsigned i = 0; i < pVCpu->hwaccm.s.TlbShootdown.cPages; i++)
2704 hmR0VmxFlushVPID(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, pVCpu->hwaccm.s.TlbShootdown.aPages[i]);
2705 }
2706 else
2707 hmR0VmxFlushVPID(pVM, pVCpu, pVM->hwaccm.s.vmx.enmFlushVPID, 0 /* GCPtr */);
2708 }
2709 }
2710 pVCpu->hwaccm.s.TlbShootdown.cPages = 0;
2711 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
2712
2713 AssertMsg(pVCpu->hwaccm.s.cTLBFlushes == pCpu->cTLBFlushes,
2714 ("Flush count mismatch for cpu %d (%x vs %x)\n", pCpu->idCpu, pVCpu->hwaccm.s.cTLBFlushes, pCpu->cTLBFlushes));
2715 AssertMsg(pCpu->uCurrentASID >= 1 && pCpu->uCurrentASID < pVM->hwaccm.s.uMaxASID,
2716 ("cpu%d uCurrentASID = %x\n", pCpu->idCpu, pCpu->uCurrentASID));
2717 AssertMsg(pVCpu->hwaccm.s.uCurrentASID >= 1 && pVCpu->hwaccm.s.uCurrentASID < pVM->hwaccm.s.uMaxASID,
2718 ("cpu%d VM uCurrentASID = %x\n", pCpu->idCpu, pVCpu->hwaccm.s.uCurrentASID));
2719
2720 int rc = VMXWriteVMCS(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hwaccm.s.uCurrentASID);
2721 AssertRC(rc);
2722
2723# ifdef VBOX_WITH_STATISTICS
2724 if (pVCpu->hwaccm.s.fForceTLBFlush)
2725 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatFlushTLBWorldSwitch);
2726 else
2727 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatNoFlushTLBWorldSwitch);
2728# endif
2729}
2730
2731
2732/**
2733 * Runs guest code in a VT-x VM.
2734 *
2735 * @returns VBox status code.
2736 * @param pVM Pointer to the VM.
2737 * @param pVCpu Pointer to the VMCPU.
2738 * @param pCtx Pointer to the guest CPU context.
2739 */
2740VMMR0DECL(int) VMXR0RunGuestCode(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
2741{
2742 STAM_PROFILE_ADV_START(&pVCpu->hwaccm.s.StatEntry, x);
2743 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hwaccm.s.StatExit1);
2744 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hwaccm.s.StatExit2);
2745
2746 VBOXSTRICTRC rc = VINF_SUCCESS;
2747 int rc2;
2748 RTGCUINTREG val;
2749 RTGCUINTREG exitReason = (RTGCUINTREG)VMX_EXIT_INVALID;
2750 RTGCUINTREG instrError, cbInstr;
2751 RTGCUINTPTR exitQualification = 0;
2752 RTGCUINTPTR intInfo = 0; /* shut up buggy gcc 4 */
2753 RTGCUINTPTR errCode, instrInfo;
2754 bool fSetupTPRCaching = false;
2755 uint64_t u64OldLSTAR = 0;
2756 uint8_t u8LastTPR = 0;
2757 RTCCUINTREG uOldEFlags = ~(RTCCUINTREG)0;
2758 unsigned cResume = 0;
2759#ifdef VBOX_STRICT
2760 RTCPUID idCpuCheck;
2761 bool fWasInLongMode = false;
2762#endif
2763#ifdef VBOX_HIGH_RES_TIMERS_HACK_IN_RING0
2764 uint64_t u64LastTime = RTTimeMilliTS();
2765#endif
2766
2767 Assert(!(pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
2768 || (pVCpu->hwaccm.s.vmx.pbVAPIC && pVM->hwaccm.s.vmx.pAPIC));
2769
2770 /*
2771 * Check if we need to use TPR shadowing.
2772 */
2773 if ( CPUMIsGuestInLongModeEx(pCtx)
2774 || ( (( pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
2775 || pVM->hwaccm.s.fTRPPatchingAllowed)
2776 && pVM->hwaccm.s.fHasIoApic)
2777 )
2778 {
2779 fSetupTPRCaching = true;
2780 }
2781
2782 Log2(("\nE"));
2783
2784#ifdef VBOX_STRICT
2785 {
2786 RTCCUINTREG val2;
2787
2788 rc2 = VMXReadVMCS(VMX_VMCS_CTRL_PIN_EXEC_CONTROLS, &val2);
2789 AssertRC(rc2);
2790 Log2(("VMX_VMCS_CTRL_PIN_EXEC_CONTROLS = %08x\n", val2));
2791
2792 /* allowed zero */
2793 if ((val2 & pVM->hwaccm.s.vmx.msr.vmx_pin_ctls.n.disallowed0) != pVM->hwaccm.s.vmx.msr.vmx_pin_ctls.n.disallowed0)
2794 Log(("Invalid VMX_VMCS_CTRL_PIN_EXEC_CONTROLS: zero\n"));
2795
2796 /* allowed one */
2797 if ((val2 & ~pVM->hwaccm.s.vmx.msr.vmx_pin_ctls.n.allowed1) != 0)
2798 Log(("Invalid VMX_VMCS_CTRL_PIN_EXEC_CONTROLS: one\n"));
2799
2800 rc2 = VMXReadVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, &val2);
2801 AssertRC(rc2);
2802 Log2(("VMX_VMCS_CTRL_PROC_EXEC_CONTROLS = %08x\n", val2));
2803
2804 /*
2805 * Must be set according to the MSR, but can be cleared if nested paging is used.
2806 */
2807 if (pVM->hwaccm.s.fNestedPaging)
2808 {
2809 val2 |= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INVLPG_EXIT
2810 | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
2811 | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT;
2812 }
2813
2814 /* allowed zero */
2815 if ((val2 & pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.disallowed0) != pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.disallowed0)
2816 Log(("Invalid VMX_VMCS_CTRL_PROC_EXEC_CONTROLS: zero\n"));
2817
2818 /* allowed one */
2819 if ((val2 & ~pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1) != 0)
2820 Log(("Invalid VMX_VMCS_CTRL_PROC_EXEC_CONTROLS: one\n"));
2821
2822 rc2 = VMXReadVMCS(VMX_VMCS_CTRL_ENTRY_CONTROLS, &val2);
2823 AssertRC(rc2);
2824 Log2(("VMX_VMCS_CTRL_ENTRY_CONTROLS = %08x\n", val2));
2825
2826 /* allowed zero */
2827 if ((val2 & pVM->hwaccm.s.vmx.msr.vmx_entry.n.disallowed0) != pVM->hwaccm.s.vmx.msr.vmx_entry.n.disallowed0)
2828 Log(("Invalid VMX_VMCS_CTRL_ENTRY_CONTROLS: zero\n"));
2829
2830 /* allowed one */
2831 if ((val2 & ~pVM->hwaccm.s.vmx.msr.vmx_entry.n.allowed1) != 0)
2832 Log(("Invalid VMX_VMCS_CTRL_ENTRY_CONTROLS: one\n"));
2833
2834 rc2 = VMXReadVMCS(VMX_VMCS_CTRL_EXIT_CONTROLS, &val2);
2835 AssertRC(rc2);
2836 Log2(("VMX_VMCS_CTRL_EXIT_CONTROLS = %08x\n", val2));
2837
2838 /* allowed zero */
2839 if ((val2 & pVM->hwaccm.s.vmx.msr.vmx_exit.n.disallowed0) != pVM->hwaccm.s.vmx.msr.vmx_exit.n.disallowed0)
2840 Log(("Invalid VMX_VMCS_CTRL_EXIT_CONTROLS: zero\n"));
2841
2842 /* allowed one */
2843 if ((val2 & ~pVM->hwaccm.s.vmx.msr.vmx_exit.n.allowed1) != 0)
2844 Log(("Invalid VMX_VMCS_CTRL_EXIT_CONTROLS: one\n"));
2845 }
2846 fWasInLongMode = CPUMIsGuestInLongModeEx(pCtx);
2847#endif /* VBOX_STRICT */
2848
2849#ifdef VBOX_WITH_CRASHDUMP_MAGIC
2850 pVCpu->hwaccm.s.vmx.VMCSCache.u64TimeEntry = RTTimeNanoTS();
2851#endif
2852
2853 /*
2854 * We can jump to this point to resume execution after determining that a VM-exit is innocent.
2855 */
2856ResumeExecution:
2857 if (!STAM_REL_PROFILE_ADV_IS_RUNNING(&pVCpu->hwaccm.s.StatEntry))
2858 STAM_REL_PROFILE_ADV_STOP_START(&pVCpu->hwaccm.s.StatExit2, &pVCpu->hwaccm.s.StatEntry, x);
2859 AssertMsg(pVCpu->hwaccm.s.idEnteredCpu == RTMpCpuId(),
2860 ("Expected %d, I'm %d; cResume=%d exitReason=%RGv exitQualification=%RGv\n",
2861 (int)pVCpu->hwaccm.s.idEnteredCpu, (int)RTMpCpuId(), cResume, exitReason, exitQualification));
2862 Assert(!HWACCMR0SuspendPending());
2863 /* Not allowed to switch modes without reloading the host state (32->64 switcher)!! */
2864 Assert(fWasInLongMode == CPUMIsGuestInLongModeEx(pCtx));
2865
2866 /*
2867 * Safety precaution; looping for too long here can have a very bad effect on the host.
2868 */
2869 if (RT_UNLIKELY(++cResume > pVM->hwaccm.s.cMaxResumeLoops))
2870 {
2871 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitMaxResume);
2872 rc = VINF_EM_RAW_INTERRUPT;
2873 goto end;
2874 }
2875
2876 /*
2877 * Check for IRQ inhibition due to instruction fusing (sti, mov ss).
2878 */
2879 if (VMCPU_FF_ISSET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
2880 {
2881 Log(("VM_FF_INHIBIT_INTERRUPTS at %RGv successor %RGv\n", (RTGCPTR)pCtx->rip, EMGetInhibitInterruptsPC(pVCpu)));
2882 if (pCtx->rip != EMGetInhibitInterruptsPC(pVCpu))
2883 {
2884 /*
2885 * Note: we intentionally don't clear VM_FF_INHIBIT_INTERRUPTS here.
2886 * Before we are able to execute this instruction in raw mode (iret to guest code) an external interrupt might
2887 * force a world switch again. Possibly allowing a guest interrupt to be dispatched in the process. This could
2888 * break the guest. Sounds very unlikely, but such timing sensitive problems are not as rare as you might think.
2889 */
2890 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
2891 /* Irq inhibition is no longer active; clear the corresponding VMX state. */
2892 rc2 = VMXWriteVMCS(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, 0);
2893 AssertRC(rc2);
2894 }
2895 }
2896 else
2897 {
2898 /* Irq inhibition is no longer active; clear the corresponding VMX state. */
2899 rc2 = VMXWriteVMCS(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, 0);
2900 AssertRC(rc2);
2901 }
2902
2903#ifdef VBOX_HIGH_RES_TIMERS_HACK_IN_RING0
2904 if (RT_UNLIKELY((cResume & 0xf) == 0))
2905 {
2906 uint64_t u64CurTime = RTTimeMilliTS();
2907
2908 if (RT_UNLIKELY(u64CurTime > u64LastTime))
2909 {
2910 u64LastTime = u64CurTime;
2911 TMTimerPollVoid(pVM, pVCpu);
2912 }
2913 }
2914#endif
2915
2916 /*
2917 * Check for pending actions that force us to go back to ring-3.
2918 */
2919 if ( VM_FF_ISPENDING(pVM, VM_FF_HWACCM_TO_R3_MASK | VM_FF_REQUEST | VM_FF_PGM_POOL_FLUSH_PENDING | VM_FF_PDM_DMA)
2920 || VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_HWACCM_TO_R3_MASK | VMCPU_FF_PGM_SYNC_CR3 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL | VMCPU_FF_REQUEST))
2921 {
2922 /* Check if a sync operation is pending. */
2923 if (VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
2924 {
2925 rc = PGMSyncCR3(pVCpu, pCtx->cr0, pCtx->cr3, pCtx->cr4, VMCPU_FF_ISSET(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
2926 if (rc != VINF_SUCCESS)
2927 {
2928 AssertRC(VBOXSTRICTRC_VAL(rc));
2929 Log(("Pending pool sync is forcing us back to ring 3; rc=%d\n", VBOXSTRICTRC_VAL(rc)));
2930 goto end;
2931 }
2932 }
2933
2934#ifdef DEBUG
2935 /* Intercept X86_XCPT_DB if stepping is enabled */
2936 if (!DBGFIsStepping(pVCpu))
2937#endif
2938 {
2939 if ( VM_FF_ISPENDING(pVM, VM_FF_HWACCM_TO_R3_MASK)
2940 || VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_HWACCM_TO_R3_MASK))
2941 {
2942 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatSwitchToR3);
2943 rc = RT_UNLIKELY(VM_FF_ISPENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
2944 goto end;
2945 }
2946 }
2947
2948 /* Pending request packets might contain actions that need immediate attention, such as pending hardware interrupts. */
2949 if ( VM_FF_ISPENDING(pVM, VM_FF_REQUEST)
2950 || VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_REQUEST))
2951 {
2952 rc = VINF_EM_PENDING_REQUEST;
2953 goto end;
2954 }
2955
2956 /* Check if a pgm pool flush is in progress. */
2957 if (VM_FF_ISPENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
2958 {
2959 rc = VINF_PGM_POOL_FLUSH_PENDING;
2960 goto end;
2961 }
2962
2963 /* Check if DMA work is pending (2nd+ run). */
2964 if (VM_FF_ISPENDING(pVM, VM_FF_PDM_DMA) && cResume > 1)
2965 {
2966 rc = VINF_EM_RAW_TO_R3;
2967 goto end;
2968 }
2969 }
2970
2971#ifdef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
2972 /*
2973 * Exit to ring-3 preemption/work is pending.
2974 *
2975 * Interrupts are disabled before the call to make sure we don't miss any interrupt
2976 * that would flag preemption (IPI, timer tick, ++). (Would've been nice to do this
2977 * further down, but hmR0VmxCheckPendingInterrupt makes that impossible.)
2978 *
2979 * Note! Interrupts must be disabled done *before* we check for TLB flushes; TLB
2980 * shootdowns rely on this.
2981 */
2982 uOldEFlags = ASMIntDisableFlags();
2983 if (RTThreadPreemptIsPending(NIL_RTTHREAD))
2984 {
2985 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitPreemptPending);
2986 rc = VINF_EM_RAW_INTERRUPT;
2987 goto end;
2988 }
2989 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
2990#endif
2991
2992 /*
2993 * When external interrupts are pending, we should exit the VM when IF is et.
2994 * Note: *After* VM_FF_INHIBIT_INTERRUPTS check!
2995 */
2996 rc = hmR0VmxCheckPendingInterrupt(pVM, pVCpu, pCtx);
2997 if (RT_FAILURE(rc))
2998 goto end;
2999
3000 /** @todo check timers?? */
3001
3002 /*
3003 * TPR caching using CR8 is only available in 64-bit mode.
3004 * Note: The 32-bit exception for AMD (X86_CPUID_AMD_FEATURE_ECX_CR8L), but this appears missing in Intel CPUs.
3005 * Note: We can't do this in LoadGuestState() as PDMApicGetTPR can jump back to ring-3 (lock)!! (no longer true) .
3006 */
3007 /** @todo query and update the TPR only when it could have been changed (mmio
3008 * access & wrsmr (x2apic) */
3009 if (fSetupTPRCaching)
3010 {
3011 /* TPR caching in CR8 */
3012 bool fPending;
3013
3014 rc2 = PDMApicGetTPR(pVCpu, &u8LastTPR, &fPending);
3015 AssertRC(rc2);
3016 /* The TPR can be found at offset 0x80 in the APIC mmio page. */
3017 pVCpu->hwaccm.s.vmx.pbVAPIC[0x80] = u8LastTPR;
3018
3019 /*
3020 * Two options here:
3021 * - external interrupt pending, but masked by the TPR value.
3022 * -> a CR8 update that lower the current TPR value should cause an exit
3023 * - no pending interrupts
3024 * -> We don't need to be explicitely notified. There are enough world switches for detecting pending interrupts.
3025 */
3026
3027 /* cr8 bits 3-0 correspond to bits 7-4 of the task priority mmio register. */
3028 rc = VMXWriteVMCS(VMX_VMCS_CTRL_TPR_THRESHOLD, (fPending) ? (u8LastTPR >> 4) : 0);
3029 AssertRC(VBOXSTRICTRC_VAL(rc));
3030
3031 if (pVM->hwaccm.s.fTPRPatchingActive)
3032 {
3033 Assert(!CPUMIsGuestInLongModeEx(pCtx));
3034 /* Our patch code uses LSTAR for TPR caching. */
3035 pCtx->msrLSTAR = u8LastTPR;
3036
3037 /** @todo r=ramshankar: we should check for MSR-bitmap support here. */
3038 if (fPending)
3039 {
3040 /* A TPR change could activate a pending interrupt, so catch lstar writes. */
3041 hmR0VmxSetMSRPermission(pVCpu, MSR_K8_LSTAR, true, false);
3042 }
3043 else
3044 {
3045 /*
3046 * No interrupts are pending, so we don't need to be explicitely notified.
3047 * There are enough world switches for detecting pending interrupts.
3048 */
3049 hmR0VmxSetMSRPermission(pVCpu, MSR_K8_LSTAR, true, true);
3050 }
3051 }
3052 }
3053
3054#ifdef LOG_ENABLED
3055 if ( pVM->hwaccm.s.fNestedPaging
3056 || pVM->hwaccm.s.vmx.fVPID)
3057 {
3058 PHMGLOBLCPUINFO pCpu = HWACCMR0GetCurrentCpu();
3059 if (pVCpu->hwaccm.s.idLastCpu != pCpu->idCpu)
3060 {
3061 LogFlow(("Force TLB flush due to rescheduling to a different cpu (%d vs %d)\n", pVCpu->hwaccm.s.idLastCpu,
3062 pCpu->idCpu));
3063 }
3064 else if (pVCpu->hwaccm.s.cTLBFlushes != pCpu->cTLBFlushes)
3065 {
3066 LogFlow(("Force TLB flush due to changed TLB flush count (%x vs %x)\n", pVCpu->hwaccm.s.cTLBFlushes,
3067 pCpu->cTLBFlushes));
3068 }
3069 else if (VMCPU_FF_ISSET(pVCpu, VMCPU_FF_TLB_FLUSH))
3070 LogFlow(("Manual TLB flush\n"));
3071 }
3072#endif
3073#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
3074 PGMRZDynMapFlushAutoSet(pVCpu);
3075#endif
3076
3077 /*
3078 * NOTE: DO NOT DO ANYTHING AFTER THIS POINT THAT MIGHT JUMP BACK TO RING-3!
3079 * (until the actual world switch)
3080 */
3081#ifdef VBOX_STRICT
3082 idCpuCheck = RTMpCpuId();
3083#endif
3084#ifdef LOG_ENABLED
3085 VMMR0LogFlushDisable(pVCpu);
3086#endif
3087
3088 /*
3089 * Save the host state first.
3090 */
3091 if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_HOST_CONTEXT)
3092 {
3093 rc = VMXR0SaveHostState(pVM, pVCpu);
3094 if (RT_UNLIKELY(rc != VINF_SUCCESS))
3095 {
3096 VMMR0LogFlushEnable(pVCpu);
3097 goto end;
3098 }
3099 }
3100
3101 /*
3102 * Load the guest state.
3103 */
3104 if (!pVCpu->hwaccm.s.fContextUseFlags)
3105 {
3106 VMXR0LoadMinimalGuestState(pVM, pVCpu, pCtx);
3107 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatLoadMinimal);
3108 }
3109 else
3110 {
3111 rc = VMXR0LoadGuestState(pVM, pVCpu, pCtx);
3112 if (RT_UNLIKELY(rc != VINF_SUCCESS))
3113 {
3114 VMMR0LogFlushEnable(pVCpu);
3115 goto end;
3116 }
3117 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatLoadFull);
3118 }
3119
3120#ifndef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
3121 /*
3122 * Disable interrupts to make sure a poke will interrupt execution.
3123 * This must be done *before* we check for TLB flushes; TLB shootdowns rely on this.
3124 */
3125 uOldEFlags = ASMIntDisableFlags();
3126 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
3127#endif
3128
3129 /* Non-register state Guest Context */
3130 /** @todo change me according to cpu state */
3131 rc2 = VMXWriteVMCS(VMX_VMCS32_GUEST_ACTIVITY_STATE, VMX_CMS_GUEST_ACTIVITY_ACTIVE);
3132 AssertRC(rc2);
3133
3134 /* Set TLB flush state as checked until we return from the world switch. */
3135 ASMAtomicWriteBool(&pVCpu->hwaccm.s.fCheckedTLBFlush, true);
3136 /* Deal with tagged TLB setup and invalidation. */
3137 pVM->hwaccm.s.vmx.pfnSetupTaggedTLB(pVM, pVCpu);
3138
3139 /*
3140 * Manual save and restore:
3141 * - General purpose registers except RIP, RSP
3142 *
3143 * Trashed:
3144 * - CR2 (we don't care)
3145 * - LDTR (reset to 0)
3146 * - DRx (presumably not changed at all)
3147 * - DR7 (reset to 0x400)
3148 * - EFLAGS (reset to RT_BIT(1); not relevant)
3149 */
3150
3151 /* All done! Let's start VM execution. */
3152 STAM_PROFILE_ADV_STOP_START(&pVCpu->hwaccm.s.StatEntry, &pVCpu->hwaccm.s.StatInGC, x);
3153 Assert(idCpuCheck == RTMpCpuId());
3154
3155#ifdef VBOX_WITH_CRASHDUMP_MAGIC
3156 pVCpu->hwaccm.s.vmx.VMCSCache.cResume = cResume;
3157 pVCpu->hwaccm.s.vmx.VMCSCache.u64TimeSwitch = RTTimeNanoTS();
3158#endif
3159
3160 /*
3161 * Save the current TPR value in the LSTAR MSR so our patches can access it.
3162 */
3163 if (pVM->hwaccm.s.fTPRPatchingActive)
3164 {
3165 Assert(pVM->hwaccm.s.fTPRPatchingActive);
3166 u64OldLSTAR = ASMRdMsr(MSR_K8_LSTAR);
3167 ASMWrMsr(MSR_K8_LSTAR, u8LastTPR);
3168 }
3169
3170 TMNotifyStartOfExecution(pVCpu);
3171
3172#ifndef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
3173 /*
3174 * Save the current Host TSC_AUX and write the guest TSC_AUX to the host, so that
3175 * RDTSCPs (that don't cause exits) reads the guest MSR. See @bugref{3324}.
3176 */
3177 if ( (pVCpu->hwaccm.s.vmx.proc_ctls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
3178 && !(pVCpu->hwaccm.s.vmx.proc_ctls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT))
3179 {
3180 pVCpu->hwaccm.s.vmx.u64HostTSCAux = ASMRdMsr(MSR_K8_TSC_AUX);
3181 uint64_t u64GuestTSCAux = 0;
3182 rc2 = CPUMQueryGuestMsr(pVCpu, MSR_K8_TSC_AUX, &u64GuestTSCAux);
3183 AssertRC(rc2);
3184 ASMWrMsr(MSR_K8_TSC_AUX, u64GuestTSCAux);
3185 }
3186#endif
3187
3188#ifdef VBOX_WITH_KERNEL_USING_XMM
3189 rc = hwaccmR0VMXStartVMWrapXMM(pVCpu->hwaccm.s.fResumeVM, pCtx, &pVCpu->hwaccm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hwaccm.s.vmx.pfnStartVM);
3190#else
3191 rc = pVCpu->hwaccm.s.vmx.pfnStartVM(pVCpu->hwaccm.s.fResumeVM, pCtx, &pVCpu->hwaccm.s.vmx.VMCSCache, pVM, pVCpu);
3192#endif
3193 ASMAtomicWriteBool(&pVCpu->hwaccm.s.fCheckedTLBFlush, false);
3194 ASMAtomicIncU32(&pVCpu->hwaccm.s.cWorldSwitchExits);
3195
3196#ifndef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
3197 /*
3198 * Restore host's TSC_AUX.
3199 */
3200 if ( (pVCpu->hwaccm.s.vmx.proc_ctls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
3201 && !(pVCpu->hwaccm.s.vmx.proc_ctls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT))
3202 {
3203 ASMWrMsr(MSR_K8_TSC_AUX, pVCpu->hwaccm.s.vmx.u64HostTSCAux);
3204 }
3205#endif
3206
3207 /* Possibly the last TSC value seen by the guest (too high) (only when we're in TSC offset mode). */
3208 if (!(pVCpu->hwaccm.s.vmx.proc_ctls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT))
3209 {
3210 TMCpuTickSetLastSeen(pVCpu,
3211 ASMReadTSC() + pVCpu->hwaccm.s.vmx.u64TSCOffset - 0x400 /* guestimate of world switch overhead in clock ticks */);
3212 }
3213
3214 TMNotifyEndOfExecution(pVCpu);
3215 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED);
3216 Assert(!(ASMGetFlags() & X86_EFL_IF));
3217
3218 /*
3219 * Restore the host LSTAR MSR if the guest could have changed it.
3220 */
3221 if (pVM->hwaccm.s.fTPRPatchingActive)
3222 {
3223 Assert(pVM->hwaccm.s.fTPRPatchingActive);
3224 pVCpu->hwaccm.s.vmx.pbVAPIC[0x80] = pCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR);
3225 ASMWrMsr(MSR_K8_LSTAR, u64OldLSTAR);
3226 }
3227
3228 STAM_PROFILE_ADV_STOP_START(&pVCpu->hwaccm.s.StatInGC, &pVCpu->hwaccm.s.StatExit1, x);
3229 ASMSetFlags(uOldEFlags);
3230#ifdef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
3231 uOldEFlags = ~(RTCCUINTREG)0;
3232#endif
3233
3234 AssertMsg(!pVCpu->hwaccm.s.vmx.VMCSCache.Write.cValidEntries, ("pVCpu->hwaccm.s.vmx.VMCSCache.Write.cValidEntries=%d\n",
3235 pVCpu->hwaccm.s.vmx.VMCSCache.Write.cValidEntries));
3236
3237 /* In case we execute a goto ResumeExecution later on. */
3238 pVCpu->hwaccm.s.fResumeVM = true;
3239 pVCpu->hwaccm.s.fForceTLBFlush = false;
3240
3241 /*
3242 * !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
3243 * IMPORTANT: WE CAN'T DO ANY LOGGING OR OPERATIONS THAT CAN DO A LONGJMP BACK TO RING 3 *BEFORE* WE'VE SYNCED BACK (MOST OF) THE GUEST STATE
3244 * !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
3245 */
3246
3247 if (RT_UNLIKELY(rc != VINF_SUCCESS))
3248 {
3249 hmR0VmxReportWorldSwitchError(pVM, pVCpu, rc, pCtx);
3250 VMMR0LogFlushEnable(pVCpu);
3251 goto end;
3252 }
3253
3254 /* Success. Query the guest state and figure out what has happened. */
3255
3256 /* Investigate why there was a VM-exit. */
3257 rc2 = VMXReadCachedVMCS(VMX_VMCS32_RO_EXIT_REASON, &exitReason);
3258 STAM_COUNTER_INC(&pVCpu->hwaccm.s.paStatExitReasonR0[exitReason & MASK_EXITREASON_STAT]);
3259
3260 exitReason &= 0xffff; /* bit 0-15 contain the exit code. */
3261 rc2 |= VMXReadCachedVMCS(VMX_VMCS32_RO_VM_INSTR_ERROR, &instrError);
3262 rc2 |= VMXReadCachedVMCS(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &cbInstr);
3263 rc2 |= VMXReadCachedVMCS(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &intInfo);
3264 /* might not be valid; depends on VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID. */
3265 rc2 |= VMXReadCachedVMCS(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERRCODE, &errCode);
3266 rc2 |= VMXReadCachedVMCS(VMX_VMCS32_RO_EXIT_INSTR_INFO, &instrInfo);
3267 rc2 |= VMXReadCachedVMCS(VMX_VMCS_RO_EXIT_QUALIFICATION, &exitQualification);
3268 AssertRC(rc2);
3269
3270 /*
3271 * Sync back the guest state.
3272 */
3273 rc2 = VMXR0SaveGuestState(pVM, pVCpu, pCtx);
3274 AssertRC(rc2);
3275
3276 /* Note! NOW IT'S SAFE FOR LOGGING! */
3277 VMMR0LogFlushEnable(pVCpu);
3278 Log2(("Raw exit reason %08x\n", exitReason));
3279#if ARCH_BITS == 64 /* for the time being */
3280 VBOXVMM_R0_HMVMX_VMEXIT(pVCpu, pCtx, exitReason);
3281#endif
3282
3283 /*
3284 * Check if an injected event was interrupted prematurely.
3285 */
3286 rc2 = VMXReadCachedVMCS(VMX_VMCS32_RO_IDT_INFO, &val);
3287 AssertRC(rc2);
3288 pVCpu->hwaccm.s.Event.intInfo = VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(val);
3289 if ( VMX_EXIT_INTERRUPTION_INFO_VALID(pVCpu->hwaccm.s.Event.intInfo)
3290 /* Ignore 'int xx' as they'll be restarted anyway. */
3291 && VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hwaccm.s.Event.intInfo) != VMX_EXIT_INTERRUPTION_INFO_TYPE_SW
3292 /* Ignore software exceptions (such as int3) as they'll reoccur when we restart the instruction anyway. */
3293 && VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hwaccm.s.Event.intInfo) != VMX_EXIT_INTERRUPTION_INFO_TYPE_SWEXCPT)
3294 {
3295 Assert(!pVCpu->hwaccm.s.Event.fPending);
3296 pVCpu->hwaccm.s.Event.fPending = true;
3297 /* Error code present? */
3298 if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(pVCpu->hwaccm.s.Event.intInfo))
3299 {
3300 rc2 = VMXReadCachedVMCS(VMX_VMCS32_RO_IDT_ERRCODE, &val);
3301 AssertRC(rc2);
3302 pVCpu->hwaccm.s.Event.errCode = val;
3303 Log(("Pending inject %RX64 at %RGv exit=%08x intInfo=%08x exitQualification=%RGv pending error=%RX64\n",
3304 pVCpu->hwaccm.s.Event.intInfo, (RTGCPTR)pCtx->rip, exitReason, intInfo, exitQualification, val));
3305 }
3306 else
3307 {
3308 Log(("Pending inject %RX64 at %RGv exit=%08x intInfo=%08x exitQualification=%RGv\n", pVCpu->hwaccm.s.Event.intInfo,
3309 (RTGCPTR)pCtx->rip, exitReason, intInfo, exitQualification));
3310 pVCpu->hwaccm.s.Event.errCode = 0;
3311 }
3312 }
3313#ifdef VBOX_STRICT
3314 else if ( VMX_EXIT_INTERRUPTION_INFO_VALID(pVCpu->hwaccm.s.Event.intInfo)
3315 /* Ignore software exceptions (such as int3) as they're reoccur when we restart the instruction anyway. */
3316 && VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hwaccm.s.Event.intInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_SWEXCPT)
3317 {
3318 Log(("Ignore pending inject %RX64 at %RGv exit=%08x intInfo=%08x exitQualification=%RGv\n",
3319 pVCpu->hwaccm.s.Event.intInfo, (RTGCPTR)pCtx->rip, exitReason, intInfo, exitQualification));
3320 }
3321
3322 if (exitReason == VMX_EXIT_ERR_INVALID_GUEST_STATE)
3323 HWACCMDumpRegs(pVM, pVCpu, pCtx);
3324#endif
3325
3326 Log2(("E%d: New EIP=%x:%RGv\n", (uint32_t)exitReason, pCtx->cs.Sel, (RTGCPTR)pCtx->rip));
3327 Log2(("Exit reason %d, exitQualification %RGv\n", (uint32_t)exitReason, exitQualification));
3328 Log2(("instrInfo=%d instrError=%d instr length=%d\n", (uint32_t)instrInfo, (uint32_t)instrError, (uint32_t)cbInstr));
3329 Log2(("Interruption error code %d\n", (uint32_t)errCode));
3330 Log2(("IntInfo = %08x\n", (uint32_t)intInfo));
3331
3332 /*
3333 * Sync back the TPR if it was changed.
3334 */
3335 if ( fSetupTPRCaching
3336 && u8LastTPR != pVCpu->hwaccm.s.vmx.pbVAPIC[0x80])
3337 {
3338 rc2 = PDMApicSetTPR(pVCpu, pVCpu->hwaccm.s.vmx.pbVAPIC[0x80]);
3339 AssertRC(rc2);
3340 }
3341
3342#ifdef DBGFTRACE_ENABLED /** @todo DTrace later. */
3343 RTTraceBufAddMsgF(pVM->CTX_SUFF(hTraceBuf), "vmexit %08x %016RX64 at %04:%08RX64 %RX64",
3344 exitReason, (uint64_t)exitQualification, pCtx->cs.Sel, pCtx->rip, (uint64_t)intInfo);
3345#endif
3346 STAM_PROFILE_ADV_STOP_START(&pVCpu->hwaccm.s.StatExit1, &pVCpu->hwaccm.s.StatExit2, x);
3347
3348 /* Some cases don't need a complete resync of the guest CPU state; handle them here. */
3349 Assert(rc == VINF_SUCCESS); /* might consider VERR_IPE_UNINITIALIZED_STATUS here later... */
3350 switch (exitReason)
3351 {
3352 case VMX_EXIT_EXCEPTION: /* 0 Exception or non-maskable interrupt (NMI). */
3353 case VMX_EXIT_EXTERNAL_IRQ: /* 1 External interrupt. */
3354 {
3355 uint32_t vector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(intInfo);
3356
3357 if (!VMX_EXIT_INTERRUPTION_INFO_VALID(intInfo))
3358 {
3359 Assert(exitReason == VMX_EXIT_EXTERNAL_IRQ);
3360#if 0 //def VBOX_WITH_VMMR0_DISABLE_PREEMPTION
3361 if ( RTThreadPreemptIsPendingTrusty()
3362 && !RTThreadPreemptIsPending(NIL_RTTHREAD))
3363 goto ResumeExecution;
3364#endif
3365 /* External interrupt; leave to allow it to be dispatched again. */
3366 rc = VINF_EM_RAW_INTERRUPT;
3367 break;
3368 }
3369 STAM_PROFILE_ADV_START(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3370 switch (VMX_EXIT_INTERRUPTION_INFO_TYPE(intInfo))
3371 {
3372 case VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI: /* Non-maskable interrupt. */
3373 /* External interrupt; leave to allow it to be dispatched again. */
3374 rc = VINF_EM_RAW_INTERRUPT;
3375 break;
3376
3377 case VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT: /* External hardware interrupt. */
3378 AssertFailed(); /* can't come here; fails the first check. */
3379 break;
3380
3381 case VMX_EXIT_INTERRUPTION_INFO_TYPE_DBEXCPT: /* Unknown why we get this type for #DB */
3382 case VMX_EXIT_INTERRUPTION_INFO_TYPE_SWEXCPT: /* Software exception. (#BP or #OF) */
3383 Assert(vector == 1 || vector == 3 || vector == 4);
3384 /* no break */
3385 case VMX_EXIT_INTERRUPTION_INFO_TYPE_HWEXCPT: /* Hardware exception. */
3386 Log2(("Hardware/software interrupt %d\n", vector));
3387 switch (vector)
3388 {
3389 case X86_XCPT_NM:
3390 {
3391 Log(("#NM fault at %RGv error code %x\n", (RTGCPTR)pCtx->rip, errCode));
3392
3393 /** @todo don't intercept #NM exceptions anymore when we've activated the guest FPU state. */
3394 /* If we sync the FPU/XMM state on-demand, then we can continue execution as if nothing has happened. */
3395 rc = CPUMR0LoadGuestFPU(pVM, pVCpu, pCtx);
3396 if (rc == VINF_SUCCESS)
3397 {
3398 Assert(CPUMIsGuestFPUStateActive(pVCpu));
3399
3400 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitShadowNM);
3401
3402 /* Continue execution. */
3403 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_GUEST_CR0;
3404
3405 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3406 goto ResumeExecution;
3407 }
3408
3409 Log(("Forward #NM fault to the guest\n"));
3410 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestNM);
3411 rc2 = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo),
3412 cbInstr, 0);
3413 AssertRC(rc2);
3414 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3415 goto ResumeExecution;
3416 }
3417
3418 case X86_XCPT_PF: /* Page fault */
3419 {
3420#ifdef VBOX_ALWAYS_TRAP_PF
3421 if (pVM->hwaccm.s.fNestedPaging)
3422 {
3423 /*
3424 * A genuine pagefault. Forward the trap to the guest by injecting the exception and resuming execution.
3425 */
3426 Log(("Guest page fault at %RGv cr2=%RGv error code %RGv rsp=%RGv\n", (RTGCPTR)pCtx->rip, exitQualification,
3427 errCode, (RTGCPTR)pCtx->rsp));
3428
3429 Assert(CPUMIsGuestInPagedProtectedModeEx(pCtx));
3430
3431 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestPF);
3432
3433 /* Now we must update CR2. */
3434 pCtx->cr2 = exitQualification;
3435 rc2 = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo),
3436 cbInstr, errCode);
3437 AssertRC(rc2);
3438
3439 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3440 goto ResumeExecution;
3441 }
3442#else
3443 Assert(!pVM->hwaccm.s.fNestedPaging);
3444#endif
3445
3446#ifdef VBOX_HWACCM_WITH_GUEST_PATCHING
3447 /* Shortcut for APIC TPR reads and writes; 32 bits guests only */
3448 if ( pVM->hwaccm.s.fTRPPatchingAllowed
3449 && pVM->hwaccm.s.pGuestPatchMem
3450 && (exitQualification & 0xfff) == 0x080
3451 && !(errCode & X86_TRAP_PF_P) /* not present */
3452 && CPUMGetGuestCPL(pVCpu) == 0
3453 && !CPUMIsGuestInLongModeEx(pCtx)
3454 && pVM->hwaccm.s.cPatches < RT_ELEMENTS(pVM->hwaccm.s.aPatches))
3455 {
3456 RTGCPHYS GCPhysApicBase, GCPhys;
3457 PDMApicGetBase(pVM, &GCPhysApicBase); /** @todo cache this */
3458 GCPhysApicBase &= PAGE_BASE_GC_MASK;
3459
3460 rc = PGMGstGetPage(pVCpu, (RTGCPTR)exitQualification, NULL, &GCPhys);
3461 if ( rc == VINF_SUCCESS
3462 && GCPhys == GCPhysApicBase)
3463 {
3464 /* Only attempt to patch the instruction once. */
3465 PHWACCMTPRPATCH pPatch = (PHWACCMTPRPATCH)RTAvloU32Get(&pVM->hwaccm.s.PatchTree, (AVLOU32KEY)pCtx->eip);
3466 if (!pPatch)
3467 {
3468 rc = VINF_EM_HWACCM_PATCH_TPR_INSTR;
3469 break;
3470 }
3471 }
3472 }
3473#endif
3474
3475 Log2(("Page fault at %RGv error code %x\n", exitQualification, errCode));
3476 /* Exit qualification contains the linear address of the page fault. */
3477 TRPMAssertTrap(pVCpu, X86_XCPT_PF, TRPM_TRAP);
3478 TRPMSetErrorCode(pVCpu, errCode);
3479 TRPMSetFaultAddress(pVCpu, exitQualification);
3480
3481 /* Shortcut for APIC TPR reads and writes. */
3482 if ( (exitQualification & 0xfff) == 0x080
3483 && !(errCode & X86_TRAP_PF_P) /* not present */
3484 && fSetupTPRCaching
3485 && (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC))
3486 {
3487 RTGCPHYS GCPhysApicBase, GCPhys;
3488 PDMApicGetBase(pVM, &GCPhysApicBase); /* @todo cache this */
3489 GCPhysApicBase &= PAGE_BASE_GC_MASK;
3490
3491 rc = PGMGstGetPage(pVCpu, (RTGCPTR)exitQualification, NULL, &GCPhys);
3492 if ( rc == VINF_SUCCESS
3493 && GCPhys == GCPhysApicBase)
3494 {
3495 Log(("Enable VT-x virtual APIC access filtering\n"));
3496 rc2 = IOMMMIOMapMMIOHCPage(pVM, GCPhysApicBase, pVM->hwaccm.s.vmx.pAPICPhys, X86_PTE_RW | X86_PTE_P);
3497 AssertRC(rc2);
3498 }
3499 }
3500
3501 /* Forward it to our trap handler first, in case our shadow pages are out of sync. */
3502 rc = PGMTrap0eHandler(pVCpu, errCode, CPUMCTX2CORE(pCtx), (RTGCPTR)exitQualification);
3503 Log2(("PGMTrap0eHandler %RGv returned %Rrc\n", (RTGCPTR)pCtx->rip, VBOXSTRICTRC_VAL(rc)));
3504
3505 if (rc == VINF_SUCCESS)
3506 { /* We've successfully synced our shadow pages, so let's just continue execution. */
3507 Log2(("Shadow page fault at %RGv cr2=%RGv error code %x\n", (RTGCPTR)pCtx->rip, exitQualification ,errCode));
3508 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitShadowPF);
3509
3510 TRPMResetTrap(pVCpu);
3511 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3512 goto ResumeExecution;
3513 }
3514 else if (rc == VINF_EM_RAW_GUEST_TRAP)
3515 {
3516 /*
3517 * A genuine pagefault. Forward the trap to the guest by injecting the exception and resuming execution.
3518 */
3519 Log2(("Forward page fault to the guest\n"));
3520
3521 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestPF);
3522 /* The error code might have been changed. */
3523 errCode = TRPMGetErrorCode(pVCpu);
3524
3525 TRPMResetTrap(pVCpu);
3526
3527 /* Now we must update CR2. */
3528 pCtx->cr2 = exitQualification;
3529 rc2 = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo),
3530 cbInstr, errCode);
3531 AssertRC(rc2);
3532
3533 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3534 goto ResumeExecution;
3535 }
3536#ifdef VBOX_STRICT
3537 if (rc != VINF_EM_RAW_EMULATE_INSTR && rc != VINF_EM_RAW_EMULATE_IO_BLOCK)
3538 Log2(("PGMTrap0eHandler failed with %d\n", VBOXSTRICTRC_VAL(rc)));
3539#endif
3540 /* Need to go back to the recompiler to emulate the instruction. */
3541 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitShadowPFEM);
3542 TRPMResetTrap(pVCpu);
3543 break;
3544 }
3545
3546 case X86_XCPT_MF: /* Floating point exception. */
3547 {
3548 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestMF);
3549 if (!(pCtx->cr0 & X86_CR0_NE))
3550 {
3551 /* old style FPU error reporting needs some extra work. */
3552 /** @todo don't fall back to the recompiler, but do it manually. */
3553 rc = VINF_EM_RAW_EMULATE_INSTR;
3554 break;
3555 }
3556 Log(("Trap %x at %04X:%RGv\n", vector, pCtx->cs.Sel, (RTGCPTR)pCtx->rip));
3557 rc2 = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo),
3558 cbInstr, errCode);
3559 AssertRC(rc2);
3560
3561 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3562 goto ResumeExecution;
3563 }
3564
3565 case X86_XCPT_DB: /* Debug exception. */
3566 {
3567 uint64_t uDR6;
3568
3569 /*
3570 * DR6, DR7.GD and IA32_DEBUGCTL.LBR are not updated yet.
3571 *
3572 * Exit qualification bits:
3573 * 3:0 B0-B3 which breakpoint condition was met
3574 * 12:4 Reserved (0)
3575 * 13 BD - debug register access detected
3576 * 14 BS - single step execution or branch taken
3577 * 63:15 Reserved (0)
3578 */
3579 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestDB);
3580
3581 /* Note that we don't support guest and host-initiated debugging at the same time. */
3582
3583 uDR6 = X86_DR6_INIT_VAL;
3584 uDR6 |= (exitQualification & (X86_DR6_B0|X86_DR6_B1|X86_DR6_B2|X86_DR6_B3|X86_DR6_BD|X86_DR6_BS));
3585 rc = DBGFRZTrap01Handler(pVM, pVCpu, CPUMCTX2CORE(pCtx), uDR6);
3586 if (rc == VINF_EM_RAW_GUEST_TRAP)
3587 {
3588 /* Update DR6 here. */
3589 pCtx->dr[6] = uDR6;
3590
3591 /* Resync DR6 if the debug state is active. */
3592 if (CPUMIsGuestDebugStateActive(pVCpu))
3593 ASMSetDR6(pCtx->dr[6]);
3594
3595 /* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
3596 pCtx->dr[7] &= ~X86_DR7_GD;
3597
3598 /* Paranoia. */
3599 pCtx->dr[7] &= 0xffffffff; /* upper 32 bits reserved */
3600 pCtx->dr[7] &= ~(RT_BIT(11) | RT_BIT(12) | RT_BIT(14) | RT_BIT(15)); /* must be zero */
3601 pCtx->dr[7] |= 0x400; /* must be one */
3602
3603 /* Resync DR7 */
3604 rc2 = VMXWriteVMCS64(VMX_VMCS64_GUEST_DR7, pCtx->dr[7]);
3605 AssertRC(rc2);
3606
3607 Log(("Trap %x (debug) at %RGv exit qualification %RX64 dr6=%x dr7=%x\n", vector, (RTGCPTR)pCtx->rip,
3608 exitQualification, (uint32_t)pCtx->dr[6], (uint32_t)pCtx->dr[7]));
3609 rc2 = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo),
3610 cbInstr, errCode);
3611 AssertRC(rc2);
3612
3613 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3614 goto ResumeExecution;
3615 }
3616 /* Return to ring 3 to deal with the debug exit code. */
3617 Log(("Debugger hardware BP at %04x:%RGv (rc=%Rrc)\n", pCtx->cs.Sel, pCtx->rip, VBOXSTRICTRC_VAL(rc)));
3618 break;
3619 }
3620
3621 case X86_XCPT_BP: /* Breakpoint. */
3622 {
3623 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestBP);
3624 rc = DBGFRZTrap03Handler(pVM, pVCpu, CPUMCTX2CORE(pCtx));
3625 if (rc == VINF_EM_RAW_GUEST_TRAP)
3626 {
3627 Log(("Guest #BP at %04x:%RGv\n", pCtx->cs.Sel, pCtx->rip));
3628 rc2 = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo),
3629 cbInstr, errCode);
3630 AssertRC(rc2);
3631 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3632 goto ResumeExecution;
3633 }
3634 if (rc == VINF_SUCCESS)
3635 {
3636 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3637 goto ResumeExecution;
3638 }
3639 Log(("Debugger BP at %04x:%RGv (rc=%Rrc)\n", pCtx->cs.Sel, pCtx->rip, VBOXSTRICTRC_VAL(rc)));
3640 break;
3641 }
3642
3643 case X86_XCPT_GP: /* General protection failure exception. */
3644 {
3645 uint32_t cbOp;
3646 PDISCPUSTATE pDis = &pVCpu->hwaccm.s.DisState;
3647
3648 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestGP);
3649#ifdef VBOX_STRICT
3650 if ( !CPUMIsGuestInRealModeEx(pCtx)
3651 || !pVM->hwaccm.s.vmx.pRealModeTSS)
3652 {
3653 Log(("Trap %x at %04X:%RGv errorCode=%RGv\n", vector, pCtx->cs.Sel, (RTGCPTR)pCtx->rip, errCode));
3654 rc2 = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo),
3655 cbInstr, errCode);
3656 AssertRC(rc2);
3657 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3658 goto ResumeExecution;
3659 }
3660#endif
3661 Assert(CPUMIsGuestInRealModeEx(pCtx));
3662
3663 LogFlow(("Real mode X86_XCPT_GP instruction emulation at %x:%RGv\n", pCtx->cs.Sel, (RTGCPTR)pCtx->rip));
3664
3665 rc2 = EMInterpretDisasOne(pVM, pVCpu, CPUMCTX2CORE(pCtx), pDis, &cbOp);
3666 if (RT_SUCCESS(rc2))
3667 {
3668 bool fUpdateRIP = true;
3669
3670 rc = VINF_SUCCESS;
3671 Assert(cbOp == pDis->cbInstr);
3672 switch (pDis->pCurInstr->uOpcode)
3673 {
3674 case OP_CLI:
3675 pCtx->eflags.Bits.u1IF = 0;
3676 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitCli);
3677 break;
3678
3679 case OP_STI:
3680 pCtx->eflags.Bits.u1IF = 1;
3681 EMSetInhibitInterruptsPC(pVCpu, pCtx->rip + pDis->cbInstr);
3682 Assert(VMCPU_FF_ISSET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
3683 rc2 = VMXWriteVMCS(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE,
3684 VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
3685 AssertRC(rc2);
3686 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitSti);
3687 break;
3688
3689 case OP_HLT:
3690 fUpdateRIP = false;
3691 rc = VINF_EM_HALT;
3692 pCtx->rip += pDis->cbInstr;
3693 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitHlt);
3694 break;
3695
3696 case OP_POPF:
3697 {
3698 RTGCPTR GCPtrStack;
3699 uint32_t cbParm;
3700 uint32_t uMask;
3701 X86EFLAGS eflags;
3702
3703 if (pDis->fPrefix & DISPREFIX_OPSIZE)
3704 {
3705 cbParm = 4;
3706 uMask = 0xffffffff;
3707 }
3708 else
3709 {
3710 cbParm = 2;
3711 uMask = 0xffff;
3712 }
3713
3714 rc2 = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pCtx), pCtx->esp & uMask, 0, &GCPtrStack);
3715 if (RT_FAILURE(rc2))
3716 {
3717 rc = VERR_EM_INTERPRETER;
3718 break;
3719 }
3720 eflags.u = 0;
3721 rc2 = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &eflags.u, cbParm);
3722 if (RT_FAILURE(rc2))
3723 {
3724 rc = VERR_EM_INTERPRETER;
3725 break;
3726 }
3727 LogFlow(("POPF %x -> %RGv mask=%x\n", eflags.u, pCtx->rsp, uMask));
3728 pCtx->eflags.u = (pCtx->eflags.u & ~(X86_EFL_POPF_BITS & uMask)) | (eflags.u & X86_EFL_POPF_BITS & uMask);
3729 /* RF cleared when popped in real mode; see pushf description in AMD manual. */
3730 pCtx->eflags.Bits.u1RF = 0;
3731 pCtx->esp += cbParm;
3732 pCtx->esp &= uMask;
3733
3734 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitPopf);
3735 break;
3736 }
3737
3738 case OP_PUSHF:
3739 {
3740 RTGCPTR GCPtrStack;
3741 uint32_t cbParm;
3742 uint32_t uMask;
3743 X86EFLAGS eflags;
3744
3745 if (pDis->fPrefix & DISPREFIX_OPSIZE)
3746 {
3747 cbParm = 4;
3748 uMask = 0xffffffff;
3749 }
3750 else
3751 {
3752 cbParm = 2;
3753 uMask = 0xffff;
3754 }
3755
3756 rc2 = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pCtx), (pCtx->esp - cbParm) & uMask, 0,
3757 &GCPtrStack);
3758 if (RT_FAILURE(rc2))
3759 {
3760 rc = VERR_EM_INTERPRETER;
3761 break;
3762 }
3763 eflags = pCtx->eflags;
3764 /* RF & VM cleared when pushed in real mode; see pushf description in AMD manual. */
3765 eflags.Bits.u1RF = 0;
3766 eflags.Bits.u1VM = 0;
3767
3768 rc2 = PGMPhysWrite(pVM, (RTGCPHYS)GCPtrStack, &eflags.u, cbParm);
3769 if (RT_FAILURE(rc2))
3770 {
3771 rc = VERR_EM_INTERPRETER;
3772 break;
3773 }
3774 LogFlow(("PUSHF %x -> %RGv\n", eflags.u, GCPtrStack));
3775 pCtx->esp -= cbParm;
3776 pCtx->esp &= uMask;
3777 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitPushf);
3778 break;
3779 }
3780
3781 case OP_IRET:
3782 {
3783 RTGCPTR GCPtrStack;
3784 uint32_t uMask = 0xffff;
3785 uint16_t aIretFrame[3];
3786
3787 if (pDis->fPrefix & (DISPREFIX_OPSIZE | DISPREFIX_ADDRSIZE))
3788 {
3789 rc = VERR_EM_INTERPRETER;
3790 break;
3791 }
3792
3793 rc2 = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pCtx), pCtx->esp & uMask, 0, &GCPtrStack);
3794 if (RT_FAILURE(rc2))
3795 {
3796 rc = VERR_EM_INTERPRETER;
3797 break;
3798 }
3799 rc2 = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &aIretFrame[0], sizeof(aIretFrame));
3800 if (RT_FAILURE(rc2))
3801 {
3802 rc = VERR_EM_INTERPRETER;
3803 break;
3804 }
3805 pCtx->ip = aIretFrame[0];
3806 pCtx->cs.Sel = aIretFrame[1];
3807 pCtx->cs.ValidSel = aIretFrame[1];
3808 pCtx->cs.u64Base = (uint32_t)pCtx->cs.Sel << 4;
3809 pCtx->eflags.u = (pCtx->eflags.u & ~(X86_EFL_POPF_BITS & uMask))
3810 | (aIretFrame[2] & X86_EFL_POPF_BITS & uMask);
3811 pCtx->sp += sizeof(aIretFrame);
3812
3813 LogFlow(("iret to %04x:%x\n", pCtx->cs.Sel, pCtx->ip));
3814 fUpdateRIP = false;
3815 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitIret);
3816 break;
3817 }
3818
3819 case OP_INT:
3820 {
3821 uint32_t intInfo2;
3822
3823 LogFlow(("Realmode: INT %x\n", pDis->Param1.uValue & 0xff));
3824 intInfo2 = pDis->Param1.uValue & 0xff;
3825 intInfo2 |= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
3826 intInfo2 |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
3827
3828 rc = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, intInfo2, cbOp, 0);
3829 AssertRC(VBOXSTRICTRC_VAL(rc));
3830 fUpdateRIP = false;
3831 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitInt);
3832 break;
3833 }
3834
3835 case OP_INTO:
3836 {
3837 if (pCtx->eflags.Bits.u1OF)
3838 {
3839 uint32_t intInfo2;
3840
3841 LogFlow(("Realmode: INTO\n"));
3842 intInfo2 = X86_XCPT_OF;
3843 intInfo2 |= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
3844 intInfo2 |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
3845
3846 rc = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, intInfo2, cbOp, 0);
3847 AssertRC(VBOXSTRICTRC_VAL(rc));
3848 fUpdateRIP = false;
3849 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitInt);
3850 }
3851 break;
3852 }
3853
3854 case OP_INT3:
3855 {
3856 uint32_t intInfo2;
3857
3858 LogFlow(("Realmode: INT 3\n"));
3859 intInfo2 = 3;
3860 intInfo2 |= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
3861 intInfo2 |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
3862
3863 rc = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, intInfo2, cbOp, 0);
3864 AssertRC(VBOXSTRICTRC_VAL(rc));
3865 fUpdateRIP = false;
3866 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitInt);
3867 break;
3868 }
3869
3870 default:
3871 rc = EMInterpretInstructionDisasState(pVCpu, pDis, CPUMCTX2CORE(pCtx), 0, EMCODETYPE_SUPERVISOR);
3872 fUpdateRIP = false;
3873 break;
3874 }
3875
3876 if (rc == VINF_SUCCESS)
3877 {
3878 if (fUpdateRIP)
3879 pCtx->rip += cbOp; /* Move on to the next instruction. */
3880
3881 /*
3882 * LIDT, LGDT can end up here. In the future CRx changes as well. Just reload the
3883 * whole context to be done with it.
3884 */
3885 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_ALL;
3886
3887 /* Only resume if successful. */
3888 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3889 goto ResumeExecution;
3890 }
3891 }
3892 else
3893 rc = VERR_EM_INTERPRETER;
3894
3895 AssertMsg(rc == VERR_EM_INTERPRETER || rc == VINF_PGM_CHANGE_MODE || rc == VINF_EM_HALT,
3896 ("Unexpected rc=%Rrc\n", VBOXSTRICTRC_VAL(rc)));
3897 break;
3898 }
3899
3900#ifdef VBOX_STRICT
3901 case X86_XCPT_XF: /* SIMD exception. */
3902 case X86_XCPT_DE: /* Divide error. */
3903 case X86_XCPT_UD: /* Unknown opcode exception. */
3904 case X86_XCPT_SS: /* Stack segment exception. */
3905 case X86_XCPT_NP: /* Segment not present exception. */
3906 {
3907 switch (vector)
3908 {
3909 case X86_XCPT_DE:
3910 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestDE);
3911 break;
3912 case X86_XCPT_UD:
3913 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestUD);
3914 break;
3915 case X86_XCPT_SS:
3916 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestSS);
3917 break;
3918 case X86_XCPT_NP:
3919 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestNP);
3920 break;
3921 case X86_XCPT_XF:
3922 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestXF);
3923 break;
3924 }
3925
3926 Log(("Trap %x at %04X:%RGv\n", vector, pCtx->cs.Sel, (RTGCPTR)pCtx->rip));
3927 rc2 = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo),
3928 cbInstr, errCode);
3929 AssertRC(rc2);
3930
3931 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3932 goto ResumeExecution;
3933 }
3934#endif
3935 default:
3936 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestXcpUnk);
3937 if ( CPUMIsGuestInRealModeEx(pCtx)
3938 && pVM->hwaccm.s.vmx.pRealModeTSS)
3939 {
3940 Log(("Real Mode Trap %x at %04x:%04X error code %x\n", vector, pCtx->cs.Sel, pCtx->eip, errCode));
3941 rc = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo),
3942 cbInstr, errCode);
3943 AssertRC(VBOXSTRICTRC_VAL(rc)); /* Strict RC check below. */
3944
3945 /* Go back to ring-3 in case of a triple fault. */
3946 if ( vector == X86_XCPT_DF
3947 && rc == VINF_EM_RESET)
3948 {
3949 break;
3950 }
3951
3952 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3953 goto ResumeExecution;
3954 }
3955 AssertMsgFailed(("Unexpected vm-exit caused by exception %x\n", vector));
3956 rc = VERR_VMX_UNEXPECTED_EXCEPTION;
3957 break;
3958 } /* switch (vector) */
3959
3960 break;
3961
3962 default:
3963 rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_CODE;
3964 AssertMsgFailed(("Unexpected interruption code %x\n", intInfo));
3965 break;
3966 }
3967
3968 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3969 break;
3970 }
3971
3972 /*
3973 * 48 EPT violation. An attempt to access memory with a guest-physical address was disallowed
3974 * by the configuration of the EPT paging structures.
3975 */
3976 case VMX_EXIT_EPT_VIOLATION:
3977 {
3978 RTGCPHYS GCPhys;
3979
3980 Assert(pVM->hwaccm.s.fNestedPaging);
3981
3982 rc2 = VMXReadVMCS64(VMX_VMCS_EXIT_PHYS_ADDR_FULL, &GCPhys);
3983 AssertRC(rc2);
3984 Assert(((exitQualification >> 7) & 3) != 2);
3985
3986 /* Determine the kind of violation. */
3987 errCode = 0;
3988 if (exitQualification & VMX_EXIT_QUALIFICATION_EPT_INSTR_FETCH)
3989 errCode |= X86_TRAP_PF_ID;
3990
3991 if (exitQualification & VMX_EXIT_QUALIFICATION_EPT_DATA_WRITE)
3992 errCode |= X86_TRAP_PF_RW;
3993
3994 /* If the page is present, then it's a page level protection fault. */
3995 if (exitQualification & VMX_EXIT_QUALIFICATION_EPT_ENTRY_PRESENT)
3996 errCode |= X86_TRAP_PF_P;
3997 else
3998 {
3999 /* Shortcut for APIC TPR reads and writes. */
4000 if ( (GCPhys & 0xfff) == 0x080
4001 && GCPhys > 0x1000000 /* to skip VGA frame buffer accesses */
4002 && fSetupTPRCaching
4003 && (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC))
4004 {
4005 RTGCPHYS GCPhysApicBase;
4006 PDMApicGetBase(pVM, &GCPhysApicBase); /* @todo cache this */
4007 GCPhysApicBase &= PAGE_BASE_GC_MASK;
4008 if (GCPhys == GCPhysApicBase + 0x80)
4009 {
4010 Log(("Enable VT-x virtual APIC access filtering\n"));
4011 rc2 = IOMMMIOMapMMIOHCPage(pVM, GCPhysApicBase, pVM->hwaccm.s.vmx.pAPICPhys, X86_PTE_RW | X86_PTE_P);
4012 AssertRC(rc2);
4013 }
4014 }
4015 }
4016 Log(("EPT Page fault %x at %RGp error code %x\n", (uint32_t)exitQualification, GCPhys, errCode));
4017
4018 /* GCPhys contains the guest physical address of the page fault. */
4019 TRPMAssertTrap(pVCpu, X86_XCPT_PF, TRPM_TRAP);
4020 TRPMSetErrorCode(pVCpu, errCode);
4021 TRPMSetFaultAddress(pVCpu, GCPhys);
4022
4023 /* Handle the pagefault trap for the nested shadow table. */
4024 rc = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, errCode, CPUMCTX2CORE(pCtx), GCPhys);
4025
4026 /*
4027 * Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment below, @bugref{6043}.
4028 */
4029 if ( rc == VINF_SUCCESS
4030 || rc == VERR_PAGE_TABLE_NOT_PRESENT
4031 || rc == VERR_PAGE_NOT_PRESENT)
4032 {
4033 /* We've successfully synced our shadow pages, so let's just continue execution. */
4034 Log2(("Shadow page fault at %RGv cr2=%RGp error code %x\n", (RTGCPTR)pCtx->rip, exitQualification , errCode));
4035 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitReasonNPF);
4036
4037 TRPMResetTrap(pVCpu);
4038 goto ResumeExecution;
4039 }
4040
4041#ifdef VBOX_STRICT
4042 if (rc != VINF_EM_RAW_EMULATE_INSTR)
4043 LogFlow(("PGMTrap0eHandlerNestedPaging at %RGv failed with %Rrc\n", (RTGCPTR)pCtx->rip, VBOXSTRICTRC_VAL(rc)));
4044#endif
4045 /* Need to go back to the recompiler to emulate the instruction. */
4046 TRPMResetTrap(pVCpu);
4047 break;
4048 }
4049
4050 case VMX_EXIT_EPT_MISCONFIG:
4051 {
4052 RTGCPHYS GCPhys;
4053
4054 Assert(pVM->hwaccm.s.fNestedPaging);
4055
4056 rc2 = VMXReadVMCS64(VMX_VMCS_EXIT_PHYS_ADDR_FULL, &GCPhys);
4057 AssertRC(rc2);
4058 Log(("VMX_EXIT_EPT_MISCONFIG for %RGp\n", GCPhys));
4059
4060 /* Shortcut for APIC TPR reads and writes. */
4061 if ( (GCPhys & 0xfff) == 0x080
4062 && GCPhys > 0x1000000 /* to skip VGA frame buffer accesses */
4063 && fSetupTPRCaching
4064 && (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC))
4065 {
4066 RTGCPHYS GCPhysApicBase;
4067 PDMApicGetBase(pVM, &GCPhysApicBase); /* @todo cache this */
4068 GCPhysApicBase &= PAGE_BASE_GC_MASK;
4069 if (GCPhys == GCPhysApicBase + 0x80)
4070 {
4071 Log(("Enable VT-x virtual APIC access filtering\n"));
4072 rc2 = IOMMMIOMapMMIOHCPage(pVM, GCPhysApicBase, pVM->hwaccm.s.vmx.pAPICPhys, X86_PTE_RW | X86_PTE_P);
4073 AssertRC(rc2);
4074 }
4075 }
4076
4077 rc = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pCtx), GCPhys, UINT32_MAX);
4078
4079 /*
4080 * If we succeed, resume execution.
4081 * Or, if fail in interpreting the instruction because we couldn't get the guest physical address
4082 * of the page containing the instruction via the guest's page tables (we would invalidate the guest page
4083 * in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
4084 * weird case. See @bugref{6043}.
4085 */
4086 if ( rc == VINF_SUCCESS
4087 || rc == VERR_PAGE_TABLE_NOT_PRESENT
4088 || rc == VERR_PAGE_NOT_PRESENT)
4089 {
4090 Log2(("PGMR0Trap0eHandlerNPMisconfig(,,,%RGp) at %RGv -> resume\n", GCPhys, (RTGCPTR)pCtx->rip));
4091 goto ResumeExecution;
4092 }
4093
4094 Log2(("PGMR0Trap0eHandlerNPMisconfig(,,,%RGp) at %RGv -> %Rrc\n", GCPhys, (RTGCPTR)pCtx->rip, VBOXSTRICTRC_VAL(rc)));
4095 break;
4096 }
4097
4098 case VMX_EXIT_IRQ_WINDOW: /* 7 Interrupt window. */
4099 /* Clear VM-exit on IF=1 change. */
4100 LogFlow(("VMX_EXIT_IRQ_WINDOW %RGv pending=%d IF=%d\n", (RTGCPTR)pCtx->rip,
4101 VMCPU_FF_ISPENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC|VMCPU_FF_INTERRUPT_PIC)), pCtx->eflags.Bits.u1IF));
4102 pVCpu->hwaccm.s.vmx.proc_ctls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_IRQ_WINDOW_EXIT;
4103 rc2 = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
4104 AssertRC(rc2);
4105 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitIrqWindow);
4106 goto ResumeExecution; /* we check for pending guest interrupts there */
4107
4108 case VMX_EXIT_WBINVD: /* 54 Guest software attempted to execute WBINVD. (conditional) */
4109 case VMX_EXIT_INVD: /* 13 Guest software attempted to execute INVD. (unconditional) */
4110 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitInvd);
4111 /* Skip instruction and continue directly. */
4112 pCtx->rip += cbInstr;
4113 /* Continue execution.*/
4114 goto ResumeExecution;
4115
4116 case VMX_EXIT_CPUID: /* 10 Guest software attempted to execute CPUID. */
4117 {
4118 Log2(("VMX: Cpuid %x\n", pCtx->eax));
4119 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitCpuid);
4120 rc = EMInterpretCpuId(pVM, pVCpu, CPUMCTX2CORE(pCtx));
4121 if (rc == VINF_SUCCESS)
4122 {
4123 /* Update EIP and continue execution. */
4124 Assert(cbInstr == 2);
4125 pCtx->rip += cbInstr;
4126 goto ResumeExecution;
4127 }
4128 AssertMsgFailed(("EMU: cpuid failed with %Rrc\n", VBOXSTRICTRC_VAL(rc)));
4129 rc = VINF_EM_RAW_EMULATE_INSTR;
4130 break;
4131 }
4132
4133 case VMX_EXIT_RDPMC: /* 15 Guest software attempted to execute RDPMC. */
4134 {
4135 Log2(("VMX: Rdpmc %x\n", pCtx->ecx));
4136 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitRdpmc);
4137 rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pCtx));
4138 if (rc == VINF_SUCCESS)
4139 {
4140 /* Update EIP and continue execution. */
4141 Assert(cbInstr == 2);
4142 pCtx->rip += cbInstr;
4143 goto ResumeExecution;
4144 }
4145 rc = VINF_EM_RAW_EMULATE_INSTR;
4146 break;
4147 }
4148
4149 case VMX_EXIT_RDTSC: /* 16 Guest software attempted to execute RDTSC. */
4150 {
4151 Log2(("VMX: Rdtsc\n"));
4152 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitRdtsc);
4153 rc = EMInterpretRdtsc(pVM, pVCpu, CPUMCTX2CORE(pCtx));
4154 if (rc == VINF_SUCCESS)
4155 {
4156 /* Update EIP and continue execution. */
4157 Assert(cbInstr == 2);
4158 pCtx->rip += cbInstr;
4159 goto ResumeExecution;
4160 }
4161 rc = VINF_EM_RAW_EMULATE_INSTR;
4162 break;
4163 }
4164
4165 case VMX_EXIT_RDTSCP: /* 51 Guest software attempted to execute RDTSCP. */
4166 {
4167 Log2(("VMX: Rdtscp\n"));
4168 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitRdtscp);
4169 rc = EMInterpretRdtscp(pVM, pVCpu, pCtx);
4170 if (rc == VINF_SUCCESS)
4171 {
4172 /* Update EIP and continue execution. */
4173 Assert(cbInstr == 3);
4174 pCtx->rip += cbInstr;
4175 goto ResumeExecution;
4176 }
4177 rc = VINF_EM_RAW_EMULATE_INSTR;
4178 break;
4179 }
4180
4181 case VMX_EXIT_INVPG: /* 14 Guest software attempted to execute INVPG. */
4182 {
4183 Log2(("VMX: invlpg\n"));
4184 Assert(!pVM->hwaccm.s.fNestedPaging);
4185
4186 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitInvpg);
4187 rc = EMInterpretInvlpg(pVM, pVCpu, CPUMCTX2CORE(pCtx), exitQualification);
4188 if (rc == VINF_SUCCESS)
4189 {
4190 /* Update EIP and continue execution. */
4191 pCtx->rip += cbInstr;
4192 goto ResumeExecution;
4193 }
4194 AssertMsg(rc == VERR_EM_INTERPRETER, ("EMU: invlpg %RGv failed with %Rrc\n", exitQualification, VBOXSTRICTRC_VAL(rc)));
4195 break;
4196 }
4197
4198 case VMX_EXIT_MONITOR: /* 39 Guest software attempted to execute MONITOR. */
4199 {
4200 Log2(("VMX: monitor\n"));
4201
4202 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitMonitor);
4203 rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pCtx));
4204 if (rc == VINF_SUCCESS)
4205 {
4206 /* Update EIP and continue execution. */
4207 pCtx->rip += cbInstr;
4208 goto ResumeExecution;
4209 }
4210 AssertMsg(rc == VERR_EM_INTERPRETER, ("EMU: monitor failed with %Rrc\n", VBOXSTRICTRC_VAL(rc)));
4211 break;
4212 }
4213
4214 case VMX_EXIT_WRMSR: /* 32 WRMSR. Guest software attempted to execute WRMSR. */
4215 /* When an interrupt is pending, we'll let MSR_K8_LSTAR writes fault in our TPR patch code. */
4216 if ( pVM->hwaccm.s.fTPRPatchingActive
4217 && pCtx->ecx == MSR_K8_LSTAR)
4218 {
4219 Assert(!CPUMIsGuestInLongModeEx(pCtx));
4220 if ((pCtx->eax & 0xff) != u8LastTPR)
4221 {
4222 Log(("VMX: Faulting MSR_K8_LSTAR write with new TPR value %x\n", pCtx->eax & 0xff));
4223
4224 /* Our patch code uses LSTAR for TPR caching. */
4225 rc2 = PDMApicSetTPR(pVCpu, pCtx->eax & 0xff);
4226 AssertRC(rc2);
4227 }
4228
4229 /* Skip the instruction and continue. */
4230 pCtx->rip += cbInstr; /* wrmsr = [0F 30] */
4231
4232 /* Only resume if successful. */
4233 goto ResumeExecution;
4234 }
4235 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_GUEST_MSR;
4236 /* no break */
4237 case VMX_EXIT_RDMSR: /* 31 RDMSR. Guest software attempted to execute RDMSR. */
4238 {
4239 STAM_COUNTER_INC((exitReason == VMX_EXIT_RDMSR) ? &pVCpu->hwaccm.s.StatExitRdmsr : &pVCpu->hwaccm.s.StatExitWrmsr);
4240
4241 /*
4242 * Note: The Intel spec. claims there's an REX version of RDMSR that's slightly different,
4243 * so we play safe by completely disassembling the instruction.
4244 */
4245 Log2(("VMX: %s\n", (exitReason == VMX_EXIT_RDMSR) ? "rdmsr" : "wrmsr"));
4246 rc = EMInterpretInstruction(pVCpu, CPUMCTX2CORE(pCtx), 0);
4247 if (rc == VINF_SUCCESS)
4248 {
4249 /* EIP has been updated already. */
4250 /* Only resume if successful. */
4251 goto ResumeExecution;
4252 }
4253 AssertMsg(rc == VERR_EM_INTERPRETER, ("EMU: %s failed with %Rrc\n",
4254 (exitReason == VMX_EXIT_RDMSR) ? "rdmsr" : "wrmsr", VBOXSTRICTRC_VAL(rc)));
4255 break;
4256 }
4257
4258 case VMX_EXIT_CRX_MOVE: /* 28 Control-register accesses. */
4259 {
4260 STAM_PROFILE_ADV_START(&pVCpu->hwaccm.s.StatExit2Sub2, y2);
4261
4262 switch (VMX_EXIT_QUALIFICATION_CRX_ACCESS(exitQualification))
4263 {
4264 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_WRITE:
4265 {
4266 Log2(("VMX: %RGv mov cr%d, x\n", (RTGCPTR)pCtx->rip, VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification)));
4267 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitCRxWrite[VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification)]);
4268 rc = EMInterpretCRxWrite(pVM, pVCpu, CPUMCTX2CORE(pCtx),
4269 VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification),
4270 VMX_EXIT_QUALIFICATION_CRX_GENREG(exitQualification));
4271 switch (VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification))
4272 {
4273 case 0:
4274 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_GUEST_CR0 | HWACCM_CHANGED_GUEST_CR3;
4275 break;
4276 case 2:
4277 break;
4278 case 3:
4279 Assert(!pVM->hwaccm.s.fNestedPaging || !CPUMIsGuestInPagedProtectedModeEx(pCtx));
4280 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_GUEST_CR3;
4281 break;
4282 case 4:
4283 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_GUEST_CR4;
4284 break;
4285 case 8:
4286 /* CR8 contains the APIC TPR */
4287 Assert(!(pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1
4288 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW));
4289 break;
4290
4291 default:
4292 AssertFailed();
4293 break;
4294 }
4295 break;
4296 }
4297
4298 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ:
4299 {
4300 Log2(("VMX: mov x, crx\n"));
4301 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitCRxRead[VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification)]);
4302
4303 Assert( !pVM->hwaccm.s.fNestedPaging
4304 || !CPUMIsGuestInPagedProtectedModeEx(pCtx)
4305 || VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification) != DISCREG_CR3);
4306
4307 /* CR8 reads only cause an exit when the TPR shadow feature isn't present. */
4308 Assert( VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification) != 8
4309 || !(pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW));
4310
4311 rc = EMInterpretCRxRead(pVM, pVCpu, CPUMCTX2CORE(pCtx),
4312 VMX_EXIT_QUALIFICATION_CRX_GENREG(exitQualification),
4313 VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification));
4314 break;
4315 }
4316
4317 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_CLTS:
4318 {
4319 Log2(("VMX: clts\n"));
4320 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitCLTS);
4321 rc = EMInterpretCLTS(pVM, pVCpu);
4322 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_GUEST_CR0;
4323 break;
4324 }
4325
4326 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_LMSW:
4327 {
4328 Log2(("VMX: lmsw %x\n", VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(exitQualification)));
4329 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitLMSW);
4330 rc = EMInterpretLMSW(pVM, pVCpu, CPUMCTX2CORE(pCtx), VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(exitQualification));
4331 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_GUEST_CR0;
4332 break;
4333 }
4334 }
4335
4336 /* Update EIP if no error occurred. */
4337 if (RT_SUCCESS(rc))
4338 pCtx->rip += cbInstr;
4339
4340 if (rc == VINF_SUCCESS)
4341 {
4342 /* Only resume if successful. */
4343 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub2, y2);
4344 goto ResumeExecution;
4345 }
4346 Assert(rc == VERR_EM_INTERPRETER || rc == VINF_PGM_CHANGE_MODE || rc == VINF_PGM_SYNC_CR3);
4347 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub2, y2);
4348 break;
4349 }
4350
4351 case VMX_EXIT_DRX_MOVE: /* 29 Debug-register accesses. */
4352 {
4353 if ( !DBGFIsStepping(pVCpu)
4354 && !CPUMIsHyperDebugStateActive(pVCpu))
4355 {
4356 /* Disable DRx move intercepts. */
4357 pVCpu->hwaccm.s.vmx.proc_ctls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT;
4358 rc2 = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
4359 AssertRC(rc2);
4360
4361 /* Save the host and load the guest debug state. */
4362 rc2 = CPUMR0LoadGuestDebugState(pVM, pVCpu, pCtx, true /* include DR6 */);
4363 AssertRC(rc2);
4364
4365#ifdef LOG_ENABLED
4366 if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(exitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
4367 {
4368 Log(("VMX_EXIT_DRX_MOVE: write DR%d genreg %d\n", VMX_EXIT_QUALIFICATION_DRX_REGISTER(exitQualification),
4369 VMX_EXIT_QUALIFICATION_DRX_GENREG(exitQualification)));
4370 }
4371 else
4372 Log(("VMX_EXIT_DRX_MOVE: read DR%d\n", VMX_EXIT_QUALIFICATION_DRX_REGISTER(exitQualification)));
4373#endif
4374
4375#ifdef VBOX_WITH_STATISTICS
4376 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatDRxContextSwitch);
4377 if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(exitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
4378 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitDRxWrite);
4379 else
4380 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitDRxRead);
4381#endif
4382
4383 goto ResumeExecution;
4384 }
4385
4386 /** @todo clear VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT after the first
4387 * time and restore DRx registers afterwards */
4388 if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(exitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
4389 {
4390 Log2(("VMX: mov DRx%d, genreg%d\n", VMX_EXIT_QUALIFICATION_DRX_REGISTER(exitQualification),
4391 VMX_EXIT_QUALIFICATION_DRX_GENREG(exitQualification)));
4392 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitDRxWrite);
4393 rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pCtx),
4394 VMX_EXIT_QUALIFICATION_DRX_REGISTER(exitQualification),
4395 VMX_EXIT_QUALIFICATION_DRX_GENREG(exitQualification));
4396 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_GUEST_DEBUG;
4397 Log2(("DR7=%08x\n", pCtx->dr[7]));
4398 }
4399 else
4400 {
4401 Log2(("VMX: mov x, DRx\n"));
4402 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitDRxRead);
4403 rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pCtx),
4404 VMX_EXIT_QUALIFICATION_DRX_GENREG(exitQualification),
4405 VMX_EXIT_QUALIFICATION_DRX_REGISTER(exitQualification));
4406 }
4407 /* Update EIP if no error occurred. */
4408 if (RT_SUCCESS(rc))
4409 pCtx->rip += cbInstr;
4410
4411 if (rc == VINF_SUCCESS)
4412 {
4413 /* Only resume if successful. */
4414 goto ResumeExecution;
4415 }
4416 Assert(rc == VERR_EM_INTERPRETER);
4417 break;
4418 }
4419
4420 /* Note: We'll get a #GP if the IO instruction isn't allowed (IOPL or TSS bitmap); no need to double check. */
4421 case VMX_EXIT_PORT_IO: /* 30 I/O instruction. */
4422 {
4423 STAM_PROFILE_ADV_START(&pVCpu->hwaccm.s.StatExit2Sub1, y1);
4424 uint32_t uPort;
4425 uint32_t uIOWidth = VMX_EXIT_QUALIFICATION_IO_WIDTH(exitQualification);
4426 bool fIOWrite = (VMX_EXIT_QUALIFICATION_IO_DIRECTION(exitQualification) == VMX_EXIT_QUALIFICATION_IO_DIRECTION_OUT);
4427
4428 /** @todo necessary to make the distinction? */
4429 if (VMX_EXIT_QUALIFICATION_IO_ENCODING(exitQualification) == VMX_EXIT_QUALIFICATION_IO_ENCODING_DX)
4430 uPort = pCtx->edx & 0xffff;
4431 else
4432 uPort = VMX_EXIT_QUALIFICATION_IO_PORT(exitQualification); /* Immediate encoding. */
4433
4434 if (RT_UNLIKELY(uIOWidth == 2 || uIOWidth >= 4)) /* paranoia */
4435 {
4436 rc = fIOWrite ? VINF_IOM_R3_IOPORT_WRITE : VINF_IOM_R3_IOPORT_READ;
4437 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub1, y1);
4438 break;
4439 }
4440
4441 uint32_t cbSize = g_aIOSize[uIOWidth];
4442 if (VMX_EXIT_QUALIFICATION_IO_STRING(exitQualification))
4443 {
4444 /* ins/outs */
4445 PDISCPUSTATE pDis = &pVCpu->hwaccm.s.DisState;
4446
4447 /* Disassemble manually to deal with segment prefixes. */
4448 /** @todo VMX_VMCS_EXIT_GUEST_LINEAR_ADDR contains the flat pointer operand of the instruction. */
4449 /** @todo VMX_VMCS32_RO_EXIT_INSTR_INFO also contains segment prefix info. */
4450 rc2 = EMInterpretDisasOne(pVM, pVCpu, CPUMCTX2CORE(pCtx), pDis, NULL);
4451 if (RT_SUCCESS(rc))
4452 {
4453 if (fIOWrite)
4454 {
4455 Log2(("IOMInterpretOUTSEx %RGv %x size=%d\n", (RTGCPTR)pCtx->rip, uPort, cbSize));
4456 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitIOStringWrite);
4457 rc = IOMInterpretOUTSEx(pVM, CPUMCTX2CORE(pCtx), uPort, pDis->fPrefix, (DISCPUMODE)pDis->uAddrMode, cbSize);
4458 }
4459 else
4460 {
4461 Log2(("IOMInterpretINSEx %RGv %x size=%d\n", (RTGCPTR)pCtx->rip, uPort, cbSize));
4462 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitIOStringRead);
4463 rc = IOMInterpretINSEx(pVM, CPUMCTX2CORE(pCtx), uPort, pDis->fPrefix, (DISCPUMODE)pDis->uAddrMode, cbSize);
4464 }
4465 }
4466 else
4467 rc = VINF_EM_RAW_EMULATE_INSTR;
4468 }
4469 else
4470 {
4471 /* Normal in/out */
4472 uint32_t uAndVal = g_aIOOpAnd[uIOWidth];
4473
4474 Assert(!VMX_EXIT_QUALIFICATION_IO_REP(exitQualification));
4475
4476 if (fIOWrite)
4477 {
4478 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitIOWrite);
4479 rc = IOMIOPortWrite(pVM, uPort, pCtx->eax & uAndVal, cbSize);
4480 if (rc == VINF_IOM_R3_IOPORT_WRITE)
4481 HWACCMR0SavePendingIOPortWrite(pVCpu, pCtx->rip, pCtx->rip + cbInstr, uPort, uAndVal, cbSize);
4482 }
4483 else
4484 {
4485 uint32_t u32Val = 0;
4486
4487 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitIORead);
4488 rc = IOMIOPortRead(pVM, uPort, &u32Val, cbSize);
4489 if (IOM_SUCCESS(rc))
4490 {
4491 /* Write back to the EAX register. */
4492 pCtx->eax = (pCtx->eax & ~uAndVal) | (u32Val & uAndVal);
4493 }
4494 else
4495 if (rc == VINF_IOM_R3_IOPORT_READ)
4496 HWACCMR0SavePendingIOPortRead(pVCpu, pCtx->rip, pCtx->rip + cbInstr, uPort, uAndVal, cbSize);
4497 }
4498 }
4499
4500 /*
4501 * Handled the I/O return codes.
4502 * (The unhandled cases end up with rc == VINF_EM_RAW_EMULATE_INSTR.)
4503 */
4504 if (IOM_SUCCESS(rc))
4505 {
4506 /* Update EIP and continue execution. */
4507 pCtx->rip += cbInstr;
4508 if (RT_LIKELY(rc == VINF_SUCCESS))
4509 {
4510 /* If any IO breakpoints are armed, then we should check if a debug trap needs to be generated. */
4511 if (pCtx->dr[7] & X86_DR7_ENABLED_MASK)
4512 {
4513 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatDRxIOCheck);
4514 for (unsigned i = 0; i < 4; i++)
4515 {
4516 unsigned uBPLen = g_aIOSize[X86_DR7_GET_LEN(pCtx->dr[7], i)];
4517
4518 if ( (uPort >= pCtx->dr[i] && uPort < pCtx->dr[i] + uBPLen)
4519 && (pCtx->dr[7] & (X86_DR7_L(i) | X86_DR7_G(i)))
4520 && (pCtx->dr[7] & X86_DR7_RW(i, X86_DR7_RW_IO)) == X86_DR7_RW(i, X86_DR7_RW_IO))
4521 {
4522 uint64_t uDR6;
4523
4524 Assert(CPUMIsGuestDebugStateActive(pVCpu));
4525
4526 uDR6 = ASMGetDR6();
4527
4528 /* Clear all breakpoint status flags and set the one we just hit. */
4529 uDR6 &= ~(X86_DR6_B0|X86_DR6_B1|X86_DR6_B2|X86_DR6_B3);
4530 uDR6 |= (uint64_t)RT_BIT(i);
4531
4532 /*
4533 * Note: AMD64 Architecture Programmer's Manual 13.1:
4534 * Bits 15:13 of the DR6 register is never cleared by the processor and must
4535 * be cleared by software after the contents have been read.
4536 */
4537 ASMSetDR6(uDR6);
4538
4539 /* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
4540 pCtx->dr[7] &= ~X86_DR7_GD;
4541
4542 /* Paranoia. */
4543 pCtx->dr[7] &= 0xffffffff; /* upper 32 bits reserved */
4544 pCtx->dr[7] &= ~(RT_BIT(11) | RT_BIT(12) | RT_BIT(14) | RT_BIT(15)); /* must be zero */
4545 pCtx->dr[7] |= 0x400; /* must be one */
4546
4547 /* Resync DR7 */
4548 rc2 = VMXWriteVMCS64(VMX_VMCS64_GUEST_DR7, pCtx->dr[7]);
4549 AssertRC(rc2);
4550
4551 /* Construct inject info. */
4552 intInfo = X86_XCPT_DB;
4553 intInfo |= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
4554 intInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HWEXCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
4555
4556 Log(("Inject IO debug trap at %RGv\n", (RTGCPTR)pCtx->rip));
4557 rc2 = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo),
4558 0 /* cbInstr */, 0 /* errCode */);
4559 AssertRC(rc2);
4560
4561 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub1, y1);
4562 goto ResumeExecution;
4563 }
4564 }
4565 }
4566 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub1, y1);
4567 goto ResumeExecution;
4568 }
4569 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub1, y1);
4570 break;
4571 }
4572
4573#ifdef VBOX_STRICT
4574 if (rc == VINF_IOM_R3_IOPORT_READ)
4575 Assert(!fIOWrite);
4576 else if (rc == VINF_IOM_R3_IOPORT_WRITE)
4577 Assert(fIOWrite);
4578 else
4579 {
4580 AssertMsg( RT_FAILURE(rc)
4581 || rc == VINF_EM_RAW_EMULATE_INSTR
4582 || rc == VINF_EM_RAW_GUEST_TRAP
4583 || rc == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", VBOXSTRICTRC_VAL(rc)));
4584 }
4585#endif
4586 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub1, y1);
4587 break;
4588 }
4589
4590 case VMX_EXIT_TPR: /* 43 TPR below threshold. Guest software executed MOV to CR8. */
4591 LogFlow(("VMX_EXIT_TPR\n"));
4592 /* RIP is already set to the next instruction and the TPR has been synced back. Just resume. */
4593 goto ResumeExecution;
4594
4595 case VMX_EXIT_APIC_ACCESS: /* 44 APIC access. Guest software attempted to access memory at a physical address
4596 on the APIC-access page. */
4597 {
4598 LogFlow(("VMX_EXIT_APIC_ACCESS\n"));
4599 unsigned uAccessType = VMX_EXIT_QUALIFICATION_APIC_ACCESS_TYPE(exitQualification);
4600
4601 switch(uAccessType)
4602 {
4603 case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
4604 case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
4605 {
4606 RTGCPHYS GCPhys;
4607 PDMApicGetBase(pVM, &GCPhys);
4608 GCPhys &= PAGE_BASE_GC_MASK;
4609 GCPhys += VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(exitQualification);
4610
4611 LogFlow(("Apic access at %RGp\n", GCPhys));
4612 rc = IOMMMIOPhysHandler(pVM, (uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ) ? 0 : X86_TRAP_PF_RW,
4613 CPUMCTX2CORE(pCtx), GCPhys);
4614 if (rc == VINF_SUCCESS)
4615 goto ResumeExecution; /* rip already updated */
4616 break;
4617 }
4618
4619 default:
4620 rc = VINF_EM_RAW_EMULATE_INSTR;
4621 break;
4622 }
4623 break;
4624 }
4625
4626 case VMX_EXIT_PREEMPTION_TIMER: /* 52 VMX-preemption timer expired. The preemption timer counted down to zero. */
4627 if (!TMTimerPollBool(pVM, pVCpu))
4628 goto ResumeExecution;
4629 rc = VINF_EM_RAW_TIMER_PENDING;
4630 break;
4631
4632 default:
4633 /* The rest is handled after syncing the entire CPU state. */
4634 break;
4635 }
4636
4637
4638 /*
4639 * Note: The guest state is not entirely synced back at this stage!
4640 */
4641
4642 /* Investigate why there was a VM-exit. (part 2) */
4643 switch (exitReason)
4644 {
4645 case VMX_EXIT_EXCEPTION: /* 0 Exception or non-maskable interrupt (NMI). */
4646 case VMX_EXIT_EXTERNAL_IRQ: /* 1 External interrupt. */
4647 case VMX_EXIT_EPT_VIOLATION:
4648 case VMX_EXIT_EPT_MISCONFIG: /* 49 EPT misconfig is used by the PGM/MMIO optimizations. */
4649 case VMX_EXIT_PREEMPTION_TIMER: /* 52 VMX-preemption timer expired. The preemption timer counted down to zero. */
4650 /* Already handled above. */
4651 break;
4652
4653 case VMX_EXIT_TRIPLE_FAULT: /* 2 Triple fault. */
4654 rc = VINF_EM_RESET; /* Triple fault equals a reset. */
4655 break;
4656
4657 case VMX_EXIT_INIT_SIGNAL: /* 3 INIT signal. */
4658 case VMX_EXIT_SIPI: /* 4 Start-up IPI (SIPI). */
4659 rc = VINF_EM_RAW_INTERRUPT;
4660 AssertFailed(); /* Can't happen. Yet. */
4661 break;
4662
4663 case VMX_EXIT_IO_SMI_IRQ: /* 5 I/O system-management interrupt (SMI). */
4664 case VMX_EXIT_SMI_IRQ: /* 6 Other SMI. */
4665 rc = VINF_EM_RAW_INTERRUPT;
4666 AssertFailed(); /* Can't happen afaik. */
4667 break;
4668
4669 case VMX_EXIT_TASK_SWITCH: /* 9 Task switch: too complicated to emulate, so fall back to the recompiler */
4670 Log(("VMX_EXIT_TASK_SWITCH: exit=%RX64\n", exitQualification));
4671 if ( (VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE(exitQualification) == VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE_IDT)
4672 && pVCpu->hwaccm.s.Event.fPending)
4673 {
4674 /* Caused by an injected interrupt. */
4675 pVCpu->hwaccm.s.Event.fPending = false;
4676
4677 Log(("VMX_EXIT_TASK_SWITCH: reassert trap %d\n", VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVCpu->hwaccm.s.Event.intInfo)));
4678 Assert(!VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(pVCpu->hwaccm.s.Event.intInfo));
4679 rc2 = TRPMAssertTrap(pVCpu, VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVCpu->hwaccm.s.Event.intInfo), TRPM_HARDWARE_INT);
4680 AssertRC(rc2);
4681 }
4682 /* else Exceptions and software interrupts can just be restarted. */
4683 rc = VERR_EM_INTERPRETER;
4684 break;
4685
4686 case VMX_EXIT_HLT: /* 12 Guest software attempted to execute HLT. */
4687 /* Check if external interrupts are pending; if so, don't switch back. */
4688 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitHlt);
4689 pCtx->rip++; /* skip hlt */
4690 if (EMShouldContinueAfterHalt(pVCpu, pCtx))
4691 goto ResumeExecution;
4692
4693 rc = VINF_EM_HALT;
4694 break;
4695
4696 case VMX_EXIT_MWAIT: /* 36 Guest software executed MWAIT. */
4697 Log2(("VMX: mwait\n"));
4698 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitMwait);
4699 rc = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pCtx));
4700 if ( rc == VINF_EM_HALT
4701 || rc == VINF_SUCCESS)
4702 {
4703 /* Update EIP and continue execution. */
4704 pCtx->rip += cbInstr;
4705
4706 /* Check if external interrupts are pending; if so, don't switch back. */
4707 if ( rc == VINF_SUCCESS
4708 || ( rc == VINF_EM_HALT
4709 && EMShouldContinueAfterHalt(pVCpu, pCtx))
4710 )
4711 goto ResumeExecution;
4712 }
4713 AssertMsg(rc == VERR_EM_INTERPRETER || rc == VINF_EM_HALT, ("EMU: mwait failed with %Rrc\n", VBOXSTRICTRC_VAL(rc)));
4714 break;
4715
4716 case VMX_EXIT_RSM: /* 17 Guest software attempted to execute RSM in SMM. */
4717 AssertFailed(); /* can't happen. */
4718 rc = VERR_EM_INTERPRETER;
4719 break;
4720
4721 case VMX_EXIT_MTF: /* 37 Exit due to Monitor Trap Flag. */
4722 LogFlow(("VMX_EXIT_MTF at %RGv\n", (RTGCPTR)pCtx->rip));
4723 pVCpu->hwaccm.s.vmx.proc_ctls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_TRAP_FLAG;
4724 rc2 = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
4725 AssertRC(rc2);
4726 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitMTF);
4727#if 0
4728 DBGFDoneStepping(pVCpu);
4729#endif
4730 rc = VINF_EM_DBG_STOP;
4731 break;
4732
4733 case VMX_EXIT_VMCALL: /* 18 Guest software executed VMCALL. */
4734 case VMX_EXIT_VMCLEAR: /* 19 Guest software executed VMCLEAR. */
4735 case VMX_EXIT_VMLAUNCH: /* 20 Guest software executed VMLAUNCH. */
4736 case VMX_EXIT_VMPTRLD: /* 21 Guest software executed VMPTRLD. */
4737 case VMX_EXIT_VMPTRST: /* 22 Guest software executed VMPTRST. */
4738 case VMX_EXIT_VMREAD: /* 23 Guest software executed VMREAD. */
4739 case VMX_EXIT_VMRESUME: /* 24 Guest software executed VMRESUME. */
4740 case VMX_EXIT_VMWRITE: /* 25 Guest software executed VMWRITE. */
4741 case VMX_EXIT_VMXOFF: /* 26 Guest software executed VMXOFF. */
4742 case VMX_EXIT_VMXON: /* 27 Guest software executed VMXON. */
4743 /** @todo inject #UD immediately */
4744 rc = VERR_EM_INTERPRETER;
4745 break;
4746
4747 case VMX_EXIT_CPUID: /* 10 Guest software attempted to execute CPUID. */
4748 case VMX_EXIT_RDTSC: /* 16 Guest software attempted to execute RDTSC. */
4749 case VMX_EXIT_INVPG: /* 14 Guest software attempted to execute INVPG. */
4750 case VMX_EXIT_CRX_MOVE: /* 28 Control-register accesses. */
4751 case VMX_EXIT_DRX_MOVE: /* 29 Debug-register accesses. */
4752 case VMX_EXIT_PORT_IO: /* 30 I/O instruction. */
4753 case VMX_EXIT_RDPMC: /* 15 Guest software attempted to execute RDPMC. */
4754 case VMX_EXIT_RDTSCP: /* 51 Guest software attempted to execute RDTSCP. */
4755 /* already handled above */
4756 AssertMsg( rc == VINF_PGM_CHANGE_MODE
4757 || rc == VINF_EM_RAW_INTERRUPT
4758 || rc == VERR_EM_INTERPRETER
4759 || rc == VINF_EM_RAW_EMULATE_INSTR
4760 || rc == VINF_PGM_SYNC_CR3
4761 || rc == VINF_IOM_R3_IOPORT_READ
4762 || rc == VINF_IOM_R3_IOPORT_WRITE
4763 || rc == VINF_EM_RAW_GUEST_TRAP
4764 || rc == VINF_TRPM_XCPT_DISPATCHED
4765 || rc == VINF_EM_RESCHEDULE_REM,
4766 ("rc = %d\n", VBOXSTRICTRC_VAL(rc)));
4767 break;
4768
4769 case VMX_EXIT_TPR: /* 43 TPR below threshold. Guest software executed MOV to CR8. */
4770 case VMX_EXIT_RDMSR: /* 31 RDMSR. Guest software attempted to execute RDMSR. */
4771 case VMX_EXIT_WRMSR: /* 32 WRMSR. Guest software attempted to execute WRMSR. */
4772 case VMX_EXIT_PAUSE: /* 40 Guest software attempted to execute PAUSE. */
4773 case VMX_EXIT_MONITOR: /* 39 Guest software attempted to execute MONITOR. */
4774 case VMX_EXIT_APIC_ACCESS: /* 44 APIC access. Guest software attempted to access memory at a physical address
4775 on the APIC-access page. */
4776 {
4777 /*
4778 * If we decided to emulate them here, then we must sync the MSRs that could have been changed (sysenter, FS/GS base)
4779 */
4780 rc = VERR_EM_INTERPRETER;
4781 break;
4782 }
4783
4784 case VMX_EXIT_IRQ_WINDOW: /* 7 Interrupt window. */
4785 Assert(rc == VINF_EM_RAW_INTERRUPT);
4786 break;
4787
4788 case VMX_EXIT_ERR_INVALID_GUEST_STATE: /* 33 VM-entry failure due to invalid guest state. */
4789 {
4790#ifdef VBOX_STRICT
4791 RTCCUINTREG val2 = 0;
4792
4793 Log(("VMX_EXIT_ERR_INVALID_GUEST_STATE\n"));
4794
4795 VMXReadVMCS(VMX_VMCS64_GUEST_RIP, &val2);
4796 Log(("Old eip %RGv new %RGv\n", (RTGCPTR)pCtx->rip, (RTGCPTR)val2));
4797
4798 VMXReadVMCS(VMX_VMCS64_GUEST_CR0, &val2);
4799 Log(("VMX_VMCS_GUEST_CR0 %RX64\n", (uint64_t)val2));
4800
4801 VMXReadVMCS(VMX_VMCS64_GUEST_CR3, &val2);
4802 Log(("VMX_VMCS_GUEST_CR3 %RX64\n", (uint64_t)val2));
4803
4804 VMXReadVMCS(VMX_VMCS64_GUEST_CR4, &val2);
4805 Log(("VMX_VMCS_GUEST_CR4 %RX64\n", (uint64_t)val2));
4806
4807 VMXReadVMCS(VMX_VMCS_GUEST_RFLAGS, &val2);
4808 Log(("VMX_VMCS_GUEST_RFLAGS %08x\n", val2));
4809
4810 VMX_LOG_SELREG(CS, "CS", val2);
4811 VMX_LOG_SELREG(DS, "DS", val2);
4812 VMX_LOG_SELREG(ES, "ES", val2);
4813 VMX_LOG_SELREG(FS, "FS", val2);
4814 VMX_LOG_SELREG(GS, "GS", val2);
4815 VMX_LOG_SELREG(SS, "SS", val2);
4816 VMX_LOG_SELREG(TR, "TR", val2);
4817 VMX_LOG_SELREG(LDTR, "LDTR", val2);
4818
4819 VMXReadVMCS(VMX_VMCS64_GUEST_GDTR_BASE, &val2);
4820 Log(("VMX_VMCS_GUEST_GDTR_BASE %RX64\n", (uint64_t)val2));
4821 VMXReadVMCS(VMX_VMCS64_GUEST_IDTR_BASE, &val2);
4822 Log(("VMX_VMCS_GUEST_IDTR_BASE %RX64\n", (uint64_t)val2));
4823#endif /* VBOX_STRICT */
4824 rc = VERR_VMX_INVALID_GUEST_STATE;
4825 break;
4826 }
4827
4828 case VMX_EXIT_ERR_MSR_LOAD: /* 34 VM-entry failure due to MSR loading. */
4829 case VMX_EXIT_ERR_MACHINE_CHECK: /* 41 VM-entry failure due to machine-check. */
4830 default:
4831 rc = VERR_VMX_UNEXPECTED_EXIT_CODE;
4832 AssertMsgFailed(("Unexpected exit code %d\n", exitReason)); /* Can't happen. */
4833 break;
4834
4835 }
4836
4837end:
4838 /* We now going back to ring-3, so clear the action flag. */
4839 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
4840
4841 /*
4842 * Signal changes for the recompiler.
4843 */
4844 CPUMSetChangedFlags(pVCpu,
4845 CPUM_CHANGED_SYSENTER_MSR
4846 | CPUM_CHANGED_LDTR
4847 | CPUM_CHANGED_GDTR
4848 | CPUM_CHANGED_IDTR
4849 | CPUM_CHANGED_TR
4850 | CPUM_CHANGED_HIDDEN_SEL_REGS);
4851
4852 /*
4853 * If we executed vmlaunch/vmresume and an external IRQ was pending, then we don't have to do a full sync the next time.
4854 */
4855 if ( exitReason == VMX_EXIT_EXTERNAL_IRQ
4856 && !VMX_EXIT_INTERRUPTION_INFO_VALID(intInfo))
4857 {
4858 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatPendingHostIrq);
4859 /* On the next entry we'll only sync the host context. */
4860 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_HOST_CONTEXT;
4861 }
4862 else
4863 {
4864 /* On the next entry we'll sync everything. */
4865 /** @todo we can do better than this */
4866 /* Not in the VINF_PGM_CHANGE_MODE though! */
4867 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_ALL;
4868 }
4869
4870 /* Translate into a less severe return code */
4871 if (rc == VERR_EM_INTERPRETER)
4872 rc = VINF_EM_RAW_EMULATE_INSTR;
4873 else if (rc == VERR_VMX_INVALID_VMCS_PTR)
4874 {
4875 /* Try to extract more information about what might have gone wrong here. */
4876 VMXGetActivateVMCS(&pVCpu->hwaccm.s.vmx.lasterror.u64VMCSPhys);
4877 pVCpu->hwaccm.s.vmx.lasterror.ulVMCSRevision = *(uint32_t *)pVCpu->hwaccm.s.vmx.pvVMCS;
4878 pVCpu->hwaccm.s.vmx.lasterror.idEnteredCpu = pVCpu->hwaccm.s.idEnteredCpu;
4879 pVCpu->hwaccm.s.vmx.lasterror.idCurrentCpu = RTMpCpuId();
4880 }
4881
4882 /* Just set the correct state here instead of trying to catch every goto above. */
4883 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED, VMCPUSTATE_STARTED_EXEC);
4884
4885#ifdef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
4886 /* Restore interrupts if we exited after disabling them. */
4887 if (uOldEFlags != ~(RTCCUINTREG)0)
4888 ASMSetFlags(uOldEFlags);
4889#endif
4890
4891 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2, x);
4892 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit1, x);
4893 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatEntry, x);
4894 Log2(("X"));
4895 return VBOXSTRICTRC_TODO(rc);
4896}
4897
4898
4899/**
4900 * Enters the VT-x session.
4901 *
4902 * @returns VBox status code.
4903 * @param pVM Pointer to the VM.
4904 * @param pVCpu Pointer to the VMCPU.
4905 * @param pCpu Pointer to the CPU info struct.
4906 */
4907VMMR0DECL(int) VMXR0Enter(PVM pVM, PVMCPU pVCpu, PHMGLOBLCPUINFO pCpu)
4908{
4909 Assert(pVM->hwaccm.s.vmx.fSupported);
4910 NOREF(pCpu);
4911
4912 unsigned cr4 = ASMGetCR4();
4913 if (!(cr4 & X86_CR4_VMXE))
4914 {
4915 AssertMsgFailed(("X86_CR4_VMXE should be set!\n"));
4916 return VERR_VMX_X86_CR4_VMXE_CLEARED;
4917 }
4918
4919 /* Activate the VMCS. */
4920 int rc = VMXActivateVMCS(pVCpu->hwaccm.s.vmx.HCPhysVMCS);
4921 if (RT_FAILURE(rc))
4922 return rc;
4923
4924 pVCpu->hwaccm.s.fResumeVM = false;
4925 return VINF_SUCCESS;
4926}
4927
4928
4929/**
4930 * Leaves the VT-x session.
4931 *
4932 * @returns VBox status code.
4933 * @param pVM Pointer to the VM.
4934 * @param pVCpu Pointer to the VMCPU.
4935 * @param pCtx Pointer to the guests CPU context.
4936 */
4937VMMR0DECL(int) VMXR0Leave(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
4938{
4939 Assert(pVM->hwaccm.s.vmx.fSupported);
4940
4941#ifdef DEBUG
4942 if (CPUMIsHyperDebugStateActive(pVCpu))
4943 {
4944 CPUMR0LoadHostDebugState(pVM, pVCpu);
4945 Assert(pVCpu->hwaccm.s.vmx.proc_ctls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT);
4946 }
4947 else
4948#endif
4949
4950 /*
4951 * Save the guest debug state if necessary.
4952 */
4953 if (CPUMIsGuestDebugStateActive(pVCpu))
4954 {
4955 CPUMR0SaveGuestDebugState(pVM, pVCpu, pCtx, true /* save DR6 */);
4956
4957 /* Enable DRx move intercepts again. */
4958 pVCpu->hwaccm.s.vmx.proc_ctls |= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT;
4959 int rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
4960 AssertRC(rc);
4961
4962 /* Resync the debug registers the next time. */
4963 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_GUEST_DEBUG;
4964 }
4965 else
4966 Assert(pVCpu->hwaccm.s.vmx.proc_ctls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT);
4967
4968 /*
4969 * Clear VMCS, marking it inactive, clearing implementation-specific data and writing
4970 * VMCS data back to memory.
4971 */
4972 int rc = VMXClearVMCS(pVCpu->hwaccm.s.vmx.HCPhysVMCS);
4973 AssertRC(rc);
4974
4975 return VINF_SUCCESS;
4976}
4977
4978
4979/**
4980 * Flush the TLB using EPT.
4981 *
4982 * @returns VBox status code.
4983 * @param pVM Pointer to the VM.
4984 * @param pVCpu Pointer to the VMCPU.
4985 * @param enmFlush Type of flush.
4986 */
4987static void hmR0VmxFlushEPT(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_EPT enmFlush)
4988{
4989 uint64_t descriptor[2];
4990
4991 LogFlow(("hmR0VmxFlushEPT %d\n", enmFlush));
4992 Assert(pVM->hwaccm.s.fNestedPaging);
4993 descriptor[0] = pVCpu->hwaccm.s.vmx.GCPhysEPTP;
4994 descriptor[1] = 0; /* MBZ. Intel spec. 33.3 VMX Instructions */
4995 int rc = VMXR0InvEPT(enmFlush, &descriptor[0]);
4996 AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %x %RGv failed with %d\n", enmFlush, pVCpu->hwaccm.s.vmx.GCPhysEPTP, rc));
4997}
4998
4999
5000/**
5001 * Flush the TLB using VPID.
5002 *
5003 * @returns VBox status code.
5004 * @param pVM Pointer to the VM.
5005 * @param pVCpu Pointer to the VMCPU (can be NULL depending on @a
5006 * enmFlush).
5007 * @param enmFlush Type of flush.
5008 * @param GCPtr Virtual address of the page to flush (can be 0 depending
5009 * on @a enmFlush).
5010 */
5011static void hmR0VmxFlushVPID(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_VPID enmFlush, RTGCPTR GCPtr)
5012{
5013#if HC_ARCH_BITS == 32
5014 /*
5015 * If we get a flush in 64-bit guest mode, then force a full TLB flush. invvpid probably takes only 32-bit addresses.
5016 */
5017 if ( CPUMIsGuestInLongMode(pVCpu)
5018 && !VMX_IS_64BIT_HOST_MODE())
5019 {
5020 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
5021 }
5022 else
5023#endif
5024 {
5025 uint64_t descriptor[2];
5026
5027 Assert(pVM->hwaccm.s.vmx.fVPID);
5028 if (enmFlush == VMX_FLUSH_VPID_ALL_CONTEXTS)
5029 {
5030 descriptor[0] = 0;
5031 descriptor[1] = 0;
5032 }
5033 else
5034 {
5035 AssertPtr(pVCpu);
5036 Assert(pVCpu->hwaccm.s.uCurrentASID != 0);
5037 descriptor[0] = pVCpu->hwaccm.s.uCurrentASID;
5038 descriptor[1] = GCPtr;
5039 }
5040 int rc = VMXR0InvVPID(enmFlush, &descriptor[0]); NOREF(rc);
5041 AssertMsg(rc == VINF_SUCCESS,
5042 ("VMXR0InvVPID %x %x %RGv failed with %d\n", enmFlush, pVCpu ? pVCpu->hwaccm.s.uCurrentASID : 0, GCPtr, rc));
5043 }
5044}
5045
5046
5047/**
5048 * Invalidates a guest page by guest virtual address. Only relevant for
5049 * EPT/VPID, otherwise there is nothing really to invalidate.
5050 *
5051 * @returns VBox status code.
5052 * @param pVM Pointer to the VM.
5053 * @param pVCpu Pointer to the VMCPU.
5054 * @param GCVirt Guest virtual address of the page to invalidate.
5055 */
5056VMMR0DECL(int) VMXR0InvalidatePage(PVM pVM, PVMCPU pVCpu, RTGCPTR GCVirt)
5057{
5058 bool fFlushPending = VMCPU_FF_ISSET(pVCpu, VMCPU_FF_TLB_FLUSH);
5059
5060 Log2(("VMXR0InvalidatePage %RGv\n", GCVirt));
5061
5062 if (!fFlushPending)
5063 {
5064 /*
5065 * We must invalidate the guest TLB entry in either case, we cannot ignore it even for the EPT case
5066 * See @bugref{6043} and @bugref{6177}
5067 *
5068 * Set the VMCPU_FF_TLB_FLUSH force flag and flush before VMENTRY in hmR0VmxSetupTLB*() as this
5069 * function maybe called in a loop with individual addresses.
5070 */
5071 if (pVM->hwaccm.s.vmx.fVPID)
5072 {
5073 /* If we can flush just this page do it, otherwise flush as little as possible. */
5074 if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVVPID_CAPS_INDIV_ADDR)
5075 hmR0VmxFlushVPID(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, GCVirt);
5076 else
5077 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
5078 }
5079 else if (pVM->hwaccm.s.fNestedPaging)
5080 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
5081 }
5082
5083 return VINF_SUCCESS;
5084}
5085
5086
5087/**
5088 * Invalidates a guest page by physical address. Only relevant for EPT/VPID,
5089 * otherwise there is nothing really to invalidate.
5090 *
5091 * NOTE: Assumes the current instruction references this physical page though a virtual address!!
5092 *
5093 * @returns VBox status code.
5094 * @param pVM Pointer to the VM.
5095 * @param pVCpu Pointer to the VMCPU.
5096 * @param GCPhys Guest physical address of the page to invalidate.
5097 */
5098VMMR0DECL(int) VMXR0InvalidatePhysPage(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys)
5099{
5100 LogFlow(("VMXR0InvalidatePhysPage %RGp\n", GCPhys));
5101
5102 /*
5103 * We cannot flush a page by guest-physical address. invvpid takes only a linear address
5104 * while invept only flushes by EPT not individual addresses. We update the force flag here
5105 * and flush before VMENTRY in hmR0VmxSetupTLB*(). This function might be called in a loop.
5106 */
5107 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
5108 return VINF_SUCCESS;
5109}
5110
5111
5112/**
5113 * Report world switch error and dump some useful debug info.
5114 *
5115 * @param pVM Pointer to the VM.
5116 * @param pVCpu Pointer to the VMCPU.
5117 * @param rc Return code.
5118 * @param pCtx Pointer to the current guest CPU context (not updated).
5119 */
5120static void hmR0VmxReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, VBOXSTRICTRC rc, PCPUMCTX pCtx)
5121{
5122 NOREF(pVM);
5123
5124 switch (VBOXSTRICTRC_VAL(rc))
5125 {
5126 case VERR_VMX_INVALID_VMXON_PTR:
5127 AssertFailed();
5128 break;
5129
5130 case VERR_VMX_UNABLE_TO_START_VM:
5131 case VERR_VMX_UNABLE_TO_RESUME_VM:
5132 {
5133 int rc2;
5134 RTCCUINTREG exitReason, instrError;
5135
5136 rc2 = VMXReadVMCS(VMX_VMCS32_RO_EXIT_REASON, &exitReason);
5137 rc2 |= VMXReadVMCS(VMX_VMCS32_RO_VM_INSTR_ERROR, &instrError);
5138 AssertRC(rc2);
5139 if (rc2 == VINF_SUCCESS)
5140 {
5141 Log(("Unable to start/resume VM for reason: %x. Instruction error %x\n", (uint32_t)exitReason,
5142 (uint32_t)instrError));
5143 Log(("Current stack %08x\n", &rc2));
5144
5145 pVCpu->hwaccm.s.vmx.lasterror.ulInstrError = instrError;
5146 pVCpu->hwaccm.s.vmx.lasterror.ulExitReason = exitReason;
5147
5148#ifdef VBOX_STRICT
5149 RTGDTR gdtr;
5150 PCX86DESCHC pDesc;
5151 RTCCUINTREG val;
5152
5153 ASMGetGDTR(&gdtr);
5154
5155 VMXReadVMCS(VMX_VMCS64_GUEST_RIP, &val);
5156 Log(("Old eip %RGv new %RGv\n", (RTGCPTR)pCtx->rip, (RTGCPTR)val));
5157 VMXReadVMCS(VMX_VMCS_CTRL_PIN_EXEC_CONTROLS, &val);
5158 Log(("VMX_VMCS_CTRL_PIN_EXEC_CONTROLS %08x\n", val));
5159 VMXReadVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, &val);
5160 Log(("VMX_VMCS_CTRL_PROC_EXEC_CONTROLS %08x\n", val));
5161 VMXReadVMCS(VMX_VMCS_CTRL_ENTRY_CONTROLS, &val);
5162 Log(("VMX_VMCS_CTRL_ENTRY_CONTROLS %08x\n", val));
5163 VMXReadVMCS(VMX_VMCS_CTRL_EXIT_CONTROLS, &val);
5164 Log(("VMX_VMCS_CTRL_EXIT_CONTROLS %08x\n", val));
5165
5166 VMXReadVMCS(VMX_VMCS_HOST_CR0, &val);
5167 Log(("VMX_VMCS_HOST_CR0 %08x\n", val));
5168 VMXReadVMCS(VMX_VMCS_HOST_CR3, &val);
5169 Log(("VMX_VMCS_HOST_CR3 %08x\n", val));
5170 VMXReadVMCS(VMX_VMCS_HOST_CR4, &val);
5171 Log(("VMX_VMCS_HOST_CR4 %08x\n", val));
5172
5173 VMXReadVMCS(VMX_VMCS16_HOST_FIELD_CS, &val);
5174 Log(("VMX_VMCS_HOST_FIELD_CS %08x\n", val));
5175 VMXReadVMCS(VMX_VMCS_GUEST_RFLAGS, &val);
5176 Log(("VMX_VMCS_GUEST_RFLAGS %08x\n", val));
5177
5178 if (val < gdtr.cbGdt)
5179 {
5180 pDesc = (PCX86DESCHC)(gdtr.pGdt + (val & X86_SEL_MASK));
5181 HWACCMR0DumpDescriptor(pDesc, val, "CS: ");
5182 }
5183
5184 VMXReadVMCS(VMX_VMCS16_HOST_FIELD_DS, &val);
5185 Log(("VMX_VMCS_HOST_FIELD_DS %08x\n", val));
5186 if (val < gdtr.cbGdt)
5187 {
5188 pDesc = (PCX86DESCHC)(gdtr.pGdt + (val & X86_SEL_MASK));
5189 HWACCMR0DumpDescriptor(pDesc, val, "DS: ");
5190 }
5191
5192 VMXReadVMCS(VMX_VMCS16_HOST_FIELD_ES, &val);
5193 Log(("VMX_VMCS_HOST_FIELD_ES %08x\n", val));
5194 if (val < gdtr.cbGdt)
5195 {
5196 pDesc = (PCX86DESCHC)(gdtr.pGdt + (val & X86_SEL_MASK));
5197 HWACCMR0DumpDescriptor(pDesc, val, "ES: ");
5198 }
5199
5200 VMXReadVMCS(VMX_VMCS16_HOST_FIELD_FS, &val);
5201 Log(("VMX_VMCS16_HOST_FIELD_FS %08x\n", val));
5202 if (val < gdtr.cbGdt)
5203 {
5204 pDesc = (PCX86DESCHC)(gdtr.pGdt + (val & X86_SEL_MASK));
5205 HWACCMR0DumpDescriptor(pDesc, val, "FS: ");
5206 }
5207
5208 VMXReadVMCS(VMX_VMCS16_HOST_FIELD_GS, &val);
5209 Log(("VMX_VMCS16_HOST_FIELD_GS %08x\n", val));
5210 if (val < gdtr.cbGdt)
5211 {
5212 pDesc = (PCX86DESCHC)(gdtr.pGdt + (val & X86_SEL_MASK));
5213 HWACCMR0DumpDescriptor(pDesc, val, "GS: ");
5214 }
5215
5216 VMXReadVMCS(VMX_VMCS16_HOST_FIELD_SS, &val);
5217 Log(("VMX_VMCS16_HOST_FIELD_SS %08x\n", val));
5218 if (val < gdtr.cbGdt)
5219 {
5220 pDesc = (PCX86DESCHC)(gdtr.pGdt + (val & X86_SEL_MASK));
5221 HWACCMR0DumpDescriptor(pDesc, val, "SS: ");
5222 }
5223
5224 VMXReadVMCS(VMX_VMCS16_HOST_FIELD_TR, &val);
5225 Log(("VMX_VMCS16_HOST_FIELD_TR %08x\n", val));
5226 if (val < gdtr.cbGdt)
5227 {
5228 pDesc = (PCX86DESCHC)(gdtr.pGdt + (val & X86_SEL_MASK));
5229 HWACCMR0DumpDescriptor(pDesc, val, "TR: ");
5230 }
5231
5232 VMXReadVMCS(VMX_VMCS_HOST_TR_BASE, &val);
5233 Log(("VMX_VMCS_HOST_TR_BASE %RHv\n", val));
5234 VMXReadVMCS(VMX_VMCS_HOST_GDTR_BASE, &val);
5235 Log(("VMX_VMCS_HOST_GDTR_BASE %RHv\n", val));
5236 VMXReadVMCS(VMX_VMCS_HOST_IDTR_BASE, &val);
5237 Log(("VMX_VMCS_HOST_IDTR_BASE %RHv\n", val));
5238 VMXReadVMCS(VMX_VMCS32_HOST_SYSENTER_CS, &val);
5239 Log(("VMX_VMCS_HOST_SYSENTER_CS %08x\n", val));
5240 VMXReadVMCS(VMX_VMCS_HOST_SYSENTER_EIP, &val);
5241 Log(("VMX_VMCS_HOST_SYSENTER_EIP %RHv\n", val));
5242 VMXReadVMCS(VMX_VMCS_HOST_SYSENTER_ESP, &val);
5243 Log(("VMX_VMCS_HOST_SYSENTER_ESP %RHv\n", val));
5244 VMXReadVMCS(VMX_VMCS_HOST_RSP, &val);
5245 Log(("VMX_VMCS_HOST_RSP %RHv\n", val));
5246 VMXReadVMCS(VMX_VMCS_HOST_RIP, &val);
5247 Log(("VMX_VMCS_HOST_RIP %RHv\n", val));
5248# if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
5249 if (VMX_IS_64BIT_HOST_MODE())
5250 {
5251 Log(("MSR_K6_EFER = %RX64\n", ASMRdMsr(MSR_K6_EFER)));
5252 Log(("MSR_K6_STAR = %RX64\n", ASMRdMsr(MSR_K6_STAR)));
5253 Log(("MSR_K8_LSTAR = %RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
5254 Log(("MSR_K8_CSTAR = %RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
5255 Log(("MSR_K8_SF_MASK = %RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
5256 Log(("MSR_K8_KERNEL_GS_BASE = %RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
5257 }
5258# endif
5259#endif /* VBOX_STRICT */
5260 }
5261 break;
5262 }
5263
5264 default:
5265 /* impossible */
5266 AssertMsgFailed(("%Rrc (%#x)\n", VBOXSTRICTRC_VAL(rc), VBOXSTRICTRC_VAL(rc)));
5267 break;
5268 }
5269}
5270
5271
5272#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
5273/**
5274 * Prepares for and executes VMLAUNCH (64 bits guest mode).
5275 *
5276 * @returns VBox status code.
5277 * @param fResume Whether to vmlauch/vmresume.
5278 * @param pCtx Pointer to the guest CPU context.
5279 * @param pCache Pointer to the VMCS cache.
5280 * @param pVM Pointer to the VM.
5281 * @param pVCpu Pointer to the VMCPU.
5282 */
5283DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
5284{
5285 uint32_t aParam[6];
5286 PHMGLOBLCPUINFO pCpu;
5287 RTHCPHYS HCPhysCpuPage;
5288 int rc;
5289
5290 pCpu = HWACCMR0GetCurrentCpu();
5291 HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
5292
5293#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5294 pCache->uPos = 1;
5295 pCache->interPD = PGMGetInterPaeCR3(pVM);
5296 pCache->pSwitcher = (uint64_t)pVM->hwaccm.s.pfnHost32ToGuest64R0;
5297#endif
5298
5299#ifdef DEBUG
5300 pCache->TestIn.HCPhysCpuPage= 0;
5301 pCache->TestIn.HCPhysVMCS = 0;
5302 pCache->TestIn.pCache = 0;
5303 pCache->TestOut.HCPhysVMCS = 0;
5304 pCache->TestOut.pCache = 0;
5305 pCache->TestOut.pCtx = 0;
5306 pCache->TestOut.eflags = 0;
5307#endif
5308
5309 aParam[0] = (uint32_t)(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
5310 aParam[1] = (uint32_t)(HCPhysCpuPage >> 32); /* Param 1: VMXON physical address - Hi. */
5311 aParam[2] = (uint32_t)(pVCpu->hwaccm.s.vmx.HCPhysVMCS); /* Param 2: VMCS physical address - Lo. */
5312 aParam[3] = (uint32_t)(pVCpu->hwaccm.s.vmx.HCPhysVMCS >> 32); /* Param 2: VMCS physical address - Hi. */
5313 aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hwaccm.s.vmx.VMCSCache);
5314 aParam[5] = 0;
5315
5316#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5317 pCtx->dr[4] = pVM->hwaccm.s.vmx.pScratchPhys + 16 + 8;
5318 *(uint32_t *)(pVM->hwaccm.s.vmx.pScratch + 16 + 8) = 1;
5319#endif
5320 rc = VMXR0Execute64BitsHandler(pVM, pVCpu, pCtx, pVM->hwaccm.s.pfnVMXGCStartVM64, 6, &aParam[0]);
5321
5322#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5323 Assert(*(uint32_t *)(pVM->hwaccm.s.vmx.pScratch + 16 + 8) == 5);
5324 Assert(pCtx->dr[4] == 10);
5325 *(uint32_t *)(pVM->hwaccm.s.vmx.pScratch + 16 + 8) = 0xff;
5326#endif
5327
5328#ifdef DEBUG
5329 AssertMsg(pCache->TestIn.HCPhysCpuPage== HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
5330 AssertMsg(pCache->TestIn.HCPhysVMCS == pVCpu->hwaccm.s.vmx.HCPhysVMCS, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVMCS,
5331 pVCpu->hwaccm.s.vmx.HCPhysVMCS));
5332 AssertMsg(pCache->TestIn.HCPhysVMCS == pCache->TestOut.HCPhysVMCS, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVMCS,
5333 pCache->TestOut.HCPhysVMCS));
5334 AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
5335 pCache->TestOut.pCache));
5336 AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hwaccm.s.vmx.VMCSCache),
5337 ("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hwaccm.s.vmx.VMCSCache)));
5338 AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
5339 pCache->TestOut.pCtx));
5340 Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
5341#endif
5342 return rc;
5343}
5344
5345
5346# ifdef VBOX_STRICT
5347static bool hmR0VmxIsValidReadField(uint32_t idxField)
5348{
5349 switch (idxField)
5350 {
5351 case VMX_VMCS64_GUEST_RIP:
5352 case VMX_VMCS64_GUEST_RSP:
5353 case VMX_VMCS_GUEST_RFLAGS:
5354 case VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE:
5355 case VMX_VMCS_CTRL_CR0_READ_SHADOW:
5356 case VMX_VMCS64_GUEST_CR0:
5357 case VMX_VMCS_CTRL_CR4_READ_SHADOW:
5358 case VMX_VMCS64_GUEST_CR4:
5359 case VMX_VMCS64_GUEST_DR7:
5360 case VMX_VMCS32_GUEST_SYSENTER_CS:
5361 case VMX_VMCS64_GUEST_SYSENTER_EIP:
5362 case VMX_VMCS64_GUEST_SYSENTER_ESP:
5363 case VMX_VMCS32_GUEST_GDTR_LIMIT:
5364 case VMX_VMCS64_GUEST_GDTR_BASE:
5365 case VMX_VMCS32_GUEST_IDTR_LIMIT:
5366 case VMX_VMCS64_GUEST_IDTR_BASE:
5367 case VMX_VMCS16_GUEST_FIELD_CS:
5368 case VMX_VMCS32_GUEST_CS_LIMIT:
5369 case VMX_VMCS64_GUEST_CS_BASE:
5370 case VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS:
5371 case VMX_VMCS16_GUEST_FIELD_DS:
5372 case VMX_VMCS32_GUEST_DS_LIMIT:
5373 case VMX_VMCS64_GUEST_DS_BASE:
5374 case VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS:
5375 case VMX_VMCS16_GUEST_FIELD_ES:
5376 case VMX_VMCS32_GUEST_ES_LIMIT:
5377 case VMX_VMCS64_GUEST_ES_BASE:
5378 case VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS:
5379 case VMX_VMCS16_GUEST_FIELD_FS:
5380 case VMX_VMCS32_GUEST_FS_LIMIT:
5381 case VMX_VMCS64_GUEST_FS_BASE:
5382 case VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS:
5383 case VMX_VMCS16_GUEST_FIELD_GS:
5384 case VMX_VMCS32_GUEST_GS_LIMIT:
5385 case VMX_VMCS64_GUEST_GS_BASE:
5386 case VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS:
5387 case VMX_VMCS16_GUEST_FIELD_SS:
5388 case VMX_VMCS32_GUEST_SS_LIMIT:
5389 case VMX_VMCS64_GUEST_SS_BASE:
5390 case VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS:
5391 case VMX_VMCS16_GUEST_FIELD_LDTR:
5392 case VMX_VMCS32_GUEST_LDTR_LIMIT:
5393 case VMX_VMCS64_GUEST_LDTR_BASE:
5394 case VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS:
5395 case VMX_VMCS16_GUEST_FIELD_TR:
5396 case VMX_VMCS32_GUEST_TR_LIMIT:
5397 case VMX_VMCS64_GUEST_TR_BASE:
5398 case VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS:
5399 case VMX_VMCS32_RO_EXIT_REASON:
5400 case VMX_VMCS32_RO_VM_INSTR_ERROR:
5401 case VMX_VMCS32_RO_EXIT_INSTR_LENGTH:
5402 case VMX_VMCS32_RO_EXIT_INTERRUPTION_ERRCODE:
5403 case VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO:
5404 case VMX_VMCS32_RO_EXIT_INSTR_INFO:
5405 case VMX_VMCS_RO_EXIT_QUALIFICATION:
5406 case VMX_VMCS32_RO_IDT_INFO:
5407 case VMX_VMCS32_RO_IDT_ERRCODE:
5408 case VMX_VMCS64_GUEST_CR3:
5409 case VMX_VMCS_EXIT_PHYS_ADDR_FULL:
5410 return true;
5411 }
5412 return false;
5413}
5414
5415
5416static bool hmR0VmxIsValidWriteField(uint32_t idxField)
5417{
5418 switch (idxField)
5419 {
5420 case VMX_VMCS64_GUEST_LDTR_BASE:
5421 case VMX_VMCS64_GUEST_TR_BASE:
5422 case VMX_VMCS64_GUEST_GDTR_BASE:
5423 case VMX_VMCS64_GUEST_IDTR_BASE:
5424 case VMX_VMCS64_GUEST_SYSENTER_EIP:
5425 case VMX_VMCS64_GUEST_SYSENTER_ESP:
5426 case VMX_VMCS64_GUEST_CR0:
5427 case VMX_VMCS64_GUEST_CR4:
5428 case VMX_VMCS64_GUEST_CR3:
5429 case VMX_VMCS64_GUEST_DR7:
5430 case VMX_VMCS64_GUEST_RIP:
5431 case VMX_VMCS64_GUEST_RSP:
5432 case VMX_VMCS64_GUEST_CS_BASE:
5433 case VMX_VMCS64_GUEST_DS_BASE:
5434 case VMX_VMCS64_GUEST_ES_BASE:
5435 case VMX_VMCS64_GUEST_FS_BASE:
5436 case VMX_VMCS64_GUEST_GS_BASE:
5437 case VMX_VMCS64_GUEST_SS_BASE:
5438 return true;
5439 }
5440 return false;
5441}
5442# endif /* VBOX_STRICT */
5443
5444
5445/**
5446 * Executes the specified handler in 64-bit mode.
5447 *
5448 * @returns VBox status code.
5449 * @param pVM Pointer to the VM.
5450 * @param pVCpu Pointer to the VMCPU.
5451 * @param pCtx Pointer to the guest CPU context.
5452 * @param pfnHandler Pointer to the RC handler function.
5453 * @param cbParam Number of parameters.
5454 * @param paParam Array of 32-bit parameters.
5455 */
5456VMMR0DECL(int) VMXR0Execute64BitsHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, RTRCPTR pfnHandler, uint32_t cbParam,
5457 uint32_t *paParam)
5458{
5459 int rc, rc2;
5460 PHMGLOBLCPUINFO pCpu;
5461 RTHCPHYS HCPhysCpuPage;
5462 RTHCUINTREG uOldEFlags;
5463
5464 AssertReturn(pVM->hwaccm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
5465 Assert(pfnHandler);
5466 Assert(pVCpu->hwaccm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hwaccm.s.vmx.VMCSCache.Write.aField));
5467 Assert(pVCpu->hwaccm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hwaccm.s.vmx.VMCSCache.Read.aField));
5468
5469#ifdef VBOX_STRICT
5470 for (unsigned i=0;i<pVCpu->hwaccm.s.vmx.VMCSCache.Write.cValidEntries;i++)
5471 Assert(hmR0VmxIsValidWriteField(pVCpu->hwaccm.s.vmx.VMCSCache.Write.aField[i]));
5472
5473 for (unsigned i=0;i<pVCpu->hwaccm.s.vmx.VMCSCache.Read.cValidEntries;i++)
5474 Assert(hmR0VmxIsValidReadField(pVCpu->hwaccm.s.vmx.VMCSCache.Read.aField[i]));
5475#endif
5476
5477 /* Disable interrupts. */
5478 uOldEFlags = ASMIntDisableFlags();
5479
5480#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
5481 RTCPUID idHostCpu = RTMpCpuId();
5482 CPUMR0SetLApic(pVM, idHostCpu);
5483#endif
5484
5485 pCpu = HWACCMR0GetCurrentCpu();
5486 HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
5487
5488 /* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
5489 VMXClearVMCS(pVCpu->hwaccm.s.vmx.HCPhysVMCS);
5490
5491 /* Leave VMX Root Mode. */
5492 VMXDisable();
5493
5494 ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
5495
5496 CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
5497 CPUMSetHyperEIP(pVCpu, pfnHandler);
5498 for (int i=(int)cbParam-1;i>=0;i--)
5499 CPUMPushHyper(pVCpu, paParam[i]);
5500
5501 STAM_PROFILE_ADV_START(&pVCpu->hwaccm.s.StatWorldSwitch3264, z);
5502
5503 /* Call switcher. */
5504 rc = pVM->hwaccm.s.pfnHost32ToGuest64R0(pVM, RT_OFFSETOF(VM, aCpus[pVCpu->idCpu].cpum) - RT_OFFSETOF(VM, cpum));
5505 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatWorldSwitch3264, z);
5506
5507 /* Make sure the VMX instructions don't cause #UD faults. */
5508 ASMSetCR4(ASMGetCR4() | X86_CR4_VMXE);
5509
5510 /* Enter VMX Root Mode */
5511 rc2 = VMXEnable(HCPhysCpuPage);
5512 if (RT_FAILURE(rc2))
5513 {
5514 ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
5515 ASMSetFlags(uOldEFlags);
5516 return VERR_VMX_VMXON_FAILED;
5517 }
5518
5519 rc2 = VMXActivateVMCS(pVCpu->hwaccm.s.vmx.HCPhysVMCS);
5520 AssertRC(rc2);
5521 Assert(!(ASMGetFlags() & X86_EFL_IF));
5522 ASMSetFlags(uOldEFlags);
5523 return rc;
5524}
5525#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL) */
5526
5527
5528#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
5529/**
5530 * Executes VMWRITE.
5531 *
5532 * @returns VBox status code
5533 * @param pVCpu Pointer to the VMCPU.
5534 * @param idxField VMCS field index.
5535 * @param u64Val 16, 32 or 64 bits value.
5536 */
5537VMMR0DECL(int) VMXWriteVMCS64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5538{
5539 int rc;
5540 switch (idxField)
5541 {
5542 case VMX_VMCS_CTRL_TSC_OFFSET_FULL:
5543 case VMX_VMCS_CTRL_IO_BITMAP_A_FULL:
5544 case VMX_VMCS_CTRL_IO_BITMAP_B_FULL:
5545 case VMX_VMCS_CTRL_MSR_BITMAP_FULL:
5546 case VMX_VMCS_CTRL_VMEXIT_MSR_STORE_FULL:
5547 case VMX_VMCS_CTRL_VMEXIT_MSR_LOAD_FULL:
5548 case VMX_VMCS_CTRL_VMENTRY_MSR_LOAD_FULL:
5549 case VMX_VMCS_CTRL_VAPIC_PAGEADDR_FULL:
5550 case VMX_VMCS_CTRL_APIC_ACCESSADDR_FULL:
5551 case VMX_VMCS_GUEST_LINK_PTR_FULL:
5552 case VMX_VMCS_GUEST_PDPTR0_FULL:
5553 case VMX_VMCS_GUEST_PDPTR1_FULL:
5554 case VMX_VMCS_GUEST_PDPTR2_FULL:
5555 case VMX_VMCS_GUEST_PDPTR3_FULL:
5556 case VMX_VMCS_GUEST_DEBUGCTL_FULL:
5557 case VMX_VMCS_GUEST_EFER_FULL:
5558 case VMX_VMCS_CTRL_EPTP_FULL:
5559 /* These fields consist of two parts, which are both writable in 32 bits mode. */
5560 rc = VMXWriteVMCS32(idxField, u64Val);
5561 rc |= VMXWriteVMCS32(idxField + 1, (uint32_t)(u64Val >> 32ULL));
5562 AssertRC(rc);
5563 return rc;
5564
5565 case VMX_VMCS64_GUEST_LDTR_BASE:
5566 case VMX_VMCS64_GUEST_TR_BASE:
5567 case VMX_VMCS64_GUEST_GDTR_BASE:
5568 case VMX_VMCS64_GUEST_IDTR_BASE:
5569 case VMX_VMCS64_GUEST_SYSENTER_EIP:
5570 case VMX_VMCS64_GUEST_SYSENTER_ESP:
5571 case VMX_VMCS64_GUEST_CR0:
5572 case VMX_VMCS64_GUEST_CR4:
5573 case VMX_VMCS64_GUEST_CR3:
5574 case VMX_VMCS64_GUEST_DR7:
5575 case VMX_VMCS64_GUEST_RIP:
5576 case VMX_VMCS64_GUEST_RSP:
5577 case VMX_VMCS64_GUEST_CS_BASE:
5578 case VMX_VMCS64_GUEST_DS_BASE:
5579 case VMX_VMCS64_GUEST_ES_BASE:
5580 case VMX_VMCS64_GUEST_FS_BASE:
5581 case VMX_VMCS64_GUEST_GS_BASE:
5582 case VMX_VMCS64_GUEST_SS_BASE:
5583 /* Queue a 64 bits value as we can't set it in 32 bits host mode. */
5584 if (u64Val >> 32ULL)
5585 rc = VMXWriteCachedVMCSEx(pVCpu, idxField, u64Val);
5586 else
5587 rc = VMXWriteVMCS32(idxField, (uint32_t)u64Val);
5588
5589 return rc;
5590
5591 default:
5592 AssertMsgFailed(("Unexpected field %x\n", idxField));
5593 return VERR_INVALID_PARAMETER;
5594 }
5595}
5596
5597
5598/**
5599 * Cache VMCS writes for performance reasons (Darwin) and for running 64 bits guests on 32 bits hosts.
5600 *
5601 * @param pVCpu Pointer to the VMCPU.
5602 * @param idxField VMCS field index.
5603 * @param u64Val 16, 32 or 64 bits value.
5604 */
5605VMMR0DECL(int) VMXWriteCachedVMCSEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5606{
5607 PVMCSCACHE pCache = &pVCpu->hwaccm.s.vmx.VMCSCache;
5608
5609 AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1,
5610 ("entries=%x\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
5611
5612 /* Make sure there are no duplicates. */
5613 for (unsigned i = 0; i < pCache->Write.cValidEntries; i++)
5614 {
5615 if (pCache->Write.aField[i] == idxField)
5616 {
5617 pCache->Write.aFieldVal[i] = u64Val;
5618 return VINF_SUCCESS;
5619 }
5620 }
5621
5622 pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
5623 pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
5624 pCache->Write.cValidEntries++;
5625 return VINF_SUCCESS;
5626}
5627
5628#endif /* HC_ARCH_BITS == 32 && !VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0 */
5629
Note: See TracBrowser for help on using the repository browser.

© 2024 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette