VirtualBox

source: vbox/trunk/src/VBox/VMM/VMMR0/HWVMXR0.cpp@ 43494

Last change on this file since 43494 was 43494, checked in by vboxsync, 12 years ago

VMM/VMMR0,R3: HM cleanup.
  • Property svn:eol-style set to native
  • Property svn:keywords set to Id Revision
File size: 219.5 KB
Line 
1/* $Id: HWVMXR0.cpp 43494 2012-10-01 14:29:11Z vboxsync $ */
2/** @file
3 * HM VMX (VT-x) - Host Context Ring-0.
4 */
5
6/*
7 * Copyright (C) 2006-2012 Oracle Corporation
8 *
9 * This file is part of VirtualBox Open Source Edition (OSE), as
10 * available from http://www.virtualbox.org. This file is free software;
11 * you can redistribute it and/or modify it under the terms of the GNU
12 * General Public License (GPL) as published by the Free Software
13 * Foundation, in version 2 as it comes in the "COPYING" file of the
14 * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15 * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16 */
17
18
19/*******************************************************************************
20* Header Files *
21*******************************************************************************/
22#define LOG_GROUP LOG_GROUP_HM
23#include <iprt/asm-amd64-x86.h>
24#include <VBox/vmm/hm.h>
25#include <VBox/vmm/pgm.h>
26#include <VBox/vmm/dbgf.h>
27#include <VBox/vmm/dbgftrace.h>
28#include <VBox/vmm/selm.h>
29#include <VBox/vmm/iom.h>
30#ifdef VBOX_WITH_REM
31# include <VBox/vmm/rem.h>
32#endif
33#include <VBox/vmm/tm.h>
34#include "HMInternal.h"
35#include <VBox/vmm/vm.h>
36#include <VBox/vmm/pdmapi.h>
37#include <VBox/err.h>
38#include <VBox/log.h>
39#include <iprt/assert.h>
40#include <iprt/param.h>
41#include <iprt/string.h>
42#include <iprt/time.h>
43#ifdef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
44# include <iprt/thread.h>
45#endif
46#include <iprt/x86.h>
47#include "HWVMXR0.h"
48
49#include "dtrace/VBoxVMM.h"
50
51
52/*******************************************************************************
53* Defined Constants And Macros *
54*******************************************************************************/
55#if defined(RT_ARCH_AMD64)
56# define VMX_IS_64BIT_HOST_MODE() (true)
57#elif defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
58# define VMX_IS_64BIT_HOST_MODE() (g_fVMXIs64bitHost != 0)
59#else
60# define VMX_IS_64BIT_HOST_MODE() (false)
61#endif
62
63
64/*******************************************************************************
65* Global Variables *
66*******************************************************************************/
67/* IO operation lookup arrays. */
68static uint32_t const g_aIOSize[4] = {1, 2, 0, 4};
69static uint32_t const g_aIOOpAnd[4] = {0xff, 0xffff, 0, 0xffffffff};
70
71#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
72/** See HMR0A.asm. */
73extern "C" uint32_t g_fVMXIs64bitHost;
74#endif
75
76
77/*******************************************************************************
78* Local Functions *
79*******************************************************************************/
80static DECLCALLBACK(void) hmR0VmxSetupTLBEPT(PVM pVM, PVMCPU pVCpu);
81static DECLCALLBACK(void) hmR0VmxSetupTLBVPID(PVM pVM, PVMCPU pVCpu);
82static DECLCALLBACK(void) hmR0VmxSetupTLBBoth(PVM pVM, PVMCPU pVCpu);
83static DECLCALLBACK(void) hmR0VmxSetupTLBDummy(PVM pVM, PVMCPU pVCpu);
84static void hmR0VmxFlushEPT(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_EPT enmFlush);
85static void hmR0VmxFlushVPID(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_VPID enmFlush, RTGCPTR GCPtr);
86static void hmR0VmxUpdateExceptionBitmap(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx);
87static void hmR0VmxSetMSRPermission(PVMCPU pVCpu, unsigned ulMSR, bool fRead, bool fWrite);
88static void hmR0VmxReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, VBOXSTRICTRC rc, PCPUMCTX pCtx);
89
90
91/**
92 * Updates error from VMCS to HMCPU's lasterror record.
93 *
94 * @param pVM Pointer to the VM.
95 * @param pVCpu Pointer to the VMCPU.
96 * @param rc The error code.
97 */
98static void hmR0VmxCheckError(PVM pVM, PVMCPU pVCpu, int rc)
99{
100 if (rc == VERR_VMX_GENERIC)
101 {
102 RTCCUINTREG instrError;
103
104 VMXReadVMCS(VMX_VMCS32_RO_VM_INSTR_ERROR, &instrError);
105 pVCpu->hm.s.vmx.lasterror.ulInstrError = instrError;
106 }
107 pVM->hm.s.lLastError = rc;
108}
109
110
111/**
112 * Sets up and activates VT-x on the current CPU.
113 *
114 * @returns VBox status code.
115 * @param pCpu Pointer to the CPU info struct.
116 * @param pVM Pointer to the VM. (can be NULL after a resume!!)
117 * @param pvCpuPage Pointer to the global CPU page.
118 * @param HCPhysCpuPage Physical address of the global CPU page.
119 * @param fEnabledByHost Set if SUPR0EnableVTx or similar was used to enable
120 * VT-x/AMD-V on the host.
121 */
122VMMR0DECL(int) VMXR0EnableCpu(PHMGLOBLCPUINFO pCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost)
123{
124 if (!fEnabledByHost)
125 {
126 AssertReturn(HCPhysCpuPage != 0 && HCPhysCpuPage != NIL_RTHCPHYS, VERR_INVALID_PARAMETER);
127 AssertReturn(pvCpuPage, VERR_INVALID_PARAMETER);
128
129 if (pVM)
130 {
131 /* Set revision dword at the beginning of the VMXON structure. */
132 *(uint32_t *)pvCpuPage = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.msr.vmx_basic_info);
133 }
134
135 /** @todo we should unmap the two pages from the virtual address space in order to prevent accidental corruption.
136 * (which can have very bad consequences!!!)
137 */
138
139 /** @todo r=bird: Why is this code different than the probing code earlier
140 * on? It just sets VMXE if needed and doesn't check that it isn't
141 * set. Mac OS X host_vmxoff may leave this set and we'll fail here
142 * and debug-assert in the calling code. This is what caused the
143 * "regression" after backing out the SUPR0EnableVTx code hours before
144 * 4.2.0GA (reboot fixed the issue). I've changed here to do the same
145 * as the init code. */
146 uint64_t uCr4 = ASMGetCR4();
147 if (!(uCr4 & X86_CR4_VMXE))
148 ASMSetCR4(ASMGetCR4() | X86_CR4_VMXE); /* Make sure the VMX instructions don't cause #UD faults. */
149
150 /*
151 * Enter VM root mode.
152 */
153 int rc = VMXEnable(HCPhysCpuPage);
154 if (RT_FAILURE(rc))
155 {
156 ASMSetCR4(uCr4);
157 return VERR_VMX_VMXON_FAILED;
158 }
159 }
160
161 /*
162 * Flush all VPIDs (in case we or any other hypervisor have been using VPIDs) so that
163 * we can avoid an explicit flush while using new VPIDs. We would still need to flush
164 * each time while reusing a VPID after hitting the MaxASID limit once.
165 */
166 if ( pVM
167 && pVM->hm.s.vmx.fVpid
168 && (pVM->hm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVVPID_CAPS_ALL_CONTEXTS))
169 {
170 hmR0VmxFlushVPID(pVM, NULL /* pvCpu */, VMX_FLUSH_VPID_ALL_CONTEXTS, 0 /* GCPtr */);
171 pCpu->fFlushAsidBeforeUse = false;
172 }
173 else
174 pCpu->fFlushAsidBeforeUse = true;
175
176 /*
177 * Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}.
178 */
179 ++pCpu->cTlbFlushes;
180
181 return VINF_SUCCESS;
182}
183
184
185/**
186 * Deactivates VT-x on the current CPU.
187 *
188 * @returns VBox status code.
189 * @param pCpu Pointer to the CPU info struct.
190 * @param pvCpuPage Pointer to the global CPU page.
191 * @param HCPhysCpuPage Physical address of the global CPU page.
192 */
193VMMR0DECL(int) VMXR0DisableCpu(PHMGLOBLCPUINFO pCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
194{
195 AssertReturn(HCPhysCpuPage != 0 && HCPhysCpuPage != NIL_RTHCPHYS, VERR_INVALID_PARAMETER);
196 AssertReturn(pvCpuPage, VERR_INVALID_PARAMETER);
197 NOREF(pCpu);
198
199 /* If we're somehow not in VMX root mode, then we shouldn't dare leaving it. */
200 if (!(ASMGetCR4() & X86_CR4_VMXE))
201 return VERR_VMX_NOT_IN_VMX_ROOT_MODE;
202
203 /* Leave VMX Root Mode. */
204 VMXDisable();
205
206 /* And clear the X86_CR4_VMXE bit. */
207 ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
208 return VINF_SUCCESS;
209}
210
211
212/**
213 * Does Ring-0 per VM VT-x initialization.
214 *
215 * @returns VBox status code.
216 * @param pVM Pointer to the VM.
217 */
218VMMR0DECL(int) VMXR0InitVM(PVM pVM)
219{
220 int rc;
221
222#ifdef LOG_ENABLED
223 SUPR0Printf("VMXR0InitVM %p\n", pVM);
224#endif
225
226 pVM->hm.s.vmx.hMemObjApicAccess = NIL_RTR0MEMOBJ;
227
228 if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
229 {
230 /* Allocate one page for the APIC physical page (serves for filtering accesses). */
231 rc = RTR0MemObjAllocCont(&pVM->hm.s.vmx.hMemObjApicAccess, PAGE_SIZE, false /* fExecutable */);
232 AssertRC(rc);
233 if (RT_FAILURE(rc))
234 return rc;
235
236 pVM->hm.s.vmx.pbApicAccess = (uint8_t *)RTR0MemObjAddress(pVM->hm.s.vmx.hMemObjApicAccess);
237 pVM->hm.s.vmx.HCPhysApicAccess = RTR0MemObjGetPagePhysAddr(pVM->hm.s.vmx.hMemObjApicAccess, 0);
238 ASMMemZero32(pVM->hm.s.vmx.pbApicAccess, PAGE_SIZE);
239 }
240 else
241 {
242 pVM->hm.s.vmx.hMemObjApicAccess = 0;
243 pVM->hm.s.vmx.pbApicAccess = 0;
244 pVM->hm.s.vmx.HCPhysApicAccess = 0;
245 }
246
247#ifdef VBOX_WITH_CRASHDUMP_MAGIC
248 {
249 rc = RTR0MemObjAllocCont(&pVM->hm.s.vmx.hMemObjScratch, PAGE_SIZE, false /* fExecutable */);
250 AssertRC(rc);
251 if (RT_FAILURE(rc))
252 return rc;
253
254 pVM->hm.s.vmx.pScratch = (uint8_t *)RTR0MemObjAddress(pVM->hm.s.vmx.hMemObjScratch);
255 pVM->hm.s.vmx.pScratchPhys = RTR0MemObjGetPagePhysAddr(pVM->hm.s.vmx.hMemObjScratch, 0);
256
257 ASMMemZero32(pVM->hm.s.vmx.pbScratch, PAGE_SIZE);
258 strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
259 *(uint64_t *)(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xDEADBEEFDEADBEEF);
260 }
261#endif
262
263 /* Allocate VMCSs for all guest CPUs. */
264 for (VMCPUID i = 0; i < pVM->cCpus; i++)
265 {
266 PVMCPU pVCpu = &pVM->aCpus[i];
267
268 pVCpu->hm.s.vmx.hMemObjVMCS = NIL_RTR0MEMOBJ;
269
270 /* Allocate one page for the VM control structure (VMCS). */
271 rc = RTR0MemObjAllocCont(&pVCpu->hm.s.vmx.hMemObjVMCS, PAGE_SIZE, false /* fExecutable */);
272 AssertRC(rc);
273 if (RT_FAILURE(rc))
274 return rc;
275
276 pVCpu->hm.s.vmx.pvVMCS = RTR0MemObjAddress(pVCpu->hm.s.vmx.hMemObjVMCS);
277 pVCpu->hm.s.vmx.HCPhysVMCS = RTR0MemObjGetPagePhysAddr(pVCpu->hm.s.vmx.hMemObjVMCS, 0);
278 ASMMemZeroPage(pVCpu->hm.s.vmx.pvVMCS);
279
280 pVCpu->hm.s.vmx.cr0_mask = 0;
281 pVCpu->hm.s.vmx.cr4_mask = 0;
282
283 /* Allocate one page for the virtual APIC page for TPR caching. */
284 rc = RTR0MemObjAllocCont(&pVCpu->hm.s.vmx.hMemObjVAPIC, PAGE_SIZE, false /* fExecutable */);
285 AssertRC(rc);
286 if (RT_FAILURE(rc))
287 return rc;
288
289 pVCpu->hm.s.vmx.pbVAPIC = (uint8_t *)RTR0MemObjAddress(pVCpu->hm.s.vmx.hMemObjVAPIC);
290 pVCpu->hm.s.vmx.HCPhysVAPIC = RTR0MemObjGetPagePhysAddr(pVCpu->hm.s.vmx.hMemObjVAPIC, 0);
291 ASMMemZeroPage(pVCpu->hm.s.vmx.pbVAPIC);
292
293 /* Allocate the MSR bitmap if this feature is supported. */
294 if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
295 {
296 rc = RTR0MemObjAllocCont(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, PAGE_SIZE, false /* fExecutable */);
297 AssertRC(rc);
298 if (RT_FAILURE(rc))
299 return rc;
300
301 pVCpu->hm.s.vmx.pvMsrBitmap = (uint8_t *)RTR0MemObjAddress(pVCpu->hm.s.vmx.hMemObjMsrBitmap);
302 pVCpu->hm.s.vmx.HCPhysMsrBitmap = RTR0MemObjGetPagePhysAddr(pVCpu->hm.s.vmx.hMemObjMsrBitmap, 0);
303 memset(pVCpu->hm.s.vmx.pvMsrBitmap, 0xff, PAGE_SIZE);
304 }
305
306#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
307 /* Allocate one page for the guest MSR load area (for preloading guest MSRs during the world switch). */
308 rc = RTR0MemObjAllocCont(&pVCpu->hm.s.vmx.hMemObjGuestMsr, PAGE_SIZE, false /* fExecutable */);
309 AssertRC(rc);
310 if (RT_FAILURE(rc))
311 return rc;
312
313 pVCpu->hm.s.vmx.pvGuestMsr = (uint8_t *)RTR0MemObjAddress(pVCpu->hm.s.vmx.hMemObjGuestMsr);
314 pVCpu->hm.s.vmx.HCPhysGuestMsr = RTR0MemObjGetPagePhysAddr(pVCpu->hm.s.vmx.hMemObjGuestMsr, 0);
315 Assert(!(pVCpu->hm.s.vmx.HCPhysGuestMsr & 0xf));
316 memset(pVCpu->hm.s.vmx.pvGuestMsr, 0, PAGE_SIZE);
317
318 /* Allocate one page for the host MSR load area (for restoring host MSRs after the world switch back). */
319 rc = RTR0MemObjAllocCont(&pVCpu->hm.s.vmx.hMemObjHostMsr, PAGE_SIZE, false /* fExecutable */);
320 AssertRC(rc);
321 if (RT_FAILURE(rc))
322 return rc;
323
324 pVCpu->hm.s.vmx.pvHostMsr = (uint8_t *)RTR0MemObjAddress(pVCpu->hm.s.vmx.hMemObjHostMsr);
325 pVCpu->hm.s.vmx.HCPhysHostMsr = RTR0MemObjGetPagePhysAddr(pVCpu->hm.s.vmx.hMemObjHostMsr, 0);
326 Assert(!(pVCpu->hm.s.vmx.HCPhysHostMsr & 0xf));
327 memset(pVCpu->hm.s.vmx.pvHostMsr, 0, PAGE_SIZE);
328#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
329
330 /* Current guest paging mode. */
331 pVCpu->hm.s.vmx.enmLastSeenGuestMode = PGMMODE_REAL;
332
333#ifdef LOG_ENABLED
334 SUPR0Printf("VMXR0InitVM %x VMCS=%x (%x)\n", pVM, pVCpu->hm.s.vmx.pvVMCS, (uint32_t)pVCpu->hm.s.vmx.HCPhysVMCS);
335#endif
336 }
337
338 return VINF_SUCCESS;
339}
340
341
342/**
343 * Does Ring-0 per VM VT-x termination.
344 *
345 * @returns VBox status code.
346 * @param pVM Pointer to the VM.
347 */
348VMMR0DECL(int) VMXR0TermVM(PVM pVM)
349{
350 for (VMCPUID i = 0; i < pVM->cCpus; i++)
351 {
352 PVMCPU pVCpu = &pVM->aCpus[i];
353
354 if (pVCpu->hm.s.vmx.hMemObjVMCS != NIL_RTR0MEMOBJ)
355 {
356 RTR0MemObjFree(pVCpu->hm.s.vmx.hMemObjVMCS, false);
357 pVCpu->hm.s.vmx.hMemObjVMCS = NIL_RTR0MEMOBJ;
358 pVCpu->hm.s.vmx.pvVMCS = 0;
359 pVCpu->hm.s.vmx.HCPhysVMCS = 0;
360 }
361 if (pVCpu->hm.s.vmx.hMemObjVAPIC != NIL_RTR0MEMOBJ)
362 {
363 RTR0MemObjFree(pVCpu->hm.s.vmx.hMemObjVAPIC, false);
364 pVCpu->hm.s.vmx.hMemObjVAPIC = NIL_RTR0MEMOBJ;
365 pVCpu->hm.s.vmx.pbVAPIC = 0;
366 pVCpu->hm.s.vmx.HCPhysVAPIC = 0;
367 }
368 if (pVCpu->hm.s.vmx.hMemObjMsrBitmap != NIL_RTR0MEMOBJ)
369 {
370 RTR0MemObjFree(pVCpu->hm.s.vmx.hMemObjMsrBitmap, false);
371 pVCpu->hm.s.vmx.hMemObjMsrBitmap = NIL_RTR0MEMOBJ;
372 pVCpu->hm.s.vmx.pvMsrBitmap = 0;
373 pVCpu->hm.s.vmx.HCPhysMsrBitmap = 0;
374 }
375#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
376 if (pVCpu->hm.s.vmx.hMemObjHostMsr != NIL_RTR0MEMOBJ)
377 {
378 RTR0MemObjFree(pVCpu->hm.s.vmx.hMemObjHostMsr, false);
379 pVCpu->hm.s.vmx.hMemObjHostMsr = NIL_RTR0MEMOBJ;
380 pVCpu->hm.s.vmx.pvHostMsr = 0;
381 pVCpu->hm.s.vmx.HCPhysHostMsr = 0;
382 }
383 if (pVCpu->hm.s.vmx.hMemObjGuestMsr != NIL_RTR0MEMOBJ)
384 {
385 RTR0MemObjFree(pVCpu->hm.s.vmx.hMemObjGuestMsr, false);
386 pVCpu->hm.s.vmx.hMemObjGuestMsr = NIL_RTR0MEMOBJ;
387 pVCpu->hm.s.vmx.pvGuestMsr = 0;
388 pVCpu->hm.s.vmx.HCPhysGuestMsr = 0;
389 }
390#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
391 }
392 if (pVM->hm.s.vmx.hMemObjApicAccess != NIL_RTR0MEMOBJ)
393 {
394 RTR0MemObjFree(pVM->hm.s.vmx.hMemObjApicAccess, false);
395 pVM->hm.s.vmx.hMemObjApicAccess = NIL_RTR0MEMOBJ;
396 pVM->hm.s.vmx.pbApicAccess = 0;
397 pVM->hm.s.vmx.HCPhysApicAccess = 0;
398 }
399#ifdef VBOX_WITH_CRASHDUMP_MAGIC
400 if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
401 {
402 ASMMemZero32(pVM->hm.s.vmx.pScratch, PAGE_SIZE);
403 RTR0MemObjFree(pVM->hm.s.vmx.hMemObjScratch, false);
404 pVM->hm.s.vmx.hMemObjScratch = NIL_RTR0MEMOBJ;
405 pVM->hm.s.vmx.pScratch = 0;
406 pVM->hm.s.vmx.pScratchPhys = 0;
407 }
408#endif
409 return VINF_SUCCESS;
410}
411
412
413/**
414 * Sets up VT-x for the specified VM.
415 *
416 * @returns VBox status code.
417 * @param pVM Pointer to the VM.
418 */
419VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
420{
421 int rc = VINF_SUCCESS;
422 uint32_t val;
423
424 AssertReturn(pVM, VERR_INVALID_PARAMETER);
425
426 /* Initialize these always, see hmR3InitFinalizeR0().*/
427 pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NONE;
428 pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NONE;
429
430 /* Determine optimal flush type for EPT. */
431 if (pVM->hm.s.fNestedPaging)
432 {
433 if (pVM->hm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVEPT)
434 {
435 if (pVM->hm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVEPT_CAPS_SINGLE_CONTEXT)
436 pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_SINGLE_CONTEXT;
437 else if (pVM->hm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVEPT_CAPS_ALL_CONTEXTS)
438 pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_ALL_CONTEXTS;
439 else
440 {
441 /*
442 * Should never really happen. EPT is supported but no suitable flush types supported.
443 * We cannot ignore EPT at this point as we've already setup Unrestricted Guest execution.
444 */
445 pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
446 return VERR_VMX_GENERIC;
447 }
448 }
449 else
450 {
451 /*
452 * Should never really happen. EPT is supported but INVEPT instruction is not supported.
453 */
454 pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
455 return VERR_VMX_GENERIC;
456 }
457 }
458
459 /* Determine optimal flush type for VPID. */
460 if (pVM->hm.s.vmx.fVpid)
461 {
462 if (pVM->hm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVVPID)
463 {
464 if (pVM->hm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVVPID_CAPS_SINGLE_CONTEXT)
465 pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_SINGLE_CONTEXT;
466 else if (pVM->hm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVVPID_CAPS_ALL_CONTEXTS)
467 pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_ALL_CONTEXTS;
468 else
469 {
470 /*
471 * Neither SINGLE nor ALL context flush types for VPID supported by the CPU.
472 * We do not handle other flush type combinations, ignore VPID capabilities.
473 */
474 if (pVM->hm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVVPID_CAPS_INDIV_ADDR)
475 Log(("VMXR0SetupVM: Only VMX_FLUSH_VPID_INDIV_ADDR supported. Ignoring VPID.\n"));
476 if (pVM->hm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVVPID_CAPS_SINGLE_CONTEXT_RETAIN_GLOBALS)
477 Log(("VMXR0SetupVM: Only VMX_FLUSH_VPID_SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
478 pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NOT_SUPPORTED;
479 pVM->hm.s.vmx.fVpid = false;
480 }
481 }
482 else
483 {
484 /*
485 * Should not really happen. EPT is supported but INVEPT is not supported.
486 * Ignore VPID capabilities as our code relies on using INVEPT for selective flushing.
487 */
488 Log(("VMXR0SetupVM: VPID supported without INVEPT support. Ignoring VPID.\n"));
489 pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NOT_SUPPORTED;
490 pVM->hm.s.vmx.fVpid = false;
491 }
492 }
493
494 for (VMCPUID i = 0; i < pVM->cCpus; i++)
495 {
496 PVMCPU pVCpu = &pVM->aCpus[i];
497
498 AssertPtr(pVCpu->hm.s.vmx.pvVMCS);
499
500 /* Set revision dword at the beginning of the VMCS structure. */
501 *(uint32_t *)pVCpu->hm.s.vmx.pvVMCS = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.msr.vmx_basic_info);
502
503 /*
504 * Clear and activate the VMCS.
505 */
506 Log(("HCPhysVMCS = %RHp\n", pVCpu->hm.s.vmx.HCPhysVMCS));
507 rc = VMXClearVMCS(pVCpu->hm.s.vmx.HCPhysVMCS);
508 if (RT_FAILURE(rc))
509 goto vmx_end;
510
511 rc = VMXActivateVMCS(pVCpu->hm.s.vmx.HCPhysVMCS);
512 if (RT_FAILURE(rc))
513 goto vmx_end;
514
515 /*
516 * VMX_VMCS_CTRL_PIN_EXEC_CONTROLS
517 * Set required bits to one and zero according to the MSR capabilities.
518 */
519 val = pVM->hm.s.vmx.msr.vmx_pin_ctls.n.disallowed0;
520 val |= VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_EXT_INT_EXIT /* External interrupts */
521 | VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_NMI_EXIT; /* Non-maskable interrupts */
522
523 /*
524 * Enable the VMX preemption timer.
525 */
526 if (pVM->hm.s.vmx.fUsePreemptTimer)
527 val |= VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_PREEMPT_TIMER;
528 val &= pVM->hm.s.vmx.msr.vmx_pin_ctls.n.allowed1;
529
530 rc = VMXWriteVMCS(VMX_VMCS_CTRL_PIN_EXEC_CONTROLS, val);
531 AssertRC(rc);
532
533 /*
534 * VMX_VMCS_CTRL_PROC_EXEC_CONTROLS
535 * Set required bits to one and zero according to the MSR capabilities.
536 */
537 val = pVM->hm.s.vmx.msr.vmx_proc_ctls.n.disallowed0;
538 /* Program which event cause VM-exits and which features we want to use. */
539 val |= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_HLT_EXIT
540 | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_TSC_OFFSET
541 | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT
542 | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_UNCOND_IO_EXIT
543 | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDPMC_EXIT
544 | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_EXIT
545 | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MWAIT_EXIT; /* don't execute mwait or else we'll idle inside
546 the guest (host thinks the cpu load is high) */
547
548 /* Without nested paging we should intercept invlpg and cr3 mov instructions. */
549 if (!pVM->hm.s.fNestedPaging)
550 {
551 val |= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INVLPG_EXIT
552 | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
553 | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT;
554 }
555
556 /*
557 * VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MWAIT_EXIT might cause a vmlaunch
558 * failure with an invalid control fields error. (combined with some other exit reasons)
559 */
560 if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
561 {
562 /* CR8 reads from the APIC shadow page; writes cause an exit is they lower the TPR below the threshold */
563 val |= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW;
564 Assert(pVM->hm.s.vmx.pbApicAccess);
565 }
566 else
567 /* Exit on CR8 reads & writes in case the TPR shadow feature isn't present. */
568 val |= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR8_STORE_EXIT | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR8_LOAD_EXIT;
569
570 if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
571 {
572 Assert(pVCpu->hm.s.vmx.HCPhysMsrBitmap);
573 val |= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS;
574 }
575
576 /* We will use the secondary control if it's present. */
577 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
578
579 /* Mask away the bits that the CPU doesn't support */
580 /** @todo make sure they don't conflict with the above requirements. */
581 val &= pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1;
582 pVCpu->hm.s.vmx.proc_ctls = val;
583
584 rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, val);
585 AssertRC(rc);
586
587 if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
588 {
589 /*
590 * VMX_VMCS_CTRL_PROC_EXEC_CONTROLS2
591 * Set required bits to one and zero according to the MSR capabilities.
592 */
593 val = pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.disallowed0;
594 val |= VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT;
595
596 if (pVM->hm.s.fNestedPaging)
597 val |= VMX_VMCS_CTRL_PROC_EXEC2_EPT;
598
599 if (pVM->hm.s.vmx.fVpid)
600 val |= VMX_VMCS_CTRL_PROC_EXEC2_VPID;
601
602 if (pVM->hm.s.fHasIoApic)
603 val |= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC;
604
605 if (pVM->hm.s.vmx.fUnrestrictedGuest)
606 val |= VMX_VMCS_CTRL_PROC_EXEC2_REAL_MODE;
607
608 if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
609 val |= VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP;
610
611 /* Mask away the bits that the CPU doesn't support */
612 /** @todo make sure they don't conflict with the above requirements. */
613 val &= pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1;
614 pVCpu->hm.s.vmx.proc_ctls2 = val;
615 rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS2, val);
616 AssertRC(rc);
617 }
618
619 /*
620 * VMX_VMCS_CTRL_CR3_TARGET_COUNT
621 * Set required bits to one and zero according to the MSR capabilities.
622 */
623 rc = VMXWriteVMCS(VMX_VMCS_CTRL_CR3_TARGET_COUNT, 0);
624 AssertRC(rc);
625
626 /*
627 * Forward all exception except #NM & #PF to the guest.
628 * We always need to check pagefaults since our shadow page table can be out of sync.
629 * And we always lazily sync the FPU & XMM state. .
630 */
631
632 /** @todo Possible optimization:
633 * Keep the FPU and XMM state current in the EM thread. That way there's no need to
634 * lazily sync anything, but the downside is that we can't use the FPU stack or XMM
635 * registers ourselves of course.
636 *
637 * Note: only possible if the current state is actually ours (X86_CR0_TS flag)
638 */
639
640 /*
641 * Don't filter page faults, all of them should cause a world switch.
642 */
643 rc = VMXWriteVMCS(VMX_VMCS_CTRL_PAGEFAULT_ERROR_MASK, 0);
644 AssertRC(rc);
645 rc = VMXWriteVMCS(VMX_VMCS_CTRL_PAGEFAULT_ERROR_MATCH, 0);
646 AssertRC(rc);
647
648 rc = VMXWriteVMCS64(VMX_VMCS_CTRL_TSC_OFFSET_FULL, 0);
649 AssertRC(rc);
650 rc = VMXWriteVMCS64(VMX_VMCS_CTRL_IO_BITMAP_A_FULL, 0);
651 AssertRC(rc);
652 rc = VMXWriteVMCS64(VMX_VMCS_CTRL_IO_BITMAP_B_FULL, 0);
653 AssertRC(rc);
654
655 /*
656 * Set the MSR bitmap address.
657 */
658 if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
659 {
660 Assert(pVCpu->hm.s.vmx.HCPhysMsrBitmap);
661
662 rc = VMXWriteVMCS64(VMX_VMCS_CTRL_MSR_BITMAP_FULL, pVCpu->hm.s.vmx.HCPhysMsrBitmap);
663 AssertRC(rc);
664
665 /*
666 * Allow the guest to directly modify these MSRs; they are loaded/stored automatically
667 * using MSR-load/store areas in the VMCS.
668 */
669 hmR0VmxSetMSRPermission(pVCpu, MSR_IA32_SYSENTER_CS, true, true);
670 hmR0VmxSetMSRPermission(pVCpu, MSR_IA32_SYSENTER_ESP, true, true);
671 hmR0VmxSetMSRPermission(pVCpu, MSR_IA32_SYSENTER_EIP, true, true);
672 hmR0VmxSetMSRPermission(pVCpu, MSR_K8_LSTAR, true, true);
673 hmR0VmxSetMSRPermission(pVCpu, MSR_K6_STAR, true, true);
674 hmR0VmxSetMSRPermission(pVCpu, MSR_K8_SF_MASK, true, true);
675 hmR0VmxSetMSRPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, true, true);
676 hmR0VmxSetMSRPermission(pVCpu, MSR_K8_GS_BASE, true, true);
677 hmR0VmxSetMSRPermission(pVCpu, MSR_K8_FS_BASE, true, true);
678 if (pVCpu->hm.s.vmx.proc_ctls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
679 hmR0VmxSetMSRPermission(pVCpu, MSR_K8_TSC_AUX, true, true);
680 }
681
682#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
683 /*
684 * Set the guest & host MSR load/store physical addresses.
685 */
686 Assert(pVCpu->hm.s.vmx.HCPhysGuestMsr);
687 rc = VMXWriteVMCS64(VMX_VMCS_CTRL_VMENTRY_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
688 AssertRC(rc);
689 rc = VMXWriteVMCS64(VMX_VMCS_CTRL_VMEXIT_MSR_STORE_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
690 AssertRC(rc);
691 Assert(pVCpu->hm.s.vmx.HCPhysHostMsr);
692 rc = VMXWriteVMCS64(VMX_VMCS_CTRL_VMEXIT_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysHostMsr);
693 AssertRC(rc);
694#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
695
696 rc = VMXWriteVMCS(VMX_VMCS_CTRL_ENTRY_MSR_LOAD_COUNT, 0);
697 AssertRC(rc);
698 rc = VMXWriteVMCS(VMX_VMCS_CTRL_EXIT_MSR_STORE_COUNT, 0);
699 AssertRC(rc);
700 rc = VMXWriteVMCS(VMX_VMCS_CTRL_EXIT_MSR_LOAD_COUNT, 0);
701 AssertRC(rc);
702
703 if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
704 {
705 Assert(pVM->hm.s.vmx.hMemObjApicAccess);
706 /* Optional */
707 rc = VMXWriteVMCS(VMX_VMCS_CTRL_TPR_THRESHOLD, 0);
708 rc |= VMXWriteVMCS64(VMX_VMCS_CTRL_VAPIC_PAGEADDR_FULL, pVCpu->hm.s.vmx.HCPhysVAPIC);
709
710 if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
711 rc |= VMXWriteVMCS64(VMX_VMCS_CTRL_APIC_ACCESSADDR_FULL, pVM->hm.s.vmx.HCPhysApicAccess);
712
713 AssertRC(rc);
714 }
715
716 /* Set link pointer to -1. Not currently used. */
717 rc = VMXWriteVMCS64(VMX_VMCS_GUEST_LINK_PTR_FULL, 0xFFFFFFFFFFFFFFFFULL);
718 AssertRC(rc);
719
720 /*
721 * Clear VMCS, marking it inactive. Clear implementation specific data and writing back
722 * VMCS data back to memory.
723 */
724 rc = VMXClearVMCS(pVCpu->hm.s.vmx.HCPhysVMCS);
725 AssertRC(rc);
726
727 /*
728 * Configure the VMCS read cache.
729 */
730 PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
731
732 VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_RIP);
733 VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_RSP);
734 VMXSetupCachedReadVMCS(pCache, VMX_VMCS_GUEST_RFLAGS);
735 VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE);
736 VMXSetupCachedReadVMCS(pCache, VMX_VMCS_CTRL_CR0_READ_SHADOW);
737 VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_CR0);
738 VMXSetupCachedReadVMCS(pCache, VMX_VMCS_CTRL_CR4_READ_SHADOW);
739 VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_CR4);
740 VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_DR7);
741 VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_GUEST_SYSENTER_CS);
742 VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_SYSENTER_EIP);
743 VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_SYSENTER_ESP);
744 VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_GUEST_GDTR_LIMIT);
745 VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_GDTR_BASE);
746 VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_GUEST_IDTR_LIMIT);
747 VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_IDTR_BASE);
748
749 VMX_SETUP_SELREG(ES, pCache);
750 VMX_SETUP_SELREG(SS, pCache);
751 VMX_SETUP_SELREG(CS, pCache);
752 VMX_SETUP_SELREG(DS, pCache);
753 VMX_SETUP_SELREG(FS, pCache);
754 VMX_SETUP_SELREG(GS, pCache);
755 VMX_SETUP_SELREG(LDTR, pCache);
756 VMX_SETUP_SELREG(TR, pCache);
757
758 /*
759 * Status code VMCS reads.
760 */
761 VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_EXIT_REASON);
762 VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_VM_INSTR_ERROR);
763 VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_EXIT_INSTR_LENGTH);
764 VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_EXIT_INTERRUPTION_ERRCODE);
765 VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO);
766 VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_EXIT_INSTR_INFO);
767 VMXSetupCachedReadVMCS(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
768 VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_IDT_INFO);
769 VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_IDT_ERRCODE);
770
771 if (pVM->hm.s.fNestedPaging)
772 {
773 VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_CR3);
774 VMXSetupCachedReadVMCS(pCache, VMX_VMCS_EXIT_PHYS_ADDR_FULL);
775 pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
776 }
777 else
778 pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
779 } /* for each VMCPU */
780
781 /*
782 * Setup the right TLB function based on CPU capabilities.
783 */
784 if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
785 pVM->hm.s.vmx.pfnSetupTaggedTlb = hmR0VmxSetupTLBBoth;
786 else if (pVM->hm.s.fNestedPaging)
787 pVM->hm.s.vmx.pfnSetupTaggedTlb = hmR0VmxSetupTLBEPT;
788 else if (pVM->hm.s.vmx.fVpid)
789 pVM->hm.s.vmx.pfnSetupTaggedTlb = hmR0VmxSetupTLBVPID;
790 else
791 pVM->hm.s.vmx.pfnSetupTaggedTlb = hmR0VmxSetupTLBDummy;
792
793vmx_end:
794 hmR0VmxCheckError(pVM, &pVM->aCpus[0], rc);
795 return rc;
796}
797
798
799/**
800 * Sets the permission bits for the specified MSR.
801 *
802 * @param pVCpu Pointer to the VMCPU.
803 * @param ulMSR The MSR value.
804 * @param fRead Whether reading is allowed.
805 * @param fWrite Whether writing is allowed.
806 */
807static void hmR0VmxSetMSRPermission(PVMCPU pVCpu, unsigned ulMSR, bool fRead, bool fWrite)
808{
809 unsigned ulBit;
810 uint8_t *pvMsrBitmap = (uint8_t *)pVCpu->hm.s.vmx.pvMsrBitmap;
811
812 /*
813 * Layout:
814 * 0x000 - 0x3ff - Low MSR read bits
815 * 0x400 - 0x7ff - High MSR read bits
816 * 0x800 - 0xbff - Low MSR write bits
817 * 0xc00 - 0xfff - High MSR write bits
818 */
819 if (ulMSR <= 0x00001FFF)
820 {
821 /* Pentium-compatible MSRs */
822 ulBit = ulMSR;
823 }
824 else if ( ulMSR >= 0xC0000000
825 && ulMSR <= 0xC0001FFF)
826 {
827 /* AMD Sixth Generation x86 Processor MSRs */
828 ulBit = (ulMSR - 0xC0000000);
829 pvMsrBitmap += 0x400;
830 }
831 else
832 {
833 AssertFailed();
834 return;
835 }
836
837 Assert(ulBit <= 0x1fff);
838 if (fRead)
839 ASMBitClear(pvMsrBitmap, ulBit);
840 else
841 ASMBitSet(pvMsrBitmap, ulBit);
842
843 if (fWrite)
844 ASMBitClear(pvMsrBitmap + 0x800, ulBit);
845 else
846 ASMBitSet(pvMsrBitmap + 0x800, ulBit);
847}
848
849
850/**
851 * Injects an event (trap or external interrupt).
852 *
853 * @returns VBox status code. Note that it may return VINF_EM_RESET to
854 * indicate a triple fault when injecting X86_XCPT_DF.
855 *
856 * @param pVM Pointer to the VM.
857 * @param pVCpu Pointer to the VMCPU.
858 * @param pCtx Pointer to the guest CPU Context.
859 * @param intInfo VMX interrupt info.
860 * @param cbInstr Opcode length of faulting instruction.
861 * @param errCode Error code (optional).
862 */
863static int hmR0VmxInjectEvent(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, uint32_t intInfo, uint32_t cbInstr, uint32_t errCode)
864{
865 int rc;
866 uint32_t iGate = VMX_EXIT_INTERRUPTION_INFO_VECTOR(intInfo);
867
868#ifdef VBOX_WITH_STATISTICS
869 STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[iGate & MASK_INJECT_IRQ_STAT]);
870#endif
871
872#ifdef VBOX_STRICT
873 if (iGate == 0xE)
874 {
875 LogFlow(("hmR0VmxInjectEvent: Injecting interrupt %d at %RGv error code=%08x CR2=%RGv intInfo=%08x\n", iGate,
876 (RTGCPTR)pCtx->rip, errCode, pCtx->cr2, intInfo));
877 }
878 else if (iGate < 0x20)
879 {
880 LogFlow(("hmR0VmxInjectEvent: Injecting interrupt %d at %RGv error code=%08x\n", iGate, (RTGCPTR)pCtx->rip,
881 errCode));
882 }
883 else
884 {
885 LogFlow(("INJ-EI: %x at %RGv\n", iGate, (RTGCPTR)pCtx->rip));
886 Assert( VMX_EXIT_INTERRUPTION_INFO_TYPE(intInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW
887 || !VMCPU_FF_ISSET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
888 Assert( VMX_EXIT_INTERRUPTION_INFO_TYPE(intInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW
889 || pCtx->eflags.u32 & X86_EFL_IF);
890 }
891#endif
892
893 if ( CPUMIsGuestInRealModeEx(pCtx)
894 && pVM->hm.s.vmx.pRealModeTSS)
895 {
896 RTGCPHYS GCPhysHandler;
897 uint16_t offset, ip;
898 RTSEL sel;
899
900 /*
901 * Injecting events doesn't work right with real mode emulation.
902 * (#GP if we try to inject external hardware interrupts)
903 * Inject the interrupt or trap directly instead.
904 *
905 * ASSUMES no access handlers for the bits we read or write below (should be safe).
906 */
907 Log(("Manual interrupt/trap '%x' inject (real mode)\n", iGate));
908
909 /*
910 * Check if the interrupt handler is present.
911 */
912 if (iGate * 4 + 3 > pCtx->idtr.cbIdt)
913 {
914 Log(("IDT cbIdt violation\n"));
915 if (iGate != X86_XCPT_DF)
916 {
917 uint32_t intInfo2;
918
919 intInfo2 = (iGate == X86_XCPT_GP) ? (uint32_t)X86_XCPT_DF : iGate;
920 intInfo2 |= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
921 intInfo2 |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
922 intInfo2 |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HWEXCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
923
924 return hmR0VmxInjectEvent(pVM, pVCpu, pCtx, intInfo2, 0, 0 /* no error code according to the Intel docs */);
925 }
926 Log(("Triple fault -> reset the VM!\n"));
927 return VINF_EM_RESET;
928 }
929 if ( VMX_EXIT_INTERRUPTION_INFO_TYPE(intInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW
930 || iGate == 3 /* Both #BP and #OF point to the instruction after. */
931 || iGate == 4)
932 {
933 ip = pCtx->ip + cbInstr;
934 }
935 else
936 ip = pCtx->ip;
937
938 /*
939 * Read the selector:offset pair of the interrupt handler.
940 */
941 GCPhysHandler = (RTGCPHYS)pCtx->idtr.pIdt + iGate * 4;
942 rc = PGMPhysSimpleReadGCPhys(pVM, &offset, GCPhysHandler, sizeof(offset)); AssertRC(rc);
943 rc = PGMPhysSimpleReadGCPhys(pVM, &sel, GCPhysHandler + 2, sizeof(sel)); AssertRC(rc);
944
945 LogFlow(("IDT handler %04X:%04X\n", sel, offset));
946
947 /*
948 * Construct the stack frame.
949 */
950 /** @todo Check stack limit. */
951 pCtx->sp -= 2;
952 LogFlow(("ss:sp %04X:%04X eflags=%x\n", pCtx->ss.Sel, pCtx->sp, pCtx->eflags.u));
953 rc = PGMPhysSimpleWriteGCPhys(pVM, pCtx->ss.u64Base + pCtx->sp, &pCtx->eflags, sizeof(uint16_t)); AssertRC(rc);
954 pCtx->sp -= 2;
955 LogFlow(("ss:sp %04X:%04X cs=%x\n", pCtx->ss.Sel, pCtx->sp, pCtx->cs.Sel));
956 rc = PGMPhysSimpleWriteGCPhys(pVM, pCtx->ss.u64Base + pCtx->sp, &pCtx->cs, sizeof(uint16_t)); AssertRC(rc);
957 pCtx->sp -= 2;
958 LogFlow(("ss:sp %04X:%04X ip=%x\n", pCtx->ss.Sel, pCtx->sp, ip));
959 rc = PGMPhysSimpleWriteGCPhys(pVM, pCtx->ss.u64Base + pCtx->sp, &ip, sizeof(ip)); AssertRC(rc);
960
961 /*
962 * Update the CPU state for executing the handler.
963 */
964 pCtx->rip = offset;
965 pCtx->cs.Sel = sel;
966 pCtx->cs.u64Base = sel << 4;
967 pCtx->eflags.u &= ~(X86_EFL_IF | X86_EFL_TF | X86_EFL_RF | X86_EFL_AC);
968
969 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_SEGMENT_REGS;
970 return VINF_SUCCESS;
971 }
972
973 /*
974 * Set event injection state.
975 */
976 rc = VMXWriteVMCS(VMX_VMCS_CTRL_ENTRY_IRQ_INFO, intInfo | (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT));
977 rc |= VMXWriteVMCS(VMX_VMCS_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
978 rc |= VMXWriteVMCS(VMX_VMCS_CTRL_ENTRY_EXCEPTION_ERRCODE, errCode);
979
980 AssertRC(rc);
981 return rc;
982}
983
984
985/**
986 * Checks for pending guest interrupts and injects them.
987 *
988 * @returns VBox status code.
989 * @param pVM Pointer to the VM.
990 * @param pVCpu Pointer to the VMCPU.
991 * @param pCtx Pointer to the guest CPU context.
992 */
993static int hmR0VmxCheckPendingInterrupt(PVM pVM, PVMCPU pVCpu, CPUMCTX *pCtx)
994{
995 int rc;
996
997 /*
998 * Dispatch any pending interrupts (injected before, but a VM exit occurred prematurely).
999 */
1000 if (pVCpu->hm.s.Event.fPending)
1001 {
1002 Log(("CPU%d: Reinjecting event %RX64 %08x at %RGv cr2=%RX64\n", pVCpu->idCpu, pVCpu->hm.s.Event.intInfo,
1003 pVCpu->hm.s.Event.errCode, (RTGCPTR)pCtx->rip, pCtx->cr2));
1004 STAM_COUNTER_INC(&pVCpu->hm.s.StatIntReinject);
1005 rc = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, pVCpu->hm.s.Event.intInfo, 0, pVCpu->hm.s.Event.errCode);
1006 AssertRC(rc);
1007
1008 pVCpu->hm.s.Event.fPending = false;
1009 return VINF_SUCCESS;
1010 }
1011
1012 /*
1013 * If an active trap is already pending, we must forward it first!
1014 */
1015 if (!TRPMHasTrap(pVCpu))
1016 {
1017 if (VMCPU_FF_TESTANDCLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI))
1018 {
1019 RTGCUINTPTR intInfo;
1020
1021 Log(("CPU%d: injecting #NMI\n", pVCpu->idCpu));
1022
1023 intInfo = X86_XCPT_NMI;
1024 intInfo |= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
1025 intInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
1026
1027 rc = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, intInfo, 0, 0);
1028 AssertRC(rc);
1029
1030 return VINF_SUCCESS;
1031 }
1032
1033 /** @todo SMI interrupts. */
1034
1035 /*
1036 * When external interrupts are pending, we should exit the VM when IF is set.
1037 */
1038 if (VMCPU_FF_ISPENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC|VMCPU_FF_INTERRUPT_PIC)))
1039 {
1040 if (!(pCtx->eflags.u32 & X86_EFL_IF))
1041 {
1042 if (!(pVCpu->hm.s.vmx.proc_ctls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_IRQ_WINDOW_EXIT))
1043 {
1044 LogFlow(("Enable irq window exit!\n"));
1045 pVCpu->hm.s.vmx.proc_ctls |= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_IRQ_WINDOW_EXIT;
1046 rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.proc_ctls);
1047 AssertRC(rc);
1048 }
1049 /* else nothing to do but wait */
1050 }
1051 else if (!VMCPU_FF_ISSET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
1052 {
1053 uint8_t u8Interrupt;
1054
1055 rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
1056 Log(("CPU%d: Dispatch interrupt: u8Interrupt=%x (%d) rc=%Rrc cs:rip=%04X:%RGv\n", pVCpu->idCpu,
1057 u8Interrupt, u8Interrupt, rc, pCtx->cs.Sel, (RTGCPTR)pCtx->rip));
1058 if (RT_SUCCESS(rc))
1059 {
1060 rc = TRPMAssertTrap(pVCpu, u8Interrupt, TRPM_HARDWARE_INT);
1061 AssertRC(rc);
1062 }
1063 else
1064 {
1065 /* Can only happen in rare cases where a pending interrupt is cleared behind our back */
1066 Assert(!VMCPU_FF_ISPENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC|VMCPU_FF_INTERRUPT_PIC)));
1067 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
1068 /* Just continue */
1069 }
1070 }
1071 else
1072 Log(("Pending interrupt blocked at %RGv by VM_FF_INHIBIT_INTERRUPTS!!\n", (RTGCPTR)pCtx->rip));
1073 }
1074 }
1075
1076#ifdef VBOX_STRICT
1077 if (TRPMHasTrap(pVCpu))
1078 {
1079 uint8_t u8Vector;
1080 rc = TRPMQueryTrapAll(pVCpu, &u8Vector, 0, 0, 0);
1081 AssertRC(rc);
1082 }
1083#endif
1084
1085 if ( (pCtx->eflags.u32 & X86_EFL_IF)
1086 && (!VMCPU_FF_ISSET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
1087 && TRPMHasTrap(pVCpu)
1088 )
1089 {
1090 uint8_t u8Vector;
1091 TRPMEVENT enmType;
1092 RTGCUINTPTR intInfo;
1093 RTGCUINT errCode;
1094
1095 /*
1096 * If a new event is pending, dispatch it now.
1097 */
1098 rc = TRPMQueryTrapAll(pVCpu, &u8Vector, &enmType, &errCode, 0);
1099 AssertRC(rc);
1100 Assert(pCtx->eflags.Bits.u1IF == 1 || enmType == TRPM_TRAP);
1101 Assert(enmType != TRPM_SOFTWARE_INT);
1102
1103 /*
1104 * Clear the pending trap.
1105 */
1106 rc = TRPMResetTrap(pVCpu);
1107 AssertRC(rc);
1108
1109 intInfo = u8Vector;
1110 intInfo |= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
1111
1112 if (enmType == TRPM_TRAP)
1113 {
1114 switch (u8Vector)
1115 {
1116 case X86_XCPT_DF:
1117 case X86_XCPT_TS:
1118 case X86_XCPT_NP:
1119 case X86_XCPT_SS:
1120 case X86_XCPT_GP:
1121 case X86_XCPT_PF:
1122 case X86_XCPT_AC:
1123 {
1124 /* Valid error codes. */
1125 intInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
1126 break;
1127 }
1128
1129 default:
1130 break;
1131 }
1132
1133 if ( u8Vector == X86_XCPT_BP
1134 || u8Vector == X86_XCPT_OF)
1135 {
1136 intInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SWEXCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
1137 }
1138 else
1139 intInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HWEXCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
1140 }
1141 else
1142 intInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
1143
1144 STAM_COUNTER_INC(&pVCpu->hm.s.StatIntInject);
1145 rc = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, intInfo, 0, errCode);
1146 AssertRC(rc);
1147 } /* if (interrupts can be dispatched) */
1148
1149 return VINF_SUCCESS;
1150}
1151
1152
1153/**
1154 * Save the host state into the VMCS.
1155 *
1156 * @returns VBox status code.
1157 * @param pVM Pointer to the VM.
1158 * @param pVCpu Pointer to the VMCPU.
1159 */
1160VMMR0DECL(int) VMXR0SaveHostState(PVM pVM, PVMCPU pVCpu)
1161{
1162 int rc = VINF_SUCCESS;
1163 NOREF(pVM);
1164
1165 /*
1166 * Host CPU Context.
1167 */
1168 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_HOST_CONTEXT)
1169 {
1170 RTIDTR idtr;
1171 RTGDTR gdtr;
1172 RTSEL SelTR;
1173 PCX86DESCHC pDesc;
1174 uintptr_t trBase;
1175 RTSEL cs;
1176 RTSEL ss;
1177 uint64_t cr3;
1178
1179 /*
1180 * Control registers.
1181 */
1182 rc = VMXWriteVMCS(VMX_VMCS_HOST_CR0, ASMGetCR0());
1183 Log2(("VMX_VMCS_HOST_CR0 %08x\n", ASMGetCR0()));
1184#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
1185 if (VMX_IS_64BIT_HOST_MODE())
1186 {
1187 cr3 = hmR0Get64bitCR3();
1188 rc |= VMXWriteVMCS64(VMX_VMCS_HOST_CR3, cr3);
1189 }
1190 else
1191#endif
1192 {
1193 cr3 = ASMGetCR3();
1194 rc |= VMXWriteVMCS(VMX_VMCS_HOST_CR3, cr3);
1195 }
1196 Log2(("VMX_VMCS_HOST_CR3 %08RX64\n", cr3));
1197 rc |= VMXWriteVMCS(VMX_VMCS_HOST_CR4, ASMGetCR4());
1198 Log2(("VMX_VMCS_HOST_CR4 %08x\n", ASMGetCR4()));
1199 AssertRC(rc);
1200
1201 /*
1202 * Selector registers.
1203 */
1204#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
1205 if (VMX_IS_64BIT_HOST_MODE())
1206 {
1207 cs = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelCS;
1208 ss = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelSS;
1209 }
1210 else
1211 {
1212 /* sysenter loads LDT cs & ss, VMX doesn't like this. Load the GDT ones (safe). */
1213 cs = (RTSEL)(uintptr_t)&SUPR0AbsKernelCS;
1214 ss = (RTSEL)(uintptr_t)&SUPR0AbsKernelSS;
1215 }
1216#else
1217 cs = ASMGetCS();
1218 ss = ASMGetSS();
1219#endif
1220 Assert(!(cs & X86_SEL_LDT)); Assert((cs & X86_SEL_RPL) == 0);
1221 Assert(!(ss & X86_SEL_LDT)); Assert((ss & X86_SEL_RPL) == 0);
1222 rc = VMXWriteVMCS(VMX_VMCS16_HOST_FIELD_CS, cs);
1223 /* Note: VMX is (again) very picky about the RPL of the selectors here; we'll restore them manually. */
1224 rc |= VMXWriteVMCS(VMX_VMCS16_HOST_FIELD_DS, 0);
1225 rc |= VMXWriteVMCS(VMX_VMCS16_HOST_FIELD_ES, 0);
1226#if HC_ARCH_BITS == 32
1227 if (!VMX_IS_64BIT_HOST_MODE())
1228 {
1229 rc |= VMXWriteVMCS(VMX_VMCS16_HOST_FIELD_FS, 0);
1230 rc |= VMXWriteVMCS(VMX_VMCS16_HOST_FIELD_GS, 0);
1231 }
1232#endif
1233 rc |= VMXWriteVMCS(VMX_VMCS16_HOST_FIELD_SS, ss);
1234 SelTR = ASMGetTR();
1235 rc |= VMXWriteVMCS(VMX_VMCS16_HOST_FIELD_TR, SelTR);
1236 AssertRC(rc);
1237 Log2(("VMX_VMCS_HOST_FIELD_CS %08x (%08x)\n", cs, ASMGetSS()));
1238 Log2(("VMX_VMCS_HOST_FIELD_DS 00000000 (%08x)\n", ASMGetDS()));
1239 Log2(("VMX_VMCS_HOST_FIELD_ES 00000000 (%08x)\n", ASMGetES()));
1240 Log2(("VMX_VMCS_HOST_FIELD_FS 00000000 (%08x)\n", ASMGetFS()));
1241 Log2(("VMX_VMCS_HOST_FIELD_GS 00000000 (%08x)\n", ASMGetGS()));
1242 Log2(("VMX_VMCS_HOST_FIELD_SS %08x (%08x)\n", ss, ASMGetSS()));
1243 Log2(("VMX_VMCS_HOST_FIELD_TR %08x\n", ASMGetTR()));
1244
1245 /*
1246 * GDTR & IDTR.
1247 */
1248#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
1249 if (VMX_IS_64BIT_HOST_MODE())
1250 {
1251 X86XDTR64 gdtr64, idtr64;
1252 hmR0Get64bitGDTRandIDTR(&gdtr64, &idtr64);
1253 rc = VMXWriteVMCS64(VMX_VMCS_HOST_GDTR_BASE, gdtr64.uAddr);
1254 rc |= VMXWriteVMCS64(VMX_VMCS_HOST_IDTR_BASE, gdtr64.uAddr);
1255 AssertRC(rc);
1256 Log2(("VMX_VMCS_HOST_GDTR_BASE %RX64\n", gdtr64.uAddr));
1257 Log2(("VMX_VMCS_HOST_IDTR_BASE %RX64\n", idtr64.uAddr));
1258 gdtr.cbGdt = gdtr64.cb;
1259 gdtr.pGdt = (uintptr_t)gdtr64.uAddr;
1260 }
1261 else
1262#endif
1263 {
1264 ASMGetGDTR(&gdtr);
1265 rc = VMXWriteVMCS(VMX_VMCS_HOST_GDTR_BASE, gdtr.pGdt);
1266 ASMGetIDTR(&idtr);
1267 rc |= VMXWriteVMCS(VMX_VMCS_HOST_IDTR_BASE, idtr.pIdt);
1268 AssertRC(rc);
1269 Log2(("VMX_VMCS_HOST_GDTR_BASE %RHv\n", gdtr.pGdt));
1270 Log2(("VMX_VMCS_HOST_IDTR_BASE %RHv\n", idtr.pIdt));
1271 }
1272
1273 /*
1274 * Save the base address of the TR selector.
1275 */
1276 if (SelTR > gdtr.cbGdt)
1277 {
1278 AssertMsgFailed(("Invalid TR selector %x. GDTR.cbGdt=%x\n", SelTR, gdtr.cbGdt));
1279 return VERR_VMX_INVALID_HOST_STATE;
1280 }
1281
1282 pDesc = (PCX86DESCHC)(gdtr.pGdt + (SelTR & X86_SEL_MASK));
1283#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
1284 if (VMX_IS_64BIT_HOST_MODE())
1285 {
1286 uint64_t trBase64 = X86DESC64_BASE((PX86DESC64)pDesc);
1287 rc = VMXWriteVMCS64(VMX_VMCS_HOST_TR_BASE, trBase64);
1288 Log2(("VMX_VMCS_HOST_TR_BASE %RX64\n", trBase64));
1289 AssertRC(rc);
1290 }
1291 else
1292#endif
1293 {
1294#if HC_ARCH_BITS == 64
1295 trBase = X86DESC64_BASE(pDesc);
1296#else
1297 trBase = X86DESC_BASE(pDesc);
1298#endif
1299 rc = VMXWriteVMCS(VMX_VMCS_HOST_TR_BASE, trBase);
1300 AssertRC(rc);
1301 Log2(("VMX_VMCS_HOST_TR_BASE %RHv\n", trBase));
1302 }
1303
1304 /*
1305 * FS base and GS base.
1306 */
1307#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
1308 if (VMX_IS_64BIT_HOST_MODE())
1309 {
1310 Log2(("MSR_K8_FS_BASE = %RX64\n", ASMRdMsr(MSR_K8_FS_BASE)));
1311 Log2(("MSR_K8_GS_BASE = %RX64\n", ASMRdMsr(MSR_K8_GS_BASE)));
1312 rc = VMXWriteVMCS64(VMX_VMCS_HOST_FS_BASE, ASMRdMsr(MSR_K8_FS_BASE));
1313 rc |= VMXWriteVMCS64(VMX_VMCS_HOST_GS_BASE, ASMRdMsr(MSR_K8_GS_BASE));
1314 }
1315#endif
1316 AssertRC(rc);
1317
1318 /*
1319 * Sysenter MSRs.
1320 */
1321 /** @todo expensive!! */
1322 rc = VMXWriteVMCS(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
1323 Log2(("VMX_VMCS_HOST_SYSENTER_CS %08x\n", ASMRdMsr_Low(MSR_IA32_SYSENTER_CS)));
1324#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
1325 if (VMX_IS_64BIT_HOST_MODE())
1326 {
1327 Log2(("VMX_VMCS_HOST_SYSENTER_EIP %RX64\n", ASMRdMsr(MSR_IA32_SYSENTER_EIP)));
1328 Log2(("VMX_VMCS_HOST_SYSENTER_ESP %RX64\n", ASMRdMsr(MSR_IA32_SYSENTER_ESP)));
1329 rc |= VMXWriteVMCS64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
1330 rc |= VMXWriteVMCS64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
1331 }
1332 else
1333 {
1334 rc |= VMXWriteVMCS(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
1335 rc |= VMXWriteVMCS(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
1336 Log2(("VMX_VMCS_HOST_SYSENTER_EIP %RX32\n", ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP)));
1337 Log2(("VMX_VMCS_HOST_SYSENTER_ESP %RX32\n", ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP)));
1338 }
1339#elif HC_ARCH_BITS == 32
1340 rc |= VMXWriteVMCS(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
1341 rc |= VMXWriteVMCS(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
1342 Log2(("VMX_VMCS_HOST_SYSENTER_EIP %RX32\n", ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP)));
1343 Log2(("VMX_VMCS_HOST_SYSENTER_ESP %RX32\n", ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP)));
1344#else
1345 Log2(("VMX_VMCS_HOST_SYSENTER_EIP %RX64\n", ASMRdMsr(MSR_IA32_SYSENTER_EIP)));
1346 Log2(("VMX_VMCS_HOST_SYSENTER_ESP %RX64\n", ASMRdMsr(MSR_IA32_SYSENTER_ESP)));
1347 rc |= VMXWriteVMCS64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
1348 rc |= VMXWriteVMCS64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
1349#endif
1350 AssertRC(rc);
1351
1352
1353#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
1354 /*
1355 * Store all host MSRs in the VM-Exit load area, so they will be reloaded after
1356 * the world switch back to the host.
1357 */
1358 PVMXMSR pMsr = (PVMXMSR)pVCpu->hm.s.vmx.pvHostMsr;
1359 unsigned idxMsr = 0;
1360
1361 uint32_t u32HostExtFeatures = ASMCpuId_EDX(0x80000001);
1362 if (u32HostExtFeatures & (X86_CPUID_EXT_FEATURE_EDX_NX | X86_CPUID_EXT_FEATURE_EDX_LONG_MODE))
1363 {
1364#if 0
1365 pMsr->u32IndexMSR = MSR_K6_EFER;
1366 pMsr->u32Reserved = 0;
1367# if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
1368 if (CPUMIsGuestInLongMode(pVCpu))
1369 {
1370 /* Must match the EFER value in our 64 bits switcher. */
1371 pMsr->u64Value = ASMRdMsr(MSR_K6_EFER) | MSR_K6_EFER_LME | MSR_K6_EFER_SCE | MSR_K6_EFER_NXE;
1372 }
1373 else
1374# endif
1375 pMsr->u64Value = ASMRdMsr(MSR_K6_EFER);
1376 pMsr++; idxMsr++;
1377#endif
1378 }
1379
1380# if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
1381 if (VMX_IS_64BIT_HOST_MODE())
1382 {
1383 pMsr->u32IndexMSR = MSR_K6_STAR;
1384 pMsr->u32Reserved = 0;
1385 pMsr->u64Value = ASMRdMsr(MSR_K6_STAR); /* legacy syscall eip, cs & ss */
1386 pMsr++; idxMsr++;
1387 pMsr->u32IndexMSR = MSR_K8_LSTAR;
1388 pMsr->u32Reserved = 0;
1389 pMsr->u64Value = ASMRdMsr(MSR_K8_LSTAR); /* 64 bits mode syscall rip */
1390 pMsr++; idxMsr++;
1391 pMsr->u32IndexMSR = MSR_K8_SF_MASK;
1392 pMsr->u32Reserved = 0;
1393 pMsr->u64Value = ASMRdMsr(MSR_K8_SF_MASK); /* syscall flag mask */
1394 pMsr++; idxMsr++;
1395
1396 /* The KERNEL_GS_BASE MSR doesn't work reliably with auto load/store. See @bugref{6208} */
1397#if 0
1398 pMsr->u32IndexMSR = MSR_K8_KERNEL_GS_BASE;
1399 pMsr->u32Reserved = 0;
1400 pMsr->u64Value = ASMRdMsr(MSR_K8_KERNEL_GS_BASE); /* swapgs exchange value */
1401 pMsr++; idxMsr++;
1402#endif
1403 }
1404# endif
1405
1406 if (pVCpu->hm.s.vmx.proc_ctls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
1407 {
1408 pMsr->u32IndexMSR = MSR_K8_TSC_AUX;
1409 pMsr->u32Reserved = 0;
1410 pMsr->u64Value = ASMRdMsr(MSR_K8_TSC_AUX);
1411 pMsr++; idxMsr++;
1412 }
1413
1414 /** @todo r=ramshankar: check IA32_VMX_MISC bits 27:25 for valid idxMsr
1415 * range. */
1416 rc = VMXWriteVMCS(VMX_VMCS_CTRL_EXIT_MSR_LOAD_COUNT, idxMsr);
1417 AssertRC(rc);
1418#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
1419
1420 pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_HOST_CONTEXT;
1421 }
1422 return rc;
1423}
1424
1425
1426/**
1427 * Loads the 4 PDPEs into the guest state when nested paging is used and the
1428 * guest operates in PAE mode.
1429 *
1430 * @returns VBox status code.
1431 * @param pVCpu Pointer to the VMCPU.
1432 * @param pCtx Pointer to the guest CPU context.
1433 */
1434static int hmR0VmxLoadPaePdpes(PVMCPU pVCpu, PCPUMCTX pCtx)
1435{
1436 if (CPUMIsGuestInPAEModeEx(pCtx))
1437 {
1438 X86PDPE aPdpes[4];
1439 int rc = PGMGstGetPaePdpes(pVCpu, &aPdpes[0]);
1440 AssertRCReturn(rc, rc);
1441
1442 rc = VMXWriteVMCS64(VMX_VMCS_GUEST_PDPTR0_FULL, aPdpes[0].u); AssertRCReturn(rc, rc);
1443 rc = VMXWriteVMCS64(VMX_VMCS_GUEST_PDPTR1_FULL, aPdpes[1].u); AssertRCReturn(rc, rc);
1444 rc = VMXWriteVMCS64(VMX_VMCS_GUEST_PDPTR2_FULL, aPdpes[2].u); AssertRCReturn(rc, rc);
1445 rc = VMXWriteVMCS64(VMX_VMCS_GUEST_PDPTR3_FULL, aPdpes[3].u); AssertRCReturn(rc, rc);
1446 }
1447 return VINF_SUCCESS;
1448}
1449
1450
1451/**
1452 * Saves the 4 PDPEs into the guest state when nested paging is used and the
1453 * guest operates in PAE mode.
1454 *
1455 * @returns VBox status code.
1456 * @param pVCpu Pointer to the VM CPU.
1457 * @param pCtx Pointer to the guest CPU context.
1458 *
1459 * @remarks Tell PGM about CR3 changes before calling this helper.
1460 */
1461static int hmR0VmxSavePaePdpes(PVMCPU pVCpu, PCPUMCTX pCtx)
1462{
1463 if (CPUMIsGuestInPAEModeEx(pCtx))
1464 {
1465 int rc;
1466 X86PDPE aPdpes[4];
1467 rc = VMXReadVMCS64(VMX_VMCS_GUEST_PDPTR0_FULL, &aPdpes[0].u); AssertRCReturn(rc, rc);
1468 rc = VMXReadVMCS64(VMX_VMCS_GUEST_PDPTR1_FULL, &aPdpes[1].u); AssertRCReturn(rc, rc);
1469 rc = VMXReadVMCS64(VMX_VMCS_GUEST_PDPTR2_FULL, &aPdpes[2].u); AssertRCReturn(rc, rc);
1470 rc = VMXReadVMCS64(VMX_VMCS_GUEST_PDPTR3_FULL, &aPdpes[3].u); AssertRCReturn(rc, rc);
1471
1472 rc = PGMGstUpdatePaePdpes(pVCpu, &aPdpes[0]);
1473 AssertRCReturn(rc, rc);
1474 }
1475 return VINF_SUCCESS;
1476}
1477
1478
1479/**
1480 * Update the exception bitmap according to the current CPU state.
1481 *
1482 * @param pVM Pointer to the VM.
1483 * @param pVCpu Pointer to the VMCPU.
1484 * @param pCtx Pointer to the guest CPU context.
1485 */
1486static void hmR0VmxUpdateExceptionBitmap(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
1487{
1488 uint32_t u32TrapMask;
1489 Assert(pCtx);
1490
1491 /*
1492 * Set up a mask for intercepting traps.
1493 */
1494 /** @todo Do we really need to always intercept #DB? */
1495 u32TrapMask = RT_BIT(X86_XCPT_DB)
1496 | RT_BIT(X86_XCPT_NM)
1497#ifdef VBOX_ALWAYS_TRAP_PF
1498 | RT_BIT(X86_XCPT_PF)
1499#endif
1500#ifdef VBOX_STRICT
1501 | RT_BIT(X86_XCPT_BP)
1502 | RT_BIT(X86_XCPT_DB)
1503 | RT_BIT(X86_XCPT_DE)
1504 | RT_BIT(X86_XCPT_NM)
1505 | RT_BIT(X86_XCPT_UD)
1506 | RT_BIT(X86_XCPT_NP)
1507 | RT_BIT(X86_XCPT_SS)
1508 | RT_BIT(X86_XCPT_GP)
1509 | RT_BIT(X86_XCPT_MF)
1510#endif
1511 ;
1512
1513 /*
1514 * Without nested paging, #PF must be intercepted to implement shadow paging.
1515 */
1516 /** @todo NP state won't change so maybe we should build the initial trap mask up front? */
1517 if (!pVM->hm.s.fNestedPaging)
1518 u32TrapMask |= RT_BIT(X86_XCPT_PF);
1519
1520 /* Catch floating point exceptions if we need to report them to the guest in a different way. */
1521 if (!(pCtx->cr0 & X86_CR0_NE))
1522 u32TrapMask |= RT_BIT(X86_XCPT_MF);
1523
1524#ifdef VBOX_STRICT
1525 Assert(u32TrapMask & RT_BIT(X86_XCPT_GP));
1526#endif
1527
1528 /*
1529 * Intercept all exceptions in real mode as none of them can be injected directly (#GP otherwise).
1530 */
1531 /** @todo Despite the claim to intercept everything, with NP we do not intercept #PF. Should we? */
1532 if ( CPUMIsGuestInRealModeEx(pCtx)
1533 && pVM->hm.s.vmx.pRealModeTSS)
1534 {
1535 u32TrapMask |= RT_BIT(X86_XCPT_DE)
1536 | RT_BIT(X86_XCPT_DB)
1537 | RT_BIT(X86_XCPT_NMI)
1538 | RT_BIT(X86_XCPT_BP)
1539 | RT_BIT(X86_XCPT_OF)
1540 | RT_BIT(X86_XCPT_BR)
1541 | RT_BIT(X86_XCPT_UD)
1542 | RT_BIT(X86_XCPT_DF)
1543 | RT_BIT(X86_XCPT_CO_SEG_OVERRUN)
1544 | RT_BIT(X86_XCPT_TS)
1545 | RT_BIT(X86_XCPT_NP)
1546 | RT_BIT(X86_XCPT_SS)
1547 | RT_BIT(X86_XCPT_GP)
1548 | RT_BIT(X86_XCPT_MF)
1549 | RT_BIT(X86_XCPT_AC)
1550 | RT_BIT(X86_XCPT_MC)
1551 | RT_BIT(X86_XCPT_XF)
1552 ;
1553 }
1554
1555 int rc = VMXWriteVMCS(VMX_VMCS_CTRL_EXCEPTION_BITMAP, u32TrapMask);
1556 AssertRC(rc);
1557}
1558
1559
1560/**
1561 * Loads a minimal guest state.
1562 *
1563 * NOTE: Don't do anything here that can cause a jump back to ring 3!!!!!
1564 *
1565 * @param pVM Pointer to the VM.
1566 * @param pVCpu Pointer to the VMCPU.
1567 * @param pCtx Pointer to the guest CPU context.
1568 */
1569VMMR0DECL(void) VMXR0LoadMinimalGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
1570{
1571 int rc;
1572 X86EFLAGS eflags;
1573
1574 Assert(!(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_ALL_GUEST));
1575
1576 /*
1577 * Load EIP, ESP and EFLAGS.
1578 */
1579 rc = VMXWriteVMCS64(VMX_VMCS64_GUEST_RIP, pCtx->rip);
1580 rc |= VMXWriteVMCS64(VMX_VMCS64_GUEST_RSP, pCtx->rsp);
1581 AssertRC(rc);
1582
1583 /*
1584 * Bits 22-31, 15, 5 & 3 must be zero. Bit 1 must be 1.
1585 */
1586 eflags = pCtx->eflags;
1587 eflags.u32 &= VMX_EFLAGS_RESERVED_0;
1588 eflags.u32 |= VMX_EFLAGS_RESERVED_1;
1589
1590 /*
1591 * Check if real mode emulation using v86 mode.
1592 */
1593 if ( CPUMIsGuestInRealModeEx(pCtx)
1594 && pVM->hm.s.vmx.pRealModeTSS)
1595 {
1596 pVCpu->hm.s.vmx.RealMode.eflags = eflags;
1597
1598 eflags.Bits.u1VM = 1;
1599 eflags.Bits.u2IOPL = 0; /* must always be 0 or else certain instructions won't cause faults. */
1600 }
1601 rc = VMXWriteVMCS(VMX_VMCS_GUEST_RFLAGS, eflags.u32);
1602 AssertRC(rc);
1603}
1604
1605
1606/**
1607 * Loads the guest state.
1608 *
1609 * NOTE: Don't do anything here that can cause a jump back to ring 3!!!!!
1610 *
1611 * @returns VBox status code.
1612 * @param pVM Pointer to the VM.
1613 * @param pVCpu Pointer to the VMCPU.
1614 * @param pCtx Pointer to the guest CPU context.
1615 */
1616VMMR0DECL(int) VMXR0LoadGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
1617{
1618 int rc = VINF_SUCCESS;
1619 RTGCUINTPTR val;
1620
1621 /*
1622 * VMX_VMCS_CTRL_ENTRY_CONTROLS
1623 * Set required bits to one and zero according to the MSR capabilities.
1624 */
1625 val = pVM->hm.s.vmx.msr.vmx_entry.n.disallowed0;
1626
1627 /*
1628 * Load guest debug controls (DR7 & IA32_DEBUGCTL_MSR).
1629 * Forced to 1 on the 'first' VT-x capable CPUs; this actually includes the newest Nehalem CPUs
1630 */
1631 val |= VMX_VMCS_CTRL_ENTRY_CONTROLS_LOAD_DEBUG;
1632
1633 if (CPUMIsGuestInLongModeEx(pCtx))
1634 val |= VMX_VMCS_CTRL_ENTRY_CONTROLS_IA64_MODE;
1635 /* else Must be zero when AMD64 is not available. */
1636
1637 /*
1638 * Mask away the bits that the CPU doesn't support.
1639 */
1640 val &= pVM->hm.s.vmx.msr.vmx_entry.n.allowed1;
1641 rc = VMXWriteVMCS(VMX_VMCS_CTRL_ENTRY_CONTROLS, val);
1642 AssertRC(rc);
1643
1644 /*
1645 * VMX_VMCS_CTRL_EXIT_CONTROLS
1646 * Set required bits to one and zero according to the MSR capabilities.
1647 */
1648 val = pVM->hm.s.vmx.msr.vmx_exit.n.disallowed0;
1649
1650 /*
1651 * Save debug controls (DR7 & IA32_DEBUGCTL_MSR)
1652 * Forced to 1 on the 'first' VT-x capable CPUs; this actually includes the newest Nehalem CPUs
1653 */
1654 val |= VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_DEBUG;
1655
1656#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
1657 if (VMX_IS_64BIT_HOST_MODE())
1658 val |= VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_AMD64;
1659 /* else Must be zero when AMD64 is not available. */
1660#elif HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
1661 if (CPUMIsGuestInLongModeEx(pCtx))
1662 val |= VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_AMD64; /* our switcher goes to long mode */
1663 else
1664 Assert(!(val & VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_AMD64));
1665#endif
1666 val &= pVM->hm.s.vmx.msr.vmx_exit.n.allowed1;
1667
1668 /*
1669 * Don't acknowledge external interrupts on VM-exit.
1670 */
1671 rc = VMXWriteVMCS(VMX_VMCS_CTRL_EXIT_CONTROLS, val);
1672 AssertRC(rc);
1673
1674 /*
1675 * Guest CPU context: ES, CS, SS, DS, FS, GS.
1676 */
1677 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SEGMENT_REGS)
1678 {
1679 if (pVM->hm.s.vmx.pRealModeTSS)
1680 {
1681 PGMMODE enmGuestMode = PGMGetGuestMode(pVCpu);
1682 if (pVCpu->hm.s.vmx.enmLastSeenGuestMode != enmGuestMode)
1683 {
1684 /*
1685 * Correct weird requirements for switching to protected mode.
1686 */
1687 if ( pVCpu->hm.s.vmx.enmLastSeenGuestMode == PGMMODE_REAL
1688 && enmGuestMode >= PGMMODE_PROTECTED)
1689 {
1690#ifdef VBOX_WITH_REM
1691 /*
1692 * Flush the recompiler code cache as it's not unlikely the guest will rewrite code
1693 * it will later execute in real mode (OpenBSD 4.0 is one such example)
1694 */
1695 REMFlushTBs(pVM);
1696#endif
1697
1698 /*
1699 * DPL of all hidden selector registers must match the current CPL (0).
1700 */
1701 pCtx->cs.Attr.n.u2Dpl = 0;
1702 pCtx->cs.Attr.n.u4Type = X86_SEL_TYPE_CODE | X86_SEL_TYPE_RW_ACC;
1703
1704 pCtx->ds.Attr.n.u2Dpl = 0;
1705 pCtx->es.Attr.n.u2Dpl = 0;
1706 pCtx->fs.Attr.n.u2Dpl = 0;
1707 pCtx->gs.Attr.n.u2Dpl = 0;
1708 pCtx->ss.Attr.n.u2Dpl = 0;
1709 }
1710 pVCpu->hm.s.vmx.enmLastSeenGuestMode = enmGuestMode;
1711 }
1712 else if ( CPUMIsGuestInRealModeEx(pCtx)
1713 && pCtx->cs.u64Base == 0xffff0000)
1714 {
1715 /* VT-x will fail with a guest invalid state otherwise... (CPU state after a reset) */
1716 pCtx->cs.u64Base = 0xf0000;
1717 pCtx->cs.Sel = 0xf000;
1718 }
1719 }
1720
1721 VMX_WRITE_SELREG(ES, es);
1722 AssertRC(rc);
1723
1724 VMX_WRITE_SELREG(CS, cs);
1725 AssertRC(rc);
1726
1727 VMX_WRITE_SELREG(SS, ss);
1728 AssertRC(rc);
1729
1730 VMX_WRITE_SELREG(DS, ds);
1731 AssertRC(rc);
1732
1733 VMX_WRITE_SELREG(FS, fs);
1734 AssertRC(rc);
1735
1736 VMX_WRITE_SELREG(GS, gs);
1737 AssertRC(rc);
1738 }
1739
1740 /*
1741 * Guest CPU context: LDTR.
1742 */
1743 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_LDTR)
1744 {
1745 if (pCtx->ldtr.Sel == 0)
1746 {
1747 rc = VMXWriteVMCS(VMX_VMCS16_GUEST_FIELD_LDTR, 0);
1748 rc |= VMXWriteVMCS(VMX_VMCS32_GUEST_LDTR_LIMIT, 0);
1749 rc |= VMXWriteVMCS64(VMX_VMCS64_GUEST_LDTR_BASE, 0);
1750 /* Note: vmlaunch will fail with 0 or just 0x02. No idea why. */
1751 rc |= VMXWriteVMCS(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, 0x82 /* present, LDT */);
1752 }
1753 else
1754 {
1755 rc = VMXWriteVMCS(VMX_VMCS16_GUEST_FIELD_LDTR, pCtx->ldtr.Sel);
1756 rc |= VMXWriteVMCS(VMX_VMCS32_GUEST_LDTR_LIMIT, pCtx->ldtr.u32Limit);
1757 rc |= VMXWriteVMCS64(VMX_VMCS64_GUEST_LDTR_BASE, pCtx->ldtr.u64Base);
1758 rc |= VMXWriteVMCS(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, pCtx->ldtr.Attr.u);
1759 }
1760 AssertRC(rc);
1761 }
1762
1763 /*
1764 * Guest CPU context: TR.
1765 */
1766 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_TR)
1767 {
1768 /*
1769 * Real mode emulation using v86 mode with CR4.VME (interrupt redirection
1770 * using the int bitmap in the TSS).
1771 */
1772 if ( CPUMIsGuestInRealModeEx(pCtx)
1773 && pVM->hm.s.vmx.pRealModeTSS)
1774 {
1775 RTGCPHYS GCPhys;
1776
1777 /* We convert it here every time as PCI regions could be reconfigured. */
1778 rc = PDMVMMDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
1779 AssertRC(rc);
1780
1781 rc = VMXWriteVMCS(VMX_VMCS16_GUEST_FIELD_TR, 0);
1782 rc |= VMXWriteVMCS(VMX_VMCS32_GUEST_TR_LIMIT, HM_VTX_TSS_SIZE);
1783 rc |= VMXWriteVMCS64(VMX_VMCS64_GUEST_TR_BASE, GCPhys /* phys = virt in this mode */);
1784
1785 X86DESCATTR attr;
1786
1787 attr.u = 0;
1788 attr.n.u1Present = 1;
1789 attr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
1790 val = attr.u;
1791 }
1792 else
1793 {
1794 rc = VMXWriteVMCS(VMX_VMCS16_GUEST_FIELD_TR, pCtx->tr.Sel);
1795 rc |= VMXWriteVMCS(VMX_VMCS32_GUEST_TR_LIMIT, pCtx->tr.u32Limit);
1796 rc |= VMXWriteVMCS64(VMX_VMCS64_GUEST_TR_BASE, pCtx->tr.u64Base);
1797
1798 val = pCtx->tr.Attr.u;
1799
1800 /* The TSS selector must be busy (REM bugs? see defect #XXXX). */
1801 if (!(val & X86_SEL_TYPE_SYS_TSS_BUSY_MASK))
1802 {
1803 if (val & 0xf)
1804 val |= X86_SEL_TYPE_SYS_TSS_BUSY_MASK;
1805 else
1806 /* Default if no TR selector has been set (otherwise vmlaunch will fail!) */
1807 val = (val & ~0xF) | X86_SEL_TYPE_SYS_386_TSS_BUSY;
1808 }
1809 AssertMsg((val & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY || (val & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY,
1810 ("%#x\n", val));
1811 }
1812 rc |= VMXWriteVMCS(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, val);
1813 AssertRC(rc);
1814 }
1815
1816 /*
1817 * Guest CPU context: GDTR.
1818 */
1819 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_GDTR)
1820 {
1821 rc = VMXWriteVMCS(VMX_VMCS32_GUEST_GDTR_LIMIT, pCtx->gdtr.cbGdt);
1822 rc |= VMXWriteVMCS64(VMX_VMCS64_GUEST_GDTR_BASE, pCtx->gdtr.pGdt);
1823 AssertRC(rc);
1824 }
1825
1826 /*
1827 * Guest CPU context: IDTR.
1828 */
1829 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_IDTR)
1830 {
1831 rc = VMXWriteVMCS(VMX_VMCS32_GUEST_IDTR_LIMIT, pCtx->idtr.cbIdt);
1832 rc |= VMXWriteVMCS64(VMX_VMCS64_GUEST_IDTR_BASE, pCtx->idtr.pIdt);
1833 AssertRC(rc);
1834 }
1835
1836 /*
1837 * Sysenter MSRs.
1838 */
1839 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_MSR)
1840 {
1841 rc = VMXWriteVMCS(VMX_VMCS32_GUEST_SYSENTER_CS, pCtx->SysEnter.cs);
1842 rc |= VMXWriteVMCS64(VMX_VMCS64_GUEST_SYSENTER_EIP, pCtx->SysEnter.eip);
1843 rc |= VMXWriteVMCS64(VMX_VMCS64_GUEST_SYSENTER_ESP, pCtx->SysEnter.esp);
1844 AssertRC(rc);
1845 }
1846
1847 /*
1848 * Guest CPU context: Control registers.
1849 */
1850 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR0)
1851 {
1852 val = pCtx->cr0;
1853 rc = VMXWriteVMCS(VMX_VMCS_CTRL_CR0_READ_SHADOW, val);
1854 Log2(("Guest CR0-shadow %08x\n", val));
1855 if (CPUMIsGuestFPUStateActive(pVCpu) == false)
1856 {
1857 /* Always use #NM exceptions to load the FPU/XMM state on demand. */
1858 val |= X86_CR0_TS | X86_CR0_ET | X86_CR0_NE | X86_CR0_MP;
1859 }
1860 else
1861 {
1862 /** @todo check if we support the old style mess correctly. */
1863 if (!(val & X86_CR0_NE))
1864 Log(("Forcing X86_CR0_NE!!!\n"));
1865
1866 val |= X86_CR0_NE; /* always turn on the native mechanism to report FPU errors (old style uses interrupts) */
1867 }
1868 /* Protected mode & paging are always enabled; we use them for emulating real and protected mode without paging too. */
1869 if (!pVM->hm.s.vmx.fUnrestrictedGuest)
1870 val |= X86_CR0_PE | X86_CR0_PG;
1871
1872 if (pVM->hm.s.fNestedPaging)
1873 {
1874 if (CPUMIsGuestInPagedProtectedModeEx(pCtx))
1875 {
1876 /* Disable CR3 read/write monitoring as we don't need it for EPT. */
1877 pVCpu->hm.s.vmx.proc_ctls &= ~( VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
1878 | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT);
1879 }
1880 else
1881 {
1882 /* Reenable CR3 read/write monitoring as our identity mapped page table is active. */
1883 pVCpu->hm.s.vmx.proc_ctls |= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
1884 | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT;
1885 }
1886 rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.proc_ctls);
1887 AssertRC(rc);
1888 }
1889 else
1890 {
1891 /* Note: We must also set this as we rely on protecting various pages for which supervisor writes must be caught. */
1892 val |= X86_CR0_WP;
1893 }
1894
1895 /* Always enable caching. */
1896 val &= ~(X86_CR0_CD|X86_CR0_NW);
1897
1898 rc |= VMXWriteVMCS64(VMX_VMCS64_GUEST_CR0, val);
1899 Log2(("Guest CR0 %08x\n", val));
1900
1901 /*
1902 * CR0 flags owned by the host; if the guests attempts to change them, then the VM will exit.
1903 */
1904 val = X86_CR0_PE /* Must monitor this bit (assumptions are made for real mode emulation) */
1905 | X86_CR0_WP /* Must monitor this bit (it must always be enabled). */
1906 | X86_CR0_PG /* Must monitor this bit (assumptions are made for real mode & protected mode without paging emulation) */
1907 | X86_CR0_CD /* Bit not restored during VM-exit! */
1908 | X86_CR0_NW /* Bit not restored during VM-exit! */
1909 | X86_CR0_NE;
1910
1911 /*
1912 * When the guest's FPU state is active, then we no longer care about the FPU related bits.
1913 */
1914 if (CPUMIsGuestFPUStateActive(pVCpu) == false)
1915 val |= X86_CR0_TS | X86_CR0_ET | X86_CR0_MP;
1916
1917 pVCpu->hm.s.vmx.cr0_mask = val;
1918
1919 rc |= VMXWriteVMCS(VMX_VMCS_CTRL_CR0_MASK, val);
1920 Log2(("Guest CR0-mask %08x\n", val));
1921 AssertRC(rc);
1922 }
1923
1924 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR4)
1925 {
1926 rc = VMXWriteVMCS(VMX_VMCS_CTRL_CR4_READ_SHADOW, pCtx->cr4);
1927 Log2(("Guest CR4-shadow %08x\n", pCtx->cr4));
1928 /* Set the required bits in cr4 too (currently X86_CR4_VMXE). */
1929 val = pCtx->cr4 | (uint32_t)pVM->hm.s.vmx.msr.vmx_cr4_fixed0;
1930
1931 if (!pVM->hm.s.fNestedPaging)
1932 {
1933 switch (pVCpu->hm.s.enmShadowMode)
1934 {
1935 case PGMMODE_REAL: /* Real mode -> emulated using v86 mode */
1936 case PGMMODE_PROTECTED: /* Protected mode, no paging -> emulated using identity mapping. */
1937 case PGMMODE_32_BIT: /* 32-bit paging. */
1938 val &= ~X86_CR4_PAE;
1939 break;
1940
1941 case PGMMODE_PAE: /* PAE paging. */
1942 case PGMMODE_PAE_NX: /* PAE paging with NX enabled. */
1943 /** Must use PAE paging as we could use physical memory > 4 GB */
1944 val |= X86_CR4_PAE;
1945 break;
1946
1947 case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
1948 case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
1949#ifdef VBOX_ENABLE_64_BITS_GUESTS
1950 break;
1951#else
1952 AssertFailed();
1953 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
1954#endif
1955 default: /* shut up gcc */
1956 AssertFailed();
1957 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
1958 }
1959 }
1960 else if ( !CPUMIsGuestInPagedProtectedModeEx(pCtx)
1961 && !pVM->hm.s.vmx.fUnrestrictedGuest)
1962 {
1963 /* We use 4 MB pages in our identity mapping page table for real and protected mode without paging. */
1964 val |= X86_CR4_PSE;
1965 /* Our identity mapping is a 32 bits page directory. */
1966 val &= ~X86_CR4_PAE;
1967 }
1968
1969 /*
1970 * Turn off VME if we're in emulated real mode.
1971 */
1972 if ( CPUMIsGuestInRealModeEx(pCtx)
1973 && pVM->hm.s.vmx.pRealModeTSS)
1974 {
1975 val &= ~X86_CR4_VME;
1976 }
1977
1978 rc |= VMXWriteVMCS64(VMX_VMCS64_GUEST_CR4, val);
1979 Log2(("Guest CR4 %08x\n", val));
1980
1981 /*
1982 * CR4 flags owned by the host; if the guests attempts to change them, then the VM will exit.
1983 */
1984 val = 0
1985 | X86_CR4_VME
1986 | X86_CR4_PAE
1987 | X86_CR4_PGE
1988 | X86_CR4_PSE
1989 | X86_CR4_VMXE;
1990 pVCpu->hm.s.vmx.cr4_mask = val;
1991
1992 rc |= VMXWriteVMCS(VMX_VMCS_CTRL_CR4_MASK, val);
1993 Log2(("Guest CR4-mask %08x\n", val));
1994 AssertRC(rc);
1995 }
1996
1997#if 0
1998 /* Enable single stepping if requested and CPU supports it. */
1999 if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_TRAP_FLAG)
2000 if (DBGFIsStepping(pVCpu))
2001 {
2002 pVCpu->hm.s.vmx.proc_ctls |= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_TRAP_FLAG;
2003 rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.proc_ctls);
2004 AssertRC(rc);
2005 }
2006#endif
2007
2008 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR3)
2009 {
2010 if (pVM->hm.s.fNestedPaging)
2011 {
2012 Assert(PGMGetHyperCR3(pVCpu));
2013 pVCpu->hm.s.vmx.GCPhysEPTP = PGMGetHyperCR3(pVCpu);
2014
2015 Assert(!(pVCpu->hm.s.vmx.GCPhysEPTP & 0xfff));
2016 /** @todo Check the IA32_VMX_EPT_VPID_CAP MSR for other supported memory types. */
2017 pVCpu->hm.s.vmx.GCPhysEPTP |= VMX_EPT_MEMTYPE_WB
2018 | (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
2019
2020 rc = VMXWriteVMCS64(VMX_VMCS_CTRL_EPTP_FULL, pVCpu->hm.s.vmx.GCPhysEPTP);
2021 AssertRC(rc);
2022
2023 if ( !CPUMIsGuestInPagedProtectedModeEx(pCtx)
2024 && !pVM->hm.s.vmx.fUnrestrictedGuest)
2025 {
2026 RTGCPHYS GCPhys;
2027
2028 /* We convert it here every time as PCI regions could be reconfigured. */
2029 rc = PDMVMMDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
2030 AssertMsgRC(rc, ("pNonPagingModeEPTPageTable = %RGv\n", pVM->hm.s.vmx.pNonPagingModeEPTPageTable));
2031
2032 /*
2033 * We use our identity mapping page table here as we need to map guest virtual to
2034 * guest physical addresses; EPT will take care of the translation to host physical addresses.
2035 */
2036 val = GCPhys;
2037 }
2038 else
2039 {
2040 /* Save the real guest CR3 in VMX_VMCS_GUEST_CR3 */
2041 val = pCtx->cr3;
2042 rc = hmR0VmxLoadPaePdpes(pVCpu, pCtx);
2043 AssertRCReturn(rc, rc);
2044 }
2045 }
2046 else
2047 {
2048 val = PGMGetHyperCR3(pVCpu);
2049 Assert(val || VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL));
2050 }
2051
2052 /* Save our shadow CR3 register. */
2053 rc = VMXWriteVMCS64(VMX_VMCS64_GUEST_CR3, val);
2054 AssertRC(rc);
2055 }
2056
2057 /*
2058 * Guest CPU context: Debug registers.
2059 */
2060 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_DEBUG)
2061 {
2062 pCtx->dr[6] |= X86_DR6_INIT_VAL; /* set all reserved bits to 1. */
2063 pCtx->dr[6] &= ~RT_BIT(12); /* must be zero. */
2064
2065 pCtx->dr[7] &= 0xffffffff; /* upper 32 bits reserved */
2066 pCtx->dr[7] &= ~(RT_BIT(11) | RT_BIT(12) | RT_BIT(14) | RT_BIT(15)); /* must be zero */
2067 pCtx->dr[7] |= 0x400; /* must be one */
2068
2069 /* Resync DR7 */
2070 rc = VMXWriteVMCS64(VMX_VMCS64_GUEST_DR7, pCtx->dr[7]);
2071 AssertRC(rc);
2072
2073#ifdef DEBUG
2074 /* Sync the hypervisor debug state now if any breakpoint is armed. */
2075 if ( CPUMGetHyperDR7(pVCpu) & (X86_DR7_ENABLED_MASK|X86_DR7_GD)
2076 && !CPUMIsHyperDebugStateActive(pVCpu)
2077 && !DBGFIsStepping(pVCpu))
2078 {
2079 /* Save the host and load the hypervisor debug state. */
2080 rc = CPUMR0LoadHyperDebugState(pVM, pVCpu, pCtx, true /* include DR6 */);
2081 AssertRC(rc);
2082
2083 /* DRx intercepts remain enabled. */
2084
2085 /* Override dr7 with the hypervisor value. */
2086 rc = VMXWriteVMCS64(VMX_VMCS64_GUEST_DR7, CPUMGetHyperDR7(pVCpu));
2087 AssertRC(rc);
2088 }
2089 else
2090#endif
2091 /* Sync the debug state now if any breakpoint is armed. */
2092 if ( (pCtx->dr[7] & (X86_DR7_ENABLED_MASK|X86_DR7_GD))
2093 && !CPUMIsGuestDebugStateActive(pVCpu)
2094 && !DBGFIsStepping(pVCpu))
2095 {
2096 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
2097
2098 /* Disable DRx move intercepts. */
2099 pVCpu->hm.s.vmx.proc_ctls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT;
2100 rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.proc_ctls);
2101 AssertRC(rc);
2102
2103 /* Save the host and load the guest debug state. */
2104 rc = CPUMR0LoadGuestDebugState(pVM, pVCpu, pCtx, true /* include DR6 */);
2105 AssertRC(rc);
2106 }
2107
2108 /* IA32_DEBUGCTL MSR. */
2109 rc = VMXWriteVMCS64(VMX_VMCS_GUEST_DEBUGCTL_FULL, 0);
2110 AssertRC(rc);
2111
2112 /** @todo do we really ever need this? */
2113 rc |= VMXWriteVMCS(VMX_VMCS_GUEST_DEBUG_EXCEPTIONS, 0);
2114 AssertRC(rc);
2115 }
2116
2117 /*
2118 * 64-bit guest mode.
2119 */
2120 if (CPUMIsGuestInLongModeEx(pCtx))
2121 {
2122#if !defined(VBOX_ENABLE_64_BITS_GUESTS)
2123 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
2124#elif HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2125 pVCpu->hm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
2126#else
2127# ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2128 if (!pVM->hm.s.fAllow64BitGuests)
2129 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
2130# endif
2131 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM64;
2132#endif
2133 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_MSR)
2134 {
2135 /* Update these as wrmsr might have changed them. */
2136 rc = VMXWriteVMCS64(VMX_VMCS64_GUEST_FS_BASE, pCtx->fs.u64Base);
2137 AssertRC(rc);
2138 rc = VMXWriteVMCS64(VMX_VMCS64_GUEST_GS_BASE, pCtx->gs.u64Base);
2139 AssertRC(rc);
2140 }
2141 }
2142 else
2143 {
2144 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
2145 }
2146
2147 hmR0VmxUpdateExceptionBitmap(pVM, pVCpu, pCtx);
2148
2149#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
2150 /*
2151 * Store all guest MSRs in the VM-entry load area, so they will be loaded
2152 * during VM-entry and restored into the VM-exit store area during VM-exit.
2153 */
2154 PVMXMSR pMsr = (PVMXMSR)pVCpu->hm.s.vmx.pvGuestMsr;
2155 unsigned idxMsr = 0;
2156
2157 uint32_t u32GstExtFeatures;
2158 uint32_t u32Temp;
2159 CPUMGetGuestCpuId(pVCpu, 0x80000001, &u32Temp, &u32Temp, &u32Temp, &u32GstExtFeatures);
2160
2161 if (u32GstExtFeatures & (X86_CPUID_EXT_FEATURE_EDX_NX | X86_CPUID_EXT_FEATURE_EDX_LONG_MODE))
2162 {
2163#if 0
2164 pMsr->u32IndexMSR = MSR_K6_EFER;
2165 pMsr->u32Reserved = 0;
2166 pMsr->u64Value = pCtx->msrEFER;
2167 /* VT-x will complain if only MSR_K6_EFER_LME is set. */
2168 if (!CPUMIsGuestInLongModeEx(pCtx))
2169 pMsr->u64Value &= ~(MSR_K6_EFER_LMA | MSR_K6_EFER_LME);
2170 pMsr++; idxMsr++;
2171#endif
2172
2173 if (u32GstExtFeatures & X86_CPUID_EXT_FEATURE_EDX_LONG_MODE)
2174 {
2175 pMsr->u32IndexMSR = MSR_K8_LSTAR;
2176 pMsr->u32Reserved = 0;
2177 pMsr->u64Value = pCtx->msrLSTAR; /* 64 bits mode syscall rip */
2178 pMsr++; idxMsr++;
2179 pMsr->u32IndexMSR = MSR_K6_STAR;
2180 pMsr->u32Reserved = 0;
2181 pMsr->u64Value = pCtx->msrSTAR; /* legacy syscall eip, cs & ss */
2182 pMsr++; idxMsr++;
2183 pMsr->u32IndexMSR = MSR_K8_SF_MASK;
2184 pMsr->u32Reserved = 0;
2185 pMsr->u64Value = pCtx->msrSFMASK; /* syscall flag mask */
2186 pMsr++; idxMsr++;
2187
2188 /* The KERNEL_GS_BASE MSR doesn't work reliably with auto load/store. See @bugref{6208} */
2189#if 0
2190 pMsr->u32IndexMSR = MSR_K8_KERNEL_GS_BASE;
2191 pMsr->u32Reserved = 0;
2192 pMsr->u64Value = pCtx->msrKERNELGSBASE; /* swapgs exchange value */
2193 pMsr++; idxMsr++;
2194#endif
2195 }
2196 }
2197
2198 if ( pVCpu->hm.s.vmx.proc_ctls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP
2199 && (u32GstExtFeatures & X86_CPUID_EXT_FEATURE_EDX_RDTSCP))
2200 {
2201 pMsr->u32IndexMSR = MSR_K8_TSC_AUX;
2202 pMsr->u32Reserved = 0;
2203 rc = CPUMQueryGuestMsr(pVCpu, MSR_K8_TSC_AUX, &pMsr->u64Value);
2204 AssertRC(rc);
2205 pMsr++; idxMsr++;
2206 }
2207
2208 pVCpu->hm.s.vmx.cCachedMsrs = idxMsr;
2209
2210 rc = VMXWriteVMCS(VMX_VMCS_CTRL_ENTRY_MSR_LOAD_COUNT, idxMsr);
2211 AssertRC(rc);
2212
2213 rc = VMXWriteVMCS(VMX_VMCS_CTRL_EXIT_MSR_STORE_COUNT, idxMsr);
2214 AssertRC(rc);
2215#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
2216
2217 bool fOffsettedTsc;
2218 if (pVM->hm.s.vmx.fUsePreemptTimer)
2219 {
2220 uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVCpu, &fOffsettedTsc, &pVCpu->hm.s.vmx.u64TSCOffset);
2221
2222 /* Make sure the returned values have sane upper and lower boundaries. */
2223 uint64_t u64CpuHz = SUPGetCpuHzFromGIP(g_pSUPGlobalInfoPage);
2224
2225 cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64 of a second */
2226 cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
2227
2228 cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
2229 uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
2230 rc = VMXWriteVMCS(VMX_VMCS32_GUEST_PREEMPTION_TIMER_VALUE, cPreemptionTickCount);
2231 AssertRC(rc);
2232 }
2233 else
2234 fOffsettedTsc = TMCpuTickCanUseRealTSC(pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset);
2235
2236 if (fOffsettedTsc)
2237 {
2238 uint64_t u64CurTSC = ASMReadTSC();
2239 if (u64CurTSC + pVCpu->hm.s.vmx.u64TSCOffset >= TMCpuTickGetLastSeen(pVCpu))
2240 {
2241 /* Note: VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT takes precedence over TSC_OFFSET, applies to RDTSCP too. */
2242 rc = VMXWriteVMCS64(VMX_VMCS_CTRL_TSC_OFFSET_FULL, pVCpu->hm.s.vmx.u64TSCOffset);
2243 AssertRC(rc);
2244
2245 pVCpu->hm.s.vmx.proc_ctls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT;
2246 rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.proc_ctls);
2247 AssertRC(rc);
2248 STAM_COUNTER_INC(&pVCpu->hm.s.StatTSCOffset);
2249 }
2250 else
2251 {
2252 /* Fall back to rdtsc, rdtscp emulation as we would otherwise pass decreasing tsc values to the guest. */
2253 LogFlow(("TSC %RX64 offset %RX64 time=%RX64 last=%RX64 (diff=%RX64, virt_tsc=%RX64)\n", u64CurTSC,
2254 pVCpu->hm.s.vmx.u64TSCOffset, u64CurTSC + pVCpu->hm.s.vmx.u64TSCOffset,
2255 TMCpuTickGetLastSeen(pVCpu), TMCpuTickGetLastSeen(pVCpu) - u64CurTSC - pVCpu->hm.s.vmx.u64TSCOffset,
2256 TMCpuTickGet(pVCpu)));
2257 pVCpu->hm.s.vmx.proc_ctls |= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT;
2258 rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.proc_ctls);
2259 AssertRC(rc);
2260 STAM_COUNTER_INC(&pVCpu->hm.s.StatTSCInterceptOverFlow);
2261 }
2262 }
2263 else
2264 {
2265 pVCpu->hm.s.vmx.proc_ctls |= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT;
2266 rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.proc_ctls);
2267 AssertRC(rc);
2268 STAM_COUNTER_INC(&pVCpu->hm.s.StatTSCIntercept);
2269 }
2270
2271 /* Done with the major changes */
2272 pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_ALL_GUEST;
2273
2274 /* Minimal guest state update (ESP, EIP, EFLAGS mostly) */
2275 VMXR0LoadMinimalGuestState(pVM, pVCpu, pCtx);
2276 return rc;
2277}
2278
2279
2280/**
2281 * Syncs back the guest state from VMCS.
2282 *
2283 * @returns VBox status code.
2284 * @param pVM Pointer to the VM.
2285 * @param pVCpu Pointer to the VMCPU.
2286 * @param pCtx Pointer to the guest CPU context.
2287 */
2288DECLINLINE(int) VMXR0SaveGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
2289{
2290 RTGCUINTREG val, valShadow;
2291 RTGCUINTPTR uInterruptState;
2292 int rc;
2293
2294 /* First sync back EIP, ESP, and EFLAGS. */
2295 rc = VMXReadCachedVMCS(VMX_VMCS64_GUEST_RIP, &val);
2296 AssertRC(rc);
2297 pCtx->rip = val;
2298 rc = VMXReadCachedVMCS(VMX_VMCS64_GUEST_RSP, &val);
2299 AssertRC(rc);
2300 pCtx->rsp = val;
2301 rc = VMXReadCachedVMCS(VMX_VMCS_GUEST_RFLAGS, &val);
2302 AssertRC(rc);
2303 pCtx->eflags.u32 = val;
2304
2305 /* Take care of instruction fusing (sti, mov ss) */
2306 rc |= VMXReadCachedVMCS(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &val);
2307 uInterruptState = val;
2308 if (uInterruptState != 0)
2309 {
2310 Assert(uInterruptState <= 2); /* only sti & mov ss */
2311 Log(("uInterruptState %x eip=%RGv\n", (uint32_t)uInterruptState, pCtx->rip));
2312 EMSetInhibitInterruptsPC(pVCpu, pCtx->rip);
2313 }
2314 else
2315 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
2316
2317 /* Control registers. */
2318 VMXReadCachedVMCS(VMX_VMCS_CTRL_CR0_READ_SHADOW, &valShadow);
2319 VMXReadCachedVMCS(VMX_VMCS64_GUEST_CR0, &val);
2320 val = (valShadow & pVCpu->hm.s.vmx.cr0_mask) | (val & ~pVCpu->hm.s.vmx.cr0_mask);
2321 CPUMSetGuestCR0(pVCpu, val);
2322
2323 VMXReadCachedVMCS(VMX_VMCS_CTRL_CR4_READ_SHADOW, &valShadow);
2324 VMXReadCachedVMCS(VMX_VMCS64_GUEST_CR4, &val);
2325 val = (valShadow & pVCpu->hm.s.vmx.cr4_mask) | (val & ~pVCpu->hm.s.vmx.cr4_mask);
2326 CPUMSetGuestCR4(pVCpu, val);
2327
2328 /*
2329 * No reason to sync back the CRx registers. They can't be changed by the guest unless in
2330 * the nested paging case where CR3 & CR4 can be changed by the guest.
2331 */
2332 if ( pVM->hm.s.fNestedPaging
2333 && CPUMIsGuestInPagedProtectedModeEx(pCtx)) /** @todo check if we will always catch mode switches and such... */
2334 {
2335 PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
2336
2337 /* Can be updated behind our back in the nested paging case. */
2338 CPUMSetGuestCR2(pVCpu, pCache->cr2);
2339
2340 VMXReadCachedVMCS(VMX_VMCS64_GUEST_CR3, &val);
2341
2342 if (val != pCtx->cr3)
2343 {
2344 CPUMSetGuestCR3(pVCpu, val);
2345 PGMUpdateCR3(pVCpu, val);
2346 }
2347 rc = hmR0VmxSavePaePdpes(pVCpu, pCtx);
2348 AssertRCReturn(rc, rc);
2349 }
2350
2351 /* Sync back DR7. */
2352 VMXReadCachedVMCS(VMX_VMCS64_GUEST_DR7, &val);
2353 pCtx->dr[7] = val;
2354
2355 /* Guest CPU context: ES, CS, SS, DS, FS, GS. */
2356 VMX_READ_SELREG(ES, es);
2357 VMX_READ_SELREG(SS, ss);
2358 VMX_READ_SELREG(CS, cs);
2359 VMX_READ_SELREG(DS, ds);
2360 VMX_READ_SELREG(FS, fs);
2361 VMX_READ_SELREG(GS, gs);
2362
2363 /* System MSRs */
2364 VMXReadCachedVMCS(VMX_VMCS32_GUEST_SYSENTER_CS, &val);
2365 pCtx->SysEnter.cs = val;
2366 VMXReadCachedVMCS(VMX_VMCS64_GUEST_SYSENTER_EIP, &val);
2367 pCtx->SysEnter.eip = val;
2368 VMXReadCachedVMCS(VMX_VMCS64_GUEST_SYSENTER_ESP, &val);
2369 pCtx->SysEnter.esp = val;
2370
2371 /* Misc. registers; must sync everything otherwise we can get out of sync when jumping to ring 3. */
2372 VMX_READ_SELREG(LDTR, ldtr);
2373
2374 VMXReadCachedVMCS(VMX_VMCS32_GUEST_GDTR_LIMIT, &val);
2375 pCtx->gdtr.cbGdt = val;
2376 VMXReadCachedVMCS(VMX_VMCS64_GUEST_GDTR_BASE, &val);
2377 pCtx->gdtr.pGdt = val;
2378
2379 VMXReadCachedVMCS(VMX_VMCS32_GUEST_IDTR_LIMIT, &val);
2380 pCtx->idtr.cbIdt = val;
2381 VMXReadCachedVMCS(VMX_VMCS64_GUEST_IDTR_BASE, &val);
2382 pCtx->idtr.pIdt = val;
2383
2384 /* Real mode emulation using v86 mode. */
2385 if ( CPUMIsGuestInRealModeEx(pCtx)
2386 && pVM->hm.s.vmx.pRealModeTSS)
2387 {
2388 /* Hide our emulation flags */
2389 pCtx->eflags.Bits.u1VM = 0;
2390
2391 /* Restore original IOPL setting as we always use 0. */
2392 pCtx->eflags.Bits.u2IOPL = pVCpu->hm.s.vmx.RealMode.eflags.Bits.u2IOPL;
2393
2394 /* Force a TR resync every time in case we switch modes. */
2395 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_TR;
2396 }
2397 else
2398 {
2399 /* In real mode we have a fake TSS, so only sync it back when it's supposed to be valid. */
2400 VMX_READ_SELREG(TR, tr);
2401 }
2402
2403#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
2404 /*
2405 * Save the possibly changed MSRs that we automatically restore and save during a world switch.
2406 */
2407 for (unsigned i = 0; i < pVCpu->hm.s.vmx.cCachedMsrs; i++)
2408 {
2409 PVMXMSR pMsr = (PVMXMSR)pVCpu->hm.s.vmx.pvGuestMsr;
2410 pMsr += i;
2411
2412 switch (pMsr->u32IndexMSR)
2413 {
2414 case MSR_K8_LSTAR:
2415 pCtx->msrLSTAR = pMsr->u64Value;
2416 break;
2417 case MSR_K6_STAR:
2418 pCtx->msrSTAR = pMsr->u64Value;
2419 break;
2420 case MSR_K8_SF_MASK:
2421 pCtx->msrSFMASK = pMsr->u64Value;
2422 break;
2423 /* The KERNEL_GS_BASE MSR doesn't work reliably with auto load/store. See @bugref{6208} */
2424#if 0
2425 case MSR_K8_KERNEL_GS_BASE:
2426 pCtx->msrKERNELGSBASE = pMsr->u64Value;
2427 break;
2428#endif
2429 case MSR_K8_TSC_AUX:
2430 CPUMSetGuestMsr(pVCpu, MSR_K8_TSC_AUX, pMsr->u64Value);
2431 break;
2432#if 0
2433 case MSR_K6_EFER:
2434 /* EFER can't be changed without causing a VM-exit. */
2435 /* Assert(pCtx->msrEFER == pMsr->u64Value); */
2436 break;
2437#endif
2438 default:
2439 AssertFailed();
2440 return VERR_HM_UNEXPECTED_LD_ST_MSR;
2441 }
2442 }
2443#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
2444 return VINF_SUCCESS;
2445}
2446
2447
2448/**
2449 * Dummy placeholder for TLB flush handling before VM-entry. Used in the case
2450 * where neither EPT nor VPID is supported by the CPU.
2451 *
2452 * @param pVM Pointer to the VM.
2453 * @param pVCpu Pointer to the VMCPU.
2454 */
2455static DECLCALLBACK(void) hmR0VmxSetupTLBDummy(PVM pVM, PVMCPU pVCpu)
2456{
2457 NOREF(pVM);
2458 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
2459 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
2460 pVCpu->hm.s.TlbShootdown.cPages = 0;
2461 return;
2462}
2463
2464
2465/**
2466 * Setup the tagged TLB for EPT+VPID.
2467 *
2468 * @param pVM Pointer to the VM.
2469 * @param pVCpu Pointer to the VMCPU.
2470 */
2471static DECLCALLBACK(void) hmR0VmxSetupTLBBoth(PVM pVM, PVMCPU pVCpu)
2472{
2473 PHMGLOBLCPUINFO pCpu;
2474
2475 Assert(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid);
2476
2477 pCpu = HMR0GetCurrentCpu();
2478
2479 /*
2480 * Force a TLB flush for the first world switch if the current CPU differs from the one we ran on last
2481 * This can happen both for start & resume due to long jumps back to ring-3.
2482 * If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
2483 * or the host Cpu is online after a suspend/resume, so we cannot reuse the current ASID anymore.
2484 */
2485 bool fNewAsid = false;
2486 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2487 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2488 {
2489 pVCpu->hm.s.fForceTLBFlush = true;
2490 fNewAsid = true;
2491 }
2492
2493 /*
2494 * Check for explicit TLB shootdowns.
2495 */
2496 if (VMCPU_FF_TESTANDCLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2497 pVCpu->hm.s.fForceTLBFlush = true;
2498
2499 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2500
2501 if (pVCpu->hm.s.fForceTLBFlush)
2502 {
2503 if (fNewAsid)
2504 {
2505 ++pCpu->uCurrentAsid;
2506 if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
2507 {
2508 pCpu->uCurrentAsid = 1; /* start at 1; host uses 0 */
2509 pCpu->cTlbFlushes++;
2510 pCpu->fFlushAsidBeforeUse = true;
2511 }
2512
2513 pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
2514 if (pCpu->fFlushAsidBeforeUse)
2515 hmR0VmxFlushVPID(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
2516 }
2517 else
2518 {
2519 if (pVM->hm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVVPID_CAPS_SINGLE_CONTEXT)
2520 hmR0VmxFlushVPID(pVM, pVCpu, VMX_FLUSH_VPID_SINGLE_CONTEXT, 0 /* GCPtr */);
2521 else
2522 hmR0VmxFlushEPT(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
2523 }
2524
2525 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2526 pVCpu->hm.s.fForceTLBFlush = false;
2527 }
2528 else
2529 {
2530 AssertMsg(pVCpu->hm.s.uCurrentAsid && pCpu->uCurrentAsid,
2531 ("hm->uCurrentAsid=%lu hm->cTlbFlushes=%lu cpu->uCurrentAsid=%lu cpu->cTlbFlushes=%lu\n",
2532 pVCpu->hm.s.uCurrentAsid, pVCpu->hm.s.cTlbFlushes,
2533 pCpu->uCurrentAsid, pCpu->cTlbFlushes));
2534
2535 /** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
2536 * not be executed. See hmQueueInvlPage() where it is commented
2537 * out. Support individual entry flushing someday. */
2538 if (VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
2539 {
2540 STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
2541
2542 /*
2543 * Flush individual guest entries using VPID from the TLB or as little as possible with EPT
2544 * as supported by the CPU.
2545 */
2546 if (pVM->hm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVVPID_CAPS_INDIV_ADDR)
2547 {
2548 for (unsigned i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
2549 hmR0VmxFlushVPID(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
2550 }
2551 else
2552 hmR0VmxFlushEPT(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
2553 }
2554 else
2555 {
2556#ifdef VBOX_WITH_STATISTICS
2557 STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTLBWorldSwitch);
2558#endif
2559 }
2560 }
2561 pVCpu->hm.s.TlbShootdown.cPages = 0;
2562 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
2563
2564 AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
2565 ("Flush count mismatch for cpu %d (%x vs %x)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
2566 AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2567 ("cpu%d uCurrentAsid = %x\n", pCpu->idCpu, pCpu->uCurrentAsid));
2568 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2569 ("cpu%d VM uCurrentAsid = %x\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2570
2571 /* Update VMCS with the VPID. */
2572 int rc = VMXWriteVMCS(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
2573 AssertRC(rc);
2574}
2575
2576
2577/**
2578 * Setup the tagged TLB for EPT only.
2579 *
2580 * @returns VBox status code.
2581 * @param pVM Pointer to the VM.
2582 * @param pVCpu Pointer to the VMCPU.
2583 */
2584static DECLCALLBACK(void) hmR0VmxSetupTLBEPT(PVM pVM, PVMCPU pVCpu)
2585{
2586 PHMGLOBLCPUINFO pCpu;
2587
2588 Assert(pVM->hm.s.fNestedPaging);
2589 Assert(!pVM->hm.s.vmx.fVpid);
2590
2591 pCpu = HMR0GetCurrentCpu();
2592
2593 /*
2594 * Force a TLB flush for the first world switch if the current CPU differs from the one we ran on last
2595 * This can happen both for start & resume due to long jumps back to ring-3.
2596 * A change in the TLB flush count implies the host Cpu is online after a suspend/resume.
2597 */
2598 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2599 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2600 {
2601 pVCpu->hm.s.fForceTLBFlush = true;
2602 }
2603
2604 /*
2605 * Check for explicit TLB shootdown flushes.
2606 */
2607 if (VMCPU_FF_TESTANDCLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2608 pVCpu->hm.s.fForceTLBFlush = true;
2609
2610 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2611 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2612
2613 if (pVCpu->hm.s.fForceTLBFlush)
2614 hmR0VmxFlushEPT(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
2615 else
2616 {
2617 /** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
2618 * not be executed. See hmQueueInvlPage() where it is commented
2619 * out. Support individual entry flushing someday. */
2620 if (VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
2621 {
2622 /*
2623 * We cannot flush individual entries without VPID support. Flush using EPT.
2624 */
2625 STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
2626 hmR0VmxFlushEPT(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
2627 }
2628 }
2629 pVCpu->hm.s.TlbShootdown.cPages= 0;
2630 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
2631
2632#ifdef VBOX_WITH_STATISTICS
2633 if (pVCpu->hm.s.fForceTLBFlush)
2634 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTLBWorldSwitch);
2635 else
2636 STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTLBWorldSwitch);
2637#endif
2638}
2639
2640
2641/**
2642 * Setup the tagged TLB for VPID.
2643 *
2644 * @returns VBox status code.
2645 * @param pVM Pointer to the VM.
2646 * @param pVCpu Pointer to the VMCPU.
2647 */
2648static DECLCALLBACK(void) hmR0VmxSetupTLBVPID(PVM pVM, PVMCPU pVCpu)
2649{
2650 PHMGLOBLCPUINFO pCpu;
2651
2652 Assert(pVM->hm.s.vmx.fVpid);
2653 Assert(!pVM->hm.s.fNestedPaging);
2654
2655 pCpu = HMR0GetCurrentCpu();
2656
2657 /*
2658 * Force a TLB flush for the first world switch if the current CPU differs from the one we ran on last
2659 * This can happen both for start & resume due to long jumps back to ring-3.
2660 * If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
2661 * or the host Cpu is online after a suspend/resume, so we cannot reuse the current ASID anymore.
2662 */
2663 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2664 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2665 {
2666 /* Force a TLB flush on VM entry. */
2667 pVCpu->hm.s.fForceTLBFlush = true;
2668 }
2669
2670 /*
2671 * Check for explicit TLB shootdown flushes.
2672 */
2673 if (VMCPU_FF_TESTANDCLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2674 pVCpu->hm.s.fForceTLBFlush = true;
2675
2676 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2677
2678 if (pVCpu->hm.s.fForceTLBFlush)
2679 {
2680 ++pCpu->uCurrentAsid;
2681 if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
2682 {
2683 pCpu->uCurrentAsid = 1; /* start at 1; host uses 0 */
2684 pCpu->cTlbFlushes++;
2685 pCpu->fFlushAsidBeforeUse = true;
2686 }
2687
2688 pVCpu->hm.s.fForceTLBFlush = false;
2689 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2690 pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
2691 if (pCpu->fFlushAsidBeforeUse)
2692 hmR0VmxFlushVPID(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
2693 }
2694 else
2695 {
2696 AssertMsg(pVCpu->hm.s.uCurrentAsid && pCpu->uCurrentAsid,
2697 ("hm->uCurrentAsid=%lu hm->cTlbFlushes=%lu cpu->uCurrentAsid=%lu cpu->cTlbFlushes=%lu\n",
2698 pVCpu->hm.s.uCurrentAsid, pVCpu->hm.s.cTlbFlushes,
2699 pCpu->uCurrentAsid, pCpu->cTlbFlushes));
2700
2701 /** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
2702 * not be executed. See hmQueueInvlPage() where it is commented
2703 * out. Support individual entry flushing someday. */
2704 if (VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
2705 {
2706 /*
2707 * Flush individual guest entries using VPID from the TLB or as little as possible with EPT
2708 * as supported by the CPU.
2709 */
2710 if (pVM->hm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVVPID_CAPS_INDIV_ADDR)
2711 {
2712 for (unsigned i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
2713 hmR0VmxFlushVPID(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
2714 }
2715 else
2716 hmR0VmxFlushVPID(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
2717 }
2718 }
2719 pVCpu->hm.s.TlbShootdown.cPages = 0;
2720 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
2721
2722 AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
2723 ("Flush count mismatch for cpu %d (%x vs %x)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
2724 AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxASID,
2725 ("cpu%d uCurrentAsid = %x\n", pCpu->idCpu, pCpu->uCurrentAsid));
2726 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxASID,
2727 ("cpu%d VM uCurrentAsid = %x\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2728
2729 int rc = VMXWriteVMCS(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
2730 AssertRC(rc);
2731
2732# ifdef VBOX_WITH_STATISTICS
2733 if (pVCpu->hm.s.fForceTLBFlush)
2734 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTLBWorldSwitch);
2735 else
2736 STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTLBWorldSwitch);
2737# endif
2738}
2739
2740
2741/**
2742 * Runs guest code in a VT-x VM.
2743 *
2744 * @returns VBox status code.
2745 * @param pVM Pointer to the VM.
2746 * @param pVCpu Pointer to the VMCPU.
2747 * @param pCtx Pointer to the guest CPU context.
2748 */
2749VMMR0DECL(int) VMXR0RunGuestCode(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
2750{
2751 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
2752 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit1);
2753 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit2);
2754
2755 VBOXSTRICTRC rc = VINF_SUCCESS;
2756 int rc2;
2757 RTGCUINTREG val;
2758 RTGCUINTREG exitReason = (RTGCUINTREG)VMX_EXIT_INVALID;
2759 RTGCUINTREG instrError, cbInstr;
2760 RTGCUINTPTR exitQualification = 0;
2761 RTGCUINTPTR intInfo = 0; /* shut up buggy gcc 4 */
2762 RTGCUINTPTR errCode, instrInfo;
2763 bool fSetupTPRCaching = false;
2764 uint64_t u64OldLSTAR = 0;
2765 uint8_t u8LastTPR = 0;
2766 RTCCUINTREG uOldEFlags = ~(RTCCUINTREG)0;
2767 unsigned cResume = 0;
2768#ifdef VBOX_STRICT
2769 RTCPUID idCpuCheck;
2770 bool fWasInLongMode = false;
2771#endif
2772#ifdef VBOX_HIGH_RES_TIMERS_HACK_IN_RING0
2773 uint64_t u64LastTime = RTTimeMilliTS();
2774#endif
2775
2776 Assert(!(pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
2777 || (pVCpu->hm.s.vmx.pbVAPIC && pVM->hm.s.vmx.pbApicAccess));
2778
2779 /*
2780 * Check if we need to use TPR shadowing.
2781 */
2782 if ( CPUMIsGuestInLongModeEx(pCtx)
2783 || ( (( pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
2784 || pVM->hm.s.fTRPPatchingAllowed)
2785 && pVM->hm.s.fHasIoApic)
2786 )
2787 {
2788 fSetupTPRCaching = true;
2789 }
2790
2791 Log2(("\nE"));
2792
2793 /* This is not ideal, but if we don't clear the event injection in the VMCS right here,
2794 * we may end up injecting some stale event into a VM, including injecting an event that
2795 * originated before a VM reset *after* the VM has been reset. See @bugref{6220}.
2796 */
2797 VMXWriteVMCS(VMX_VMCS_CTRL_ENTRY_IRQ_INFO, 0);
2798
2799#ifdef VBOX_STRICT
2800 {
2801 RTCCUINTREG val2;
2802
2803 rc2 = VMXReadVMCS(VMX_VMCS_CTRL_PIN_EXEC_CONTROLS, &val2);
2804 AssertRC(rc2);
2805 Log2(("VMX_VMCS_CTRL_PIN_EXEC_CONTROLS = %08x\n", val2));
2806
2807 /* allowed zero */
2808 if ((val2 & pVM->hm.s.vmx.msr.vmx_pin_ctls.n.disallowed0) != pVM->hm.s.vmx.msr.vmx_pin_ctls.n.disallowed0)
2809 Log(("Invalid VMX_VMCS_CTRL_PIN_EXEC_CONTROLS: zero\n"));
2810
2811 /* allowed one */
2812 if ((val2 & ~pVM->hm.s.vmx.msr.vmx_pin_ctls.n.allowed1) != 0)
2813 Log(("Invalid VMX_VMCS_CTRL_PIN_EXEC_CONTROLS: one\n"));
2814
2815 rc2 = VMXReadVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, &val2);
2816 AssertRC(rc2);
2817 Log2(("VMX_VMCS_CTRL_PROC_EXEC_CONTROLS = %08x\n", val2));
2818
2819 /*
2820 * Must be set according to the MSR, but can be cleared if nested paging is used.
2821 */
2822 if (pVM->hm.s.fNestedPaging)
2823 {
2824 val2 |= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INVLPG_EXIT
2825 | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
2826 | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT;
2827 }
2828
2829 /* allowed zero */
2830 if ((val2 & pVM->hm.s.vmx.msr.vmx_proc_ctls.n.disallowed0) != pVM->hm.s.vmx.msr.vmx_proc_ctls.n.disallowed0)
2831 Log(("Invalid VMX_VMCS_CTRL_PROC_EXEC_CONTROLS: zero\n"));
2832
2833 /* allowed one */
2834 if ((val2 & ~pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1) != 0)
2835 Log(("Invalid VMX_VMCS_CTRL_PROC_EXEC_CONTROLS: one\n"));
2836
2837 rc2 = VMXReadVMCS(VMX_VMCS_CTRL_ENTRY_CONTROLS, &val2);
2838 AssertRC(rc2);
2839 Log2(("VMX_VMCS_CTRL_ENTRY_CONTROLS = %08x\n", val2));
2840
2841 /* allowed zero */
2842 if ((val2 & pVM->hm.s.vmx.msr.vmx_entry.n.disallowed0) != pVM->hm.s.vmx.msr.vmx_entry.n.disallowed0)
2843 Log(("Invalid VMX_VMCS_CTRL_ENTRY_CONTROLS: zero\n"));
2844
2845 /* allowed one */
2846 if ((val2 & ~pVM->hm.s.vmx.msr.vmx_entry.n.allowed1) != 0)
2847 Log(("Invalid VMX_VMCS_CTRL_ENTRY_CONTROLS: one\n"));
2848
2849 rc2 = VMXReadVMCS(VMX_VMCS_CTRL_EXIT_CONTROLS, &val2);
2850 AssertRC(rc2);
2851 Log2(("VMX_VMCS_CTRL_EXIT_CONTROLS = %08x\n", val2));
2852
2853 /* allowed zero */
2854 if ((val2 & pVM->hm.s.vmx.msr.vmx_exit.n.disallowed0) != pVM->hm.s.vmx.msr.vmx_exit.n.disallowed0)
2855 Log(("Invalid VMX_VMCS_CTRL_EXIT_CONTROLS: zero\n"));
2856
2857 /* allowed one */
2858 if ((val2 & ~pVM->hm.s.vmx.msr.vmx_exit.n.allowed1) != 0)
2859 Log(("Invalid VMX_VMCS_CTRL_EXIT_CONTROLS: one\n"));
2860 }
2861 fWasInLongMode = CPUMIsGuestInLongModeEx(pCtx);
2862#endif /* VBOX_STRICT */
2863
2864#ifdef VBOX_WITH_CRASHDUMP_MAGIC
2865 pVCpu->hm.s.vmx.VMCSCache.u64TimeEntry = RTTimeNanoTS();
2866#endif
2867
2868 /*
2869 * We can jump to this point to resume execution after determining that a VM-exit is innocent.
2870 */
2871ResumeExecution:
2872 if (!STAM_REL_PROFILE_ADV_IS_RUNNING(&pVCpu->hm.s.StatEntry))
2873 STAM_REL_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit2, &pVCpu->hm.s.StatEntry, x);
2874 AssertMsg(pVCpu->hm.s.idEnteredCpu == RTMpCpuId(),
2875 ("Expected %d, I'm %d; cResume=%d exitReason=%RGv exitQualification=%RGv\n",
2876 (int)pVCpu->hm.s.idEnteredCpu, (int)RTMpCpuId(), cResume, exitReason, exitQualification));
2877 Assert(!HMR0SuspendPending());
2878 /* Not allowed to switch modes without reloading the host state (32->64 switcher)!! */
2879 Assert(fWasInLongMode == CPUMIsGuestInLongModeEx(pCtx));
2880
2881 /*
2882 * Safety precaution; looping for too long here can have a very bad effect on the host.
2883 */
2884 if (RT_UNLIKELY(++cResume > pVM->hm.s.cMaxResumeLoops))
2885 {
2886 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMaxResume);
2887 rc = VINF_EM_RAW_INTERRUPT;
2888 goto end;
2889 }
2890
2891 /*
2892 * Check for IRQ inhibition due to instruction fusing (sti, mov ss).
2893 */
2894 if (VMCPU_FF_ISSET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
2895 {
2896 Log(("VM_FF_INHIBIT_INTERRUPTS at %RGv successor %RGv\n", (RTGCPTR)pCtx->rip, EMGetInhibitInterruptsPC(pVCpu)));
2897 if (pCtx->rip != EMGetInhibitInterruptsPC(pVCpu))
2898 {
2899 /*
2900 * Note: we intentionally don't clear VM_FF_INHIBIT_INTERRUPTS here.
2901 * Before we are able to execute this instruction in raw mode (iret to guest code) an external interrupt might
2902 * force a world switch again. Possibly allowing a guest interrupt to be dispatched in the process. This could
2903 * break the guest. Sounds very unlikely, but such timing sensitive problems are not as rare as you might think.
2904 */
2905 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
2906 /* Irq inhibition is no longer active; clear the corresponding VMX state. */
2907 rc2 = VMXWriteVMCS(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, 0);
2908 AssertRC(rc2);
2909 }
2910 }
2911 else
2912 {
2913 /* Irq inhibition is no longer active; clear the corresponding VMX state. */
2914 rc2 = VMXWriteVMCS(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, 0);
2915 AssertRC(rc2);
2916 }
2917
2918#ifdef VBOX_HIGH_RES_TIMERS_HACK_IN_RING0
2919 if (RT_UNLIKELY((cResume & 0xf) == 0))
2920 {
2921 uint64_t u64CurTime = RTTimeMilliTS();
2922
2923 if (RT_UNLIKELY(u64CurTime > u64LastTime))
2924 {
2925 u64LastTime = u64CurTime;
2926 TMTimerPollVoid(pVM, pVCpu);
2927 }
2928 }
2929#endif
2930
2931 /*
2932 * Check for pending actions that force us to go back to ring-3.
2933 */
2934 if ( VM_FF_ISPENDING(pVM, VM_FF_HM_TO_R3_MASK | VM_FF_REQUEST | VM_FF_PGM_POOL_FLUSH_PENDING | VM_FF_PDM_DMA)
2935 || VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK | VMCPU_FF_PGM_SYNC_CR3 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL | VMCPU_FF_REQUEST))
2936 {
2937 /* Check if a sync operation is pending. */
2938 if (VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
2939 {
2940 rc = PGMSyncCR3(pVCpu, pCtx->cr0, pCtx->cr3, pCtx->cr4, VMCPU_FF_ISSET(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
2941 if (rc != VINF_SUCCESS)
2942 {
2943 AssertRC(VBOXSTRICTRC_VAL(rc));
2944 Log(("Pending pool sync is forcing us back to ring 3; rc=%d\n", VBOXSTRICTRC_VAL(rc)));
2945 goto end;
2946 }
2947 }
2948
2949#ifdef DEBUG
2950 /* Intercept X86_XCPT_DB if stepping is enabled */
2951 if (!DBGFIsStepping(pVCpu))
2952#endif
2953 {
2954 if ( VM_FF_ISPENDING(pVM, VM_FF_HM_TO_R3_MASK)
2955 || VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
2956 {
2957 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchToR3);
2958 rc = RT_UNLIKELY(VM_FF_ISPENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
2959 goto end;
2960 }
2961 }
2962
2963 /* Pending request packets might contain actions that need immediate attention, such as pending hardware interrupts. */
2964 if ( VM_FF_ISPENDING(pVM, VM_FF_REQUEST)
2965 || VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_REQUEST))
2966 {
2967 rc = VINF_EM_PENDING_REQUEST;
2968 goto end;
2969 }
2970
2971 /* Check if a pgm pool flush is in progress. */
2972 if (VM_FF_ISPENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
2973 {
2974 rc = VINF_PGM_POOL_FLUSH_PENDING;
2975 goto end;
2976 }
2977
2978 /* Check if DMA work is pending (2nd+ run). */
2979 if (VM_FF_ISPENDING(pVM, VM_FF_PDM_DMA) && cResume > 1)
2980 {
2981 rc = VINF_EM_RAW_TO_R3;
2982 goto end;
2983 }
2984 }
2985
2986#ifdef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
2987 /*
2988 * Exit to ring-3 preemption/work is pending.
2989 *
2990 * Interrupts are disabled before the call to make sure we don't miss any interrupt
2991 * that would flag preemption (IPI, timer tick, ++). (Would've been nice to do this
2992 * further down, but hmR0VmxCheckPendingInterrupt makes that impossible.)
2993 *
2994 * Note! Interrupts must be disabled done *before* we check for TLB flushes; TLB
2995 * shootdowns rely on this.
2996 */
2997 uOldEFlags = ASMIntDisableFlags();
2998 if (RTThreadPreemptIsPending(NIL_RTTHREAD))
2999 {
3000 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptPending);
3001 rc = VINF_EM_RAW_INTERRUPT;
3002 goto end;
3003 }
3004 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
3005#endif
3006
3007 /*
3008 * When external interrupts are pending, we should exit the VM when IF is set.
3009 * Note: *After* VM_FF_INHIBIT_INTERRUPTS check!
3010 */
3011 rc = hmR0VmxCheckPendingInterrupt(pVM, pVCpu, pCtx);
3012 if (RT_FAILURE(rc))
3013 goto end;
3014
3015 /** @todo check timers?? */
3016
3017 /*
3018 * TPR caching using CR8 is only available in 64-bit mode.
3019 * Note: The 32-bit exception for AMD (X86_CPUID_AMD_FEATURE_ECX_CR8L), but this appears missing in Intel CPUs.
3020 * Note: We can't do this in LoadGuestState() as PDMApicGetTPR can jump back to ring-3 (lock)!! (no longer true) .
3021 */
3022 /** @todo query and update the TPR only when it could have been changed (mmio
3023 * access & wrsmr (x2apic) */
3024 if (fSetupTPRCaching)
3025 {
3026 /* TPR caching in CR8 */
3027 bool fPending;
3028
3029 rc2 = PDMApicGetTPR(pVCpu, &u8LastTPR, &fPending);
3030 AssertRC(rc2);
3031 /* The TPR can be found at offset 0x80 in the APIC mmio page. */
3032 pVCpu->hm.s.vmx.pbVAPIC[0x80] = u8LastTPR;
3033
3034 /*
3035 * Two options here:
3036 * - external interrupt pending, but masked by the TPR value.
3037 * -> a CR8 update that lower the current TPR value should cause an exit
3038 * - no pending interrupts
3039 * -> We don't need to be explicitely notified. There are enough world switches for detecting pending interrupts.
3040 */
3041
3042 /* cr8 bits 3-0 correspond to bits 7-4 of the task priority mmio register. */
3043 rc = VMXWriteVMCS(VMX_VMCS_CTRL_TPR_THRESHOLD, (fPending) ? (u8LastTPR >> 4) : 0);
3044 AssertRC(VBOXSTRICTRC_VAL(rc));
3045
3046 if (pVM->hm.s.fTPRPatchingActive)
3047 {
3048 Assert(!CPUMIsGuestInLongModeEx(pCtx));
3049 /* Our patch code uses LSTAR for TPR caching. */
3050 pCtx->msrLSTAR = u8LastTPR;
3051
3052 /** @todo r=ramshankar: we should check for MSR-bitmap support here. */
3053 if (fPending)
3054 {
3055 /* A TPR change could activate a pending interrupt, so catch lstar writes. */
3056 hmR0VmxSetMSRPermission(pVCpu, MSR_K8_LSTAR, true, false);
3057 }
3058 else
3059 {
3060 /*
3061 * No interrupts are pending, so we don't need to be explicitely notified.
3062 * There are enough world switches for detecting pending interrupts.
3063 */
3064 hmR0VmxSetMSRPermission(pVCpu, MSR_K8_LSTAR, true, true);
3065 }
3066 }
3067 }
3068
3069#ifdef LOG_ENABLED
3070 if ( pVM->hm.s.fNestedPaging
3071 || pVM->hm.s.vmx.fVpid)
3072 {
3073 PHMGLOBLCPUINFO pCpu = HMR0GetCurrentCpu();
3074 if (pVCpu->hm.s.idLastCpu != pCpu->idCpu)
3075 {
3076 LogFlow(("Force TLB flush due to rescheduling to a different cpu (%d vs %d)\n", pVCpu->hm.s.idLastCpu,
3077 pCpu->idCpu));
3078 }
3079 else if (pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
3080 {
3081 LogFlow(("Force TLB flush due to changed TLB flush count (%x vs %x)\n", pVCpu->hm.s.cTlbFlushes,
3082 pCpu->cTlbFlushes));
3083 }
3084 else if (VMCPU_FF_ISSET(pVCpu, VMCPU_FF_TLB_FLUSH))
3085 LogFlow(("Manual TLB flush\n"));
3086 }
3087#endif
3088#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
3089 PGMRZDynMapFlushAutoSet(pVCpu);
3090#endif
3091
3092 /*
3093 * NOTE: DO NOT DO ANYTHING AFTER THIS POINT THAT MIGHT JUMP BACK TO RING-3!
3094 * (until the actual world switch)
3095 */
3096#ifdef VBOX_STRICT
3097 idCpuCheck = RTMpCpuId();
3098#endif
3099#ifdef LOG_ENABLED
3100 VMMR0LogFlushDisable(pVCpu);
3101#endif
3102
3103 /*
3104 * Save the host state first.
3105 */
3106 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_HOST_CONTEXT)
3107 {
3108 rc = VMXR0SaveHostState(pVM, pVCpu);
3109 if (RT_UNLIKELY(rc != VINF_SUCCESS))
3110 {
3111 VMMR0LogFlushEnable(pVCpu);
3112 goto end;
3113 }
3114 }
3115
3116 /*
3117 * Load the guest state.
3118 */
3119 if (!pVCpu->hm.s.fContextUseFlags)
3120 {
3121 VMXR0LoadMinimalGuestState(pVM, pVCpu, pCtx);
3122 STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadMinimal);
3123 }
3124 else
3125 {
3126 rc = VMXR0LoadGuestState(pVM, pVCpu, pCtx);
3127 if (RT_UNLIKELY(rc != VINF_SUCCESS))
3128 {
3129 VMMR0LogFlushEnable(pVCpu);
3130 goto end;
3131 }
3132 STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadFull);
3133 }
3134
3135#ifndef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
3136 /*
3137 * Disable interrupts to make sure a poke will interrupt execution.
3138 * This must be done *before* we check for TLB flushes; TLB shootdowns rely on this.
3139 */
3140 uOldEFlags = ASMIntDisableFlags();
3141 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
3142#endif
3143
3144 /* Non-register state Guest Context */
3145 /** @todo change me according to cpu state */
3146 rc2 = VMXWriteVMCS(VMX_VMCS32_GUEST_ACTIVITY_STATE, VMX_CMS_GUEST_ACTIVITY_ACTIVE);
3147 AssertRC(rc2);
3148
3149 /* Set TLB flush state as checked until we return from the world switch. */
3150 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true);
3151 /* Deal with tagged TLB setup and invalidation. */
3152 pVM->hm.s.vmx.pfnSetupTaggedTlb(pVM, pVCpu);
3153
3154 /*
3155 * Manual save and restore:
3156 * - General purpose registers except RIP, RSP
3157 *
3158 * Trashed:
3159 * - CR2 (we don't care)
3160 * - LDTR (reset to 0)
3161 * - DRx (presumably not changed at all)
3162 * - DR7 (reset to 0x400)
3163 * - EFLAGS (reset to RT_BIT(1); not relevant)
3164 */
3165
3166 /* All done! Let's start VM execution. */
3167 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
3168 Assert(idCpuCheck == RTMpCpuId());
3169
3170#ifdef VBOX_WITH_CRASHDUMP_MAGIC
3171 pVCpu->hm.s.vmx.VMCSCache.cResume = cResume;
3172 pVCpu->hm.s.vmx.VMCSCache.u64TimeSwitch = RTTimeNanoTS();
3173#endif
3174
3175 /*
3176 * Save the current TPR value in the LSTAR MSR so our patches can access it.
3177 */
3178 if (pVM->hm.s.fTPRPatchingActive)
3179 {
3180 Assert(pVM->hm.s.fTPRPatchingActive);
3181 u64OldLSTAR = ASMRdMsr(MSR_K8_LSTAR);
3182 ASMWrMsr(MSR_K8_LSTAR, u8LastTPR);
3183 }
3184
3185 TMNotifyStartOfExecution(pVCpu);
3186
3187#ifndef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
3188 /*
3189 * Save the current Host TSC_AUX and write the guest TSC_AUX to the host, so that
3190 * RDTSCPs (that don't cause exits) reads the guest MSR. See @bugref{3324}.
3191 */
3192 if ( (pVCpu->hm.s.vmx.proc_ctls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
3193 && !(pVCpu->hm.s.vmx.proc_ctls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT))
3194 {
3195 pVCpu->hm.s.u64HostTSCAux = ASMRdMsr(MSR_K8_TSC_AUX);
3196 uint64_t u64GuestTSCAux = 0;
3197 rc2 = CPUMQueryGuestMsr(pVCpu, MSR_K8_TSC_AUX, &u64GuestTSCAux);
3198 AssertRC(rc2);
3199 ASMWrMsr(MSR_K8_TSC_AUX, u64GuestTSCAux);
3200 }
3201#endif
3202
3203#ifdef VBOX_WITH_KERNEL_USING_XMM
3204 rc = hmR0VMXStartVMWrapXMM(pVCpu->hm.s.fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hm.s.vmx.pfnStartVM);
3205#else
3206 rc = pVCpu->hm.s.vmx.pfnStartVM(pVCpu->hm.s.fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu);
3207#endif
3208 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false);
3209 ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits);
3210
3211 /* Possibly the last TSC value seen by the guest (too high) (only when we're in TSC offset mode). */
3212 if (!(pVCpu->hm.s.vmx.proc_ctls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT))
3213 {
3214#ifndef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
3215 /* Restore host's TSC_AUX. */
3216 if (pVCpu->hm.s.vmx.proc_ctls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
3217 ASMWrMsr(MSR_K8_TSC_AUX, pVCpu->hm.s.u64HostTSCAux);
3218#endif
3219
3220 TMCpuTickSetLastSeen(pVCpu,
3221 ASMReadTSC() + pVCpu->hm.s.vmx.u64TSCOffset - 0x400 /* guestimate of world switch overhead in clock ticks */);
3222 }
3223
3224 TMNotifyEndOfExecution(pVCpu);
3225 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED);
3226 Assert(!(ASMGetFlags() & X86_EFL_IF));
3227
3228 /*
3229 * Restore the host LSTAR MSR if the guest could have changed it.
3230 */
3231 if (pVM->hm.s.fTPRPatchingActive)
3232 {
3233 Assert(pVM->hm.s.fTPRPatchingActive);
3234 pVCpu->hm.s.vmx.pbVAPIC[0x80] = pCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR);
3235 ASMWrMsr(MSR_K8_LSTAR, u64OldLSTAR);
3236 }
3237
3238 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatExit1, x);
3239 ASMSetFlags(uOldEFlags);
3240#ifdef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
3241 uOldEFlags = ~(RTCCUINTREG)0;
3242#endif
3243
3244 AssertMsg(!pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries, ("pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries=%d\n",
3245 pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries));
3246
3247 /* In case we execute a goto ResumeExecution later on. */
3248 pVCpu->hm.s.fResumeVM = true;
3249 pVCpu->hm.s.fForceTLBFlush = false;
3250
3251 /*
3252 * !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
3253 * IMPORTANT: WE CAN'T DO ANY LOGGING OR OPERATIONS THAT CAN DO A LONGJMP BACK TO RING 3 *BEFORE* WE'VE SYNCED BACK (MOST OF) THE GUEST STATE
3254 * !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
3255 */
3256
3257 if (RT_UNLIKELY(rc != VINF_SUCCESS))
3258 {
3259 hmR0VmxReportWorldSwitchError(pVM, pVCpu, rc, pCtx);
3260 VMMR0LogFlushEnable(pVCpu);
3261 goto end;
3262 }
3263
3264 /* Success. Query the guest state and figure out what has happened. */
3265
3266 /* Investigate why there was a VM-exit. */
3267 rc2 = VMXReadCachedVMCS(VMX_VMCS32_RO_EXIT_REASON, &exitReason);
3268 STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[exitReason & MASK_EXITREASON_STAT]);
3269
3270 exitReason &= 0xffff; /* bit 0-15 contain the exit code. */
3271 rc2 |= VMXReadCachedVMCS(VMX_VMCS32_RO_VM_INSTR_ERROR, &instrError);
3272 rc2 |= VMXReadCachedVMCS(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &cbInstr);
3273 rc2 |= VMXReadCachedVMCS(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &intInfo);
3274 /* might not be valid; depends on VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID. */
3275 rc2 |= VMXReadCachedVMCS(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERRCODE, &errCode);
3276 rc2 |= VMXReadCachedVMCS(VMX_VMCS32_RO_EXIT_INSTR_INFO, &instrInfo);
3277 rc2 |= VMXReadCachedVMCS(VMX_VMCS_RO_EXIT_QUALIFICATION, &exitQualification);
3278 AssertRC(rc2);
3279
3280 /*
3281 * Sync back the guest state.
3282 */
3283 rc2 = VMXR0SaveGuestState(pVM, pVCpu, pCtx);
3284 AssertRC(rc2);
3285
3286 /* Note! NOW IT'S SAFE FOR LOGGING! */
3287 VMMR0LogFlushEnable(pVCpu);
3288 Log2(("Raw exit reason %08x\n", exitReason));
3289#if ARCH_BITS == 64 /* for the time being */
3290 VBOXVMM_R0_HMVMX_VMEXIT(pVCpu, pCtx, exitReason);
3291#endif
3292
3293 /*
3294 * Check if an injected event was interrupted prematurely.
3295 */
3296 rc2 = VMXReadCachedVMCS(VMX_VMCS32_RO_IDT_INFO, &val);
3297 AssertRC(rc2);
3298 pVCpu->hm.s.Event.intInfo = VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(val);
3299 if ( VMX_EXIT_INTERRUPTION_INFO_VALID(pVCpu->hm.s.Event.intInfo)
3300 /* Ignore 'int xx' as they'll be restarted anyway. */
3301 && VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hm.s.Event.intInfo) != VMX_EXIT_INTERRUPTION_INFO_TYPE_SW
3302 /* Ignore software exceptions (such as int3) as they'll reoccur when we restart the instruction anyway. */
3303 && VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hm.s.Event.intInfo) != VMX_EXIT_INTERRUPTION_INFO_TYPE_SWEXCPT)
3304 {
3305 Assert(!pVCpu->hm.s.Event.fPending);
3306 pVCpu->hm.s.Event.fPending = true;
3307 /* Error code present? */
3308 if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.intInfo))
3309 {
3310 rc2 = VMXReadCachedVMCS(VMX_VMCS32_RO_IDT_ERRCODE, &val);
3311 AssertRC(rc2);
3312 pVCpu->hm.s.Event.errCode = val;
3313 Log(("Pending inject %RX64 at %RGv exit=%08x intInfo=%08x exitQualification=%RGv pending error=%RX64\n",
3314 pVCpu->hm.s.Event.intInfo, (RTGCPTR)pCtx->rip, exitReason, intInfo, exitQualification, val));
3315 }
3316 else
3317 {
3318 Log(("Pending inject %RX64 at %RGv exit=%08x intInfo=%08x exitQualification=%RGv\n", pVCpu->hm.s.Event.intInfo,
3319 (RTGCPTR)pCtx->rip, exitReason, intInfo, exitQualification));
3320 pVCpu->hm.s.Event.errCode = 0;
3321 }
3322 }
3323#ifdef VBOX_STRICT
3324 else if ( VMX_EXIT_INTERRUPTION_INFO_VALID(pVCpu->hm.s.Event.intInfo)
3325 /* Ignore software exceptions (such as int3) as they're reoccur when we restart the instruction anyway. */
3326 && VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hm.s.Event.intInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_SWEXCPT)
3327 {
3328 Log(("Ignore pending inject %RX64 at %RGv exit=%08x intInfo=%08x exitQualification=%RGv\n",
3329 pVCpu->hm.s.Event.intInfo, (RTGCPTR)pCtx->rip, exitReason, intInfo, exitQualification));
3330 }
3331
3332 if (exitReason == VMX_EXIT_ERR_INVALID_GUEST_STATE)
3333 HMDumpRegs(pVM, pVCpu, pCtx);
3334#endif
3335
3336 Log2(("E%d: New EIP=%x:%RGv\n", (uint32_t)exitReason, pCtx->cs.Sel, (RTGCPTR)pCtx->rip));
3337 Log2(("Exit reason %d, exitQualification %RGv\n", (uint32_t)exitReason, exitQualification));
3338 Log2(("instrInfo=%d instrError=%d instr length=%d\n", (uint32_t)instrInfo, (uint32_t)instrError, (uint32_t)cbInstr));
3339 Log2(("Interruption error code %d\n", (uint32_t)errCode));
3340 Log2(("IntInfo = %08x\n", (uint32_t)intInfo));
3341
3342 /*
3343 * Sync back the TPR if it was changed.
3344 */
3345 if ( fSetupTPRCaching
3346 && u8LastTPR != pVCpu->hm.s.vmx.pbVAPIC[0x80])
3347 {
3348 rc2 = PDMApicSetTPR(pVCpu, pVCpu->hm.s.vmx.pbVAPIC[0x80]);
3349 AssertRC(rc2);
3350 }
3351
3352#ifdef DBGFTRACE_ENABLED /** @todo DTrace later. */
3353 RTTraceBufAddMsgF(pVM->CTX_SUFF(hTraceBuf), "vmexit %08x %016RX64 at %04:%08RX64 %RX64",
3354 exitReason, (uint64_t)exitQualification, pCtx->cs.Sel, pCtx->rip, (uint64_t)intInfo);
3355#endif
3356 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
3357
3358 /* Some cases don't need a complete resync of the guest CPU state; handle them here. */
3359 Assert(rc == VINF_SUCCESS); /* might consider VERR_IPE_UNINITIALIZED_STATUS here later... */
3360 switch (exitReason)
3361 {
3362 case VMX_EXIT_EXCEPTION: /* 0 Exception or non-maskable interrupt (NMI). */
3363 case VMX_EXIT_EXTERNAL_IRQ: /* 1 External interrupt. */
3364 {
3365 uint32_t vector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(intInfo);
3366
3367 if (!VMX_EXIT_INTERRUPTION_INFO_VALID(intInfo))
3368 {
3369 Assert(exitReason == VMX_EXIT_EXTERNAL_IRQ);
3370#if 0 //def VBOX_WITH_VMMR0_DISABLE_PREEMPTION
3371 if ( RTThreadPreemptIsPendingTrusty()
3372 && !RTThreadPreemptIsPending(NIL_RTTHREAD))
3373 goto ResumeExecution;
3374#endif
3375 /* External interrupt; leave to allow it to be dispatched again. */
3376 rc = VINF_EM_RAW_INTERRUPT;
3377 break;
3378 }
3379 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExit2Sub3, y3);
3380 switch (VMX_EXIT_INTERRUPTION_INFO_TYPE(intInfo))
3381 {
3382 case VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI: /* Non-maskable interrupt. */
3383 /* External interrupt; leave to allow it to be dispatched again. */
3384 rc = VINF_EM_RAW_INTERRUPT;
3385 break;
3386
3387 case VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT: /* External hardware interrupt. */
3388 AssertFailed(); /* can't come here; fails the first check. */
3389 break;
3390
3391 case VMX_EXIT_INTERRUPTION_INFO_TYPE_DBEXCPT: /* Unknown why we get this type for #DB */
3392 case VMX_EXIT_INTERRUPTION_INFO_TYPE_SWEXCPT: /* Software exception. (#BP or #OF) */
3393 Assert(vector == 1 || vector == 3 || vector == 4);
3394 /* no break */
3395 case VMX_EXIT_INTERRUPTION_INFO_TYPE_HWEXCPT: /* Hardware exception. */
3396 Log2(("Hardware/software interrupt %d\n", vector));
3397 switch (vector)
3398 {
3399 case X86_XCPT_NM:
3400 {
3401 Log(("#NM fault at %RGv error code %x\n", (RTGCPTR)pCtx->rip, errCode));
3402
3403 /** @todo don't intercept #NM exceptions anymore when we've activated the guest FPU state. */
3404 /* If we sync the FPU/XMM state on-demand, then we can continue execution as if nothing has happened. */
3405 rc = CPUMR0LoadGuestFPU(pVM, pVCpu, pCtx);
3406 if (rc == VINF_SUCCESS)
3407 {
3408 Assert(CPUMIsGuestFPUStateActive(pVCpu));
3409
3410 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowNM);
3411
3412 /* Continue execution. */
3413 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_CR0;
3414
3415 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2Sub3, y3);
3416 goto ResumeExecution;
3417 }
3418
3419 Log(("Forward #NM fault to the guest\n"));
3420 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
3421 rc2 = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo),
3422 cbInstr, 0);
3423 AssertRC(rc2);
3424 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2Sub3, y3);
3425 goto ResumeExecution;
3426 }
3427
3428 case X86_XCPT_PF: /* Page fault */
3429 {
3430#ifdef VBOX_ALWAYS_TRAP_PF
3431 if (pVM->hm.s.fNestedPaging)
3432 {
3433 /*
3434 * A genuine pagefault. Forward the trap to the guest by injecting the exception and resuming execution.
3435 */
3436 Log(("Guest page fault at %RGv cr2=%RGv error code %RGv rsp=%RGv\n", (RTGCPTR)pCtx->rip, exitQualification,
3437 errCode, (RTGCPTR)pCtx->rsp));
3438
3439 Assert(CPUMIsGuestInPagedProtectedModeEx(pCtx));
3440
3441 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
3442
3443 /* Now we must update CR2. */
3444 pCtx->cr2 = exitQualification;
3445 rc2 = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo),
3446 cbInstr, errCode);
3447 AssertRC(rc2);
3448
3449 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2Sub3, y3);
3450 goto ResumeExecution;
3451 }
3452#else
3453 Assert(!pVM->hm.s.fNestedPaging);
3454#endif
3455
3456#ifdef VBOX_HM_WITH_GUEST_PATCHING
3457 /* Shortcut for APIC TPR reads and writes; 32 bits guests only */
3458 if ( pVM->hm.s.fTRPPatchingAllowed
3459 && pVM->hm.s.pGuestPatchMem
3460 && (exitQualification & 0xfff) == 0x080
3461 && !(errCode & X86_TRAP_PF_P) /* not present */
3462 && CPUMGetGuestCPL(pVCpu) == 0
3463 && !CPUMIsGuestInLongModeEx(pCtx)
3464 && pVM->hm.s.cPatches < RT_ELEMENTS(pVM->hm.s.aPatches))
3465 {
3466 RTGCPHYS GCPhysApicBase, GCPhys;
3467 PDMApicGetBase(pVM, &GCPhysApicBase); /** @todo cache this */
3468 GCPhysApicBase &= PAGE_BASE_GC_MASK;
3469
3470 rc = PGMGstGetPage(pVCpu, (RTGCPTR)exitQualification, NULL, &GCPhys);
3471 if ( rc == VINF_SUCCESS
3472 && GCPhys == GCPhysApicBase)
3473 {
3474 /* Only attempt to patch the instruction once. */
3475 PHMTPRPATCH pPatch = (PHMTPRPATCH)RTAvloU32Get(&pVM->hm.s.PatchTree, (AVLOU32KEY)pCtx->eip);
3476 if (!pPatch)
3477 {
3478 rc = VINF_EM_HM_PATCH_TPR_INSTR;
3479 break;
3480 }
3481 }
3482 }
3483#endif
3484
3485 Log2(("Page fault at %RGv error code %x\n", exitQualification, errCode));
3486 /* Exit qualification contains the linear address of the page fault. */
3487 TRPMAssertTrap(pVCpu, X86_XCPT_PF, TRPM_TRAP);
3488 TRPMSetErrorCode(pVCpu, errCode);
3489 TRPMSetFaultAddress(pVCpu, exitQualification);
3490
3491 /* Shortcut for APIC TPR reads and writes. */
3492 if ( (exitQualification & 0xfff) == 0x080
3493 && !(errCode & X86_TRAP_PF_P) /* not present */
3494 && fSetupTPRCaching
3495 && (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC))
3496 {
3497 RTGCPHYS GCPhysApicBase, GCPhys;
3498 PDMApicGetBase(pVM, &GCPhysApicBase); /* @todo cache this */
3499 GCPhysApicBase &= PAGE_BASE_GC_MASK;
3500
3501 rc = PGMGstGetPage(pVCpu, (RTGCPTR)exitQualification, NULL, &GCPhys);
3502 if ( rc == VINF_SUCCESS
3503 && GCPhys == GCPhysApicBase)
3504 {
3505 Log(("Enable VT-x virtual APIC access filtering\n"));
3506 rc2 = IOMMMIOMapMMIOHCPage(pVM, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW | X86_PTE_P);
3507 AssertRC(rc2);
3508 }
3509 }
3510
3511 /* Forward it to our trap handler first, in case our shadow pages are out of sync. */
3512 rc = PGMTrap0eHandler(pVCpu, errCode, CPUMCTX2CORE(pCtx), (RTGCPTR)exitQualification);
3513 Log2(("PGMTrap0eHandler %RGv returned %Rrc\n", (RTGCPTR)pCtx->rip, VBOXSTRICTRC_VAL(rc)));
3514
3515 if (rc == VINF_SUCCESS)
3516 { /* We've successfully synced our shadow pages, so let's just continue execution. */
3517 Log2(("Shadow page fault at %RGv cr2=%RGv error code %x\n", (RTGCPTR)pCtx->rip, exitQualification ,errCode));
3518 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
3519
3520 TRPMResetTrap(pVCpu);
3521 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2Sub3, y3);
3522 goto ResumeExecution;
3523 }
3524 else if (rc == VINF_EM_RAW_GUEST_TRAP)
3525 {
3526 /*
3527 * A genuine pagefault. Forward the trap to the guest by injecting the exception and resuming execution.
3528 */
3529 Log2(("Forward page fault to the guest\n"));
3530
3531 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
3532 /* The error code might have been changed. */
3533 errCode = TRPMGetErrorCode(pVCpu);
3534
3535 TRPMResetTrap(pVCpu);
3536
3537 /* Now we must update CR2. */
3538 pCtx->cr2 = exitQualification;
3539 rc2 = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo),
3540 cbInstr, errCode);
3541 AssertRC(rc2);
3542
3543 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2Sub3, y3);
3544 goto ResumeExecution;
3545 }
3546#ifdef VBOX_STRICT
3547 if (rc != VINF_EM_RAW_EMULATE_INSTR && rc != VINF_EM_RAW_EMULATE_IO_BLOCK)
3548 Log2(("PGMTrap0eHandler failed with %d\n", VBOXSTRICTRC_VAL(rc)));
3549#endif
3550 /* Need to go back to the recompiler to emulate the instruction. */
3551 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
3552 TRPMResetTrap(pVCpu);
3553 break;
3554 }
3555
3556 case X86_XCPT_MF: /* Floating point exception. */
3557 {
3558 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
3559 if (!(pCtx->cr0 & X86_CR0_NE))
3560 {
3561 /* old style FPU error reporting needs some extra work. */
3562 /** @todo don't fall back to the recompiler, but do it manually. */
3563 rc = VINF_EM_RAW_EMULATE_INSTR;
3564 break;
3565 }
3566 Log(("Trap %x at %04X:%RGv\n", vector, pCtx->cs.Sel, (RTGCPTR)pCtx->rip));
3567 rc2 = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo),
3568 cbInstr, errCode);
3569 AssertRC(rc2);
3570
3571 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2Sub3, y3);
3572 goto ResumeExecution;
3573 }
3574
3575 case X86_XCPT_DB: /* Debug exception. */
3576 {
3577 uint64_t uDR6;
3578
3579 /*
3580 * DR6, DR7.GD and IA32_DEBUGCTL.LBR are not updated yet.
3581 *
3582 * Exit qualification bits:
3583 * 3:0 B0-B3 which breakpoint condition was met
3584 * 12:4 Reserved (0)
3585 * 13 BD - debug register access detected
3586 * 14 BS - single step execution or branch taken
3587 * 63:15 Reserved (0)
3588 */
3589 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
3590
3591 /* Note that we don't support guest and host-initiated debugging at the same time. */
3592
3593 uDR6 = X86_DR6_INIT_VAL;
3594 uDR6 |= (exitQualification & (X86_DR6_B0|X86_DR6_B1|X86_DR6_B2|X86_DR6_B3|X86_DR6_BD|X86_DR6_BS));
3595 rc = DBGFRZTrap01Handler(pVM, pVCpu, CPUMCTX2CORE(pCtx), uDR6);
3596 if (rc == VINF_EM_RAW_GUEST_TRAP)
3597 {
3598 /* Update DR6 here. */
3599 pCtx->dr[6] = uDR6;
3600
3601 /* Resync DR6 if the debug state is active. */
3602 if (CPUMIsGuestDebugStateActive(pVCpu))
3603 ASMSetDR6(pCtx->dr[6]);
3604
3605 /* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
3606 pCtx->dr[7] &= ~X86_DR7_GD;
3607
3608 /* Paranoia. */
3609 pCtx->dr[7] &= 0xffffffff; /* upper 32 bits reserved */
3610 pCtx->dr[7] &= ~(RT_BIT(11) | RT_BIT(12) | RT_BIT(14) | RT_BIT(15)); /* must be zero */
3611 pCtx->dr[7] |= 0x400; /* must be one */
3612
3613 /* Resync DR7 */
3614 rc2 = VMXWriteVMCS64(VMX_VMCS64_GUEST_DR7, pCtx->dr[7]);
3615 AssertRC(rc2);
3616
3617 Log(("Trap %x (debug) at %RGv exit qualification %RX64 dr6=%x dr7=%x\n", vector, (RTGCPTR)pCtx->rip,
3618 exitQualification, (uint32_t)pCtx->dr[6], (uint32_t)pCtx->dr[7]));
3619 rc2 = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo),
3620 cbInstr, errCode);
3621 AssertRC(rc2);
3622
3623 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2Sub3, y3);
3624 goto ResumeExecution;
3625 }
3626 /* Return to ring 3 to deal with the debug exit code. */
3627 Log(("Debugger hardware BP at %04x:%RGv (rc=%Rrc)\n", pCtx->cs.Sel, pCtx->rip, VBOXSTRICTRC_VAL(rc)));
3628 break;
3629 }
3630
3631 case X86_XCPT_BP: /* Breakpoint. */
3632 {
3633 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
3634 rc = DBGFRZTrap03Handler(pVM, pVCpu, CPUMCTX2CORE(pCtx));
3635 if (rc == VINF_EM_RAW_GUEST_TRAP)
3636 {
3637 Log(("Guest #BP at %04x:%RGv\n", pCtx->cs.Sel, pCtx->rip));
3638 rc2 = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo),
3639 cbInstr, errCode);
3640 AssertRC(rc2);
3641 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2Sub3, y3);
3642 goto ResumeExecution;
3643 }
3644 if (rc == VINF_SUCCESS)
3645 {
3646 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2Sub3, y3);
3647 goto ResumeExecution;
3648 }
3649 Log(("Debugger BP at %04x:%RGv (rc=%Rrc)\n", pCtx->cs.Sel, pCtx->rip, VBOXSTRICTRC_VAL(rc)));
3650 break;
3651 }
3652
3653 case X86_XCPT_GP: /* General protection failure exception. */
3654 {
3655 uint32_t cbOp;
3656 PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
3657
3658 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
3659#ifdef VBOX_STRICT
3660 if ( !CPUMIsGuestInRealModeEx(pCtx)
3661 || !pVM->hm.s.vmx.pRealModeTSS)
3662 {
3663 Log(("Trap %x at %04X:%RGv errorCode=%RGv\n", vector, pCtx->cs.Sel, (RTGCPTR)pCtx->rip, errCode));
3664 rc2 = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo),
3665 cbInstr, errCode);
3666 AssertRC(rc2);
3667 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2Sub3, y3);
3668 goto ResumeExecution;
3669 }
3670#endif
3671 Assert(CPUMIsGuestInRealModeEx(pCtx));
3672
3673 LogFlow(("Real mode X86_XCPT_GP instruction emulation at %x:%RGv\n", pCtx->cs.Sel, (RTGCPTR)pCtx->rip));
3674
3675 rc2 = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbOp);
3676 if (RT_SUCCESS(rc2))
3677 {
3678 bool fUpdateRIP = true;
3679
3680 rc = VINF_SUCCESS;
3681 Assert(cbOp == pDis->cbInstr);
3682 switch (pDis->pCurInstr->uOpcode)
3683 {
3684 case OP_CLI:
3685 pCtx->eflags.Bits.u1IF = 0;
3686 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCli);
3687 break;
3688
3689 case OP_STI:
3690 pCtx->eflags.Bits.u1IF = 1;
3691 EMSetInhibitInterruptsPC(pVCpu, pCtx->rip + pDis->cbInstr);
3692 Assert(VMCPU_FF_ISSET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
3693 rc2 = VMXWriteVMCS(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE,
3694 VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
3695 AssertRC(rc2);
3696 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitSti);
3697 break;
3698
3699 case OP_HLT:
3700 fUpdateRIP = false;
3701 rc = VINF_EM_HALT;
3702 pCtx->rip += pDis->cbInstr;
3703 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
3704 break;
3705
3706 case OP_POPF:
3707 {
3708 RTGCPTR GCPtrStack;
3709 uint32_t cbParm;
3710 uint32_t uMask;
3711 X86EFLAGS eflags;
3712
3713 if (pDis->fPrefix & DISPREFIX_OPSIZE)
3714 {
3715 cbParm = 4;
3716 uMask = 0xffffffff;
3717 }
3718 else
3719 {
3720 cbParm = 2;
3721 uMask = 0xffff;
3722 }
3723
3724 rc2 = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pCtx), pCtx->esp & uMask, 0, &GCPtrStack);
3725 if (RT_FAILURE(rc2))
3726 {
3727 rc = VERR_EM_INTERPRETER;
3728 break;
3729 }
3730 eflags.u = 0;
3731 rc2 = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &eflags.u, cbParm);
3732 if (RT_FAILURE(rc2))
3733 {
3734 rc = VERR_EM_INTERPRETER;
3735 break;
3736 }
3737 LogFlow(("POPF %x -> %RGv mask=%x\n", eflags.u, pCtx->rsp, uMask));
3738 pCtx->eflags.u = (pCtx->eflags.u & ~(X86_EFL_POPF_BITS & uMask))
3739 | (eflags.u & X86_EFL_POPF_BITS & uMask);
3740 /* RF cleared when popped in real mode; see pushf description in AMD manual. */
3741 pCtx->eflags.Bits.u1RF = 0;
3742 pCtx->esp += cbParm;
3743 pCtx->esp &= uMask;
3744
3745 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPopf);
3746 break;
3747 }
3748
3749 case OP_PUSHF:
3750 {
3751 RTGCPTR GCPtrStack;
3752 uint32_t cbParm;
3753 uint32_t uMask;
3754 X86EFLAGS eflags;
3755
3756 if (pDis->fPrefix & DISPREFIX_OPSIZE)
3757 {
3758 cbParm = 4;
3759 uMask = 0xffffffff;
3760 }
3761 else
3762 {
3763 cbParm = 2;
3764 uMask = 0xffff;
3765 }
3766
3767 rc2 = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pCtx), (pCtx->esp - cbParm) & uMask, 0,
3768 &GCPtrStack);
3769 if (RT_FAILURE(rc2))
3770 {
3771 rc = VERR_EM_INTERPRETER;
3772 break;
3773 }
3774 eflags = pCtx->eflags;
3775 /* RF & VM cleared when pushed in real mode; see pushf description in AMD manual. */
3776 eflags.Bits.u1RF = 0;
3777 eflags.Bits.u1VM = 0;
3778
3779 rc2 = PGMPhysWrite(pVM, (RTGCPHYS)GCPtrStack, &eflags.u, cbParm);
3780 if (RT_FAILURE(rc2))
3781 {
3782 rc = VERR_EM_INTERPRETER;
3783 break;
3784 }
3785 LogFlow(("PUSHF %x -> %RGv\n", eflags.u, GCPtrStack));
3786 pCtx->esp -= cbParm;
3787 pCtx->esp &= uMask;
3788 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPushf);
3789 break;
3790 }
3791
3792 case OP_IRET:
3793 {
3794 RTGCPTR GCPtrStack;
3795 uint32_t uMask = 0xffff;
3796 uint16_t aIretFrame[3];
3797
3798 if (pDis->fPrefix & (DISPREFIX_OPSIZE | DISPREFIX_ADDRSIZE))
3799 {
3800 rc = VERR_EM_INTERPRETER;
3801 break;
3802 }
3803
3804 rc2 = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pCtx), pCtx->esp & uMask, 0, &GCPtrStack);
3805 if (RT_FAILURE(rc2))
3806 {
3807 rc = VERR_EM_INTERPRETER;
3808 break;
3809 }
3810 rc2 = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &aIretFrame[0], sizeof(aIretFrame));
3811 if (RT_FAILURE(rc2))
3812 {
3813 rc = VERR_EM_INTERPRETER;
3814 break;
3815 }
3816 pCtx->ip = aIretFrame[0];
3817 pCtx->cs.Sel = aIretFrame[1];
3818 pCtx->cs.ValidSel = aIretFrame[1];
3819 pCtx->cs.u64Base = (uint32_t)pCtx->cs.Sel << 4;
3820 pCtx->eflags.u = (pCtx->eflags.u & ~(X86_EFL_POPF_BITS & uMask))
3821 | (aIretFrame[2] & X86_EFL_POPF_BITS & uMask);
3822 pCtx->sp += sizeof(aIretFrame);
3823
3824 LogFlow(("iret to %04x:%x\n", pCtx->cs.Sel, pCtx->ip));
3825 fUpdateRIP = false;
3826 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIret);
3827 break;
3828 }
3829
3830 case OP_INT:
3831 {
3832 uint32_t intInfo2;
3833
3834 LogFlow(("Realmode: INT %x\n", pDis->Param1.uValue & 0xff));
3835 intInfo2 = pDis->Param1.uValue & 0xff;
3836 intInfo2 |= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
3837 intInfo2 |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
3838
3839 rc = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, intInfo2, cbOp, 0);
3840 AssertRC(VBOXSTRICTRC_VAL(rc));
3841 fUpdateRIP = false;
3842 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
3843 break;
3844 }
3845
3846 case OP_INTO:
3847 {
3848 if (pCtx->eflags.Bits.u1OF)
3849 {
3850 uint32_t intInfo2;
3851
3852 LogFlow(("Realmode: INTO\n"));
3853 intInfo2 = X86_XCPT_OF;
3854 intInfo2 |= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
3855 intInfo2 |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
3856
3857 rc = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, intInfo2, cbOp, 0);
3858 AssertRC(VBOXSTRICTRC_VAL(rc));
3859 fUpdateRIP = false;
3860 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
3861 }
3862 break;
3863 }
3864
3865 case OP_INT3:
3866 {
3867 uint32_t intInfo2;
3868
3869 LogFlow(("Realmode: INT 3\n"));
3870 intInfo2 = 3;
3871 intInfo2 |= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
3872 intInfo2 |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
3873
3874 rc = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, intInfo2, cbOp, 0);
3875 AssertRC(VBOXSTRICTRC_VAL(rc));
3876 fUpdateRIP = false;
3877 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
3878 break;
3879 }
3880
3881 default:
3882 rc = EMInterpretInstructionDisasState(pVCpu, pDis, CPUMCTX2CORE(pCtx), 0, EMCODETYPE_SUPERVISOR);
3883 fUpdateRIP = false;
3884 break;
3885 }
3886
3887 if (rc == VINF_SUCCESS)
3888 {
3889 if (fUpdateRIP)
3890 pCtx->rip += cbOp; /* Move on to the next instruction. */
3891
3892 /*
3893 * LIDT, LGDT can end up here. In the future CRx changes as well. Just reload the
3894 * whole context to be done with it.
3895 */
3896 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_ALL;
3897
3898 /* Only resume if successful. */
3899 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2Sub3, y3);
3900 goto ResumeExecution;
3901 }
3902 }
3903 else
3904 rc = VERR_EM_INTERPRETER;
3905
3906 AssertMsg(rc == VERR_EM_INTERPRETER || rc == VINF_PGM_CHANGE_MODE || rc == VINF_EM_HALT,
3907 ("Unexpected rc=%Rrc\n", VBOXSTRICTRC_VAL(rc)));
3908 break;
3909 }
3910
3911#ifdef VBOX_STRICT
3912 case X86_XCPT_XF: /* SIMD exception. */
3913 case X86_XCPT_DE: /* Divide error. */
3914 case X86_XCPT_UD: /* Unknown opcode exception. */
3915 case X86_XCPT_SS: /* Stack segment exception. */
3916 case X86_XCPT_NP: /* Segment not present exception. */
3917 {
3918 switch (vector)
3919 {
3920 case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE); break;
3921 case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD); break;
3922 case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS); break;
3923 case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP); break;
3924 case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF); break;
3925 }
3926
3927 Log(("Trap %x at %04X:%RGv\n", vector, pCtx->cs.Sel, (RTGCPTR)pCtx->rip));
3928 rc2 = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo),
3929 cbInstr, errCode);
3930 AssertRC(rc2);
3931
3932 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2Sub3, y3);
3933 goto ResumeExecution;
3934 }
3935#endif
3936 default:
3937 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
3938 if ( CPUMIsGuestInRealModeEx(pCtx)
3939 && pVM->hm.s.vmx.pRealModeTSS)
3940 {
3941 Log(("Real Mode Trap %x at %04x:%04X error code %x\n", vector, pCtx->cs.Sel, pCtx->eip, errCode));
3942 rc = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo),
3943 cbInstr, errCode);
3944 AssertRC(VBOXSTRICTRC_VAL(rc)); /* Strict RC check below. */
3945
3946 /* Go back to ring-3 in case of a triple fault. */
3947 if ( vector == X86_XCPT_DF
3948 && rc == VINF_EM_RESET)
3949 {
3950 break;
3951 }
3952
3953 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2Sub3, y3);
3954 goto ResumeExecution;
3955 }
3956 AssertMsgFailed(("Unexpected vm-exit caused by exception %x\n", vector));
3957 rc = VERR_VMX_UNEXPECTED_EXCEPTION;
3958 break;
3959 } /* switch (vector) */
3960
3961 break;
3962
3963 default:
3964 rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_CODE;
3965 AssertMsgFailed(("Unexpected interruption code %x\n", intInfo));
3966 break;
3967 }
3968
3969 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2Sub3, y3);
3970 break;
3971 }
3972
3973 /*
3974 * 48 EPT violation. An attempt to access memory with a guest-physical address was disallowed
3975 * by the configuration of the EPT paging structures.
3976 */
3977 case VMX_EXIT_EPT_VIOLATION:
3978 {
3979 RTGCPHYS GCPhys;
3980
3981 Assert(pVM->hm.s.fNestedPaging);
3982
3983 rc2 = VMXReadVMCS64(VMX_VMCS_EXIT_PHYS_ADDR_FULL, &GCPhys);
3984 AssertRC(rc2);
3985 Assert(((exitQualification >> 7) & 3) != 2);
3986
3987 /* Determine the kind of violation. */
3988 errCode = 0;
3989 if (exitQualification & VMX_EXIT_QUALIFICATION_EPT_INSTR_FETCH)
3990 errCode |= X86_TRAP_PF_ID;
3991
3992 if (exitQualification & VMX_EXIT_QUALIFICATION_EPT_DATA_WRITE)
3993 errCode |= X86_TRAP_PF_RW;
3994
3995 /* If the page is present, then it's a page level protection fault. */
3996 if (exitQualification & VMX_EXIT_QUALIFICATION_EPT_ENTRY_PRESENT)
3997 errCode |= X86_TRAP_PF_P;
3998 else
3999 {
4000 /* Shortcut for APIC TPR reads and writes. */
4001 if ( (GCPhys & 0xfff) == 0x080
4002 && GCPhys > 0x1000000 /* to skip VGA frame buffer accesses */
4003 && fSetupTPRCaching
4004 && (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC))
4005 {
4006 RTGCPHYS GCPhysApicBase;
4007 PDMApicGetBase(pVM, &GCPhysApicBase); /* @todo cache this */
4008 GCPhysApicBase &= PAGE_BASE_GC_MASK;
4009 if (GCPhys == GCPhysApicBase + 0x80)
4010 {
4011 Log(("Enable VT-x virtual APIC access filtering\n"));
4012 rc2 = IOMMMIOMapMMIOHCPage(pVM, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW | X86_PTE_P);
4013 AssertRC(rc2);
4014 }
4015 }
4016 }
4017 Log(("EPT Page fault %x at %RGp error code %x\n", (uint32_t)exitQualification, GCPhys, errCode));
4018
4019 /* GCPhys contains the guest physical address of the page fault. */
4020 TRPMAssertTrap(pVCpu, X86_XCPT_PF, TRPM_TRAP);
4021 TRPMSetErrorCode(pVCpu, errCode);
4022 TRPMSetFaultAddress(pVCpu, GCPhys);
4023
4024 /* Handle the pagefault trap for the nested shadow table. */
4025 rc = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, errCode, CPUMCTX2CORE(pCtx), GCPhys);
4026
4027 /*
4028 * Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment below, @bugref{6043}.
4029 */
4030 if ( rc == VINF_SUCCESS
4031 || rc == VERR_PAGE_TABLE_NOT_PRESENT
4032 || rc == VERR_PAGE_NOT_PRESENT)
4033 {
4034 /* We've successfully synced our shadow pages, so let's just continue execution. */
4035 Log2(("Shadow page fault at %RGv cr2=%RGp error code %x\n", (RTGCPTR)pCtx->rip, exitQualification , errCode));
4036 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNPF);
4037
4038 TRPMResetTrap(pVCpu);
4039 goto ResumeExecution;
4040 }
4041
4042#ifdef VBOX_STRICT
4043 if (rc != VINF_EM_RAW_EMULATE_INSTR)
4044 LogFlow(("PGMTrap0eHandlerNestedPaging at %RGv failed with %Rrc\n", (RTGCPTR)pCtx->rip, VBOXSTRICTRC_VAL(rc)));
4045#endif
4046 /* Need to go back to the recompiler to emulate the instruction. */
4047 TRPMResetTrap(pVCpu);
4048 break;
4049 }
4050
4051 case VMX_EXIT_EPT_MISCONFIG:
4052 {
4053 RTGCPHYS GCPhys;
4054
4055 Assert(pVM->hm.s.fNestedPaging);
4056
4057 rc2 = VMXReadVMCS64(VMX_VMCS_EXIT_PHYS_ADDR_FULL, &GCPhys);
4058 AssertRC(rc2);
4059 Log(("VMX_EXIT_EPT_MISCONFIG for %RGp\n", GCPhys));
4060
4061 /* Shortcut for APIC TPR reads and writes. */
4062 if ( (GCPhys & 0xfff) == 0x080
4063 && GCPhys > 0x1000000 /* to skip VGA frame buffer accesses */
4064 && fSetupTPRCaching
4065 && (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC))
4066 {
4067 RTGCPHYS GCPhysApicBase;
4068 PDMApicGetBase(pVM, &GCPhysApicBase); /* @todo cache this */
4069 GCPhysApicBase &= PAGE_BASE_GC_MASK;
4070 if (GCPhys == GCPhysApicBase + 0x80)
4071 {
4072 Log(("Enable VT-x virtual APIC access filtering\n"));
4073 rc2 = IOMMMIOMapMMIOHCPage(pVM, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW | X86_PTE_P);
4074 AssertRC(rc2);
4075 }
4076 }
4077
4078 rc = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pCtx), GCPhys, UINT32_MAX);
4079
4080 /*
4081 * If we succeed, resume execution.
4082 * Or, if fail in interpreting the instruction because we couldn't get the guest physical address
4083 * of the page containing the instruction via the guest's page tables (we would invalidate the guest page
4084 * in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
4085 * weird case. See @bugref{6043}.
4086 */
4087 if ( rc == VINF_SUCCESS
4088 || rc == VERR_PAGE_TABLE_NOT_PRESENT
4089 || rc == VERR_PAGE_NOT_PRESENT)
4090 {
4091 Log2(("PGMR0Trap0eHandlerNPMisconfig(,,,%RGp) at %RGv -> resume\n", GCPhys, (RTGCPTR)pCtx->rip));
4092 goto ResumeExecution;
4093 }
4094
4095 Log2(("PGMR0Trap0eHandlerNPMisconfig(,,,%RGp) at %RGv -> %Rrc\n", GCPhys, (RTGCPTR)pCtx->rip, VBOXSTRICTRC_VAL(rc)));
4096 break;
4097 }
4098
4099 case VMX_EXIT_IRQ_WINDOW: /* 7 Interrupt window. */
4100 /* Clear VM-exit on IF=1 change. */
4101 LogFlow(("VMX_EXIT_IRQ_WINDOW %RGv pending=%d IF=%d\n", (RTGCPTR)pCtx->rip,
4102 VMCPU_FF_ISPENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC|VMCPU_FF_INTERRUPT_PIC)), pCtx->eflags.Bits.u1IF));
4103 pVCpu->hm.s.vmx.proc_ctls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_IRQ_WINDOW_EXIT;
4104 rc2 = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.proc_ctls);
4105 AssertRC(rc2);
4106 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIrqWindow);
4107 goto ResumeExecution; /* we check for pending guest interrupts there */
4108
4109 case VMX_EXIT_WBINVD: /* 54 Guest software attempted to execute WBINVD. (conditional) */
4110 case VMX_EXIT_INVD: /* 13 Guest software attempted to execute INVD. (unconditional) */
4111 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvd);
4112 /* Skip instruction and continue directly. */
4113 pCtx->rip += cbInstr;
4114 /* Continue execution.*/
4115 goto ResumeExecution;
4116
4117 case VMX_EXIT_CPUID: /* 10 Guest software attempted to execute CPUID. */
4118 {
4119 Log2(("VMX: Cpuid %x\n", pCtx->eax));
4120 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCpuid);
4121 rc = EMInterpretCpuId(pVM, pVCpu, CPUMCTX2CORE(pCtx));
4122 if (rc == VINF_SUCCESS)
4123 {
4124 /* Update EIP and continue execution. */
4125 Assert(cbInstr == 2);
4126 pCtx->rip += cbInstr;
4127 goto ResumeExecution;
4128 }
4129 AssertMsgFailed(("EMU: cpuid failed with %Rrc\n", VBOXSTRICTRC_VAL(rc)));
4130 rc = VINF_EM_RAW_EMULATE_INSTR;
4131 break;
4132 }
4133
4134 case VMX_EXIT_RDPMC: /* 15 Guest software attempted to execute RDPMC. */
4135 {
4136 Log2(("VMX: Rdpmc %x\n", pCtx->ecx));
4137 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdpmc);
4138 rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pCtx));
4139 if (rc == VINF_SUCCESS)
4140 {
4141 /* Update EIP and continue execution. */
4142 Assert(cbInstr == 2);
4143 pCtx->rip += cbInstr;
4144 goto ResumeExecution;
4145 }
4146 rc = VINF_EM_RAW_EMULATE_INSTR;
4147 break;
4148 }
4149
4150 case VMX_EXIT_RDTSC: /* 16 Guest software attempted to execute RDTSC. */
4151 {
4152 Log2(("VMX: Rdtsc\n"));
4153 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
4154 rc = EMInterpretRdtsc(pVM, pVCpu, CPUMCTX2CORE(pCtx));
4155 if (rc == VINF_SUCCESS)
4156 {
4157 /* Update EIP and continue execution. */
4158 Assert(cbInstr == 2);
4159 pCtx->rip += cbInstr;
4160 goto ResumeExecution;
4161 }
4162 rc = VINF_EM_RAW_EMULATE_INSTR;
4163 break;
4164 }
4165
4166 case VMX_EXIT_RDTSCP: /* 51 Guest software attempted to execute RDTSCP. */
4167 {
4168 Log2(("VMX: Rdtscp\n"));
4169 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtscp);
4170 rc = EMInterpretRdtscp(pVM, pVCpu, pCtx);
4171 if (rc == VINF_SUCCESS)
4172 {
4173 /* Update EIP and continue execution. */
4174 Assert(cbInstr == 3);
4175 pCtx->rip += cbInstr;
4176 goto ResumeExecution;
4177 }
4178 rc = VINF_EM_RAW_EMULATE_INSTR;
4179 break;
4180 }
4181
4182 case VMX_EXIT_INVLPG: /* 14 Guest software attempted to execute INVLPG. */
4183 {
4184 Log2(("VMX: invlpg\n"));
4185 Assert(!pVM->hm.s.fNestedPaging);
4186
4187 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvlpg);
4188 rc = EMInterpretInvlpg(pVM, pVCpu, CPUMCTX2CORE(pCtx), exitQualification);
4189 if (rc == VINF_SUCCESS)
4190 {
4191 /* Update EIP and continue execution. */
4192 pCtx->rip += cbInstr;
4193 goto ResumeExecution;
4194 }
4195 AssertMsg(rc == VERR_EM_INTERPRETER, ("EMU: invlpg %RGv failed with %Rrc\n", exitQualification, VBOXSTRICTRC_VAL(rc)));
4196 break;
4197 }
4198
4199 case VMX_EXIT_MONITOR: /* 39 Guest software attempted to execute MONITOR. */
4200 {
4201 Log2(("VMX: monitor\n"));
4202
4203 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMonitor);
4204 rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pCtx));
4205 if (rc == VINF_SUCCESS)
4206 {
4207 /* Update EIP and continue execution. */
4208 pCtx->rip += cbInstr;
4209 goto ResumeExecution;
4210 }
4211 AssertMsg(rc == VERR_EM_INTERPRETER, ("EMU: monitor failed with %Rrc\n", VBOXSTRICTRC_VAL(rc)));
4212 break;
4213 }
4214
4215 case VMX_EXIT_WRMSR: /* 32 WRMSR. Guest software attempted to execute WRMSR. */
4216 /* When an interrupt is pending, we'll let MSR_K8_LSTAR writes fault in our TPR patch code. */
4217 if ( pVM->hm.s.fTPRPatchingActive
4218 && pCtx->ecx == MSR_K8_LSTAR)
4219 {
4220 Assert(!CPUMIsGuestInLongModeEx(pCtx));
4221 if ((pCtx->eax & 0xff) != u8LastTPR)
4222 {
4223 Log(("VMX: Faulting MSR_K8_LSTAR write with new TPR value %x\n", pCtx->eax & 0xff));
4224
4225 /* Our patch code uses LSTAR for TPR caching. */
4226 rc2 = PDMApicSetTPR(pVCpu, pCtx->eax & 0xff);
4227 AssertRC(rc2);
4228 }
4229
4230 /* Skip the instruction and continue. */
4231 pCtx->rip += cbInstr; /* wrmsr = [0F 30] */
4232
4233 /* Only resume if successful. */
4234 goto ResumeExecution;
4235 }
4236 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_MSR;
4237 /* no break */
4238 case VMX_EXIT_RDMSR: /* 31 RDMSR. Guest software attempted to execute RDMSR. */
4239 {
4240 STAM_COUNTER_INC((exitReason == VMX_EXIT_RDMSR) ? &pVCpu->hm.s.StatExitRdmsr : &pVCpu->hm.s.StatExitWrmsr);
4241
4242 /*
4243 * Note: The Intel spec. claims there's an REX version of RDMSR that's slightly different,
4244 * so we play safe by completely disassembling the instruction.
4245 */
4246 Log2(("VMX: %s\n", (exitReason == VMX_EXIT_RDMSR) ? "rdmsr" : "wrmsr"));
4247 rc = EMInterpretInstruction(pVCpu, CPUMCTX2CORE(pCtx), 0);
4248 if (rc == VINF_SUCCESS)
4249 {
4250 /* EIP has been updated already. */
4251 /* Only resume if successful. */
4252 goto ResumeExecution;
4253 }
4254 AssertMsg(rc == VERR_EM_INTERPRETER, ("EMU: %s failed with %Rrc\n",
4255 (exitReason == VMX_EXIT_RDMSR) ? "rdmsr" : "wrmsr", VBOXSTRICTRC_VAL(rc)));
4256 break;
4257 }
4258
4259 case VMX_EXIT_CRX_MOVE: /* 28 Control-register accesses. */
4260 {
4261 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExit2Sub2, y2);
4262
4263 switch (VMX_EXIT_QUALIFICATION_CRX_ACCESS(exitQualification))
4264 {
4265 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_WRITE:
4266 {
4267 Log2(("VMX: %RGv mov cr%d, x\n", (RTGCPTR)pCtx->rip, VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification)));
4268 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxWrite[VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification)]);
4269 rc = EMInterpretCRxWrite(pVM, pVCpu, CPUMCTX2CORE(pCtx),
4270 VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification),
4271 VMX_EXIT_QUALIFICATION_CRX_GENREG(exitQualification));
4272 switch (VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification))
4273 {
4274 case 0:
4275 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_CR0 | HM_CHANGED_GUEST_CR3;
4276 break;
4277 case 2:
4278 break;
4279 case 3:
4280 Assert(!pVM->hm.s.fNestedPaging || !CPUMIsGuestInPagedProtectedModeEx(pCtx));
4281 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_CR3;
4282 break;
4283 case 4:
4284 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_CR4;
4285 break;
4286 case 8:
4287 /* CR8 contains the APIC TPR */
4288 Assert(!(pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1
4289 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW));
4290 break;
4291
4292 default:
4293 AssertFailed();
4294 break;
4295 }
4296 break;
4297 }
4298
4299 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ:
4300 {
4301 Log2(("VMX: mov x, crx\n"));
4302 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxRead[VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification)]);
4303
4304 Assert( !pVM->hm.s.fNestedPaging
4305 || !CPUMIsGuestInPagedProtectedModeEx(pCtx)
4306 || VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification) != DISCREG_CR3);
4307
4308 /* CR8 reads only cause an exit when the TPR shadow feature isn't present. */
4309 Assert( VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification) != 8
4310 || !(pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW));
4311
4312 rc = EMInterpretCRxRead(pVM, pVCpu, CPUMCTX2CORE(pCtx),
4313 VMX_EXIT_QUALIFICATION_CRX_GENREG(exitQualification),
4314 VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification));
4315 break;
4316 }
4317
4318 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_CLTS:
4319 {
4320 Log2(("VMX: clts\n"));
4321 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCLTS);
4322 rc = EMInterpretCLTS(pVM, pVCpu);
4323 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_CR0;
4324 break;
4325 }
4326
4327 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_LMSW:
4328 {
4329 Log2(("VMX: lmsw %x\n", VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(exitQualification)));
4330 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLMSW);
4331 rc = EMInterpretLMSW(pVM, pVCpu, CPUMCTX2CORE(pCtx), VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(exitQualification));
4332 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_CR0;
4333 break;
4334 }
4335 }
4336
4337 /* Update EIP if no error occurred. */
4338 if (RT_SUCCESS(rc))
4339 pCtx->rip += cbInstr;
4340
4341 if (rc == VINF_SUCCESS)
4342 {
4343 /* Only resume if successful. */
4344 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2Sub2, y2);
4345 goto ResumeExecution;
4346 }
4347 Assert(rc == VERR_EM_INTERPRETER || rc == VINF_PGM_CHANGE_MODE || rc == VINF_PGM_SYNC_CR3);
4348 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2Sub2, y2);
4349 break;
4350 }
4351
4352 case VMX_EXIT_DRX_MOVE: /* 29 Debug-register accesses. */
4353 {
4354 if ( !DBGFIsStepping(pVCpu)
4355 && !CPUMIsHyperDebugStateActive(pVCpu))
4356 {
4357 /* Disable DRx move intercepts. */
4358 pVCpu->hm.s.vmx.proc_ctls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT;
4359 rc2 = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.proc_ctls);
4360 AssertRC(rc2);
4361
4362 /* Save the host and load the guest debug state. */
4363 rc2 = CPUMR0LoadGuestDebugState(pVM, pVCpu, pCtx, true /* include DR6 */);
4364 AssertRC(rc2);
4365
4366#ifdef LOG_ENABLED
4367 if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(exitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
4368 {
4369 Log(("VMX_EXIT_DRX_MOVE: write DR%d genreg %d\n", VMX_EXIT_QUALIFICATION_DRX_REGISTER(exitQualification),
4370 VMX_EXIT_QUALIFICATION_DRX_GENREG(exitQualification)));
4371 }
4372 else
4373 Log(("VMX_EXIT_DRX_MOVE: read DR%d\n", VMX_EXIT_QUALIFICATION_DRX_REGISTER(exitQualification)));
4374#endif
4375
4376#ifdef VBOX_WITH_STATISTICS
4377 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
4378 if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(exitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
4379 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
4380 else
4381 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
4382#endif
4383
4384 goto ResumeExecution;
4385 }
4386
4387 /** @todo clear VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT after the first
4388 * time and restore DRx registers afterwards */
4389 if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(exitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
4390 {
4391 Log2(("VMX: mov DRx%d, genreg%d\n", VMX_EXIT_QUALIFICATION_DRX_REGISTER(exitQualification),
4392 VMX_EXIT_QUALIFICATION_DRX_GENREG(exitQualification)));
4393 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
4394 rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pCtx),
4395 VMX_EXIT_QUALIFICATION_DRX_REGISTER(exitQualification),
4396 VMX_EXIT_QUALIFICATION_DRX_GENREG(exitQualification));
4397 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_DEBUG;
4398 Log2(("DR7=%08x\n", pCtx->dr[7]));
4399 }
4400 else
4401 {
4402 Log2(("VMX: mov x, DRx\n"));
4403 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
4404 rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pCtx),
4405 VMX_EXIT_QUALIFICATION_DRX_GENREG(exitQualification),
4406 VMX_EXIT_QUALIFICATION_DRX_REGISTER(exitQualification));
4407 }
4408 /* Update EIP if no error occurred. */
4409 if (RT_SUCCESS(rc))
4410 pCtx->rip += cbInstr;
4411
4412 if (rc == VINF_SUCCESS)
4413 {
4414 /* Only resume if successful. */
4415 goto ResumeExecution;
4416 }
4417 Assert(rc == VERR_EM_INTERPRETER);
4418 break;
4419 }
4420
4421 /* Note: We'll get a #GP if the IO instruction isn't allowed (IOPL or TSS bitmap); no need to double check. */
4422 case VMX_EXIT_PORT_IO: /* 30 I/O instruction. */
4423 {
4424 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExit2Sub1, y1);
4425 uint32_t uPort;
4426 uint32_t uIOWidth = VMX_EXIT_QUALIFICATION_IO_WIDTH(exitQualification);
4427 bool fIOWrite = (VMX_EXIT_QUALIFICATION_IO_DIRECTION(exitQualification) == VMX_EXIT_QUALIFICATION_IO_DIRECTION_OUT);
4428
4429 /** @todo necessary to make the distinction? */
4430 if (VMX_EXIT_QUALIFICATION_IO_ENCODING(exitQualification) == VMX_EXIT_QUALIFICATION_IO_ENCODING_DX)
4431 uPort = pCtx->edx & 0xffff;
4432 else
4433 uPort = VMX_EXIT_QUALIFICATION_IO_PORT(exitQualification); /* Immediate encoding. */
4434
4435 if (RT_UNLIKELY(uIOWidth == 2 || uIOWidth >= 4)) /* paranoia */
4436 {
4437 rc = fIOWrite ? VINF_IOM_R3_IOPORT_WRITE : VINF_IOM_R3_IOPORT_READ;
4438 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2Sub1, y1);
4439 break;
4440 }
4441
4442 uint32_t cbSize = g_aIOSize[uIOWidth];
4443 if (VMX_EXIT_QUALIFICATION_IO_STRING(exitQualification))
4444 {
4445 /* ins/outs */
4446 PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
4447
4448 /* Disassemble manually to deal with segment prefixes. */
4449 /** @todo VMX_VMCS_EXIT_GUEST_LINEAR_ADDR contains the flat pointer operand of the instruction. */
4450 /** @todo VMX_VMCS32_RO_EXIT_INSTR_INFO also contains segment prefix info. */
4451 rc2 = EMInterpretDisasCurrent(pVM, pVCpu, pDis, NULL);
4452 if (RT_SUCCESS(rc))
4453 {
4454 if (fIOWrite)
4455 {
4456 Log2(("IOMInterpretOUTSEx %RGv %x size=%d\n", (RTGCPTR)pCtx->rip, uPort, cbSize));
4457 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringWrite);
4458 rc = IOMInterpretOUTSEx(pVM, CPUMCTX2CORE(pCtx), uPort, pDis->fPrefix, (DISCPUMODE)pDis->uAddrMode, cbSize);
4459 }
4460 else
4461 {
4462 Log2(("IOMInterpretINSEx %RGv %x size=%d\n", (RTGCPTR)pCtx->rip, uPort, cbSize));
4463 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringRead);
4464 rc = IOMInterpretINSEx(pVM, CPUMCTX2CORE(pCtx), uPort, pDis->fPrefix, (DISCPUMODE)pDis->uAddrMode, cbSize);
4465 }
4466 }
4467 else
4468 rc = VINF_EM_RAW_EMULATE_INSTR;
4469 }
4470 else
4471 {
4472 /* Normal in/out */
4473 uint32_t uAndVal = g_aIOOpAnd[uIOWidth];
4474
4475 Assert(!VMX_EXIT_QUALIFICATION_IO_REP(exitQualification));
4476
4477 if (fIOWrite)
4478 {
4479 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
4480 rc = IOMIOPortWrite(pVM, uPort, pCtx->eax & uAndVal, cbSize);
4481 if (rc == VINF_IOM_R3_IOPORT_WRITE)
4482 HMR0SavePendingIOPortWrite(pVCpu, pCtx->rip, pCtx->rip + cbInstr, uPort, uAndVal, cbSize);
4483 }
4484 else
4485 {
4486 uint32_t u32Val = 0;
4487
4488 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
4489 rc = IOMIOPortRead(pVM, uPort, &u32Val, cbSize);
4490 if (IOM_SUCCESS(rc))
4491 {
4492 /* Write back to the EAX register. */
4493 pCtx->eax = (pCtx->eax & ~uAndVal) | (u32Val & uAndVal);
4494 }
4495 else
4496 if (rc == VINF_IOM_R3_IOPORT_READ)
4497 HMR0SavePendingIOPortRead(pVCpu, pCtx->rip, pCtx->rip + cbInstr, uPort, uAndVal, cbSize);
4498 }
4499 }
4500
4501 /*
4502 * Handled the I/O return codes.
4503 * (The unhandled cases end up with rc == VINF_EM_RAW_EMULATE_INSTR.)
4504 */
4505 if (IOM_SUCCESS(rc))
4506 {
4507 /* Update EIP and continue execution. */
4508 pCtx->rip += cbInstr;
4509 if (RT_LIKELY(rc == VINF_SUCCESS))
4510 {
4511 /* If any IO breakpoints are armed, then we should check if a debug trap needs to be generated. */
4512 if (pCtx->dr[7] & X86_DR7_ENABLED_MASK)
4513 {
4514 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIOCheck);
4515 for (unsigned i = 0; i < 4; i++)
4516 {
4517 unsigned uBPLen = g_aIOSize[X86_DR7_GET_LEN(pCtx->dr[7], i)];
4518
4519 if ( (uPort >= pCtx->dr[i] && uPort < pCtx->dr[i] + uBPLen)
4520 && (pCtx->dr[7] & (X86_DR7_L(i) | X86_DR7_G(i)))
4521 && (pCtx->dr[7] & X86_DR7_RW(i, X86_DR7_RW_IO)) == X86_DR7_RW(i, X86_DR7_RW_IO))
4522 {
4523 uint64_t uDR6;
4524
4525 Assert(CPUMIsGuestDebugStateActive(pVCpu));
4526
4527 uDR6 = ASMGetDR6();
4528
4529 /* Clear all breakpoint status flags and set the one we just hit. */
4530 uDR6 &= ~(X86_DR6_B0|X86_DR6_B1|X86_DR6_B2|X86_DR6_B3);
4531 uDR6 |= (uint64_t)RT_BIT(i);
4532
4533 /*
4534 * Note: AMD64 Architecture Programmer's Manual 13.1:
4535 * Bits 15:13 of the DR6 register is never cleared by the processor and must
4536 * be cleared by software after the contents have been read.
4537 */
4538 ASMSetDR6(uDR6);
4539
4540 /* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
4541 pCtx->dr[7] &= ~X86_DR7_GD;
4542
4543 /* Paranoia. */
4544 pCtx->dr[7] &= 0xffffffff; /* upper 32 bits reserved */
4545 pCtx->dr[7] &= ~(RT_BIT(11) | RT_BIT(12) | RT_BIT(14) | RT_BIT(15)); /* must be zero */
4546 pCtx->dr[7] |= 0x400; /* must be one */
4547
4548 /* Resync DR7 */
4549 rc2 = VMXWriteVMCS64(VMX_VMCS64_GUEST_DR7, pCtx->dr[7]);
4550 AssertRC(rc2);
4551
4552 /* Construct inject info. */
4553 intInfo = X86_XCPT_DB;
4554 intInfo |= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
4555 intInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HWEXCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
4556
4557 Log(("Inject IO debug trap at %RGv\n", (RTGCPTR)pCtx->rip));
4558 rc2 = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo),
4559 0 /* cbInstr */, 0 /* errCode */);
4560 AssertRC(rc2);
4561
4562 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2Sub1, y1);
4563 goto ResumeExecution;
4564 }
4565 }
4566 }
4567 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2Sub1, y1);
4568 goto ResumeExecution;
4569 }
4570 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2Sub1, y1);
4571 break;
4572 }
4573
4574#ifdef VBOX_STRICT
4575 if (rc == VINF_IOM_R3_IOPORT_READ)
4576 Assert(!fIOWrite);
4577 else if (rc == VINF_IOM_R3_IOPORT_WRITE)
4578 Assert(fIOWrite);
4579 else
4580 {
4581 AssertMsg( RT_FAILURE(rc)
4582 || rc == VINF_EM_RAW_EMULATE_INSTR
4583 || rc == VINF_EM_RAW_GUEST_TRAP
4584 || rc == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", VBOXSTRICTRC_VAL(rc)));
4585 }
4586#endif
4587 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2Sub1, y1);
4588 break;
4589 }
4590
4591 case VMX_EXIT_TPR: /* 43 TPR below threshold. Guest software executed MOV to CR8. */
4592 LogFlow(("VMX_EXIT_TPR\n"));
4593 /* RIP is already set to the next instruction and the TPR has been synced back. Just resume. */
4594 goto ResumeExecution;
4595
4596 case VMX_EXIT_APIC_ACCESS: /* 44 APIC access. Guest software attempted to access memory at a physical address
4597 on the APIC-access page. */
4598 {
4599 LogFlow(("VMX_EXIT_APIC_ACCESS\n"));
4600 unsigned uAccessType = VMX_EXIT_QUALIFICATION_APIC_ACCESS_TYPE(exitQualification);
4601
4602 switch (uAccessType)
4603 {
4604 case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
4605 case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
4606 {
4607 RTGCPHYS GCPhys;
4608 PDMApicGetBase(pVM, &GCPhys);
4609 GCPhys &= PAGE_BASE_GC_MASK;
4610 GCPhys += VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(exitQualification);
4611
4612 LogFlow(("Apic access at %RGp\n", GCPhys));
4613 rc = IOMMMIOPhysHandler(pVM, (uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ) ? 0 : X86_TRAP_PF_RW,
4614 CPUMCTX2CORE(pCtx), GCPhys);
4615 if (rc == VINF_SUCCESS)
4616 goto ResumeExecution; /* rip already updated */
4617 break;
4618 }
4619
4620 default:
4621 rc = VINF_EM_RAW_EMULATE_INSTR;
4622 break;
4623 }
4624 break;
4625 }
4626
4627 case VMX_EXIT_PREEMPTION_TIMER: /* 52 VMX-preemption timer expired. The preemption timer counted down to zero. */
4628 if (!TMTimerPollBool(pVM, pVCpu))
4629 goto ResumeExecution;
4630 rc = VINF_EM_RAW_TIMER_PENDING;
4631 break;
4632
4633 default:
4634 /* The rest is handled after syncing the entire CPU state. */
4635 break;
4636 }
4637
4638
4639 /*
4640 * Note: The guest state is not entirely synced back at this stage!
4641 */
4642
4643 /* Investigate why there was a VM-exit. (part 2) */
4644 switch (exitReason)
4645 {
4646 case VMX_EXIT_EXCEPTION: /* 0 Exception or non-maskable interrupt (NMI). */
4647 case VMX_EXIT_EXTERNAL_IRQ: /* 1 External interrupt. */
4648 case VMX_EXIT_EPT_VIOLATION:
4649 case VMX_EXIT_EPT_MISCONFIG: /* 49 EPT misconfig is used by the PGM/MMIO optimizations. */
4650 case VMX_EXIT_PREEMPTION_TIMER: /* 52 VMX-preemption timer expired. The preemption timer counted down to zero. */
4651 /* Already handled above. */
4652 break;
4653
4654 case VMX_EXIT_TRIPLE_FAULT: /* 2 Triple fault. */
4655 rc = VINF_EM_RESET; /* Triple fault equals a reset. */
4656 break;
4657
4658 case VMX_EXIT_INIT_SIGNAL: /* 3 INIT signal. */
4659 case VMX_EXIT_SIPI: /* 4 Start-up IPI (SIPI). */
4660 rc = VINF_EM_RAW_INTERRUPT;
4661 AssertFailed(); /* Can't happen. Yet. */
4662 break;
4663
4664 case VMX_EXIT_IO_SMI_IRQ: /* 5 I/O system-management interrupt (SMI). */
4665 case VMX_EXIT_SMI_IRQ: /* 6 Other SMI. */
4666 rc = VINF_EM_RAW_INTERRUPT;
4667 AssertFailed(); /* Can't happen afaik. */
4668 break;
4669
4670 case VMX_EXIT_TASK_SWITCH: /* 9 Task switch: too complicated to emulate, so fall back to the recompiler */
4671 Log(("VMX_EXIT_TASK_SWITCH: exit=%RX64\n", exitQualification));
4672 if ( (VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE(exitQualification) == VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE_IDT)
4673 && pVCpu->hm.s.Event.fPending)
4674 {
4675 /* Caused by an injected interrupt. */
4676 pVCpu->hm.s.Event.fPending = false;
4677
4678 Log(("VMX_EXIT_TASK_SWITCH: reassert trap %d\n", VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVCpu->hm.s.Event.intInfo)));
4679 Assert(!VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.intInfo));
4680 rc2 = TRPMAssertTrap(pVCpu, VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVCpu->hm.s.Event.intInfo), TRPM_HARDWARE_INT);
4681 AssertRC(rc2);
4682 }
4683 /* else Exceptions and software interrupts can just be restarted. */
4684 rc = VERR_EM_INTERPRETER;
4685 break;
4686
4687 case VMX_EXIT_HLT: /* 12 Guest software attempted to execute HLT. */
4688 /* Check if external interrupts are pending; if so, don't switch back. */
4689 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
4690 pCtx->rip++; /* skip hlt */
4691 if (EMShouldContinueAfterHalt(pVCpu, pCtx))
4692 goto ResumeExecution;
4693
4694 rc = VINF_EM_HALT;
4695 break;
4696
4697 case VMX_EXIT_MWAIT: /* 36 Guest software executed MWAIT. */
4698 Log2(("VMX: mwait\n"));
4699 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMwait);
4700 rc = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pCtx));
4701 if ( rc == VINF_EM_HALT
4702 || rc == VINF_SUCCESS)
4703 {
4704 /* Update EIP and continue execution. */
4705 pCtx->rip += cbInstr;
4706
4707 /* Check if external interrupts are pending; if so, don't switch back. */
4708 if ( rc == VINF_SUCCESS
4709 || ( rc == VINF_EM_HALT
4710 && EMShouldContinueAfterHalt(pVCpu, pCtx))
4711 )
4712 goto ResumeExecution;
4713 }
4714 AssertMsg(rc == VERR_EM_INTERPRETER || rc == VINF_EM_HALT, ("EMU: mwait failed with %Rrc\n", VBOXSTRICTRC_VAL(rc)));
4715 break;
4716
4717 case VMX_EXIT_RSM: /* 17 Guest software attempted to execute RSM in SMM. */
4718 AssertFailed(); /* can't happen. */
4719 rc = VERR_EM_INTERPRETER;
4720 break;
4721
4722 case VMX_EXIT_MTF: /* 37 Exit due to Monitor Trap Flag. */
4723 LogFlow(("VMX_EXIT_MTF at %RGv\n", (RTGCPTR)pCtx->rip));
4724 pVCpu->hm.s.vmx.proc_ctls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_TRAP_FLAG;
4725 rc2 = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.proc_ctls);
4726 AssertRC(rc2);
4727 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMTF);
4728#if 0
4729 DBGFDoneStepping(pVCpu);
4730#endif
4731 rc = VINF_EM_DBG_STOP;
4732 break;
4733
4734 case VMX_EXIT_VMCALL: /* 18 Guest software executed VMCALL. */
4735 case VMX_EXIT_VMCLEAR: /* 19 Guest software executed VMCLEAR. */
4736 case VMX_EXIT_VMLAUNCH: /* 20 Guest software executed VMLAUNCH. */
4737 case VMX_EXIT_VMPTRLD: /* 21 Guest software executed VMPTRLD. */
4738 case VMX_EXIT_VMPTRST: /* 22 Guest software executed VMPTRST. */
4739 case VMX_EXIT_VMREAD: /* 23 Guest software executed VMREAD. */
4740 case VMX_EXIT_VMRESUME: /* 24 Guest software executed VMRESUME. */
4741 case VMX_EXIT_VMWRITE: /* 25 Guest software executed VMWRITE. */
4742 case VMX_EXIT_VMXOFF: /* 26 Guest software executed VMXOFF. */
4743 case VMX_EXIT_VMXON: /* 27 Guest software executed VMXON. */
4744 /** @todo inject #UD immediately */
4745 rc = VERR_EM_INTERPRETER;
4746 break;
4747
4748 case VMX_EXIT_CPUID: /* 10 Guest software attempted to execute CPUID. */
4749 case VMX_EXIT_RDTSC: /* 16 Guest software attempted to execute RDTSC. */
4750 case VMX_EXIT_INVLPG: /* 14 Guest software attempted to execute INVLPG. */
4751 case VMX_EXIT_CRX_MOVE: /* 28 Control-register accesses. */
4752 case VMX_EXIT_DRX_MOVE: /* 29 Debug-register accesses. */
4753 case VMX_EXIT_PORT_IO: /* 30 I/O instruction. */
4754 case VMX_EXIT_RDPMC: /* 15 Guest software attempted to execute RDPMC. */
4755 case VMX_EXIT_RDTSCP: /* 51 Guest software attempted to execute RDTSCP. */
4756 /* already handled above */
4757 AssertMsg( rc == VINF_PGM_CHANGE_MODE
4758 || rc == VINF_EM_RAW_INTERRUPT
4759 || rc == VERR_EM_INTERPRETER
4760 || rc == VINF_EM_RAW_EMULATE_INSTR
4761 || rc == VINF_PGM_SYNC_CR3
4762 || rc == VINF_IOM_R3_IOPORT_READ
4763 || rc == VINF_IOM_R3_IOPORT_WRITE
4764 || rc == VINF_EM_RAW_GUEST_TRAP
4765 || rc == VINF_TRPM_XCPT_DISPATCHED
4766 || rc == VINF_EM_RESCHEDULE_REM,
4767 ("rc = %d\n", VBOXSTRICTRC_VAL(rc)));
4768 break;
4769
4770 case VMX_EXIT_TPR: /* 43 TPR below threshold. Guest software executed MOV to CR8. */
4771 case VMX_EXIT_RDMSR: /* 31 RDMSR. Guest software attempted to execute RDMSR. */
4772 case VMX_EXIT_WRMSR: /* 32 WRMSR. Guest software attempted to execute WRMSR. */
4773 case VMX_EXIT_PAUSE: /* 40 Guest software attempted to execute PAUSE. */
4774 case VMX_EXIT_MONITOR: /* 39 Guest software attempted to execute MONITOR. */
4775 case VMX_EXIT_APIC_ACCESS: /* 44 APIC access. Guest software attempted to access memory at a physical address
4776 on the APIC-access page. */
4777 {
4778 /*
4779 * If we decided to emulate them here, then we must sync the MSRs that could have been changed (sysenter, FS/GS base)
4780 */
4781 rc = VERR_EM_INTERPRETER;
4782 break;
4783 }
4784
4785 case VMX_EXIT_IRQ_WINDOW: /* 7 Interrupt window. */
4786 Assert(rc == VINF_EM_RAW_INTERRUPT);
4787 break;
4788
4789 case VMX_EXIT_ERR_INVALID_GUEST_STATE: /* 33 VM-entry failure due to invalid guest state. */
4790 {
4791#ifdef VBOX_STRICT
4792 RTCCUINTREG val2 = 0;
4793
4794 Log(("VMX_EXIT_ERR_INVALID_GUEST_STATE\n"));
4795
4796 VMXReadVMCS(VMX_VMCS64_GUEST_RIP, &val2);
4797 Log(("Old eip %RGv new %RGv\n", (RTGCPTR)pCtx->rip, (RTGCPTR)val2));
4798
4799 VMXReadVMCS(VMX_VMCS64_GUEST_CR0, &val2);
4800 Log(("VMX_VMCS_GUEST_CR0 %RX64\n", (uint64_t)val2));
4801
4802 VMXReadVMCS(VMX_VMCS64_GUEST_CR3, &val2);
4803 Log(("VMX_VMCS_GUEST_CR3 %RX64\n", (uint64_t)val2));
4804
4805 VMXReadVMCS(VMX_VMCS64_GUEST_CR4, &val2);
4806 Log(("VMX_VMCS_GUEST_CR4 %RX64\n", (uint64_t)val2));
4807
4808 VMXReadVMCS(VMX_VMCS_GUEST_RFLAGS, &val2);
4809 Log(("VMX_VMCS_GUEST_RFLAGS %08x\n", val2));
4810
4811 VMX_LOG_SELREG(CS, "CS", val2);
4812 VMX_LOG_SELREG(DS, "DS", val2);
4813 VMX_LOG_SELREG(ES, "ES", val2);
4814 VMX_LOG_SELREG(FS, "FS", val2);
4815 VMX_LOG_SELREG(GS, "GS", val2);
4816 VMX_LOG_SELREG(SS, "SS", val2);
4817 VMX_LOG_SELREG(TR, "TR", val2);
4818 VMX_LOG_SELREG(LDTR, "LDTR", val2);
4819
4820 VMXReadVMCS(VMX_VMCS64_GUEST_GDTR_BASE, &val2);
4821 Log(("VMX_VMCS_GUEST_GDTR_BASE %RX64\n", (uint64_t)val2));
4822 VMXReadVMCS(VMX_VMCS64_GUEST_IDTR_BASE, &val2);
4823 Log(("VMX_VMCS_GUEST_IDTR_BASE %RX64\n", (uint64_t)val2));
4824#endif /* VBOX_STRICT */
4825 rc = VERR_VMX_INVALID_GUEST_STATE;
4826 break;
4827 }
4828
4829 case VMX_EXIT_ERR_MSR_LOAD: /* 34 VM-entry failure due to MSR loading. */
4830 case VMX_EXIT_ERR_MACHINE_CHECK: /* 41 VM-entry failure due to machine-check. */
4831 default:
4832 rc = VERR_VMX_UNEXPECTED_EXIT_CODE;
4833 AssertMsgFailed(("Unexpected exit code %d\n", exitReason)); /* Can't happen. */
4834 break;
4835
4836 }
4837
4838end:
4839 /* We now going back to ring-3, so clear the action flag. */
4840 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
4841
4842 /*
4843 * Signal changes for the recompiler.
4844 */
4845 CPUMSetChangedFlags(pVCpu,
4846 CPUM_CHANGED_SYSENTER_MSR
4847 | CPUM_CHANGED_LDTR
4848 | CPUM_CHANGED_GDTR
4849 | CPUM_CHANGED_IDTR
4850 | CPUM_CHANGED_TR
4851 | CPUM_CHANGED_HIDDEN_SEL_REGS);
4852
4853 /*
4854 * If we executed vmlaunch/vmresume and an external IRQ was pending, then we don't have to do a full sync the next time.
4855 */
4856 if ( exitReason == VMX_EXIT_EXTERNAL_IRQ
4857 && !VMX_EXIT_INTERRUPTION_INFO_VALID(intInfo))
4858 {
4859 STAM_COUNTER_INC(&pVCpu->hm.s.StatPendingHostIrq);
4860 /* On the next entry we'll only sync the host context. */
4861 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_HOST_CONTEXT;
4862 }
4863 else
4864 {
4865 /* On the next entry we'll sync everything. */
4866 /** @todo we can do better than this */
4867 /* Not in the VINF_PGM_CHANGE_MODE though! */
4868 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_ALL;
4869 }
4870
4871 /* Translate into a less severe return code */
4872 if (rc == VERR_EM_INTERPRETER)
4873 rc = VINF_EM_RAW_EMULATE_INSTR;
4874 else if (rc == VERR_VMX_INVALID_VMCS_PTR)
4875 {
4876 /* Try to extract more information about what might have gone wrong here. */
4877 VMXGetActivateVMCS(&pVCpu->hm.s.vmx.lasterror.u64VMCSPhys);
4878 pVCpu->hm.s.vmx.lasterror.ulVMCSRevision = *(uint32_t *)pVCpu->hm.s.vmx.pvVMCS;
4879 pVCpu->hm.s.vmx.lasterror.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
4880 pVCpu->hm.s.vmx.lasterror.idCurrentCpu = RTMpCpuId();
4881 }
4882
4883 /* Just set the correct state here instead of trying to catch every goto above. */
4884 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED, VMCPUSTATE_STARTED_EXEC);
4885
4886#ifdef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
4887 /* Restore interrupts if we exited after disabling them. */
4888 if (uOldEFlags != ~(RTCCUINTREG)0)
4889 ASMSetFlags(uOldEFlags);
4890#endif
4891
4892 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
4893 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
4894 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
4895 Log2(("X"));
4896 return VBOXSTRICTRC_TODO(rc);
4897}
4898
4899
4900/**
4901 * Enters the VT-x session.
4902 *
4903 * @returns VBox status code.
4904 * @param pVM Pointer to the VM.
4905 * @param pVCpu Pointer to the VMCPU.
4906 * @param pCpu Pointer to the CPU info struct.
4907 */
4908VMMR0DECL(int) VMXR0Enter(PVM pVM, PVMCPU pVCpu, PHMGLOBLCPUINFO pCpu)
4909{
4910 Assert(pVM->hm.s.vmx.fSupported);
4911 NOREF(pCpu);
4912
4913 unsigned cr4 = ASMGetCR4();
4914 if (!(cr4 & X86_CR4_VMXE))
4915 {
4916 AssertMsgFailed(("X86_CR4_VMXE should be set!\n"));
4917 return VERR_VMX_X86_CR4_VMXE_CLEARED;
4918 }
4919
4920 /* Activate the VMCS. */
4921 int rc = VMXActivateVMCS(pVCpu->hm.s.vmx.HCPhysVMCS);
4922 if (RT_FAILURE(rc))
4923 return rc;
4924
4925 pVCpu->hm.s.fResumeVM = false;
4926 return VINF_SUCCESS;
4927}
4928
4929
4930/**
4931 * Leaves the VT-x session.
4932 *
4933 * @returns VBox status code.
4934 * @param pVM Pointer to the VM.
4935 * @param pVCpu Pointer to the VMCPU.
4936 * @param pCtx Pointer to the guests CPU context.
4937 */
4938VMMR0DECL(int) VMXR0Leave(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
4939{
4940 Assert(pVM->hm.s.vmx.fSupported);
4941
4942#ifdef DEBUG
4943 if (CPUMIsHyperDebugStateActive(pVCpu))
4944 {
4945 CPUMR0LoadHostDebugState(pVM, pVCpu);
4946 Assert(pVCpu->hm.s.vmx.proc_ctls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT);
4947 }
4948 else
4949#endif
4950
4951 /*
4952 * Save the guest debug state if necessary.
4953 */
4954 if (CPUMIsGuestDebugStateActive(pVCpu))
4955 {
4956 CPUMR0SaveGuestDebugState(pVM, pVCpu, pCtx, true /* save DR6 */);
4957
4958 /* Enable DRx move intercepts again. */
4959 pVCpu->hm.s.vmx.proc_ctls |= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT;
4960 int rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.proc_ctls);
4961 AssertRC(rc);
4962
4963 /* Resync the debug registers the next time. */
4964 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_DEBUG;
4965 }
4966 else
4967 Assert(pVCpu->hm.s.vmx.proc_ctls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT);
4968
4969 /*
4970 * Clear VMCS, marking it inactive, clearing implementation-specific data and writing
4971 * VMCS data back to memory.
4972 */
4973 int rc = VMXClearVMCS(pVCpu->hm.s.vmx.HCPhysVMCS);
4974 AssertRC(rc);
4975
4976 return VINF_SUCCESS;
4977}
4978
4979
4980/**
4981 * Flush the TLB using EPT.
4982 *
4983 * @returns VBox status code.
4984 * @param pVM Pointer to the VM.
4985 * @param pVCpu Pointer to the VMCPU.
4986 * @param enmFlush Type of flush.
4987 */
4988static void hmR0VmxFlushEPT(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_EPT enmFlush)
4989{
4990 uint64_t descriptor[2];
4991
4992 LogFlow(("hmR0VmxFlushEPT %d\n", enmFlush));
4993 Assert(pVM->hm.s.fNestedPaging);
4994 descriptor[0] = pVCpu->hm.s.vmx.GCPhysEPTP;
4995 descriptor[1] = 0; /* MBZ. Intel spec. 33.3 VMX Instructions */
4996 int rc = VMXR0InvEPT(enmFlush, &descriptor[0]);
4997 AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %x %RGv failed with %d\n", enmFlush, pVCpu->hm.s.vmx.GCPhysEPTP, rc));
4998#ifdef VBOX_WITH_STATISTICS
4999 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNP);
5000#endif
5001}
5002
5003
5004/**
5005 * Flush the TLB using VPID.
5006 *
5007 * @returns VBox status code.
5008 * @param pVM Pointer to the VM.
5009 * @param pVCpu Pointer to the VMCPU (can be NULL depending on @a
5010 * enmFlush).
5011 * @param enmFlush Type of flush.
5012 * @param GCPtr Virtual address of the page to flush (can be 0 depending
5013 * on @a enmFlush).
5014 */
5015static void hmR0VmxFlushVPID(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_VPID enmFlush, RTGCPTR GCPtr)
5016{
5017 uint64_t descriptor[2];
5018
5019 Assert(pVM->hm.s.vmx.fVpid);
5020 if (enmFlush == VMX_FLUSH_VPID_ALL_CONTEXTS)
5021 {
5022 descriptor[0] = 0;
5023 descriptor[1] = 0;
5024 }
5025 else
5026 {
5027 AssertPtr(pVCpu);
5028 AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
5029 AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
5030 descriptor[0] = pVCpu->hm.s.uCurrentAsid;
5031 descriptor[1] = GCPtr;
5032 }
5033 int rc = VMXR0InvVPID(enmFlush, &descriptor[0]); NOREF(rc);
5034 AssertMsg(rc == VINF_SUCCESS,
5035 ("VMXR0InvVPID %x %x %RGv failed with %d\n", enmFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
5036#ifdef VBOX_WITH_STATISTICS
5037 if (pVCpu)
5038 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushASID);
5039#endif
5040}
5041
5042
5043/**
5044 * Invalidates a guest page by guest virtual address. Only relevant for
5045 * EPT/VPID, otherwise there is nothing really to invalidate.
5046 *
5047 * @returns VBox status code.
5048 * @param pVM Pointer to the VM.
5049 * @param pVCpu Pointer to the VMCPU.
5050 * @param GCVirt Guest virtual address of the page to invalidate.
5051 */
5052VMMR0DECL(int) VMXR0InvalidatePage(PVM pVM, PVMCPU pVCpu, RTGCPTR GCVirt)
5053{
5054 bool fFlushPending = VMCPU_FF_ISSET(pVCpu, VMCPU_FF_TLB_FLUSH);
5055
5056 Log2(("VMXR0InvalidatePage %RGv\n", GCVirt));
5057
5058 if (!fFlushPending)
5059 {
5060 /*
5061 * We must invalidate the guest TLB entry in either case, we cannot ignore it even for the EPT case
5062 * See @bugref{6043} and @bugref{6177}
5063 *
5064 * Set the VMCPU_FF_TLB_FLUSH force flag and flush before VMENTRY in hmR0VmxSetupTLB*() as this
5065 * function maybe called in a loop with individual addresses.
5066 */
5067 if (pVM->hm.s.vmx.fVpid)
5068 {
5069 /* If we can flush just this page do it, otherwise flush as little as possible. */
5070 if (pVM->hm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVVPID_CAPS_INDIV_ADDR)
5071 hmR0VmxFlushVPID(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, GCVirt);
5072 else
5073 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
5074 }
5075 else if (pVM->hm.s.fNestedPaging)
5076 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
5077 }
5078
5079 return VINF_SUCCESS;
5080}
5081
5082
5083/**
5084 * Invalidates a guest page by physical address. Only relevant for EPT/VPID,
5085 * otherwise there is nothing really to invalidate.
5086 *
5087 * NOTE: Assumes the current instruction references this physical page though a virtual address!!
5088 *
5089 * @returns VBox status code.
5090 * @param pVM Pointer to the VM.
5091 * @param pVCpu Pointer to the VMCPU.
5092 * @param GCPhys Guest physical address of the page to invalidate.
5093 */
5094VMMR0DECL(int) VMXR0InvalidatePhysPage(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys)
5095{
5096 LogFlow(("VMXR0InvalidatePhysPage %RGp\n", GCPhys));
5097
5098 /*
5099 * We cannot flush a page by guest-physical address. invvpid takes only a linear address
5100 * while invept only flushes by EPT not individual addresses. We update the force flag here
5101 * and flush before VMENTRY in hmR0VmxSetupTLB*(). This function might be called in a loop.
5102 */
5103 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
5104 return VINF_SUCCESS;
5105}
5106
5107
5108/**
5109 * Report world switch error and dump some useful debug info.
5110 *
5111 * @param pVM Pointer to the VM.
5112 * @param pVCpu Pointer to the VMCPU.
5113 * @param rc Return code.
5114 * @param pCtx Pointer to the current guest CPU context (not updated).
5115 */
5116static void hmR0VmxReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, VBOXSTRICTRC rc, PCPUMCTX pCtx)
5117{
5118 NOREF(pVM);
5119
5120 switch (VBOXSTRICTRC_VAL(rc))
5121 {
5122 case VERR_VMX_INVALID_VMXON_PTR:
5123 AssertFailed();
5124 break;
5125
5126 case VERR_VMX_UNABLE_TO_START_VM:
5127 case VERR_VMX_UNABLE_TO_RESUME_VM:
5128 {
5129 int rc2;
5130 RTCCUINTREG exitReason, instrError;
5131
5132 rc2 = VMXReadVMCS(VMX_VMCS32_RO_EXIT_REASON, &exitReason);
5133 rc2 |= VMXReadVMCS(VMX_VMCS32_RO_VM_INSTR_ERROR, &instrError);
5134 AssertRC(rc2);
5135 if (rc2 == VINF_SUCCESS)
5136 {
5137 Log(("Unable to start/resume VM for reason: %x. Instruction error %x\n", (uint32_t)exitReason,
5138 (uint32_t)instrError));
5139 Log(("Current stack %08x\n", &rc2));
5140
5141 pVCpu->hm.s.vmx.lasterror.ulInstrError = instrError;
5142 pVCpu->hm.s.vmx.lasterror.ulExitReason = exitReason;
5143
5144#ifdef VBOX_STRICT
5145 RTGDTR gdtr;
5146 PCX86DESCHC pDesc;
5147 RTCCUINTREG val;
5148
5149 ASMGetGDTR(&gdtr);
5150
5151 VMXReadVMCS(VMX_VMCS64_GUEST_RIP, &val);
5152 Log(("Old eip %RGv new %RGv\n", (RTGCPTR)pCtx->rip, (RTGCPTR)val));
5153 VMXReadVMCS(VMX_VMCS_CTRL_PIN_EXEC_CONTROLS, &val);
5154 Log(("VMX_VMCS_CTRL_PIN_EXEC_CONTROLS %08x\n", val));
5155 VMXReadVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, &val);
5156 Log(("VMX_VMCS_CTRL_PROC_EXEC_CONTROLS %08x\n", val));
5157 VMXReadVMCS(VMX_VMCS_CTRL_ENTRY_CONTROLS, &val);
5158 Log(("VMX_VMCS_CTRL_ENTRY_CONTROLS %08x\n", val));
5159 VMXReadVMCS(VMX_VMCS_CTRL_EXIT_CONTROLS, &val);
5160 Log(("VMX_VMCS_CTRL_EXIT_CONTROLS %08x\n", val));
5161
5162 VMXReadVMCS(VMX_VMCS_HOST_CR0, &val);
5163 Log(("VMX_VMCS_HOST_CR0 %08x\n", val));
5164 VMXReadVMCS(VMX_VMCS_HOST_CR3, &val);
5165 Log(("VMX_VMCS_HOST_CR3 %08x\n", val));
5166 VMXReadVMCS(VMX_VMCS_HOST_CR4, &val);
5167 Log(("VMX_VMCS_HOST_CR4 %08x\n", val));
5168
5169 VMXReadVMCS(VMX_VMCS16_HOST_FIELD_CS, &val);
5170 Log(("VMX_VMCS_HOST_FIELD_CS %08x\n", val));
5171 VMXReadVMCS(VMX_VMCS_GUEST_RFLAGS, &val);
5172 Log(("VMX_VMCS_GUEST_RFLAGS %08x\n", val));
5173
5174 if (val < gdtr.cbGdt)
5175 {
5176 pDesc = (PCX86DESCHC)(gdtr.pGdt + (val & X86_SEL_MASK));
5177 HMR0DumpDescriptor(pDesc, val, "CS: ");
5178 }
5179
5180 VMXReadVMCS(VMX_VMCS16_HOST_FIELD_DS, &val);
5181 Log(("VMX_VMCS_HOST_FIELD_DS %08x\n", val));
5182 if (val < gdtr.cbGdt)
5183 {
5184 pDesc = (PCX86DESCHC)(gdtr.pGdt + (val & X86_SEL_MASK));
5185 HMR0DumpDescriptor(pDesc, val, "DS: ");
5186 }
5187
5188 VMXReadVMCS(VMX_VMCS16_HOST_FIELD_ES, &val);
5189 Log(("VMX_VMCS_HOST_FIELD_ES %08x\n", val));
5190 if (val < gdtr.cbGdt)
5191 {
5192 pDesc = (PCX86DESCHC)(gdtr.pGdt + (val & X86_SEL_MASK));
5193 HMR0DumpDescriptor(pDesc, val, "ES: ");
5194 }
5195
5196 VMXReadVMCS(VMX_VMCS16_HOST_FIELD_FS, &val);
5197 Log(("VMX_VMCS16_HOST_FIELD_FS %08x\n", val));
5198 if (val < gdtr.cbGdt)
5199 {
5200 pDesc = (PCX86DESCHC)(gdtr.pGdt + (val & X86_SEL_MASK));
5201 HMR0DumpDescriptor(pDesc, val, "FS: ");
5202 }
5203
5204 VMXReadVMCS(VMX_VMCS16_HOST_FIELD_GS, &val);
5205 Log(("VMX_VMCS16_HOST_FIELD_GS %08x\n", val));
5206 if (val < gdtr.cbGdt)
5207 {
5208 pDesc = (PCX86DESCHC)(gdtr.pGdt + (val & X86_SEL_MASK));
5209 HMR0DumpDescriptor(pDesc, val, "GS: ");
5210 }
5211
5212 VMXReadVMCS(VMX_VMCS16_HOST_FIELD_SS, &val);
5213 Log(("VMX_VMCS16_HOST_FIELD_SS %08x\n", val));
5214 if (val < gdtr.cbGdt)
5215 {
5216 pDesc = (PCX86DESCHC)(gdtr.pGdt + (val & X86_SEL_MASK));
5217 HMR0DumpDescriptor(pDesc, val, "SS: ");
5218 }
5219
5220 VMXReadVMCS(VMX_VMCS16_HOST_FIELD_TR, &val);
5221 Log(("VMX_VMCS16_HOST_FIELD_TR %08x\n", val));
5222 if (val < gdtr.cbGdt)
5223 {
5224 pDesc = (PCX86DESCHC)(gdtr.pGdt + (val & X86_SEL_MASK));
5225 HMR0DumpDescriptor(pDesc, val, "TR: ");
5226 }
5227
5228 VMXReadVMCS(VMX_VMCS_HOST_TR_BASE, &val);
5229 Log(("VMX_VMCS_HOST_TR_BASE %RHv\n", val));
5230 VMXReadVMCS(VMX_VMCS_HOST_GDTR_BASE, &val);
5231 Log(("VMX_VMCS_HOST_GDTR_BASE %RHv\n", val));
5232 VMXReadVMCS(VMX_VMCS_HOST_IDTR_BASE, &val);
5233 Log(("VMX_VMCS_HOST_IDTR_BASE %RHv\n", val));
5234 VMXReadVMCS(VMX_VMCS32_HOST_SYSENTER_CS, &val);
5235 Log(("VMX_VMCS_HOST_SYSENTER_CS %08x\n", val));
5236 VMXReadVMCS(VMX_VMCS_HOST_SYSENTER_EIP, &val);
5237 Log(("VMX_VMCS_HOST_SYSENTER_EIP %RHv\n", val));
5238 VMXReadVMCS(VMX_VMCS_HOST_SYSENTER_ESP, &val);
5239 Log(("VMX_VMCS_HOST_SYSENTER_ESP %RHv\n", val));
5240 VMXReadVMCS(VMX_VMCS_HOST_RSP, &val);
5241 Log(("VMX_VMCS_HOST_RSP %RHv\n", val));
5242 VMXReadVMCS(VMX_VMCS_HOST_RIP, &val);
5243 Log(("VMX_VMCS_HOST_RIP %RHv\n", val));
5244# if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
5245 if (VMX_IS_64BIT_HOST_MODE())
5246 {
5247 Log(("MSR_K6_EFER = %RX64\n", ASMRdMsr(MSR_K6_EFER)));
5248 Log(("MSR_K6_STAR = %RX64\n", ASMRdMsr(MSR_K6_STAR)));
5249 Log(("MSR_K8_LSTAR = %RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
5250 Log(("MSR_K8_CSTAR = %RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
5251 Log(("MSR_K8_SF_MASK = %RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
5252 Log(("MSR_K8_KERNEL_GS_BASE = %RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
5253 }
5254# endif
5255#endif /* VBOX_STRICT */
5256 }
5257 break;
5258 }
5259
5260 default:
5261 /* impossible */
5262 AssertMsgFailed(("%Rrc (%#x)\n", VBOXSTRICTRC_VAL(rc), VBOXSTRICTRC_VAL(rc)));
5263 break;
5264 }
5265}
5266
5267
5268#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
5269/**
5270 * Prepares for and executes VMLAUNCH (64 bits guest mode).
5271 *
5272 * @returns VBox status code.
5273 * @param fResume Whether to vmlauch/vmresume.
5274 * @param pCtx Pointer to the guest CPU context.
5275 * @param pCache Pointer to the VMCS cache.
5276 * @param pVM Pointer to the VM.
5277 * @param pVCpu Pointer to the VMCPU.
5278 */
5279DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
5280{
5281 uint32_t aParam[6];
5282 PHMGLOBLCPUINFO pCpu;
5283 RTHCPHYS HCPhysCpuPage;
5284 int rc;
5285
5286 pCpu = HMR0GetCurrentCpu();
5287 HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
5288
5289#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5290 pCache->uPos = 1;
5291 pCache->interPD = PGMGetInterPaeCR3(pVM);
5292 pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
5293#endif
5294
5295#ifdef DEBUG
5296 pCache->TestIn.HCPhysCpuPage= 0;
5297 pCache->TestIn.HCPhysVMCS = 0;
5298 pCache->TestIn.pCache = 0;
5299 pCache->TestOut.HCPhysVMCS = 0;
5300 pCache->TestOut.pCache = 0;
5301 pCache->TestOut.pCtx = 0;
5302 pCache->TestOut.eflags = 0;
5303#endif
5304
5305 aParam[0] = (uint32_t)(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
5306 aParam[1] = (uint32_t)(HCPhysCpuPage >> 32); /* Param 1: VMXON physical address - Hi. */
5307 aParam[2] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVMCS); /* Param 2: VMCS physical address - Lo. */
5308 aParam[3] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVMCS >> 32); /* Param 2: VMCS physical address - Hi. */
5309 aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache);
5310 aParam[5] = 0;
5311
5312#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5313 pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
5314 *(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
5315#endif
5316 rc = VMXR0Execute64BitsHandler(pVM, pVCpu, pCtx, pVM->hm.s.pfnVMXGCStartVM64, 6, &aParam[0]);
5317
5318#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5319 Assert(*(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
5320 Assert(pCtx->dr[4] == 10);
5321 *(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
5322#endif
5323
5324#ifdef DEBUG
5325 AssertMsg(pCache->TestIn.HCPhysCpuPage== HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
5326 AssertMsg(pCache->TestIn.HCPhysVMCS == pVCpu->hm.s.vmx.HCPhysVMCS, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVMCS,
5327 pVCpu->hm.s.vmx.HCPhysVMCS));
5328 AssertMsg(pCache->TestIn.HCPhysVMCS == pCache->TestOut.HCPhysVMCS, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVMCS,
5329 pCache->TestOut.HCPhysVMCS));
5330 AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
5331 pCache->TestOut.pCache));
5332 AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache),
5333 ("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache)));
5334 AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
5335 pCache->TestOut.pCtx));
5336 Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
5337#endif
5338 return rc;
5339}
5340
5341
5342# ifdef VBOX_STRICT
5343static bool hmR0VmxIsValidReadField(uint32_t idxField)
5344{
5345 switch (idxField)
5346 {
5347 case VMX_VMCS64_GUEST_RIP:
5348 case VMX_VMCS64_GUEST_RSP:
5349 case VMX_VMCS_GUEST_RFLAGS:
5350 case VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE:
5351 case VMX_VMCS_CTRL_CR0_READ_SHADOW:
5352 case VMX_VMCS64_GUEST_CR0:
5353 case VMX_VMCS_CTRL_CR4_READ_SHADOW:
5354 case VMX_VMCS64_GUEST_CR4:
5355 case VMX_VMCS64_GUEST_DR7:
5356 case VMX_VMCS32_GUEST_SYSENTER_CS:
5357 case VMX_VMCS64_GUEST_SYSENTER_EIP:
5358 case VMX_VMCS64_GUEST_SYSENTER_ESP:
5359 case VMX_VMCS32_GUEST_GDTR_LIMIT:
5360 case VMX_VMCS64_GUEST_GDTR_BASE:
5361 case VMX_VMCS32_GUEST_IDTR_LIMIT:
5362 case VMX_VMCS64_GUEST_IDTR_BASE:
5363 case VMX_VMCS16_GUEST_FIELD_CS:
5364 case VMX_VMCS32_GUEST_CS_LIMIT:
5365 case VMX_VMCS64_GUEST_CS_BASE:
5366 case VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS:
5367 case VMX_VMCS16_GUEST_FIELD_DS:
5368 case VMX_VMCS32_GUEST_DS_LIMIT:
5369 case VMX_VMCS64_GUEST_DS_BASE:
5370 case VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS:
5371 case VMX_VMCS16_GUEST_FIELD_ES:
5372 case VMX_VMCS32_GUEST_ES_LIMIT:
5373 case VMX_VMCS64_GUEST_ES_BASE:
5374 case VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS:
5375 case VMX_VMCS16_GUEST_FIELD_FS:
5376 case VMX_VMCS32_GUEST_FS_LIMIT:
5377 case VMX_VMCS64_GUEST_FS_BASE:
5378 case VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS:
5379 case VMX_VMCS16_GUEST_FIELD_GS:
5380 case VMX_VMCS32_GUEST_GS_LIMIT:
5381 case VMX_VMCS64_GUEST_GS_BASE:
5382 case VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS:
5383 case VMX_VMCS16_GUEST_FIELD_SS:
5384 case VMX_VMCS32_GUEST_SS_LIMIT:
5385 case VMX_VMCS64_GUEST_SS_BASE:
5386 case VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS:
5387 case VMX_VMCS16_GUEST_FIELD_LDTR:
5388 case VMX_VMCS32_GUEST_LDTR_LIMIT:
5389 case VMX_VMCS64_GUEST_LDTR_BASE:
5390 case VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS:
5391 case VMX_VMCS16_GUEST_FIELD_TR:
5392 case VMX_VMCS32_GUEST_TR_LIMIT:
5393 case VMX_VMCS64_GUEST_TR_BASE:
5394 case VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS:
5395 case VMX_VMCS32_RO_EXIT_REASON:
5396 case VMX_VMCS32_RO_VM_INSTR_ERROR:
5397 case VMX_VMCS32_RO_EXIT_INSTR_LENGTH:
5398 case VMX_VMCS32_RO_EXIT_INTERRUPTION_ERRCODE:
5399 case VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO:
5400 case VMX_VMCS32_RO_EXIT_INSTR_INFO:
5401 case VMX_VMCS_RO_EXIT_QUALIFICATION:
5402 case VMX_VMCS32_RO_IDT_INFO:
5403 case VMX_VMCS32_RO_IDT_ERRCODE:
5404 case VMX_VMCS64_GUEST_CR3:
5405 case VMX_VMCS_EXIT_PHYS_ADDR_FULL:
5406 return true;
5407 }
5408 return false;
5409}
5410
5411
5412static bool hmR0VmxIsValidWriteField(uint32_t idxField)
5413{
5414 switch (idxField)
5415 {
5416 case VMX_VMCS64_GUEST_LDTR_BASE:
5417 case VMX_VMCS64_GUEST_TR_BASE:
5418 case VMX_VMCS64_GUEST_GDTR_BASE:
5419 case VMX_VMCS64_GUEST_IDTR_BASE:
5420 case VMX_VMCS64_GUEST_SYSENTER_EIP:
5421 case VMX_VMCS64_GUEST_SYSENTER_ESP:
5422 case VMX_VMCS64_GUEST_CR0:
5423 case VMX_VMCS64_GUEST_CR4:
5424 case VMX_VMCS64_GUEST_CR3:
5425 case VMX_VMCS64_GUEST_DR7:
5426 case VMX_VMCS64_GUEST_RIP:
5427 case VMX_VMCS64_GUEST_RSP:
5428 case VMX_VMCS64_GUEST_CS_BASE:
5429 case VMX_VMCS64_GUEST_DS_BASE:
5430 case VMX_VMCS64_GUEST_ES_BASE:
5431 case VMX_VMCS64_GUEST_FS_BASE:
5432 case VMX_VMCS64_GUEST_GS_BASE:
5433 case VMX_VMCS64_GUEST_SS_BASE:
5434 return true;
5435 }
5436 return false;
5437}
5438# endif /* VBOX_STRICT */
5439
5440
5441/**
5442 * Executes the specified handler in 64-bit mode.
5443 *
5444 * @returns VBox status code.
5445 * @param pVM Pointer to the VM.
5446 * @param pVCpu Pointer to the VMCPU.
5447 * @param pCtx Pointer to the guest CPU context.
5448 * @param pfnHandler Pointer to the RC handler function.
5449 * @param cbParam Number of parameters.
5450 * @param paParam Array of 32-bit parameters.
5451 */
5452VMMR0DECL(int) VMXR0Execute64BitsHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, RTRCPTR pfnHandler, uint32_t cbParam,
5453 uint32_t *paParam)
5454{
5455 int rc, rc2;
5456 PHMGLOBLCPUINFO pCpu;
5457 RTHCPHYS HCPhysCpuPage;
5458 RTHCUINTREG uOldEFlags;
5459
5460 AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
5461 Assert(pfnHandler);
5462 Assert(pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Write.aField));
5463 Assert(pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Read.aField));
5464
5465#ifdef VBOX_STRICT
5466 for (unsigned i=0;i<pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries;i++)
5467 Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VMCSCache.Write.aField[i]));
5468
5469 for (unsigned i=0;i<pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries;i++)
5470 Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VMCSCache.Read.aField[i]));
5471#endif
5472
5473 /* Disable interrupts. */
5474 uOldEFlags = ASMIntDisableFlags();
5475
5476#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
5477 RTCPUID idHostCpu = RTMpCpuId();
5478 CPUMR0SetLApic(pVM, idHostCpu);
5479#endif
5480
5481 pCpu = HMR0GetCurrentCpu();
5482 HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
5483
5484 /* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
5485 VMXClearVMCS(pVCpu->hm.s.vmx.HCPhysVMCS);
5486
5487 /* Leave VMX Root Mode. */
5488 VMXDisable();
5489
5490 ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
5491
5492 CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
5493 CPUMSetHyperEIP(pVCpu, pfnHandler);
5494 for (int i=(int)cbParam-1;i>=0;i--)
5495 CPUMPushHyper(pVCpu, paParam[i]);
5496
5497 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
5498
5499 /* Call switcher. */
5500 rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_OFFSETOF(VM, aCpus[pVCpu->idCpu].cpum) - RT_OFFSETOF(VM, cpum));
5501 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
5502
5503 /* Make sure the VMX instructions don't cause #UD faults. */
5504 ASMSetCR4(ASMGetCR4() | X86_CR4_VMXE);
5505
5506 /* Enter VMX Root Mode */
5507 rc2 = VMXEnable(HCPhysCpuPage);
5508 if (RT_FAILURE(rc2))
5509 {
5510 ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
5511 ASMSetFlags(uOldEFlags);
5512 return VERR_VMX_VMXON_FAILED;
5513 }
5514
5515 rc2 = VMXActivateVMCS(pVCpu->hm.s.vmx.HCPhysVMCS);
5516 AssertRC(rc2);
5517 Assert(!(ASMGetFlags() & X86_EFL_IF));
5518 ASMSetFlags(uOldEFlags);
5519 return rc;
5520}
5521#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL) */
5522
5523
5524#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
5525/**
5526 * Executes VMWRITE.
5527 *
5528 * @returns VBox status code
5529 * @param pVCpu Pointer to the VMCPU.
5530 * @param idxField VMCS field index.
5531 * @param u64Val 16, 32 or 64 bits value.
5532 */
5533VMMR0DECL(int) VMXWriteVMCS64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5534{
5535 int rc;
5536 switch (idxField)
5537 {
5538 case VMX_VMCS_CTRL_TSC_OFFSET_FULL:
5539 case VMX_VMCS_CTRL_IO_BITMAP_A_FULL:
5540 case VMX_VMCS_CTRL_IO_BITMAP_B_FULL:
5541 case VMX_VMCS_CTRL_MSR_BITMAP_FULL:
5542 case VMX_VMCS_CTRL_VMEXIT_MSR_STORE_FULL:
5543 case VMX_VMCS_CTRL_VMEXIT_MSR_LOAD_FULL:
5544 case VMX_VMCS_CTRL_VMENTRY_MSR_LOAD_FULL:
5545 case VMX_VMCS_CTRL_VAPIC_PAGEADDR_FULL:
5546 case VMX_VMCS_CTRL_APIC_ACCESSADDR_FULL:
5547 case VMX_VMCS_GUEST_LINK_PTR_FULL:
5548 case VMX_VMCS_GUEST_PDPTR0_FULL:
5549 case VMX_VMCS_GUEST_PDPTR1_FULL:
5550 case VMX_VMCS_GUEST_PDPTR2_FULL:
5551 case VMX_VMCS_GUEST_PDPTR3_FULL:
5552 case VMX_VMCS_GUEST_DEBUGCTL_FULL:
5553 case VMX_VMCS_GUEST_EFER_FULL:
5554 case VMX_VMCS_CTRL_EPTP_FULL:
5555 /* These fields consist of two parts, which are both writable in 32 bits mode. */
5556 rc = VMXWriteVMCS32(idxField, u64Val);
5557 rc |= VMXWriteVMCS32(idxField + 1, (uint32_t)(u64Val >> 32ULL));
5558 AssertRC(rc);
5559 return rc;
5560
5561 case VMX_VMCS64_GUEST_LDTR_BASE:
5562 case VMX_VMCS64_GUEST_TR_BASE:
5563 case VMX_VMCS64_GUEST_GDTR_BASE:
5564 case VMX_VMCS64_GUEST_IDTR_BASE:
5565 case VMX_VMCS64_GUEST_SYSENTER_EIP:
5566 case VMX_VMCS64_GUEST_SYSENTER_ESP:
5567 case VMX_VMCS64_GUEST_CR0:
5568 case VMX_VMCS64_GUEST_CR4:
5569 case VMX_VMCS64_GUEST_CR3:
5570 case VMX_VMCS64_GUEST_DR7:
5571 case VMX_VMCS64_GUEST_RIP:
5572 case VMX_VMCS64_GUEST_RSP:
5573 case VMX_VMCS64_GUEST_CS_BASE:
5574 case VMX_VMCS64_GUEST_DS_BASE:
5575 case VMX_VMCS64_GUEST_ES_BASE:
5576 case VMX_VMCS64_GUEST_FS_BASE:
5577 case VMX_VMCS64_GUEST_GS_BASE:
5578 case VMX_VMCS64_GUEST_SS_BASE:
5579 /* Queue a 64 bits value as we can't set it in 32 bits host mode. */
5580 if (u64Val >> 32ULL)
5581 rc = VMXWriteCachedVMCSEx(pVCpu, idxField, u64Val);
5582 else
5583 rc = VMXWriteVMCS32(idxField, (uint32_t)u64Val);
5584
5585 return rc;
5586
5587 default:
5588 AssertMsgFailed(("Unexpected field %x\n", idxField));
5589 return VERR_INVALID_PARAMETER;
5590 }
5591}
5592
5593
5594/**
5595 * Cache VMCS writes for performance reasons (Darwin) and for running 64 bits guests on 32 bits hosts.
5596 *
5597 * @param pVCpu Pointer to the VMCPU.
5598 * @param idxField VMCS field index.
5599 * @param u64Val 16, 32 or 64 bits value.
5600 */
5601VMMR0DECL(int) VMXWriteCachedVMCSEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5602{
5603 PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5604
5605 AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1,
5606 ("entries=%x\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
5607
5608 /* Make sure there are no duplicates. */
5609 for (unsigned i = 0; i < pCache->Write.cValidEntries; i++)
5610 {
5611 if (pCache->Write.aField[i] == idxField)
5612 {
5613 pCache->Write.aFieldVal[i] = u64Val;
5614 return VINF_SUCCESS;
5615 }
5616 }
5617
5618 pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
5619 pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
5620 pCache->Write.cValidEntries++;
5621 return VINF_SUCCESS;
5622}
5623
5624#endif /* HC_ARCH_BITS == 32 && !VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0 */
5625
Note: See TracBrowser for help on using the repository browser.

© 2024 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette