VirtualBox

source: vbox/trunk/src/VBox/VMM/VMMR0/HWVMXR0.cpp@ 43708

Last change on this file since 43708 was 43708, checked in by vboxsync, 12 years ago

burn fix
  • Property svn:eol-style set to native
  • Property svn:keywords set to Id Revision
File size: 221.1 KB
Line 
1/* $Id: HWVMXR0.cpp 43708 2012-10-23 08:26:47Z vboxsync $ */
2/** @file
3 * HM VMX (VT-x) - Host Context Ring-0.
4 */
5
6/*
7 * Copyright (C) 2006-2012 Oracle Corporation
8 *
9 * This file is part of VirtualBox Open Source Edition (OSE), as
10 * available from http://www.virtualbox.org. This file is free software;
11 * you can redistribute it and/or modify it under the terms of the GNU
12 * General Public License (GPL) as published by the Free Software
13 * Foundation, in version 2 as it comes in the "COPYING" file of the
14 * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15 * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16 */
17
18
19/*******************************************************************************
20* Header Files *
21*******************************************************************************/
22#define LOG_GROUP LOG_GROUP_HM
23#include <iprt/asm-amd64-x86.h>
24#include <VBox/vmm/hm.h>
25#include <VBox/vmm/pgm.h>
26#include <VBox/vmm/dbgf.h>
27#include <VBox/vmm/dbgftrace.h>
28#include <VBox/vmm/selm.h>
29#include <VBox/vmm/iom.h>
30#ifdef VBOX_WITH_REM
31# include <VBox/vmm/rem.h>
32#endif
33#include <VBox/vmm/tm.h>
34#include "HMInternal.h"
35#include <VBox/vmm/vm.h>
36#include <VBox/vmm/pdmapi.h>
37#include <VBox/err.h>
38#include <VBox/log.h>
39#include <iprt/assert.h>
40#include <iprt/param.h>
41#include <iprt/string.h>
42#include <iprt/time.h>
43#ifdef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
44# include <iprt/thread.h>
45#endif
46#include <iprt/x86.h>
47#include "HWVMXR0.h"
48
49#include "dtrace/VBoxVMM.h"
50
51
52/*******************************************************************************
53* Defined Constants And Macros *
54*******************************************************************************/
55#if defined(RT_ARCH_AMD64)
56# define VMX_IS_64BIT_HOST_MODE() (true)
57#elif defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
58# define VMX_IS_64BIT_HOST_MODE() (g_fVMXIs64bitHost != 0)
59#else
60# define VMX_IS_64BIT_HOST_MODE() (false)
61#endif
62
63
64/*******************************************************************************
65* Global Variables *
66*******************************************************************************/
67/* IO operation lookup arrays. */
68static uint32_t const g_aIOSize[4] = {1, 2, 0, 4};
69static uint32_t const g_aIOOpAnd[4] = {0xff, 0xffff, 0, 0xffffffff};
70
71#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
72/** See HMR0A.asm. */
73extern "C" uint32_t g_fVMXIs64bitHost;
74#endif
75
76
77/*******************************************************************************
78* Local Functions *
79*******************************************************************************/
80static DECLCALLBACK(void) hmR0VmxSetupTLBEPT(PVM pVM, PVMCPU pVCpu);
81static DECLCALLBACK(void) hmR0VmxSetupTLBVPID(PVM pVM, PVMCPU pVCpu);
82static DECLCALLBACK(void) hmR0VmxSetupTLBBoth(PVM pVM, PVMCPU pVCpu);
83static DECLCALLBACK(void) hmR0VmxSetupTLBDummy(PVM pVM, PVMCPU pVCpu);
84static void hmR0VmxFlushEPT(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_EPT enmFlush);
85static void hmR0VmxFlushVPID(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_VPID enmFlush, RTGCPTR GCPtr);
86static void hmR0VmxUpdateExceptionBitmap(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx);
87static void hmR0VmxSetMSRPermission(PVMCPU pVCpu, unsigned ulMSR, bool fRead, bool fWrite);
88static void hmR0VmxReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, VBOXSTRICTRC rc, PCPUMCTX pCtx);
89
90
91/**
92 * Updates error from VMCS to HMCPU's lasterror record.
93 *
94 * @param pVM Pointer to the VM.
95 * @param pVCpu Pointer to the VMCPU.
96 * @param rc The error code.
97 */
98static void hmR0VmxCheckError(PVM pVM, PVMCPU pVCpu, int rc)
99{
100 if (rc == VERR_VMX_GENERIC)
101 {
102 RTCCUINTREG instrError;
103
104 VMXReadVMCS(VMX_VMCS32_RO_VM_INSTR_ERROR, &instrError);
105 pVCpu->hm.s.vmx.lasterror.ulInstrError = instrError;
106 }
107 pVM->hm.s.lLastError = rc;
108}
109
110
111/**
112 * Sets up and activates VT-x on the current CPU.
113 *
114 * @returns VBox status code.
115 * @param pCpu Pointer to the CPU info struct.
116 * @param pVM Pointer to the VM. (can be NULL after a resume!!)
117 * @param pvCpuPage Pointer to the global CPU page.
118 * @param HCPhysCpuPage Physical address of the global CPU page.
119 * @param fEnabledByHost Set if SUPR0EnableVTx or similar was used to enable
120 * VT-x/AMD-V on the host.
121 */
122VMMR0DECL(int) VMXR0EnableCpu(PHMGLOBLCPUINFO pCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost)
123{
124 if (!fEnabledByHost)
125 {
126 AssertReturn(HCPhysCpuPage != 0 && HCPhysCpuPage != NIL_RTHCPHYS, VERR_INVALID_PARAMETER);
127 AssertReturn(pvCpuPage, VERR_INVALID_PARAMETER);
128
129 if (pVM)
130 {
131 /* Set revision dword at the beginning of the VMXON structure. */
132 *(uint32_t *)pvCpuPage = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.msr.vmx_basic_info);
133 }
134
135 /** @todo we should unmap the two pages from the virtual address space in order to prevent accidental corruption.
136 * (which can have very bad consequences!!!)
137 */
138
139 /** @todo r=bird: Why is this code different than the probing code earlier
140 * on? It just sets VMXE if needed and doesn't check that it isn't
141 * set. Mac OS X host_vmxoff may leave this set and we'll fail here
142 * and debug-assert in the calling code. This is what caused the
143 * "regression" after backing out the SUPR0EnableVTx code hours before
144 * 4.2.0GA (reboot fixed the issue). I've changed here to do the same
145 * as the init code. */
146 uint64_t uCr4 = ASMGetCR4();
147 if (!(uCr4 & X86_CR4_VMXE))
148 ASMSetCR4(ASMGetCR4() | X86_CR4_VMXE); /* Make sure the VMX instructions don't cause #UD faults. */
149
150 /*
151 * Enter VM root mode.
152 */
153 int rc = VMXEnable(HCPhysCpuPage);
154 if (RT_FAILURE(rc))
155 {
156 ASMSetCR4(uCr4);
157 return VERR_VMX_VMXON_FAILED;
158 }
159 }
160
161 /*
162 * Flush all VPIDs (in case we or any other hypervisor have been using VPIDs) so that
163 * we can avoid an explicit flush while using new VPIDs. We would still need to flush
164 * each time while reusing a VPID after hitting the MaxASID limit once.
165 */
166 if ( pVM
167 && pVM->hm.s.vmx.fVpid
168 && (pVM->hm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVVPID_CAPS_ALL_CONTEXTS))
169 {
170 hmR0VmxFlushVPID(pVM, NULL /* pvCpu */, VMX_FLUSH_VPID_ALL_CONTEXTS, 0 /* GCPtr */);
171 pCpu->fFlushAsidBeforeUse = false;
172 }
173 else
174 pCpu->fFlushAsidBeforeUse = true;
175
176 /*
177 * Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}.
178 */
179 ++pCpu->cTlbFlushes;
180
181 return VINF_SUCCESS;
182}
183
184
185/**
186 * Deactivates VT-x on the current CPU.
187 *
188 * @returns VBox status code.
189 * @param pCpu Pointer to the CPU info struct.
190 * @param pvCpuPage Pointer to the global CPU page.
191 * @param HCPhysCpuPage Physical address of the global CPU page.
192 */
193VMMR0DECL(int) VMXR0DisableCpu(PHMGLOBLCPUINFO pCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
194{
195 AssertReturn(HCPhysCpuPage != 0 && HCPhysCpuPage != NIL_RTHCPHYS, VERR_INVALID_PARAMETER);
196 AssertReturn(pvCpuPage, VERR_INVALID_PARAMETER);
197 NOREF(pCpu);
198
199 /* If we're somehow not in VMX root mode, then we shouldn't dare leaving it. */
200 if (!(ASMGetCR4() & X86_CR4_VMXE))
201 return VERR_VMX_NOT_IN_VMX_ROOT_MODE;
202
203 /* Leave VMX Root Mode. */
204 VMXDisable();
205
206 /* And clear the X86_CR4_VMXE bit. */
207 ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
208 return VINF_SUCCESS;
209}
210
211
212/**
213 * Does Ring-0 per VM VT-x initialization.
214 *
215 * @returns VBox status code.
216 * @param pVM Pointer to the VM.
217 */
218VMMR0DECL(int) VMXR0InitVM(PVM pVM)
219{
220 int rc;
221
222#ifdef LOG_ENABLED
223 SUPR0Printf("VMXR0InitVM %p\n", pVM);
224#endif
225
226 pVM->hm.s.vmx.hMemObjApicAccess = NIL_RTR0MEMOBJ;
227
228 if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
229 {
230 /* Allocate one page for the APIC physical page (serves for filtering accesses). */
231 rc = RTR0MemObjAllocCont(&pVM->hm.s.vmx.hMemObjApicAccess, PAGE_SIZE, false /* fExecutable */);
232 AssertRC(rc);
233 if (RT_FAILURE(rc))
234 return rc;
235
236 pVM->hm.s.vmx.pbApicAccess = (uint8_t *)RTR0MemObjAddress(pVM->hm.s.vmx.hMemObjApicAccess);
237 pVM->hm.s.vmx.HCPhysApicAccess = RTR0MemObjGetPagePhysAddr(pVM->hm.s.vmx.hMemObjApicAccess, 0);
238 ASMMemZero32(pVM->hm.s.vmx.pbApicAccess, PAGE_SIZE);
239 }
240 else
241 {
242 pVM->hm.s.vmx.hMemObjApicAccess = 0;
243 pVM->hm.s.vmx.pbApicAccess = 0;
244 pVM->hm.s.vmx.HCPhysApicAccess = 0;
245 }
246
247#ifdef VBOX_WITH_CRASHDUMP_MAGIC
248 {
249 rc = RTR0MemObjAllocCont(&pVM->hm.s.vmx.hMemObjScratch, PAGE_SIZE, false /* fExecutable */);
250 AssertRC(rc);
251 if (RT_FAILURE(rc))
252 return rc;
253
254 pVM->hm.s.vmx.pScratch = (uint8_t *)RTR0MemObjAddress(pVM->hm.s.vmx.hMemObjScratch);
255 pVM->hm.s.vmx.pScratchPhys = RTR0MemObjGetPagePhysAddr(pVM->hm.s.vmx.hMemObjScratch, 0);
256
257 ASMMemZero32(pVM->hm.s.vmx.pbScratch, PAGE_SIZE);
258 strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
259 *(uint64_t *)(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xDEADBEEFDEADBEEF);
260 }
261#endif
262
263 /* Allocate VMCSs for all guest CPUs. */
264 for (VMCPUID i = 0; i < pVM->cCpus; i++)
265 {
266 PVMCPU pVCpu = &pVM->aCpus[i];
267
268 pVCpu->hm.s.vmx.hMemObjVMCS = NIL_RTR0MEMOBJ;
269
270 /* Allocate one page for the VM control structure (VMCS). */
271 rc = RTR0MemObjAllocCont(&pVCpu->hm.s.vmx.hMemObjVMCS, PAGE_SIZE, false /* fExecutable */);
272 AssertRC(rc);
273 if (RT_FAILURE(rc))
274 return rc;
275
276 pVCpu->hm.s.vmx.pvVMCS = RTR0MemObjAddress(pVCpu->hm.s.vmx.hMemObjVMCS);
277 pVCpu->hm.s.vmx.HCPhysVMCS = RTR0MemObjGetPagePhysAddr(pVCpu->hm.s.vmx.hMemObjVMCS, 0);
278 ASMMemZeroPage(pVCpu->hm.s.vmx.pvVMCS);
279
280 pVCpu->hm.s.vmx.cr0_mask = 0;
281 pVCpu->hm.s.vmx.cr4_mask = 0;
282
283 /* Allocate one page for the virtual APIC page for TPR caching. */
284 rc = RTR0MemObjAllocCont(&pVCpu->hm.s.vmx.hMemObjVAPIC, PAGE_SIZE, false /* fExecutable */);
285 AssertRC(rc);
286 if (RT_FAILURE(rc))
287 return rc;
288
289 pVCpu->hm.s.vmx.pbVAPIC = (uint8_t *)RTR0MemObjAddress(pVCpu->hm.s.vmx.hMemObjVAPIC);
290 pVCpu->hm.s.vmx.HCPhysVAPIC = RTR0MemObjGetPagePhysAddr(pVCpu->hm.s.vmx.hMemObjVAPIC, 0);
291 ASMMemZeroPage(pVCpu->hm.s.vmx.pbVAPIC);
292
293 /* Allocate the MSR bitmap if this feature is supported. */
294 if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
295 {
296 rc = RTR0MemObjAllocCont(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, PAGE_SIZE, false /* fExecutable */);
297 AssertRC(rc);
298 if (RT_FAILURE(rc))
299 return rc;
300
301 pVCpu->hm.s.vmx.pvMsrBitmap = (uint8_t *)RTR0MemObjAddress(pVCpu->hm.s.vmx.hMemObjMsrBitmap);
302 pVCpu->hm.s.vmx.HCPhysMsrBitmap = RTR0MemObjGetPagePhysAddr(pVCpu->hm.s.vmx.hMemObjMsrBitmap, 0);
303 memset(pVCpu->hm.s.vmx.pvMsrBitmap, 0xff, PAGE_SIZE);
304 }
305
306#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
307 /* Allocate one page for the guest MSR load area (for preloading guest MSRs during the world switch). */
308 rc = RTR0MemObjAllocCont(&pVCpu->hm.s.vmx.hMemObjGuestMsr, PAGE_SIZE, false /* fExecutable */);
309 AssertRC(rc);
310 if (RT_FAILURE(rc))
311 return rc;
312
313 pVCpu->hm.s.vmx.pvGuestMsr = (uint8_t *)RTR0MemObjAddress(pVCpu->hm.s.vmx.hMemObjGuestMsr);
314 pVCpu->hm.s.vmx.HCPhysGuestMsr = RTR0MemObjGetPagePhysAddr(pVCpu->hm.s.vmx.hMemObjGuestMsr, 0);
315 Assert(!(pVCpu->hm.s.vmx.HCPhysGuestMsr & 0xf));
316 memset(pVCpu->hm.s.vmx.pvGuestMsr, 0, PAGE_SIZE);
317
318 /* Allocate one page for the host MSR load area (for restoring host MSRs after the world switch back). */
319 rc = RTR0MemObjAllocCont(&pVCpu->hm.s.vmx.hMemObjHostMsr, PAGE_SIZE, false /* fExecutable */);
320 AssertRC(rc);
321 if (RT_FAILURE(rc))
322 return rc;
323
324 pVCpu->hm.s.vmx.pvHostMsr = (uint8_t *)RTR0MemObjAddress(pVCpu->hm.s.vmx.hMemObjHostMsr);
325 pVCpu->hm.s.vmx.HCPhysHostMsr = RTR0MemObjGetPagePhysAddr(pVCpu->hm.s.vmx.hMemObjHostMsr, 0);
326 Assert(!(pVCpu->hm.s.vmx.HCPhysHostMsr & 0xf));
327 memset(pVCpu->hm.s.vmx.pvHostMsr, 0, PAGE_SIZE);
328#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
329
330 /* Current guest paging mode. */
331 pVCpu->hm.s.vmx.enmLastSeenGuestMode = PGMMODE_REAL;
332
333#ifdef LOG_ENABLED
334 SUPR0Printf("VMXR0InitVM %x VMCS=%x (%x)\n", pVM, pVCpu->hm.s.vmx.pvVMCS, (uint32_t)pVCpu->hm.s.vmx.HCPhysVMCS);
335#endif
336 }
337
338 return VINF_SUCCESS;
339}
340
341
342/**
343 * Does Ring-0 per VM VT-x termination.
344 *
345 * @returns VBox status code.
346 * @param pVM Pointer to the VM.
347 */
348VMMR0DECL(int) VMXR0TermVM(PVM pVM)
349{
350 for (VMCPUID i = 0; i < pVM->cCpus; i++)
351 {
352 PVMCPU pVCpu = &pVM->aCpus[i];
353
354 if (pVCpu->hm.s.vmx.hMemObjVMCS != NIL_RTR0MEMOBJ)
355 {
356 RTR0MemObjFree(pVCpu->hm.s.vmx.hMemObjVMCS, false);
357 pVCpu->hm.s.vmx.hMemObjVMCS = NIL_RTR0MEMOBJ;
358 pVCpu->hm.s.vmx.pvVMCS = 0;
359 pVCpu->hm.s.vmx.HCPhysVMCS = 0;
360 }
361 if (pVCpu->hm.s.vmx.hMemObjVAPIC != NIL_RTR0MEMOBJ)
362 {
363 RTR0MemObjFree(pVCpu->hm.s.vmx.hMemObjVAPIC, false);
364 pVCpu->hm.s.vmx.hMemObjVAPIC = NIL_RTR0MEMOBJ;
365 pVCpu->hm.s.vmx.pbVAPIC = 0;
366 pVCpu->hm.s.vmx.HCPhysVAPIC = 0;
367 }
368 if (pVCpu->hm.s.vmx.hMemObjMsrBitmap != NIL_RTR0MEMOBJ)
369 {
370 RTR0MemObjFree(pVCpu->hm.s.vmx.hMemObjMsrBitmap, false);
371 pVCpu->hm.s.vmx.hMemObjMsrBitmap = NIL_RTR0MEMOBJ;
372 pVCpu->hm.s.vmx.pvMsrBitmap = 0;
373 pVCpu->hm.s.vmx.HCPhysMsrBitmap = 0;
374 }
375#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
376 if (pVCpu->hm.s.vmx.hMemObjHostMsr != NIL_RTR0MEMOBJ)
377 {
378 RTR0MemObjFree(pVCpu->hm.s.vmx.hMemObjHostMsr, false);
379 pVCpu->hm.s.vmx.hMemObjHostMsr = NIL_RTR0MEMOBJ;
380 pVCpu->hm.s.vmx.pvHostMsr = 0;
381 pVCpu->hm.s.vmx.HCPhysHostMsr = 0;
382 }
383 if (pVCpu->hm.s.vmx.hMemObjGuestMsr != NIL_RTR0MEMOBJ)
384 {
385 RTR0MemObjFree(pVCpu->hm.s.vmx.hMemObjGuestMsr, false);
386 pVCpu->hm.s.vmx.hMemObjGuestMsr = NIL_RTR0MEMOBJ;
387 pVCpu->hm.s.vmx.pvGuestMsr = 0;
388 pVCpu->hm.s.vmx.HCPhysGuestMsr = 0;
389 }
390#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
391 }
392 if (pVM->hm.s.vmx.hMemObjApicAccess != NIL_RTR0MEMOBJ)
393 {
394 RTR0MemObjFree(pVM->hm.s.vmx.hMemObjApicAccess, false);
395 pVM->hm.s.vmx.hMemObjApicAccess = NIL_RTR0MEMOBJ;
396 pVM->hm.s.vmx.pbApicAccess = 0;
397 pVM->hm.s.vmx.HCPhysApicAccess = 0;
398 }
399#ifdef VBOX_WITH_CRASHDUMP_MAGIC
400 if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
401 {
402 ASMMemZero32(pVM->hm.s.vmx.pScratch, PAGE_SIZE);
403 RTR0MemObjFree(pVM->hm.s.vmx.hMemObjScratch, false);
404 pVM->hm.s.vmx.hMemObjScratch = NIL_RTR0MEMOBJ;
405 pVM->hm.s.vmx.pScratch = 0;
406 pVM->hm.s.vmx.pScratchPhys = 0;
407 }
408#endif
409 return VINF_SUCCESS;
410}
411
412
413/**
414 * Sets up VT-x for the specified VM.
415 *
416 * @returns VBox status code.
417 * @param pVM Pointer to the VM.
418 */
419VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
420{
421 int rc = VINF_SUCCESS;
422 uint32_t val;
423
424 AssertReturn(pVM, VERR_INVALID_PARAMETER);
425
426 /* Initialize these always, see hmR3InitFinalizeR0().*/
427 pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NONE;
428 pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NONE;
429
430 /* Determine optimal flush type for EPT. */
431 if (pVM->hm.s.fNestedPaging)
432 {
433 if (pVM->hm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVEPT)
434 {
435 if (pVM->hm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVEPT_CAPS_SINGLE_CONTEXT)
436 pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_SINGLE_CONTEXT;
437 else if (pVM->hm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVEPT_CAPS_ALL_CONTEXTS)
438 pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_ALL_CONTEXTS;
439 else
440 {
441 /*
442 * Should never really happen. EPT is supported but no suitable flush types supported.
443 * We cannot ignore EPT at this point as we've already setup Unrestricted Guest execution.
444 */
445 pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
446 return VERR_VMX_GENERIC;
447 }
448 }
449 else
450 {
451 /*
452 * Should never really happen. EPT is supported but INVEPT instruction is not supported.
453 */
454 pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
455 return VERR_VMX_GENERIC;
456 }
457 }
458
459 /* Determine optimal flush type for VPID. */
460 if (pVM->hm.s.vmx.fVpid)
461 {
462 if (pVM->hm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVVPID)
463 {
464 if (pVM->hm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVVPID_CAPS_SINGLE_CONTEXT)
465 pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_SINGLE_CONTEXT;
466 else if (pVM->hm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVVPID_CAPS_ALL_CONTEXTS)
467 pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_ALL_CONTEXTS;
468 else
469 {
470 /*
471 * Neither SINGLE nor ALL context flush types for VPID supported by the CPU.
472 * We do not handle other flush type combinations, ignore VPID capabilities.
473 */
474 if (pVM->hm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVVPID_CAPS_INDIV_ADDR)
475 Log(("VMXR0SetupVM: Only VMX_FLUSH_VPID_INDIV_ADDR supported. Ignoring VPID.\n"));
476 if (pVM->hm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVVPID_CAPS_SINGLE_CONTEXT_RETAIN_GLOBALS)
477 Log(("VMXR0SetupVM: Only VMX_FLUSH_VPID_SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
478 pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NOT_SUPPORTED;
479 pVM->hm.s.vmx.fVpid = false;
480 }
481 }
482 else
483 {
484 /*
485 * Should not really happen. EPT is supported but INVEPT is not supported.
486 * Ignore VPID capabilities as our code relies on using INVEPT for selective flushing.
487 */
488 Log(("VMXR0SetupVM: VPID supported without INVEPT support. Ignoring VPID.\n"));
489 pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NOT_SUPPORTED;
490 pVM->hm.s.vmx.fVpid = false;
491 }
492 }
493
494 for (VMCPUID i = 0; i < pVM->cCpus; i++)
495 {
496 PVMCPU pVCpu = &pVM->aCpus[i];
497
498 AssertPtr(pVCpu->hm.s.vmx.pvVMCS);
499
500 /* Set revision dword at the beginning of the VMCS structure. */
501 *(uint32_t *)pVCpu->hm.s.vmx.pvVMCS = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.msr.vmx_basic_info);
502
503 /*
504 * Clear and activate the VMCS.
505 */
506 Log(("HCPhysVMCS = %RHp\n", pVCpu->hm.s.vmx.HCPhysVMCS));
507 rc = VMXClearVMCS(pVCpu->hm.s.vmx.HCPhysVMCS);
508 if (RT_FAILURE(rc))
509 goto vmx_end;
510
511 rc = VMXActivateVMCS(pVCpu->hm.s.vmx.HCPhysVMCS);
512 if (RT_FAILURE(rc))
513 goto vmx_end;
514
515 /*
516 * VMX_VMCS_CTRL_PIN_EXEC_CONTROLS
517 * Set required bits to one and zero according to the MSR capabilities.
518 */
519 val = pVM->hm.s.vmx.msr.vmx_pin_ctls.n.disallowed0;
520 val |= VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_EXT_INT_EXIT /* External interrupts */
521 | VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_NMI_EXIT; /* Non-maskable interrupts */
522
523 /*
524 * Enable the VMX preemption timer.
525 */
526 if (pVM->hm.s.vmx.fUsePreemptTimer)
527 val |= VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_PREEMPT_TIMER;
528 val &= pVM->hm.s.vmx.msr.vmx_pin_ctls.n.allowed1;
529
530 rc = VMXWriteVMCS(VMX_VMCS32_CTRL_PIN_EXEC_CONTROLS, val);
531 AssertRC(rc);
532
533 /*
534 * VMX_VMCS_CTRL_PROC_EXEC_CONTROLS
535 * Set required bits to one and zero according to the MSR capabilities.
536 */
537 val = pVM->hm.s.vmx.msr.vmx_proc_ctls.n.disallowed0;
538 /* Program which event cause VM-exits and which features we want to use. */
539 val |= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_HLT_EXIT
540 | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_TSC_OFFSET
541 | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT
542 | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_UNCOND_IO_EXIT
543 | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDPMC_EXIT
544 | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_EXIT
545 | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MWAIT_EXIT; /* don't execute mwait or else we'll idle inside
546 the guest (host thinks the cpu load is high) */
547
548 /* Without nested paging we should intercept invlpg and cr3 mov instructions. */
549 if (!pVM->hm.s.fNestedPaging)
550 {
551 val |= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INVLPG_EXIT
552 | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
553 | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT;
554 }
555
556 /*
557 * VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MWAIT_EXIT might cause a vmlaunch
558 * failure with an invalid control fields error. (combined with some other exit reasons)
559 */
560 if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
561 {
562 /* CR8 reads from the APIC shadow page; writes cause an exit is they lower the TPR below the threshold */
563 val |= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW;
564 Assert(pVM->hm.s.vmx.pbApicAccess);
565 }
566 else
567 /* Exit on CR8 reads & writes in case the TPR shadow feature isn't present. */
568 val |= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR8_STORE_EXIT | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR8_LOAD_EXIT;
569
570 if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
571 {
572 Assert(pVCpu->hm.s.vmx.HCPhysMsrBitmap);
573 val |= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS;
574 }
575
576 /* We will use the secondary control if it's present. */
577 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
578
579 /* Mask away the bits that the CPU doesn't support */
580 /** @todo make sure they don't conflict with the above requirements. */
581 val &= pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1;
582 pVCpu->hm.s.vmx.proc_ctls = val;
583
584 rc = VMXWriteVMCS(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, val);
585 AssertRC(rc);
586
587 if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
588 {
589 /*
590 * VMX_VMCS_CTRL_PROC_EXEC_CONTROLS2
591 * Set required bits to one and zero according to the MSR capabilities.
592 */
593 val = pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.disallowed0;
594 val |= VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT;
595
596 if (pVM->hm.s.fNestedPaging)
597 val |= VMX_VMCS_CTRL_PROC_EXEC2_EPT;
598
599 if (pVM->hm.s.vmx.fVpid)
600 val |= VMX_VMCS_CTRL_PROC_EXEC2_VPID;
601
602 if (pVM->hm.s.fHasIoApic)
603 val |= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC;
604
605 if (pVM->hm.s.vmx.fUnrestrictedGuest)
606 val |= VMX_VMCS_CTRL_PROC_EXEC2_REAL_MODE;
607
608 if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
609 val |= VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP;
610
611 /* Mask away the bits that the CPU doesn't support */
612 /** @todo make sure they don't conflict with the above requirements. */
613 val &= pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1;
614 pVCpu->hm.s.vmx.proc_ctls2 = val;
615 rc = VMXWriteVMCS(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS2, val);
616 AssertRC(rc);
617 }
618
619 /*
620 * VMX_VMCS_CTRL_CR3_TARGET_COUNT
621 * Set required bits to one and zero according to the MSR capabilities.
622 */
623 rc = VMXWriteVMCS(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, 0);
624 AssertRC(rc);
625
626 /*
627 * Forward all exception except #NM & #PF to the guest.
628 * We always need to check pagefaults since our shadow page table can be out of sync.
629 * And we always lazily sync the FPU & XMM state. .
630 */
631
632 /** @todo Possible optimization:
633 * Keep the FPU and XMM state current in the EM thread. That way there's no need to
634 * lazily sync anything, but the downside is that we can't use the FPU stack or XMM
635 * registers ourselves of course.
636 *
637 * Note: only possible if the current state is actually ours (X86_CR0_TS flag)
638 */
639
640 /*
641 * Don't filter page faults, all of them should cause a world switch.
642 */
643 rc = VMXWriteVMCS(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, 0);
644 AssertRC(rc);
645 rc = VMXWriteVMCS(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, 0);
646 AssertRC(rc);
647
648 rc = VMXWriteVMCS64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, 0);
649 AssertRC(rc);
650 rc = VMXWriteVMCS64(VMX_VMCS64_CTRL_IO_BITMAP_A_FULL, 0);
651 AssertRC(rc);
652 rc = VMXWriteVMCS64(VMX_VMCS64_CTRL_IO_BITMAP_B_FULL, 0);
653 AssertRC(rc);
654
655 /*
656 * Set the MSR bitmap address.
657 */
658 if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
659 {
660 Assert(pVCpu->hm.s.vmx.HCPhysMsrBitmap);
661
662 rc = VMXWriteVMCS64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, pVCpu->hm.s.vmx.HCPhysMsrBitmap);
663 AssertRC(rc);
664
665 /*
666 * Allow the guest to directly modify these MSRs; they are loaded/stored automatically
667 * using MSR-load/store areas in the VMCS.
668 */
669 hmR0VmxSetMSRPermission(pVCpu, MSR_IA32_SYSENTER_CS, true, true);
670 hmR0VmxSetMSRPermission(pVCpu, MSR_IA32_SYSENTER_ESP, true, true);
671 hmR0VmxSetMSRPermission(pVCpu, MSR_IA32_SYSENTER_EIP, true, true);
672 hmR0VmxSetMSRPermission(pVCpu, MSR_K8_LSTAR, true, true);
673 hmR0VmxSetMSRPermission(pVCpu, MSR_K6_STAR, true, true);
674 hmR0VmxSetMSRPermission(pVCpu, MSR_K8_SF_MASK, true, true);
675 hmR0VmxSetMSRPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, true, true);
676 hmR0VmxSetMSRPermission(pVCpu, MSR_K8_GS_BASE, true, true);
677 hmR0VmxSetMSRPermission(pVCpu, MSR_K8_FS_BASE, true, true);
678 if (pVCpu->hm.s.vmx.proc_ctls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
679 hmR0VmxSetMSRPermission(pVCpu, MSR_K8_TSC_AUX, true, true);
680 }
681
682#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
683 /*
684 * Set the guest & host MSR load/store physical addresses.
685 */
686 Assert(pVCpu->hm.s.vmx.HCPhysGuestMsr);
687 rc = VMXWriteVMCS64(VMX_VMCS64_CTRL_VMENTRY_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
688 AssertRC(rc);
689 rc = VMXWriteVMCS64(VMX_VMCS64_CTRL_VMEXIT_MSR_STORE_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
690 AssertRC(rc);
691 Assert(pVCpu->hm.s.vmx.HCPhysHostMsr);
692 rc = VMXWriteVMCS64(VMX_VMCS64_CTRL_VMEXIT_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysHostMsr);
693 AssertRC(rc);
694#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
695
696 rc = VMXWriteVMCS(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, 0);
697 AssertRC(rc);
698 rc = VMXWriteVMCS(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, 0);
699 AssertRC(rc);
700 rc = VMXWriteVMCS(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, 0);
701 AssertRC(rc);
702
703 if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
704 {
705 Assert(pVM->hm.s.vmx.hMemObjApicAccess);
706 /* Optional */
707 rc = VMXWriteVMCS(VMX_VMCS32_CTRL_TPR_THRESHOLD, 0);
708 rc |= VMXWriteVMCS64(VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL, pVCpu->hm.s.vmx.HCPhysVAPIC);
709
710 if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
711 rc |= VMXWriteVMCS64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, pVM->hm.s.vmx.HCPhysApicAccess);
712
713 AssertRC(rc);
714 }
715
716 /* Set link pointer to -1. Not currently used. */
717 rc = VMXWriteVMCS64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, 0xFFFFFFFFFFFFFFFFULL);
718 AssertRC(rc);
719
720 /*
721 * Clear VMCS, marking it inactive. Clear implementation specific data and writing back
722 * VMCS data back to memory.
723 */
724 rc = VMXClearVMCS(pVCpu->hm.s.vmx.HCPhysVMCS);
725 AssertRC(rc);
726
727 /*
728 * Configure the VMCS read cache.
729 */
730 PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
731
732 VMXSetupCachedReadVMCS(pCache, VMX_VMCS_GUEST_RIP);
733 VMXSetupCachedReadVMCS(pCache, VMX_VMCS_GUEST_RSP);
734 VMXSetupCachedReadVMCS(pCache, VMX_VMCS_GUEST_RFLAGS);
735 VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE);
736 VMXSetupCachedReadVMCS(pCache, VMX_VMCS_CTRL_CR0_READ_SHADOW);
737 VMXSetupCachedReadVMCS(pCache, VMX_VMCS_GUEST_CR0);
738 VMXSetupCachedReadVMCS(pCache, VMX_VMCS_CTRL_CR4_READ_SHADOW);
739 VMXSetupCachedReadVMCS(pCache, VMX_VMCS_GUEST_CR4);
740 VMXSetupCachedReadVMCS(pCache, VMX_VMCS_GUEST_DR7);
741 VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_GUEST_SYSENTER_CS);
742 VMXSetupCachedReadVMCS(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
743 VMXSetupCachedReadVMCS(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
744 VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_GUEST_GDTR_LIMIT);
745 VMXSetupCachedReadVMCS(pCache, VMX_VMCS_GUEST_GDTR_BASE);
746 VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_GUEST_IDTR_LIMIT);
747 VMXSetupCachedReadVMCS(pCache, VMX_VMCS_GUEST_IDTR_BASE);
748
749 VMX_SETUP_SELREG(ES, pCache);
750 VMX_SETUP_SELREG(SS, pCache);
751 VMX_SETUP_SELREG(CS, pCache);
752 VMX_SETUP_SELREG(DS, pCache);
753 VMX_SETUP_SELREG(FS, pCache);
754 VMX_SETUP_SELREG(GS, pCache);
755 VMX_SETUP_SELREG(LDTR, pCache);
756 VMX_SETUP_SELREG(TR, pCache);
757
758 /*
759 * Status code VMCS reads.
760 */
761 VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_EXIT_REASON);
762 VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_VM_INSTR_ERROR);
763 VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_EXIT_INSTR_LENGTH);
764 VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_EXIT_INTERRUPTION_ERRCODE);
765 VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO);
766 VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_EXIT_INSTR_INFO);
767 VMXSetupCachedReadVMCS(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
768 VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_IDT_INFO);
769 VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_IDT_ERRCODE);
770
771 if (pVM->hm.s.fNestedPaging)
772 {
773 VMXSetupCachedReadVMCS(pCache, VMX_VMCS_GUEST_CR3);
774 VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL);
775 pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
776 }
777 else
778 pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
779 } /* for each VMCPU */
780
781 /*
782 * Setup the right TLB function based on CPU capabilities.
783 */
784 if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
785 pVM->hm.s.vmx.pfnSetupTaggedTlb = hmR0VmxSetupTLBBoth;
786 else if (pVM->hm.s.fNestedPaging)
787 pVM->hm.s.vmx.pfnSetupTaggedTlb = hmR0VmxSetupTLBEPT;
788 else if (pVM->hm.s.vmx.fVpid)
789 pVM->hm.s.vmx.pfnSetupTaggedTlb = hmR0VmxSetupTLBVPID;
790 else
791 pVM->hm.s.vmx.pfnSetupTaggedTlb = hmR0VmxSetupTLBDummy;
792
793vmx_end:
794 hmR0VmxCheckError(pVM, &pVM->aCpus[0], rc);
795 return rc;
796}
797
798
799/**
800 * Sets the permission bits for the specified MSR.
801 *
802 * @param pVCpu Pointer to the VMCPU.
803 * @param ulMSR The MSR value.
804 * @param fRead Whether reading is allowed.
805 * @param fWrite Whether writing is allowed.
806 */
807static void hmR0VmxSetMSRPermission(PVMCPU pVCpu, unsigned ulMSR, bool fRead, bool fWrite)
808{
809 unsigned ulBit;
810 uint8_t *pvMsrBitmap = (uint8_t *)pVCpu->hm.s.vmx.pvMsrBitmap;
811
812 /*
813 * Layout:
814 * 0x000 - 0x3ff - Low MSR read bits
815 * 0x400 - 0x7ff - High MSR read bits
816 * 0x800 - 0xbff - Low MSR write bits
817 * 0xc00 - 0xfff - High MSR write bits
818 */
819 if (ulMSR <= 0x00001FFF)
820 {
821 /* Pentium-compatible MSRs */
822 ulBit = ulMSR;
823 }
824 else if ( ulMSR >= 0xC0000000
825 && ulMSR <= 0xC0001FFF)
826 {
827 /* AMD Sixth Generation x86 Processor MSRs */
828 ulBit = (ulMSR - 0xC0000000);
829 pvMsrBitmap += 0x400;
830 }
831 else
832 {
833 AssertFailed();
834 return;
835 }
836
837 Assert(ulBit <= 0x1fff);
838 if (fRead)
839 ASMBitClear(pvMsrBitmap, ulBit);
840 else
841 ASMBitSet(pvMsrBitmap, ulBit);
842
843 if (fWrite)
844 ASMBitClear(pvMsrBitmap + 0x800, ulBit);
845 else
846 ASMBitSet(pvMsrBitmap + 0x800, ulBit);
847}
848
849
850/**
851 * Injects an event (trap or external interrupt).
852 *
853 * @returns VBox status code. Note that it may return VINF_EM_RESET to
854 * indicate a triple fault when injecting X86_XCPT_DF.
855 *
856 * @param pVM Pointer to the VM.
857 * @param pVCpu Pointer to the VMCPU.
858 * @param pCtx Pointer to the guest CPU Context.
859 * @param intInfo VMX interrupt info.
860 * @param cbInstr Opcode length of faulting instruction.
861 * @param errCode Error code (optional).
862 */
863static int hmR0VmxInjectEvent(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, uint32_t intInfo, uint32_t cbInstr, uint32_t errCode)
864{
865 int rc;
866 uint32_t iGate = VMX_EXIT_INTERRUPTION_INFO_VECTOR(intInfo);
867
868#ifdef VBOX_WITH_STATISTICS
869 STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[iGate & MASK_INJECT_IRQ_STAT]);
870#endif
871
872#ifdef VBOX_STRICT
873 if (iGate == 0xE)
874 {
875 LogFlow(("hmR0VmxInjectEvent: Injecting interrupt %d at %RGv error code=%08x CR2=%RGv intInfo=%08x\n", iGate,
876 (RTGCPTR)pCtx->rip, errCode, pCtx->cr2, intInfo));
877 }
878 else if (iGate < 0x20)
879 {
880 LogFlow(("hmR0VmxInjectEvent: Injecting interrupt %d at %RGv error code=%08x\n", iGate, (RTGCPTR)pCtx->rip,
881 errCode));
882 }
883 else
884 {
885 LogFlow(("INJ-EI: %x at %RGv\n", iGate, (RTGCPTR)pCtx->rip));
886 Assert( VMX_EXIT_INTERRUPTION_INFO_TYPE(intInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW
887 || !VMCPU_FF_ISSET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
888 Assert( VMX_EXIT_INTERRUPTION_INFO_TYPE(intInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW
889 || pCtx->eflags.u32 & X86_EFL_IF);
890 }
891#endif
892
893 if ( CPUMIsGuestInRealModeEx(pCtx)
894 && pVM->hm.s.vmx.pRealModeTSS)
895 {
896 RTGCPHYS GCPhysHandler;
897 uint16_t offset, ip;
898 RTSEL sel;
899
900 /*
901 * Injecting events doesn't work right with real mode emulation.
902 * (#GP if we try to inject external hardware interrupts)
903 * Inject the interrupt or trap directly instead.
904 *
905 * ASSUMES no access handlers for the bits we read or write below (should be safe).
906 */
907 Log(("Manual interrupt/trap '%x' inject (real mode)\n", iGate));
908
909 /*
910 * Check if the interrupt handler is present.
911 */
912 if (iGate * 4 + 3 > pCtx->idtr.cbIdt)
913 {
914 Log(("IDT cbIdt violation\n"));
915 if (iGate != X86_XCPT_DF)
916 {
917 uint32_t intInfo2;
918
919 intInfo2 = (iGate == X86_XCPT_GP) ? (uint32_t)X86_XCPT_DF : iGate;
920 intInfo2 |= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
921 intInfo2 |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
922 intInfo2 |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HWEXCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
923
924 return hmR0VmxInjectEvent(pVM, pVCpu, pCtx, intInfo2, 0, 0 /* no error code according to the Intel docs */);
925 }
926 Log(("Triple fault -> reset the VM!\n"));
927 return VINF_EM_RESET;
928 }
929 if ( VMX_EXIT_INTERRUPTION_INFO_TYPE(intInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW
930 || iGate == 3 /* Both #BP and #OF point to the instruction after. */
931 || iGate == 4)
932 {
933 ip = pCtx->ip + cbInstr;
934 }
935 else
936 ip = pCtx->ip;
937
938 /*
939 * Read the selector:offset pair of the interrupt handler.
940 */
941 GCPhysHandler = (RTGCPHYS)pCtx->idtr.pIdt + iGate * 4;
942 rc = PGMPhysSimpleReadGCPhys(pVM, &offset, GCPhysHandler, sizeof(offset)); AssertRC(rc);
943 rc = PGMPhysSimpleReadGCPhys(pVM, &sel, GCPhysHandler + 2, sizeof(sel)); AssertRC(rc);
944
945 LogFlow(("IDT handler %04X:%04X\n", sel, offset));
946
947 /*
948 * Construct the stack frame.
949 */
950 /** @todo Check stack limit. */
951 pCtx->sp -= 2;
952 LogFlow(("ss:sp %04X:%04X eflags=%x\n", pCtx->ss.Sel, pCtx->sp, pCtx->eflags.u));
953 rc = PGMPhysSimpleWriteGCPhys(pVM, pCtx->ss.u64Base + pCtx->sp, &pCtx->eflags, sizeof(uint16_t)); AssertRC(rc);
954 pCtx->sp -= 2;
955 LogFlow(("ss:sp %04X:%04X cs=%x\n", pCtx->ss.Sel, pCtx->sp, pCtx->cs.Sel));
956 rc = PGMPhysSimpleWriteGCPhys(pVM, pCtx->ss.u64Base + pCtx->sp, &pCtx->cs, sizeof(uint16_t)); AssertRC(rc);
957 pCtx->sp -= 2;
958 LogFlow(("ss:sp %04X:%04X ip=%x\n", pCtx->ss.Sel, pCtx->sp, ip));
959 rc = PGMPhysSimpleWriteGCPhys(pVM, pCtx->ss.u64Base + pCtx->sp, &ip, sizeof(ip)); AssertRC(rc);
960
961 /*
962 * Update the CPU state for executing the handler.
963 */
964 pCtx->rip = offset;
965 pCtx->cs.Sel = sel;
966 pCtx->cs.u64Base = sel << 4;
967 pCtx->eflags.u &= ~(X86_EFL_IF | X86_EFL_TF | X86_EFL_RF | X86_EFL_AC);
968
969 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_SEGMENT_REGS;
970 return VINF_SUCCESS;
971 }
972
973 /*
974 * Set event injection state.
975 */
976 rc = VMXWriteVMCS(VMX_VMCS32_CTRL_ENTRY_IRQ_INFO, intInfo | (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT));
977 rc |= VMXWriteVMCS(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
978 rc |= VMXWriteVMCS(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, errCode);
979
980 AssertRC(rc);
981 return rc;
982}
983
984
985/**
986 * Checks for pending guest interrupts and injects them.
987 *
988 * @returns VBox status code.
989 * @param pVM Pointer to the VM.
990 * @param pVCpu Pointer to the VMCPU.
991 * @param pCtx Pointer to the guest CPU context.
992 */
993static int hmR0VmxCheckPendingInterrupt(PVM pVM, PVMCPU pVCpu, CPUMCTX *pCtx)
994{
995 int rc;
996
997 /*
998 * Dispatch any pending interrupts (injected before, but a VM exit occurred prematurely).
999 */
1000 if (pVCpu->hm.s.Event.fPending)
1001 {
1002 Log(("CPU%d: Reinjecting event %RX64 %08x at %RGv cr2=%RX64\n", pVCpu->idCpu, pVCpu->hm.s.Event.intInfo,
1003 pVCpu->hm.s.Event.errCode, (RTGCPTR)pCtx->rip, pCtx->cr2));
1004 STAM_COUNTER_INC(&pVCpu->hm.s.StatIntReinject);
1005 rc = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, pVCpu->hm.s.Event.intInfo, 0, pVCpu->hm.s.Event.errCode);
1006 AssertRC(rc);
1007
1008 pVCpu->hm.s.Event.fPending = false;
1009 return VINF_SUCCESS;
1010 }
1011
1012 /*
1013 * If an active trap is already pending, we must forward it first!
1014 */
1015 if (!TRPMHasTrap(pVCpu))
1016 {
1017 if (VMCPU_FF_TESTANDCLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI))
1018 {
1019 RTGCUINTPTR intInfo;
1020
1021 Log(("CPU%d: injecting #NMI\n", pVCpu->idCpu));
1022
1023 intInfo = X86_XCPT_NMI;
1024 intInfo |= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
1025 intInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
1026
1027 rc = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, intInfo, 0, 0);
1028 AssertRC(rc);
1029
1030 return VINF_SUCCESS;
1031 }
1032
1033 /** @todo SMI interrupts. */
1034
1035 /*
1036 * When external interrupts are pending, we should exit the VM when IF is set.
1037 */
1038 if (VMCPU_FF_ISPENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC|VMCPU_FF_INTERRUPT_PIC)))
1039 {
1040 if (!(pCtx->eflags.u32 & X86_EFL_IF))
1041 {
1042 if (!(pVCpu->hm.s.vmx.proc_ctls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_IRQ_WINDOW_EXIT))
1043 {
1044 LogFlow(("Enable irq window exit!\n"));
1045 pVCpu->hm.s.vmx.proc_ctls |= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_IRQ_WINDOW_EXIT;
1046 rc = VMXWriteVMCS(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.proc_ctls);
1047 AssertRC(rc);
1048 }
1049 /* else nothing to do but wait */
1050 }
1051 else if (!VMCPU_FF_ISSET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
1052 {
1053 uint8_t u8Interrupt;
1054
1055 rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
1056 Log(("CPU%d: Dispatch interrupt: u8Interrupt=%x (%d) rc=%Rrc cs:rip=%04X:%RGv\n", pVCpu->idCpu,
1057 u8Interrupt, u8Interrupt, rc, pCtx->cs.Sel, (RTGCPTR)pCtx->rip));
1058 if (RT_SUCCESS(rc))
1059 {
1060 rc = TRPMAssertTrap(pVCpu, u8Interrupt, TRPM_HARDWARE_INT);
1061 AssertRC(rc);
1062 }
1063 else
1064 {
1065 /* Can only happen in rare cases where a pending interrupt is cleared behind our back */
1066 Assert(!VMCPU_FF_ISPENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC|VMCPU_FF_INTERRUPT_PIC)));
1067 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
1068 /* Just continue */
1069 }
1070 }
1071 else
1072 Log(("Pending interrupt blocked at %RGv by VM_FF_INHIBIT_INTERRUPTS!!\n", (RTGCPTR)pCtx->rip));
1073 }
1074 }
1075
1076#ifdef VBOX_STRICT
1077 if (TRPMHasTrap(pVCpu))
1078 {
1079 uint8_t u8Vector;
1080 rc = TRPMQueryTrapAll(pVCpu, &u8Vector, 0, 0, 0);
1081 AssertRC(rc);
1082 }
1083#endif
1084
1085 if ( (pCtx->eflags.u32 & X86_EFL_IF)
1086 && (!VMCPU_FF_ISSET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
1087 && TRPMHasTrap(pVCpu)
1088 )
1089 {
1090 uint8_t u8Vector;
1091 TRPMEVENT enmType;
1092 RTGCUINTPTR intInfo;
1093 RTGCUINT errCode;
1094
1095 /*
1096 * If a new event is pending, dispatch it now.
1097 */
1098 rc = TRPMQueryTrapAll(pVCpu, &u8Vector, &enmType, &errCode, 0);
1099 AssertRC(rc);
1100 Assert(pCtx->eflags.Bits.u1IF == 1 || enmType == TRPM_TRAP);
1101 Assert(enmType != TRPM_SOFTWARE_INT);
1102
1103 /*
1104 * Clear the pending trap.
1105 */
1106 rc = TRPMResetTrap(pVCpu);
1107 AssertRC(rc);
1108
1109 intInfo = u8Vector;
1110 intInfo |= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
1111
1112 if (enmType == TRPM_TRAP)
1113 {
1114 switch (u8Vector)
1115 {
1116 case X86_XCPT_DF:
1117 case X86_XCPT_TS:
1118 case X86_XCPT_NP:
1119 case X86_XCPT_SS:
1120 case X86_XCPT_GP:
1121 case X86_XCPT_PF:
1122 case X86_XCPT_AC:
1123 {
1124 /* Valid error codes. */
1125 intInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
1126 break;
1127 }
1128
1129 default:
1130 break;
1131 }
1132
1133 if ( u8Vector == X86_XCPT_BP
1134 || u8Vector == X86_XCPT_OF)
1135 {
1136 intInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SWEXCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
1137 }
1138 else
1139 intInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HWEXCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
1140 }
1141 else
1142 intInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
1143
1144 STAM_COUNTER_INC(&pVCpu->hm.s.StatIntInject);
1145 rc = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, intInfo, 0, errCode);
1146 AssertRC(rc);
1147 } /* if (interrupts can be dispatched) */
1148
1149 return VINF_SUCCESS;
1150}
1151
1152/**
1153 * Checks for pending VMX events and converts them to TRPM. Before we execute any instruction
1154 * outside of VMX, any pending VMX event must be converted so that it can be delivered properly.
1155 *
1156 * @returns VBox status code.
1157 * @param pVCpu Pointer to the VMCPU.
1158 */
1159static int hmR0VmxCheckPendingEvent(PVMCPU pVCpu)
1160{
1161 if (pVCpu->hm.s.Event.fPending)
1162 {
1163 TRPMEVENT enmTrapType;
1164
1165 /* If a trap was already pending, we did something wrong! */
1166 Assert((TRPMQueryTrap(pVCpu, NULL, NULL) == VERR_TRPM_NO_ACTIVE_TRAP));
1167
1168 /*
1169 * Clear the pending event and move it over to TRPM for the rest
1170 * of the world to see.
1171 */
1172 pVCpu->hm.s.Event.fPending = false;
1173 switch (VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hm.s.Event.intInfo))
1174 {
1175 case VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT:
1176 case VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI:
1177 enmTrapType = TRPM_HARDWARE_INT;
1178 break;
1179 case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW:
1180 case VMX_EXIT_INTERRUPTION_INFO_TYPE_SWEXCPT:
1181 case VMX_EXIT_INTERRUPTION_INFO_TYPE_DBEXCPT:
1182 enmTrapType = TRPM_SOFTWARE_INT;
1183 break;
1184 case VMX_EXIT_INTERRUPTION_INFO_TYPE_HWEXCPT:
1185 enmTrapType = TRPM_TRAP;
1186 break;
1187 default:
1188 enmTrapType = TRPM_32BIT_HACK; /* Can't get here. */
1189 AssertFailed();
1190 }
1191 TRPMAssertTrap(pVCpu, VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVCpu->hm.s.Event.intInfo), enmTrapType);
1192 if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.intInfo))
1193 TRPMSetErrorCode(pVCpu, pVCpu->hm.s.Event.errCode);
1194 //@todo: Is there any situation where we need to call TRPMSetFaultAddress()?
1195 }
1196 return VINF_SUCCESS;
1197}
1198
1199/**
1200 * Save the host state into the VMCS.
1201 *
1202 * @returns VBox status code.
1203 * @param pVM Pointer to the VM.
1204 * @param pVCpu Pointer to the VMCPU.
1205 */
1206VMMR0DECL(int) VMXR0SaveHostState(PVM pVM, PVMCPU pVCpu)
1207{
1208 int rc = VINF_SUCCESS;
1209 NOREF(pVM);
1210
1211 /*
1212 * Host CPU Context.
1213 */
1214 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_HOST_CONTEXT)
1215 {
1216 RTIDTR idtr;
1217 RTGDTR gdtr;
1218 RTSEL SelTR;
1219 PCX86DESCHC pDesc;
1220 uintptr_t trBase;
1221 RTSEL cs;
1222 RTSEL ss;
1223 uint64_t cr3;
1224
1225 /*
1226 * Control registers.
1227 */
1228 rc = VMXWriteVMCS(VMX_VMCS_HOST_CR0, ASMGetCR0());
1229 Log2(("VMX_VMCS_HOST_CR0 %08x\n", ASMGetCR0()));
1230#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
1231 if (VMX_IS_64BIT_HOST_MODE())
1232 {
1233 cr3 = hmR0Get64bitCR3();
1234 rc |= VMXWriteVMCS64(VMX_VMCS_HOST_CR3, cr3);
1235 }
1236 else
1237#endif
1238 {
1239 cr3 = ASMGetCR3();
1240 rc |= VMXWriteVMCS(VMX_VMCS_HOST_CR3, cr3);
1241 }
1242 Log2(("VMX_VMCS_HOST_CR3 %08RX64\n", cr3));
1243 rc |= VMXWriteVMCS(VMX_VMCS_HOST_CR4, ASMGetCR4());
1244 Log2(("VMX_VMCS_HOST_CR4 %08x\n", ASMGetCR4()));
1245 AssertRC(rc);
1246
1247 /*
1248 * Selector registers.
1249 */
1250#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
1251 if (VMX_IS_64BIT_HOST_MODE())
1252 {
1253 cs = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelCS;
1254 ss = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelSS;
1255 }
1256 else
1257 {
1258 /* sysenter loads LDT cs & ss, VMX doesn't like this. Load the GDT ones (safe). */
1259 cs = (RTSEL)(uintptr_t)&SUPR0AbsKernelCS;
1260 ss = (RTSEL)(uintptr_t)&SUPR0AbsKernelSS;
1261 }
1262#else
1263 cs = ASMGetCS();
1264 ss = ASMGetSS();
1265#endif
1266 Assert(!(cs & X86_SEL_LDT)); Assert((cs & X86_SEL_RPL) == 0);
1267 Assert(!(ss & X86_SEL_LDT)); Assert((ss & X86_SEL_RPL) == 0);
1268 rc = VMXWriteVMCS(VMX_VMCS16_HOST_FIELD_CS, cs);
1269 /* Note: VMX is (again) very picky about the RPL of the selectors here; we'll restore them manually. */
1270 rc |= VMXWriteVMCS(VMX_VMCS16_HOST_FIELD_DS, 0);
1271 rc |= VMXWriteVMCS(VMX_VMCS16_HOST_FIELD_ES, 0);
1272#if HC_ARCH_BITS == 32
1273 if (!VMX_IS_64BIT_HOST_MODE())
1274 {
1275 rc |= VMXWriteVMCS(VMX_VMCS16_HOST_FIELD_FS, 0);
1276 rc |= VMXWriteVMCS(VMX_VMCS16_HOST_FIELD_GS, 0);
1277 }
1278#endif
1279 rc |= VMXWriteVMCS(VMX_VMCS16_HOST_FIELD_SS, ss);
1280 SelTR = ASMGetTR();
1281 rc |= VMXWriteVMCS(VMX_VMCS16_HOST_FIELD_TR, SelTR);
1282 AssertRC(rc);
1283 Log2(("VMX_VMCS_HOST_FIELD_CS %08x (%08x)\n", cs, ASMGetSS()));
1284 Log2(("VMX_VMCS_HOST_FIELD_DS 00000000 (%08x)\n", ASMGetDS()));
1285 Log2(("VMX_VMCS_HOST_FIELD_ES 00000000 (%08x)\n", ASMGetES()));
1286 Log2(("VMX_VMCS_HOST_FIELD_FS 00000000 (%08x)\n", ASMGetFS()));
1287 Log2(("VMX_VMCS_HOST_FIELD_GS 00000000 (%08x)\n", ASMGetGS()));
1288 Log2(("VMX_VMCS_HOST_FIELD_SS %08x (%08x)\n", ss, ASMGetSS()));
1289 Log2(("VMX_VMCS_HOST_FIELD_TR %08x\n", ASMGetTR()));
1290
1291 /*
1292 * GDTR & IDTR.
1293 */
1294#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
1295 if (VMX_IS_64BIT_HOST_MODE())
1296 {
1297 X86XDTR64 gdtr64, idtr64;
1298 hmR0Get64bitGDTRandIDTR(&gdtr64, &idtr64);
1299 rc = VMXWriteVMCS64(VMX_VMCS_HOST_GDTR_BASE, gdtr64.uAddr);
1300 rc |= VMXWriteVMCS64(VMX_VMCS_HOST_IDTR_BASE, gdtr64.uAddr);
1301 AssertRC(rc);
1302 Log2(("VMX_VMCS_HOST_GDTR_BASE %RX64\n", gdtr64.uAddr));
1303 Log2(("VMX_VMCS_HOST_IDTR_BASE %RX64\n", idtr64.uAddr));
1304 gdtr.cbGdt = gdtr64.cb;
1305 gdtr.pGdt = (uintptr_t)gdtr64.uAddr;
1306 }
1307 else
1308#endif
1309 {
1310 ASMGetGDTR(&gdtr);
1311 rc = VMXWriteVMCS(VMX_VMCS_HOST_GDTR_BASE, gdtr.pGdt);
1312 ASMGetIDTR(&idtr);
1313 rc |= VMXWriteVMCS(VMX_VMCS_HOST_IDTR_BASE, idtr.pIdt);
1314 AssertRC(rc);
1315 Log2(("VMX_VMCS_HOST_GDTR_BASE %RHv\n", gdtr.pGdt));
1316 Log2(("VMX_VMCS_HOST_IDTR_BASE %RHv\n", idtr.pIdt));
1317 }
1318
1319 /*
1320 * Save the base address of the TR selector.
1321 */
1322 if (SelTR > gdtr.cbGdt)
1323 {
1324 AssertMsgFailed(("Invalid TR selector %x. GDTR.cbGdt=%x\n", SelTR, gdtr.cbGdt));
1325 return VERR_VMX_INVALID_HOST_STATE;
1326 }
1327
1328 pDesc = (PCX86DESCHC)(gdtr.pGdt + (SelTR & X86_SEL_MASK));
1329#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
1330 if (VMX_IS_64BIT_HOST_MODE())
1331 {
1332 uint64_t trBase64 = X86DESC64_BASE((PX86DESC64)pDesc);
1333 rc = VMXWriteVMCS64(VMX_VMCS_HOST_TR_BASE, trBase64);
1334 Log2(("VMX_VMCS_HOST_TR_BASE %RX64\n", trBase64));
1335 AssertRC(rc);
1336 }
1337 else
1338#endif
1339 {
1340#if HC_ARCH_BITS == 64
1341 trBase = X86DESC64_BASE(pDesc);
1342#else
1343 trBase = X86DESC_BASE(pDesc);
1344#endif
1345 rc = VMXWriteVMCS(VMX_VMCS_HOST_TR_BASE, trBase);
1346 AssertRC(rc);
1347 Log2(("VMX_VMCS_HOST_TR_BASE %RHv\n", trBase));
1348 }
1349
1350 /*
1351 * FS base and GS base.
1352 */
1353#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
1354 if (VMX_IS_64BIT_HOST_MODE())
1355 {
1356 Log2(("MSR_K8_FS_BASE = %RX64\n", ASMRdMsr(MSR_K8_FS_BASE)));
1357 Log2(("MSR_K8_GS_BASE = %RX64\n", ASMRdMsr(MSR_K8_GS_BASE)));
1358 rc = VMXWriteVMCS64(VMX_VMCS_HOST_FS_BASE, ASMRdMsr(MSR_K8_FS_BASE));
1359 rc |= VMXWriteVMCS64(VMX_VMCS_HOST_GS_BASE, ASMRdMsr(MSR_K8_GS_BASE));
1360 }
1361#endif
1362 AssertRC(rc);
1363
1364 /*
1365 * Sysenter MSRs.
1366 */
1367 /** @todo expensive!! */
1368 rc = VMXWriteVMCS(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
1369 Log2(("VMX_VMCS_HOST_SYSENTER_CS %08x\n", ASMRdMsr_Low(MSR_IA32_SYSENTER_CS)));
1370#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
1371 if (VMX_IS_64BIT_HOST_MODE())
1372 {
1373 Log2(("VMX_VMCS_HOST_SYSENTER_EIP %RX64\n", ASMRdMsr(MSR_IA32_SYSENTER_EIP)));
1374 Log2(("VMX_VMCS_HOST_SYSENTER_ESP %RX64\n", ASMRdMsr(MSR_IA32_SYSENTER_ESP)));
1375 rc |= VMXWriteVMCS64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
1376 rc |= VMXWriteVMCS64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
1377 }
1378 else
1379 {
1380 rc |= VMXWriteVMCS(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
1381 rc |= VMXWriteVMCS(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
1382 Log2(("VMX_VMCS_HOST_SYSENTER_EIP %RX32\n", ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP)));
1383 Log2(("VMX_VMCS_HOST_SYSENTER_ESP %RX32\n", ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP)));
1384 }
1385#elif HC_ARCH_BITS == 32
1386 rc |= VMXWriteVMCS(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
1387 rc |= VMXWriteVMCS(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
1388 Log2(("VMX_VMCS_HOST_SYSENTER_EIP %RX32\n", ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP)));
1389 Log2(("VMX_VMCS_HOST_SYSENTER_ESP %RX32\n", ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP)));
1390#else
1391 Log2(("VMX_VMCS_HOST_SYSENTER_EIP %RX64\n", ASMRdMsr(MSR_IA32_SYSENTER_EIP)));
1392 Log2(("VMX_VMCS_HOST_SYSENTER_ESP %RX64\n", ASMRdMsr(MSR_IA32_SYSENTER_ESP)));
1393 rc |= VMXWriteVMCS64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
1394 rc |= VMXWriteVMCS64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
1395#endif
1396 AssertRC(rc);
1397
1398
1399#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
1400 /*
1401 * Store all host MSRs in the VM-Exit load area, so they will be reloaded after
1402 * the world switch back to the host.
1403 */
1404 PVMXMSR pMsr = (PVMXMSR)pVCpu->hm.s.vmx.pvHostMsr;
1405 unsigned idxMsr = 0;
1406
1407 uint32_t u32HostExtFeatures = ASMCpuId_EDX(0x80000001);
1408 if (u32HostExtFeatures & (X86_CPUID_EXT_FEATURE_EDX_NX | X86_CPUID_EXT_FEATURE_EDX_LONG_MODE))
1409 {
1410#if 0
1411 pMsr->u32IndexMSR = MSR_K6_EFER;
1412 pMsr->u32Reserved = 0;
1413# if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
1414 if (CPUMIsGuestInLongMode(pVCpu))
1415 {
1416 /* Must match the EFER value in our 64 bits switcher. */
1417 pMsr->u64Value = ASMRdMsr(MSR_K6_EFER) | MSR_K6_EFER_LME | MSR_K6_EFER_SCE | MSR_K6_EFER_NXE;
1418 }
1419 else
1420# endif
1421 pMsr->u64Value = ASMRdMsr(MSR_K6_EFER);
1422 pMsr++; idxMsr++;
1423#endif
1424 }
1425
1426# if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
1427 if (VMX_IS_64BIT_HOST_MODE())
1428 {
1429 pMsr->u32IndexMSR = MSR_K6_STAR;
1430 pMsr->u32Reserved = 0;
1431 pMsr->u64Value = ASMRdMsr(MSR_K6_STAR); /* legacy syscall eip, cs & ss */
1432 pMsr++; idxMsr++;
1433 pMsr->u32IndexMSR = MSR_K8_LSTAR;
1434 pMsr->u32Reserved = 0;
1435 pMsr->u64Value = ASMRdMsr(MSR_K8_LSTAR); /* 64 bits mode syscall rip */
1436 pMsr++; idxMsr++;
1437 pMsr->u32IndexMSR = MSR_K8_SF_MASK;
1438 pMsr->u32Reserved = 0;
1439 pMsr->u64Value = ASMRdMsr(MSR_K8_SF_MASK); /* syscall flag mask */
1440 pMsr++; idxMsr++;
1441
1442 /* The KERNEL_GS_BASE MSR doesn't work reliably with auto load/store. See @bugref{6208} */
1443#if 0
1444 pMsr->u32IndexMSR = MSR_K8_KERNEL_GS_BASE;
1445 pMsr->u32Reserved = 0;
1446 pMsr->u64Value = ASMRdMsr(MSR_K8_KERNEL_GS_BASE); /* swapgs exchange value */
1447 pMsr++; idxMsr++;
1448#endif
1449 }
1450# endif
1451
1452 if (pVCpu->hm.s.vmx.proc_ctls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
1453 {
1454 pMsr->u32IndexMSR = MSR_K8_TSC_AUX;
1455 pMsr->u32Reserved = 0;
1456 pMsr->u64Value = ASMRdMsr(MSR_K8_TSC_AUX);
1457 pMsr++; idxMsr++;
1458 }
1459
1460 /** @todo r=ramshankar: check IA32_VMX_MISC bits 27:25 for valid idxMsr
1461 * range. */
1462 rc = VMXWriteVMCS(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, idxMsr);
1463 AssertRC(rc);
1464#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
1465
1466 pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_HOST_CONTEXT;
1467 }
1468 return rc;
1469}
1470
1471
1472/**
1473 * Loads the 4 PDPEs into the guest state when nested paging is used and the
1474 * guest operates in PAE mode.
1475 *
1476 * @returns VBox status code.
1477 * @param pVCpu Pointer to the VMCPU.
1478 * @param pCtx Pointer to the guest CPU context.
1479 */
1480static int hmR0VmxLoadPaePdpes(PVMCPU pVCpu, PCPUMCTX pCtx)
1481{
1482 if (CPUMIsGuestInPAEModeEx(pCtx))
1483 {
1484 X86PDPE aPdpes[4];
1485 int rc = PGMGstGetPaePdpes(pVCpu, &aPdpes[0]);
1486 AssertRCReturn(rc, rc);
1487
1488 rc = VMXWriteVMCS64(VMX_VMCS64_GUEST_PDPTE0_FULL, aPdpes[0].u); AssertRCReturn(rc, rc);
1489 rc = VMXWriteVMCS64(VMX_VMCS64_GUEST_PDPTE1_FULL, aPdpes[1].u); AssertRCReturn(rc, rc);
1490 rc = VMXWriteVMCS64(VMX_VMCS64_GUEST_PDPTE2_FULL, aPdpes[2].u); AssertRCReturn(rc, rc);
1491 rc = VMXWriteVMCS64(VMX_VMCS64_GUEST_PDPTE3_FULL, aPdpes[3].u); AssertRCReturn(rc, rc);
1492 }
1493 return VINF_SUCCESS;
1494}
1495
1496
1497/**
1498 * Saves the 4 PDPEs into the guest state when nested paging is used and the
1499 * guest operates in PAE mode.
1500 *
1501 * @returns VBox status code.
1502 * @param pVCpu Pointer to the VM CPU.
1503 * @param pCtx Pointer to the guest CPU context.
1504 *
1505 * @remarks Tell PGM about CR3 changes before calling this helper.
1506 */
1507static int hmR0VmxSavePaePdpes(PVMCPU pVCpu, PCPUMCTX pCtx)
1508{
1509 if (CPUMIsGuestInPAEModeEx(pCtx))
1510 {
1511 int rc;
1512 X86PDPE aPdpes[4];
1513 rc = VMXReadVMCS64(VMX_VMCS64_GUEST_PDPTE0_FULL, &aPdpes[0].u); AssertRCReturn(rc, rc);
1514 rc = VMXReadVMCS64(VMX_VMCS64_GUEST_PDPTE1_FULL, &aPdpes[1].u); AssertRCReturn(rc, rc);
1515 rc = VMXReadVMCS64(VMX_VMCS64_GUEST_PDPTE2_FULL, &aPdpes[2].u); AssertRCReturn(rc, rc);
1516 rc = VMXReadVMCS64(VMX_VMCS64_GUEST_PDPTE3_FULL, &aPdpes[3].u); AssertRCReturn(rc, rc);
1517
1518 rc = PGMGstUpdatePaePdpes(pVCpu, &aPdpes[0]);
1519 AssertRCReturn(rc, rc);
1520 }
1521 return VINF_SUCCESS;
1522}
1523
1524
1525/**
1526 * Update the exception bitmap according to the current CPU state.
1527 *
1528 * @param pVM Pointer to the VM.
1529 * @param pVCpu Pointer to the VMCPU.
1530 * @param pCtx Pointer to the guest CPU context.
1531 */
1532static void hmR0VmxUpdateExceptionBitmap(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
1533{
1534 uint32_t u32TrapMask;
1535 Assert(pCtx);
1536
1537 /*
1538 * Set up a mask for intercepting traps.
1539 */
1540 /** @todo Do we really need to always intercept #DB? */
1541 u32TrapMask = RT_BIT(X86_XCPT_DB)
1542 | RT_BIT(X86_XCPT_NM)
1543#ifdef VBOX_ALWAYS_TRAP_PF
1544 | RT_BIT(X86_XCPT_PF)
1545#endif
1546#ifdef VBOX_STRICT
1547 | RT_BIT(X86_XCPT_BP)
1548 | RT_BIT(X86_XCPT_DB)
1549 | RT_BIT(X86_XCPT_DE)
1550 | RT_BIT(X86_XCPT_NM)
1551 | RT_BIT(X86_XCPT_UD)
1552 | RT_BIT(X86_XCPT_NP)
1553 | RT_BIT(X86_XCPT_SS)
1554 | RT_BIT(X86_XCPT_GP)
1555 | RT_BIT(X86_XCPT_MF)
1556#endif
1557 ;
1558
1559 /*
1560 * Without nested paging, #PF must be intercepted to implement shadow paging.
1561 */
1562 /** @todo NP state won't change so maybe we should build the initial trap mask up front? */
1563 if (!pVM->hm.s.fNestedPaging)
1564 u32TrapMask |= RT_BIT(X86_XCPT_PF);
1565
1566 /* Catch floating point exceptions if we need to report them to the guest in a different way. */
1567 if (!(pCtx->cr0 & X86_CR0_NE))
1568 u32TrapMask |= RT_BIT(X86_XCPT_MF);
1569
1570#ifdef VBOX_STRICT
1571 Assert(u32TrapMask & RT_BIT(X86_XCPT_GP));
1572#endif
1573
1574 /*
1575 * Intercept all exceptions in real mode as none of them can be injected directly (#GP otherwise).
1576 */
1577 /** @todo Despite the claim to intercept everything, with NP we do not intercept #PF. Should we? */
1578 if ( CPUMIsGuestInRealModeEx(pCtx)
1579 && pVM->hm.s.vmx.pRealModeTSS)
1580 {
1581 u32TrapMask |= RT_BIT(X86_XCPT_DE)
1582 | RT_BIT(X86_XCPT_DB)
1583 | RT_BIT(X86_XCPT_NMI)
1584 | RT_BIT(X86_XCPT_BP)
1585 | RT_BIT(X86_XCPT_OF)
1586 | RT_BIT(X86_XCPT_BR)
1587 | RT_BIT(X86_XCPT_UD)
1588 | RT_BIT(X86_XCPT_DF)
1589 | RT_BIT(X86_XCPT_CO_SEG_OVERRUN)
1590 | RT_BIT(X86_XCPT_TS)
1591 | RT_BIT(X86_XCPT_NP)
1592 | RT_BIT(X86_XCPT_SS)
1593 | RT_BIT(X86_XCPT_GP)
1594 | RT_BIT(X86_XCPT_MF)
1595 | RT_BIT(X86_XCPT_AC)
1596 | RT_BIT(X86_XCPT_MC)
1597 | RT_BIT(X86_XCPT_XF)
1598 ;
1599 }
1600
1601 int rc = VMXWriteVMCS(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32TrapMask);
1602 AssertRC(rc);
1603}
1604
1605
1606/**
1607 * Loads a minimal guest state.
1608 *
1609 * NOTE: Don't do anything here that can cause a jump back to ring 3!!!!!
1610 *
1611 * @param pVM Pointer to the VM.
1612 * @param pVCpu Pointer to the VMCPU.
1613 * @param pCtx Pointer to the guest CPU context.
1614 */
1615VMMR0DECL(void) VMXR0LoadMinimalGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
1616{
1617 int rc;
1618 X86EFLAGS eflags;
1619
1620 Assert(!(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_ALL_GUEST));
1621
1622 /*
1623 * Load EIP, ESP and EFLAGS.
1624 */
1625 rc = VMXWriteVMCS64(VMX_VMCS_GUEST_RIP, pCtx->rip);
1626 rc |= VMXWriteVMCS64(VMX_VMCS_GUEST_RSP, pCtx->rsp);
1627 AssertRC(rc);
1628
1629 /*
1630 * Bits 22-31, 15, 5 & 3 must be zero. Bit 1 must be 1.
1631 */
1632 eflags = pCtx->eflags;
1633 eflags.u32 &= VMX_EFLAGS_RESERVED_0;
1634 eflags.u32 |= VMX_EFLAGS_RESERVED_1;
1635
1636 /*
1637 * Check if real mode emulation using v86 mode.
1638 */
1639 if ( CPUMIsGuestInRealModeEx(pCtx)
1640 && pVM->hm.s.vmx.pRealModeTSS)
1641 {
1642 pVCpu->hm.s.vmx.RealMode.eflags = eflags;
1643
1644 eflags.Bits.u1VM = 1;
1645 eflags.Bits.u2IOPL = 0; /* must always be 0 or else certain instructions won't cause faults. */
1646 }
1647 rc = VMXWriteVMCS(VMX_VMCS_GUEST_RFLAGS, eflags.u32);
1648 AssertRC(rc);
1649}
1650
1651
1652/**
1653 * Loads the guest state.
1654 *
1655 * NOTE: Don't do anything here that can cause a jump back to ring 3!!!!!
1656 *
1657 * @returns VBox status code.
1658 * @param pVM Pointer to the VM.
1659 * @param pVCpu Pointer to the VMCPU.
1660 * @param pCtx Pointer to the guest CPU context.
1661 */
1662VMMR0DECL(int) VMXR0LoadGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
1663{
1664 int rc = VINF_SUCCESS;
1665 RTGCUINTPTR val;
1666
1667 /*
1668 * VMX_VMCS_CTRL_ENTRY_CONTROLS
1669 * Set required bits to one and zero according to the MSR capabilities.
1670 */
1671 val = pVM->hm.s.vmx.msr.vmx_entry.n.disallowed0;
1672
1673 /*
1674 * Load guest debug controls (DR7 & IA32_DEBUGCTL_MSR).
1675 * Forced to 1 on the 'first' VT-x capable CPUs; this actually includes the newest Nehalem CPUs
1676 */
1677 val |= VMX_VMCS_CTRL_ENTRY_CONTROLS_LOAD_DEBUG;
1678
1679 if (CPUMIsGuestInLongModeEx(pCtx))
1680 val |= VMX_VMCS_CTRL_ENTRY_CONTROLS_IA64_MODE;
1681 /* else Must be zero when AMD64 is not available. */
1682
1683 /*
1684 * Mask away the bits that the CPU doesn't support.
1685 */
1686 val &= pVM->hm.s.vmx.msr.vmx_entry.n.allowed1;
1687 rc = VMXWriteVMCS(VMX_VMCS32_CTRL_ENTRY_CONTROLS, val);
1688 AssertRC(rc);
1689
1690 /*
1691 * VMX_VMCS_CTRL_EXIT_CONTROLS
1692 * Set required bits to one and zero according to the MSR capabilities.
1693 */
1694 val = pVM->hm.s.vmx.msr.vmx_exit.n.disallowed0;
1695
1696 /*
1697 * Save debug controls (DR7 & IA32_DEBUGCTL_MSR)
1698 * Forced to 1 on the 'first' VT-x capable CPUs; this actually includes the newest Nehalem CPUs
1699 */
1700 val |= VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_DEBUG;
1701
1702#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
1703 if (VMX_IS_64BIT_HOST_MODE())
1704 val |= VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_AMD64;
1705 /* else Must be zero when AMD64 is not available. */
1706#elif HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
1707 if (CPUMIsGuestInLongModeEx(pCtx))
1708 val |= VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_AMD64; /* our switcher goes to long mode */
1709 else
1710 Assert(!(val & VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_AMD64));
1711#endif
1712 val &= pVM->hm.s.vmx.msr.vmx_exit.n.allowed1;
1713
1714 /*
1715 * Don't acknowledge external interrupts on VM-exit.
1716 */
1717 rc = VMXWriteVMCS(VMX_VMCS32_CTRL_EXIT_CONTROLS, val);
1718 AssertRC(rc);
1719
1720 /*
1721 * Guest CPU context: ES, CS, SS, DS, FS, GS.
1722 */
1723 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SEGMENT_REGS)
1724 {
1725 if (pVM->hm.s.vmx.pRealModeTSS)
1726 {
1727 PGMMODE enmGuestMode = PGMGetGuestMode(pVCpu);
1728 if (pVCpu->hm.s.vmx.enmLastSeenGuestMode != enmGuestMode)
1729 {
1730 /*
1731 * Correct weird requirements for switching to protected mode.
1732 */
1733 if ( pVCpu->hm.s.vmx.enmLastSeenGuestMode == PGMMODE_REAL
1734 && enmGuestMode >= PGMMODE_PROTECTED)
1735 {
1736#ifdef VBOX_WITH_REM
1737 /*
1738 * Flush the recompiler code cache as it's not unlikely the guest will rewrite code
1739 * it will later execute in real mode (OpenBSD 4.0 is one such example)
1740 */
1741 REMFlushTBs(pVM);
1742#endif
1743
1744 /*
1745 * DPL of all hidden selector registers must match the current CPL (0).
1746 */
1747 pCtx->cs.Attr.n.u2Dpl = 0;
1748 pCtx->cs.Attr.n.u4Type = X86_SEL_TYPE_CODE | X86_SEL_TYPE_RW_ACC;
1749
1750 pCtx->ds.Attr.n.u2Dpl = 0;
1751 pCtx->es.Attr.n.u2Dpl = 0;
1752 pCtx->fs.Attr.n.u2Dpl = 0;
1753 pCtx->gs.Attr.n.u2Dpl = 0;
1754 pCtx->ss.Attr.n.u2Dpl = 0;
1755 }
1756 pVCpu->hm.s.vmx.enmLastSeenGuestMode = enmGuestMode;
1757 }
1758 }
1759
1760 VMX_WRITE_SELREG(ES, es);
1761 AssertRC(rc);
1762
1763 VMX_WRITE_SELREG(CS, cs);
1764 AssertRC(rc);
1765
1766 VMX_WRITE_SELREG(SS, ss);
1767 AssertRC(rc);
1768
1769 VMX_WRITE_SELREG(DS, ds);
1770 AssertRC(rc);
1771
1772 VMX_WRITE_SELREG(FS, fs);
1773 AssertRC(rc);
1774
1775 VMX_WRITE_SELREG(GS, gs);
1776 AssertRC(rc);
1777 }
1778
1779 /*
1780 * Guest CPU context: LDTR.
1781 */
1782 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_LDTR)
1783 {
1784 if (pCtx->ldtr.Sel == 0)
1785 {
1786 rc = VMXWriteVMCS(VMX_VMCS16_GUEST_FIELD_LDTR, 0);
1787 rc |= VMXWriteVMCS(VMX_VMCS32_GUEST_LDTR_LIMIT, 0);
1788 rc |= VMXWriteVMCS64(VMX_VMCS_GUEST_LDTR_BASE, 0); /* @todo removing "64" in the function should be the same. */
1789 /* Note: vmlaunch will fail with 0 or just 0x02. No idea why. */
1790 rc |= VMXWriteVMCS(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, 0x82 /* present, LDT */);
1791 }
1792 else
1793 {
1794 rc = VMXWriteVMCS(VMX_VMCS16_GUEST_FIELD_LDTR, pCtx->ldtr.Sel);
1795 rc |= VMXWriteVMCS(VMX_VMCS32_GUEST_LDTR_LIMIT, pCtx->ldtr.u32Limit);
1796 rc |= VMXWriteVMCS64(VMX_VMCS_GUEST_LDTR_BASE, pCtx->ldtr.u64Base); /* @todo removing "64" and it should be the same */
1797 rc |= VMXWriteVMCS(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, pCtx->ldtr.Attr.u);
1798 }
1799 AssertRC(rc);
1800 }
1801
1802 /*
1803 * Guest CPU context: TR.
1804 */
1805 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_TR)
1806 {
1807 /*
1808 * Real mode emulation using v86 mode with CR4.VME (interrupt redirection
1809 * using the int bitmap in the TSS).
1810 */
1811 if ( CPUMIsGuestInRealModeEx(pCtx)
1812 && pVM->hm.s.vmx.pRealModeTSS)
1813 {
1814 RTGCPHYS GCPhys;
1815
1816 /* We convert it here every time as PCI regions could be reconfigured. */
1817 rc = PDMVMMDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
1818 AssertRC(rc);
1819
1820 rc = VMXWriteVMCS(VMX_VMCS16_GUEST_FIELD_TR, 0);
1821 rc |= VMXWriteVMCS(VMX_VMCS32_GUEST_TR_LIMIT, HM_VTX_TSS_SIZE);
1822 rc |= VMXWriteVMCS64(VMX_VMCS_GUEST_TR_BASE, GCPhys /* phys = virt in this mode */);
1823
1824 X86DESCATTR attr;
1825
1826 attr.u = 0;
1827 attr.n.u1Present = 1;
1828 attr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
1829 val = attr.u;
1830 }
1831 else
1832 {
1833 rc = VMXWriteVMCS(VMX_VMCS16_GUEST_FIELD_TR, pCtx->tr.Sel);
1834 rc |= VMXWriteVMCS(VMX_VMCS32_GUEST_TR_LIMIT, pCtx->tr.u32Limit);
1835 rc |= VMXWriteVMCS64(VMX_VMCS_GUEST_TR_BASE, pCtx->tr.u64Base);
1836
1837 val = pCtx->tr.Attr.u;
1838
1839 /* The TSS selector must be busy (REM bugs? see defect #XXXX). */
1840 if (!(val & X86_SEL_TYPE_SYS_TSS_BUSY_MASK))
1841 {
1842 if (val & 0xf)
1843 val |= X86_SEL_TYPE_SYS_TSS_BUSY_MASK;
1844 else
1845 /* Default if no TR selector has been set (otherwise vmlaunch will fail!) */
1846 val = (val & ~0xF) | X86_SEL_TYPE_SYS_386_TSS_BUSY;
1847 }
1848 AssertMsg((val & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY || (val & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY,
1849 ("%#x\n", val));
1850 }
1851 rc |= VMXWriteVMCS(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, val);
1852 AssertRC(rc);
1853 }
1854
1855 /*
1856 * Guest CPU context: GDTR.
1857 */
1858 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_GDTR)
1859 {
1860 rc = VMXWriteVMCS(VMX_VMCS32_GUEST_GDTR_LIMIT, pCtx->gdtr.cbGdt);
1861 rc |= VMXWriteVMCS64(VMX_VMCS_GUEST_GDTR_BASE, pCtx->gdtr.pGdt);
1862 AssertRC(rc);
1863 }
1864
1865 /*
1866 * Guest CPU context: IDTR.
1867 */
1868 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_IDTR)
1869 {
1870 rc = VMXWriteVMCS(VMX_VMCS32_GUEST_IDTR_LIMIT, pCtx->idtr.cbIdt);
1871 rc |= VMXWriteVMCS64(VMX_VMCS_GUEST_IDTR_BASE, pCtx->idtr.pIdt);
1872 AssertRC(rc);
1873 }
1874
1875 /*
1876 * Sysenter MSRs.
1877 */
1878 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_MSR)
1879 {
1880 rc = VMXWriteVMCS(VMX_VMCS32_GUEST_SYSENTER_CS, pCtx->SysEnter.cs);
1881 rc |= VMXWriteVMCS64(VMX_VMCS_GUEST_SYSENTER_EIP, pCtx->SysEnter.eip);
1882 rc |= VMXWriteVMCS64(VMX_VMCS_GUEST_SYSENTER_ESP, pCtx->SysEnter.esp);
1883 AssertRC(rc);
1884 }
1885
1886 /*
1887 * Guest CPU context: Control registers.
1888 */
1889 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR0)
1890 {
1891 val = pCtx->cr0;
1892 rc = VMXWriteVMCS(VMX_VMCS_CTRL_CR0_READ_SHADOW, val);
1893 Log2(("Guest CR0-shadow %08x\n", val));
1894 if (CPUMIsGuestFPUStateActive(pVCpu) == false)
1895 {
1896 /* Always use #NM exceptions to load the FPU/XMM state on demand. */
1897 val |= X86_CR0_TS | X86_CR0_ET | X86_CR0_NE | X86_CR0_MP;
1898 }
1899 else
1900 {
1901 /** @todo check if we support the old style mess correctly. */
1902 if (!(val & X86_CR0_NE))
1903 Log(("Forcing X86_CR0_NE!!!\n"));
1904
1905 val |= X86_CR0_NE; /* always turn on the native mechanism to report FPU errors (old style uses interrupts) */
1906 }
1907 /* Protected mode & paging are always enabled; we use them for emulating real and protected mode without paging too. */
1908 if (!pVM->hm.s.vmx.fUnrestrictedGuest)
1909 val |= X86_CR0_PE | X86_CR0_PG;
1910
1911 if (pVM->hm.s.fNestedPaging)
1912 {
1913 if (CPUMIsGuestInPagedProtectedModeEx(pCtx))
1914 {
1915 /* Disable CR3 read/write monitoring as we don't need it for EPT. */
1916 pVCpu->hm.s.vmx.proc_ctls &= ~( VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
1917 | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT);
1918 }
1919 else
1920 {
1921 /* Reenable CR3 read/write monitoring as our identity mapped page table is active. */
1922 pVCpu->hm.s.vmx.proc_ctls |= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
1923 | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT;
1924 }
1925 rc = VMXWriteVMCS(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.proc_ctls);
1926 AssertRC(rc);
1927 }
1928 else
1929 {
1930 /* Note: We must also set this as we rely on protecting various pages for which supervisor writes must be caught. */
1931 val |= X86_CR0_WP;
1932 }
1933
1934 /* Always enable caching. */
1935 val &= ~(X86_CR0_CD|X86_CR0_NW);
1936
1937 rc |= VMXWriteVMCS64(VMX_VMCS_GUEST_CR0, val);
1938 Log2(("Guest CR0 %08x\n", val));
1939
1940 /*
1941 * CR0 flags owned by the host; if the guests attempts to change them, then the VM will exit.
1942 */
1943 val = X86_CR0_PE /* Must monitor this bit (assumptions are made for real mode emulation) */
1944 | X86_CR0_WP /* Must monitor this bit (it must always be enabled). */
1945 | X86_CR0_PG /* Must monitor this bit (assumptions are made for real mode & protected mode without paging emulation) */
1946 | X86_CR0_CD /* Bit not restored during VM-exit! */
1947 | X86_CR0_NW /* Bit not restored during VM-exit! */
1948 | X86_CR0_NE;
1949
1950 /*
1951 * When the guest's FPU state is active, then we no longer care about the FPU related bits.
1952 */
1953 if (CPUMIsGuestFPUStateActive(pVCpu) == false)
1954 val |= X86_CR0_TS | X86_CR0_ET | X86_CR0_MP;
1955
1956 pVCpu->hm.s.vmx.cr0_mask = val;
1957
1958 rc |= VMXWriteVMCS(VMX_VMCS_CTRL_CR0_MASK, val);
1959 Log2(("Guest CR0-mask %08x\n", val));
1960 AssertRC(rc);
1961 }
1962
1963 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR4)
1964 {
1965 rc = VMXWriteVMCS(VMX_VMCS_CTRL_CR4_READ_SHADOW, pCtx->cr4);
1966 Log2(("Guest CR4-shadow %08x\n", pCtx->cr4));
1967 /* Set the required bits in cr4 too (currently X86_CR4_VMXE). */
1968 val = pCtx->cr4 | (uint32_t)pVM->hm.s.vmx.msr.vmx_cr4_fixed0;
1969
1970 if (!pVM->hm.s.fNestedPaging)
1971 {
1972 switch (pVCpu->hm.s.enmShadowMode)
1973 {
1974 case PGMMODE_REAL: /* Real mode -> emulated using v86 mode */
1975 case PGMMODE_PROTECTED: /* Protected mode, no paging -> emulated using identity mapping. */
1976 case PGMMODE_32_BIT: /* 32-bit paging. */
1977 val &= ~X86_CR4_PAE;
1978 break;
1979
1980 case PGMMODE_PAE: /* PAE paging. */
1981 case PGMMODE_PAE_NX: /* PAE paging with NX enabled. */
1982 /** Must use PAE paging as we could use physical memory > 4 GB */
1983 val |= X86_CR4_PAE;
1984 break;
1985
1986 case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
1987 case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
1988#ifdef VBOX_ENABLE_64_BITS_GUESTS
1989 break;
1990#else
1991 AssertFailed();
1992 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
1993#endif
1994 default: /* shut up gcc */
1995 AssertFailed();
1996 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
1997 }
1998 }
1999 else if ( !CPUMIsGuestInPagedProtectedModeEx(pCtx)
2000 && !pVM->hm.s.vmx.fUnrestrictedGuest)
2001 {
2002 /* We use 4 MB pages in our identity mapping page table for real and protected mode without paging. */
2003 val |= X86_CR4_PSE;
2004 /* Our identity mapping is a 32 bits page directory. */
2005 val &= ~X86_CR4_PAE;
2006 }
2007
2008 /*
2009 * Turn off VME if we're in emulated real mode.
2010 */
2011 if ( CPUMIsGuestInRealModeEx(pCtx)
2012 && pVM->hm.s.vmx.pRealModeTSS)
2013 {
2014 val &= ~X86_CR4_VME;
2015 }
2016
2017 rc |= VMXWriteVMCS64(VMX_VMCS_GUEST_CR4, val);
2018 Log2(("Guest CR4 %08x\n", val));
2019
2020 /*
2021 * CR4 flags owned by the host; if the guests attempts to change them, then the VM will exit.
2022 */
2023 val = 0
2024 | X86_CR4_VME
2025 | X86_CR4_PAE
2026 | X86_CR4_PGE
2027 | X86_CR4_PSE
2028 | X86_CR4_VMXE;
2029 pVCpu->hm.s.vmx.cr4_mask = val;
2030
2031 rc |= VMXWriteVMCS(VMX_VMCS_CTRL_CR4_MASK, val);
2032 Log2(("Guest CR4-mask %08x\n", val));
2033 AssertRC(rc);
2034 }
2035
2036#if 0
2037 /* Enable single stepping if requested and CPU supports it. */
2038 if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_TRAP_FLAG)
2039 if (DBGFIsStepping(pVCpu))
2040 {
2041 pVCpu->hm.s.vmx.proc_ctls |= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_TRAP_FLAG;
2042 rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.proc_ctls);
2043 AssertRC(rc);
2044 }
2045#endif
2046
2047 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR3)
2048 {
2049 if (pVM->hm.s.fNestedPaging)
2050 {
2051 Assert(PGMGetHyperCR3(pVCpu));
2052 pVCpu->hm.s.vmx.GCPhysEPTP = PGMGetHyperCR3(pVCpu);
2053
2054 Assert(!(pVCpu->hm.s.vmx.GCPhysEPTP & 0xfff));
2055 /** @todo Check the IA32_VMX_EPT_VPID_CAP MSR for other supported memory types. */
2056 pVCpu->hm.s.vmx.GCPhysEPTP |= VMX_EPT_MEMTYPE_WB
2057 | (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
2058
2059 rc = VMXWriteVMCS64(VMX_VMCS64_CTRL_EPTP_FULL, pVCpu->hm.s.vmx.GCPhysEPTP);
2060 AssertRC(rc);
2061
2062 if ( !CPUMIsGuestInPagedProtectedModeEx(pCtx)
2063 && !pVM->hm.s.vmx.fUnrestrictedGuest)
2064 {
2065 RTGCPHYS GCPhys;
2066
2067 /* We convert it here every time as PCI regions could be reconfigured. */
2068 rc = PDMVMMDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
2069 AssertMsgRC(rc, ("pNonPagingModeEPTPageTable = %RGv\n", pVM->hm.s.vmx.pNonPagingModeEPTPageTable));
2070
2071 /*
2072 * We use our identity mapping page table here as we need to map guest virtual to
2073 * guest physical addresses; EPT will take care of the translation to host physical addresses.
2074 */
2075 val = GCPhys;
2076 }
2077 else
2078 {
2079 /* Save the real guest CR3 in VMX_VMCS_GUEST_CR3 */
2080 val = pCtx->cr3;
2081 rc = hmR0VmxLoadPaePdpes(pVCpu, pCtx);
2082 AssertRCReturn(rc, rc);
2083 }
2084 }
2085 else
2086 {
2087 val = PGMGetHyperCR3(pVCpu);
2088 Assert(val || VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL));
2089 }
2090
2091 /* Save our shadow CR3 register. */
2092 rc = VMXWriteVMCS64(VMX_VMCS_GUEST_CR3, val);
2093 AssertRC(rc);
2094 }
2095
2096 /*
2097 * Guest CPU context: Debug registers.
2098 */
2099 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_DEBUG)
2100 {
2101 pCtx->dr[6] |= X86_DR6_INIT_VAL; /* set all reserved bits to 1. */
2102 pCtx->dr[6] &= ~RT_BIT(12); /* must be zero. */
2103
2104 pCtx->dr[7] &= 0xffffffff; /* upper 32 bits reserved */
2105 pCtx->dr[7] &= ~(RT_BIT(11) | RT_BIT(12) | RT_BIT(14) | RT_BIT(15)); /* must be zero */
2106 pCtx->dr[7] |= 0x400; /* must be one */
2107
2108 /* Resync DR7 */
2109 rc = VMXWriteVMCS64(VMX_VMCS_GUEST_DR7, pCtx->dr[7]);
2110 AssertRC(rc);
2111
2112#ifdef DEBUG
2113 /* Sync the hypervisor debug state now if any breakpoint is armed. */
2114 if ( CPUMGetHyperDR7(pVCpu) & (X86_DR7_ENABLED_MASK|X86_DR7_GD)
2115 && !CPUMIsHyperDebugStateActive(pVCpu)
2116 && !DBGFIsStepping(pVCpu))
2117 {
2118 /* Save the host and load the hypervisor debug state. */
2119 rc = CPUMR0LoadHyperDebugState(pVM, pVCpu, pCtx, true /* include DR6 */);
2120 AssertRC(rc);
2121
2122 /* DRx intercepts remain enabled. */
2123
2124 /* Override dr7 with the hypervisor value. */
2125 rc = VMXWriteVMCS64(VMX_VMCS_GUEST_DR7, CPUMGetHyperDR7(pVCpu));
2126 AssertRC(rc);
2127 }
2128 else
2129#endif
2130 /* Sync the debug state now if any breakpoint is armed. */
2131 if ( (pCtx->dr[7] & (X86_DR7_ENABLED_MASK|X86_DR7_GD))
2132 && !CPUMIsGuestDebugStateActive(pVCpu)
2133 && !DBGFIsStepping(pVCpu))
2134 {
2135 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
2136
2137 /* Disable DRx move intercepts. */
2138 pVCpu->hm.s.vmx.proc_ctls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT;
2139 rc = VMXWriteVMCS(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.proc_ctls);
2140 AssertRC(rc);
2141
2142 /* Save the host and load the guest debug state. */
2143 rc = CPUMR0LoadGuestDebugState(pVM, pVCpu, pCtx, true /* include DR6 */);
2144 AssertRC(rc);
2145 }
2146
2147 /* IA32_DEBUGCTL MSR. */
2148 rc = VMXWriteVMCS64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, 0);
2149 AssertRC(rc);
2150
2151 /** @todo do we really ever need this? */
2152 rc |= VMXWriteVMCS(VMX_VMCS_GUEST_DEBUG_EXCEPTIONS, 0);
2153 AssertRC(rc);
2154 }
2155
2156 /*
2157 * 64-bit guest mode.
2158 */
2159 if (CPUMIsGuestInLongModeEx(pCtx))
2160 {
2161#if !defined(VBOX_ENABLE_64_BITS_GUESTS)
2162 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
2163#elif HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2164 pVCpu->hm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
2165#else
2166# ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2167 if (!pVM->hm.s.fAllow64BitGuests)
2168 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
2169# endif
2170 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM64;
2171#endif
2172 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_MSR)
2173 {
2174 /* Update these as wrmsr might have changed them. */
2175 rc = VMXWriteVMCS64(VMX_VMCS_GUEST_FS_BASE, pCtx->fs.u64Base);
2176 AssertRC(rc);
2177 rc = VMXWriteVMCS64(VMX_VMCS_GUEST_GS_BASE, pCtx->gs.u64Base);
2178 AssertRC(rc);
2179 }
2180 }
2181 else
2182 {
2183 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
2184 }
2185
2186 hmR0VmxUpdateExceptionBitmap(pVM, pVCpu, pCtx);
2187
2188#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
2189 /*
2190 * Store all guest MSRs in the VM-entry load area, so they will be loaded
2191 * during VM-entry and restored into the VM-exit store area during VM-exit.
2192 */
2193 PVMXMSR pMsr = (PVMXMSR)pVCpu->hm.s.vmx.pvGuestMsr;
2194 unsigned idxMsr = 0;
2195
2196 uint32_t u32GstExtFeatures;
2197 uint32_t u32Temp;
2198 CPUMGetGuestCpuId(pVCpu, 0x80000001, &u32Temp, &u32Temp, &u32Temp, &u32GstExtFeatures);
2199
2200 if (u32GstExtFeatures & (X86_CPUID_EXT_FEATURE_EDX_NX | X86_CPUID_EXT_FEATURE_EDX_LONG_MODE))
2201 {
2202#if 0
2203 pMsr->u32IndexMSR = MSR_K6_EFER;
2204 pMsr->u32Reserved = 0;
2205 pMsr->u64Value = pCtx->msrEFER;
2206 /* VT-x will complain if only MSR_K6_EFER_LME is set. */
2207 if (!CPUMIsGuestInLongModeEx(pCtx))
2208 pMsr->u64Value &= ~(MSR_K6_EFER_LMA | MSR_K6_EFER_LME);
2209 pMsr++; idxMsr++;
2210#endif
2211
2212 if (u32GstExtFeatures & X86_CPUID_EXT_FEATURE_EDX_LONG_MODE)
2213 {
2214 pMsr->u32IndexMSR = MSR_K8_LSTAR;
2215 pMsr->u32Reserved = 0;
2216 pMsr->u64Value = pCtx->msrLSTAR; /* 64 bits mode syscall rip */
2217 pMsr++; idxMsr++;
2218 pMsr->u32IndexMSR = MSR_K6_STAR;
2219 pMsr->u32Reserved = 0;
2220 pMsr->u64Value = pCtx->msrSTAR; /* legacy syscall eip, cs & ss */
2221 pMsr++; idxMsr++;
2222 pMsr->u32IndexMSR = MSR_K8_SF_MASK;
2223 pMsr->u32Reserved = 0;
2224 pMsr->u64Value = pCtx->msrSFMASK; /* syscall flag mask */
2225 pMsr++; idxMsr++;
2226
2227 /* The KERNEL_GS_BASE MSR doesn't work reliably with auto load/store. See @bugref{6208} */
2228#if 0
2229 pMsr->u32IndexMSR = MSR_K8_KERNEL_GS_BASE;
2230 pMsr->u32Reserved = 0;
2231 pMsr->u64Value = pCtx->msrKERNELGSBASE; /* swapgs exchange value */
2232 pMsr++; idxMsr++;
2233#endif
2234 }
2235 }
2236
2237 if ( pVCpu->hm.s.vmx.proc_ctls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP
2238 && (u32GstExtFeatures & X86_CPUID_EXT_FEATURE_EDX_RDTSCP))
2239 {
2240 pMsr->u32IndexMSR = MSR_K8_TSC_AUX;
2241 pMsr->u32Reserved = 0;
2242 rc = CPUMQueryGuestMsr(pVCpu, MSR_K8_TSC_AUX, &pMsr->u64Value);
2243 AssertRC(rc);
2244 pMsr++; idxMsr++;
2245 }
2246
2247 pVCpu->hm.s.vmx.cCachedMsrs = idxMsr;
2248
2249 rc = VMXWriteVMCS(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, idxMsr);
2250 AssertRC(rc);
2251
2252 rc = VMXWriteVMCS(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, idxMsr);
2253 AssertRC(rc);
2254#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
2255
2256 bool fOffsettedTsc;
2257 if (pVM->hm.s.vmx.fUsePreemptTimer)
2258 {
2259 uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVCpu, &fOffsettedTsc, &pVCpu->hm.s.vmx.u64TSCOffset);
2260
2261 /* Make sure the returned values have sane upper and lower boundaries. */
2262 uint64_t u64CpuHz = SUPGetCpuHzFromGIP(g_pSUPGlobalInfoPage);
2263
2264 cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64 of a second */
2265 cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
2266
2267 cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
2268 uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
2269 rc = VMXWriteVMCS(VMX_VMCS32_GUEST_PREEMPTION_TIMER_VALUE, cPreemptionTickCount);
2270 AssertRC(rc);
2271 }
2272 else
2273 fOffsettedTsc = TMCpuTickCanUseRealTSC(pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset);
2274
2275 if (fOffsettedTsc)
2276 {
2277 uint64_t u64CurTSC = ASMReadTSC();
2278 if (u64CurTSC + pVCpu->hm.s.vmx.u64TSCOffset >= TMCpuTickGetLastSeen(pVCpu))
2279 {
2280 /* Note: VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT takes precedence over TSC_OFFSET, applies to RDTSCP too. */
2281 rc = VMXWriteVMCS64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, pVCpu->hm.s.vmx.u64TSCOffset);
2282 AssertRC(rc);
2283
2284 pVCpu->hm.s.vmx.proc_ctls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT;
2285 rc = VMXWriteVMCS(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.proc_ctls);
2286 AssertRC(rc);
2287 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
2288 }
2289 else
2290 {
2291 /* Fall back to rdtsc, rdtscp emulation as we would otherwise pass decreasing tsc values to the guest. */
2292 LogFlow(("TSC %RX64 offset %RX64 time=%RX64 last=%RX64 (diff=%RX64, virt_tsc=%RX64)\n", u64CurTSC,
2293 pVCpu->hm.s.vmx.u64TSCOffset, u64CurTSC + pVCpu->hm.s.vmx.u64TSCOffset,
2294 TMCpuTickGetLastSeen(pVCpu), TMCpuTickGetLastSeen(pVCpu) - u64CurTSC - pVCpu->hm.s.vmx.u64TSCOffset,
2295 TMCpuTickGet(pVCpu)));
2296 pVCpu->hm.s.vmx.proc_ctls |= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT;
2297 rc = VMXWriteVMCS(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.proc_ctls);
2298 AssertRC(rc);
2299 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscInterceptOverFlow);
2300 }
2301 }
2302 else
2303 {
2304 pVCpu->hm.s.vmx.proc_ctls |= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT;
2305 rc = VMXWriteVMCS(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.proc_ctls);
2306 AssertRC(rc);
2307 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
2308 }
2309
2310 /* Done with the major changes */
2311 pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_ALL_GUEST;
2312
2313 /* Minimal guest state update (ESP, EIP, EFLAGS mostly) */
2314 VMXR0LoadMinimalGuestState(pVM, pVCpu, pCtx);
2315 return rc;
2316}
2317
2318
2319/**
2320 * Syncs back the guest state from VMCS.
2321 *
2322 * @returns VBox status code.
2323 * @param pVM Pointer to the VM.
2324 * @param pVCpu Pointer to the VMCPU.
2325 * @param pCtx Pointer to the guest CPU context.
2326 */
2327DECLINLINE(int) VMXR0SaveGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
2328{
2329 RTGCUINTREG val, valShadow;
2330 RTGCUINTPTR uInterruptState;
2331 int rc;
2332
2333 /* First sync back EIP, ESP, and EFLAGS. */
2334 rc = VMXReadCachedVMCS(VMX_VMCS_GUEST_RIP, &val);
2335 AssertRC(rc);
2336 pCtx->rip = val;
2337 rc = VMXReadCachedVMCS(VMX_VMCS_GUEST_RSP, &val);
2338 AssertRC(rc);
2339 pCtx->rsp = val;
2340 rc = VMXReadCachedVMCS(VMX_VMCS_GUEST_RFLAGS, &val);
2341 AssertRC(rc);
2342 pCtx->eflags.u32 = val;
2343
2344 /* Take care of instruction fusing (sti, mov ss) */
2345 rc |= VMXReadCachedVMCS(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &val);
2346 uInterruptState = val;
2347 if (uInterruptState != 0)
2348 {
2349 Assert(uInterruptState <= 2); /* only sti & mov ss */
2350 Log(("uInterruptState %x eip=%RGv\n", (uint32_t)uInterruptState, pCtx->rip));
2351 EMSetInhibitInterruptsPC(pVCpu, pCtx->rip);
2352 }
2353 else
2354 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
2355
2356 /* Control registers. */
2357 VMXReadCachedVMCS(VMX_VMCS_CTRL_CR0_READ_SHADOW, &valShadow);
2358 VMXReadCachedVMCS(VMX_VMCS_GUEST_CR0, &val);
2359 val = (valShadow & pVCpu->hm.s.vmx.cr0_mask) | (val & ~pVCpu->hm.s.vmx.cr0_mask);
2360 CPUMSetGuestCR0(pVCpu, val);
2361
2362 VMXReadCachedVMCS(VMX_VMCS_CTRL_CR4_READ_SHADOW, &valShadow);
2363 VMXReadCachedVMCS(VMX_VMCS_GUEST_CR4, &val);
2364 val = (valShadow & pVCpu->hm.s.vmx.cr4_mask) | (val & ~pVCpu->hm.s.vmx.cr4_mask);
2365 CPUMSetGuestCR4(pVCpu, val);
2366
2367 /*
2368 * No reason to sync back the CRx registers. They can't be changed by the guest unless in
2369 * the nested paging case where CR3 & CR4 can be changed by the guest.
2370 */
2371 if ( pVM->hm.s.fNestedPaging
2372 && CPUMIsGuestInPagedProtectedModeEx(pCtx)) /** @todo check if we will always catch mode switches and such... */
2373 {
2374 PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
2375
2376 /* Can be updated behind our back in the nested paging case. */
2377 CPUMSetGuestCR2(pVCpu, pCache->cr2);
2378
2379 VMXReadCachedVMCS(VMX_VMCS_GUEST_CR3, &val);
2380
2381 if (val != pCtx->cr3)
2382 {
2383 CPUMSetGuestCR3(pVCpu, val);
2384 PGMUpdateCR3(pVCpu, val);
2385 }
2386 rc = hmR0VmxSavePaePdpes(pVCpu, pCtx);
2387 AssertRCReturn(rc, rc);
2388 }
2389
2390 /* Sync back DR7. */
2391 VMXReadCachedVMCS(VMX_VMCS_GUEST_DR7, &val);
2392 pCtx->dr[7] = val;
2393
2394 /* Guest CPU context: ES, CS, SS, DS, FS, GS. */
2395 VMX_READ_SELREG(ES, es);
2396 VMX_READ_SELREG(SS, ss);
2397 VMX_READ_SELREG(CS, cs);
2398 VMX_READ_SELREG(DS, ds);
2399 VMX_READ_SELREG(FS, fs);
2400 VMX_READ_SELREG(GS, gs);
2401
2402 /* System MSRs */
2403 VMXReadCachedVMCS(VMX_VMCS32_GUEST_SYSENTER_CS, &val);
2404 pCtx->SysEnter.cs = val;
2405 VMXReadCachedVMCS(VMX_VMCS_GUEST_SYSENTER_EIP, &val);
2406 pCtx->SysEnter.eip = val;
2407 VMXReadCachedVMCS(VMX_VMCS_GUEST_SYSENTER_ESP, &val);
2408 pCtx->SysEnter.esp = val;
2409
2410 /* Misc. registers; must sync everything otherwise we can get out of sync when jumping to ring 3. */
2411 VMX_READ_SELREG(LDTR, ldtr);
2412
2413 VMXReadCachedVMCS(VMX_VMCS32_GUEST_GDTR_LIMIT, &val);
2414 pCtx->gdtr.cbGdt = val;
2415 VMXReadCachedVMCS(VMX_VMCS_GUEST_GDTR_BASE, &val);
2416 pCtx->gdtr.pGdt = val;
2417
2418 VMXReadCachedVMCS(VMX_VMCS32_GUEST_IDTR_LIMIT, &val);
2419 pCtx->idtr.cbIdt = val;
2420 VMXReadCachedVMCS(VMX_VMCS_GUEST_IDTR_BASE, &val);
2421 pCtx->idtr.pIdt = val;
2422
2423 /* Real mode emulation using v86 mode. */
2424 if ( CPUMIsGuestInRealModeEx(pCtx)
2425 && pVM->hm.s.vmx.pRealModeTSS)
2426 {
2427 /* Hide our emulation flags */
2428 pCtx->eflags.Bits.u1VM = 0;
2429
2430 /* Restore original IOPL setting as we always use 0. */
2431 pCtx->eflags.Bits.u2IOPL = pVCpu->hm.s.vmx.RealMode.eflags.Bits.u2IOPL;
2432
2433 /* Force a TR resync every time in case we switch modes. */
2434 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_TR;
2435 }
2436 else
2437 {
2438 /* In real mode we have a fake TSS, so only sync it back when it's supposed to be valid. */
2439 VMX_READ_SELREG(TR, tr);
2440 }
2441
2442#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
2443 /*
2444 * Save the possibly changed MSRs that we automatically restore and save during a world switch.
2445 */
2446 for (unsigned i = 0; i < pVCpu->hm.s.vmx.cCachedMsrs; i++)
2447 {
2448 PVMXMSR pMsr = (PVMXMSR)pVCpu->hm.s.vmx.pvGuestMsr;
2449 pMsr += i;
2450
2451 switch (pMsr->u32IndexMSR)
2452 {
2453 case MSR_K8_LSTAR:
2454 pCtx->msrLSTAR = pMsr->u64Value;
2455 break;
2456 case MSR_K6_STAR:
2457 pCtx->msrSTAR = pMsr->u64Value;
2458 break;
2459 case MSR_K8_SF_MASK:
2460 pCtx->msrSFMASK = pMsr->u64Value;
2461 break;
2462 /* The KERNEL_GS_BASE MSR doesn't work reliably with auto load/store. See @bugref{6208} */
2463#if 0
2464 case MSR_K8_KERNEL_GS_BASE:
2465 pCtx->msrKERNELGSBASE = pMsr->u64Value;
2466 break;
2467#endif
2468 case MSR_K8_TSC_AUX:
2469 CPUMSetGuestMsr(pVCpu, MSR_K8_TSC_AUX, pMsr->u64Value);
2470 break;
2471#if 0
2472 case MSR_K6_EFER:
2473 /* EFER can't be changed without causing a VM-exit. */
2474 /* Assert(pCtx->msrEFER == pMsr->u64Value); */
2475 break;
2476#endif
2477 default:
2478 AssertFailed();
2479 return VERR_HM_UNEXPECTED_LD_ST_MSR;
2480 }
2481 }
2482#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
2483 return VINF_SUCCESS;
2484}
2485
2486
2487/**
2488 * Dummy placeholder for TLB flush handling before VM-entry. Used in the case
2489 * where neither EPT nor VPID is supported by the CPU.
2490 *
2491 * @param pVM Pointer to the VM.
2492 * @param pVCpu Pointer to the VMCPU.
2493 */
2494static DECLCALLBACK(void) hmR0VmxSetupTLBDummy(PVM pVM, PVMCPU pVCpu)
2495{
2496 NOREF(pVM);
2497 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
2498 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
2499 pVCpu->hm.s.TlbShootdown.cPages = 0;
2500 return;
2501}
2502
2503
2504/**
2505 * Setup the tagged TLB for EPT+VPID.
2506 *
2507 * @param pVM Pointer to the VM.
2508 * @param pVCpu Pointer to the VMCPU.
2509 */
2510static DECLCALLBACK(void) hmR0VmxSetupTLBBoth(PVM pVM, PVMCPU pVCpu)
2511{
2512 PHMGLOBLCPUINFO pCpu;
2513
2514 Assert(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid);
2515
2516 pCpu = HMR0GetCurrentCpu();
2517
2518 /*
2519 * Force a TLB flush for the first world switch if the current CPU differs from the one we ran on last
2520 * This can happen both for start & resume due to long jumps back to ring-3.
2521 * If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
2522 * or the host Cpu is online after a suspend/resume, so we cannot reuse the current ASID anymore.
2523 */
2524 bool fNewAsid = false;
2525 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2526 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2527 {
2528 pVCpu->hm.s.fForceTLBFlush = true;
2529 fNewAsid = true;
2530 }
2531
2532 /*
2533 * Check for explicit TLB shootdowns.
2534 */
2535 if (VMCPU_FF_TESTANDCLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2536 pVCpu->hm.s.fForceTLBFlush = true;
2537
2538 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2539
2540 if (pVCpu->hm.s.fForceTLBFlush)
2541 {
2542 if (fNewAsid)
2543 {
2544 ++pCpu->uCurrentAsid;
2545 if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
2546 {
2547 pCpu->uCurrentAsid = 1; /* start at 1; host uses 0 */
2548 pCpu->cTlbFlushes++;
2549 pCpu->fFlushAsidBeforeUse = true;
2550 }
2551
2552 pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
2553 if (pCpu->fFlushAsidBeforeUse)
2554 hmR0VmxFlushVPID(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
2555 }
2556 else
2557 {
2558 if (pVM->hm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVVPID_CAPS_SINGLE_CONTEXT)
2559 hmR0VmxFlushVPID(pVM, pVCpu, VMX_FLUSH_VPID_SINGLE_CONTEXT, 0 /* GCPtr */);
2560 else
2561 hmR0VmxFlushEPT(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
2562 }
2563
2564 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2565 pVCpu->hm.s.fForceTLBFlush = false;
2566 }
2567 else
2568 {
2569 AssertMsg(pVCpu->hm.s.uCurrentAsid && pCpu->uCurrentAsid,
2570 ("hm->uCurrentAsid=%lu hm->cTlbFlushes=%lu cpu->uCurrentAsid=%lu cpu->cTlbFlushes=%lu\n",
2571 pVCpu->hm.s.uCurrentAsid, pVCpu->hm.s.cTlbFlushes,
2572 pCpu->uCurrentAsid, pCpu->cTlbFlushes));
2573
2574 /** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
2575 * not be executed. See hmQueueInvlPage() where it is commented
2576 * out. Support individual entry flushing someday. */
2577 if (VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
2578 {
2579 STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
2580
2581 /*
2582 * Flush individual guest entries using VPID from the TLB or as little as possible with EPT
2583 * as supported by the CPU.
2584 */
2585 if (pVM->hm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVVPID_CAPS_INDIV_ADDR)
2586 {
2587 for (unsigned i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
2588 hmR0VmxFlushVPID(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
2589 }
2590 else
2591 hmR0VmxFlushEPT(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
2592 }
2593 else
2594 STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
2595 }
2596
2597 pVCpu->hm.s.TlbShootdown.cPages = 0;
2598 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
2599
2600 AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
2601 ("Flush count mismatch for cpu %d (%x vs %x)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
2602 AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2603 ("cpu%d uCurrentAsid = %x\n", pCpu->idCpu, pCpu->uCurrentAsid));
2604 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2605 ("cpu%d VM uCurrentAsid = %x\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2606
2607 /* Update VMCS with the VPID. */
2608 int rc = VMXWriteVMCS(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
2609 AssertRC(rc);
2610}
2611
2612
2613/**
2614 * Setup the tagged TLB for EPT only.
2615 *
2616 * @returns VBox status code.
2617 * @param pVM Pointer to the VM.
2618 * @param pVCpu Pointer to the VMCPU.
2619 */
2620static DECLCALLBACK(void) hmR0VmxSetupTLBEPT(PVM pVM, PVMCPU pVCpu)
2621{
2622 PHMGLOBLCPUINFO pCpu;
2623
2624 Assert(pVM->hm.s.fNestedPaging);
2625 Assert(!pVM->hm.s.vmx.fVpid);
2626
2627 pCpu = HMR0GetCurrentCpu();
2628
2629 /*
2630 * Force a TLB flush for the first world switch if the current CPU differs from the one we ran on last
2631 * This can happen both for start & resume due to long jumps back to ring-3.
2632 * A change in the TLB flush count implies the host Cpu is online after a suspend/resume.
2633 */
2634 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2635 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2636 {
2637 pVCpu->hm.s.fForceTLBFlush = true;
2638 }
2639
2640 /*
2641 * Check for explicit TLB shootdown flushes.
2642 */
2643 if (VMCPU_FF_TESTANDCLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2644 pVCpu->hm.s.fForceTLBFlush = true;
2645
2646 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2647 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2648
2649 if (pVCpu->hm.s.fForceTLBFlush)
2650 hmR0VmxFlushEPT(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
2651 else
2652 {
2653 /** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
2654 * not be executed. See hmQueueInvlPage() where it is commented
2655 * out. Support individual entry flushing someday. */
2656 if (VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
2657 {
2658 /*
2659 * We cannot flush individual entries without VPID support. Flush using EPT.
2660 */
2661 STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
2662 hmR0VmxFlushEPT(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
2663 }
2664 }
2665 pVCpu->hm.s.TlbShootdown.cPages= 0;
2666 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
2667
2668#ifdef VBOX_WITH_STATISTICS
2669 /** @todo r=ramshankar: this is not accurate anymore with the VPID+EPT
2670 * handling. Should be fixed later. */
2671 if (pVCpu->hm.s.fForceTLBFlush)
2672 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2673 else
2674 STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
2675#endif
2676}
2677
2678
2679/**
2680 * Setup the tagged TLB for VPID.
2681 *
2682 * @returns VBox status code.
2683 * @param pVM Pointer to the VM.
2684 * @param pVCpu Pointer to the VMCPU.
2685 */
2686static DECLCALLBACK(void) hmR0VmxSetupTLBVPID(PVM pVM, PVMCPU pVCpu)
2687{
2688 PHMGLOBLCPUINFO pCpu;
2689
2690 Assert(pVM->hm.s.vmx.fVpid);
2691 Assert(!pVM->hm.s.fNestedPaging);
2692
2693 pCpu = HMR0GetCurrentCpu();
2694
2695 /*
2696 * Force a TLB flush for the first world switch if the current CPU differs from the one we ran on last
2697 * This can happen both for start & resume due to long jumps back to ring-3.
2698 * If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
2699 * or the host Cpu is online after a suspend/resume, so we cannot reuse the current ASID anymore.
2700 */
2701 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2702 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2703 {
2704 /* Force a TLB flush on VM entry. */
2705 pVCpu->hm.s.fForceTLBFlush = true;
2706 }
2707
2708 /*
2709 * Check for explicit TLB shootdown flushes.
2710 */
2711 if (VMCPU_FF_TESTANDCLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2712 pVCpu->hm.s.fForceTLBFlush = true;
2713
2714 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2715
2716 if (pVCpu->hm.s.fForceTLBFlush)
2717 {
2718 ++pCpu->uCurrentAsid;
2719 if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
2720 {
2721 pCpu->uCurrentAsid = 1; /* start at 1; host uses 0 */
2722 pCpu->cTlbFlushes++;
2723 pCpu->fFlushAsidBeforeUse = true;
2724 }
2725
2726 pVCpu->hm.s.fForceTLBFlush = false;
2727 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2728 pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
2729 if (pCpu->fFlushAsidBeforeUse)
2730 hmR0VmxFlushVPID(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
2731 }
2732 else
2733 {
2734 AssertMsg(pVCpu->hm.s.uCurrentAsid && pCpu->uCurrentAsid,
2735 ("hm->uCurrentAsid=%lu hm->cTlbFlushes=%lu cpu->uCurrentAsid=%lu cpu->cTlbFlushes=%lu\n",
2736 pVCpu->hm.s.uCurrentAsid, pVCpu->hm.s.cTlbFlushes,
2737 pCpu->uCurrentAsid, pCpu->cTlbFlushes));
2738
2739 /** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
2740 * not be executed. See hmQueueInvlPage() where it is commented
2741 * out. Support individual entry flushing someday. */
2742 if (VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
2743 {
2744 /*
2745 * Flush individual guest entries using VPID from the TLB or as little as possible with EPT
2746 * as supported by the CPU.
2747 */
2748 if (pVM->hm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVVPID_CAPS_INDIV_ADDR)
2749 {
2750 for (unsigned i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
2751 hmR0VmxFlushVPID(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
2752 }
2753 else
2754 hmR0VmxFlushVPID(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
2755 }
2756 }
2757 pVCpu->hm.s.TlbShootdown.cPages = 0;
2758 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
2759
2760 AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
2761 ("Flush count mismatch for cpu %d (%x vs %x)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
2762 AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2763 ("cpu%d uCurrentAsid = %x\n", pCpu->idCpu, pCpu->uCurrentAsid));
2764 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2765 ("cpu%d VM uCurrentAsid = %x\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2766
2767 int rc = VMXWriteVMCS(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
2768 AssertRC(rc);
2769
2770# ifdef VBOX_WITH_STATISTICS
2771 /** @todo r=ramshankar: this is not accurate anymore with EPT+VPID handling.
2772 * Should be fixed later. */
2773 if (pVCpu->hm.s.fForceTLBFlush)
2774 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2775 else
2776 STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
2777# endif
2778}
2779
2780
2781/**
2782 * Runs guest code in a VT-x VM.
2783 *
2784 * @returns VBox status code.
2785 * @param pVM Pointer to the VM.
2786 * @param pVCpu Pointer to the VMCPU.
2787 * @param pCtx Pointer to the guest CPU context.
2788 */
2789VMMR0DECL(int) VMXR0RunGuestCode(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
2790{
2791 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
2792 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit1);
2793 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit2);
2794
2795 VBOXSTRICTRC rc = VINF_SUCCESS;
2796 int rc2;
2797 RTGCUINTREG val;
2798 RTGCUINTREG exitReason = (RTGCUINTREG)VMX_EXIT_INVALID;
2799 RTGCUINTREG instrError, cbInstr;
2800 RTGCUINTPTR exitQualification = 0;
2801 RTGCUINTPTR intInfo = 0; /* shut up buggy gcc 4 */
2802 RTGCUINTPTR errCode, instrInfo;
2803 bool fSetupTPRCaching = false;
2804 uint64_t u64OldLSTAR = 0;
2805 uint8_t u8LastTPR = 0;
2806 RTCCUINTREG uOldEFlags = ~(RTCCUINTREG)0;
2807 unsigned cResume = 0;
2808#ifdef VBOX_STRICT
2809 RTCPUID idCpuCheck;
2810 bool fWasInLongMode = false;
2811#endif
2812#ifdef VBOX_HIGH_RES_TIMERS_HACK_IN_RING0
2813 uint64_t u64LastTime = RTTimeMilliTS();
2814#endif
2815
2816 Assert(!(pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
2817 || (pVCpu->hm.s.vmx.pbVAPIC && pVM->hm.s.vmx.pbApicAccess));
2818
2819 /*
2820 * Check if we need to use TPR shadowing.
2821 */
2822 if ( CPUMIsGuestInLongModeEx(pCtx)
2823 || ( (( pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
2824 || pVM->hm.s.fTRPPatchingAllowed)
2825 && pVM->hm.s.fHasIoApic)
2826 )
2827 {
2828 fSetupTPRCaching = true;
2829 }
2830
2831 Log2(("\nE"));
2832
2833 /* This is not ideal, but if we don't clear the event injection in the VMCS right here,
2834 * we may end up injecting some stale event into a VM, including injecting an event that
2835 * originated before a VM reset *after* the VM has been reset. See @bugref{6220}.
2836 */
2837 VMXWriteVMCS(VMX_VMCS32_CTRL_ENTRY_IRQ_INFO, 0);
2838
2839#ifdef VBOX_STRICT
2840 {
2841 RTCCUINTREG val2;
2842
2843 rc2 = VMXReadVMCS(VMX_VMCS32_CTRL_PIN_EXEC_CONTROLS, &val2);
2844 AssertRC(rc2);
2845 Log2(("VMX_VMCS_CTRL_PIN_EXEC_CONTROLS = %08x\n", val2));
2846
2847 /* allowed zero */
2848 if ((val2 & pVM->hm.s.vmx.msr.vmx_pin_ctls.n.disallowed0) != pVM->hm.s.vmx.msr.vmx_pin_ctls.n.disallowed0)
2849 Log(("Invalid VMX_VMCS_CTRL_PIN_EXEC_CONTROLS: zero\n"));
2850
2851 /* allowed one */
2852 if ((val2 & ~pVM->hm.s.vmx.msr.vmx_pin_ctls.n.allowed1) != 0)
2853 Log(("Invalid VMX_VMCS_CTRL_PIN_EXEC_CONTROLS: one\n"));
2854
2855 rc2 = VMXReadVMCS(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, &val2);
2856 AssertRC(rc2);
2857 Log2(("VMX_VMCS_CTRL_PROC_EXEC_CONTROLS = %08x\n", val2));
2858
2859 /*
2860 * Must be set according to the MSR, but can be cleared if nested paging is used.
2861 */
2862 if (pVM->hm.s.fNestedPaging)
2863 {
2864 val2 |= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INVLPG_EXIT
2865 | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
2866 | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT;
2867 }
2868
2869 /* allowed zero */
2870 if ((val2 & pVM->hm.s.vmx.msr.vmx_proc_ctls.n.disallowed0) != pVM->hm.s.vmx.msr.vmx_proc_ctls.n.disallowed0)
2871 Log(("Invalid VMX_VMCS_CTRL_PROC_EXEC_CONTROLS: zero\n"));
2872
2873 /* allowed one */
2874 if ((val2 & ~pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1) != 0)
2875 Log(("Invalid VMX_VMCS_CTRL_PROC_EXEC_CONTROLS: one\n"));
2876
2877 rc2 = VMXReadVMCS(VMX_VMCS32_CTRL_ENTRY_CONTROLS, &val2);
2878 AssertRC(rc2);
2879 Log2(("VMX_VMCS_CTRL_ENTRY_CONTROLS = %08x\n", val2));
2880
2881 /* allowed zero */
2882 if ((val2 & pVM->hm.s.vmx.msr.vmx_entry.n.disallowed0) != pVM->hm.s.vmx.msr.vmx_entry.n.disallowed0)
2883 Log(("Invalid VMX_VMCS_CTRL_ENTRY_CONTROLS: zero\n"));
2884
2885 /* allowed one */
2886 if ((val2 & ~pVM->hm.s.vmx.msr.vmx_entry.n.allowed1) != 0)
2887 Log(("Invalid VMX_VMCS_CTRL_ENTRY_CONTROLS: one\n"));
2888
2889 rc2 = VMXReadVMCS(VMX_VMCS32_CTRL_EXIT_CONTROLS, &val2);
2890 AssertRC(rc2);
2891 Log2(("VMX_VMCS_CTRL_EXIT_CONTROLS = %08x\n", val2));
2892
2893 /* allowed zero */
2894 if ((val2 & pVM->hm.s.vmx.msr.vmx_exit.n.disallowed0) != pVM->hm.s.vmx.msr.vmx_exit.n.disallowed0)
2895 Log(("Invalid VMX_VMCS_CTRL_EXIT_CONTROLS: zero\n"));
2896
2897 /* allowed one */
2898 if ((val2 & ~pVM->hm.s.vmx.msr.vmx_exit.n.allowed1) != 0)
2899 Log(("Invalid VMX_VMCS_CTRL_EXIT_CONTROLS: one\n"));
2900 }
2901 fWasInLongMode = CPUMIsGuestInLongModeEx(pCtx);
2902#endif /* VBOX_STRICT */
2903
2904#ifdef VBOX_WITH_CRASHDUMP_MAGIC
2905 pVCpu->hm.s.vmx.VMCSCache.u64TimeEntry = RTTimeNanoTS();
2906#endif
2907
2908 /*
2909 * We can jump to this point to resume execution after determining that a VM-exit is innocent.
2910 */
2911ResumeExecution:
2912 if (!STAM_REL_PROFILE_ADV_IS_RUNNING(&pVCpu->hm.s.StatEntry))
2913 STAM_REL_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit2, &pVCpu->hm.s.StatEntry, x);
2914 AssertMsg(pVCpu->hm.s.idEnteredCpu == RTMpCpuId(),
2915 ("Expected %d, I'm %d; cResume=%d exitReason=%RGv exitQualification=%RGv\n",
2916 (int)pVCpu->hm.s.idEnteredCpu, (int)RTMpCpuId(), cResume, exitReason, exitQualification));
2917 Assert(!HMR0SuspendPending());
2918 /* Not allowed to switch modes without reloading the host state (32->64 switcher)!! */
2919 Assert(fWasInLongMode == CPUMIsGuestInLongModeEx(pCtx));
2920
2921 /*
2922 * Safety precaution; looping for too long here can have a very bad effect on the host.
2923 */
2924 if (RT_UNLIKELY(++cResume > pVM->hm.s.cMaxResumeLoops))
2925 {
2926 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMaxResume);
2927 rc = VINF_EM_RAW_INTERRUPT;
2928 goto end;
2929 }
2930
2931 /*
2932 * Check for IRQ inhibition due to instruction fusing (sti, mov ss).
2933 */
2934 if (VMCPU_FF_ISSET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
2935 {
2936 Log(("VM_FF_INHIBIT_INTERRUPTS at %RGv successor %RGv\n", (RTGCPTR)pCtx->rip, EMGetInhibitInterruptsPC(pVCpu)));
2937 if (pCtx->rip != EMGetInhibitInterruptsPC(pVCpu))
2938 {
2939 /*
2940 * Note: we intentionally don't clear VM_FF_INHIBIT_INTERRUPTS here.
2941 * Before we are able to execute this instruction in raw mode (iret to guest code) an external interrupt might
2942 * force a world switch again. Possibly allowing a guest interrupt to be dispatched in the process. This could
2943 * break the guest. Sounds very unlikely, but such timing sensitive problems are not as rare as you might think.
2944 */
2945 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
2946 /* Irq inhibition is no longer active; clear the corresponding VMX state. */
2947 rc2 = VMXWriteVMCS(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, 0);
2948 AssertRC(rc2);
2949 }
2950 }
2951 else
2952 {
2953 /* Irq inhibition is no longer active; clear the corresponding VMX state. */
2954 rc2 = VMXWriteVMCS(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, 0);
2955 AssertRC(rc2);
2956 }
2957
2958#ifdef VBOX_HIGH_RES_TIMERS_HACK_IN_RING0
2959 if (RT_UNLIKELY((cResume & 0xf) == 0))
2960 {
2961 uint64_t u64CurTime = RTTimeMilliTS();
2962
2963 if (RT_UNLIKELY(u64CurTime > u64LastTime))
2964 {
2965 u64LastTime = u64CurTime;
2966 TMTimerPollVoid(pVM, pVCpu);
2967 }
2968 }
2969#endif
2970
2971 /*
2972 * Check for pending actions that force us to go back to ring-3.
2973 */
2974 if ( VM_FF_ISPENDING(pVM, VM_FF_HM_TO_R3_MASK | VM_FF_REQUEST | VM_FF_PGM_POOL_FLUSH_PENDING | VM_FF_PDM_DMA)
2975 || VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK | VMCPU_FF_PGM_SYNC_CR3 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL | VMCPU_FF_REQUEST))
2976 {
2977 /* Check if a sync operation is pending. */
2978 if (VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
2979 {
2980 rc = PGMSyncCR3(pVCpu, pCtx->cr0, pCtx->cr3, pCtx->cr4, VMCPU_FF_ISSET(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
2981 if (rc != VINF_SUCCESS)
2982 {
2983 AssertRC(VBOXSTRICTRC_VAL(rc));
2984 Log(("Pending pool sync is forcing us back to ring 3; rc=%d\n", VBOXSTRICTRC_VAL(rc)));
2985 goto end;
2986 }
2987 }
2988
2989#ifdef DEBUG
2990 /* Intercept X86_XCPT_DB if stepping is enabled */
2991 if (!DBGFIsStepping(pVCpu))
2992#endif
2993 {
2994 if ( VM_FF_ISPENDING(pVM, VM_FF_HM_TO_R3_MASK)
2995 || VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
2996 {
2997 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchToR3);
2998 rc = RT_UNLIKELY(VM_FF_ISPENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
2999 goto end;
3000 }
3001 }
3002
3003 /* Pending request packets might contain actions that need immediate attention, such as pending hardware interrupts. */
3004 if ( VM_FF_ISPENDING(pVM, VM_FF_REQUEST)
3005 || VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_REQUEST))
3006 {
3007 rc = VINF_EM_PENDING_REQUEST;
3008 goto end;
3009 }
3010
3011 /* Check if a pgm pool flush is in progress. */
3012 if (VM_FF_ISPENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
3013 {
3014 rc = VINF_PGM_POOL_FLUSH_PENDING;
3015 goto end;
3016 }
3017
3018 /* Check if DMA work is pending (2nd+ run). */
3019 if (VM_FF_ISPENDING(pVM, VM_FF_PDM_DMA) && cResume > 1)
3020 {
3021 rc = VINF_EM_RAW_TO_R3;
3022 goto end;
3023 }
3024 }
3025
3026#ifdef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
3027 /*
3028 * Exit to ring-3 preemption/work is pending.
3029 *
3030 * Interrupts are disabled before the call to make sure we don't miss any interrupt
3031 * that would flag preemption (IPI, timer tick, ++). (Would've been nice to do this
3032 * further down, but hmR0VmxCheckPendingInterrupt makes that impossible.)
3033 *
3034 * Note! Interrupts must be disabled done *before* we check for TLB flushes; TLB
3035 * shootdowns rely on this.
3036 */
3037 uOldEFlags = ASMIntDisableFlags();
3038 if (RTThreadPreemptIsPending(NIL_RTTHREAD))
3039 {
3040 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptPending);
3041 rc = VINF_EM_RAW_INTERRUPT;
3042 goto end;
3043 }
3044 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
3045#endif
3046
3047 /*
3048 * When external interrupts are pending, we should exit the VM when IF is set.
3049 * Note: *After* VM_FF_INHIBIT_INTERRUPTS check!
3050 */
3051 rc = hmR0VmxCheckPendingInterrupt(pVM, pVCpu, pCtx);
3052 if (RT_FAILURE(rc))
3053 goto end;
3054
3055 /** @todo check timers?? */
3056
3057 /*
3058 * TPR caching using CR8 is only available in 64-bit mode.
3059 * Note: The 32-bit exception for AMD (X86_CPUID_AMD_FEATURE_ECX_CR8L), but this appears missing in Intel CPUs.
3060 * Note: We can't do this in LoadGuestState() as PDMApicGetTPR can jump back to ring-3 (lock)!! (no longer true) .
3061 */
3062 /** @todo query and update the TPR only when it could have been changed (mmio
3063 * access & wrsmr (x2apic) */
3064 if (fSetupTPRCaching)
3065 {
3066 /* TPR caching in CR8 */
3067 bool fPending;
3068
3069 rc2 = PDMApicGetTPR(pVCpu, &u8LastTPR, &fPending);
3070 AssertRC(rc2);
3071 /* The TPR can be found at offset 0x80 in the APIC mmio page. */
3072 pVCpu->hm.s.vmx.pbVAPIC[0x80] = u8LastTPR;
3073
3074 /*
3075 * Two options here:
3076 * - external interrupt pending, but masked by the TPR value.
3077 * -> a CR8 update that lower the current TPR value should cause an exit
3078 * - no pending interrupts
3079 * -> We don't need to be explicitely notified. There are enough world switches for detecting pending interrupts.
3080 */
3081
3082 /* cr8 bits 3-0 correspond to bits 7-4 of the task priority mmio register. */
3083 rc = VMXWriteVMCS(VMX_VMCS32_CTRL_TPR_THRESHOLD, (fPending) ? (u8LastTPR >> 4) : 0);
3084 AssertRC(VBOXSTRICTRC_VAL(rc));
3085
3086 if (pVM->hm.s.fTPRPatchingActive)
3087 {
3088 Assert(!CPUMIsGuestInLongModeEx(pCtx));
3089 /* Our patch code uses LSTAR for TPR caching. */
3090 pCtx->msrLSTAR = u8LastTPR;
3091
3092 /** @todo r=ramshankar: we should check for MSR-bitmap support here. */
3093 if (fPending)
3094 {
3095 /* A TPR change could activate a pending interrupt, so catch lstar writes. */
3096 hmR0VmxSetMSRPermission(pVCpu, MSR_K8_LSTAR, true, false);
3097 }
3098 else
3099 {
3100 /*
3101 * No interrupts are pending, so we don't need to be explicitely notified.
3102 * There are enough world switches for detecting pending interrupts.
3103 */
3104 hmR0VmxSetMSRPermission(pVCpu, MSR_K8_LSTAR, true, true);
3105 }
3106 }
3107 }
3108
3109#ifdef LOG_ENABLED
3110 if ( pVM->hm.s.fNestedPaging
3111 || pVM->hm.s.vmx.fVpid)
3112 {
3113 PHMGLOBLCPUINFO pCpu = HMR0GetCurrentCpu();
3114 if (pVCpu->hm.s.idLastCpu != pCpu->idCpu)
3115 {
3116 LogFlow(("Force TLB flush due to rescheduling to a different cpu (%d vs %d)\n", pVCpu->hm.s.idLastCpu,
3117 pCpu->idCpu));
3118 }
3119 else if (pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
3120 {
3121 LogFlow(("Force TLB flush due to changed TLB flush count (%x vs %x)\n", pVCpu->hm.s.cTlbFlushes,
3122 pCpu->cTlbFlushes));
3123 }
3124 else if (VMCPU_FF_ISSET(pVCpu, VMCPU_FF_TLB_FLUSH))
3125 LogFlow(("Manual TLB flush\n"));
3126 }
3127#endif
3128#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
3129 PGMRZDynMapFlushAutoSet(pVCpu);
3130#endif
3131
3132 /*
3133 * NOTE: DO NOT DO ANYTHING AFTER THIS POINT THAT MIGHT JUMP BACK TO RING-3!
3134 * (until the actual world switch)
3135 */
3136#ifdef VBOX_STRICT
3137 idCpuCheck = RTMpCpuId();
3138#endif
3139#ifdef LOG_ENABLED
3140 VMMR0LogFlushDisable(pVCpu);
3141#endif
3142
3143 /*
3144 * Save the host state first.
3145 */
3146 if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_HOST_CONTEXT)
3147 {
3148 rc = VMXR0SaveHostState(pVM, pVCpu);
3149 if (RT_UNLIKELY(rc != VINF_SUCCESS))
3150 {
3151 VMMR0LogFlushEnable(pVCpu);
3152 goto end;
3153 }
3154 }
3155
3156 /*
3157 * Load the guest state.
3158 */
3159 if (!pVCpu->hm.s.fContextUseFlags)
3160 {
3161 VMXR0LoadMinimalGuestState(pVM, pVCpu, pCtx);
3162 STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadMinimal);
3163 }
3164 else
3165 {
3166 rc = VMXR0LoadGuestState(pVM, pVCpu, pCtx);
3167 if (RT_UNLIKELY(rc != VINF_SUCCESS))
3168 {
3169 VMMR0LogFlushEnable(pVCpu);
3170 goto end;
3171 }
3172 STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadFull);
3173 }
3174
3175#ifndef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
3176 /*
3177 * Disable interrupts to make sure a poke will interrupt execution.
3178 * This must be done *before* we check for TLB flushes; TLB shootdowns rely on this.
3179 */
3180 uOldEFlags = ASMIntDisableFlags();
3181 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
3182#endif
3183
3184 /* Non-register state Guest Context */
3185 /** @todo change me according to cpu state */
3186 rc2 = VMXWriteVMCS(VMX_VMCS32_GUEST_ACTIVITY_STATE, VMX_CMS_GUEST_ACTIVITY_ACTIVE);
3187 AssertRC(rc2);
3188
3189 /* Set TLB flush state as checked until we return from the world switch. */
3190 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true);
3191 /* Deal with tagged TLB setup and invalidation. */
3192 pVM->hm.s.vmx.pfnSetupTaggedTlb(pVM, pVCpu);
3193
3194 /*
3195 * Manual save and restore:
3196 * - General purpose registers except RIP, RSP
3197 *
3198 * Trashed:
3199 * - CR2 (we don't care)
3200 * - LDTR (reset to 0)
3201 * - DRx (presumably not changed at all)
3202 * - DR7 (reset to 0x400)
3203 * - EFLAGS (reset to RT_BIT(1); not relevant)
3204 */
3205
3206 /* All done! Let's start VM execution. */
3207 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
3208 Assert(idCpuCheck == RTMpCpuId());
3209
3210#ifdef VBOX_WITH_CRASHDUMP_MAGIC
3211 pVCpu->hm.s.vmx.VMCSCache.cResume = cResume;
3212 pVCpu->hm.s.vmx.VMCSCache.u64TimeSwitch = RTTimeNanoTS();
3213#endif
3214
3215 /*
3216 * Save the current TPR value in the LSTAR MSR so our patches can access it.
3217 */
3218 if (pVM->hm.s.fTPRPatchingActive)
3219 {
3220 Assert(pVM->hm.s.fTPRPatchingActive);
3221 u64OldLSTAR = ASMRdMsr(MSR_K8_LSTAR);
3222 ASMWrMsr(MSR_K8_LSTAR, u8LastTPR);
3223 }
3224
3225 TMNotifyStartOfExecution(pVCpu);
3226
3227#ifndef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
3228 /*
3229 * Save the current Host TSC_AUX and write the guest TSC_AUX to the host, so that
3230 * RDTSCPs (that don't cause exits) reads the guest MSR. See @bugref{3324}.
3231 */
3232 if ( (pVCpu->hm.s.vmx.proc_ctls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
3233 && !(pVCpu->hm.s.vmx.proc_ctls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT))
3234 {
3235 pVCpu->hm.s.u64HostTSCAux = ASMRdMsr(MSR_K8_TSC_AUX);
3236 uint64_t u64GuestTSCAux = 0;
3237 rc2 = CPUMQueryGuestMsr(pVCpu, MSR_K8_TSC_AUX, &u64GuestTSCAux);
3238 AssertRC(rc2);
3239 ASMWrMsr(MSR_K8_TSC_AUX, u64GuestTSCAux);
3240 }
3241#endif
3242
3243#ifdef VBOX_WITH_KERNEL_USING_XMM
3244 rc = hmR0VMXStartVMWrapXMM(pVCpu->hm.s.fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hm.s.vmx.pfnStartVM);
3245#else
3246 rc = pVCpu->hm.s.vmx.pfnStartVM(pVCpu->hm.s.fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu);
3247#endif
3248 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false);
3249 ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits);
3250
3251 /* Possibly the last TSC value seen by the guest (too high) (only when we're in TSC offset mode). */
3252 if (!(pVCpu->hm.s.vmx.proc_ctls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT))
3253 {
3254#ifndef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
3255 /* Restore host's TSC_AUX. */
3256 if (pVCpu->hm.s.vmx.proc_ctls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
3257 ASMWrMsr(MSR_K8_TSC_AUX, pVCpu->hm.s.u64HostTSCAux);
3258#endif
3259
3260 TMCpuTickSetLastSeen(pVCpu,
3261 ASMReadTSC() + pVCpu->hm.s.vmx.u64TSCOffset - 0x400 /* guestimate of world switch overhead in clock ticks */);
3262 }
3263
3264 TMNotifyEndOfExecution(pVCpu);
3265 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED);
3266 Assert(!(ASMGetFlags() & X86_EFL_IF));
3267
3268 /*
3269 * Restore the host LSTAR MSR if the guest could have changed it.
3270 */
3271 if (pVM->hm.s.fTPRPatchingActive)
3272 {
3273 Assert(pVM->hm.s.fTPRPatchingActive);
3274 pVCpu->hm.s.vmx.pbVAPIC[0x80] = pCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR);
3275 ASMWrMsr(MSR_K8_LSTAR, u64OldLSTAR);
3276 }
3277
3278 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatExit1, x);
3279 ASMSetFlags(uOldEFlags);
3280#ifdef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
3281 uOldEFlags = ~(RTCCUINTREG)0;
3282#endif
3283
3284 AssertMsg(!pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries, ("pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries=%d\n",
3285 pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries));
3286
3287 /* In case we execute a goto ResumeExecution later on. */
3288 pVCpu->hm.s.fResumeVM = true;
3289 pVCpu->hm.s.fForceTLBFlush = false;
3290
3291 /*
3292 * !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
3293 * IMPORTANT: WE CAN'T DO ANY LOGGING OR OPERATIONS THAT CAN DO A LONGJMP BACK TO RING 3 *BEFORE* WE'VE SYNCED BACK (MOST OF) THE GUEST STATE
3294 * !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
3295 */
3296
3297 if (RT_UNLIKELY(rc != VINF_SUCCESS))
3298 {
3299 hmR0VmxReportWorldSwitchError(pVM, pVCpu, rc, pCtx);
3300 VMMR0LogFlushEnable(pVCpu);
3301 goto end;
3302 }
3303
3304 /* Success. Query the guest state and figure out what has happened. */
3305
3306 /* Investigate why there was a VM-exit. */
3307 rc2 = VMXReadCachedVMCS(VMX_VMCS32_RO_EXIT_REASON, &exitReason);
3308 STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[exitReason & MASK_EXITREASON_STAT]);
3309
3310 exitReason &= 0xffff; /* bit 0-15 contain the exit code. */
3311 rc2 |= VMXReadCachedVMCS(VMX_VMCS32_RO_VM_INSTR_ERROR, &instrError);
3312 rc2 |= VMXReadCachedVMCS(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &cbInstr);
3313 rc2 |= VMXReadCachedVMCS(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &intInfo);
3314 /* might not be valid; depends on VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID. */
3315 rc2 |= VMXReadCachedVMCS(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERRCODE, &errCode);
3316 rc2 |= VMXReadCachedVMCS(VMX_VMCS32_RO_EXIT_INSTR_INFO, &instrInfo);
3317 rc2 |= VMXReadCachedVMCS(VMX_VMCS_RO_EXIT_QUALIFICATION, &exitQualification);
3318 AssertRC(rc2);
3319
3320 /*
3321 * Sync back the guest state.
3322 */
3323 rc2 = VMXR0SaveGuestState(pVM, pVCpu, pCtx);
3324 AssertRC(rc2);
3325
3326 /* Note! NOW IT'S SAFE FOR LOGGING! */
3327 VMMR0LogFlushEnable(pVCpu);
3328 Log2(("Raw exit reason %08x\n", exitReason));
3329#if ARCH_BITS == 64 /* for the time being */
3330 VBOXVMM_R0_HMVMX_VMEXIT(pVCpu, pCtx, exitReason);
3331#endif
3332
3333 /*
3334 * Check if an injected event was interrupted prematurely.
3335 */
3336 rc2 = VMXReadCachedVMCS(VMX_VMCS32_RO_IDT_INFO, &val);
3337 AssertRC(rc2);
3338 pVCpu->hm.s.Event.intInfo = VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(val);
3339 if ( VMX_EXIT_INTERRUPTION_INFO_VALID(pVCpu->hm.s.Event.intInfo)
3340 /* Ignore 'int xx' as they'll be restarted anyway. */
3341 && VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hm.s.Event.intInfo) != VMX_EXIT_INTERRUPTION_INFO_TYPE_SW
3342 /* Ignore software exceptions (such as int3) as they'll reoccur when we restart the instruction anyway. */
3343 && VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hm.s.Event.intInfo) != VMX_EXIT_INTERRUPTION_INFO_TYPE_SWEXCPT)
3344 {
3345 Assert(!pVCpu->hm.s.Event.fPending);
3346 pVCpu->hm.s.Event.fPending = true;
3347 /* Error code present? */
3348 if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.intInfo))
3349 {
3350 rc2 = VMXReadCachedVMCS(VMX_VMCS32_RO_IDT_ERRCODE, &val);
3351 AssertRC(rc2);
3352 pVCpu->hm.s.Event.errCode = val;
3353 Log(("Pending inject %RX64 at %RGv exit=%08x intInfo=%08x exitQualification=%RGv pending error=%RX64\n",
3354 pVCpu->hm.s.Event.intInfo, (RTGCPTR)pCtx->rip, exitReason, intInfo, exitQualification, val));
3355 }
3356 else
3357 {
3358 Log(("Pending inject %RX64 at %RGv exit=%08x intInfo=%08x exitQualification=%RGv\n", pVCpu->hm.s.Event.intInfo,
3359 (RTGCPTR)pCtx->rip, exitReason, intInfo, exitQualification));
3360 pVCpu->hm.s.Event.errCode = 0;
3361 }
3362 }
3363#ifdef VBOX_STRICT
3364 else if ( VMX_EXIT_INTERRUPTION_INFO_VALID(pVCpu->hm.s.Event.intInfo)
3365 /* Ignore software exceptions (such as int3) as they're reoccur when we restart the instruction anyway. */
3366 && VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hm.s.Event.intInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_SWEXCPT)
3367 {
3368 Log(("Ignore pending inject %RX64 at %RGv exit=%08x intInfo=%08x exitQualification=%RGv\n",
3369 pVCpu->hm.s.Event.intInfo, (RTGCPTR)pCtx->rip, exitReason, intInfo, exitQualification));
3370 }
3371
3372 if (exitReason == VMX_EXIT_ERR_INVALID_GUEST_STATE)
3373 HMDumpRegs(pVM, pVCpu, pCtx);
3374#endif
3375
3376 Log2(("E%d: New EIP=%x:%RGv\n", (uint32_t)exitReason, pCtx->cs.Sel, (RTGCPTR)pCtx->rip));
3377 Log2(("Exit reason %d, exitQualification %RGv\n", (uint32_t)exitReason, exitQualification));
3378 Log2(("instrInfo=%d instrError=%d instr length=%d\n", (uint32_t)instrInfo, (uint32_t)instrError, (uint32_t)cbInstr));
3379 Log2(("Interruption error code %d\n", (uint32_t)errCode));
3380 Log2(("IntInfo = %08x\n", (uint32_t)intInfo));
3381
3382 /*
3383 * Sync back the TPR if it was changed.
3384 */
3385 if ( fSetupTPRCaching
3386 && u8LastTPR != pVCpu->hm.s.vmx.pbVAPIC[0x80])
3387 {
3388 rc2 = PDMApicSetTPR(pVCpu, pVCpu->hm.s.vmx.pbVAPIC[0x80]);
3389 AssertRC(rc2);
3390 }
3391
3392#ifdef DBGFTRACE_ENABLED /** @todo DTrace later. */
3393 RTTraceBufAddMsgF(pVM->CTX_SUFF(hTraceBuf), "vmexit %08x %016RX64 at %04:%08RX64 %RX64",
3394 exitReason, (uint64_t)exitQualification, pCtx->cs.Sel, pCtx->rip, (uint64_t)intInfo);
3395#endif
3396 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
3397
3398 /* Some cases don't need a complete resync of the guest CPU state; handle them here. */
3399 Assert(rc == VINF_SUCCESS); /* might consider VERR_IPE_UNINITIALIZED_STATUS here later... */
3400 switch (exitReason)
3401 {
3402 case VMX_EXIT_EXCEPTION: /* 0 Exception or non-maskable interrupt (NMI). */
3403 case VMX_EXIT_EXTERNAL_IRQ: /* 1 External interrupt. */
3404 {
3405 uint32_t vector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(intInfo);
3406
3407 if (!VMX_EXIT_INTERRUPTION_INFO_VALID(intInfo))
3408 {
3409 Assert(exitReason == VMX_EXIT_EXTERNAL_IRQ);
3410 /* External interrupt; leave to allow it to be dispatched again. */
3411 rc = VINF_EM_RAW_INTERRUPT;
3412 break;
3413 }
3414 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExit2Sub3, y3);
3415 switch (VMX_EXIT_INTERRUPTION_INFO_TYPE(intInfo))
3416 {
3417 case VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI: /* Non-maskable interrupt. */
3418 /* External interrupt; leave to allow it to be dispatched again. */
3419 rc = VINF_EM_RAW_INTERRUPT;
3420 break;
3421
3422 case VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT: /* External hardware interrupt. */
3423 AssertFailed(); /* can't come here; fails the first check. */
3424 break;
3425
3426 case VMX_EXIT_INTERRUPTION_INFO_TYPE_DBEXCPT: /* Unknown why we get this type for #DB */
3427 case VMX_EXIT_INTERRUPTION_INFO_TYPE_SWEXCPT: /* Software exception. (#BP or #OF) */
3428 Assert(vector == 1 || vector == 3 || vector == 4);
3429 /* no break */
3430 case VMX_EXIT_INTERRUPTION_INFO_TYPE_HWEXCPT: /* Hardware exception. */
3431 Log2(("Hardware/software interrupt %d\n", vector));
3432 switch (vector)
3433 {
3434 case X86_XCPT_NM:
3435 {
3436 Log(("#NM fault at %RGv error code %x\n", (RTGCPTR)pCtx->rip, errCode));
3437
3438 /** @todo don't intercept #NM exceptions anymore when we've activated the guest FPU state. */
3439 /* If we sync the FPU/XMM state on-demand, then we can continue execution as if nothing has happened. */
3440 rc = CPUMR0LoadGuestFPU(pVM, pVCpu, pCtx);
3441 if (rc == VINF_SUCCESS)
3442 {
3443 Assert(CPUMIsGuestFPUStateActive(pVCpu));
3444
3445 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowNM);
3446
3447 /* Continue execution. */
3448 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_CR0;
3449
3450 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2Sub3, y3);
3451 goto ResumeExecution;
3452 }
3453
3454 Log(("Forward #NM fault to the guest\n"));
3455 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
3456 rc2 = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo),
3457 cbInstr, 0);
3458 AssertRC(rc2);
3459 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2Sub3, y3);
3460 goto ResumeExecution;
3461 }
3462
3463 case X86_XCPT_PF: /* Page fault */
3464 {
3465#ifdef VBOX_ALWAYS_TRAP_PF
3466 if (pVM->hm.s.fNestedPaging)
3467 {
3468 /*
3469 * A genuine pagefault. Forward the trap to the guest by injecting the exception and resuming execution.
3470 */
3471 Log(("Guest page fault at %RGv cr2=%RGv error code %RGv rsp=%RGv\n", (RTGCPTR)pCtx->rip, exitQualification,
3472 errCode, (RTGCPTR)pCtx->rsp));
3473
3474 Assert(CPUMIsGuestInPagedProtectedModeEx(pCtx));
3475
3476 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
3477
3478 /* Now we must update CR2. */
3479 pCtx->cr2 = exitQualification;
3480 rc2 = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo),
3481 cbInstr, errCode);
3482 AssertRC(rc2);
3483
3484 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2Sub3, y3);
3485 goto ResumeExecution;
3486 }
3487#else
3488 Assert(!pVM->hm.s.fNestedPaging);
3489#endif
3490
3491#ifdef VBOX_HM_WITH_GUEST_PATCHING
3492 /* Shortcut for APIC TPR reads and writes; 32 bits guests only */
3493 if ( pVM->hm.s.fTRPPatchingAllowed
3494 && pVM->hm.s.pGuestPatchMem
3495 && (exitQualification & 0xfff) == 0x080
3496 && !(errCode & X86_TRAP_PF_P) /* not present */
3497 && CPUMGetGuestCPL(pVCpu) == 0
3498 && !CPUMIsGuestInLongModeEx(pCtx)
3499 && pVM->hm.s.cPatches < RT_ELEMENTS(pVM->hm.s.aPatches))
3500 {
3501 RTGCPHYS GCPhysApicBase, GCPhys;
3502 GCPhysApicBase = pCtx->msrApicBase;
3503 GCPhysApicBase &= PAGE_BASE_GC_MASK;
3504
3505 rc = PGMGstGetPage(pVCpu, (RTGCPTR)exitQualification, NULL, &GCPhys);
3506 if ( rc == VINF_SUCCESS
3507 && GCPhys == GCPhysApicBase)
3508 {
3509 /* Only attempt to patch the instruction once. */
3510 PHMTPRPATCH pPatch = (PHMTPRPATCH)RTAvloU32Get(&pVM->hm.s.PatchTree, (AVLOU32KEY)pCtx->eip);
3511 if (!pPatch)
3512 {
3513 rc = VINF_EM_HM_PATCH_TPR_INSTR;
3514 break;
3515 }
3516 }
3517 }
3518#endif
3519
3520 Log2(("Page fault at %RGv error code %x\n", exitQualification, errCode));
3521 /* Exit qualification contains the linear address of the page fault. */
3522 TRPMAssertTrap(pVCpu, X86_XCPT_PF, TRPM_TRAP);
3523 TRPMSetErrorCode(pVCpu, errCode);
3524 TRPMSetFaultAddress(pVCpu, exitQualification);
3525
3526 /* Shortcut for APIC TPR reads and writes. */
3527 if ( (exitQualification & 0xfff) == 0x080
3528 && !(errCode & X86_TRAP_PF_P) /* not present */
3529 && fSetupTPRCaching
3530 && (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC))
3531 {
3532 RTGCPHYS GCPhysApicBase, GCPhys;
3533 GCPhysApicBase = pCtx->msrApicBase;
3534 GCPhysApicBase &= PAGE_BASE_GC_MASK;
3535
3536 rc = PGMGstGetPage(pVCpu, (RTGCPTR)exitQualification, NULL, &GCPhys);
3537 if ( rc == VINF_SUCCESS
3538 && GCPhys == GCPhysApicBase)
3539 {
3540 Log(("Enable VT-x virtual APIC access filtering\n"));
3541 rc2 = IOMMMIOMapMMIOHCPage(pVM, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW | X86_PTE_P);
3542 AssertRC(rc2);
3543 }
3544 }
3545
3546 /* Forward it to our trap handler first, in case our shadow pages are out of sync. */
3547 rc = PGMTrap0eHandler(pVCpu, errCode, CPUMCTX2CORE(pCtx), (RTGCPTR)exitQualification);
3548 Log2(("PGMTrap0eHandler %RGv returned %Rrc\n", (RTGCPTR)pCtx->rip, VBOXSTRICTRC_VAL(rc)));
3549
3550 if (rc == VINF_SUCCESS)
3551 { /* We've successfully synced our shadow pages, so let's just continue execution. */
3552 Log2(("Shadow page fault at %RGv cr2=%RGv error code %x\n", (RTGCPTR)pCtx->rip, exitQualification ,errCode));
3553 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
3554
3555 TRPMResetTrap(pVCpu);
3556 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2Sub3, y3);
3557 goto ResumeExecution;
3558 }
3559 else if (rc == VINF_EM_RAW_GUEST_TRAP)
3560 {
3561 /*
3562 * A genuine pagefault. Forward the trap to the guest by injecting the exception and resuming execution.
3563 */
3564 Log2(("Forward page fault to the guest\n"));
3565
3566 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
3567 /* The error code might have been changed. */
3568 errCode = TRPMGetErrorCode(pVCpu);
3569
3570 TRPMResetTrap(pVCpu);
3571
3572 /* Now we must update CR2. */
3573 pCtx->cr2 = exitQualification;
3574 rc2 = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo),
3575 cbInstr, errCode);
3576 AssertRC(rc2);
3577
3578 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2Sub3, y3);
3579 goto ResumeExecution;
3580 }
3581#ifdef VBOX_STRICT
3582 if (rc != VINF_EM_RAW_EMULATE_INSTR && rc != VINF_EM_RAW_EMULATE_IO_BLOCK)
3583 Log2(("PGMTrap0eHandler failed with %d\n", VBOXSTRICTRC_VAL(rc)));
3584#endif
3585 /* Need to go back to the recompiler to emulate the instruction. */
3586 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
3587 TRPMResetTrap(pVCpu);
3588
3589 /* If event delivery caused the #PF (shadow or not), tell TRPM. */
3590 hmR0VmxCheckPendingEvent(pVCpu);
3591 break;
3592 }
3593
3594 case X86_XCPT_MF: /* Floating point exception. */
3595 {
3596 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
3597 if (!(pCtx->cr0 & X86_CR0_NE))
3598 {
3599 /* old style FPU error reporting needs some extra work. */
3600 /** @todo don't fall back to the recompiler, but do it manually. */
3601 rc = VINF_EM_RAW_EMULATE_INSTR;
3602 break;
3603 }
3604 Log(("Trap %x at %04X:%RGv\n", vector, pCtx->cs.Sel, (RTGCPTR)pCtx->rip));
3605 rc2 = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo),
3606 cbInstr, errCode);
3607 AssertRC(rc2);
3608
3609 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2Sub3, y3);
3610 goto ResumeExecution;
3611 }
3612
3613 case X86_XCPT_DB: /* Debug exception. */
3614 {
3615 uint64_t uDR6;
3616
3617 /*
3618 * DR6, DR7.GD and IA32_DEBUGCTL.LBR are not updated yet.
3619 *
3620 * Exit qualification bits:
3621 * 3:0 B0-B3 which breakpoint condition was met
3622 * 12:4 Reserved (0)
3623 * 13 BD - debug register access detected
3624 * 14 BS - single step execution or branch taken
3625 * 63:15 Reserved (0)
3626 */
3627 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
3628
3629 /* Note that we don't support guest and host-initiated debugging at the same time. */
3630
3631 uDR6 = X86_DR6_INIT_VAL;
3632 uDR6 |= (exitQualification & (X86_DR6_B0|X86_DR6_B1|X86_DR6_B2|X86_DR6_B3|X86_DR6_BD|X86_DR6_BS));
3633 rc = DBGFRZTrap01Handler(pVM, pVCpu, CPUMCTX2CORE(pCtx), uDR6);
3634 if (rc == VINF_EM_RAW_GUEST_TRAP)
3635 {
3636 /* Update DR6 here. */
3637 pCtx->dr[6] = uDR6;
3638
3639 /* Resync DR6 if the debug state is active. */
3640 if (CPUMIsGuestDebugStateActive(pVCpu))
3641 ASMSetDR6(pCtx->dr[6]);
3642
3643 /* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
3644 pCtx->dr[7] &= ~X86_DR7_GD;
3645
3646 /* Paranoia. */
3647 pCtx->dr[7] &= 0xffffffff; /* upper 32 bits reserved */
3648 pCtx->dr[7] &= ~(RT_BIT(11) | RT_BIT(12) | RT_BIT(14) | RT_BIT(15)); /* must be zero */
3649 pCtx->dr[7] |= 0x400; /* must be one */
3650
3651 /* Resync DR7 */
3652 rc2 = VMXWriteVMCS64(VMX_VMCS_GUEST_DR7, pCtx->dr[7]);
3653 AssertRC(rc2);
3654
3655 Log(("Trap %x (debug) at %RGv exit qualification %RX64 dr6=%x dr7=%x\n", vector, (RTGCPTR)pCtx->rip,
3656 exitQualification, (uint32_t)pCtx->dr[6], (uint32_t)pCtx->dr[7]));
3657 rc2 = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo),
3658 cbInstr, errCode);
3659 AssertRC(rc2);
3660
3661 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2Sub3, y3);
3662 goto ResumeExecution;
3663 }
3664 /* Return to ring 3 to deal with the debug exit code. */
3665 Log(("Debugger hardware BP at %04x:%RGv (rc=%Rrc)\n", pCtx->cs.Sel, pCtx->rip, VBOXSTRICTRC_VAL(rc)));
3666 break;
3667 }
3668
3669 case X86_XCPT_BP: /* Breakpoint. */
3670 {
3671 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
3672 rc = DBGFRZTrap03Handler(pVM, pVCpu, CPUMCTX2CORE(pCtx));
3673 if (rc == VINF_EM_RAW_GUEST_TRAP)
3674 {
3675 Log(("Guest #BP at %04x:%RGv\n", pCtx->cs.Sel, pCtx->rip));
3676 rc2 = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo),
3677 cbInstr, errCode);
3678 AssertRC(rc2);
3679 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2Sub3, y3);
3680 goto ResumeExecution;
3681 }
3682 if (rc == VINF_SUCCESS)
3683 {
3684 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2Sub3, y3);
3685 goto ResumeExecution;
3686 }
3687 Log(("Debugger BP at %04x:%RGv (rc=%Rrc)\n", pCtx->cs.Sel, pCtx->rip, VBOXSTRICTRC_VAL(rc)));
3688 break;
3689 }
3690
3691 case X86_XCPT_GP: /* General protection failure exception. */
3692 {
3693 uint32_t cbOp;
3694 PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
3695
3696 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
3697#ifdef VBOX_STRICT
3698 if ( !CPUMIsGuestInRealModeEx(pCtx)
3699 || !pVM->hm.s.vmx.pRealModeTSS)
3700 {
3701 Log(("Trap %x at %04X:%RGv errorCode=%RGv\n", vector, pCtx->cs.Sel, (RTGCPTR)pCtx->rip, errCode));
3702 rc2 = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo),
3703 cbInstr, errCode);
3704 AssertRC(rc2);
3705 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2Sub3, y3);
3706 goto ResumeExecution;
3707 }
3708#endif
3709 Assert(CPUMIsGuestInRealModeEx(pCtx));
3710
3711 LogFlow(("Real mode X86_XCPT_GP instruction emulation at %x:%RGv\n", pCtx->cs.Sel, (RTGCPTR)pCtx->rip));
3712
3713 rc2 = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbOp);
3714 if (RT_SUCCESS(rc2))
3715 {
3716 bool fUpdateRIP = true;
3717
3718 rc = VINF_SUCCESS;
3719 Assert(cbOp == pDis->cbInstr);
3720 switch (pDis->pCurInstr->uOpcode)
3721 {
3722 case OP_CLI:
3723 pCtx->eflags.Bits.u1IF = 0;
3724 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCli);
3725 break;
3726
3727 case OP_STI:
3728 pCtx->eflags.Bits.u1IF = 1;
3729 EMSetInhibitInterruptsPC(pVCpu, pCtx->rip + pDis->cbInstr);
3730 Assert(VMCPU_FF_ISSET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
3731 rc2 = VMXWriteVMCS(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE,
3732 VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
3733 AssertRC(rc2);
3734 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitSti);
3735 break;
3736
3737 case OP_HLT:
3738 fUpdateRIP = false;
3739 rc = VINF_EM_HALT;
3740 pCtx->rip += pDis->cbInstr;
3741 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
3742 break;
3743
3744 case OP_POPF:
3745 {
3746 RTGCPTR GCPtrStack;
3747 uint32_t cbParm;
3748 uint32_t uMask;
3749 X86EFLAGS eflags;
3750
3751 if (pDis->fPrefix & DISPREFIX_OPSIZE)
3752 {
3753 cbParm = 4;
3754 uMask = 0xffffffff;
3755 }
3756 else
3757 {
3758 cbParm = 2;
3759 uMask = 0xffff;
3760 }
3761
3762 rc2 = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pCtx), pCtx->esp & uMask, 0, &GCPtrStack);
3763 if (RT_FAILURE(rc2))
3764 {
3765 rc = VERR_EM_INTERPRETER;
3766 break;
3767 }
3768 eflags.u = 0;
3769 rc2 = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &eflags.u, cbParm);
3770 if (RT_FAILURE(rc2))
3771 {
3772 rc = VERR_EM_INTERPRETER;
3773 break;
3774 }
3775 LogFlow(("POPF %x -> %RGv mask=%x\n", eflags.u, pCtx->rsp, uMask));
3776 pCtx->eflags.u = (pCtx->eflags.u & ~(X86_EFL_POPF_BITS & uMask))
3777 | (eflags.u & X86_EFL_POPF_BITS & uMask);
3778 /* RF cleared when popped in real mode; see pushf description in AMD manual. */
3779 pCtx->eflags.Bits.u1RF = 0;
3780 pCtx->esp += cbParm;
3781 pCtx->esp &= uMask;
3782
3783 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPopf);
3784 break;
3785 }
3786
3787 case OP_PUSHF:
3788 {
3789 RTGCPTR GCPtrStack;
3790 uint32_t cbParm;
3791 uint32_t uMask;
3792 X86EFLAGS eflags;
3793
3794 if (pDis->fPrefix & DISPREFIX_OPSIZE)
3795 {
3796 cbParm = 4;
3797 uMask = 0xffffffff;
3798 }
3799 else
3800 {
3801 cbParm = 2;
3802 uMask = 0xffff;
3803 }
3804
3805 rc2 = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pCtx), (pCtx->esp - cbParm) & uMask, 0,
3806 &GCPtrStack);
3807 if (RT_FAILURE(rc2))
3808 {
3809 rc = VERR_EM_INTERPRETER;
3810 break;
3811 }
3812 eflags = pCtx->eflags;
3813 /* RF & VM cleared when pushed in real mode; see pushf description in AMD manual. */
3814 eflags.Bits.u1RF = 0;
3815 eflags.Bits.u1VM = 0;
3816
3817 rc2 = PGMPhysWrite(pVM, (RTGCPHYS)GCPtrStack, &eflags.u, cbParm);
3818 if (RT_FAILURE(rc2))
3819 {
3820 rc = VERR_EM_INTERPRETER;
3821 break;
3822 }
3823 LogFlow(("PUSHF %x -> %RGv\n", eflags.u, GCPtrStack));
3824 pCtx->esp -= cbParm;
3825 pCtx->esp &= uMask;
3826 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPushf);
3827 break;
3828 }
3829
3830 case OP_IRET:
3831 {
3832 RTGCPTR GCPtrStack;
3833 uint32_t uMask = 0xffff;
3834 uint16_t aIretFrame[3];
3835
3836 if (pDis->fPrefix & (DISPREFIX_OPSIZE | DISPREFIX_ADDRSIZE))
3837 {
3838 rc = VERR_EM_INTERPRETER;
3839 break;
3840 }
3841
3842 rc2 = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pCtx), pCtx->esp & uMask, 0, &GCPtrStack);
3843 if (RT_FAILURE(rc2))
3844 {
3845 rc = VERR_EM_INTERPRETER;
3846 break;
3847 }
3848 rc2 = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &aIretFrame[0], sizeof(aIretFrame));
3849 if (RT_FAILURE(rc2))
3850 {
3851 rc = VERR_EM_INTERPRETER;
3852 break;
3853 }
3854 pCtx->ip = aIretFrame[0];
3855 pCtx->cs.Sel = aIretFrame[1];
3856 pCtx->cs.ValidSel = aIretFrame[1];
3857 pCtx->cs.u64Base = (uint32_t)pCtx->cs.Sel << 4;
3858 pCtx->eflags.u = (pCtx->eflags.u & ~(X86_EFL_POPF_BITS & uMask))
3859 | (aIretFrame[2] & X86_EFL_POPF_BITS & uMask);
3860 pCtx->sp += sizeof(aIretFrame);
3861
3862 LogFlow(("iret to %04x:%x\n", pCtx->cs.Sel, pCtx->ip));
3863 fUpdateRIP = false;
3864 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIret);
3865 break;
3866 }
3867
3868 case OP_INT:
3869 {
3870 uint32_t intInfo2;
3871
3872 LogFlow(("Realmode: INT %x\n", pDis->Param1.uValue & 0xff));
3873 intInfo2 = pDis->Param1.uValue & 0xff;
3874 intInfo2 |= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
3875 intInfo2 |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
3876
3877 rc = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, intInfo2, cbOp, 0);
3878 AssertRC(VBOXSTRICTRC_VAL(rc));
3879 fUpdateRIP = false;
3880 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
3881 break;
3882 }
3883
3884 case OP_INTO:
3885 {
3886 if (pCtx->eflags.Bits.u1OF)
3887 {
3888 uint32_t intInfo2;
3889
3890 LogFlow(("Realmode: INTO\n"));
3891 intInfo2 = X86_XCPT_OF;
3892 intInfo2 |= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
3893 intInfo2 |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
3894
3895 rc = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, intInfo2, cbOp, 0);
3896 AssertRC(VBOXSTRICTRC_VAL(rc));
3897 fUpdateRIP = false;
3898 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
3899 }
3900 break;
3901 }
3902
3903 case OP_INT3:
3904 {
3905 uint32_t intInfo2;
3906
3907 LogFlow(("Realmode: INT 3\n"));
3908 intInfo2 = 3;
3909 intInfo2 |= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
3910 intInfo2 |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
3911
3912 rc = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, intInfo2, cbOp, 0);
3913 AssertRC(VBOXSTRICTRC_VAL(rc));
3914 fUpdateRIP = false;
3915 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
3916 break;
3917 }
3918
3919 default:
3920 rc = EMInterpretInstructionDisasState(pVCpu, pDis, CPUMCTX2CORE(pCtx), 0, EMCODETYPE_SUPERVISOR);
3921 fUpdateRIP = false;
3922 break;
3923 }
3924
3925 if (rc == VINF_SUCCESS)
3926 {
3927 if (fUpdateRIP)
3928 pCtx->rip += cbOp; /* Move on to the next instruction. */
3929
3930 /*
3931 * LIDT, LGDT can end up here. In the future CRx changes as well. Just reload the
3932 * whole context to be done with it.
3933 */
3934 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_ALL;
3935
3936 /* Only resume if successful. */
3937 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2Sub3, y3);
3938 goto ResumeExecution;
3939 }
3940 }
3941 else
3942 rc = VERR_EM_INTERPRETER;
3943
3944 AssertMsg(rc == VERR_EM_INTERPRETER || rc == VINF_PGM_CHANGE_MODE || rc == VINF_EM_HALT,
3945 ("Unexpected rc=%Rrc\n", VBOXSTRICTRC_VAL(rc)));
3946 break;
3947 }
3948
3949#ifdef VBOX_STRICT
3950 case X86_XCPT_XF: /* SIMD exception. */
3951 case X86_XCPT_DE: /* Divide error. */
3952 case X86_XCPT_UD: /* Unknown opcode exception. */
3953 case X86_XCPT_SS: /* Stack segment exception. */
3954 case X86_XCPT_NP: /* Segment not present exception. */
3955 {
3956 switch (vector)
3957 {
3958 case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE); break;
3959 case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD); break;
3960 case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS); break;
3961 case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP); break;
3962 case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF); break;
3963 }
3964
3965 Log(("Trap %x at %04X:%RGv\n", vector, pCtx->cs.Sel, (RTGCPTR)pCtx->rip));
3966 rc2 = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo),
3967 cbInstr, errCode);
3968 AssertRC(rc2);
3969
3970 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2Sub3, y3);
3971 goto ResumeExecution;
3972 }
3973#endif
3974 default:
3975 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
3976 if ( CPUMIsGuestInRealModeEx(pCtx)
3977 && pVM->hm.s.vmx.pRealModeTSS)
3978 {
3979 Log(("Real Mode Trap %x at %04x:%04X error code %x\n", vector, pCtx->cs.Sel, pCtx->eip, errCode));
3980 rc = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo),
3981 cbInstr, errCode);
3982 AssertRC(VBOXSTRICTRC_VAL(rc)); /* Strict RC check below. */
3983
3984 /* Go back to ring-3 in case of a triple fault. */
3985 if ( vector == X86_XCPT_DF
3986 && rc == VINF_EM_RESET)
3987 {
3988 break;
3989 }
3990
3991 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2Sub3, y3);
3992 goto ResumeExecution;
3993 }
3994 AssertMsgFailed(("Unexpected vm-exit caused by exception %x\n", vector));
3995 rc = VERR_VMX_UNEXPECTED_EXCEPTION;
3996 break;
3997 } /* switch (vector) */
3998
3999 break;
4000
4001 default:
4002 rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_CODE;
4003 AssertMsgFailed(("Unexpected interruption code %x\n", intInfo));
4004 break;
4005 }
4006
4007 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2Sub3, y3);
4008 break;
4009 }
4010
4011 /*
4012 * 48 EPT violation. An attempt to access memory with a guest-physical address was disallowed
4013 * by the configuration of the EPT paging structures.
4014 */
4015 case VMX_EXIT_EPT_VIOLATION:
4016 {
4017 RTGCPHYS GCPhys;
4018
4019 Assert(pVM->hm.s.fNestedPaging);
4020
4021 rc2 = VMXReadVMCS64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
4022 AssertRC(rc2);
4023 Assert(((exitQualification >> 7) & 3) != 2);
4024
4025 /* Determine the kind of violation. */
4026 errCode = 0;
4027 if (exitQualification & VMX_EXIT_QUALIFICATION_EPT_INSTR_FETCH)
4028 errCode |= X86_TRAP_PF_ID;
4029
4030 if (exitQualification & VMX_EXIT_QUALIFICATION_EPT_DATA_WRITE)
4031 errCode |= X86_TRAP_PF_RW;
4032
4033 /* If the page is present, then it's a page level protection fault. */
4034 if (exitQualification & VMX_EXIT_QUALIFICATION_EPT_ENTRY_PRESENT)
4035 errCode |= X86_TRAP_PF_P;
4036 else
4037 {
4038 /* Shortcut for APIC TPR reads and writes. */
4039 if ( (GCPhys & 0xfff) == 0x080
4040 && GCPhys > 0x1000000 /* to skip VGA frame buffer accesses */
4041 && fSetupTPRCaching
4042 && (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC))
4043 {
4044 RTGCPHYS GCPhysApicBase;
4045 GCPhysApicBase = pCtx->msrApicBase;
4046 GCPhysApicBase &= PAGE_BASE_GC_MASK;
4047 if (GCPhys == GCPhysApicBase + 0x80)
4048 {
4049 Log(("Enable VT-x virtual APIC access filtering\n"));
4050 rc2 = IOMMMIOMapMMIOHCPage(pVM, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW | X86_PTE_P);
4051 AssertRC(rc2);
4052 }
4053 }
4054 }
4055 Log(("EPT Page fault %x at %RGp error code %x\n", (uint32_t)exitQualification, GCPhys, errCode));
4056
4057 /* GCPhys contains the guest physical address of the page fault. */
4058 TRPMAssertTrap(pVCpu, X86_XCPT_PF, TRPM_TRAP);
4059 TRPMSetErrorCode(pVCpu, errCode);
4060 TRPMSetFaultAddress(pVCpu, GCPhys);
4061
4062 /* Handle the pagefault trap for the nested shadow table. */
4063 rc = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, errCode, CPUMCTX2CORE(pCtx), GCPhys);
4064
4065 /*
4066 * Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment below, @bugref{6043}.
4067 */
4068 if ( rc == VINF_SUCCESS
4069 || rc == VERR_PAGE_TABLE_NOT_PRESENT
4070 || rc == VERR_PAGE_NOT_PRESENT)
4071 {
4072 /* We've successfully synced our shadow pages, so let's just continue execution. */
4073 Log2(("Shadow page fault at %RGv cr2=%RGp error code %x\n", (RTGCPTR)pCtx->rip, exitQualification , errCode));
4074 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
4075
4076 TRPMResetTrap(pVCpu);
4077 goto ResumeExecution;
4078 }
4079
4080#ifdef VBOX_STRICT
4081 if (rc != VINF_EM_RAW_EMULATE_INSTR)
4082 LogFlow(("PGMTrap0eHandlerNestedPaging at %RGv failed with %Rrc\n", (RTGCPTR)pCtx->rip, VBOXSTRICTRC_VAL(rc)));
4083#endif
4084 /* Need to go back to the recompiler to emulate the instruction. */
4085 TRPMResetTrap(pVCpu);
4086 break;
4087 }
4088
4089 case VMX_EXIT_EPT_MISCONFIG:
4090 {
4091 RTGCPHYS GCPhys;
4092
4093 Assert(pVM->hm.s.fNestedPaging);
4094
4095 rc2 = VMXReadVMCS64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
4096 AssertRC(rc2);
4097 Log(("VMX_EXIT_EPT_MISCONFIG for %RGp\n", GCPhys));
4098
4099 /* Shortcut for APIC TPR reads and writes. */
4100 if ( (GCPhys & 0xfff) == 0x080
4101 && GCPhys > 0x1000000 /* to skip VGA frame buffer accesses */
4102 && fSetupTPRCaching
4103 && (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC))
4104 {
4105 RTGCPHYS GCPhysApicBase = pCtx->msrApicBase;
4106 GCPhysApicBase &= PAGE_BASE_GC_MASK;
4107 if (GCPhys == GCPhysApicBase + 0x80)
4108 {
4109 Log(("Enable VT-x virtual APIC access filtering\n"));
4110 rc2 = IOMMMIOMapMMIOHCPage(pVM, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW | X86_PTE_P);
4111 AssertRC(rc2);
4112 }
4113 }
4114
4115 rc = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pCtx), GCPhys, UINT32_MAX);
4116
4117 /*
4118 * If we succeed, resume execution.
4119 * Or, if fail in interpreting the instruction because we couldn't get the guest physical address
4120 * of the page containing the instruction via the guest's page tables (we would invalidate the guest page
4121 * in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
4122 * weird case. See @bugref{6043}.
4123 */
4124 if ( rc == VINF_SUCCESS
4125 || rc == VERR_PAGE_TABLE_NOT_PRESENT
4126 || rc == VERR_PAGE_NOT_PRESENT)
4127 {
4128 Log2(("PGMR0Trap0eHandlerNPMisconfig(,,,%RGp) at %RGv -> resume\n", GCPhys, (RTGCPTR)pCtx->rip));
4129 goto ResumeExecution;
4130 }
4131
4132 Log2(("PGMR0Trap0eHandlerNPMisconfig(,,,%RGp) at %RGv -> %Rrc\n", GCPhys, (RTGCPTR)pCtx->rip, VBOXSTRICTRC_VAL(rc)));
4133 break;
4134 }
4135
4136 case VMX_EXIT_IRQ_WINDOW: /* 7 Interrupt window. */
4137 /* Clear VM-exit on IF=1 change. */
4138 LogFlow(("VMX_EXIT_IRQ_WINDOW %RGv pending=%d IF=%d\n", (RTGCPTR)pCtx->rip,
4139 VMCPU_FF_ISPENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC|VMCPU_FF_INTERRUPT_PIC)), pCtx->eflags.Bits.u1IF));
4140 pVCpu->hm.s.vmx.proc_ctls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_IRQ_WINDOW_EXIT;
4141 rc2 = VMXWriteVMCS(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.proc_ctls);
4142 AssertRC(rc2);
4143 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIrqWindow);
4144 goto ResumeExecution; /* we check for pending guest interrupts there */
4145
4146 case VMX_EXIT_WBINVD: /* 54 Guest software attempted to execute WBINVD. (conditional) */
4147 case VMX_EXIT_INVD: /* 13 Guest software attempted to execute INVD. (unconditional) */
4148 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvd);
4149 /* Skip instruction and continue directly. */
4150 pCtx->rip += cbInstr;
4151 /* Continue execution.*/
4152 goto ResumeExecution;
4153
4154 case VMX_EXIT_CPUID: /* 10 Guest software attempted to execute CPUID. */
4155 {
4156 Log2(("VMX: Cpuid %x\n", pCtx->eax));
4157 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCpuid);
4158 rc = EMInterpretCpuId(pVM, pVCpu, CPUMCTX2CORE(pCtx));
4159 if (rc == VINF_SUCCESS)
4160 {
4161 /* Update EIP and continue execution. */
4162 Assert(cbInstr == 2);
4163 pCtx->rip += cbInstr;
4164 goto ResumeExecution;
4165 }
4166 AssertMsgFailed(("EMU: cpuid failed with %Rrc\n", VBOXSTRICTRC_VAL(rc)));
4167 rc = VINF_EM_RAW_EMULATE_INSTR;
4168 break;
4169 }
4170
4171 case VMX_EXIT_RDPMC: /* 15 Guest software attempted to execute RDPMC. */
4172 {
4173 Log2(("VMX: Rdpmc %x\n", pCtx->ecx));
4174 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdpmc);
4175 rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pCtx));
4176 if (rc == VINF_SUCCESS)
4177 {
4178 /* Update EIP and continue execution. */
4179 Assert(cbInstr == 2);
4180 pCtx->rip += cbInstr;
4181 goto ResumeExecution;
4182 }
4183 rc = VINF_EM_RAW_EMULATE_INSTR;
4184 break;
4185 }
4186
4187 case VMX_EXIT_RDTSC: /* 16 Guest software attempted to execute RDTSC. */
4188 {
4189 Log2(("VMX: Rdtsc\n"));
4190 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
4191 rc = EMInterpretRdtsc(pVM, pVCpu, CPUMCTX2CORE(pCtx));
4192 if (rc == VINF_SUCCESS)
4193 {
4194 /* Update EIP and continue execution. */
4195 Assert(cbInstr == 2);
4196 pCtx->rip += cbInstr;
4197 goto ResumeExecution;
4198 }
4199 rc = VINF_EM_RAW_EMULATE_INSTR;
4200 break;
4201 }
4202
4203 case VMX_EXIT_RDTSCP: /* 51 Guest software attempted to execute RDTSCP. */
4204 {
4205 Log2(("VMX: Rdtscp\n"));
4206 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtscp);
4207 rc = EMInterpretRdtscp(pVM, pVCpu, pCtx);
4208 if (rc == VINF_SUCCESS)
4209 {
4210 /* Update EIP and continue execution. */
4211 Assert(cbInstr == 3);
4212 pCtx->rip += cbInstr;
4213 goto ResumeExecution;
4214 }
4215 rc = VINF_EM_RAW_EMULATE_INSTR;
4216 break;
4217 }
4218
4219 case VMX_EXIT_INVLPG: /* 14 Guest software attempted to execute INVLPG. */
4220 {
4221 Log2(("VMX: invlpg\n"));
4222 Assert(!pVM->hm.s.fNestedPaging);
4223
4224 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvlpg);
4225 rc = EMInterpretInvlpg(pVM, pVCpu, CPUMCTX2CORE(pCtx), exitQualification);
4226 if (rc == VINF_SUCCESS)
4227 {
4228 /* Update EIP and continue execution. */
4229 pCtx->rip += cbInstr;
4230 goto ResumeExecution;
4231 }
4232 AssertMsg(rc == VERR_EM_INTERPRETER, ("EMU: invlpg %RGv failed with %Rrc\n", exitQualification, VBOXSTRICTRC_VAL(rc)));
4233 break;
4234 }
4235
4236 case VMX_EXIT_MONITOR: /* 39 Guest software attempted to execute MONITOR. */
4237 {
4238 Log2(("VMX: monitor\n"));
4239
4240 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMonitor);
4241 rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pCtx));
4242 if (rc == VINF_SUCCESS)
4243 {
4244 /* Update EIP and continue execution. */
4245 pCtx->rip += cbInstr;
4246 goto ResumeExecution;
4247 }
4248 AssertMsg(rc == VERR_EM_INTERPRETER, ("EMU: monitor failed with %Rrc\n", VBOXSTRICTRC_VAL(rc)));
4249 break;
4250 }
4251
4252 case VMX_EXIT_WRMSR: /* 32 WRMSR. Guest software attempted to execute WRMSR. */
4253 /* When an interrupt is pending, we'll let MSR_K8_LSTAR writes fault in our TPR patch code. */
4254 if ( pVM->hm.s.fTPRPatchingActive
4255 && pCtx->ecx == MSR_K8_LSTAR)
4256 {
4257 Assert(!CPUMIsGuestInLongModeEx(pCtx));
4258 if ((pCtx->eax & 0xff) != u8LastTPR)
4259 {
4260 Log(("VMX: Faulting MSR_K8_LSTAR write with new TPR value %x\n", pCtx->eax & 0xff));
4261
4262 /* Our patch code uses LSTAR for TPR caching. */
4263 rc2 = PDMApicSetTPR(pVCpu, pCtx->eax & 0xff);
4264 AssertRC(rc2);
4265 }
4266
4267 /* Skip the instruction and continue. */
4268 pCtx->rip += cbInstr; /* wrmsr = [0F 30] */
4269
4270 /* Only resume if successful. */
4271 goto ResumeExecution;
4272 }
4273 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_MSR;
4274 /* no break */
4275 case VMX_EXIT_RDMSR: /* 31 RDMSR. Guest software attempted to execute RDMSR. */
4276 {
4277 STAM_COUNTER_INC((exitReason == VMX_EXIT_RDMSR) ? &pVCpu->hm.s.StatExitRdmsr : &pVCpu->hm.s.StatExitWrmsr);
4278
4279 /*
4280 * Note: The Intel spec. claims there's an REX version of RDMSR that's slightly different,
4281 * so we play safe by completely disassembling the instruction.
4282 */
4283 Log2(("VMX: %s\n", (exitReason == VMX_EXIT_RDMSR) ? "rdmsr" : "wrmsr"));
4284 rc = EMInterpretInstruction(pVCpu, CPUMCTX2CORE(pCtx), 0);
4285 if (rc == VINF_SUCCESS)
4286 {
4287 /* EIP has been updated already. */
4288 /* Only resume if successful. */
4289 goto ResumeExecution;
4290 }
4291 AssertMsg(rc == VERR_EM_INTERPRETER, ("EMU: %s failed with %Rrc\n",
4292 (exitReason == VMX_EXIT_RDMSR) ? "rdmsr" : "wrmsr", VBOXSTRICTRC_VAL(rc)));
4293 break;
4294 }
4295
4296 case VMX_EXIT_CRX_MOVE: /* 28 Control-register accesses. */
4297 {
4298 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExit2Sub2, y2);
4299
4300 switch (VMX_EXIT_QUALIFICATION_CRX_ACCESS(exitQualification))
4301 {
4302 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_WRITE:
4303 {
4304 Log2(("VMX: %RGv mov cr%d, x\n", (RTGCPTR)pCtx->rip, VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification)));
4305 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxWrite[VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification)]);
4306 rc = EMInterpretCRxWrite(pVM, pVCpu, CPUMCTX2CORE(pCtx),
4307 VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification),
4308 VMX_EXIT_QUALIFICATION_CRX_GENREG(exitQualification));
4309 switch (VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification))
4310 {
4311 case 0:
4312 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_CR0 | HM_CHANGED_GUEST_CR3;
4313 break;
4314 case 2:
4315 break;
4316 case 3:
4317 Assert(!pVM->hm.s.fNestedPaging || !CPUMIsGuestInPagedProtectedModeEx(pCtx));
4318 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_CR3;
4319 break;
4320 case 4:
4321 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_CR4;
4322 break;
4323 case 8:
4324 /* CR8 contains the APIC TPR */
4325 Assert(!(pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1
4326 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW));
4327 break;
4328
4329 default:
4330 AssertFailed();
4331 break;
4332 }
4333 break;
4334 }
4335
4336 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ:
4337 {
4338 Log2(("VMX: mov x, crx\n"));
4339 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxRead[VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification)]);
4340
4341 Assert( !pVM->hm.s.fNestedPaging
4342 || !CPUMIsGuestInPagedProtectedModeEx(pCtx)
4343 || VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification) != DISCREG_CR3);
4344
4345 /* CR8 reads only cause an exit when the TPR shadow feature isn't present. */
4346 Assert( VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification) != 8
4347 || !(pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW));
4348
4349 rc = EMInterpretCRxRead(pVM, pVCpu, CPUMCTX2CORE(pCtx),
4350 VMX_EXIT_QUALIFICATION_CRX_GENREG(exitQualification),
4351 VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification));
4352 break;
4353 }
4354
4355 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_CLTS:
4356 {
4357 Log2(("VMX: clts\n"));
4358 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
4359 rc = EMInterpretCLTS(pVM, pVCpu);
4360 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_CR0;
4361 break;
4362 }
4363
4364 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_LMSW:
4365 {
4366 Log2(("VMX: lmsw %x\n", VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(exitQualification)));
4367 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLMSW);
4368 rc = EMInterpretLMSW(pVM, pVCpu, CPUMCTX2CORE(pCtx), VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(exitQualification));
4369 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_CR0;
4370 break;
4371 }
4372 }
4373
4374 /* Update EIP if no error occurred. */
4375 if (RT_SUCCESS(rc))
4376 pCtx->rip += cbInstr;
4377
4378 if (rc == VINF_SUCCESS)
4379 {
4380 /* Only resume if successful. */
4381 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2Sub2, y2);
4382 goto ResumeExecution;
4383 }
4384 Assert(rc == VERR_EM_INTERPRETER || rc == VINF_PGM_CHANGE_MODE || rc == VINF_PGM_SYNC_CR3);
4385 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2Sub2, y2);
4386 break;
4387 }
4388
4389 case VMX_EXIT_DRX_MOVE: /* 29 Debug-register accesses. */
4390 {
4391 if ( !DBGFIsStepping(pVCpu)
4392 && !CPUMIsHyperDebugStateActive(pVCpu))
4393 {
4394 /* Disable DRx move intercepts. */
4395 pVCpu->hm.s.vmx.proc_ctls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT;
4396 rc2 = VMXWriteVMCS(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.proc_ctls);
4397 AssertRC(rc2);
4398
4399 /* Save the host and load the guest debug state. */
4400 rc2 = CPUMR0LoadGuestDebugState(pVM, pVCpu, pCtx, true /* include DR6 */);
4401 AssertRC(rc2);
4402
4403#ifdef LOG_ENABLED
4404 if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(exitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
4405 {
4406 Log(("VMX_EXIT_DRX_MOVE: write DR%d genreg %d\n", VMX_EXIT_QUALIFICATION_DRX_REGISTER(exitQualification),
4407 VMX_EXIT_QUALIFICATION_DRX_GENREG(exitQualification)));
4408 }
4409 else
4410 Log(("VMX_EXIT_DRX_MOVE: read DR%d\n", VMX_EXIT_QUALIFICATION_DRX_REGISTER(exitQualification)));
4411#endif
4412
4413#ifdef VBOX_WITH_STATISTICS
4414 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
4415 if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(exitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
4416 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
4417 else
4418 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
4419#endif
4420
4421 goto ResumeExecution;
4422 }
4423
4424 /** @todo clear VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT after the first
4425 * time and restore DRx registers afterwards */
4426 if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(exitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
4427 {
4428 Log2(("VMX: mov DRx%d, genreg%d\n", VMX_EXIT_QUALIFICATION_DRX_REGISTER(exitQualification),
4429 VMX_EXIT_QUALIFICATION_DRX_GENREG(exitQualification)));
4430 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
4431 rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pCtx),
4432 VMX_EXIT_QUALIFICATION_DRX_REGISTER(exitQualification),
4433 VMX_EXIT_QUALIFICATION_DRX_GENREG(exitQualification));
4434 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_DEBUG;
4435 Log2(("DR7=%08x\n", pCtx->dr[7]));
4436 }
4437 else
4438 {
4439 Log2(("VMX: mov x, DRx\n"));
4440 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
4441 rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pCtx),
4442 VMX_EXIT_QUALIFICATION_DRX_GENREG(exitQualification),
4443 VMX_EXIT_QUALIFICATION_DRX_REGISTER(exitQualification));
4444 }
4445 /* Update EIP if no error occurred. */
4446 if (RT_SUCCESS(rc))
4447 pCtx->rip += cbInstr;
4448
4449 if (rc == VINF_SUCCESS)
4450 {
4451 /* Only resume if successful. */
4452 goto ResumeExecution;
4453 }
4454 Assert(rc == VERR_EM_INTERPRETER);
4455 break;
4456 }
4457
4458 /* Note: We'll get a #GP if the IO instruction isn't allowed (IOPL or TSS bitmap); no need to double check. */
4459 case VMX_EXIT_PORT_IO: /* 30 I/O instruction. */
4460 {
4461 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExit2Sub1, y1);
4462 uint32_t uPort;
4463 uint32_t uIOWidth = VMX_EXIT_QUALIFICATION_IO_WIDTH(exitQualification);
4464 bool fIOWrite = (VMX_EXIT_QUALIFICATION_IO_DIRECTION(exitQualification) == VMX_EXIT_QUALIFICATION_IO_DIRECTION_OUT);
4465
4466 /** @todo necessary to make the distinction? */
4467 if (VMX_EXIT_QUALIFICATION_IO_ENCODING(exitQualification) == VMX_EXIT_QUALIFICATION_IO_ENCODING_DX)
4468 uPort = pCtx->edx & 0xffff;
4469 else
4470 uPort = VMX_EXIT_QUALIFICATION_IO_PORT(exitQualification); /* Immediate encoding. */
4471
4472 if (RT_UNLIKELY(uIOWidth == 2 || uIOWidth >= 4)) /* paranoia */
4473 {
4474 rc = fIOWrite ? VINF_IOM_R3_IOPORT_WRITE : VINF_IOM_R3_IOPORT_READ;
4475 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2Sub1, y1);
4476 break;
4477 }
4478
4479 uint32_t cbSize = g_aIOSize[uIOWidth];
4480 if (VMX_EXIT_QUALIFICATION_IO_STRING(exitQualification))
4481 {
4482 /* ins/outs */
4483 PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
4484
4485 /* Disassemble manually to deal with segment prefixes. */
4486 /** @todo VMX_VMCS_EXIT_GUEST_LINEAR_ADDR contains the flat pointer operand of the instruction. */
4487 /** @todo VMX_VMCS32_RO_EXIT_INSTR_INFO also contains segment prefix info. */
4488 rc2 = EMInterpretDisasCurrent(pVM, pVCpu, pDis, NULL);
4489 if (RT_SUCCESS(rc))
4490 {
4491 if (fIOWrite)
4492 {
4493 Log2(("IOMInterpretOUTSEx %RGv %x size=%d\n", (RTGCPTR)pCtx->rip, uPort, cbSize));
4494 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringWrite);
4495 rc = IOMInterpretOUTSEx(pVM, CPUMCTX2CORE(pCtx), uPort, pDis->fPrefix, (DISCPUMODE)pDis->uAddrMode, cbSize);
4496 }
4497 else
4498 {
4499 Log2(("IOMInterpretINSEx %RGv %x size=%d\n", (RTGCPTR)pCtx->rip, uPort, cbSize));
4500 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringRead);
4501 rc = IOMInterpretINSEx(pVM, CPUMCTX2CORE(pCtx), uPort, pDis->fPrefix, (DISCPUMODE)pDis->uAddrMode, cbSize);
4502 }
4503 }
4504 else
4505 rc = VINF_EM_RAW_EMULATE_INSTR;
4506 }
4507 else
4508 {
4509 /* Normal in/out */
4510 uint32_t uAndVal = g_aIOOpAnd[uIOWidth];
4511
4512 Assert(!VMX_EXIT_QUALIFICATION_IO_REP(exitQualification));
4513
4514 if (fIOWrite)
4515 {
4516 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
4517 rc = IOMIOPortWrite(pVM, uPort, pCtx->eax & uAndVal, cbSize);
4518 if (rc == VINF_IOM_R3_IOPORT_WRITE)
4519 HMR0SavePendingIOPortWrite(pVCpu, pCtx->rip, pCtx->rip + cbInstr, uPort, uAndVal, cbSize);
4520 }
4521 else
4522 {
4523 uint32_t u32Val = 0;
4524
4525 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
4526 rc = IOMIOPortRead(pVM, uPort, &u32Val, cbSize);
4527 if (IOM_SUCCESS(rc))
4528 {
4529 /* Write back to the EAX register. */
4530 pCtx->eax = (pCtx->eax & ~uAndVal) | (u32Val & uAndVal);
4531 }
4532 else
4533 if (rc == VINF_IOM_R3_IOPORT_READ)
4534 HMR0SavePendingIOPortRead(pVCpu, pCtx->rip, pCtx->rip + cbInstr, uPort, uAndVal, cbSize);
4535 }
4536 }
4537
4538 /*
4539 * Handled the I/O return codes.
4540 * (The unhandled cases end up with rc == VINF_EM_RAW_EMULATE_INSTR.)
4541 */
4542 if (IOM_SUCCESS(rc))
4543 {
4544 /* Update EIP and continue execution. */
4545 pCtx->rip += cbInstr;
4546 if (RT_LIKELY(rc == VINF_SUCCESS))
4547 {
4548 /* If any IO breakpoints are armed, then we should check if a debug trap needs to be generated. */
4549 if (pCtx->dr[7] & X86_DR7_ENABLED_MASK)
4550 {
4551 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
4552 for (unsigned i = 0; i < 4; i++)
4553 {
4554 unsigned uBPLen = g_aIOSize[X86_DR7_GET_LEN(pCtx->dr[7], i)];
4555
4556 if ( (uPort >= pCtx->dr[i] && uPort < pCtx->dr[i] + uBPLen)
4557 && (pCtx->dr[7] & (X86_DR7_L(i) | X86_DR7_G(i)))
4558 && (pCtx->dr[7] & X86_DR7_RW(i, X86_DR7_RW_IO)) == X86_DR7_RW(i, X86_DR7_RW_IO))
4559 {
4560 uint64_t uDR6;
4561
4562 Assert(CPUMIsGuestDebugStateActive(pVCpu));
4563
4564 uDR6 = ASMGetDR6();
4565
4566 /* Clear all breakpoint status flags and set the one we just hit. */
4567 uDR6 &= ~(X86_DR6_B0|X86_DR6_B1|X86_DR6_B2|X86_DR6_B3);
4568 uDR6 |= (uint64_t)RT_BIT(i);
4569
4570 /*
4571 * Note: AMD64 Architecture Programmer's Manual 13.1:
4572 * Bits 15:13 of the DR6 register is never cleared by the processor and must
4573 * be cleared by software after the contents have been read.
4574 */
4575 ASMSetDR6(uDR6);
4576
4577 /* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
4578 pCtx->dr[7] &= ~X86_DR7_GD;
4579
4580 /* Paranoia. */
4581 pCtx->dr[7] &= 0xffffffff; /* upper 32 bits reserved */
4582 pCtx->dr[7] &= ~(RT_BIT(11) | RT_BIT(12) | RT_BIT(14) | RT_BIT(15)); /* must be zero */
4583 pCtx->dr[7] |= 0x400; /* must be one */
4584
4585 /* Resync DR7 */
4586 rc2 = VMXWriteVMCS64(VMX_VMCS_GUEST_DR7, pCtx->dr[7]);
4587 AssertRC(rc2);
4588
4589 /* Construct inject info. */
4590 intInfo = X86_XCPT_DB;
4591 intInfo |= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
4592 intInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HWEXCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
4593
4594 Log(("Inject IO debug trap at %RGv\n", (RTGCPTR)pCtx->rip));
4595 rc2 = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo),
4596 0 /* cbInstr */, 0 /* errCode */);
4597 AssertRC(rc2);
4598
4599 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2Sub1, y1);
4600 goto ResumeExecution;
4601 }
4602 }
4603 }
4604 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2Sub1, y1);
4605 goto ResumeExecution;
4606 }
4607 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2Sub1, y1);
4608 break;
4609 }
4610
4611#ifdef VBOX_STRICT
4612 if (rc == VINF_IOM_R3_IOPORT_READ)
4613 Assert(!fIOWrite);
4614 else if (rc == VINF_IOM_R3_IOPORT_WRITE)
4615 Assert(fIOWrite);
4616 else
4617 {
4618 AssertMsg( RT_FAILURE(rc)
4619 || rc == VINF_EM_RAW_EMULATE_INSTR
4620 || rc == VINF_EM_RAW_GUEST_TRAP
4621 || rc == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", VBOXSTRICTRC_VAL(rc)));
4622 }
4623#endif
4624 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2Sub1, y1);
4625 break;
4626 }
4627
4628 case VMX_EXIT_TPR: /* 43 TPR below threshold. Guest software executed MOV to CR8. */
4629 LogFlow(("VMX_EXIT_TPR\n"));
4630 /* RIP is already set to the next instruction and the TPR has been synced back. Just resume. */
4631 goto ResumeExecution;
4632
4633 case VMX_EXIT_APIC_ACCESS: /* 44 APIC access. Guest software attempted to access memory at a physical address
4634 on the APIC-access page. */
4635 {
4636 LogFlow(("VMX_EXIT_APIC_ACCESS\n"));
4637 unsigned uAccessType = VMX_EXIT_QUALIFICATION_APIC_ACCESS_TYPE(exitQualification);
4638
4639 switch (uAccessType)
4640 {
4641 case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
4642 case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
4643 {
4644 RTGCPHYS GCPhys = pCtx->msrApicBase;
4645 GCPhys &= PAGE_BASE_GC_MASK;
4646 GCPhys += VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(exitQualification);
4647
4648 LogFlow(("Apic access at %RGp\n", GCPhys));
4649 rc = IOMMMIOPhysHandler(pVM, (uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ) ? 0 : X86_TRAP_PF_RW,
4650 CPUMCTX2CORE(pCtx), GCPhys);
4651 if (rc == VINF_SUCCESS)
4652 goto ResumeExecution; /* rip already updated */
4653 break;
4654 }
4655
4656 default:
4657 rc = VINF_EM_RAW_EMULATE_INSTR;
4658 break;
4659 }
4660 break;
4661 }
4662
4663 case VMX_EXIT_PREEMPTION_TIMER: /* 52 VMX-preemption timer expired. The preemption timer counted down to zero. */
4664 if (!TMTimerPollBool(pVM, pVCpu))
4665 goto ResumeExecution;
4666 rc = VINF_EM_RAW_TIMER_PENDING;
4667 break;
4668
4669 default:
4670 /* The rest is handled after syncing the entire CPU state. */
4671 break;
4672 }
4673
4674
4675 /*
4676 * Note: The guest state is not entirely synced back at this stage!
4677 */
4678
4679 /* Investigate why there was a VM-exit. (part 2) */
4680 switch (exitReason)
4681 {
4682 case VMX_EXIT_EXCEPTION: /* 0 Exception or non-maskable interrupt (NMI). */
4683 case VMX_EXIT_EXTERNAL_IRQ: /* 1 External interrupt. */
4684 case VMX_EXIT_EPT_VIOLATION:
4685 case VMX_EXIT_EPT_MISCONFIG: /* 49 EPT misconfig is used by the PGM/MMIO optimizations. */
4686 case VMX_EXIT_PREEMPTION_TIMER: /* 52 VMX-preemption timer expired. The preemption timer counted down to zero. */
4687 /* Already handled above. */
4688 break;
4689
4690 case VMX_EXIT_TRIPLE_FAULT: /* 2 Triple fault. */
4691 rc = VINF_EM_RESET; /* Triple fault equals a reset. */
4692 break;
4693
4694 case VMX_EXIT_INIT_SIGNAL: /* 3 INIT signal. */
4695 case VMX_EXIT_SIPI: /* 4 Start-up IPI (SIPI). */
4696 rc = VINF_EM_RAW_INTERRUPT;
4697 AssertFailed(); /* Can't happen. Yet. */
4698 break;
4699
4700 case VMX_EXIT_IO_SMI_IRQ: /* 5 I/O system-management interrupt (SMI). */
4701 case VMX_EXIT_SMI_IRQ: /* 6 Other SMI. */
4702 rc = VINF_EM_RAW_INTERRUPT;
4703 AssertFailed(); /* Can't happen afaik. */
4704 break;
4705
4706 case VMX_EXIT_TASK_SWITCH: /* 9 Task switch: too complicated to emulate, so fall back to the recompiler */
4707 Log(("VMX_EXIT_TASK_SWITCH: exit=%RX64\n", exitQualification));
4708 if ( (VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE(exitQualification) == VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE_IDT)
4709 && pVCpu->hm.s.Event.fPending)
4710 {
4711 /* Caused by an injected interrupt. */
4712 pVCpu->hm.s.Event.fPending = false;
4713
4714 Log(("VMX_EXIT_TASK_SWITCH: reassert trap %d\n", VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVCpu->hm.s.Event.intInfo)));
4715 Assert(!VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.intInfo));
4716 //@todo: Why do we assume this had to be a hardware interrupt? What about software interrupts or exceptions?
4717 rc2 = TRPMAssertTrap(pVCpu, VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVCpu->hm.s.Event.intInfo), TRPM_HARDWARE_INT);
4718 AssertRC(rc2);
4719 }
4720 /* else Exceptions and software interrupts can just be restarted. */
4721 rc = VERR_EM_INTERPRETER;
4722 break;
4723
4724 case VMX_EXIT_HLT: /* 12 Guest software attempted to execute HLT. */
4725 /* Check if external interrupts are pending; if so, don't switch back. */
4726 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
4727 pCtx->rip++; /* skip hlt */
4728 if (EMShouldContinueAfterHalt(pVCpu, pCtx))
4729 goto ResumeExecution;
4730
4731 rc = VINF_EM_HALT;
4732 break;
4733
4734 case VMX_EXIT_MWAIT: /* 36 Guest software executed MWAIT. */
4735 Log2(("VMX: mwait\n"));
4736 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMwait);
4737 rc = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pCtx));
4738 if ( rc == VINF_EM_HALT
4739 || rc == VINF_SUCCESS)
4740 {
4741 /* Update EIP and continue execution. */
4742 pCtx->rip += cbInstr;
4743
4744 /* Check if external interrupts are pending; if so, don't switch back. */
4745 if ( rc == VINF_SUCCESS
4746 || ( rc == VINF_EM_HALT
4747 && EMShouldContinueAfterHalt(pVCpu, pCtx))
4748 )
4749 goto ResumeExecution;
4750 }
4751 AssertMsg(rc == VERR_EM_INTERPRETER || rc == VINF_EM_HALT, ("EMU: mwait failed with %Rrc\n", VBOXSTRICTRC_VAL(rc)));
4752 break;
4753
4754 case VMX_EXIT_RSM: /* 17 Guest software attempted to execute RSM in SMM. */
4755 AssertFailed(); /* can't happen. */
4756 rc = VERR_EM_INTERPRETER;
4757 break;
4758
4759 case VMX_EXIT_MTF: /* 37 Exit due to Monitor Trap Flag. */
4760 LogFlow(("VMX_EXIT_MTF at %RGv\n", (RTGCPTR)pCtx->rip));
4761 pVCpu->hm.s.vmx.proc_ctls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_TRAP_FLAG;
4762 rc2 = VMXWriteVMCS(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.proc_ctls);
4763 AssertRC(rc2);
4764 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMtf);
4765#if 0
4766 DBGFDoneStepping(pVCpu);
4767#endif
4768 rc = VINF_EM_DBG_STOP;
4769 break;
4770
4771 case VMX_EXIT_VMCALL: /* 18 Guest software executed VMCALL. */
4772 case VMX_EXIT_VMCLEAR: /* 19 Guest software executed VMCLEAR. */
4773 case VMX_EXIT_VMLAUNCH: /* 20 Guest software executed VMLAUNCH. */
4774 case VMX_EXIT_VMPTRLD: /* 21 Guest software executed VMPTRLD. */
4775 case VMX_EXIT_VMPTRST: /* 22 Guest software executed VMPTRST. */
4776 case VMX_EXIT_VMREAD: /* 23 Guest software executed VMREAD. */
4777 case VMX_EXIT_VMRESUME: /* 24 Guest software executed VMRESUME. */
4778 case VMX_EXIT_VMWRITE: /* 25 Guest software executed VMWRITE. */
4779 case VMX_EXIT_VMXOFF: /* 26 Guest software executed VMXOFF. */
4780 case VMX_EXIT_VMXON: /* 27 Guest software executed VMXON. */
4781 /** @todo inject #UD immediately */
4782 rc = VERR_EM_INTERPRETER;
4783 break;
4784
4785 case VMX_EXIT_CPUID: /* 10 Guest software attempted to execute CPUID. */
4786 case VMX_EXIT_RDTSC: /* 16 Guest software attempted to execute RDTSC. */
4787 case VMX_EXIT_INVLPG: /* 14 Guest software attempted to execute INVLPG. */
4788 case VMX_EXIT_CRX_MOVE: /* 28 Control-register accesses. */
4789 case VMX_EXIT_DRX_MOVE: /* 29 Debug-register accesses. */
4790 case VMX_EXIT_PORT_IO: /* 30 I/O instruction. */
4791 case VMX_EXIT_RDPMC: /* 15 Guest software attempted to execute RDPMC. */
4792 case VMX_EXIT_RDTSCP: /* 51 Guest software attempted to execute RDTSCP. */
4793 /* already handled above */
4794 AssertMsg( rc == VINF_PGM_CHANGE_MODE
4795 || rc == VINF_EM_RAW_INTERRUPT
4796 || rc == VERR_EM_INTERPRETER
4797 || rc == VINF_EM_RAW_EMULATE_INSTR
4798 || rc == VINF_PGM_SYNC_CR3
4799 || rc == VINF_IOM_R3_IOPORT_READ
4800 || rc == VINF_IOM_R3_IOPORT_WRITE
4801 || rc == VINF_EM_RAW_GUEST_TRAP
4802 || rc == VINF_TRPM_XCPT_DISPATCHED
4803 || rc == VINF_EM_RESCHEDULE_REM,
4804 ("rc = %d\n", VBOXSTRICTRC_VAL(rc)));
4805 break;
4806
4807 case VMX_EXIT_TPR: /* 43 TPR below threshold. Guest software executed MOV to CR8. */
4808 case VMX_EXIT_RDMSR: /* 31 RDMSR. Guest software attempted to execute RDMSR. */
4809 case VMX_EXIT_WRMSR: /* 32 WRMSR. Guest software attempted to execute WRMSR. */
4810 case VMX_EXIT_PAUSE: /* 40 Guest software attempted to execute PAUSE. */
4811 case VMX_EXIT_MONITOR: /* 39 Guest software attempted to execute MONITOR. */
4812 case VMX_EXIT_APIC_ACCESS: /* 44 APIC access. Guest software attempted to access memory at a physical address
4813 on the APIC-access page. */
4814 {
4815 /*
4816 * If we decided to emulate them here, then we must sync the MSRs that could have been changed (sysenter, FS/GS base)
4817 */
4818 rc = VERR_EM_INTERPRETER;
4819 break;
4820 }
4821
4822 case VMX_EXIT_IRQ_WINDOW: /* 7 Interrupt window. */
4823 Assert(rc == VINF_EM_RAW_INTERRUPT);
4824 break;
4825
4826 case VMX_EXIT_ERR_INVALID_GUEST_STATE: /* 33 VM-entry failure due to invalid guest state. */
4827 {
4828#ifdef VBOX_STRICT
4829 RTCCUINTREG val2 = 0;
4830
4831 Log(("VMX_EXIT_ERR_INVALID_GUEST_STATE\n"));
4832
4833 VMXReadVMCS(VMX_VMCS_GUEST_RIP, &val2);
4834 Log(("Old eip %RGv new %RGv\n", (RTGCPTR)pCtx->rip, (RTGCPTR)val2));
4835
4836 VMXReadVMCS(VMX_VMCS_GUEST_CR0, &val2);
4837 Log(("VMX_VMCS_GUEST_CR0 %RX64\n", (uint64_t)val2));
4838
4839 VMXReadVMCS(VMX_VMCS_GUEST_CR3, &val2);
4840 Log(("VMX_VMCS_GUEST_CR3 %RX64\n", (uint64_t)val2));
4841
4842 VMXReadVMCS(VMX_VMCS_GUEST_CR4, &val2);
4843 Log(("VMX_VMCS_GUEST_CR4 %RX64\n", (uint64_t)val2));
4844
4845 VMXReadVMCS(VMX_VMCS_GUEST_RFLAGS, &val2);
4846 Log(("VMX_VMCS_GUEST_RFLAGS %08x\n", val2));
4847
4848 VMX_LOG_SELREG(CS, "CS", val2);
4849 VMX_LOG_SELREG(DS, "DS", val2);
4850 VMX_LOG_SELREG(ES, "ES", val2);
4851 VMX_LOG_SELREG(FS, "FS", val2);
4852 VMX_LOG_SELREG(GS, "GS", val2);
4853 VMX_LOG_SELREG(SS, "SS", val2);
4854 VMX_LOG_SELREG(TR, "TR", val2);
4855 VMX_LOG_SELREG(LDTR, "LDTR", val2);
4856
4857 VMXReadVMCS(VMX_VMCS_GUEST_GDTR_BASE, &val2);
4858 Log(("VMX_VMCS_GUEST_GDTR_BASE %RX64\n", (uint64_t)val2));
4859 VMXReadVMCS(VMX_VMCS_GUEST_IDTR_BASE, &val2);
4860 Log(("VMX_VMCS_GUEST_IDTR_BASE %RX64\n", (uint64_t)val2));
4861#endif /* VBOX_STRICT */
4862 rc = VERR_VMX_INVALID_GUEST_STATE;
4863 break;
4864 }
4865
4866 case VMX_EXIT_ERR_MSR_LOAD: /* 34 VM-entry failure due to MSR loading. */
4867 case VMX_EXIT_ERR_MACHINE_CHECK: /* 41 VM-entry failure due to machine-check. */
4868 default:
4869 rc = VERR_VMX_UNEXPECTED_EXIT_CODE;
4870 AssertMsgFailed(("Unexpected exit code %d\n", exitReason)); /* Can't happen. */
4871 break;
4872
4873 }
4874
4875end:
4876 /* We now going back to ring-3, so clear the action flag. */
4877 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
4878
4879 /*
4880 * Signal changes for the recompiler.
4881 */
4882 CPUMSetChangedFlags(pVCpu,
4883 CPUM_CHANGED_SYSENTER_MSR
4884 | CPUM_CHANGED_LDTR
4885 | CPUM_CHANGED_GDTR
4886 | CPUM_CHANGED_IDTR
4887 | CPUM_CHANGED_TR
4888 | CPUM_CHANGED_HIDDEN_SEL_REGS);
4889
4890 /*
4891 * If we executed vmlaunch/vmresume and an external IRQ was pending, then we don't have to do a full sync the next time.
4892 */
4893 if ( exitReason == VMX_EXIT_EXTERNAL_IRQ
4894 && !VMX_EXIT_INTERRUPTION_INFO_VALID(intInfo))
4895 {
4896 STAM_COUNTER_INC(&pVCpu->hm.s.StatPendingHostIrq);
4897 /* On the next entry we'll only sync the host context. */
4898 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_HOST_CONTEXT;
4899 }
4900 else
4901 {
4902 /* On the next entry we'll sync everything. */
4903 /** @todo we can do better than this */
4904 /* Not in the VINF_PGM_CHANGE_MODE though! */
4905 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_ALL;
4906 }
4907
4908 /* Translate into a less severe return code */
4909 if (rc == VERR_EM_INTERPRETER)
4910 rc = VINF_EM_RAW_EMULATE_INSTR;
4911 else if (rc == VERR_VMX_INVALID_VMCS_PTR)
4912 {
4913 /* Try to extract more information about what might have gone wrong here. */
4914 VMXGetActivateVMCS(&pVCpu->hm.s.vmx.lasterror.u64VMCSPhys);
4915 pVCpu->hm.s.vmx.lasterror.ulVMCSRevision = *(uint32_t *)pVCpu->hm.s.vmx.pvVMCS;
4916 pVCpu->hm.s.vmx.lasterror.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
4917 pVCpu->hm.s.vmx.lasterror.idCurrentCpu = RTMpCpuId();
4918 }
4919
4920 /* Just set the correct state here instead of trying to catch every goto above. */
4921 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED, VMCPUSTATE_STARTED_EXEC);
4922
4923#ifdef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
4924 /* Restore interrupts if we exited after disabling them. */
4925 if (uOldEFlags != ~(RTCCUINTREG)0)
4926 ASMSetFlags(uOldEFlags);
4927#endif
4928
4929 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
4930 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
4931 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
4932 Log2(("X"));
4933 return VBOXSTRICTRC_TODO(rc);
4934}
4935
4936
4937/**
4938 * Enters the VT-x session.
4939 *
4940 * @returns VBox status code.
4941 * @param pVM Pointer to the VM.
4942 * @param pVCpu Pointer to the VMCPU.
4943 * @param pCpu Pointer to the CPU info struct.
4944 */
4945VMMR0DECL(int) VMXR0Enter(PVM pVM, PVMCPU pVCpu, PHMGLOBLCPUINFO pCpu)
4946{
4947 Assert(pVM->hm.s.vmx.fSupported);
4948 NOREF(pCpu);
4949
4950 unsigned cr4 = ASMGetCR4();
4951 if (!(cr4 & X86_CR4_VMXE))
4952 {
4953 AssertMsgFailed(("X86_CR4_VMXE should be set!\n"));
4954 return VERR_VMX_X86_CR4_VMXE_CLEARED;
4955 }
4956
4957 /* Activate the VMCS. */
4958 int rc = VMXActivateVMCS(pVCpu->hm.s.vmx.HCPhysVMCS);
4959 if (RT_FAILURE(rc))
4960 return rc;
4961
4962 pVCpu->hm.s.fResumeVM = false;
4963 return VINF_SUCCESS;
4964}
4965
4966
4967/**
4968 * Leaves the VT-x session.
4969 *
4970 * @returns VBox status code.
4971 * @param pVM Pointer to the VM.
4972 * @param pVCpu Pointer to the VMCPU.
4973 * @param pCtx Pointer to the guests CPU context.
4974 */
4975VMMR0DECL(int) VMXR0Leave(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
4976{
4977 Assert(pVM->hm.s.vmx.fSupported);
4978
4979#ifdef DEBUG
4980 if (CPUMIsHyperDebugStateActive(pVCpu))
4981 {
4982 CPUMR0LoadHostDebugState(pVM, pVCpu);
4983 Assert(pVCpu->hm.s.vmx.proc_ctls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT);
4984 }
4985 else
4986#endif
4987
4988 /*
4989 * Save the guest debug state if necessary.
4990 */
4991 if (CPUMIsGuestDebugStateActive(pVCpu))
4992 {
4993 CPUMR0SaveGuestDebugState(pVM, pVCpu, pCtx, true /* save DR6 */);
4994
4995 /* Enable DRx move intercepts again. */
4996 pVCpu->hm.s.vmx.proc_ctls |= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT;
4997 int rc = VMXWriteVMCS(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.proc_ctls);
4998 AssertRC(rc);
4999
5000 /* Resync the debug registers the next time. */
5001 pVCpu->hm.s.fContextUseFlags |= HM_CHANGED_GUEST_DEBUG;
5002 }
5003 else
5004 Assert(pVCpu->hm.s.vmx.proc_ctls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT);
5005
5006 /*
5007 * Clear VMCS, marking it inactive, clearing implementation-specific data and writing
5008 * VMCS data back to memory.
5009 */
5010 int rc = VMXClearVMCS(pVCpu->hm.s.vmx.HCPhysVMCS);
5011 AssertRC(rc);
5012
5013 return VINF_SUCCESS;
5014}
5015
5016
5017/**
5018 * Flush the TLB using EPT.
5019 *
5020 * @returns VBox status code.
5021 * @param pVM Pointer to the VM.
5022 * @param pVCpu Pointer to the VMCPU.
5023 * @param enmFlush Type of flush.
5024 */
5025static void hmR0VmxFlushEPT(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_EPT enmFlush)
5026{
5027 uint64_t descriptor[2];
5028
5029 LogFlow(("hmR0VmxFlushEPT %d\n", enmFlush));
5030 Assert(pVM->hm.s.fNestedPaging);
5031 descriptor[0] = pVCpu->hm.s.vmx.GCPhysEPTP;
5032 descriptor[1] = 0; /* MBZ. Intel spec. 33.3 VMX Instructions */
5033 int rc = VMXR0InvEPT(enmFlush, &descriptor[0]);
5034 AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %x %RGv failed with %d\n", enmFlush, pVCpu->hm.s.vmx.GCPhysEPTP, rc));
5035#ifdef VBOX_WITH_STATISTICS
5036 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
5037#endif
5038}
5039
5040
5041/**
5042 * Flush the TLB using VPID.
5043 *
5044 * @returns VBox status code.
5045 * @param pVM Pointer to the VM.
5046 * @param pVCpu Pointer to the VMCPU (can be NULL depending on @a
5047 * enmFlush).
5048 * @param enmFlush Type of flush.
5049 * @param GCPtr Virtual address of the page to flush (can be 0 depending
5050 * on @a enmFlush).
5051 */
5052static void hmR0VmxFlushVPID(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_VPID enmFlush, RTGCPTR GCPtr)
5053{
5054 uint64_t descriptor[2];
5055
5056 Assert(pVM->hm.s.vmx.fVpid);
5057 if (enmFlush == VMX_FLUSH_VPID_ALL_CONTEXTS)
5058 {
5059 descriptor[0] = 0;
5060 descriptor[1] = 0;
5061 }
5062 else
5063 {
5064 AssertPtr(pVCpu);
5065 AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
5066 AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
5067 descriptor[0] = pVCpu->hm.s.uCurrentAsid;
5068 descriptor[1] = GCPtr;
5069 }
5070 int rc = VMXR0InvVPID(enmFlush, &descriptor[0]); NOREF(rc);
5071 AssertMsg(rc == VINF_SUCCESS,
5072 ("VMXR0InvVPID %x %x %RGv failed with %d\n", enmFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
5073#ifdef VBOX_WITH_STATISTICS
5074 if (pVCpu)
5075 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
5076#endif
5077}
5078
5079
5080/**
5081 * Invalidates a guest page by guest virtual address. Only relevant for
5082 * EPT/VPID, otherwise there is nothing really to invalidate.
5083 *
5084 * @returns VBox status code.
5085 * @param pVM Pointer to the VM.
5086 * @param pVCpu Pointer to the VMCPU.
5087 * @param GCVirt Guest virtual address of the page to invalidate.
5088 */
5089VMMR0DECL(int) VMXR0InvalidatePage(PVM pVM, PVMCPU pVCpu, RTGCPTR GCVirt)
5090{
5091 bool fFlushPending = VMCPU_FF_ISSET(pVCpu, VMCPU_FF_TLB_FLUSH);
5092
5093 Log2(("VMXR0InvalidatePage %RGv\n", GCVirt));
5094
5095 if (!fFlushPending)
5096 {
5097 /*
5098 * We must invalidate the guest TLB entry in either case, we cannot ignore it even for the EPT case
5099 * See @bugref{6043} and @bugref{6177}
5100 *
5101 * Set the VMCPU_FF_TLB_FLUSH force flag and flush before VMENTRY in hmR0VmxSetupTLB*() as this
5102 * function maybe called in a loop with individual addresses.
5103 */
5104 if (pVM->hm.s.vmx.fVpid)
5105 {
5106 /* If we can flush just this page do it, otherwise flush as little as possible. */
5107 if (pVM->hm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVVPID_CAPS_INDIV_ADDR)
5108 hmR0VmxFlushVPID(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, GCVirt);
5109 else
5110 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
5111 }
5112 else if (pVM->hm.s.fNestedPaging)
5113 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
5114 }
5115
5116 return VINF_SUCCESS;
5117}
5118
5119
5120/**
5121 * Invalidates a guest page by physical address. Only relevant for EPT/VPID,
5122 * otherwise there is nothing really to invalidate.
5123 *
5124 * NOTE: Assumes the current instruction references this physical page though a virtual address!!
5125 *
5126 * @returns VBox status code.
5127 * @param pVM Pointer to the VM.
5128 * @param pVCpu Pointer to the VMCPU.
5129 * @param GCPhys Guest physical address of the page to invalidate.
5130 */
5131VMMR0DECL(int) VMXR0InvalidatePhysPage(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys)
5132{
5133 LogFlow(("VMXR0InvalidatePhysPage %RGp\n", GCPhys));
5134
5135 /*
5136 * We cannot flush a page by guest-physical address. invvpid takes only a linear address
5137 * while invept only flushes by EPT not individual addresses. We update the force flag here
5138 * and flush before VMENTRY in hmR0VmxSetupTLB*(). This function might be called in a loop.
5139 */
5140 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
5141 return VINF_SUCCESS;
5142}
5143
5144
5145/**
5146 * Report world switch error and dump some useful debug info.
5147 *
5148 * @param pVM Pointer to the VM.
5149 * @param pVCpu Pointer to the VMCPU.
5150 * @param rc Return code.
5151 * @param pCtx Pointer to the current guest CPU context (not updated).
5152 */
5153static void hmR0VmxReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, VBOXSTRICTRC rc, PCPUMCTX pCtx)
5154{
5155 NOREF(pVM);
5156
5157 switch (VBOXSTRICTRC_VAL(rc))
5158 {
5159 case VERR_VMX_INVALID_VMXON_PTR:
5160 AssertFailed();
5161 break;
5162
5163 case VERR_VMX_UNABLE_TO_START_VM:
5164 case VERR_VMX_UNABLE_TO_RESUME_VM:
5165 {
5166 int rc2;
5167 RTCCUINTREG exitReason, instrError;
5168
5169 rc2 = VMXReadVMCS(VMX_VMCS32_RO_EXIT_REASON, &exitReason);
5170 rc2 |= VMXReadVMCS(VMX_VMCS32_RO_VM_INSTR_ERROR, &instrError);
5171 AssertRC(rc2);
5172 if (rc2 == VINF_SUCCESS)
5173 {
5174 Log(("Unable to start/resume VM for reason: %x. Instruction error %x\n", (uint32_t)exitReason,
5175 (uint32_t)instrError));
5176 Log(("Current stack %08x\n", &rc2));
5177
5178 pVCpu->hm.s.vmx.lasterror.ulInstrError = instrError;
5179 pVCpu->hm.s.vmx.lasterror.ulExitReason = exitReason;
5180
5181#ifdef VBOX_STRICT
5182 RTGDTR gdtr;
5183 PCX86DESCHC pDesc;
5184 RTCCUINTREG val;
5185
5186 ASMGetGDTR(&gdtr);
5187
5188 VMXReadVMCS(VMX_VMCS_GUEST_RIP, &val);
5189 Log(("Old eip %RGv new %RGv\n", (RTGCPTR)pCtx->rip, (RTGCPTR)val));
5190 VMXReadVMCS(VMX_VMCS32_CTRL_PIN_EXEC_CONTROLS, &val);
5191 Log(("VMX_VMCS_CTRL_PIN_EXEC_CONTROLS %08x\n", val));
5192 VMXReadVMCS(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, &val);
5193 Log(("VMX_VMCS_CTRL_PROC_EXEC_CONTROLS %08x\n", val));
5194 VMXReadVMCS(VMX_VMCS32_CTRL_ENTRY_CONTROLS, &val);
5195 Log(("VMX_VMCS_CTRL_ENTRY_CONTROLS %08x\n", val));
5196 VMXReadVMCS(VMX_VMCS32_CTRL_EXIT_CONTROLS, &val);
5197 Log(("VMX_VMCS_CTRL_EXIT_CONTROLS %08x\n", val));
5198
5199 VMXReadVMCS(VMX_VMCS_HOST_CR0, &val);
5200 Log(("VMX_VMCS_HOST_CR0 %08x\n", val));
5201 VMXReadVMCS(VMX_VMCS_HOST_CR3, &val);
5202 Log(("VMX_VMCS_HOST_CR3 %08x\n", val));
5203 VMXReadVMCS(VMX_VMCS_HOST_CR4, &val);
5204 Log(("VMX_VMCS_HOST_CR4 %08x\n", val));
5205
5206 VMXReadVMCS(VMX_VMCS16_HOST_FIELD_CS, &val);
5207 Log(("VMX_VMCS_HOST_FIELD_CS %08x\n", val));
5208 VMXReadVMCS(VMX_VMCS_GUEST_RFLAGS, &val);
5209 Log(("VMX_VMCS_GUEST_RFLAGS %08x\n", val));
5210
5211 if (val < gdtr.cbGdt)
5212 {
5213 pDesc = (PCX86DESCHC)(gdtr.pGdt + (val & X86_SEL_MASK));
5214 HMR0DumpDescriptor(pDesc, val, "CS: ");
5215 }
5216
5217 VMXReadVMCS(VMX_VMCS16_HOST_FIELD_DS, &val);
5218 Log(("VMX_VMCS_HOST_FIELD_DS %08x\n", val));
5219 if (val < gdtr.cbGdt)
5220 {
5221 pDesc = (PCX86DESCHC)(gdtr.pGdt + (val & X86_SEL_MASK));
5222 HMR0DumpDescriptor(pDesc, val, "DS: ");
5223 }
5224
5225 VMXReadVMCS(VMX_VMCS16_HOST_FIELD_ES, &val);
5226 Log(("VMX_VMCS_HOST_FIELD_ES %08x\n", val));
5227 if (val < gdtr.cbGdt)
5228 {
5229 pDesc = (PCX86DESCHC)(gdtr.pGdt + (val & X86_SEL_MASK));
5230 HMR0DumpDescriptor(pDesc, val, "ES: ");
5231 }
5232
5233 VMXReadVMCS(VMX_VMCS16_HOST_FIELD_FS, &val);
5234 Log(("VMX_VMCS16_HOST_FIELD_FS %08x\n", val));
5235 if (val < gdtr.cbGdt)
5236 {
5237 pDesc = (PCX86DESCHC)(gdtr.pGdt + (val & X86_SEL_MASK));
5238 HMR0DumpDescriptor(pDesc, val, "FS: ");
5239 }
5240
5241 VMXReadVMCS(VMX_VMCS16_HOST_FIELD_GS, &val);
5242 Log(("VMX_VMCS16_HOST_FIELD_GS %08x\n", val));
5243 if (val < gdtr.cbGdt)
5244 {
5245 pDesc = (PCX86DESCHC)(gdtr.pGdt + (val & X86_SEL_MASK));
5246 HMR0DumpDescriptor(pDesc, val, "GS: ");
5247 }
5248
5249 VMXReadVMCS(VMX_VMCS16_HOST_FIELD_SS, &val);
5250 Log(("VMX_VMCS16_HOST_FIELD_SS %08x\n", val));
5251 if (val < gdtr.cbGdt)
5252 {
5253 pDesc = (PCX86DESCHC)(gdtr.pGdt + (val & X86_SEL_MASK));
5254 HMR0DumpDescriptor(pDesc, val, "SS: ");
5255 }
5256
5257 VMXReadVMCS(VMX_VMCS16_HOST_FIELD_TR, &val);
5258 Log(("VMX_VMCS16_HOST_FIELD_TR %08x\n", val));
5259 if (val < gdtr.cbGdt)
5260 {
5261 pDesc = (PCX86DESCHC)(gdtr.pGdt + (val & X86_SEL_MASK));
5262 HMR0DumpDescriptor(pDesc, val, "TR: ");
5263 }
5264
5265 VMXReadVMCS(VMX_VMCS_HOST_TR_BASE, &val);
5266 Log(("VMX_VMCS_HOST_TR_BASE %RHv\n", val));
5267 VMXReadVMCS(VMX_VMCS_HOST_GDTR_BASE, &val);
5268 Log(("VMX_VMCS_HOST_GDTR_BASE %RHv\n", val));
5269 VMXReadVMCS(VMX_VMCS_HOST_IDTR_BASE, &val);
5270 Log(("VMX_VMCS_HOST_IDTR_BASE %RHv\n", val));
5271 VMXReadVMCS(VMX_VMCS32_HOST_SYSENTER_CS, &val);
5272 Log(("VMX_VMCS_HOST_SYSENTER_CS %08x\n", val));
5273 VMXReadVMCS(VMX_VMCS_HOST_SYSENTER_EIP, &val);
5274 Log(("VMX_VMCS_HOST_SYSENTER_EIP %RHv\n", val));
5275 VMXReadVMCS(VMX_VMCS_HOST_SYSENTER_ESP, &val);
5276 Log(("VMX_VMCS_HOST_SYSENTER_ESP %RHv\n", val));
5277 VMXReadVMCS(VMX_VMCS_HOST_RSP, &val);
5278 Log(("VMX_VMCS_HOST_RSP %RHv\n", val));
5279 VMXReadVMCS(VMX_VMCS_HOST_RIP, &val);
5280 Log(("VMX_VMCS_HOST_RIP %RHv\n", val));
5281# if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
5282 if (VMX_IS_64BIT_HOST_MODE())
5283 {
5284 Log(("MSR_K6_EFER = %RX64\n", ASMRdMsr(MSR_K6_EFER)));
5285 Log(("MSR_K6_STAR = %RX64\n", ASMRdMsr(MSR_K6_STAR)));
5286 Log(("MSR_K8_LSTAR = %RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
5287 Log(("MSR_K8_CSTAR = %RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
5288 Log(("MSR_K8_SF_MASK = %RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
5289 Log(("MSR_K8_KERNEL_GS_BASE = %RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
5290 }
5291# endif
5292#endif /* VBOX_STRICT */
5293 }
5294 break;
5295 }
5296
5297 default:
5298 /* impossible */
5299 AssertMsgFailed(("%Rrc (%#x)\n", VBOXSTRICTRC_VAL(rc), VBOXSTRICTRC_VAL(rc)));
5300 break;
5301 }
5302}
5303
5304
5305#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
5306/**
5307 * Prepares for and executes VMLAUNCH (64 bits guest mode).
5308 *
5309 * @returns VBox status code.
5310 * @param fResume Whether to vmlauch/vmresume.
5311 * @param pCtx Pointer to the guest CPU context.
5312 * @param pCache Pointer to the VMCS cache.
5313 * @param pVM Pointer to the VM.
5314 * @param pVCpu Pointer to the VMCPU.
5315 */
5316DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
5317{
5318 uint32_t aParam[6];
5319 PHMGLOBLCPUINFO pCpu;
5320 RTHCPHYS HCPhysCpuPage;
5321 int rc;
5322
5323 pCpu = HMR0GetCurrentCpu();
5324 HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
5325
5326#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5327 pCache->uPos = 1;
5328 pCache->interPD = PGMGetInterPaeCR3(pVM);
5329 pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
5330#endif
5331
5332#ifdef DEBUG
5333 pCache->TestIn.HCPhysCpuPage= 0;
5334 pCache->TestIn.HCPhysVMCS = 0;
5335 pCache->TestIn.pCache = 0;
5336 pCache->TestOut.HCPhysVMCS = 0;
5337 pCache->TestOut.pCache = 0;
5338 pCache->TestOut.pCtx = 0;
5339 pCache->TestOut.eflags = 0;
5340#endif
5341
5342 aParam[0] = (uint32_t)(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
5343 aParam[1] = (uint32_t)(HCPhysCpuPage >> 32); /* Param 1: VMXON physical address - Hi. */
5344 aParam[2] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVMCS); /* Param 2: VMCS physical address - Lo. */
5345 aParam[3] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVMCS >> 32); /* Param 2: VMCS physical address - Hi. */
5346 aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache);
5347 aParam[5] = 0;
5348
5349#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5350 pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
5351 *(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
5352#endif
5353 rc = VMXR0Execute64BitsHandler(pVM, pVCpu, pCtx, pVM->hm.s.pfnVMXGCStartVM64, 6, &aParam[0]);
5354
5355#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5356 Assert(*(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
5357 Assert(pCtx->dr[4] == 10);
5358 *(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
5359#endif
5360
5361#ifdef DEBUG
5362 AssertMsg(pCache->TestIn.HCPhysCpuPage== HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
5363 AssertMsg(pCache->TestIn.HCPhysVMCS == pVCpu->hm.s.vmx.HCPhysVMCS, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVMCS,
5364 pVCpu->hm.s.vmx.HCPhysVMCS));
5365 AssertMsg(pCache->TestIn.HCPhysVMCS == pCache->TestOut.HCPhysVMCS, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVMCS,
5366 pCache->TestOut.HCPhysVMCS));
5367 AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
5368 pCache->TestOut.pCache));
5369 AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache),
5370 ("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache)));
5371 AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
5372 pCache->TestOut.pCtx));
5373 Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
5374#endif
5375 return rc;
5376}
5377
5378
5379# ifdef VBOX_STRICT
5380static bool hmR0VmxIsValidReadField(uint32_t idxField)
5381{
5382 switch (idxField)
5383 {
5384 case VMX_VMCS_GUEST_RIP:
5385 case VMX_VMCS_GUEST_RSP:
5386 case VMX_VMCS_GUEST_RFLAGS:
5387 case VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE:
5388 case VMX_VMCS_CTRL_CR0_READ_SHADOW:
5389 case VMX_VMCS_GUEST_CR0:
5390 case VMX_VMCS_CTRL_CR4_READ_SHADOW:
5391 case VMX_VMCS_GUEST_CR4:
5392 case VMX_VMCS_GUEST_DR7:
5393 case VMX_VMCS32_GUEST_SYSENTER_CS:
5394 case VMX_VMCS_GUEST_SYSENTER_EIP:
5395 case VMX_VMCS_GUEST_SYSENTER_ESP:
5396 case VMX_VMCS32_GUEST_GDTR_LIMIT:
5397 case VMX_VMCS_GUEST_GDTR_BASE:
5398 case VMX_VMCS32_GUEST_IDTR_LIMIT:
5399 case VMX_VMCS_GUEST_IDTR_BASE:
5400 case VMX_VMCS16_GUEST_FIELD_CS:
5401 case VMX_VMCS32_GUEST_CS_LIMIT:
5402 case VMX_VMCS_GUEST_CS_BASE:
5403 case VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS:
5404 case VMX_VMCS16_GUEST_FIELD_DS:
5405 case VMX_VMCS32_GUEST_DS_LIMIT:
5406 case VMX_VMCS_GUEST_DS_BASE:
5407 case VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS:
5408 case VMX_VMCS16_GUEST_FIELD_ES:
5409 case VMX_VMCS32_GUEST_ES_LIMIT:
5410 case VMX_VMCS_GUEST_ES_BASE:
5411 case VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS:
5412 case VMX_VMCS16_GUEST_FIELD_FS:
5413 case VMX_VMCS32_GUEST_FS_LIMIT:
5414 case VMX_VMCS_GUEST_FS_BASE:
5415 case VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS:
5416 case VMX_VMCS16_GUEST_FIELD_GS:
5417 case VMX_VMCS32_GUEST_GS_LIMIT:
5418 case VMX_VMCS_GUEST_GS_BASE:
5419 case VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS:
5420 case VMX_VMCS16_GUEST_FIELD_SS:
5421 case VMX_VMCS32_GUEST_SS_LIMIT:
5422 case VMX_VMCS_GUEST_SS_BASE:
5423 case VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS:
5424 case VMX_VMCS16_GUEST_FIELD_LDTR:
5425 case VMX_VMCS32_GUEST_LDTR_LIMIT:
5426 case VMX_VMCS_GUEST_LDTR_BASE:
5427 case VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS:
5428 case VMX_VMCS16_GUEST_FIELD_TR:
5429 case VMX_VMCS32_GUEST_TR_LIMIT:
5430 case VMX_VMCS_GUEST_TR_BASE:
5431 case VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS:
5432 case VMX_VMCS32_RO_EXIT_REASON:
5433 case VMX_VMCS32_RO_VM_INSTR_ERROR:
5434 case VMX_VMCS32_RO_EXIT_INSTR_LENGTH:
5435 case VMX_VMCS32_RO_EXIT_INTERRUPTION_ERRCODE:
5436 case VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO:
5437 case VMX_VMCS32_RO_EXIT_INSTR_INFO:
5438 case VMX_VMCS_RO_EXIT_QUALIFICATION:
5439 case VMX_VMCS32_RO_IDT_INFO:
5440 case VMX_VMCS32_RO_IDT_ERRCODE:
5441 case VMX_VMCS_GUEST_CR3:
5442 case VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL:
5443 return true;
5444 }
5445 return false;
5446}
5447
5448
5449static bool hmR0VmxIsValidWriteField(uint32_t idxField)
5450{
5451 switch (idxField)
5452 {
5453 case VMX_VMCS_GUEST_LDTR_BASE:
5454 case VMX_VMCS_GUEST_TR_BASE:
5455 case VMX_VMCS_GUEST_GDTR_BASE:
5456 case VMX_VMCS_GUEST_IDTR_BASE:
5457 case VMX_VMCS_GUEST_SYSENTER_EIP:
5458 case VMX_VMCS_GUEST_SYSENTER_ESP:
5459 case VMX_VMCS_GUEST_CR0:
5460 case VMX_VMCS_GUEST_CR4:
5461 case VMX_VMCS_GUEST_CR3:
5462 case VMX_VMCS_GUEST_DR7:
5463 case VMX_VMCS_GUEST_RIP:
5464 case VMX_VMCS_GUEST_RSP:
5465 case VMX_VMCS_GUEST_CS_BASE:
5466 case VMX_VMCS_GUEST_DS_BASE:
5467 case VMX_VMCS_GUEST_ES_BASE:
5468 case VMX_VMCS_GUEST_FS_BASE:
5469 case VMX_VMCS_GUEST_GS_BASE:
5470 case VMX_VMCS_GUEST_SS_BASE:
5471 return true;
5472 }
5473 return false;
5474}
5475# endif /* VBOX_STRICT */
5476
5477
5478/**
5479 * Executes the specified handler in 64-bit mode.
5480 *
5481 * @returns VBox status code.
5482 * @param pVM Pointer to the VM.
5483 * @param pVCpu Pointer to the VMCPU.
5484 * @param pCtx Pointer to the guest CPU context.
5485 * @param pfnHandler Pointer to the RC handler function.
5486 * @param cbParam Number of parameters.
5487 * @param paParam Array of 32-bit parameters.
5488 */
5489VMMR0DECL(int) VMXR0Execute64BitsHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, RTRCPTR pfnHandler, uint32_t cbParam,
5490 uint32_t *paParam)
5491{
5492 int rc, rc2;
5493 PHMGLOBLCPUINFO pCpu;
5494 RTHCPHYS HCPhysCpuPage;
5495 RTHCUINTREG uOldEFlags;
5496
5497 AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
5498 Assert(pfnHandler);
5499 Assert(pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Write.aField));
5500 Assert(pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Read.aField));
5501
5502#ifdef VBOX_STRICT
5503 for (unsigned i=0;i<pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries;i++)
5504 Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VMCSCache.Write.aField[i]));
5505
5506 for (unsigned i=0;i<pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries;i++)
5507 Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VMCSCache.Read.aField[i]));
5508#endif
5509
5510 /* Disable interrupts. */
5511 uOldEFlags = ASMIntDisableFlags();
5512
5513#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
5514 RTCPUID idHostCpu = RTMpCpuId();
5515 CPUMR0SetLApic(pVM, idHostCpu);
5516#endif
5517
5518 pCpu = HMR0GetCurrentCpu();
5519 HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
5520
5521 /* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
5522 VMXClearVMCS(pVCpu->hm.s.vmx.HCPhysVMCS);
5523
5524 /* Leave VMX Root Mode. */
5525 VMXDisable();
5526
5527 ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
5528
5529 CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
5530 CPUMSetHyperEIP(pVCpu, pfnHandler);
5531 for (int i=(int)cbParam-1;i>=0;i--)
5532 CPUMPushHyper(pVCpu, paParam[i]);
5533
5534 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
5535
5536 /* Call switcher. */
5537 rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_OFFSETOF(VM, aCpus[pVCpu->idCpu].cpum) - RT_OFFSETOF(VM, cpum));
5538 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
5539
5540 /* Make sure the VMX instructions don't cause #UD faults. */
5541 ASMSetCR4(ASMGetCR4() | X86_CR4_VMXE);
5542
5543 /* Enter VMX Root Mode */
5544 rc2 = VMXEnable(HCPhysCpuPage);
5545 if (RT_FAILURE(rc2))
5546 {
5547 ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
5548 ASMSetFlags(uOldEFlags);
5549 return VERR_VMX_VMXON_FAILED;
5550 }
5551
5552 rc2 = VMXActivateVMCS(pVCpu->hm.s.vmx.HCPhysVMCS);
5553 AssertRC(rc2);
5554 Assert(!(ASMGetFlags() & X86_EFL_IF));
5555 ASMSetFlags(uOldEFlags);
5556 return rc;
5557}
5558#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL) */
5559
5560
5561#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
5562/**
5563 * Executes VMWRITE.
5564 *
5565 * @returns VBox status code
5566 * @param pVCpu Pointer to the VMCPU.
5567 * @param idxField VMCS field index.
5568 * @param u64Val 16, 32 or 64 bits value.
5569 */
5570VMMR0DECL(int) VMXWriteVMCS64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5571{
5572 int rc;
5573 switch (idxField)
5574 {
5575 case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
5576 case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
5577 case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
5578 case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
5579 case VMX_VMCS64_CTRL_VMEXIT_MSR_STORE_FULL:
5580 case VMX_VMCS64_CTRL_VMEXIT_MSR_LOAD_FULL:
5581 case VMX_VMCS64_CTRL_VMENTRY_MSR_LOAD_FULL:
5582 case VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL:
5583 case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
5584 case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
5585 case VMX_VMCS64_GUEST_PDPTE0_FULL:
5586 case VMX_VMCS64_GUEST_PDPTE1_FULL:
5587 case VMX_VMCS64_GUEST_PDPTE2_FULL:
5588 case VMX_VMCS64_GUEST_PDPTE3_FULL:
5589 case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
5590 case VMX_VMCS64_GUEST_EFER_FULL:
5591 case VMX_VMCS64_CTRL_EPTP_FULL:
5592 /* These fields consist of two parts, which are both writable in 32 bits mode. */
5593 rc = VMXWriteVMCS32(idxField, u64Val);
5594 rc |= VMXWriteVMCS32(idxField + 1, (uint32_t)(u64Val >> 32ULL));
5595 AssertRC(rc);
5596 return rc;
5597
5598 case VMX_VMCS_GUEST_LDTR_BASE:
5599 case VMX_VMCS_GUEST_TR_BASE:
5600 case VMX_VMCS_GUEST_GDTR_BASE:
5601 case VMX_VMCS_GUEST_IDTR_BASE:
5602 case VMX_VMCS_GUEST_SYSENTER_EIP:
5603 case VMX_VMCS_GUEST_SYSENTER_ESP:
5604 case VMX_VMCS_GUEST_CR0:
5605 case VMX_VMCS_GUEST_CR4:
5606 case VMX_VMCS_GUEST_CR3:
5607 case VMX_VMCS_GUEST_DR7:
5608 case VMX_VMCS_GUEST_RIP:
5609 case VMX_VMCS_GUEST_RSP:
5610 case VMX_VMCS_GUEST_CS_BASE:
5611 case VMX_VMCS_GUEST_DS_BASE:
5612 case VMX_VMCS_GUEST_ES_BASE:
5613 case VMX_VMCS_GUEST_FS_BASE:
5614 case VMX_VMCS_GUEST_GS_BASE:
5615 case VMX_VMCS_GUEST_SS_BASE:
5616 /* Queue a 64 bits value as we can't set it in 32 bits host mode. */
5617 if (u64Val >> 32ULL)
5618 rc = VMXWriteCachedVMCSEx(pVCpu, idxField, u64Val);
5619 else
5620 rc = VMXWriteVMCS32(idxField, (uint32_t)u64Val);
5621
5622 return rc;
5623
5624 default:
5625 AssertMsgFailed(("Unexpected field %x\n", idxField));
5626 return VERR_INVALID_PARAMETER;
5627 }
5628}
5629
5630
5631/**
5632 * Cache VMCS writes for running 64 bits guests on 32 bits hosts.
5633 *
5634 * @param pVCpu Pointer to the VMCPU.
5635 * @param idxField VMCS field index.
5636 * @param u64Val 16, 32 or 64 bits value.
5637 */
5638VMMR0DECL(int) VMXWriteCachedVMCSEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5639{
5640 PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5641
5642 AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1,
5643 ("entries=%x\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
5644
5645 /* Make sure there are no duplicates. */
5646 for (unsigned i = 0; i < pCache->Write.cValidEntries; i++)
5647 {
5648 if (pCache->Write.aField[i] == idxField)
5649 {
5650 pCache->Write.aFieldVal[i] = u64Val;
5651 return VINF_SUCCESS;
5652 }
5653 }
5654
5655 pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
5656 pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
5657 pCache->Write.cValidEntries++;
5658 return VINF_SUCCESS;
5659}
5660
5661#endif /* HC_ARCH_BITS == 32 && !VBOX_WITH_HYBRID_32BIT_KERNEL */
5662
Note: See TracBrowser for help on using the repository browser.

© 2024 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette