VirtualBox

source: vbox/trunk/src/VBox/VMM/VMMR0/HWVMXR0.cpp@ 42186

Last change on this file since 42186 was 42186, checked in by vboxsync, 13 years ago

SELM,DIS,CPUM,EM: Hidden selector register cleanups.
  • Property svn:eol-style set to native
  • Property svn:keywords set to Id Revision
File size: 220.4 KB
Line 
1/* $Id: HWVMXR0.cpp 42186 2012-07-17 13:32:15Z vboxsync $ */
2/** @file
3 * HM VMX (VT-x) - Host Context Ring-0.
4 */
5
6/*
7 * Copyright (C) 2006-2012 Oracle Corporation
8 *
9 * This file is part of VirtualBox Open Source Edition (OSE), as
10 * available from http://www.virtualbox.org. This file is free software;
11 * you can redistribute it and/or modify it under the terms of the GNU
12 * General Public License (GPL) as published by the Free Software
13 * Foundation, in version 2 as it comes in the "COPYING" file of the
14 * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15 * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16 */
17
18
19/*******************************************************************************
20* Header Files *
21*******************************************************************************/
22#define LOG_GROUP LOG_GROUP_HWACCM
23#include <iprt/asm-amd64-x86.h>
24#include <VBox/vmm/hwaccm.h>
25#include <VBox/vmm/pgm.h>
26#include <VBox/vmm/dbgf.h>
27#include <VBox/vmm/dbgftrace.h>
28#include <VBox/vmm/selm.h>
29#include <VBox/vmm/iom.h>
30#ifdef VBOX_WITH_REM
31# include <VBox/vmm/rem.h>
32#endif
33#include <VBox/vmm/tm.h>
34#include "HWACCMInternal.h"
35#include <VBox/vmm/vm.h>
36#include <VBox/vmm/pdmapi.h>
37#include <VBox/err.h>
38#include <VBox/log.h>
39#include <iprt/assert.h>
40#include <iprt/param.h>
41#include <iprt/string.h>
42#include <iprt/time.h>
43#ifdef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
44# include <iprt/thread.h>
45#endif
46#include <iprt/x86.h>
47#include "HWVMXR0.h"
48
49#include "dtrace/VBoxVMM.h"
50
51
52/*******************************************************************************
53* Defined Constants And Macros *
54*******************************************************************************/
55#if defined(RT_ARCH_AMD64)
56# define VMX_IS_64BIT_HOST_MODE() (true)
57#elif defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
58# define VMX_IS_64BIT_HOST_MODE() (g_fVMXIs64bitHost != 0)
59#else
60# define VMX_IS_64BIT_HOST_MODE() (false)
61#endif
62
63
64/*******************************************************************************
65* Global Variables *
66*******************************************************************************/
67/* IO operation lookup arrays. */
68static uint32_t const g_aIOSize[4] = {1, 2, 0, 4};
69static uint32_t const g_aIOOpAnd[4] = {0xff, 0xffff, 0, 0xffffffff};
70
71#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
72/** See HWACCMR0A.asm. */
73extern "C" uint32_t g_fVMXIs64bitHost;
74#endif
75
76
77/*******************************************************************************
78* Local Functions *
79*******************************************************************************/
80static DECLCALLBACK(void) hmR0VmxSetupTLBEPT(PVM pVM, PVMCPU pVCpu);
81static DECLCALLBACK(void) hmR0VmxSetupTLBVPID(PVM pVM, PVMCPU pVCpu);
82static DECLCALLBACK(void) hmR0VmxSetupTLBBoth(PVM pVM, PVMCPU pVCpu);
83static DECLCALLBACK(void) hmR0VmxSetupTLBDummy(PVM pVM, PVMCPU pVCpu);
84static void hmR0VmxFlushEPT(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_EPT enmFlush);
85static void hmR0VmxFlushVPID(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_VPID enmFlush, RTGCPTR GCPtr);
86static void hmR0VmxUpdateExceptionBitmap(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx);
87static void hmR0VmxSetMSRPermission(PVMCPU pVCpu, unsigned ulMSR, bool fRead, bool fWrite);
88static void hmR0VmxReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, VBOXSTRICTRC rc, PCPUMCTX pCtx);
89
90
91/**
92 * Updates error from VMCS to HWACCMCPU's lasterror record.
93 *
94 * @param pVM Pointer to the VM.
95 * @param pVCpu Pointer to the VMCPU.
96 * @param rc The error code.
97 */
98static void hmR0VmxCheckError(PVM pVM, PVMCPU pVCpu, int rc)
99{
100 if (rc == VERR_VMX_GENERIC)
101 {
102 RTCCUINTREG instrError;
103
104 VMXReadVMCS(VMX_VMCS32_RO_VM_INSTR_ERROR, &instrError);
105 pVCpu->hwaccm.s.vmx.lasterror.ulInstrError = instrError;
106 }
107 pVM->hwaccm.s.lLastError = rc;
108}
109
110
111/**
112 * Sets up and activates VT-x on the current CPU.
113 *
114 * @returns VBox status code.
115 * @param pCpu Pointer to the CPU info struct.
116 * @param pVM Pointer to the VM. (can be NULL after a resume!!)
117 * @param pvCpuPage Pointer to the global CPU page.
118 * @param HCPhysCpuPage Physical address of the global CPU page.
119 */
120VMMR0DECL(int) VMXR0EnableCpu(PHMGLOBLCPUINFO pCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
121{
122 AssertReturn(HCPhysCpuPage != 0 && HCPhysCpuPage != NIL_RTHCPHYS, VERR_INVALID_PARAMETER);
123 AssertReturn(pvCpuPage, VERR_INVALID_PARAMETER);
124
125 if (pVM)
126 {
127 /* Set revision dword at the beginning of the VMXON structure. */
128 *(uint32_t *)pvCpuPage = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hwaccm.s.vmx.msr.vmx_basic_info);
129 }
130
131 /** @todo we should unmap the two pages from the virtual address space in order to prevent accidental corruption.
132 * (which can have very bad consequences!!!)
133 */
134
135 if (ASMGetCR4() & X86_CR4_VMXE)
136 return VERR_VMX_IN_VMX_ROOT_MODE;
137
138 ASMSetCR4(ASMGetCR4() | X86_CR4_VMXE); /* Make sure the VMX instructions don't cause #UD faults. */
139
140 /*
141 * Enter VM root mode.
142 */
143 int rc = VMXEnable(HCPhysCpuPage);
144 if (RT_FAILURE(rc))
145 {
146 ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
147 return VERR_VMX_VMXON_FAILED;
148 }
149
150 /*
151 * Flush all VPIDs (in case we or any other hypervisor have been using VPIDs) so that
152 * we can avoid an explicit flush while using new VPIDs. We would still need to flush
153 * each time while reusing a VPID after hitting the MaxASID limit once.
154 */
155 if ( pVM
156 && pVM->hwaccm.s.vmx.fVPID
157 && (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVVPID_CAPS_ALL_CONTEXTS))
158 {
159 hmR0VmxFlushVPID(pVM, NULL /* pvCpu */, VMX_FLUSH_VPID_ALL_CONTEXTS, 0 /* GCPtr */);
160 pCpu->fFlushASIDBeforeUse = false;
161 }
162 else
163 pCpu->fFlushASIDBeforeUse = true;
164
165 /*
166 * Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}.
167 */
168 ++pCpu->cTLBFlushes;
169
170 return VINF_SUCCESS;
171}
172
173
174/**
175 * Deactivates VT-x on the current CPU.
176 *
177 * @returns VBox status code.
178 * @param pCpu Pointer to the CPU info struct.
179 * @param pvCpuPage Pointer to the global CPU page.
180 * @param HCPhysCpuPage Physical address of the global CPU page.
181 */
182VMMR0DECL(int) VMXR0DisableCpu(PHMGLOBLCPUINFO pCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
183{
184 AssertReturn(HCPhysCpuPage != 0 && HCPhysCpuPage != NIL_RTHCPHYS, VERR_INVALID_PARAMETER);
185 AssertReturn(pvCpuPage, VERR_INVALID_PARAMETER);
186 NOREF(pCpu);
187
188 /* If we're somehow not in VMX root mode, then we shouldn't dare leaving it. */
189 if (!(ASMGetCR4() & X86_CR4_VMXE))
190 return VERR_VMX_NOT_IN_VMX_ROOT_MODE;
191
192 /* Leave VMX Root Mode. */
193 VMXDisable();
194
195 /* And clear the X86_CR4_VMXE bit. */
196 ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
197 return VINF_SUCCESS;
198}
199
200
201/**
202 * Does Ring-0 per VM VT-x initialization.
203 *
204 * @returns VBox status code.
205 * @param pVM Pointer to the VM.
206 */
207VMMR0DECL(int) VMXR0InitVM(PVM pVM)
208{
209 int rc;
210
211#ifdef LOG_ENABLED
212 SUPR0Printf("VMXR0InitVM %p\n", pVM);
213#endif
214
215 pVM->hwaccm.s.vmx.pMemObjAPIC = NIL_RTR0MEMOBJ;
216
217 if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
218 {
219 /* Allocate one page for the APIC physical page (serves for filtering accesses). */
220 rc = RTR0MemObjAllocCont(&pVM->hwaccm.s.vmx.pMemObjAPIC, PAGE_SIZE, true /* executable R0 mapping */);
221 AssertRC(rc);
222 if (RT_FAILURE(rc))
223 return rc;
224
225 pVM->hwaccm.s.vmx.pAPIC = (uint8_t *)RTR0MemObjAddress(pVM->hwaccm.s.vmx.pMemObjAPIC);
226 pVM->hwaccm.s.vmx.pAPICPhys = RTR0MemObjGetPagePhysAddr(pVM->hwaccm.s.vmx.pMemObjAPIC, 0);
227 ASMMemZero32(pVM->hwaccm.s.vmx.pAPIC, PAGE_SIZE);
228 }
229 else
230 {
231 pVM->hwaccm.s.vmx.pMemObjAPIC = 0;
232 pVM->hwaccm.s.vmx.pAPIC = 0;
233 pVM->hwaccm.s.vmx.pAPICPhys = 0;
234 }
235
236#ifdef VBOX_WITH_CRASHDUMP_MAGIC
237 {
238 rc = RTR0MemObjAllocCont(&pVM->hwaccm.s.vmx.pMemObjScratch, PAGE_SIZE, true /* executable R0 mapping */);
239 AssertRC(rc);
240 if (RT_FAILURE(rc))
241 return rc;
242
243 pVM->hwaccm.s.vmx.pScratch = (uint8_t *)RTR0MemObjAddress(pVM->hwaccm.s.vmx.pMemObjScratch);
244 pVM->hwaccm.s.vmx.pScratchPhys = RTR0MemObjGetPagePhysAddr(pVM->hwaccm.s.vmx.pMemObjScratch, 0);
245
246 ASMMemZero32(pVM->hwaccm.s.vmx.pScratch, PAGE_SIZE);
247 strcpy((char *)pVM->hwaccm.s.vmx.pScratch, "SCRATCH Magic");
248 *(uint64_t *)(pVM->hwaccm.s.vmx.pScratch + 16) = UINT64_C(0xDEADBEEFDEADBEEF);
249 }
250#endif
251
252 /* Allocate VMCSs for all guest CPUs. */
253 for (VMCPUID i = 0; i < pVM->cCpus; i++)
254 {
255 PVMCPU pVCpu = &pVM->aCpus[i];
256
257 pVCpu->hwaccm.s.vmx.hMemObjVMCS = NIL_RTR0MEMOBJ;
258
259 /* Allocate one page for the VM control structure (VMCS). */
260 rc = RTR0MemObjAllocCont(&pVCpu->hwaccm.s.vmx.hMemObjVMCS, PAGE_SIZE, true /* executable R0 mapping */);
261 AssertRC(rc);
262 if (RT_FAILURE(rc))
263 return rc;
264
265 pVCpu->hwaccm.s.vmx.pvVMCS = RTR0MemObjAddress(pVCpu->hwaccm.s.vmx.hMemObjVMCS);
266 pVCpu->hwaccm.s.vmx.HCPhysVMCS = RTR0MemObjGetPagePhysAddr(pVCpu->hwaccm.s.vmx.hMemObjVMCS, 0);
267 ASMMemZeroPage(pVCpu->hwaccm.s.vmx.pvVMCS);
268
269 pVCpu->hwaccm.s.vmx.cr0_mask = 0;
270 pVCpu->hwaccm.s.vmx.cr4_mask = 0;
271
272 /* Allocate one page for the virtual APIC page for TPR caching. */
273 rc = RTR0MemObjAllocCont(&pVCpu->hwaccm.s.vmx.hMemObjVAPIC, PAGE_SIZE, true /* executable R0 mapping */);
274 AssertRC(rc);
275 if (RT_FAILURE(rc))
276 return rc;
277
278 pVCpu->hwaccm.s.vmx.pbVAPIC = (uint8_t *)RTR0MemObjAddress(pVCpu->hwaccm.s.vmx.hMemObjVAPIC);
279 pVCpu->hwaccm.s.vmx.HCPhysVAPIC = RTR0MemObjGetPagePhysAddr(pVCpu->hwaccm.s.vmx.hMemObjVAPIC, 0);
280 ASMMemZeroPage(pVCpu->hwaccm.s.vmx.pbVAPIC);
281
282 /* Allocate the MSR bitmap if this feature is supported. */
283 if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
284 {
285 rc = RTR0MemObjAllocCont(&pVCpu->hwaccm.s.vmx.pMemObjMSRBitmap, PAGE_SIZE, true /* executable R0 mapping */);
286 AssertRC(rc);
287 if (RT_FAILURE(rc))
288 return rc;
289
290 pVCpu->hwaccm.s.vmx.pMSRBitmap = (uint8_t *)RTR0MemObjAddress(pVCpu->hwaccm.s.vmx.pMemObjMSRBitmap);
291 pVCpu->hwaccm.s.vmx.pMSRBitmapPhys = RTR0MemObjGetPagePhysAddr(pVCpu->hwaccm.s.vmx.pMemObjMSRBitmap, 0);
292 memset(pVCpu->hwaccm.s.vmx.pMSRBitmap, 0xff, PAGE_SIZE);
293 }
294
295#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
296 /* Allocate one page for the guest MSR load area (for preloading guest MSRs during the world switch). */
297 rc = RTR0MemObjAllocCont(&pVCpu->hwaccm.s.vmx.pMemObjGuestMSR, PAGE_SIZE, true /* executable R0 mapping */);
298 AssertRC(rc);
299 if (RT_FAILURE(rc))
300 return rc;
301
302 pVCpu->hwaccm.s.vmx.pGuestMSR = (uint8_t *)RTR0MemObjAddress(pVCpu->hwaccm.s.vmx.pMemObjGuestMSR);
303 pVCpu->hwaccm.s.vmx.pGuestMSRPhys = RTR0MemObjGetPagePhysAddr(pVCpu->hwaccm.s.vmx.pMemObjGuestMSR, 0);
304 memset(pVCpu->hwaccm.s.vmx.pGuestMSR, 0, PAGE_SIZE);
305
306 /* Allocate one page for the host MSR load area (for restoring host MSRs after the world switch back). */
307 rc = RTR0MemObjAllocCont(&pVCpu->hwaccm.s.vmx.pMemObjHostMSR, PAGE_SIZE, true /* executable R0 mapping */);
308 AssertRC(rc);
309 if (RT_FAILURE(rc))
310 return rc;
311
312 pVCpu->hwaccm.s.vmx.pHostMSR = (uint8_t *)RTR0MemObjAddress(pVCpu->hwaccm.s.vmx.pMemObjHostMSR);
313 pVCpu->hwaccm.s.vmx.pHostMSRPhys = RTR0MemObjGetPagePhysAddr(pVCpu->hwaccm.s.vmx.pMemObjHostMSR, 0);
314 memset(pVCpu->hwaccm.s.vmx.pHostMSR, 0, PAGE_SIZE);
315#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
316
317 /* Current guest paging mode. */
318 pVCpu->hwaccm.s.vmx.enmLastSeenGuestMode = PGMMODE_REAL;
319
320#ifdef LOG_ENABLED
321 SUPR0Printf("VMXR0InitVM %x VMCS=%x (%x)\n", pVM, pVCpu->hwaccm.s.vmx.pvVMCS, (uint32_t)pVCpu->hwaccm.s.vmx.HCPhysVMCS);
322#endif
323 }
324
325 return VINF_SUCCESS;
326}
327
328
329/**
330 * Does Ring-0 per VM VT-x termination.
331 *
332 * @returns VBox status code.
333 * @param pVM Pointer to the VM.
334 */
335VMMR0DECL(int) VMXR0TermVM(PVM pVM)
336{
337 for (VMCPUID i = 0; i < pVM->cCpus; i++)
338 {
339 PVMCPU pVCpu = &pVM->aCpus[i];
340
341 if (pVCpu->hwaccm.s.vmx.hMemObjVMCS != NIL_RTR0MEMOBJ)
342 {
343 RTR0MemObjFree(pVCpu->hwaccm.s.vmx.hMemObjVMCS, false);
344 pVCpu->hwaccm.s.vmx.hMemObjVMCS = NIL_RTR0MEMOBJ;
345 pVCpu->hwaccm.s.vmx.pvVMCS = 0;
346 pVCpu->hwaccm.s.vmx.HCPhysVMCS = 0;
347 }
348 if (pVCpu->hwaccm.s.vmx.hMemObjVAPIC != NIL_RTR0MEMOBJ)
349 {
350 RTR0MemObjFree(pVCpu->hwaccm.s.vmx.hMemObjVAPIC, false);
351 pVCpu->hwaccm.s.vmx.hMemObjVAPIC = NIL_RTR0MEMOBJ;
352 pVCpu->hwaccm.s.vmx.pbVAPIC = 0;
353 pVCpu->hwaccm.s.vmx.HCPhysVAPIC = 0;
354 }
355 if (pVCpu->hwaccm.s.vmx.pMemObjMSRBitmap != NIL_RTR0MEMOBJ)
356 {
357 RTR0MemObjFree(pVCpu->hwaccm.s.vmx.pMemObjMSRBitmap, false);
358 pVCpu->hwaccm.s.vmx.pMemObjMSRBitmap = NIL_RTR0MEMOBJ;
359 pVCpu->hwaccm.s.vmx.pMSRBitmap = 0;
360 pVCpu->hwaccm.s.vmx.pMSRBitmapPhys = 0;
361 }
362#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
363 if (pVCpu->hwaccm.s.vmx.pMemObjHostMSR != NIL_RTR0MEMOBJ)
364 {
365 RTR0MemObjFree(pVCpu->hwaccm.s.vmx.pMemObjHostMSR, false);
366 pVCpu->hwaccm.s.vmx.pMemObjHostMSR = NIL_RTR0MEMOBJ;
367 pVCpu->hwaccm.s.vmx.pHostMSR = 0;
368 pVCpu->hwaccm.s.vmx.pHostMSRPhys = 0;
369 }
370 if (pVCpu->hwaccm.s.vmx.pMemObjGuestMSR != NIL_RTR0MEMOBJ)
371 {
372 RTR0MemObjFree(pVCpu->hwaccm.s.vmx.pMemObjGuestMSR, false);
373 pVCpu->hwaccm.s.vmx.pMemObjGuestMSR = NIL_RTR0MEMOBJ;
374 pVCpu->hwaccm.s.vmx.pGuestMSR = 0;
375 pVCpu->hwaccm.s.vmx.pGuestMSRPhys = 0;
376 }
377#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
378 }
379 if (pVM->hwaccm.s.vmx.pMemObjAPIC != NIL_RTR0MEMOBJ)
380 {
381 RTR0MemObjFree(pVM->hwaccm.s.vmx.pMemObjAPIC, false);
382 pVM->hwaccm.s.vmx.pMemObjAPIC = NIL_RTR0MEMOBJ;
383 pVM->hwaccm.s.vmx.pAPIC = 0;
384 pVM->hwaccm.s.vmx.pAPICPhys = 0;
385 }
386#ifdef VBOX_WITH_CRASHDUMP_MAGIC
387 if (pVM->hwaccm.s.vmx.pMemObjScratch != NIL_RTR0MEMOBJ)
388 {
389 ASMMemZero32(pVM->hwaccm.s.vmx.pScratch, PAGE_SIZE);
390 RTR0MemObjFree(pVM->hwaccm.s.vmx.pMemObjScratch, false);
391 pVM->hwaccm.s.vmx.pMemObjScratch = NIL_RTR0MEMOBJ;
392 pVM->hwaccm.s.vmx.pScratch = 0;
393 pVM->hwaccm.s.vmx.pScratchPhys = 0;
394 }
395#endif
396 return VINF_SUCCESS;
397}
398
399
400/**
401 * Sets up VT-x for the specified VM.
402 *
403 * @returns VBox status code.
404 * @param pVM Pointer to the VM.
405 */
406VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
407{
408 int rc = VINF_SUCCESS;
409 uint32_t val;
410
411 AssertReturn(pVM, VERR_INVALID_PARAMETER);
412
413 /* Initialize these always, see hwaccmR3InitFinalizeR0().*/
414 pVM->hwaccm.s.vmx.enmFlushEPT = VMX_FLUSH_EPT_NONE;
415 pVM->hwaccm.s.vmx.enmFlushVPID = VMX_FLUSH_VPID_NONE;
416
417 /* Determine optimal flush type for EPT. */
418 if (pVM->hwaccm.s.fNestedPaging)
419 {
420 if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVEPT_CAPS_SINGLE_CONTEXT)
421 pVM->hwaccm.s.vmx.enmFlushEPT = VMX_FLUSH_EPT_SINGLE_CONTEXT;
422 else if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVEPT_CAPS_ALL_CONTEXTS)
423 pVM->hwaccm.s.vmx.enmFlushEPT = VMX_FLUSH_EPT_ALL_CONTEXTS;
424 else
425 {
426 /*
427 * Should never really happen. EPT is supported but no suitable flush types supported.
428 * We cannot ignore EPT at this point as we've already setup Unrestricted Guest execution.
429 */
430 pVM->hwaccm.s.vmx.enmFlushEPT = VMX_FLUSH_EPT_NOT_SUPPORTED;
431 return VERR_VMX_GENERIC;
432 }
433 }
434
435 /* Determine optimal flush type for VPID. */
436 if (pVM->hwaccm.s.vmx.fVPID)
437 {
438 if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVVPID_CAPS_SINGLE_CONTEXT)
439 pVM->hwaccm.s.vmx.enmFlushVPID = VMX_FLUSH_VPID_SINGLE_CONTEXT;
440 else if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVVPID_CAPS_ALL_CONTEXTS)
441 pVM->hwaccm.s.vmx.enmFlushVPID = VMX_FLUSH_VPID_ALL_CONTEXTS;
442 else
443 {
444 /*
445 * Neither SINGLE nor ALL context flush types for VPID supported by the CPU.
446 * We do not handle other flush type combinations, ignore VPID capabilities.
447 */
448 if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVVPID_CAPS_INDIV_ADDR)
449 Log(("VMXR0SetupVM: Only VMX_FLUSH_VPID_INDIV_ADDR supported. Ignoring VPID.\n"));
450 if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVVPID_CAPS_SINGLE_CONTEXT_RETAIN_GLOBALS)
451 Log(("VMXR0SetupVM: Only VMX_FLUSH_VPID_SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
452 pVM->hwaccm.s.vmx.enmFlushVPID = VMX_FLUSH_VPID_NOT_SUPPORTED;
453 pVM->hwaccm.s.vmx.fVPID = false;
454 }
455 }
456
457 for (VMCPUID i = 0; i < pVM->cCpus; i++)
458 {
459 PVMCPU pVCpu = &pVM->aCpus[i];
460
461 AssertPtr(pVCpu->hwaccm.s.vmx.pvVMCS);
462
463 /* Set revision dword at the beginning of the VMCS structure. */
464 *(uint32_t *)pVCpu->hwaccm.s.vmx.pvVMCS = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hwaccm.s.vmx.msr.vmx_basic_info);
465
466 /*
467 * Clear and activate the VMCS.
468 */
469 Log(("HCPhysVMCS = %RHp\n", pVCpu->hwaccm.s.vmx.HCPhysVMCS));
470 rc = VMXClearVMCS(pVCpu->hwaccm.s.vmx.HCPhysVMCS);
471 if (RT_FAILURE(rc))
472 goto vmx_end;
473
474 rc = VMXActivateVMCS(pVCpu->hwaccm.s.vmx.HCPhysVMCS);
475 if (RT_FAILURE(rc))
476 goto vmx_end;
477
478 /*
479 * VMX_VMCS_CTRL_PIN_EXEC_CONTROLS
480 * Set required bits to one and zero according to the MSR capabilities.
481 */
482 val = pVM->hwaccm.s.vmx.msr.vmx_pin_ctls.n.disallowed0;
483 val |= VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_EXT_INT_EXIT /* External interrupts */
484 | VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_NMI_EXIT; /* Non-maskable interrupts */
485
486 /*
487 * Enable the VMX preemption timer.
488 */
489 if (pVM->hwaccm.s.vmx.fUsePreemptTimer)
490 val |= VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_PREEMPT_TIMER;
491 val &= pVM->hwaccm.s.vmx.msr.vmx_pin_ctls.n.allowed1;
492
493 rc = VMXWriteVMCS(VMX_VMCS_CTRL_PIN_EXEC_CONTROLS, val);
494 AssertRC(rc);
495
496 /*
497 * VMX_VMCS_CTRL_PROC_EXEC_CONTROLS
498 * Set required bits to one and zero according to the MSR capabilities.
499 */
500 val = pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.disallowed0;
501 /* Program which event cause VM-exits and which features we want to use. */
502 val |= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_HLT_EXIT
503 | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_TSC_OFFSET
504 | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT
505 | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_UNCOND_IO_EXIT
506 | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDPMC_EXIT
507 | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_EXIT
508 | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MWAIT_EXIT; /* don't execute mwait or else we'll idle inside
509 the guest (host thinks the cpu load is high) */
510
511 /* Without nested paging we should intercept invlpg and cr3 mov instructions. */
512 if (!pVM->hwaccm.s.fNestedPaging)
513 {
514 val |= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INVLPG_EXIT
515 | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
516 | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT;
517 }
518
519 /*
520 * VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MWAIT_EXIT might cause a vmlaunch
521 * failure with an invalid control fields error. (combined with some other exit reasons)
522 */
523 if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
524 {
525 /* CR8 reads from the APIC shadow page; writes cause an exit is they lower the TPR below the threshold */
526 val |= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW;
527 Assert(pVM->hwaccm.s.vmx.pAPIC);
528 }
529 else
530 /* Exit on CR8 reads & writes in case the TPR shadow feature isn't present. */
531 val |= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR8_STORE_EXIT | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR8_LOAD_EXIT;
532
533 if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
534 {
535 Assert(pVCpu->hwaccm.s.vmx.pMSRBitmapPhys);
536 val |= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS;
537 }
538
539 /* We will use the secondary control if it's present. */
540 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
541
542 /* Mask away the bits that the CPU doesn't support */
543 /** @todo make sure they don't conflict with the above requirements. */
544 val &= pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1;
545 pVCpu->hwaccm.s.vmx.proc_ctls = val;
546
547 rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, val);
548 AssertRC(rc);
549
550 if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
551 {
552 /*
553 * VMX_VMCS_CTRL_PROC_EXEC_CONTROLS2
554 * Set required bits to one and zero according to the MSR capabilities.
555 */
556 val = pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2.n.disallowed0;
557 val |= VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT;
558
559 if (pVM->hwaccm.s.fNestedPaging)
560 val |= VMX_VMCS_CTRL_PROC_EXEC2_EPT;
561
562 if (pVM->hwaccm.s.vmx.fVPID)
563 val |= VMX_VMCS_CTRL_PROC_EXEC2_VPID;
564
565 if (pVM->hwaccm.s.fHasIoApic)
566 val |= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC;
567
568 if (pVM->hwaccm.s.vmx.fUnrestrictedGuest)
569 val |= VMX_VMCS_CTRL_PROC_EXEC2_REAL_MODE;
570
571 if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
572 val |= VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP;
573
574 /* Mask away the bits that the CPU doesn't support */
575 /** @todo make sure they don't conflict with the above requirements. */
576 val &= pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2.n.allowed1;
577 pVCpu->hwaccm.s.vmx.proc_ctls2 = val;
578 rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS2, val);
579 AssertRC(rc);
580 }
581
582 /*
583 * VMX_VMCS_CTRL_CR3_TARGET_COUNT
584 * Set required bits to one and zero according to the MSR capabilities.
585 */
586 rc = VMXWriteVMCS(VMX_VMCS_CTRL_CR3_TARGET_COUNT, 0);
587 AssertRC(rc);
588
589 /*
590 * Forward all exception except #NM & #PF to the guest.
591 * We always need to check pagefaults since our shadow page table can be out of sync.
592 * And we always lazily sync the FPU & XMM state. .
593 */
594
595 /** @todo Possible optimization:
596 * Keep the FPU and XMM state current in the EM thread. That way there's no need to
597 * lazily sync anything, but the downside is that we can't use the FPU stack or XMM
598 * registers ourselves of course.
599 *
600 * Note: only possible if the current state is actually ours (X86_CR0_TS flag)
601 */
602
603 /*
604 * Don't filter page faults, all of them should cause a world switch.
605 */
606 rc = VMXWriteVMCS(VMX_VMCS_CTRL_PAGEFAULT_ERROR_MASK, 0);
607 AssertRC(rc);
608 rc = VMXWriteVMCS(VMX_VMCS_CTRL_PAGEFAULT_ERROR_MATCH, 0);
609 AssertRC(rc);
610
611 rc = VMXWriteVMCS64(VMX_VMCS_CTRL_TSC_OFFSET_FULL, 0);
612 AssertRC(rc);
613 rc = VMXWriteVMCS64(VMX_VMCS_CTRL_IO_BITMAP_A_FULL, 0);
614 AssertRC(rc);
615 rc = VMXWriteVMCS64(VMX_VMCS_CTRL_IO_BITMAP_B_FULL, 0);
616 AssertRC(rc);
617
618 /*
619 * Set the MSR bitmap address.
620 */
621 if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
622 {
623 Assert(pVCpu->hwaccm.s.vmx.pMSRBitmapPhys);
624
625 rc = VMXWriteVMCS64(VMX_VMCS_CTRL_MSR_BITMAP_FULL, pVCpu->hwaccm.s.vmx.pMSRBitmapPhys);
626 AssertRC(rc);
627
628 /*
629 * Allow the guest to directly modify these MSRs; they are loaded/stored automatically
630 * using MSR-load/store areas in the VMCS.
631 */
632 hmR0VmxSetMSRPermission(pVCpu, MSR_IA32_SYSENTER_CS, true, true);
633 hmR0VmxSetMSRPermission(pVCpu, MSR_IA32_SYSENTER_ESP, true, true);
634 hmR0VmxSetMSRPermission(pVCpu, MSR_IA32_SYSENTER_EIP, true, true);
635 hmR0VmxSetMSRPermission(pVCpu, MSR_K8_LSTAR, true, true);
636 hmR0VmxSetMSRPermission(pVCpu, MSR_K6_STAR, true, true);
637 hmR0VmxSetMSRPermission(pVCpu, MSR_K8_SF_MASK, true, true);
638 hmR0VmxSetMSRPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, true, true);
639 hmR0VmxSetMSRPermission(pVCpu, MSR_K8_GS_BASE, true, true);
640 hmR0VmxSetMSRPermission(pVCpu, MSR_K8_FS_BASE, true, true);
641 if (pVCpu->hwaccm.s.vmx.proc_ctls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
642 hmR0VmxSetMSRPermission(pVCpu, MSR_K8_TSC_AUX, true, true);
643 }
644
645#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
646 /*
647 * Set the guest & host MSR load/store physical addresses.
648 */
649 Assert(pVCpu->hwaccm.s.vmx.pGuestMSRPhys);
650 rc = VMXWriteVMCS64(VMX_VMCS_CTRL_VMENTRY_MSR_LOAD_FULL, pVCpu->hwaccm.s.vmx.pGuestMSRPhys);
651 AssertRC(rc);
652 rc = VMXWriteVMCS64(VMX_VMCS_CTRL_VMEXIT_MSR_STORE_FULL, pVCpu->hwaccm.s.vmx.pGuestMSRPhys);
653 AssertRC(rc);
654 Assert(pVCpu->hwaccm.s.vmx.pHostMSRPhys);
655 rc = VMXWriteVMCS64(VMX_VMCS_CTRL_VMEXIT_MSR_LOAD_FULL, pVCpu->hwaccm.s.vmx.pHostMSRPhys);
656 AssertRC(rc);
657#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
658
659 rc = VMXWriteVMCS(VMX_VMCS_CTRL_ENTRY_MSR_LOAD_COUNT, 0);
660 AssertRC(rc);
661 rc = VMXWriteVMCS(VMX_VMCS_CTRL_EXIT_MSR_STORE_COUNT, 0);
662 AssertRC(rc);
663 rc = VMXWriteVMCS(VMX_VMCS_CTRL_EXIT_MSR_LOAD_COUNT, 0);
664 AssertRC(rc);
665
666 if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
667 {
668 Assert(pVM->hwaccm.s.vmx.pMemObjAPIC);
669 /* Optional */
670 rc = VMXWriteVMCS(VMX_VMCS_CTRL_TPR_THRESHOLD, 0);
671 rc |= VMXWriteVMCS64(VMX_VMCS_CTRL_VAPIC_PAGEADDR_FULL, pVCpu->hwaccm.s.vmx.HCPhysVAPIC);
672
673 if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
674 rc |= VMXWriteVMCS64(VMX_VMCS_CTRL_APIC_ACCESSADDR_FULL, pVM->hwaccm.s.vmx.pAPICPhys);
675
676 AssertRC(rc);
677 }
678
679 /* Set link pointer to -1. Not currently used. */
680 rc = VMXWriteVMCS64(VMX_VMCS_GUEST_LINK_PTR_FULL, 0xFFFFFFFFFFFFFFFFULL);
681 AssertRC(rc);
682
683 /*
684 * Clear VMCS, marking it inactive. Clear implementation specific data and writing back
685 * VMCS data back to memory.
686 */
687 rc = VMXClearVMCS(pVCpu->hwaccm.s.vmx.HCPhysVMCS);
688 AssertRC(rc);
689
690 /*
691 * Configure the VMCS read cache.
692 */
693 PVMCSCACHE pCache = &pVCpu->hwaccm.s.vmx.VMCSCache;
694
695 VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_RIP);
696 VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_RSP);
697 VMXSetupCachedReadVMCS(pCache, VMX_VMCS_GUEST_RFLAGS);
698 VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE);
699 VMXSetupCachedReadVMCS(pCache, VMX_VMCS_CTRL_CR0_READ_SHADOW);
700 VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_CR0);
701 VMXSetupCachedReadVMCS(pCache, VMX_VMCS_CTRL_CR4_READ_SHADOW);
702 VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_CR4);
703 VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_DR7);
704 VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_GUEST_SYSENTER_CS);
705 VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_SYSENTER_EIP);
706 VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_SYSENTER_ESP);
707 VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_GUEST_GDTR_LIMIT);
708 VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_GDTR_BASE);
709 VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_GUEST_IDTR_LIMIT);
710 VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_IDTR_BASE);
711
712 VMX_SETUP_SELREG(ES, pCache);
713 VMX_SETUP_SELREG(SS, pCache);
714 VMX_SETUP_SELREG(CS, pCache);
715 VMX_SETUP_SELREG(DS, pCache);
716 VMX_SETUP_SELREG(FS, pCache);
717 VMX_SETUP_SELREG(GS, pCache);
718 VMX_SETUP_SELREG(LDTR, pCache);
719 VMX_SETUP_SELREG(TR, pCache);
720
721 /*
722 * Status code VMCS reads.
723 */
724 VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_EXIT_REASON);
725 VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_VM_INSTR_ERROR);
726 VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_EXIT_INSTR_LENGTH);
727 VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_EXIT_INTERRUPTION_ERRCODE);
728 VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO);
729 VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_EXIT_INSTR_INFO);
730 VMXSetupCachedReadVMCS(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
731 VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_IDT_INFO);
732 VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_IDT_ERRCODE);
733
734 if (pVM->hwaccm.s.fNestedPaging)
735 {
736 VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_CR3);
737 VMXSetupCachedReadVMCS(pCache, VMX_VMCS_EXIT_PHYS_ADDR_FULL);
738 pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
739 }
740 else
741 pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
742 } /* for each VMCPU */
743
744 /*
745 * Setup the right TLB function based on CPU capabilities.
746 */
747 if (pVM->hwaccm.s.fNestedPaging && pVM->hwaccm.s.vmx.fVPID)
748 pVM->hwaccm.s.vmx.pfnSetupTaggedTLB = hmR0VmxSetupTLBBoth;
749 else if (pVM->hwaccm.s.fNestedPaging)
750 pVM->hwaccm.s.vmx.pfnSetupTaggedTLB = hmR0VmxSetupTLBEPT;
751 else if (pVM->hwaccm.s.vmx.fVPID)
752 pVM->hwaccm.s.vmx.pfnSetupTaggedTLB = hmR0VmxSetupTLBVPID;
753 else
754 pVM->hwaccm.s.vmx.pfnSetupTaggedTLB = hmR0VmxSetupTLBDummy;
755
756vmx_end:
757 hmR0VmxCheckError(pVM, &pVM->aCpus[0], rc);
758 return rc;
759}
760
761
762/**
763 * Sets the permission bits for the specified MSR.
764 *
765 * @param pVCpu Pointer to the VMCPU.
766 * @param ulMSR The MSR value.
767 * @param fRead Whether reading is allowed.
768 * @param fWrite Whether writing is allowed.
769 */
770static void hmR0VmxSetMSRPermission(PVMCPU pVCpu, unsigned ulMSR, bool fRead, bool fWrite)
771{
772 unsigned ulBit;
773 uint8_t *pMSRBitmap = (uint8_t *)pVCpu->hwaccm.s.vmx.pMSRBitmap;
774
775 /*
776 * Layout:
777 * 0x000 - 0x3ff - Low MSR read bits
778 * 0x400 - 0x7ff - High MSR read bits
779 * 0x800 - 0xbff - Low MSR write bits
780 * 0xc00 - 0xfff - High MSR write bits
781 */
782 if (ulMSR <= 0x00001FFF)
783 {
784 /* Pentium-compatible MSRs */
785 ulBit = ulMSR;
786 }
787 else if ( ulMSR >= 0xC0000000
788 && ulMSR <= 0xC0001FFF)
789 {
790 /* AMD Sixth Generation x86 Processor MSRs */
791 ulBit = (ulMSR - 0xC0000000);
792 pMSRBitmap += 0x400;
793 }
794 else
795 {
796 AssertFailed();
797 return;
798 }
799
800 Assert(ulBit <= 0x1fff);
801 if (fRead)
802 ASMBitClear(pMSRBitmap, ulBit);
803 else
804 ASMBitSet(pMSRBitmap, ulBit);
805
806 if (fWrite)
807 ASMBitClear(pMSRBitmap + 0x800, ulBit);
808 else
809 ASMBitSet(pMSRBitmap + 0x800, ulBit);
810}
811
812
813/**
814 * Injects an event (trap or external interrupt).
815 *
816 * @returns VBox status code. Note that it may return VINF_EM_RESET to
817 * indicate a triple fault when injecting X86_XCPT_DF.
818 *
819 * @param pVM Pointer to the VM.
820 * @param pVCpu Pointer to the VMCPU.
821 * @param pCtx Pointer to the guest CPU Context.
822 * @param intInfo VMX interrupt info.
823 * @param cbInstr Opcode length of faulting instruction.
824 * @param errCode Error code (optional).
825 */
826static int hmR0VmxInjectEvent(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, uint32_t intInfo, uint32_t cbInstr, uint32_t errCode)
827{
828 int rc;
829 uint32_t iGate = VMX_EXIT_INTERRUPTION_INFO_VECTOR(intInfo);
830
831#ifdef VBOX_WITH_STATISTICS
832 STAM_COUNTER_INC(&pVCpu->hwaccm.s.paStatInjectedIrqsR0[iGate & MASK_INJECT_IRQ_STAT]);
833#endif
834
835#ifdef VBOX_STRICT
836 if (iGate == 0xE)
837 {
838 LogFlow(("hmR0VmxInjectEvent: Injecting interrupt %d at %RGv error code=%08x CR2=%RGv intInfo=%08x\n", iGate,
839 (RTGCPTR)pCtx->rip, errCode, pCtx->cr2, intInfo));
840 }
841 else if (iGate < 0x20)
842 {
843 LogFlow(("hmR0VmxInjectEvent: Injecting interrupt %d at %RGv error code=%08x\n", iGate, (RTGCPTR)pCtx->rip,
844 errCode));
845 }
846 else
847 {
848 LogFlow(("INJ-EI: %x at %RGv\n", iGate, (RTGCPTR)pCtx->rip));
849 Assert( VMX_EXIT_INTERRUPTION_INFO_TYPE(intInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW
850 || !VMCPU_FF_ISSET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
851 Assert( VMX_EXIT_INTERRUPTION_INFO_TYPE(intInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW
852 || pCtx->eflags.u32 & X86_EFL_IF);
853 }
854#endif
855
856 if ( CPUMIsGuestInRealModeEx(pCtx)
857 && pVM->hwaccm.s.vmx.pRealModeTSS)
858 {
859 RTGCPHYS GCPhysHandler;
860 uint16_t offset, ip;
861 RTSEL sel;
862
863 /*
864 * Injecting events doesn't work right with real mode emulation.
865 * (#GP if we try to inject external hardware interrupts)
866 * Inject the interrupt or trap directly instead.
867 *
868 * ASSUMES no access handlers for the bits we read or write below (should be safe).
869 */
870 Log(("Manual interrupt/trap '%x' inject (real mode)\n", iGate));
871
872 /*
873 * Check if the interrupt handler is present.
874 */
875 if (iGate * 4 + 3 > pCtx->idtr.cbIdt)
876 {
877 Log(("IDT cbIdt violation\n"));
878 if (iGate != X86_XCPT_DF)
879 {
880 uint32_t intInfo2;
881
882 intInfo2 = (iGate == X86_XCPT_GP) ? (uint32_t)X86_XCPT_DF : iGate;
883 intInfo2 |= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
884 intInfo2 |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
885 intInfo2 |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HWEXCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
886
887 return hmR0VmxInjectEvent(pVM, pVCpu, pCtx, intInfo2, 0, 0 /* no error code according to the Intel docs */);
888 }
889 Log(("Triple fault -> reset the VM!\n"));
890 return VINF_EM_RESET;
891 }
892 if ( VMX_EXIT_INTERRUPTION_INFO_TYPE(intInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW
893 || iGate == 3 /* Both #BP and #OF point to the instruction after. */
894 || iGate == 4)
895 {
896 ip = pCtx->ip + cbInstr;
897 }
898 else
899 ip = pCtx->ip;
900
901 /*
902 * Read the selector:offset pair of the interrupt handler.
903 */
904 GCPhysHandler = (RTGCPHYS)pCtx->idtr.pIdt + iGate * 4;
905 rc = PGMPhysSimpleReadGCPhys(pVM, &offset, GCPhysHandler, sizeof(offset)); AssertRC(rc);
906 rc = PGMPhysSimpleReadGCPhys(pVM, &sel, GCPhysHandler + 2, sizeof(sel)); AssertRC(rc);
907
908 LogFlow(("IDT handler %04X:%04X\n", sel, offset));
909
910 /*
911 * Construct the stack frame.
912 */
913 /** @todo Check stack limit. */
914 pCtx->sp -= 2;
915 LogFlow(("ss:sp %04X:%04X eflags=%x\n", pCtx->ss.Sel, pCtx->sp, pCtx->eflags.u));
916 rc = PGMPhysSimpleWriteGCPhys(pVM, pCtx->ss.u64Base + pCtx->sp, &pCtx->eflags, sizeof(uint16_t)); AssertRC(rc);
917 pCtx->sp -= 2;
918 LogFlow(("ss:sp %04X:%04X cs=%x\n", pCtx->ss.Sel, pCtx->sp, pCtx->cs.Sel));
919 rc = PGMPhysSimpleWriteGCPhys(pVM, pCtx->ss.u64Base + pCtx->sp, &pCtx->cs, sizeof(uint16_t)); AssertRC(rc);
920 pCtx->sp -= 2;
921 LogFlow(("ss:sp %04X:%04X ip=%x\n", pCtx->ss.Sel, pCtx->sp, ip));
922 rc = PGMPhysSimpleWriteGCPhys(pVM, pCtx->ss.u64Base + pCtx->sp, &ip, sizeof(ip)); AssertRC(rc);
923
924 /*
925 * Update the CPU state for executing the handler.
926 */
927 pCtx->rip = offset;
928 pCtx->cs.Sel = sel;
929 pCtx->cs.u64Base = sel << 4;
930 pCtx->eflags.u &= ~(X86_EFL_IF | X86_EFL_TF | X86_EFL_RF | X86_EFL_AC);
931
932 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_GUEST_SEGMENT_REGS;
933 return VINF_SUCCESS;
934 }
935
936 /*
937 * Set event injection state.
938 */
939 rc = VMXWriteVMCS(VMX_VMCS_CTRL_ENTRY_IRQ_INFO, intInfo | (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT));
940 rc |= VMXWriteVMCS(VMX_VMCS_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
941 rc |= VMXWriteVMCS(VMX_VMCS_CTRL_ENTRY_EXCEPTION_ERRCODE, errCode);
942
943 AssertRC(rc);
944 return rc;
945}
946
947
948/**
949 * Checks for pending guest interrupts and injects them.
950 *
951 * @returns VBox status code.
952 * @param pVM Pointer to the VM.
953 * @param pVCpu Pointer to the VMCPU.
954 * @param pCtx Pointer to the guest CPU context.
955 */
956static int hmR0VmxCheckPendingInterrupt(PVM pVM, PVMCPU pVCpu, CPUMCTX *pCtx)
957{
958 int rc;
959
960 /*
961 * Dispatch any pending interrupts (injected before, but a VM exit occurred prematurely).
962 */
963 if (pVCpu->hwaccm.s.Event.fPending)
964 {
965 Log(("CPU%d: Reinjecting event %RX64 %08x at %RGv cr2=%RX64\n", pVCpu->idCpu, pVCpu->hwaccm.s.Event.intInfo,
966 pVCpu->hwaccm.s.Event.errCode, (RTGCPTR)pCtx->rip, pCtx->cr2));
967 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatIntReinject);
968 rc = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, pVCpu->hwaccm.s.Event.intInfo, 0, pVCpu->hwaccm.s.Event.errCode);
969 AssertRC(rc);
970
971 pVCpu->hwaccm.s.Event.fPending = false;
972 return VINF_SUCCESS;
973 }
974
975 /*
976 * If an active trap is already pending, we must forward it first!
977 */
978 if (!TRPMHasTrap(pVCpu))
979 {
980 if (VMCPU_FF_TESTANDCLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI))
981 {
982 RTGCUINTPTR intInfo;
983
984 Log(("CPU%d: injecting #NMI\n", pVCpu->idCpu));
985
986 intInfo = X86_XCPT_NMI;
987 intInfo |= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
988 intInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
989
990 rc = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, intInfo, 0, 0);
991 AssertRC(rc);
992
993 return VINF_SUCCESS;
994 }
995
996 /** @todo SMI interrupts. */
997
998 /*
999 * When external interrupts are pending, we should exit the VM when IF is set.
1000 */
1001 if (VMCPU_FF_ISPENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC|VMCPU_FF_INTERRUPT_PIC)))
1002 {
1003 if (!(pCtx->eflags.u32 & X86_EFL_IF))
1004 {
1005 if (!(pVCpu->hwaccm.s.vmx.proc_ctls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_IRQ_WINDOW_EXIT))
1006 {
1007 LogFlow(("Enable irq window exit!\n"));
1008 pVCpu->hwaccm.s.vmx.proc_ctls |= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_IRQ_WINDOW_EXIT;
1009 rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
1010 AssertRC(rc);
1011 }
1012 /* else nothing to do but wait */
1013 }
1014 else if (!VMCPU_FF_ISSET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
1015 {
1016 uint8_t u8Interrupt;
1017
1018 rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
1019 Log(("CPU%d: Dispatch interrupt: u8Interrupt=%x (%d) rc=%Rrc cs:rip=%04X:%RGv\n", pVCpu->idCpu,
1020 u8Interrupt, u8Interrupt, rc, pCtx->cs.Sel, (RTGCPTR)pCtx->rip));
1021 if (RT_SUCCESS(rc))
1022 {
1023 rc = TRPMAssertTrap(pVCpu, u8Interrupt, TRPM_HARDWARE_INT);
1024 AssertRC(rc);
1025 }
1026 else
1027 {
1028 /* Can only happen in rare cases where a pending interrupt is cleared behind our back */
1029 Assert(!VMCPU_FF_ISPENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC|VMCPU_FF_INTERRUPT_PIC)));
1030 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatSwitchGuestIrq);
1031 /* Just continue */
1032 }
1033 }
1034 else
1035 Log(("Pending interrupt blocked at %RGv by VM_FF_INHIBIT_INTERRUPTS!!\n", (RTGCPTR)pCtx->rip));
1036 }
1037 }
1038
1039#ifdef VBOX_STRICT
1040 if (TRPMHasTrap(pVCpu))
1041 {
1042 uint8_t u8Vector;
1043 rc = TRPMQueryTrapAll(pVCpu, &u8Vector, 0, 0, 0);
1044 AssertRC(rc);
1045 }
1046#endif
1047
1048 if ( (pCtx->eflags.u32 & X86_EFL_IF)
1049 && (!VMCPU_FF_ISSET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
1050 && TRPMHasTrap(pVCpu)
1051 )
1052 {
1053 uint8_t u8Vector;
1054 TRPMEVENT enmType;
1055 RTGCUINTPTR intInfo;
1056 RTGCUINT errCode;
1057
1058 /*
1059 * If a new event is pending, dispatch it now.
1060 */
1061 rc = TRPMQueryTrapAll(pVCpu, &u8Vector, &enmType, &errCode, 0);
1062 AssertRC(rc);
1063 Assert(pCtx->eflags.Bits.u1IF == 1 || enmType == TRPM_TRAP);
1064 Assert(enmType != TRPM_SOFTWARE_INT);
1065
1066 /*
1067 * Clear the pending trap.
1068 */
1069 rc = TRPMResetTrap(pVCpu);
1070 AssertRC(rc);
1071
1072 intInfo = u8Vector;
1073 intInfo |= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
1074
1075 if (enmType == TRPM_TRAP)
1076 {
1077 switch (u8Vector)
1078 {
1079 case X86_XCPT_DF:
1080 case X86_XCPT_TS:
1081 case X86_XCPT_NP:
1082 case X86_XCPT_SS:
1083 case X86_XCPT_GP:
1084 case X86_XCPT_PF:
1085 case X86_XCPT_AC:
1086 {
1087 /* Valid error codes. */
1088 intInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
1089 break;
1090 }
1091
1092 default:
1093 break;
1094 }
1095
1096 if ( u8Vector == X86_XCPT_BP
1097 || u8Vector == X86_XCPT_OF)
1098 {
1099 intInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SWEXCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
1100 }
1101 else
1102 intInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HWEXCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
1103 }
1104 else
1105 intInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
1106
1107 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatIntInject);
1108 rc = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, intInfo, 0, errCode);
1109 AssertRC(rc);
1110 } /* if (interrupts can be dispatched) */
1111
1112 return VINF_SUCCESS;
1113}
1114
1115
1116/**
1117 * Save the host state into the VMCS.
1118 *
1119 * @returns VBox status code.
1120 * @param pVM Pointer to the VM.
1121 * @param pVCpu Pointer to the VMCPU.
1122 */
1123VMMR0DECL(int) VMXR0SaveHostState(PVM pVM, PVMCPU pVCpu)
1124{
1125 int rc = VINF_SUCCESS;
1126 NOREF(pVM);
1127
1128 /*
1129 * Host CPU Context.
1130 */
1131 if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_HOST_CONTEXT)
1132 {
1133 RTIDTR idtr;
1134 RTGDTR gdtr;
1135 RTSEL SelTR;
1136 PCX86DESCHC pDesc;
1137 uintptr_t trBase;
1138 RTSEL cs;
1139 RTSEL ss;
1140 uint64_t cr3;
1141
1142 /*
1143 * Control registers.
1144 */
1145 rc = VMXWriteVMCS(VMX_VMCS_HOST_CR0, ASMGetCR0());
1146 Log2(("VMX_VMCS_HOST_CR0 %08x\n", ASMGetCR0()));
1147#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
1148 if (VMX_IS_64BIT_HOST_MODE())
1149 {
1150 cr3 = hwaccmR0Get64bitCR3();
1151 rc |= VMXWriteVMCS64(VMX_VMCS_HOST_CR3, cr3);
1152 }
1153 else
1154#endif
1155 {
1156 cr3 = ASMGetCR3();
1157 rc |= VMXWriteVMCS(VMX_VMCS_HOST_CR3, cr3);
1158 }
1159 Log2(("VMX_VMCS_HOST_CR3 %08RX64\n", cr3));
1160 rc |= VMXWriteVMCS(VMX_VMCS_HOST_CR4, ASMGetCR4());
1161 Log2(("VMX_VMCS_HOST_CR4 %08x\n", ASMGetCR4()));
1162 AssertRC(rc);
1163
1164 /*
1165 * Selector registers.
1166 */
1167#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
1168 if (VMX_IS_64BIT_HOST_MODE())
1169 {
1170 cs = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelCS;
1171 ss = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelSS;
1172 }
1173 else
1174 {
1175 /* sysenter loads LDT cs & ss, VMX doesn't like this. Load the GDT ones (safe). */
1176 cs = (RTSEL)(uintptr_t)&SUPR0AbsKernelCS;
1177 ss = (RTSEL)(uintptr_t)&SUPR0AbsKernelSS;
1178 }
1179#else
1180 cs = ASMGetCS();
1181 ss = ASMGetSS();
1182#endif
1183 Assert(!(cs & X86_SEL_LDT)); Assert((cs & X86_SEL_RPL) == 0);
1184 Assert(!(ss & X86_SEL_LDT)); Assert((ss & X86_SEL_RPL) == 0);
1185 rc = VMXWriteVMCS(VMX_VMCS16_HOST_FIELD_CS, cs);
1186 /* Note: VMX is (again) very picky about the RPL of the selectors here; we'll restore them manually. */
1187 rc |= VMXWriteVMCS(VMX_VMCS16_HOST_FIELD_DS, 0);
1188 rc |= VMXWriteVMCS(VMX_VMCS16_HOST_FIELD_ES, 0);
1189#if HC_ARCH_BITS == 32
1190 if (!VMX_IS_64BIT_HOST_MODE())
1191 {
1192 rc |= VMXWriteVMCS(VMX_VMCS16_HOST_FIELD_FS, 0);
1193 rc |= VMXWriteVMCS(VMX_VMCS16_HOST_FIELD_GS, 0);
1194 }
1195#endif
1196 rc |= VMXWriteVMCS(VMX_VMCS16_HOST_FIELD_SS, ss);
1197 SelTR = ASMGetTR();
1198 rc |= VMXWriteVMCS(VMX_VMCS16_HOST_FIELD_TR, SelTR);
1199 AssertRC(rc);
1200 Log2(("VMX_VMCS_HOST_FIELD_CS %08x (%08x)\n", cs, ASMGetSS()));
1201 Log2(("VMX_VMCS_HOST_FIELD_DS 00000000 (%08x)\n", ASMGetDS()));
1202 Log2(("VMX_VMCS_HOST_FIELD_ES 00000000 (%08x)\n", ASMGetES()));
1203 Log2(("VMX_VMCS_HOST_FIELD_FS 00000000 (%08x)\n", ASMGetFS()));
1204 Log2(("VMX_VMCS_HOST_FIELD_GS 00000000 (%08x)\n", ASMGetGS()));
1205 Log2(("VMX_VMCS_HOST_FIELD_SS %08x (%08x)\n", ss, ASMGetSS()));
1206 Log2(("VMX_VMCS_HOST_FIELD_TR %08x\n", ASMGetTR()));
1207
1208 /*
1209 * GDTR & IDTR.
1210 */
1211#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
1212 if (VMX_IS_64BIT_HOST_MODE())
1213 {
1214 X86XDTR64 gdtr64, idtr64;
1215 hwaccmR0Get64bitGDTRandIDTR(&gdtr64, &idtr64);
1216 rc = VMXWriteVMCS64(VMX_VMCS_HOST_GDTR_BASE, gdtr64.uAddr);
1217 rc |= VMXWriteVMCS64(VMX_VMCS_HOST_IDTR_BASE, gdtr64.uAddr);
1218 AssertRC(rc);
1219 Log2(("VMX_VMCS_HOST_GDTR_BASE %RX64\n", gdtr64.uAddr));
1220 Log2(("VMX_VMCS_HOST_IDTR_BASE %RX64\n", idtr64.uAddr));
1221 gdtr.cbGdt = gdtr64.cb;
1222 gdtr.pGdt = (uintptr_t)gdtr64.uAddr;
1223 }
1224 else
1225#endif
1226 {
1227 ASMGetGDTR(&gdtr);
1228 rc = VMXWriteVMCS(VMX_VMCS_HOST_GDTR_BASE, gdtr.pGdt);
1229 ASMGetIDTR(&idtr);
1230 rc |= VMXWriteVMCS(VMX_VMCS_HOST_IDTR_BASE, idtr.pIdt);
1231 AssertRC(rc);
1232 Log2(("VMX_VMCS_HOST_GDTR_BASE %RHv\n", gdtr.pGdt));
1233 Log2(("VMX_VMCS_HOST_IDTR_BASE %RHv\n", idtr.pIdt));
1234 }
1235
1236 /*
1237 * Save the base address of the TR selector.
1238 */
1239 if (SelTR > gdtr.cbGdt)
1240 {
1241 AssertMsgFailed(("Invalid TR selector %x. GDTR.cbGdt=%x\n", SelTR, gdtr.cbGdt));
1242 return VERR_VMX_INVALID_HOST_STATE;
1243 }
1244
1245 pDesc = (PCX86DESCHC)(gdtr.pGdt + (SelTR & X86_SEL_MASK));
1246#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
1247 if (VMX_IS_64BIT_HOST_MODE())
1248 {
1249 uint64_t trBase64 = X86DESC64_BASE(*(PX86DESC64)pDesc);
1250 rc = VMXWriteVMCS64(VMX_VMCS_HOST_TR_BASE, trBase64);
1251 Log2(("VMX_VMCS_HOST_TR_BASE %RX64\n", trBase64));
1252 AssertRC(rc);
1253 }
1254 else
1255#endif
1256 {
1257#if HC_ARCH_BITS == 64
1258 trBase = X86DESC64_BASE(*pDesc);
1259#else
1260 trBase = X86DESC_BASE(*pDesc);
1261#endif
1262 rc = VMXWriteVMCS(VMX_VMCS_HOST_TR_BASE, trBase);
1263 AssertRC(rc);
1264 Log2(("VMX_VMCS_HOST_TR_BASE %RHv\n", trBase));
1265 }
1266
1267 /*
1268 * FS base and GS base.
1269 */
1270#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
1271 if (VMX_IS_64BIT_HOST_MODE())
1272 {
1273 Log2(("MSR_K8_FS_BASE = %RX64\n", ASMRdMsr(MSR_K8_FS_BASE)));
1274 Log2(("MSR_K8_GS_BASE = %RX64\n", ASMRdMsr(MSR_K8_GS_BASE)));
1275 rc = VMXWriteVMCS64(VMX_VMCS_HOST_FS_BASE, ASMRdMsr(MSR_K8_FS_BASE));
1276 rc |= VMXWriteVMCS64(VMX_VMCS_HOST_GS_BASE, ASMRdMsr(MSR_K8_GS_BASE));
1277 }
1278#endif
1279 AssertRC(rc);
1280
1281 /*
1282 * Sysenter MSRs.
1283 */
1284 /** @todo expensive!! */
1285 rc = VMXWriteVMCS(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
1286 Log2(("VMX_VMCS_HOST_SYSENTER_CS %08x\n", ASMRdMsr_Low(MSR_IA32_SYSENTER_CS)));
1287#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
1288 if (VMX_IS_64BIT_HOST_MODE())
1289 {
1290 Log2(("VMX_VMCS_HOST_SYSENTER_EIP %RX64\n", ASMRdMsr(MSR_IA32_SYSENTER_EIP)));
1291 Log2(("VMX_VMCS_HOST_SYSENTER_ESP %RX64\n", ASMRdMsr(MSR_IA32_SYSENTER_ESP)));
1292 rc |= VMXWriteVMCS64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
1293 rc |= VMXWriteVMCS64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
1294 }
1295 else
1296 {
1297 rc |= VMXWriteVMCS(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
1298 rc |= VMXWriteVMCS(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
1299 Log2(("VMX_VMCS_HOST_SYSENTER_EIP %RX32\n", ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP)));
1300 Log2(("VMX_VMCS_HOST_SYSENTER_ESP %RX32\n", ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP)));
1301 }
1302#elif HC_ARCH_BITS == 32
1303 rc |= VMXWriteVMCS(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
1304 rc |= VMXWriteVMCS(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
1305 Log2(("VMX_VMCS_HOST_SYSENTER_EIP %RX32\n", ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP)));
1306 Log2(("VMX_VMCS_HOST_SYSENTER_ESP %RX32\n", ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP)));
1307#else
1308 Log2(("VMX_VMCS_HOST_SYSENTER_EIP %RX64\n", ASMRdMsr(MSR_IA32_SYSENTER_EIP)));
1309 Log2(("VMX_VMCS_HOST_SYSENTER_ESP %RX64\n", ASMRdMsr(MSR_IA32_SYSENTER_ESP)));
1310 rc |= VMXWriteVMCS64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
1311 rc |= VMXWriteVMCS64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
1312#endif
1313 AssertRC(rc);
1314
1315
1316#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
1317 /*
1318 * Store all host MSRs in the VM-Exit load area, so they will be reloaded after
1319 * the world switch back to the host.
1320 */
1321 PVMXMSR pMsr = (PVMXMSR)pVCpu->hwaccm.s.vmx.pHostMSR;
1322 unsigned idxMsr = 0;
1323
1324 /*
1325 * Check if EFER MSR present.
1326 */
1327 uint32_t u32HostExtFeatures = ASMCpuId_EDX(0x80000001);
1328 if (u32HostExtFeatures & (X86_CPUID_EXT_FEATURE_EDX_NX | X86_CPUID_EXT_FEATURE_EDX_LONG_MODE))
1329 {
1330 if (u32HostExtFeatures & X86_CPUID_EXT_FEATURE_EDX_SYSCALL)
1331 {
1332 pMsr->u32IndexMSR = MSR_K6_STAR;
1333 pMsr->u32Reserved = 0;
1334 pMsr->u64Value = ASMRdMsr(MSR_K6_STAR); /* legacy syscall eip, cs & ss */
1335 pMsr++; idxMsr++;
1336 }
1337
1338 pMsr->u32IndexMSR = MSR_K6_EFER;
1339 pMsr->u32Reserved = 0;
1340# if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
1341 if (CPUMIsGuestInLongMode(pVCpu))
1342 {
1343 /* Must match the EFER value in our 64 bits switcher. */
1344 pMsr->u64Value = ASMRdMsr(MSR_K6_EFER) | MSR_K6_EFER_LME | MSR_K6_EFER_SCE | MSR_K6_EFER_NXE;
1345 }
1346 else
1347# endif
1348 pMsr->u64Value = ASMRdMsr(MSR_K6_EFER);
1349 pMsr++; idxMsr++;
1350 }
1351
1352# if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
1353 if (VMX_IS_64BIT_HOST_MODE())
1354 {
1355 pMsr->u32IndexMSR = MSR_K8_LSTAR;
1356 pMsr->u32Reserved = 0;
1357 pMsr->u64Value = ASMRdMsr(MSR_K8_LSTAR); /* 64 bits mode syscall rip */
1358 pMsr++; idxMsr++;
1359 pMsr->u32IndexMSR = MSR_K8_SF_MASK;
1360 pMsr->u32Reserved = 0;
1361 pMsr->u64Value = ASMRdMsr(MSR_K8_SF_MASK); /* syscall flag mask */
1362 pMsr++; idxMsr++;
1363
1364 /* The KERNEL_GS_BASE MSR doesn't work reliably with auto load/store. See @bugref{6208} */
1365#if 0
1366 pMsr->u32IndexMSR = MSR_K8_KERNEL_GS_BASE;
1367 pMsr->u32Reserved = 0;
1368 pMsr->u64Value = ASMRdMsr(MSR_K8_KERNEL_GS_BASE); /* swapgs exchange value */
1369 pMsr++; idxMsr++;
1370#endif
1371 }
1372# endif
1373
1374 if (pVCpu->hwaccm.s.vmx.proc_ctls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
1375 {
1376 pMsr->u32IndexMSR = MSR_K8_TSC_AUX;
1377 pMsr->u32Reserved = 0;
1378 pMsr->u64Value = ASMRdMsr(MSR_K8_TSC_AUX);
1379 pMsr++; idxMsr++;
1380 }
1381
1382 /** @todo r=ramshankar: check IA32_VMX_MISC bits 27:25 for valid idxMsr
1383 * range. */
1384 rc = VMXWriteVMCS(VMX_VMCS_CTRL_EXIT_MSR_LOAD_COUNT, idxMsr);
1385 AssertRC(rc);
1386
1387 pVCpu->hwaccm.s.vmx.cCachedMSRs = idxMsr;
1388#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
1389
1390 pVCpu->hwaccm.s.fContextUseFlags &= ~HWACCM_CHANGED_HOST_CONTEXT;
1391 }
1392 return rc;
1393}
1394
1395
1396/**
1397 * Loads the 4 PDPEs into the guest state when nested paging is used and the
1398 * guest operates in PAE mode.
1399 *
1400 * @returns VBox status code.
1401 * @param pVCpu Pointer to the VMCPU.
1402 * @param pCtx Pointer to the guest CPU context.
1403 */
1404static int hmR0VmxLoadPaePdpes(PVMCPU pVCpu, PCPUMCTX pCtx)
1405{
1406 if (CPUMIsGuestInPAEModeEx(pCtx))
1407 {
1408 X86PDPE aPdpes[4];
1409 int rc = PGMGstGetPaePdpes(pVCpu, &aPdpes[0]);
1410 AssertRCReturn(rc, rc);
1411
1412 rc = VMXWriteVMCS64(VMX_VMCS_GUEST_PDPTR0_FULL, aPdpes[0].u); AssertRCReturn(rc, rc);
1413 rc = VMXWriteVMCS64(VMX_VMCS_GUEST_PDPTR1_FULL, aPdpes[1].u); AssertRCReturn(rc, rc);
1414 rc = VMXWriteVMCS64(VMX_VMCS_GUEST_PDPTR2_FULL, aPdpes[2].u); AssertRCReturn(rc, rc);
1415 rc = VMXWriteVMCS64(VMX_VMCS_GUEST_PDPTR3_FULL, aPdpes[3].u); AssertRCReturn(rc, rc);
1416 }
1417 return VINF_SUCCESS;
1418}
1419
1420
1421/**
1422 * Saves the 4 PDPEs into the guest state when nested paging is used and the
1423 * guest operates in PAE mode.
1424 *
1425 * @returns VBox status code.
1426 * @param pVCpu Pointer to the VM CPU.
1427 * @param pCtx Pointer to the guest CPU context.
1428 *
1429 * @remarks Tell PGM about CR3 changes before calling this helper.
1430 */
1431static int hmR0VmxSavePaePdpes(PVMCPU pVCpu, PCPUMCTX pCtx)
1432{
1433 if (CPUMIsGuestInPAEModeEx(pCtx))
1434 {
1435 int rc;
1436 X86PDPE aPdpes[4];
1437 rc = VMXReadVMCS64(VMX_VMCS_GUEST_PDPTR0_FULL, &aPdpes[0].u); AssertRCReturn(rc, rc);
1438 rc = VMXReadVMCS64(VMX_VMCS_GUEST_PDPTR1_FULL, &aPdpes[1].u); AssertRCReturn(rc, rc);
1439 rc = VMXReadVMCS64(VMX_VMCS_GUEST_PDPTR2_FULL, &aPdpes[2].u); AssertRCReturn(rc, rc);
1440 rc = VMXReadVMCS64(VMX_VMCS_GUEST_PDPTR3_FULL, &aPdpes[3].u); AssertRCReturn(rc, rc);
1441
1442 rc = PGMGstUpdatePaePdpes(pVCpu, &aPdpes[0]);
1443 AssertRCReturn(rc, rc);
1444 }
1445 return VINF_SUCCESS;
1446}
1447
1448
1449/**
1450 * Update the exception bitmap according to the current CPU state.
1451 *
1452 * @param pVM Pointer to the VM.
1453 * @param pVCpu Pointer to the VMCPU.
1454 * @param pCtx Pointer to the guest CPU context.
1455 */
1456static void hmR0VmxUpdateExceptionBitmap(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
1457{
1458 uint32_t u32TrapMask;
1459 Assert(pCtx);
1460
1461 /*
1462 * Set up a mask for intercepting traps.
1463 */
1464 /** @todo Do we really need to always intercept #DB? */
1465 u32TrapMask = RT_BIT(X86_XCPT_DB)
1466 | RT_BIT(X86_XCPT_NM)
1467#ifdef VBOX_ALWAYS_TRAP_PF
1468 | RT_BIT(X86_XCPT_PF)
1469#endif
1470#ifdef VBOX_STRICT
1471 | RT_BIT(X86_XCPT_BP)
1472 | RT_BIT(X86_XCPT_DB)
1473 | RT_BIT(X86_XCPT_DE)
1474 | RT_BIT(X86_XCPT_NM)
1475 | RT_BIT(X86_XCPT_UD)
1476 | RT_BIT(X86_XCPT_NP)
1477 | RT_BIT(X86_XCPT_SS)
1478 | RT_BIT(X86_XCPT_GP)
1479 | RT_BIT(X86_XCPT_MF)
1480#endif
1481 ;
1482
1483 /*
1484 * Without nested paging, #PF must be intercepted to implement shadow paging.
1485 */
1486 /** @todo NP state won't change so maybe we should build the initial trap mask up front? */
1487 if (!pVM->hwaccm.s.fNestedPaging)
1488 u32TrapMask |= RT_BIT(X86_XCPT_PF);
1489
1490 /* Catch floating point exceptions if we need to report them to the guest in a different way. */
1491 if (!(pCtx->cr0 & X86_CR0_NE))
1492 u32TrapMask |= RT_BIT(X86_XCPT_MF);
1493
1494#ifdef VBOX_STRICT
1495 Assert(u32TrapMask & RT_BIT(X86_XCPT_GP));
1496#endif
1497
1498 /*
1499 * Intercept all exceptions in real mode as none of them can be injected directly (#GP otherwise).
1500 */
1501 /** @todo Despite the claim to intercept everything, with NP we do not intercept #PF. Should we? */
1502 if ( CPUMIsGuestInRealModeEx(pCtx)
1503 && pVM->hwaccm.s.vmx.pRealModeTSS)
1504 {
1505 u32TrapMask |= RT_BIT(X86_XCPT_DE)
1506 | RT_BIT(X86_XCPT_DB)
1507 | RT_BIT(X86_XCPT_NMI)
1508 | RT_BIT(X86_XCPT_BP)
1509 | RT_BIT(X86_XCPT_OF)
1510 | RT_BIT(X86_XCPT_BR)
1511 | RT_BIT(X86_XCPT_UD)
1512 | RT_BIT(X86_XCPT_DF)
1513 | RT_BIT(X86_XCPT_CO_SEG_OVERRUN)
1514 | RT_BIT(X86_XCPT_TS)
1515 | RT_BIT(X86_XCPT_NP)
1516 | RT_BIT(X86_XCPT_SS)
1517 | RT_BIT(X86_XCPT_GP)
1518 | RT_BIT(X86_XCPT_MF)
1519 | RT_BIT(X86_XCPT_AC)
1520 | RT_BIT(X86_XCPT_MC)
1521 | RT_BIT(X86_XCPT_XF)
1522 ;
1523 }
1524
1525 int rc = VMXWriteVMCS(VMX_VMCS_CTRL_EXCEPTION_BITMAP, u32TrapMask);
1526 AssertRC(rc);
1527}
1528
1529
1530/**
1531 * Loads a minimal guest state.
1532 *
1533 * NOTE: Don't do anything here that can cause a jump back to ring 3!!!!!
1534 *
1535 * @param pVM Pointer to the VM.
1536 * @param pVCpu Pointer to the VMCPU.
1537 * @param pCtx Pointer to the guest CPU context.
1538 */
1539VMMR0DECL(void) VMXR0LoadMinimalGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
1540{
1541 int rc;
1542 X86EFLAGS eflags;
1543
1544 Assert(!(pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_ALL_GUEST));
1545
1546 /*
1547 * Load EIP, ESP and EFLAGS.
1548 */
1549 rc = VMXWriteVMCS64(VMX_VMCS64_GUEST_RIP, pCtx->rip);
1550 rc |= VMXWriteVMCS64(VMX_VMCS64_GUEST_RSP, pCtx->rsp);
1551 AssertRC(rc);
1552
1553 /*
1554 * Bits 22-31, 15, 5 & 3 must be zero. Bit 1 must be 1.
1555 */
1556 eflags = pCtx->eflags;
1557 eflags.u32 &= VMX_EFLAGS_RESERVED_0;
1558 eflags.u32 |= VMX_EFLAGS_RESERVED_1;
1559
1560 /*
1561 * Check if real mode emulation using v86 mode.
1562 */
1563 if ( CPUMIsGuestInRealModeEx(pCtx)
1564 && pVM->hwaccm.s.vmx.pRealModeTSS)
1565 {
1566 pVCpu->hwaccm.s.vmx.RealMode.eflags = eflags;
1567
1568 eflags.Bits.u1VM = 1;
1569 eflags.Bits.u2IOPL = 0; /* must always be 0 or else certain instructions won't cause faults. */
1570 }
1571 rc = VMXWriteVMCS(VMX_VMCS_GUEST_RFLAGS, eflags.u32);
1572 AssertRC(rc);
1573}
1574
1575
1576/**
1577 * Loads the guest state.
1578 *
1579 * NOTE: Don't do anything here that can cause a jump back to ring 3!!!!!
1580 *
1581 * @returns VBox status code.
1582 * @param pVM Pointer to the VM.
1583 * @param pVCpu Pointer to the VMCPU.
1584 * @param pCtx Pointer to the guest CPU context.
1585 */
1586VMMR0DECL(int) VMXR0LoadGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
1587{
1588 int rc = VINF_SUCCESS;
1589 RTGCUINTPTR val;
1590
1591 /*
1592 * VMX_VMCS_CTRL_ENTRY_CONTROLS
1593 * Set required bits to one and zero according to the MSR capabilities.
1594 */
1595 val = pVM->hwaccm.s.vmx.msr.vmx_entry.n.disallowed0;
1596
1597 /*
1598 * Load guest debug controls (DR7 & IA32_DEBUGCTL_MSR).
1599 * Forced to 1 on the 'first' VT-x capable CPUs; this actually includes the newest Nehalem CPUs
1600 */
1601 val |= VMX_VMCS_CTRL_ENTRY_CONTROLS_LOAD_DEBUG;
1602
1603 if (CPUMIsGuestInLongModeEx(pCtx))
1604 val |= VMX_VMCS_CTRL_ENTRY_CONTROLS_IA64_MODE;
1605 /* else Must be zero when AMD64 is not available. */
1606
1607 /*
1608 * Mask away the bits that the CPU doesn't support.
1609 */
1610 val &= pVM->hwaccm.s.vmx.msr.vmx_entry.n.allowed1;
1611 rc = VMXWriteVMCS(VMX_VMCS_CTRL_ENTRY_CONTROLS, val);
1612 AssertRC(rc);
1613
1614 /*
1615 * VMX_VMCS_CTRL_EXIT_CONTROLS
1616 * Set required bits to one and zero according to the MSR capabilities.
1617 */
1618 val = pVM->hwaccm.s.vmx.msr.vmx_exit.n.disallowed0;
1619
1620 /*
1621 * Save debug controls (DR7 & IA32_DEBUGCTL_MSR)
1622 * Forced to 1 on the 'first' VT-x capable CPUs; this actually includes the newest Nehalem CPUs
1623 */
1624 val |= VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_DEBUG;
1625
1626#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
1627 if (VMX_IS_64BIT_HOST_MODE())
1628 val |= VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_AMD64;
1629 /* else Must be zero when AMD64 is not available. */
1630#elif HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
1631 if (CPUMIsGuestInLongModeEx(pCtx))
1632 val |= VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_AMD64; /* our switcher goes to long mode */
1633 else
1634 Assert(!(val & VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_AMD64));
1635#endif
1636 val &= pVM->hwaccm.s.vmx.msr.vmx_exit.n.allowed1;
1637
1638 /*
1639 * Don't acknowledge external interrupts on VM-exit.
1640 */
1641 rc = VMXWriteVMCS(VMX_VMCS_CTRL_EXIT_CONTROLS, val);
1642 AssertRC(rc);
1643
1644 /*
1645 * Guest CPU context: ES, CS, SS, DS, FS, GS.
1646 */
1647 if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_SEGMENT_REGS)
1648 {
1649 if (pVM->hwaccm.s.vmx.pRealModeTSS)
1650 {
1651 PGMMODE enmGuestMode = PGMGetGuestMode(pVCpu);
1652 if (pVCpu->hwaccm.s.vmx.enmLastSeenGuestMode != enmGuestMode)
1653 {
1654 /*
1655 * Correct weird requirements for switching to protected mode.
1656 */
1657 if ( pVCpu->hwaccm.s.vmx.enmLastSeenGuestMode == PGMMODE_REAL
1658 && enmGuestMode >= PGMMODE_PROTECTED)
1659 {
1660#ifdef VBOX_WITH_REM
1661 /*
1662 * Flush the recompiler code cache as it's not unlikely the guest will rewrite code
1663 * it will later execute in real mode (OpenBSD 4.0 is one such example)
1664 */
1665 REMFlushTBs(pVM);
1666#endif
1667
1668 /*
1669 * DPL of all hidden selector registers must match the current CPL (0).
1670 */
1671 pCtx->cs.Attr.n.u2Dpl = 0;
1672 pCtx->cs.Attr.n.u4Type = X86_SEL_TYPE_CODE | X86_SEL_TYPE_RW_ACC;
1673
1674 pCtx->ds.Attr.n.u2Dpl = 0;
1675 pCtx->es.Attr.n.u2Dpl = 0;
1676 pCtx->fs.Attr.n.u2Dpl = 0;
1677 pCtx->gs.Attr.n.u2Dpl = 0;
1678 pCtx->ss.Attr.n.u2Dpl = 0;
1679 }
1680 pVCpu->hwaccm.s.vmx.enmLastSeenGuestMode = enmGuestMode;
1681 }
1682 else if ( CPUMIsGuestInRealModeEx(pCtx)
1683 && pCtx->cs.u64Base == 0xffff0000)
1684 {
1685 /* VT-x will fail with a guest invalid state otherwise... (CPU state after a reset) */
1686 pCtx->cs.u64Base = 0xf0000;
1687 pCtx->cs.Sel = 0xf000;
1688 }
1689 }
1690
1691 VMX_WRITE_SELREG(ES, es);
1692 AssertRC(rc);
1693
1694 VMX_WRITE_SELREG(CS, cs);
1695 AssertRC(rc);
1696
1697 VMX_WRITE_SELREG(SS, ss);
1698 AssertRC(rc);
1699
1700 VMX_WRITE_SELREG(DS, ds);
1701 AssertRC(rc);
1702
1703 VMX_WRITE_SELREG(FS, fs);
1704 AssertRC(rc);
1705
1706 VMX_WRITE_SELREG(GS, gs);
1707 AssertRC(rc);
1708 }
1709
1710 /*
1711 * Guest CPU context: LDTR.
1712 */
1713 if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_LDTR)
1714 {
1715 if (pCtx->ldtr.Sel == 0)
1716 {
1717 rc = VMXWriteVMCS(VMX_VMCS16_GUEST_FIELD_LDTR, 0);
1718 rc |= VMXWriteVMCS(VMX_VMCS32_GUEST_LDTR_LIMIT, 0);
1719 rc |= VMXWriteVMCS64(VMX_VMCS64_GUEST_LDTR_BASE, 0);
1720 /* Note: vmlaunch will fail with 0 or just 0x02. No idea why. */
1721 rc |= VMXWriteVMCS(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, 0x82 /* present, LDT */);
1722 }
1723 else
1724 {
1725 rc = VMXWriteVMCS(VMX_VMCS16_GUEST_FIELD_LDTR, pCtx->ldtr.Sel);
1726 rc |= VMXWriteVMCS(VMX_VMCS32_GUEST_LDTR_LIMIT, pCtx->ldtr.u32Limit);
1727 rc |= VMXWriteVMCS64(VMX_VMCS64_GUEST_LDTR_BASE, pCtx->ldtr.u64Base);
1728 rc |= VMXWriteVMCS(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, pCtx->ldtr.Attr.u);
1729 }
1730 AssertRC(rc);
1731 }
1732
1733 /*
1734 * Guest CPU context: TR.
1735 */
1736 if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_TR)
1737 {
1738 /*
1739 * Real mode emulation using v86 mode with CR4.VME (interrupt redirection
1740 * using the int bitmap in the TSS).
1741 */
1742 if ( CPUMIsGuestInRealModeEx(pCtx)
1743 && pVM->hwaccm.s.vmx.pRealModeTSS)
1744 {
1745 RTGCPHYS GCPhys;
1746
1747 /* We convert it here every time as PCI regions could be reconfigured. */
1748 rc = PDMVMMDevHeapR3ToGCPhys(pVM, pVM->hwaccm.s.vmx.pRealModeTSS, &GCPhys);
1749 AssertRC(rc);
1750
1751 rc = VMXWriteVMCS(VMX_VMCS16_GUEST_FIELD_TR, 0);
1752 rc |= VMXWriteVMCS(VMX_VMCS32_GUEST_TR_LIMIT, HWACCM_VTX_TSS_SIZE);
1753 rc |= VMXWriteVMCS64(VMX_VMCS64_GUEST_TR_BASE, GCPhys /* phys = virt in this mode */);
1754
1755 X86DESCATTR attr;
1756
1757 attr.u = 0;
1758 attr.n.u1Present = 1;
1759 attr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
1760 val = attr.u;
1761 }
1762 else
1763 {
1764 rc = VMXWriteVMCS(VMX_VMCS16_GUEST_FIELD_TR, pCtx->tr.Sel);
1765 rc |= VMXWriteVMCS(VMX_VMCS32_GUEST_TR_LIMIT, pCtx->tr.u32Limit);
1766 rc |= VMXWriteVMCS64(VMX_VMCS64_GUEST_TR_BASE, pCtx->tr.u64Base);
1767
1768 val = pCtx->tr.Attr.u;
1769
1770 /* The TSS selector must be busy (REM bugs? see defect #XXXX). */
1771 if (!(val & X86_SEL_TYPE_SYS_TSS_BUSY_MASK))
1772 {
1773 if (val & 0xf)
1774 val |= X86_SEL_TYPE_SYS_TSS_BUSY_MASK;
1775 else
1776 /* Default if no TR selector has been set (otherwise vmlaunch will fail!) */
1777 val = (val & ~0xF) | X86_SEL_TYPE_SYS_386_TSS_BUSY;
1778 }
1779 AssertMsg((val & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY || (val & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY,
1780 ("%#x\n", val));
1781 }
1782 rc |= VMXWriteVMCS(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, val);
1783 AssertRC(rc);
1784 }
1785
1786 /*
1787 * Guest CPU context: GDTR.
1788 */
1789 if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_GDTR)
1790 {
1791 rc = VMXWriteVMCS(VMX_VMCS32_GUEST_GDTR_LIMIT, pCtx->gdtr.cbGdt);
1792 rc |= VMXWriteVMCS64(VMX_VMCS64_GUEST_GDTR_BASE, pCtx->gdtr.pGdt);
1793 AssertRC(rc);
1794 }
1795
1796 /*
1797 * Guest CPU context: IDTR.
1798 */
1799 if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_IDTR)
1800 {
1801 rc = VMXWriteVMCS(VMX_VMCS32_GUEST_IDTR_LIMIT, pCtx->idtr.cbIdt);
1802 rc |= VMXWriteVMCS64(VMX_VMCS64_GUEST_IDTR_BASE, pCtx->idtr.pIdt);
1803 AssertRC(rc);
1804 }
1805
1806 /*
1807 * Sysenter MSRs.
1808 */
1809 if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_MSR)
1810 {
1811 rc = VMXWriteVMCS(VMX_VMCS32_GUEST_SYSENTER_CS, pCtx->SysEnter.cs);
1812 rc |= VMXWriteVMCS64(VMX_VMCS64_GUEST_SYSENTER_EIP, pCtx->SysEnter.eip);
1813 rc |= VMXWriteVMCS64(VMX_VMCS64_GUEST_SYSENTER_ESP, pCtx->SysEnter.esp);
1814 AssertRC(rc);
1815 }
1816
1817 /*
1818 * Guest CPU context: Control registers.
1819 */
1820 if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_CR0)
1821 {
1822 val = pCtx->cr0;
1823 rc = VMXWriteVMCS(VMX_VMCS_CTRL_CR0_READ_SHADOW, val);
1824 Log2(("Guest CR0-shadow %08x\n", val));
1825 if (CPUMIsGuestFPUStateActive(pVCpu) == false)
1826 {
1827 /* Always use #NM exceptions to load the FPU/XMM state on demand. */
1828 val |= X86_CR0_TS | X86_CR0_ET | X86_CR0_NE | X86_CR0_MP;
1829 }
1830 else
1831 {
1832 /** @todo check if we support the old style mess correctly. */
1833 if (!(val & X86_CR0_NE))
1834 Log(("Forcing X86_CR0_NE!!!\n"));
1835
1836 val |= X86_CR0_NE; /* always turn on the native mechanism to report FPU errors (old style uses interrupts) */
1837 }
1838 /* Protected mode & paging are always enabled; we use them for emulating real and protected mode without paging too. */
1839 if (!pVM->hwaccm.s.vmx.fUnrestrictedGuest)
1840 val |= X86_CR0_PE | X86_CR0_PG;
1841
1842 if (pVM->hwaccm.s.fNestedPaging)
1843 {
1844 if (CPUMIsGuestInPagedProtectedModeEx(pCtx))
1845 {
1846 /* Disable CR3 read/write monitoring as we don't need it for EPT. */
1847 pVCpu->hwaccm.s.vmx.proc_ctls &= ~( VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
1848 | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT);
1849 }
1850 else
1851 {
1852 /* Reenable CR3 read/write monitoring as our identity mapped page table is active. */
1853 pVCpu->hwaccm.s.vmx.proc_ctls |= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
1854 | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT;
1855 }
1856 rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
1857 AssertRC(rc);
1858 }
1859 else
1860 {
1861 /* Note: We must also set this as we rely on protecting various pages for which supervisor writes must be caught. */
1862 val |= X86_CR0_WP;
1863 }
1864
1865 /* Always enable caching. */
1866 val &= ~(X86_CR0_CD|X86_CR0_NW);
1867
1868 rc |= VMXWriteVMCS64(VMX_VMCS64_GUEST_CR0, val);
1869 Log2(("Guest CR0 %08x\n", val));
1870
1871 /*
1872 * CR0 flags owned by the host; if the guests attempts to change them, then the VM will exit.
1873 */
1874 val = X86_CR0_PE /* Must monitor this bit (assumptions are made for real mode emulation) */
1875 | X86_CR0_WP /* Must monitor this bit (it must always be enabled). */
1876 | X86_CR0_PG /* Must monitor this bit (assumptions are made for real mode & protected mode without paging emulation) */
1877 | X86_CR0_CD /* Bit not restored during VM-exit! */
1878 | X86_CR0_NW /* Bit not restored during VM-exit! */
1879 | X86_CR0_NE;
1880
1881 /*
1882 * When the guest's FPU state is active, then we no longer care about the FPU related bits.
1883 */
1884 if (CPUMIsGuestFPUStateActive(pVCpu) == false)
1885 val |= X86_CR0_TS | X86_CR0_ET | X86_CR0_MP;
1886
1887 pVCpu->hwaccm.s.vmx.cr0_mask = val;
1888
1889 rc |= VMXWriteVMCS(VMX_VMCS_CTRL_CR0_MASK, val);
1890 Log2(("Guest CR0-mask %08x\n", val));
1891 AssertRC(rc);
1892 }
1893
1894 if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_CR4)
1895 {
1896 rc = VMXWriteVMCS(VMX_VMCS_CTRL_CR4_READ_SHADOW, pCtx->cr4);
1897 Log2(("Guest CR4-shadow %08x\n", pCtx->cr4));
1898 /* Set the required bits in cr4 too (currently X86_CR4_VMXE). */
1899 val = pCtx->cr4 | (uint32_t)pVM->hwaccm.s.vmx.msr.vmx_cr4_fixed0;
1900
1901 if (!pVM->hwaccm.s.fNestedPaging)
1902 {
1903 switch(pVCpu->hwaccm.s.enmShadowMode)
1904 {
1905 case PGMMODE_REAL: /* Real mode -> emulated using v86 mode */
1906 case PGMMODE_PROTECTED: /* Protected mode, no paging -> emulated using identity mapping. */
1907 case PGMMODE_32_BIT: /* 32-bit paging. */
1908 val &= ~X86_CR4_PAE;
1909 break;
1910
1911 case PGMMODE_PAE: /* PAE paging. */
1912 case PGMMODE_PAE_NX: /* PAE paging with NX enabled. */
1913 /** Must use PAE paging as we could use physical memory > 4 GB */
1914 val |= X86_CR4_PAE;
1915 break;
1916
1917 case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
1918 case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
1919#ifdef VBOX_ENABLE_64_BITS_GUESTS
1920 break;
1921#else
1922 AssertFailed();
1923 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
1924#endif
1925 default: /* shut up gcc */
1926 AssertFailed();
1927 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
1928 }
1929 }
1930 else if ( !CPUMIsGuestInPagedProtectedModeEx(pCtx)
1931 && !pVM->hwaccm.s.vmx.fUnrestrictedGuest)
1932 {
1933 /* We use 4 MB pages in our identity mapping page table for real and protected mode without paging. */
1934 val |= X86_CR4_PSE;
1935 /* Our identity mapping is a 32 bits page directory. */
1936 val &= ~X86_CR4_PAE;
1937 }
1938
1939 /*
1940 * Turn off VME if we're in emulated real mode.
1941 */
1942 if ( CPUMIsGuestInRealModeEx(pCtx)
1943 && pVM->hwaccm.s.vmx.pRealModeTSS)
1944 {
1945 val &= ~X86_CR4_VME;
1946 }
1947
1948 rc |= VMXWriteVMCS64(VMX_VMCS64_GUEST_CR4, val);
1949 Log2(("Guest CR4 %08x\n", val));
1950
1951 /*
1952 * CR4 flags owned by the host; if the guests attempts to change them, then the VM will exit.
1953 */
1954 val = 0
1955 | X86_CR4_VME
1956 | X86_CR4_PAE
1957 | X86_CR4_PGE
1958 | X86_CR4_PSE
1959 | X86_CR4_VMXE;
1960 pVCpu->hwaccm.s.vmx.cr4_mask = val;
1961
1962 rc |= VMXWriteVMCS(VMX_VMCS_CTRL_CR4_MASK, val);
1963 Log2(("Guest CR4-mask %08x\n", val));
1964 AssertRC(rc);
1965 }
1966
1967#if 0
1968 /* Enable single stepping if requested and CPU supports it. */
1969 if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_TRAP_FLAG)
1970 if (DBGFIsStepping(pVCpu))
1971 {
1972 pVCpu->hwaccm.s.vmx.proc_ctls |= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_TRAP_FLAG;
1973 rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
1974 AssertRC(rc);
1975 }
1976#endif
1977
1978 if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_CR3)
1979 {
1980 if (pVM->hwaccm.s.fNestedPaging)
1981 {
1982 Assert(PGMGetHyperCR3(pVCpu));
1983 pVCpu->hwaccm.s.vmx.GCPhysEPTP = PGMGetHyperCR3(pVCpu);
1984
1985 Assert(!(pVCpu->hwaccm.s.vmx.GCPhysEPTP & 0xfff));
1986 /** @todo Check the IA32_VMX_EPT_VPID_CAP MSR for other supported memory types. */
1987 pVCpu->hwaccm.s.vmx.GCPhysEPTP |= VMX_EPT_MEMTYPE_WB
1988 | (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
1989
1990 rc = VMXWriteVMCS64(VMX_VMCS_CTRL_EPTP_FULL, pVCpu->hwaccm.s.vmx.GCPhysEPTP);
1991 AssertRC(rc);
1992
1993 if ( !CPUMIsGuestInPagedProtectedModeEx(pCtx)
1994 && !pVM->hwaccm.s.vmx.fUnrestrictedGuest)
1995 {
1996 RTGCPHYS GCPhys;
1997
1998 /* We convert it here every time as PCI regions could be reconfigured. */
1999 rc = PDMVMMDevHeapR3ToGCPhys(pVM, pVM->hwaccm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
2000 AssertMsgRC(rc, ("pNonPagingModeEPTPageTable = %RGv\n", pVM->hwaccm.s.vmx.pNonPagingModeEPTPageTable));
2001
2002 /*
2003 * We use our identity mapping page table here as we need to map guest virtual to
2004 * guest physical addresses; EPT will take care of the translation to host physical addresses.
2005 */
2006 val = GCPhys;
2007 }
2008 else
2009 {
2010 /* Save the real guest CR3 in VMX_VMCS_GUEST_CR3 */
2011 val = pCtx->cr3;
2012 rc = hmR0VmxLoadPaePdpes(pVCpu, pCtx);
2013 AssertRCReturn(rc, rc);
2014 }
2015 }
2016 else
2017 {
2018 val = PGMGetHyperCR3(pVCpu);
2019 Assert(val || VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL));
2020 }
2021
2022 /* Save our shadow CR3 register. */
2023 rc = VMXWriteVMCS64(VMX_VMCS64_GUEST_CR3, val);
2024 AssertRC(rc);
2025 }
2026
2027 /*
2028 * Guest CPU context: Debug registers.
2029 */
2030 if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_DEBUG)
2031 {
2032 pCtx->dr[6] |= X86_DR6_INIT_VAL; /* set all reserved bits to 1. */
2033 pCtx->dr[6] &= ~RT_BIT(12); /* must be zero. */
2034
2035 pCtx->dr[7] &= 0xffffffff; /* upper 32 bits reserved */
2036 pCtx->dr[7] &= ~(RT_BIT(11) | RT_BIT(12) | RT_BIT(14) | RT_BIT(15)); /* must be zero */
2037 pCtx->dr[7] |= 0x400; /* must be one */
2038
2039 /* Resync DR7 */
2040 rc = VMXWriteVMCS64(VMX_VMCS64_GUEST_DR7, pCtx->dr[7]);
2041 AssertRC(rc);
2042
2043#ifdef DEBUG
2044 /* Sync the hypervisor debug state now if any breakpoint is armed. */
2045 if ( CPUMGetHyperDR7(pVCpu) & (X86_DR7_ENABLED_MASK|X86_DR7_GD)
2046 && !CPUMIsHyperDebugStateActive(pVCpu)
2047 && !DBGFIsStepping(pVCpu))
2048 {
2049 /* Save the host and load the hypervisor debug state. */
2050 rc = CPUMR0LoadHyperDebugState(pVM, pVCpu, pCtx, true /* include DR6 */);
2051 AssertRC(rc);
2052
2053 /* DRx intercepts remain enabled. */
2054
2055 /* Override dr7 with the hypervisor value. */
2056 rc = VMXWriteVMCS64(VMX_VMCS64_GUEST_DR7, CPUMGetHyperDR7(pVCpu));
2057 AssertRC(rc);
2058 }
2059 else
2060#endif
2061 /* Sync the debug state now if any breakpoint is armed. */
2062 if ( (pCtx->dr[7] & (X86_DR7_ENABLED_MASK|X86_DR7_GD))
2063 && !CPUMIsGuestDebugStateActive(pVCpu)
2064 && !DBGFIsStepping(pVCpu))
2065 {
2066 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatDRxArmed);
2067
2068 /* Disable DRx move intercepts. */
2069 pVCpu->hwaccm.s.vmx.proc_ctls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT;
2070 rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
2071 AssertRC(rc);
2072
2073 /* Save the host and load the guest debug state. */
2074 rc = CPUMR0LoadGuestDebugState(pVM, pVCpu, pCtx, true /* include DR6 */);
2075 AssertRC(rc);
2076 }
2077
2078 /* IA32_DEBUGCTL MSR. */
2079 rc = VMXWriteVMCS64(VMX_VMCS_GUEST_DEBUGCTL_FULL, 0);
2080 AssertRC(rc);
2081
2082 /** @todo do we really ever need this? */
2083 rc |= VMXWriteVMCS(VMX_VMCS_GUEST_DEBUG_EXCEPTIONS, 0);
2084 AssertRC(rc);
2085 }
2086
2087 /*
2088 * 64-bit guest mode.
2089 */
2090 if (CPUMIsGuestInLongModeEx(pCtx))
2091 {
2092#if !defined(VBOX_ENABLE_64_BITS_GUESTS)
2093 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
2094#elif HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2095 pVCpu->hwaccm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
2096#else
2097# ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2098 if (!pVM->hwaccm.s.fAllow64BitGuests)
2099 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
2100# endif
2101 pVCpu->hwaccm.s.vmx.pfnStartVM = VMXR0StartVM64;
2102#endif
2103 if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_MSR)
2104 {
2105 /* Update these as wrmsr might have changed them. */
2106 rc = VMXWriteVMCS64(VMX_VMCS64_GUEST_FS_BASE, pCtx->fs.u64Base);
2107 AssertRC(rc);
2108 rc = VMXWriteVMCS64(VMX_VMCS64_GUEST_GS_BASE, pCtx->gs.u64Base);
2109 AssertRC(rc);
2110 }
2111 }
2112 else
2113 {
2114 pVCpu->hwaccm.s.vmx.pfnStartVM = VMXR0StartVM32;
2115 }
2116
2117 hmR0VmxUpdateExceptionBitmap(pVM, pVCpu, pCtx);
2118
2119#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
2120 /*
2121 * Store all guest MSRs in the VM-entry load area, so they will be loaded
2122 * during the world switch.
2123 */
2124 PVMXMSR pMsr = (PVMXMSR)pVCpu->hwaccm.s.vmx.pGuestMSR;
2125 unsigned idxMsr = 0;
2126
2127 uint32_t u32GstExtFeatures;
2128 uint32_t u32Temp;
2129 CPUMGetGuestCpuId(pVCpu, 0x80000001, &u32Temp, &u32Temp, &u32Temp, &u32GstExtFeatures);
2130
2131 /*
2132 * Check if EFER MSR present.
2133 */
2134 if (u32GstExtFeatures & (X86_CPUID_EXT_FEATURE_EDX_NX | X86_CPUID_EXT_FEATURE_EDX_LONG_MODE))
2135 {
2136 pMsr->u32IndexMSR = MSR_K6_EFER;
2137 pMsr->u32Reserved = 0;
2138 pMsr->u64Value = pCtx->msrEFER;
2139 /* VT-x will complain if only MSR_K6_EFER_LME is set. */
2140 if (!CPUMIsGuestInLongModeEx(pCtx))
2141 pMsr->u64Value &= ~(MSR_K6_EFER_LMA | MSR_K6_EFER_LME);
2142 pMsr++; idxMsr++;
2143
2144 if (u32GstExtFeatures & X86_CPUID_EXT_FEATURE_EDX_LONG_MODE)
2145 {
2146 pMsr->u32IndexMSR = MSR_K8_LSTAR;
2147 pMsr->u32Reserved = 0;
2148 pMsr->u64Value = pCtx->msrLSTAR; /* 64 bits mode syscall rip */
2149 pMsr++; idxMsr++;
2150 pMsr->u32IndexMSR = MSR_K6_STAR;
2151 pMsr->u32Reserved = 0;
2152 pMsr->u64Value = pCtx->msrSTAR; /* legacy syscall eip, cs & ss */
2153 pMsr++; idxMsr++;
2154 pMsr->u32IndexMSR = MSR_K8_SF_MASK;
2155 pMsr->u32Reserved = 0;
2156 pMsr->u64Value = pCtx->msrSFMASK; /* syscall flag mask */
2157 pMsr++; idxMsr++;
2158
2159 /* The KERNEL_GS_BASE MSR doesn't work reliably with auto load/store. See @bugref{6208} */
2160#if 0
2161 pMsr->u32IndexMSR = MSR_K8_KERNEL_GS_BASE;
2162 pMsr->u32Reserved = 0;
2163 pMsr->u64Value = pCtx->msrKERNELGSBASE; /* swapgs exchange value */
2164 pMsr++; idxMsr++;
2165#endif
2166 }
2167 }
2168
2169 if ( pVCpu->hwaccm.s.vmx.proc_ctls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP
2170 && (u32GstExtFeatures & X86_CPUID_EXT_FEATURE_EDX_RDTSCP))
2171 {
2172 pMsr->u32IndexMSR = MSR_K8_TSC_AUX;
2173 pMsr->u32Reserved = 0;
2174 rc = CPUMQueryGuestMsr(pVCpu, MSR_K8_TSC_AUX, &pMsr->u64Value);
2175 AssertRC(rc);
2176 pMsr++; idxMsr++;
2177 }
2178
2179 /*
2180 * The number of host MSRs saved must be identical to the number of guest MSRs loaded.
2181 * It's not a VT-x requirement but how it's practically used here.
2182 */
2183 Assert(pVCpu->hwaccm.s.vmx.cCachedMSRs == idxMsr);
2184
2185 rc = VMXWriteVMCS(VMX_VMCS_CTRL_ENTRY_MSR_LOAD_COUNT, idxMsr);
2186 AssertRC(rc);
2187
2188 rc = VMXWriteVMCS(VMX_VMCS_CTRL_EXIT_MSR_STORE_COUNT, idxMsr);
2189 AssertRC(rc);
2190#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
2191
2192 bool fOffsettedTsc;
2193 if (pVM->hwaccm.s.vmx.fUsePreemptTimer)
2194 {
2195 uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVCpu, &fOffsettedTsc, &pVCpu->hwaccm.s.vmx.u64TSCOffset);
2196
2197 /* Make sure the returned values have sane upper and lower boundaries. */
2198 uint64_t u64CpuHz = SUPGetCpuHzFromGIP(g_pSUPGlobalInfoPage);
2199
2200 cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64 of a second */
2201 cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
2202
2203 cTicksToDeadline >>= pVM->hwaccm.s.vmx.cPreemptTimerShift;
2204 uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
2205 rc = VMXWriteVMCS(VMX_VMCS32_GUEST_PREEMPTION_TIMER_VALUE, cPreemptionTickCount);
2206 AssertRC(rc);
2207 }
2208 else
2209 fOffsettedTsc = TMCpuTickCanUseRealTSC(pVCpu, &pVCpu->hwaccm.s.vmx.u64TSCOffset);
2210
2211 if (fOffsettedTsc)
2212 {
2213 uint64_t u64CurTSC = ASMReadTSC();
2214 if (u64CurTSC + pVCpu->hwaccm.s.vmx.u64TSCOffset >= TMCpuTickGetLastSeen(pVCpu))
2215 {
2216 /* Note: VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT takes precedence over TSC_OFFSET, applies to RDTSCP too. */
2217 rc = VMXWriteVMCS64(VMX_VMCS_CTRL_TSC_OFFSET_FULL, pVCpu->hwaccm.s.vmx.u64TSCOffset);
2218 AssertRC(rc);
2219
2220 pVCpu->hwaccm.s.vmx.proc_ctls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT;
2221 rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
2222 AssertRC(rc);
2223 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatTSCOffset);
2224 }
2225 else
2226 {
2227 /* Fall back to rdtsc, rdtscp emulation as we would otherwise pass decreasing tsc values to the guest. */
2228 LogFlow(("TSC %RX64 offset %RX64 time=%RX64 last=%RX64 (diff=%RX64, virt_tsc=%RX64)\n", u64CurTSC,
2229 pVCpu->hwaccm.s.vmx.u64TSCOffset, u64CurTSC + pVCpu->hwaccm.s.vmx.u64TSCOffset,
2230 TMCpuTickGetLastSeen(pVCpu), TMCpuTickGetLastSeen(pVCpu) - u64CurTSC - pVCpu->hwaccm.s.vmx.u64TSCOffset,
2231 TMCpuTickGet(pVCpu)));
2232 pVCpu->hwaccm.s.vmx.proc_ctls |= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT;
2233 rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
2234 AssertRC(rc);
2235 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatTSCInterceptOverFlow);
2236 }
2237 }
2238 else
2239 {
2240 pVCpu->hwaccm.s.vmx.proc_ctls |= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT;
2241 rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
2242 AssertRC(rc);
2243 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatTSCIntercept);
2244 }
2245
2246 /* Done with the major changes */
2247 pVCpu->hwaccm.s.fContextUseFlags &= ~HWACCM_CHANGED_ALL_GUEST;
2248
2249 /* Minimal guest state update (ESP, EIP, EFLAGS mostly) */
2250 VMXR0LoadMinimalGuestState(pVM, pVCpu, pCtx);
2251 return rc;
2252}
2253
2254
2255/**
2256 * Syncs back the guest state from VMCS.
2257 *
2258 * @returns VBox status code.
2259 * @param pVM Pointer to the VM.
2260 * @param pVCpu Pointer to the VMCPU.
2261 * @param pCtx Pointer to the guest CPU context.
2262 */
2263DECLINLINE(int) VMXR0SaveGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
2264{
2265 RTGCUINTREG val, valShadow;
2266 RTGCUINTPTR uInterruptState;
2267 int rc;
2268
2269 /* First sync back EIP, ESP, and EFLAGS. */
2270 rc = VMXReadCachedVMCS(VMX_VMCS64_GUEST_RIP, &val);
2271 AssertRC(rc);
2272 pCtx->rip = val;
2273 rc = VMXReadCachedVMCS(VMX_VMCS64_GUEST_RSP, &val);
2274 AssertRC(rc);
2275 pCtx->rsp = val;
2276 rc = VMXReadCachedVMCS(VMX_VMCS_GUEST_RFLAGS, &val);
2277 AssertRC(rc);
2278 pCtx->eflags.u32 = val;
2279
2280 /* Take care of instruction fusing (sti, mov ss) */
2281 rc |= VMXReadCachedVMCS(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &val);
2282 uInterruptState = val;
2283 if (uInterruptState != 0)
2284 {
2285 Assert(uInterruptState <= 2); /* only sti & mov ss */
2286 Log(("uInterruptState %x eip=%RGv\n", (uint32_t)uInterruptState, pCtx->rip));
2287 EMSetInhibitInterruptsPC(pVCpu, pCtx->rip);
2288 }
2289 else
2290 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
2291
2292 /* Control registers. */
2293 VMXReadCachedVMCS(VMX_VMCS_CTRL_CR0_READ_SHADOW, &valShadow);
2294 VMXReadCachedVMCS(VMX_VMCS64_GUEST_CR0, &val);
2295 val = (valShadow & pVCpu->hwaccm.s.vmx.cr0_mask) | (val & ~pVCpu->hwaccm.s.vmx.cr0_mask);
2296 CPUMSetGuestCR0(pVCpu, val);
2297
2298 VMXReadCachedVMCS(VMX_VMCS_CTRL_CR4_READ_SHADOW, &valShadow);
2299 VMXReadCachedVMCS(VMX_VMCS64_GUEST_CR4, &val);
2300 val = (valShadow & pVCpu->hwaccm.s.vmx.cr4_mask) | (val & ~pVCpu->hwaccm.s.vmx.cr4_mask);
2301 CPUMSetGuestCR4(pVCpu, val);
2302
2303 /*
2304 * No reason to sync back the CRx registers. They can't be changed by the guest unless in
2305 * the nested paging case where CR3 & CR4 can be changed by the guest.
2306 */
2307 if ( pVM->hwaccm.s.fNestedPaging
2308 && CPUMIsGuestInPagedProtectedModeEx(pCtx)) /** @todo check if we will always catch mode switches and such... */
2309 {
2310 PVMCSCACHE pCache = &pVCpu->hwaccm.s.vmx.VMCSCache;
2311
2312 /* Can be updated behind our back in the nested paging case. */
2313 CPUMSetGuestCR2(pVCpu, pCache->cr2);
2314
2315 VMXReadCachedVMCS(VMX_VMCS64_GUEST_CR3, &val);
2316
2317 if (val != pCtx->cr3)
2318 {
2319 CPUMSetGuestCR3(pVCpu, val);
2320 PGMUpdateCR3(pVCpu, val);
2321 }
2322 rc = hmR0VmxSavePaePdpes(pVCpu, pCtx);
2323 AssertRCReturn(rc, rc);
2324 }
2325
2326 /* Sync back DR7. */
2327 VMXReadCachedVMCS(VMX_VMCS64_GUEST_DR7, &val);
2328 pCtx->dr[7] = val;
2329
2330 /* Guest CPU context: ES, CS, SS, DS, FS, GS. */
2331 VMX_READ_SELREG(ES, es);
2332 VMX_READ_SELREG(SS, ss);
2333 VMX_READ_SELREG(CS, cs);
2334 VMX_READ_SELREG(DS, ds);
2335 VMX_READ_SELREG(FS, fs);
2336 VMX_READ_SELREG(GS, gs);
2337
2338 /* System MSRs */
2339 VMXReadCachedVMCS(VMX_VMCS32_GUEST_SYSENTER_CS, &val);
2340 pCtx->SysEnter.cs = val;
2341 VMXReadCachedVMCS(VMX_VMCS64_GUEST_SYSENTER_EIP, &val);
2342 pCtx->SysEnter.eip = val;
2343 VMXReadCachedVMCS(VMX_VMCS64_GUEST_SYSENTER_ESP, &val);
2344 pCtx->SysEnter.esp = val;
2345
2346 /* Misc. registers; must sync everything otherwise we can get out of sync when jumping to ring 3. */
2347 VMX_READ_SELREG(LDTR, ldtr);
2348
2349 VMXReadCachedVMCS(VMX_VMCS32_GUEST_GDTR_LIMIT, &val);
2350 pCtx->gdtr.cbGdt = val;
2351 VMXReadCachedVMCS(VMX_VMCS64_GUEST_GDTR_BASE, &val);
2352 pCtx->gdtr.pGdt = val;
2353
2354 VMXReadCachedVMCS(VMX_VMCS32_GUEST_IDTR_LIMIT, &val);
2355 pCtx->idtr.cbIdt = val;
2356 VMXReadCachedVMCS(VMX_VMCS64_GUEST_IDTR_BASE, &val);
2357 pCtx->idtr.pIdt = val;
2358
2359 /* Real mode emulation using v86 mode. */
2360 if ( CPUMIsGuestInRealModeEx(pCtx)
2361 && pVM->hwaccm.s.vmx.pRealModeTSS)
2362 {
2363 /* Hide our emulation flags */
2364 pCtx->eflags.Bits.u1VM = 0;
2365
2366 /* Restore original IOPL setting as we always use 0. */
2367 pCtx->eflags.Bits.u2IOPL = pVCpu->hwaccm.s.vmx.RealMode.eflags.Bits.u2IOPL;
2368
2369 /* Force a TR resync every time in case we switch modes. */
2370 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_GUEST_TR;
2371 }
2372 else
2373 {
2374 /* In real mode we have a fake TSS, so only sync it back when it's supposed to be valid. */
2375 VMX_READ_SELREG(TR, tr);
2376 }
2377
2378#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
2379 /*
2380 * Save the possibly changed MSRs that we automatically restore and save during a world switch.
2381 */
2382 for (unsigned i = 0; i < pVCpu->hwaccm.s.vmx.cCachedMSRs; i++)
2383 {
2384 PVMXMSR pMsr = (PVMXMSR)pVCpu->hwaccm.s.vmx.pGuestMSR;
2385 pMsr += i;
2386
2387 switch (pMsr->u32IndexMSR)
2388 {
2389 case MSR_K8_LSTAR:
2390 pCtx->msrLSTAR = pMsr->u64Value;
2391 break;
2392 case MSR_K6_STAR:
2393 pCtx->msrSTAR = pMsr->u64Value;
2394 break;
2395 case MSR_K8_SF_MASK:
2396 pCtx->msrSFMASK = pMsr->u64Value;
2397 break;
2398
2399 /* The KERNEL_GS_BASE MSR doesn't work reliably with auto load/store. See @bugref{6208} */
2400#if 0
2401 case MSR_K8_KERNEL_GS_BASE:
2402 pCtx->msrKERNELGSBASE = pMsr->u64Value;
2403 break;
2404#endif
2405 case MSR_K8_TSC_AUX:
2406 CPUMSetGuestMsr(pVCpu, MSR_K8_TSC_AUX, pMsr->u64Value);
2407 break;
2408
2409 case MSR_K6_EFER:
2410 /* EFER can't be changed without causing a VM-exit. */
2411 /* Assert(pCtx->msrEFER == pMsr->u64Value); */
2412 break;
2413 default:
2414 AssertFailed();
2415 return VERR_HM_UNEXPECTED_LD_ST_MSR;
2416 }
2417 }
2418#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
2419 return VINF_SUCCESS;
2420}
2421
2422
2423/**
2424 * Dummy placeholder for TLB flush handling before VM-entry. Used in the case
2425 * where neither EPT nor VPID is supported by the CPU.
2426 *
2427 * @param pVM Pointer to the VM.
2428 * @param pVCpu Pointer to the VMCPU.
2429 */
2430static DECLCALLBACK(void) hmR0VmxSetupTLBDummy(PVM pVM, PVMCPU pVCpu)
2431{
2432 NOREF(pVM);
2433 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
2434 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
2435 pVCpu->hwaccm.s.TlbShootdown.cPages = 0;
2436 return;
2437}
2438
2439
2440/**
2441 * Setup the tagged TLB for EPT+VPID.
2442 *
2443 * @param pVM Pointer to the VM.
2444 * @param pVCpu Pointer to the VMCPU.
2445 */
2446static DECLCALLBACK(void) hmR0VmxSetupTLBBoth(PVM pVM, PVMCPU pVCpu)
2447{
2448 PHMGLOBLCPUINFO pCpu;
2449
2450 Assert(pVM->hwaccm.s.fNestedPaging && pVM->hwaccm.s.vmx.fVPID);
2451
2452 pCpu = HWACCMR0GetCurrentCpu();
2453
2454 /*
2455 * Force a TLB flush for the first world switch if the current CPU differs from the one we ran on last
2456 * This can happen both for start & resume due to long jumps back to ring-3.
2457 * If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
2458 * or the host Cpu is online after a suspend/resume, so we cannot reuse the current ASID anymore.
2459 */
2460 bool fNewASID = false;
2461 if ( pVCpu->hwaccm.s.idLastCpu != pCpu->idCpu
2462 || pVCpu->hwaccm.s.cTLBFlushes != pCpu->cTLBFlushes)
2463 {
2464 pVCpu->hwaccm.s.fForceTLBFlush = true;
2465 fNewASID = true;
2466 }
2467
2468 /*
2469 * Check for explicit TLB shootdowns.
2470 */
2471 if (VMCPU_FF_TESTANDCLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2472 pVCpu->hwaccm.s.fForceTLBFlush = true;
2473
2474 pVCpu->hwaccm.s.idLastCpu = pCpu->idCpu;
2475
2476 if (pVCpu->hwaccm.s.fForceTLBFlush)
2477 {
2478 if (fNewASID)
2479 {
2480 ++pCpu->uCurrentASID;
2481 if (pCpu->uCurrentASID >= pVM->hwaccm.s.uMaxASID)
2482 {
2483 pCpu->uCurrentASID = 1; /* start at 1; host uses 0 */
2484 pCpu->cTLBFlushes++;
2485 pCpu->fFlushASIDBeforeUse = true;
2486 }
2487
2488 pVCpu->hwaccm.s.uCurrentASID = pCpu->uCurrentASID;
2489 if (pCpu->fFlushASIDBeforeUse)
2490 {
2491 hmR0VmxFlushVPID(pVM, pVCpu, pVM->hwaccm.s.vmx.enmFlushVPID, 0 /* GCPtr */);
2492#ifdef VBOX_WITH_STATISTICS
2493 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatFlushASID);
2494#endif
2495 }
2496 }
2497 else
2498 {
2499 if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVVPID_CAPS_SINGLE_CONTEXT)
2500 hmR0VmxFlushVPID(pVM, pVCpu, VMX_FLUSH_VPID_SINGLE_CONTEXT, 0 /* GCPtr */);
2501 else
2502 hmR0VmxFlushEPT(pVM, pVCpu, pVM->hwaccm.s.vmx.enmFlushEPT);
2503
2504#ifdef VBOX_WITH_STATISTICS
2505 /*
2506 * This is not terribly accurate (i.e. we don't have any StatFlushEPT counter). We currently count these
2507 * as ASID flushes too, better than including them under StatFlushTLBWorldSwitch.
2508 */
2509 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatFlushASID);
2510#endif
2511 }
2512
2513 pVCpu->hwaccm.s.cTLBFlushes = pCpu->cTLBFlushes;
2514 pVCpu->hwaccm.s.fForceTLBFlush = false;
2515 }
2516 else
2517 {
2518 AssertMsg(pVCpu->hwaccm.s.uCurrentASID && pCpu->uCurrentASID,
2519 ("hwaccm->uCurrentASID=%lu hwaccm->cTLBFlushes=%lu cpu->uCurrentASID=%lu cpu->cTLBFlushes=%lu\n",
2520 pVCpu->hwaccm.s.uCurrentASID, pVCpu->hwaccm.s.cTLBFlushes,
2521 pCpu->uCurrentASID, pCpu->cTLBFlushes));
2522
2523 /** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
2524 * not be executed. See hwaccmQueueInvlPage() where it is commented
2525 * out. Support individual entry flushing someday. */
2526 if (VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
2527 {
2528 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatTlbShootdown);
2529
2530 /*
2531 * Flush individual guest entries using VPID from the TLB or as little as possible with EPT
2532 * as supported by the CPU.
2533 */
2534 if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVVPID_CAPS_INDIV_ADDR)
2535 {
2536 for (unsigned i = 0; i < pVCpu->hwaccm.s.TlbShootdown.cPages; i++)
2537 hmR0VmxFlushVPID(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, pVCpu->hwaccm.s.TlbShootdown.aPages[i]);
2538 }
2539 else
2540 hmR0VmxFlushEPT(pVM, pVCpu, pVM->hwaccm.s.vmx.enmFlushEPT);
2541 }
2542 else
2543 {
2544#ifdef VBOX_WITH_STATISTICS
2545 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatNoFlushTLBWorldSwitch);
2546#endif
2547 }
2548 }
2549 pVCpu->hwaccm.s.TlbShootdown.cPages = 0;
2550 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
2551
2552 AssertMsg(pVCpu->hwaccm.s.cTLBFlushes == pCpu->cTLBFlushes,
2553 ("Flush count mismatch for cpu %d (%x vs %x)\n", pCpu->idCpu, pVCpu->hwaccm.s.cTLBFlushes, pCpu->cTLBFlushes));
2554 AssertMsg(pCpu->uCurrentASID >= 1 && pCpu->uCurrentASID < pVM->hwaccm.s.uMaxASID,
2555 ("cpu%d uCurrentASID = %x\n", pCpu->idCpu, pCpu->uCurrentASID));
2556 AssertMsg(pVCpu->hwaccm.s.uCurrentASID >= 1 && pVCpu->hwaccm.s.uCurrentASID < pVM->hwaccm.s.uMaxASID,
2557 ("cpu%d VM uCurrentASID = %x\n", pCpu->idCpu, pVCpu->hwaccm.s.uCurrentASID));
2558
2559 /* Update VMCS with the VPID. */
2560 int rc = VMXWriteVMCS(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hwaccm.s.uCurrentASID);
2561 AssertRC(rc);
2562}
2563
2564
2565/**
2566 * Setup the tagged TLB for EPT only.
2567 *
2568 * @returns VBox status code.
2569 * @param pVM Pointer to the VM.
2570 * @param pVCpu Pointer to the VMCPU.
2571 */
2572static DECLCALLBACK(void) hmR0VmxSetupTLBEPT(PVM pVM, PVMCPU pVCpu)
2573{
2574 PHMGLOBLCPUINFO pCpu;
2575
2576 Assert(pVM->hwaccm.s.fNestedPaging);
2577 Assert(!pVM->hwaccm.s.vmx.fVPID);
2578
2579 pCpu = HWACCMR0GetCurrentCpu();
2580
2581 /*
2582 * Force a TLB flush for the first world switch if the current CPU differs from the one we ran on last
2583 * This can happen both for start & resume due to long jumps back to ring-3.
2584 * A change in the TLB flush count implies the host Cpu is online after a suspend/resume.
2585 */
2586 if ( pVCpu->hwaccm.s.idLastCpu != pCpu->idCpu
2587 || pVCpu->hwaccm.s.cTLBFlushes != pCpu->cTLBFlushes)
2588 {
2589 pVCpu->hwaccm.s.fForceTLBFlush = true;
2590 }
2591
2592 /*
2593 * Check for explicit TLB shootdown flushes.
2594 */
2595 if (VMCPU_FF_TESTANDCLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2596 pVCpu->hwaccm.s.fForceTLBFlush = true;
2597
2598 pVCpu->hwaccm.s.idLastCpu = pCpu->idCpu;
2599 pVCpu->hwaccm.s.cTLBFlushes = pCpu->cTLBFlushes;
2600
2601 if (pVCpu->hwaccm.s.fForceTLBFlush)
2602 hmR0VmxFlushEPT(pVM, pVCpu, pVM->hwaccm.s.vmx.enmFlushEPT);
2603 else
2604 {
2605 /** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
2606 * not be executed. See hwaccmQueueInvlPage() where it is commented
2607 * out. Support individual entry flushing someday. */
2608 if (VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
2609 {
2610 /*
2611 * We cannot flush individual entries without VPID support. Flush using EPT.
2612 */
2613 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatTlbShootdown);
2614 hmR0VmxFlushEPT(pVM, pVCpu, pVM->hwaccm.s.vmx.enmFlushEPT);
2615 }
2616 }
2617 pVCpu->hwaccm.s.TlbShootdown.cPages= 0;
2618 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
2619
2620#ifdef VBOX_WITH_STATISTICS
2621 if (pVCpu->hwaccm.s.fForceTLBFlush)
2622 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatFlushTLBWorldSwitch);
2623 else
2624 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatNoFlushTLBWorldSwitch);
2625#endif
2626}
2627
2628
2629/**
2630 * Setup the tagged TLB for VPID.
2631 *
2632 * @returns VBox status code.
2633 * @param pVM Pointer to the VM.
2634 * @param pVCpu Pointer to the VMCPU.
2635 */
2636static DECLCALLBACK(void) hmR0VmxSetupTLBVPID(PVM pVM, PVMCPU pVCpu)
2637{
2638 PHMGLOBLCPUINFO pCpu;
2639
2640 Assert(pVM->hwaccm.s.vmx.fVPID);
2641 Assert(!pVM->hwaccm.s.fNestedPaging);
2642
2643 pCpu = HWACCMR0GetCurrentCpu();
2644
2645 /*
2646 * Force a TLB flush for the first world switch if the current CPU differs from the one we ran on last
2647 * This can happen both for start & resume due to long jumps back to ring-3.
2648 * If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
2649 * or the host Cpu is online after a suspend/resume, so we cannot reuse the current ASID anymore.
2650 */
2651 if ( pVCpu->hwaccm.s.idLastCpu != pCpu->idCpu
2652 || pVCpu->hwaccm.s.cTLBFlushes != pCpu->cTLBFlushes)
2653 {
2654 /* Force a TLB flush on VM entry. */
2655 pVCpu->hwaccm.s.fForceTLBFlush = true;
2656 }
2657
2658 /*
2659 * Check for explicit TLB shootdown flushes.
2660 */
2661 if (VMCPU_FF_TESTANDCLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2662 pVCpu->hwaccm.s.fForceTLBFlush = true;
2663
2664 pVCpu->hwaccm.s.idLastCpu = pCpu->idCpu;
2665
2666 if (pVCpu->hwaccm.s.fForceTLBFlush)
2667 {
2668 ++pCpu->uCurrentASID;
2669 if (pCpu->uCurrentASID >= pVM->hwaccm.s.uMaxASID)
2670 {
2671 pCpu->uCurrentASID = 1; /* start at 1; host uses 0 */
2672 pCpu->cTLBFlushes++;
2673 pCpu->fFlushASIDBeforeUse = true;
2674 }
2675 else
2676 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatFlushASID);
2677
2678 pVCpu->hwaccm.s.fForceTLBFlush = false;
2679 pVCpu->hwaccm.s.cTLBFlushes = pCpu->cTLBFlushes;
2680 pVCpu->hwaccm.s.uCurrentASID = pCpu->uCurrentASID;
2681 if (pCpu->fFlushASIDBeforeUse)
2682 hmR0VmxFlushVPID(pVM, pVCpu, pVM->hwaccm.s.vmx.enmFlushVPID, 0 /* GCPtr */);
2683 }
2684 else
2685 {
2686 AssertMsg(pVCpu->hwaccm.s.uCurrentASID && pCpu->uCurrentASID,
2687 ("hwaccm->uCurrentASID=%lu hwaccm->cTLBFlushes=%lu cpu->uCurrentASID=%lu cpu->cTLBFlushes=%lu\n",
2688 pVCpu->hwaccm.s.uCurrentASID, pVCpu->hwaccm.s.cTLBFlushes,
2689 pCpu->uCurrentASID, pCpu->cTLBFlushes));
2690
2691 /** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
2692 * not be executed. See hwaccmQueueInvlPage() where it is commented
2693 * out. Support individual entry flushing someday. */
2694 if (VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
2695 {
2696 /*
2697 * Flush individual guest entries using VPID from the TLB or as little as possible with EPT
2698 * as supported by the CPU.
2699 */
2700 if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVVPID_CAPS_INDIV_ADDR)
2701 {
2702 for (unsigned i = 0; i < pVCpu->hwaccm.s.TlbShootdown.cPages; i++)
2703 hmR0VmxFlushVPID(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, pVCpu->hwaccm.s.TlbShootdown.aPages[i]);
2704 }
2705 else
2706 hmR0VmxFlushVPID(pVM, pVCpu, pVM->hwaccm.s.vmx.enmFlushVPID, 0 /* GCPtr */);
2707 }
2708 }
2709 pVCpu->hwaccm.s.TlbShootdown.cPages = 0;
2710 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
2711
2712 AssertMsg(pVCpu->hwaccm.s.cTLBFlushes == pCpu->cTLBFlushes,
2713 ("Flush count mismatch for cpu %d (%x vs %x)\n", pCpu->idCpu, pVCpu->hwaccm.s.cTLBFlushes, pCpu->cTLBFlushes));
2714 AssertMsg(pCpu->uCurrentASID >= 1 && pCpu->uCurrentASID < pVM->hwaccm.s.uMaxASID,
2715 ("cpu%d uCurrentASID = %x\n", pCpu->idCpu, pCpu->uCurrentASID));
2716 AssertMsg(pVCpu->hwaccm.s.uCurrentASID >= 1 && pVCpu->hwaccm.s.uCurrentASID < pVM->hwaccm.s.uMaxASID,
2717 ("cpu%d VM uCurrentASID = %x\n", pCpu->idCpu, pVCpu->hwaccm.s.uCurrentASID));
2718
2719 int rc = VMXWriteVMCS(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hwaccm.s.uCurrentASID);
2720 AssertRC(rc);
2721
2722# ifdef VBOX_WITH_STATISTICS
2723 if (pVCpu->hwaccm.s.fForceTLBFlush)
2724 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatFlushTLBWorldSwitch);
2725 else
2726 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatNoFlushTLBWorldSwitch);
2727# endif
2728}
2729
2730
2731/**
2732 * Runs guest code in a VT-x VM.
2733 *
2734 * @returns VBox status code.
2735 * @param pVM Pointer to the VM.
2736 * @param pVCpu Pointer to the VMCPU.
2737 * @param pCtx Pointer to the guest CPU context.
2738 */
2739VMMR0DECL(int) VMXR0RunGuestCode(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
2740{
2741 STAM_PROFILE_ADV_START(&pVCpu->hwaccm.s.StatEntry, x);
2742 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hwaccm.s.StatExit1);
2743 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hwaccm.s.StatExit2);
2744
2745 VBOXSTRICTRC rc = VINF_SUCCESS;
2746 int rc2;
2747 RTGCUINTREG val;
2748 RTGCUINTREG exitReason = (RTGCUINTREG)VMX_EXIT_INVALID;
2749 RTGCUINTREG instrError, cbInstr;
2750 RTGCUINTPTR exitQualification = 0;
2751 RTGCUINTPTR intInfo = 0; /* shut up buggy gcc 4 */
2752 RTGCUINTPTR errCode, instrInfo;
2753 bool fSetupTPRCaching = false;
2754 uint64_t u64OldLSTAR = 0;
2755 uint8_t u8LastTPR = 0;
2756 RTCCUINTREG uOldEFlags = ~(RTCCUINTREG)0;
2757 unsigned cResume = 0;
2758#ifdef VBOX_STRICT
2759 RTCPUID idCpuCheck;
2760 bool fWasInLongMode = false;
2761#endif
2762#ifdef VBOX_HIGH_RES_TIMERS_HACK_IN_RING0
2763 uint64_t u64LastTime = RTTimeMilliTS();
2764#endif
2765
2766 Assert(!(pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
2767 || (pVCpu->hwaccm.s.vmx.pbVAPIC && pVM->hwaccm.s.vmx.pAPIC));
2768
2769 /*
2770 * Check if we need to use TPR shadowing.
2771 */
2772 if ( CPUMIsGuestInLongModeEx(pCtx)
2773 || ( (( pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
2774 || pVM->hwaccm.s.fTRPPatchingAllowed)
2775 && pVM->hwaccm.s.fHasIoApic)
2776 )
2777 {
2778 fSetupTPRCaching = true;
2779 }
2780
2781 Log2(("\nE"));
2782
2783#ifdef VBOX_STRICT
2784 {
2785 RTCCUINTREG val2;
2786
2787 rc2 = VMXReadVMCS(VMX_VMCS_CTRL_PIN_EXEC_CONTROLS, &val2);
2788 AssertRC(rc2);
2789 Log2(("VMX_VMCS_CTRL_PIN_EXEC_CONTROLS = %08x\n", val2));
2790
2791 /* allowed zero */
2792 if ((val2 & pVM->hwaccm.s.vmx.msr.vmx_pin_ctls.n.disallowed0) != pVM->hwaccm.s.vmx.msr.vmx_pin_ctls.n.disallowed0)
2793 Log(("Invalid VMX_VMCS_CTRL_PIN_EXEC_CONTROLS: zero\n"));
2794
2795 /* allowed one */
2796 if ((val2 & ~pVM->hwaccm.s.vmx.msr.vmx_pin_ctls.n.allowed1) != 0)
2797 Log(("Invalid VMX_VMCS_CTRL_PIN_EXEC_CONTROLS: one\n"));
2798
2799 rc2 = VMXReadVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, &val2);
2800 AssertRC(rc2);
2801 Log2(("VMX_VMCS_CTRL_PROC_EXEC_CONTROLS = %08x\n", val2));
2802
2803 /*
2804 * Must be set according to the MSR, but can be cleared if nested paging is used.
2805 */
2806 if (pVM->hwaccm.s.fNestedPaging)
2807 {
2808 val2 |= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INVLPG_EXIT
2809 | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
2810 | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT;
2811 }
2812
2813 /* allowed zero */
2814 if ((val2 & pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.disallowed0) != pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.disallowed0)
2815 Log(("Invalid VMX_VMCS_CTRL_PROC_EXEC_CONTROLS: zero\n"));
2816
2817 /* allowed one */
2818 if ((val2 & ~pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1) != 0)
2819 Log(("Invalid VMX_VMCS_CTRL_PROC_EXEC_CONTROLS: one\n"));
2820
2821 rc2 = VMXReadVMCS(VMX_VMCS_CTRL_ENTRY_CONTROLS, &val2);
2822 AssertRC(rc2);
2823 Log2(("VMX_VMCS_CTRL_ENTRY_CONTROLS = %08x\n", val2));
2824
2825 /* allowed zero */
2826 if ((val2 & pVM->hwaccm.s.vmx.msr.vmx_entry.n.disallowed0) != pVM->hwaccm.s.vmx.msr.vmx_entry.n.disallowed0)
2827 Log(("Invalid VMX_VMCS_CTRL_ENTRY_CONTROLS: zero\n"));
2828
2829 /* allowed one */
2830 if ((val2 & ~pVM->hwaccm.s.vmx.msr.vmx_entry.n.allowed1) != 0)
2831 Log(("Invalid VMX_VMCS_CTRL_ENTRY_CONTROLS: one\n"));
2832
2833 rc2 = VMXReadVMCS(VMX_VMCS_CTRL_EXIT_CONTROLS, &val2);
2834 AssertRC(rc2);
2835 Log2(("VMX_VMCS_CTRL_EXIT_CONTROLS = %08x\n", val2));
2836
2837 /* allowed zero */
2838 if ((val2 & pVM->hwaccm.s.vmx.msr.vmx_exit.n.disallowed0) != pVM->hwaccm.s.vmx.msr.vmx_exit.n.disallowed0)
2839 Log(("Invalid VMX_VMCS_CTRL_EXIT_CONTROLS: zero\n"));
2840
2841 /* allowed one */
2842 if ((val2 & ~pVM->hwaccm.s.vmx.msr.vmx_exit.n.allowed1) != 0)
2843 Log(("Invalid VMX_VMCS_CTRL_EXIT_CONTROLS: one\n"));
2844 }
2845 fWasInLongMode = CPUMIsGuestInLongModeEx(pCtx);
2846#endif /* VBOX_STRICT */
2847
2848#ifdef VBOX_WITH_CRASHDUMP_MAGIC
2849 pVCpu->hwaccm.s.vmx.VMCSCache.u64TimeEntry = RTTimeNanoTS();
2850#endif
2851
2852 /*
2853 * We can jump to this point to resume execution after determining that a VM-exit is innocent.
2854 */
2855ResumeExecution:
2856 if (!STAM_REL_PROFILE_ADV_IS_RUNNING(&pVCpu->hwaccm.s.StatEntry))
2857 STAM_REL_PROFILE_ADV_STOP_START(&pVCpu->hwaccm.s.StatExit2, &pVCpu->hwaccm.s.StatEntry, x);
2858 AssertMsg(pVCpu->hwaccm.s.idEnteredCpu == RTMpCpuId(),
2859 ("Expected %d, I'm %d; cResume=%d exitReason=%RGv exitQualification=%RGv\n",
2860 (int)pVCpu->hwaccm.s.idEnteredCpu, (int)RTMpCpuId(), cResume, exitReason, exitQualification));
2861 Assert(!HWACCMR0SuspendPending());
2862 /* Not allowed to switch modes without reloading the host state (32->64 switcher)!! */
2863 Assert(fWasInLongMode == CPUMIsGuestInLongModeEx(pCtx));
2864
2865 /*
2866 * Safety precaution; looping for too long here can have a very bad effect on the host.
2867 */
2868 if (RT_UNLIKELY(++cResume > pVM->hwaccm.s.cMaxResumeLoops))
2869 {
2870 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitMaxResume);
2871 rc = VINF_EM_RAW_INTERRUPT;
2872 goto end;
2873 }
2874
2875 /*
2876 * Check for IRQ inhibition due to instruction fusing (sti, mov ss).
2877 */
2878 if (VMCPU_FF_ISSET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
2879 {
2880 Log(("VM_FF_INHIBIT_INTERRUPTS at %RGv successor %RGv\n", (RTGCPTR)pCtx->rip, EMGetInhibitInterruptsPC(pVCpu)));
2881 if (pCtx->rip != EMGetInhibitInterruptsPC(pVCpu))
2882 {
2883 /*
2884 * Note: we intentionally don't clear VM_FF_INHIBIT_INTERRUPTS here.
2885 * Before we are able to execute this instruction in raw mode (iret to guest code) an external interrupt might
2886 * force a world switch again. Possibly allowing a guest interrupt to be dispatched in the process. This could
2887 * break the guest. Sounds very unlikely, but such timing sensitive problems are not as rare as you might think.
2888 */
2889 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
2890 /* Irq inhibition is no longer active; clear the corresponding VMX state. */
2891 rc2 = VMXWriteVMCS(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, 0);
2892 AssertRC(rc2);
2893 }
2894 }
2895 else
2896 {
2897 /* Irq inhibition is no longer active; clear the corresponding VMX state. */
2898 rc2 = VMXWriteVMCS(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, 0);
2899 AssertRC(rc2);
2900 }
2901
2902#ifdef VBOX_HIGH_RES_TIMERS_HACK_IN_RING0
2903 if (RT_UNLIKELY((cResume & 0xf) == 0))
2904 {
2905 uint64_t u64CurTime = RTTimeMilliTS();
2906
2907 if (RT_UNLIKELY(u64CurTime > u64LastTime))
2908 {
2909 u64LastTime = u64CurTime;
2910 TMTimerPollVoid(pVM, pVCpu);
2911 }
2912 }
2913#endif
2914
2915 /*
2916 * Check for pending actions that force us to go back to ring-3.
2917 */
2918 if ( VM_FF_ISPENDING(pVM, VM_FF_HWACCM_TO_R3_MASK | VM_FF_REQUEST | VM_FF_PGM_POOL_FLUSH_PENDING | VM_FF_PDM_DMA)
2919 || VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_HWACCM_TO_R3_MASK | VMCPU_FF_PGM_SYNC_CR3 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL | VMCPU_FF_REQUEST))
2920 {
2921 /* Check if a sync operation is pending. */
2922 if (VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
2923 {
2924 rc = PGMSyncCR3(pVCpu, pCtx->cr0, pCtx->cr3, pCtx->cr4, VMCPU_FF_ISSET(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
2925 if (rc != VINF_SUCCESS)
2926 {
2927 AssertRC(VBOXSTRICTRC_VAL(rc));
2928 Log(("Pending pool sync is forcing us back to ring 3; rc=%d\n", VBOXSTRICTRC_VAL(rc)));
2929 goto end;
2930 }
2931 }
2932
2933#ifdef DEBUG
2934 /* Intercept X86_XCPT_DB if stepping is enabled */
2935 if (!DBGFIsStepping(pVCpu))
2936#endif
2937 {
2938 if ( VM_FF_ISPENDING(pVM, VM_FF_HWACCM_TO_R3_MASK)
2939 || VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_HWACCM_TO_R3_MASK))
2940 {
2941 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatSwitchToR3);
2942 rc = RT_UNLIKELY(VM_FF_ISPENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
2943 goto end;
2944 }
2945 }
2946
2947 /* Pending request packets might contain actions that need immediate attention, such as pending hardware interrupts. */
2948 if ( VM_FF_ISPENDING(pVM, VM_FF_REQUEST)
2949 || VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_REQUEST))
2950 {
2951 rc = VINF_EM_PENDING_REQUEST;
2952 goto end;
2953 }
2954
2955 /* Check if a pgm pool flush is in progress. */
2956 if (VM_FF_ISPENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
2957 {
2958 rc = VINF_PGM_POOL_FLUSH_PENDING;
2959 goto end;
2960 }
2961
2962 /* Check if DMA work is pending (2nd+ run). */
2963 if (VM_FF_ISPENDING(pVM, VM_FF_PDM_DMA) && cResume > 1)
2964 {
2965 rc = VINF_EM_RAW_TO_R3;
2966 goto end;
2967 }
2968 }
2969
2970#ifdef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
2971 /*
2972 * Exit to ring-3 preemption/work is pending.
2973 *
2974 * Interrupts are disabled before the call to make sure we don't miss any interrupt
2975 * that would flag preemption (IPI, timer tick, ++). (Would've been nice to do this
2976 * further down, but hmR0VmxCheckPendingInterrupt makes that impossible.)
2977 *
2978 * Note! Interrupts must be disabled done *before* we check for TLB flushes; TLB
2979 * shootdowns rely on this.
2980 */
2981 uOldEFlags = ASMIntDisableFlags();
2982 if (RTThreadPreemptIsPending(NIL_RTTHREAD))
2983 {
2984 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitPreemptPending);
2985 rc = VINF_EM_RAW_INTERRUPT;
2986 goto end;
2987 }
2988 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
2989#endif
2990
2991 /*
2992 * When external interrupts are pending, we should exit the VM when IF is et.
2993 * Note: *After* VM_FF_INHIBIT_INTERRUPTS check!
2994 */
2995 rc = hmR0VmxCheckPendingInterrupt(pVM, pVCpu, pCtx);
2996 if (RT_FAILURE(rc))
2997 goto end;
2998
2999 /** @todo check timers?? */
3000
3001 /*
3002 * TPR caching using CR8 is only available in 64-bit mode.
3003 * Note: The 32-bit exception for AMD (X86_CPUID_AMD_FEATURE_ECX_CR8L), but this appears missing in Intel CPUs.
3004 * Note: We can't do this in LoadGuestState() as PDMApicGetTPR can jump back to ring-3 (lock)!! (no longer true) .
3005 */
3006 /** @todo query and update the TPR only when it could have been changed (mmio
3007 * access & wrsmr (x2apic) */
3008 if (fSetupTPRCaching)
3009 {
3010 /* TPR caching in CR8 */
3011 bool fPending;
3012
3013 rc2 = PDMApicGetTPR(pVCpu, &u8LastTPR, &fPending);
3014 AssertRC(rc2);
3015 /* The TPR can be found at offset 0x80 in the APIC mmio page. */
3016 pVCpu->hwaccm.s.vmx.pbVAPIC[0x80] = u8LastTPR;
3017
3018 /*
3019 * Two options here:
3020 * - external interrupt pending, but masked by the TPR value.
3021 * -> a CR8 update that lower the current TPR value should cause an exit
3022 * - no pending interrupts
3023 * -> We don't need to be explicitely notified. There are enough world switches for detecting pending interrupts.
3024 */
3025
3026 /* cr8 bits 3-0 correspond to bits 7-4 of the task priority mmio register. */
3027 rc = VMXWriteVMCS(VMX_VMCS_CTRL_TPR_THRESHOLD, (fPending) ? (u8LastTPR >> 4) : 0);
3028 AssertRC(VBOXSTRICTRC_VAL(rc));
3029
3030 if (pVM->hwaccm.s.fTPRPatchingActive)
3031 {
3032 Assert(!CPUMIsGuestInLongModeEx(pCtx));
3033 /* Our patch code uses LSTAR for TPR caching. */
3034 pCtx->msrLSTAR = u8LastTPR;
3035
3036 /** @todo r=ramshankar: we should check for MSR-bitmap support here. */
3037 if (fPending)
3038 {
3039 /* A TPR change could activate a pending interrupt, so catch lstar writes. */
3040 hmR0VmxSetMSRPermission(pVCpu, MSR_K8_LSTAR, true, false);
3041 }
3042 else
3043 {
3044 /*
3045 * No interrupts are pending, so we don't need to be explicitely notified.
3046 * There are enough world switches for detecting pending interrupts.
3047 */
3048 hmR0VmxSetMSRPermission(pVCpu, MSR_K8_LSTAR, true, true);
3049 }
3050 }
3051 }
3052
3053#ifdef LOG_ENABLED
3054 if ( pVM->hwaccm.s.fNestedPaging
3055 || pVM->hwaccm.s.vmx.fVPID)
3056 {
3057 PHMGLOBLCPUINFO pCpu = HWACCMR0GetCurrentCpu();
3058 if (pVCpu->hwaccm.s.idLastCpu != pCpu->idCpu)
3059 {
3060 LogFlow(("Force TLB flush due to rescheduling to a different cpu (%d vs %d)\n", pVCpu->hwaccm.s.idLastCpu,
3061 pCpu->idCpu));
3062 }
3063 else if (pVCpu->hwaccm.s.cTLBFlushes != pCpu->cTLBFlushes)
3064 {
3065 LogFlow(("Force TLB flush due to changed TLB flush count (%x vs %x)\n", pVCpu->hwaccm.s.cTLBFlushes,
3066 pCpu->cTLBFlushes));
3067 }
3068 else if (VMCPU_FF_ISSET(pVCpu, VMCPU_FF_TLB_FLUSH))
3069 LogFlow(("Manual TLB flush\n"));
3070 }
3071#endif
3072#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
3073 PGMRZDynMapFlushAutoSet(pVCpu);
3074#endif
3075
3076 /*
3077 * NOTE: DO NOT DO ANYTHING AFTER THIS POINT THAT MIGHT JUMP BACK TO RING-3!
3078 * (until the actual world switch)
3079 */
3080#ifdef VBOX_STRICT
3081 idCpuCheck = RTMpCpuId();
3082#endif
3083#ifdef LOG_ENABLED
3084 VMMR0LogFlushDisable(pVCpu);
3085#endif
3086
3087 /*
3088 * Save the host state first.
3089 */
3090 if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_HOST_CONTEXT)
3091 {
3092 rc = VMXR0SaveHostState(pVM, pVCpu);
3093 if (RT_UNLIKELY(rc != VINF_SUCCESS))
3094 {
3095 VMMR0LogFlushEnable(pVCpu);
3096 goto end;
3097 }
3098 }
3099
3100 /*
3101 * Load the guest state.
3102 */
3103 if (!pVCpu->hwaccm.s.fContextUseFlags)
3104 {
3105 VMXR0LoadMinimalGuestState(pVM, pVCpu, pCtx);
3106 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatLoadMinimal);
3107 }
3108 else
3109 {
3110 rc = VMXR0LoadGuestState(pVM, pVCpu, pCtx);
3111 if (RT_UNLIKELY(rc != VINF_SUCCESS))
3112 {
3113 VMMR0LogFlushEnable(pVCpu);
3114 goto end;
3115 }
3116 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatLoadFull);
3117 }
3118
3119#ifndef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
3120 /*
3121 * Disable interrupts to make sure a poke will interrupt execution.
3122 * This must be done *before* we check for TLB flushes; TLB shootdowns rely on this.
3123 */
3124 uOldEFlags = ASMIntDisableFlags();
3125 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
3126#endif
3127
3128 /* Non-register state Guest Context */
3129 /** @todo change me according to cpu state */
3130 rc2 = VMXWriteVMCS(VMX_VMCS32_GUEST_ACTIVITY_STATE, VMX_CMS_GUEST_ACTIVITY_ACTIVE);
3131 AssertRC(rc2);
3132
3133 /* Set TLB flush state as checked until we return from the world switch. */
3134 ASMAtomicWriteBool(&pVCpu->hwaccm.s.fCheckedTLBFlush, true);
3135 /* Deal with tagged TLB setup and invalidation. */
3136 pVM->hwaccm.s.vmx.pfnSetupTaggedTLB(pVM, pVCpu);
3137
3138 /*
3139 * Manual save and restore:
3140 * - General purpose registers except RIP, RSP
3141 *
3142 * Trashed:
3143 * - CR2 (we don't care)
3144 * - LDTR (reset to 0)
3145 * - DRx (presumably not changed at all)
3146 * - DR7 (reset to 0x400)
3147 * - EFLAGS (reset to RT_BIT(1); not relevant)
3148 */
3149
3150 /* All done! Let's start VM execution. */
3151 STAM_PROFILE_ADV_STOP_START(&pVCpu->hwaccm.s.StatEntry, &pVCpu->hwaccm.s.StatInGC, x);
3152 Assert(idCpuCheck == RTMpCpuId());
3153
3154#ifdef VBOX_WITH_CRASHDUMP_MAGIC
3155 pVCpu->hwaccm.s.vmx.VMCSCache.cResume = cResume;
3156 pVCpu->hwaccm.s.vmx.VMCSCache.u64TimeSwitch = RTTimeNanoTS();
3157#endif
3158
3159 /*
3160 * Save the current TPR value in the LSTAR MSR so our patches can access it.
3161 */
3162 if (pVM->hwaccm.s.fTPRPatchingActive)
3163 {
3164 Assert(pVM->hwaccm.s.fTPRPatchingActive);
3165 u64OldLSTAR = ASMRdMsr(MSR_K8_LSTAR);
3166 ASMWrMsr(MSR_K8_LSTAR, u8LastTPR);
3167 }
3168
3169 TMNotifyStartOfExecution(pVCpu);
3170
3171#ifndef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
3172 /*
3173 * Save the current Host TSC_AUX and write the guest TSC_AUX to the host, so that
3174 * RDTSCPs (that don't cause exits) reads the guest MSR. See @bugref{3324}.
3175 */
3176 if ( (pVCpu->hwaccm.s.vmx.proc_ctls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
3177 && !(pVCpu->hwaccm.s.vmx.proc_ctls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT))
3178 {
3179 pVCpu->hwaccm.s.u64HostTSCAux = ASMRdMsr(MSR_K8_TSC_AUX);
3180 uint64_t u64GuestTSCAux = 0;
3181 rc2 = CPUMQueryGuestMsr(pVCpu, MSR_K8_TSC_AUX, &u64GuestTSCAux);
3182 AssertRC(rc2);
3183 ASMWrMsr(MSR_K8_TSC_AUX, u64GuestTSCAux);
3184 }
3185#endif
3186
3187#ifdef VBOX_WITH_KERNEL_USING_XMM
3188 rc = hwaccmR0VMXStartVMWrapXMM(pVCpu->hwaccm.s.fResumeVM, pCtx, &pVCpu->hwaccm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hwaccm.s.vmx.pfnStartVM);
3189#else
3190 rc = pVCpu->hwaccm.s.vmx.pfnStartVM(pVCpu->hwaccm.s.fResumeVM, pCtx, &pVCpu->hwaccm.s.vmx.VMCSCache, pVM, pVCpu);
3191#endif
3192 ASMAtomicWriteBool(&pVCpu->hwaccm.s.fCheckedTLBFlush, false);
3193 ASMAtomicIncU32(&pVCpu->hwaccm.s.cWorldSwitchExits);
3194
3195 /* Possibly the last TSC value seen by the guest (too high) (only when we're in TSC offset mode). */
3196 if (!(pVCpu->hwaccm.s.vmx.proc_ctls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT))
3197 {
3198#ifndef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
3199 /* Restore host's TSC_AUX. */
3200 if (pVCpu->hwaccm.s.vmx.proc_ctls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
3201 ASMWrMsr(MSR_K8_TSC_AUX, pVCpu->hwaccm.s.u64HostTSCAux);
3202#endif
3203
3204 TMCpuTickSetLastSeen(pVCpu,
3205 ASMReadTSC() + pVCpu->hwaccm.s.vmx.u64TSCOffset - 0x400 /* guestimate of world switch overhead in clock ticks */);
3206 }
3207
3208 TMNotifyEndOfExecution(pVCpu);
3209 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED);
3210 Assert(!(ASMGetFlags() & X86_EFL_IF));
3211
3212 /*
3213 * Restore the host LSTAR MSR if the guest could have changed it.
3214 */
3215 if (pVM->hwaccm.s.fTPRPatchingActive)
3216 {
3217 Assert(pVM->hwaccm.s.fTPRPatchingActive);
3218 pVCpu->hwaccm.s.vmx.pbVAPIC[0x80] = pCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR);
3219 ASMWrMsr(MSR_K8_LSTAR, u64OldLSTAR);
3220 }
3221
3222 STAM_PROFILE_ADV_STOP_START(&pVCpu->hwaccm.s.StatInGC, &pVCpu->hwaccm.s.StatExit1, x);
3223 ASMSetFlags(uOldEFlags);
3224#ifdef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
3225 uOldEFlags = ~(RTCCUINTREG)0;
3226#endif
3227
3228 AssertMsg(!pVCpu->hwaccm.s.vmx.VMCSCache.Write.cValidEntries, ("pVCpu->hwaccm.s.vmx.VMCSCache.Write.cValidEntries=%d\n",
3229 pVCpu->hwaccm.s.vmx.VMCSCache.Write.cValidEntries));
3230
3231 /* In case we execute a goto ResumeExecution later on. */
3232 pVCpu->hwaccm.s.fResumeVM = true;
3233 pVCpu->hwaccm.s.fForceTLBFlush = false;
3234
3235 /*
3236 * !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
3237 * IMPORTANT: WE CAN'T DO ANY LOGGING OR OPERATIONS THAT CAN DO A LONGJMP BACK TO RING 3 *BEFORE* WE'VE SYNCED BACK (MOST OF) THE GUEST STATE
3238 * !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
3239 */
3240
3241 if (RT_UNLIKELY(rc != VINF_SUCCESS))
3242 {
3243 hmR0VmxReportWorldSwitchError(pVM, pVCpu, rc, pCtx);
3244 VMMR0LogFlushEnable(pVCpu);
3245 goto end;
3246 }
3247
3248 /* Success. Query the guest state and figure out what has happened. */
3249
3250 /* Investigate why there was a VM-exit. */
3251 rc2 = VMXReadCachedVMCS(VMX_VMCS32_RO_EXIT_REASON, &exitReason);
3252 STAM_COUNTER_INC(&pVCpu->hwaccm.s.paStatExitReasonR0[exitReason & MASK_EXITREASON_STAT]);
3253
3254 exitReason &= 0xffff; /* bit 0-15 contain the exit code. */
3255 rc2 |= VMXReadCachedVMCS(VMX_VMCS32_RO_VM_INSTR_ERROR, &instrError);
3256 rc2 |= VMXReadCachedVMCS(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &cbInstr);
3257 rc2 |= VMXReadCachedVMCS(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &intInfo);
3258 /* might not be valid; depends on VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID. */
3259 rc2 |= VMXReadCachedVMCS(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERRCODE, &errCode);
3260 rc2 |= VMXReadCachedVMCS(VMX_VMCS32_RO_EXIT_INSTR_INFO, &instrInfo);
3261 rc2 |= VMXReadCachedVMCS(VMX_VMCS_RO_EXIT_QUALIFICATION, &exitQualification);
3262 AssertRC(rc2);
3263
3264 /*
3265 * Sync back the guest state.
3266 */
3267 rc2 = VMXR0SaveGuestState(pVM, pVCpu, pCtx);
3268 AssertRC(rc2);
3269
3270 /* Note! NOW IT'S SAFE FOR LOGGING! */
3271 VMMR0LogFlushEnable(pVCpu);
3272 Log2(("Raw exit reason %08x\n", exitReason));
3273#if ARCH_BITS == 64 /* for the time being */
3274 VBOXVMM_R0_HMVMX_VMEXIT(pVCpu, pCtx, exitReason);
3275#endif
3276
3277 /*
3278 * Check if an injected event was interrupted prematurely.
3279 */
3280 rc2 = VMXReadCachedVMCS(VMX_VMCS32_RO_IDT_INFO, &val);
3281 AssertRC(rc2);
3282 pVCpu->hwaccm.s.Event.intInfo = VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(val);
3283 if ( VMX_EXIT_INTERRUPTION_INFO_VALID(pVCpu->hwaccm.s.Event.intInfo)
3284 /* Ignore 'int xx' as they'll be restarted anyway. */
3285 && VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hwaccm.s.Event.intInfo) != VMX_EXIT_INTERRUPTION_INFO_TYPE_SW
3286 /* Ignore software exceptions (such as int3) as they'll reoccur when we restart the instruction anyway. */
3287 && VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hwaccm.s.Event.intInfo) != VMX_EXIT_INTERRUPTION_INFO_TYPE_SWEXCPT)
3288 {
3289 Assert(!pVCpu->hwaccm.s.Event.fPending);
3290 pVCpu->hwaccm.s.Event.fPending = true;
3291 /* Error code present? */
3292 if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(pVCpu->hwaccm.s.Event.intInfo))
3293 {
3294 rc2 = VMXReadCachedVMCS(VMX_VMCS32_RO_IDT_ERRCODE, &val);
3295 AssertRC(rc2);
3296 pVCpu->hwaccm.s.Event.errCode = val;
3297 Log(("Pending inject %RX64 at %RGv exit=%08x intInfo=%08x exitQualification=%RGv pending error=%RX64\n",
3298 pVCpu->hwaccm.s.Event.intInfo, (RTGCPTR)pCtx->rip, exitReason, intInfo, exitQualification, val));
3299 }
3300 else
3301 {
3302 Log(("Pending inject %RX64 at %RGv exit=%08x intInfo=%08x exitQualification=%RGv\n", pVCpu->hwaccm.s.Event.intInfo,
3303 (RTGCPTR)pCtx->rip, exitReason, intInfo, exitQualification));
3304 pVCpu->hwaccm.s.Event.errCode = 0;
3305 }
3306 }
3307#ifdef VBOX_STRICT
3308 else if ( VMX_EXIT_INTERRUPTION_INFO_VALID(pVCpu->hwaccm.s.Event.intInfo)
3309 /* Ignore software exceptions (such as int3) as they're reoccur when we restart the instruction anyway. */
3310 && VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hwaccm.s.Event.intInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_SWEXCPT)
3311 {
3312 Log(("Ignore pending inject %RX64 at %RGv exit=%08x intInfo=%08x exitQualification=%RGv\n",
3313 pVCpu->hwaccm.s.Event.intInfo, (RTGCPTR)pCtx->rip, exitReason, intInfo, exitQualification));
3314 }
3315
3316 if (exitReason == VMX_EXIT_ERR_INVALID_GUEST_STATE)
3317 HWACCMDumpRegs(pVM, pVCpu, pCtx);
3318#endif
3319
3320 Log2(("E%d: New EIP=%x:%RGv\n", (uint32_t)exitReason, pCtx->cs.Sel, (RTGCPTR)pCtx->rip));
3321 Log2(("Exit reason %d, exitQualification %RGv\n", (uint32_t)exitReason, exitQualification));
3322 Log2(("instrInfo=%d instrError=%d instr length=%d\n", (uint32_t)instrInfo, (uint32_t)instrError, (uint32_t)cbInstr));
3323 Log2(("Interruption error code %d\n", (uint32_t)errCode));
3324 Log2(("IntInfo = %08x\n", (uint32_t)intInfo));
3325
3326 /*
3327 * Sync back the TPR if it was changed.
3328 */
3329 if ( fSetupTPRCaching
3330 && u8LastTPR != pVCpu->hwaccm.s.vmx.pbVAPIC[0x80])
3331 {
3332 rc2 = PDMApicSetTPR(pVCpu, pVCpu->hwaccm.s.vmx.pbVAPIC[0x80]);
3333 AssertRC(rc2);
3334 }
3335
3336#ifdef DBGFTRACE_ENABLED /** @todo DTrace later. */
3337 RTTraceBufAddMsgF(pVM->CTX_SUFF(hTraceBuf), "vmexit %08x %016RX64 at %04:%08RX64 %RX64",
3338 exitReason, (uint64_t)exitQualification, pCtx->cs.Sel, pCtx->rip, (uint64_t)intInfo);
3339#endif
3340 STAM_PROFILE_ADV_STOP_START(&pVCpu->hwaccm.s.StatExit1, &pVCpu->hwaccm.s.StatExit2, x);
3341
3342 /* Some cases don't need a complete resync of the guest CPU state; handle them here. */
3343 Assert(rc == VINF_SUCCESS); /* might consider VERR_IPE_UNINITIALIZED_STATUS here later... */
3344 switch (exitReason)
3345 {
3346 case VMX_EXIT_EXCEPTION: /* 0 Exception or non-maskable interrupt (NMI). */
3347 case VMX_EXIT_EXTERNAL_IRQ: /* 1 External interrupt. */
3348 {
3349 uint32_t vector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(intInfo);
3350
3351 if (!VMX_EXIT_INTERRUPTION_INFO_VALID(intInfo))
3352 {
3353 Assert(exitReason == VMX_EXIT_EXTERNAL_IRQ);
3354#if 0 //def VBOX_WITH_VMMR0_DISABLE_PREEMPTION
3355 if ( RTThreadPreemptIsPendingTrusty()
3356 && !RTThreadPreemptIsPending(NIL_RTTHREAD))
3357 goto ResumeExecution;
3358#endif
3359 /* External interrupt; leave to allow it to be dispatched again. */
3360 rc = VINF_EM_RAW_INTERRUPT;
3361 break;
3362 }
3363 STAM_PROFILE_ADV_START(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3364 switch (VMX_EXIT_INTERRUPTION_INFO_TYPE(intInfo))
3365 {
3366 case VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI: /* Non-maskable interrupt. */
3367 /* External interrupt; leave to allow it to be dispatched again. */
3368 rc = VINF_EM_RAW_INTERRUPT;
3369 break;
3370
3371 case VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT: /* External hardware interrupt. */
3372 AssertFailed(); /* can't come here; fails the first check. */
3373 break;
3374
3375 case VMX_EXIT_INTERRUPTION_INFO_TYPE_DBEXCPT: /* Unknown why we get this type for #DB */
3376 case VMX_EXIT_INTERRUPTION_INFO_TYPE_SWEXCPT: /* Software exception. (#BP or #OF) */
3377 Assert(vector == 1 || vector == 3 || vector == 4);
3378 /* no break */
3379 case VMX_EXIT_INTERRUPTION_INFO_TYPE_HWEXCPT: /* Hardware exception. */
3380 Log2(("Hardware/software interrupt %d\n", vector));
3381 switch (vector)
3382 {
3383 case X86_XCPT_NM:
3384 {
3385 Log(("#NM fault at %RGv error code %x\n", (RTGCPTR)pCtx->rip, errCode));
3386
3387 /** @todo don't intercept #NM exceptions anymore when we've activated the guest FPU state. */
3388 /* If we sync the FPU/XMM state on-demand, then we can continue execution as if nothing has happened. */
3389 rc = CPUMR0LoadGuestFPU(pVM, pVCpu, pCtx);
3390 if (rc == VINF_SUCCESS)
3391 {
3392 Assert(CPUMIsGuestFPUStateActive(pVCpu));
3393
3394 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitShadowNM);
3395
3396 /* Continue execution. */
3397 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_GUEST_CR0;
3398
3399 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3400 goto ResumeExecution;
3401 }
3402
3403 Log(("Forward #NM fault to the guest\n"));
3404 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestNM);
3405 rc2 = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo),
3406 cbInstr, 0);
3407 AssertRC(rc2);
3408 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3409 goto ResumeExecution;
3410 }
3411
3412 case X86_XCPT_PF: /* Page fault */
3413 {
3414#ifdef VBOX_ALWAYS_TRAP_PF
3415 if (pVM->hwaccm.s.fNestedPaging)
3416 {
3417 /*
3418 * A genuine pagefault. Forward the trap to the guest by injecting the exception and resuming execution.
3419 */
3420 Log(("Guest page fault at %RGv cr2=%RGv error code %RGv rsp=%RGv\n", (RTGCPTR)pCtx->rip, exitQualification,
3421 errCode, (RTGCPTR)pCtx->rsp));
3422
3423 Assert(CPUMIsGuestInPagedProtectedModeEx(pCtx));
3424
3425 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestPF);
3426
3427 /* Now we must update CR2. */
3428 pCtx->cr2 = exitQualification;
3429 rc2 = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo),
3430 cbInstr, errCode);
3431 AssertRC(rc2);
3432
3433 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3434 goto ResumeExecution;
3435 }
3436#else
3437 Assert(!pVM->hwaccm.s.fNestedPaging);
3438#endif
3439
3440#ifdef VBOX_HWACCM_WITH_GUEST_PATCHING
3441 /* Shortcut for APIC TPR reads and writes; 32 bits guests only */
3442 if ( pVM->hwaccm.s.fTRPPatchingAllowed
3443 && pVM->hwaccm.s.pGuestPatchMem
3444 && (exitQualification & 0xfff) == 0x080
3445 && !(errCode & X86_TRAP_PF_P) /* not present */
3446 && CPUMGetGuestCPL(pVCpu) == 0
3447 && !CPUMIsGuestInLongModeEx(pCtx)
3448 && pVM->hwaccm.s.cPatches < RT_ELEMENTS(pVM->hwaccm.s.aPatches))
3449 {
3450 RTGCPHYS GCPhysApicBase, GCPhys;
3451 PDMApicGetBase(pVM, &GCPhysApicBase); /** @todo cache this */
3452 GCPhysApicBase &= PAGE_BASE_GC_MASK;
3453
3454 rc = PGMGstGetPage(pVCpu, (RTGCPTR)exitQualification, NULL, &GCPhys);
3455 if ( rc == VINF_SUCCESS
3456 && GCPhys == GCPhysApicBase)
3457 {
3458 /* Only attempt to patch the instruction once. */
3459 PHWACCMTPRPATCH pPatch = (PHWACCMTPRPATCH)RTAvloU32Get(&pVM->hwaccm.s.PatchTree, (AVLOU32KEY)pCtx->eip);
3460 if (!pPatch)
3461 {
3462 rc = VINF_EM_HWACCM_PATCH_TPR_INSTR;
3463 break;
3464 }
3465 }
3466 }
3467#endif
3468
3469 Log2(("Page fault at %RGv error code %x\n", exitQualification, errCode));
3470 /* Exit qualification contains the linear address of the page fault. */
3471 TRPMAssertTrap(pVCpu, X86_XCPT_PF, TRPM_TRAP);
3472 TRPMSetErrorCode(pVCpu, errCode);
3473 TRPMSetFaultAddress(pVCpu, exitQualification);
3474
3475 /* Shortcut for APIC TPR reads and writes. */
3476 if ( (exitQualification & 0xfff) == 0x080
3477 && !(errCode & X86_TRAP_PF_P) /* not present */
3478 && fSetupTPRCaching
3479 && (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC))
3480 {
3481 RTGCPHYS GCPhysApicBase, GCPhys;
3482 PDMApicGetBase(pVM, &GCPhysApicBase); /* @todo cache this */
3483 GCPhysApicBase &= PAGE_BASE_GC_MASK;
3484
3485 rc = PGMGstGetPage(pVCpu, (RTGCPTR)exitQualification, NULL, &GCPhys);
3486 if ( rc == VINF_SUCCESS
3487 && GCPhys == GCPhysApicBase)
3488 {
3489 Log(("Enable VT-x virtual APIC access filtering\n"));
3490 rc2 = IOMMMIOMapMMIOHCPage(pVM, GCPhysApicBase, pVM->hwaccm.s.vmx.pAPICPhys, X86_PTE_RW | X86_PTE_P);
3491 AssertRC(rc2);
3492 }
3493 }
3494
3495 /* Forward it to our trap handler first, in case our shadow pages are out of sync. */
3496 rc = PGMTrap0eHandler(pVCpu, errCode, CPUMCTX2CORE(pCtx), (RTGCPTR)exitQualification);
3497 Log2(("PGMTrap0eHandler %RGv returned %Rrc\n", (RTGCPTR)pCtx->rip, VBOXSTRICTRC_VAL(rc)));
3498
3499 if (rc == VINF_SUCCESS)
3500 { /* We've successfully synced our shadow pages, so let's just continue execution. */
3501 Log2(("Shadow page fault at %RGv cr2=%RGv error code %x\n", (RTGCPTR)pCtx->rip, exitQualification ,errCode));
3502 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitShadowPF);
3503
3504 TRPMResetTrap(pVCpu);
3505 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3506 goto ResumeExecution;
3507 }
3508 else if (rc == VINF_EM_RAW_GUEST_TRAP)
3509 {
3510 /*
3511 * A genuine pagefault. Forward the trap to the guest by injecting the exception and resuming execution.
3512 */
3513 Log2(("Forward page fault to the guest\n"));
3514
3515 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestPF);
3516 /* The error code might have been changed. */
3517 errCode = TRPMGetErrorCode(pVCpu);
3518
3519 TRPMResetTrap(pVCpu);
3520
3521 /* Now we must update CR2. */
3522 pCtx->cr2 = exitQualification;
3523 rc2 = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo),
3524 cbInstr, errCode);
3525 AssertRC(rc2);
3526
3527 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3528 goto ResumeExecution;
3529 }
3530#ifdef VBOX_STRICT
3531 if (rc != VINF_EM_RAW_EMULATE_INSTR && rc != VINF_EM_RAW_EMULATE_IO_BLOCK)
3532 Log2(("PGMTrap0eHandler failed with %d\n", VBOXSTRICTRC_VAL(rc)));
3533#endif
3534 /* Need to go back to the recompiler to emulate the instruction. */
3535 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitShadowPFEM);
3536 TRPMResetTrap(pVCpu);
3537 break;
3538 }
3539
3540 case X86_XCPT_MF: /* Floating point exception. */
3541 {
3542 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestMF);
3543 if (!(pCtx->cr0 & X86_CR0_NE))
3544 {
3545 /* old style FPU error reporting needs some extra work. */
3546 /** @todo don't fall back to the recompiler, but do it manually. */
3547 rc = VINF_EM_RAW_EMULATE_INSTR;
3548 break;
3549 }
3550 Log(("Trap %x at %04X:%RGv\n", vector, pCtx->cs.Sel, (RTGCPTR)pCtx->rip));
3551 rc2 = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo),
3552 cbInstr, errCode);
3553 AssertRC(rc2);
3554
3555 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3556 goto ResumeExecution;
3557 }
3558
3559 case X86_XCPT_DB: /* Debug exception. */
3560 {
3561 uint64_t uDR6;
3562
3563 /*
3564 * DR6, DR7.GD and IA32_DEBUGCTL.LBR are not updated yet.
3565 *
3566 * Exit qualification bits:
3567 * 3:0 B0-B3 which breakpoint condition was met
3568 * 12:4 Reserved (0)
3569 * 13 BD - debug register access detected
3570 * 14 BS - single step execution or branch taken
3571 * 63:15 Reserved (0)
3572 */
3573 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestDB);
3574
3575 /* Note that we don't support guest and host-initiated debugging at the same time. */
3576
3577 uDR6 = X86_DR6_INIT_VAL;
3578 uDR6 |= (exitQualification & (X86_DR6_B0|X86_DR6_B1|X86_DR6_B2|X86_DR6_B3|X86_DR6_BD|X86_DR6_BS));
3579 rc = DBGFRZTrap01Handler(pVM, pVCpu, CPUMCTX2CORE(pCtx), uDR6);
3580 if (rc == VINF_EM_RAW_GUEST_TRAP)
3581 {
3582 /* Update DR6 here. */
3583 pCtx->dr[6] = uDR6;
3584
3585 /* Resync DR6 if the debug state is active. */
3586 if (CPUMIsGuestDebugStateActive(pVCpu))
3587 ASMSetDR6(pCtx->dr[6]);
3588
3589 /* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
3590 pCtx->dr[7] &= ~X86_DR7_GD;
3591
3592 /* Paranoia. */
3593 pCtx->dr[7] &= 0xffffffff; /* upper 32 bits reserved */
3594 pCtx->dr[7] &= ~(RT_BIT(11) | RT_BIT(12) | RT_BIT(14) | RT_BIT(15)); /* must be zero */
3595 pCtx->dr[7] |= 0x400; /* must be one */
3596
3597 /* Resync DR7 */
3598 rc2 = VMXWriteVMCS64(VMX_VMCS64_GUEST_DR7, pCtx->dr[7]);
3599 AssertRC(rc2);
3600
3601 Log(("Trap %x (debug) at %RGv exit qualification %RX64 dr6=%x dr7=%x\n", vector, (RTGCPTR)pCtx->rip,
3602 exitQualification, (uint32_t)pCtx->dr[6], (uint32_t)pCtx->dr[7]));
3603 rc2 = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo),
3604 cbInstr, errCode);
3605 AssertRC(rc2);
3606
3607 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3608 goto ResumeExecution;
3609 }
3610 /* Return to ring 3 to deal with the debug exit code. */
3611 Log(("Debugger hardware BP at %04x:%RGv (rc=%Rrc)\n", pCtx->cs.Sel, pCtx->rip, VBOXSTRICTRC_VAL(rc)));
3612 break;
3613 }
3614
3615 case X86_XCPT_BP: /* Breakpoint. */
3616 {
3617 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestBP);
3618 rc = DBGFRZTrap03Handler(pVM, pVCpu, CPUMCTX2CORE(pCtx));
3619 if (rc == VINF_EM_RAW_GUEST_TRAP)
3620 {
3621 Log(("Guest #BP at %04x:%RGv\n", pCtx->cs.Sel, pCtx->rip));
3622 rc2 = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo),
3623 cbInstr, errCode);
3624 AssertRC(rc2);
3625 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3626 goto ResumeExecution;
3627 }
3628 if (rc == VINF_SUCCESS)
3629 {
3630 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3631 goto ResumeExecution;
3632 }
3633 Log(("Debugger BP at %04x:%RGv (rc=%Rrc)\n", pCtx->cs.Sel, pCtx->rip, VBOXSTRICTRC_VAL(rc)));
3634 break;
3635 }
3636
3637 case X86_XCPT_GP: /* General protection failure exception. */
3638 {
3639 uint32_t cbOp;
3640 PDISCPUSTATE pDis = &pVCpu->hwaccm.s.DisState;
3641
3642 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestGP);
3643#ifdef VBOX_STRICT
3644 if ( !CPUMIsGuestInRealModeEx(pCtx)
3645 || !pVM->hwaccm.s.vmx.pRealModeTSS)
3646 {
3647 Log(("Trap %x at %04X:%RGv errorCode=%RGv\n", vector, pCtx->cs.Sel, (RTGCPTR)pCtx->rip, errCode));
3648 rc2 = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo),
3649 cbInstr, errCode);
3650 AssertRC(rc2);
3651 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3652 goto ResumeExecution;
3653 }
3654#endif
3655 Assert(CPUMIsGuestInRealModeEx(pCtx));
3656
3657 LogFlow(("Real mode X86_XCPT_GP instruction emulation at %x:%RGv\n", pCtx->cs.Sel, (RTGCPTR)pCtx->rip));
3658
3659 rc2 = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbOp);
3660 if (RT_SUCCESS(rc2))
3661 {
3662 bool fUpdateRIP = true;
3663
3664 rc = VINF_SUCCESS;
3665 Assert(cbOp == pDis->cbInstr);
3666 switch (pDis->pCurInstr->uOpcode)
3667 {
3668 case OP_CLI:
3669 pCtx->eflags.Bits.u1IF = 0;
3670 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitCli);
3671 break;
3672
3673 case OP_STI:
3674 pCtx->eflags.Bits.u1IF = 1;
3675 EMSetInhibitInterruptsPC(pVCpu, pCtx->rip + pDis->cbInstr);
3676 Assert(VMCPU_FF_ISSET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
3677 rc2 = VMXWriteVMCS(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE,
3678 VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
3679 AssertRC(rc2);
3680 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitSti);
3681 break;
3682
3683 case OP_HLT:
3684 fUpdateRIP = false;
3685 rc = VINF_EM_HALT;
3686 pCtx->rip += pDis->cbInstr;
3687 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitHlt);
3688 break;
3689
3690 case OP_POPF:
3691 {
3692 RTGCPTR GCPtrStack;
3693 uint32_t cbParm;
3694 uint32_t uMask;
3695 X86EFLAGS eflags;
3696
3697 if (pDis->fPrefix & DISPREFIX_OPSIZE)
3698 {
3699 cbParm = 4;
3700 uMask = 0xffffffff;
3701 }
3702 else
3703 {
3704 cbParm = 2;
3705 uMask = 0xffff;
3706 }
3707
3708 rc2 = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pCtx), pCtx->esp & uMask, 0, &GCPtrStack);
3709 if (RT_FAILURE(rc2))
3710 {
3711 rc = VERR_EM_INTERPRETER;
3712 break;
3713 }
3714 eflags.u = 0;
3715 rc2 = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &eflags.u, cbParm);
3716 if (RT_FAILURE(rc2))
3717 {
3718 rc = VERR_EM_INTERPRETER;
3719 break;
3720 }
3721 LogFlow(("POPF %x -> %RGv mask=%x\n", eflags.u, pCtx->rsp, uMask));
3722 pCtx->eflags.u = (pCtx->eflags.u & ~(X86_EFL_POPF_BITS & uMask)) | (eflags.u & X86_EFL_POPF_BITS & uMask);
3723 /* RF cleared when popped in real mode; see pushf description in AMD manual. */
3724 pCtx->eflags.Bits.u1RF = 0;
3725 pCtx->esp += cbParm;
3726 pCtx->esp &= uMask;
3727
3728 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitPopf);
3729 break;
3730 }
3731
3732 case OP_PUSHF:
3733 {
3734 RTGCPTR GCPtrStack;
3735 uint32_t cbParm;
3736 uint32_t uMask;
3737 X86EFLAGS eflags;
3738
3739 if (pDis->fPrefix & DISPREFIX_OPSIZE)
3740 {
3741 cbParm = 4;
3742 uMask = 0xffffffff;
3743 }
3744 else
3745 {
3746 cbParm = 2;
3747 uMask = 0xffff;
3748 }
3749
3750 rc2 = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pCtx), (pCtx->esp - cbParm) & uMask, 0,
3751 &GCPtrStack);
3752 if (RT_FAILURE(rc2))
3753 {
3754 rc = VERR_EM_INTERPRETER;
3755 break;
3756 }
3757 eflags = pCtx->eflags;
3758 /* RF & VM cleared when pushed in real mode; see pushf description in AMD manual. */
3759 eflags.Bits.u1RF = 0;
3760 eflags.Bits.u1VM = 0;
3761
3762 rc2 = PGMPhysWrite(pVM, (RTGCPHYS)GCPtrStack, &eflags.u, cbParm);
3763 if (RT_FAILURE(rc2))
3764 {
3765 rc = VERR_EM_INTERPRETER;
3766 break;
3767 }
3768 LogFlow(("PUSHF %x -> %RGv\n", eflags.u, GCPtrStack));
3769 pCtx->esp -= cbParm;
3770 pCtx->esp &= uMask;
3771 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitPushf);
3772 break;
3773 }
3774
3775 case OP_IRET:
3776 {
3777 RTGCPTR GCPtrStack;
3778 uint32_t uMask = 0xffff;
3779 uint16_t aIretFrame[3];
3780
3781 if (pDis->fPrefix & (DISPREFIX_OPSIZE | DISPREFIX_ADDRSIZE))
3782 {
3783 rc = VERR_EM_INTERPRETER;
3784 break;
3785 }
3786
3787 rc2 = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pCtx), pCtx->esp & uMask, 0, &GCPtrStack);
3788 if (RT_FAILURE(rc2))
3789 {
3790 rc = VERR_EM_INTERPRETER;
3791 break;
3792 }
3793 rc2 = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &aIretFrame[0], sizeof(aIretFrame));
3794 if (RT_FAILURE(rc2))
3795 {
3796 rc = VERR_EM_INTERPRETER;
3797 break;
3798 }
3799 pCtx->ip = aIretFrame[0];
3800 pCtx->cs.Sel = aIretFrame[1];
3801 pCtx->cs.ValidSel = aIretFrame[1];
3802 pCtx->cs.u64Base = (uint32_t)pCtx->cs.Sel << 4;
3803 pCtx->eflags.u = (pCtx->eflags.u & ~(X86_EFL_POPF_BITS & uMask))
3804 | (aIretFrame[2] & X86_EFL_POPF_BITS & uMask);
3805 pCtx->sp += sizeof(aIretFrame);
3806
3807 LogFlow(("iret to %04x:%x\n", pCtx->cs.Sel, pCtx->ip));
3808 fUpdateRIP = false;
3809 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitIret);
3810 break;
3811 }
3812
3813 case OP_INT:
3814 {
3815 uint32_t intInfo2;
3816
3817 LogFlow(("Realmode: INT %x\n", pDis->Param1.uValue & 0xff));
3818 intInfo2 = pDis->Param1.uValue & 0xff;
3819 intInfo2 |= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
3820 intInfo2 |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
3821
3822 rc = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, intInfo2, cbOp, 0);
3823 AssertRC(VBOXSTRICTRC_VAL(rc));
3824 fUpdateRIP = false;
3825 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitInt);
3826 break;
3827 }
3828
3829 case OP_INTO:
3830 {
3831 if (pCtx->eflags.Bits.u1OF)
3832 {
3833 uint32_t intInfo2;
3834
3835 LogFlow(("Realmode: INTO\n"));
3836 intInfo2 = X86_XCPT_OF;
3837 intInfo2 |= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
3838 intInfo2 |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
3839
3840 rc = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, intInfo2, cbOp, 0);
3841 AssertRC(VBOXSTRICTRC_VAL(rc));
3842 fUpdateRIP = false;
3843 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitInt);
3844 }
3845 break;
3846 }
3847
3848 case OP_INT3:
3849 {
3850 uint32_t intInfo2;
3851
3852 LogFlow(("Realmode: INT 3\n"));
3853 intInfo2 = 3;
3854 intInfo2 |= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
3855 intInfo2 |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
3856
3857 rc = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, intInfo2, cbOp, 0);
3858 AssertRC(VBOXSTRICTRC_VAL(rc));
3859 fUpdateRIP = false;
3860 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitInt);
3861 break;
3862 }
3863
3864 default:
3865 rc = EMInterpretInstructionDisasState(pVCpu, pDis, CPUMCTX2CORE(pCtx), 0, EMCODETYPE_SUPERVISOR);
3866 fUpdateRIP = false;
3867 break;
3868 }
3869
3870 if (rc == VINF_SUCCESS)
3871 {
3872 if (fUpdateRIP)
3873 pCtx->rip += cbOp; /* Move on to the next instruction. */
3874
3875 /*
3876 * LIDT, LGDT can end up here. In the future CRx changes as well. Just reload the
3877 * whole context to be done with it.
3878 */
3879 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_ALL;
3880
3881 /* Only resume if successful. */
3882 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3883 goto ResumeExecution;
3884 }
3885 }
3886 else
3887 rc = VERR_EM_INTERPRETER;
3888
3889 AssertMsg(rc == VERR_EM_INTERPRETER || rc == VINF_PGM_CHANGE_MODE || rc == VINF_EM_HALT,
3890 ("Unexpected rc=%Rrc\n", VBOXSTRICTRC_VAL(rc)));
3891 break;
3892 }
3893
3894#ifdef VBOX_STRICT
3895 case X86_XCPT_XF: /* SIMD exception. */
3896 case X86_XCPT_DE: /* Divide error. */
3897 case X86_XCPT_UD: /* Unknown opcode exception. */
3898 case X86_XCPT_SS: /* Stack segment exception. */
3899 case X86_XCPT_NP: /* Segment not present exception. */
3900 {
3901 switch (vector)
3902 {
3903 case X86_XCPT_DE:
3904 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestDE);
3905 break;
3906 case X86_XCPT_UD:
3907 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestUD);
3908 break;
3909 case X86_XCPT_SS:
3910 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestSS);
3911 break;
3912 case X86_XCPT_NP:
3913 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestNP);
3914 break;
3915 case X86_XCPT_XF:
3916 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestXF);
3917 break;
3918 }
3919
3920 Log(("Trap %x at %04X:%RGv\n", vector, pCtx->cs.Sel, (RTGCPTR)pCtx->rip));
3921 rc2 = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo),
3922 cbInstr, errCode);
3923 AssertRC(rc2);
3924
3925 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3926 goto ResumeExecution;
3927 }
3928#endif
3929 default:
3930 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestXcpUnk);
3931 if ( CPUMIsGuestInRealModeEx(pCtx)
3932 && pVM->hwaccm.s.vmx.pRealModeTSS)
3933 {
3934 Log(("Real Mode Trap %x at %04x:%04X error code %x\n", vector, pCtx->cs.Sel, pCtx->eip, errCode));
3935 rc = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo),
3936 cbInstr, errCode);
3937 AssertRC(VBOXSTRICTRC_VAL(rc)); /* Strict RC check below. */
3938
3939 /* Go back to ring-3 in case of a triple fault. */
3940 if ( vector == X86_XCPT_DF
3941 && rc == VINF_EM_RESET)
3942 {
3943 break;
3944 }
3945
3946 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3947 goto ResumeExecution;
3948 }
3949 AssertMsgFailed(("Unexpected vm-exit caused by exception %x\n", vector));
3950 rc = VERR_VMX_UNEXPECTED_EXCEPTION;
3951 break;
3952 } /* switch (vector) */
3953
3954 break;
3955
3956 default:
3957 rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_CODE;
3958 AssertMsgFailed(("Unexpected interruption code %x\n", intInfo));
3959 break;
3960 }
3961
3962 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3963 break;
3964 }
3965
3966 /*
3967 * 48 EPT violation. An attempt to access memory with a guest-physical address was disallowed
3968 * by the configuration of the EPT paging structures.
3969 */
3970 case VMX_EXIT_EPT_VIOLATION:
3971 {
3972 RTGCPHYS GCPhys;
3973
3974 Assert(pVM->hwaccm.s.fNestedPaging);
3975
3976 rc2 = VMXReadVMCS64(VMX_VMCS_EXIT_PHYS_ADDR_FULL, &GCPhys);
3977 AssertRC(rc2);
3978 Assert(((exitQualification >> 7) & 3) != 2);
3979
3980 /* Determine the kind of violation. */
3981 errCode = 0;
3982 if (exitQualification & VMX_EXIT_QUALIFICATION_EPT_INSTR_FETCH)
3983 errCode |= X86_TRAP_PF_ID;
3984
3985 if (exitQualification & VMX_EXIT_QUALIFICATION_EPT_DATA_WRITE)
3986 errCode |= X86_TRAP_PF_RW;
3987
3988 /* If the page is present, then it's a page level protection fault. */
3989 if (exitQualification & VMX_EXIT_QUALIFICATION_EPT_ENTRY_PRESENT)
3990 errCode |= X86_TRAP_PF_P;
3991 else
3992 {
3993 /* Shortcut for APIC TPR reads and writes. */
3994 if ( (GCPhys & 0xfff) == 0x080
3995 && GCPhys > 0x1000000 /* to skip VGA frame buffer accesses */
3996 && fSetupTPRCaching
3997 && (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC))
3998 {
3999 RTGCPHYS GCPhysApicBase;
4000 PDMApicGetBase(pVM, &GCPhysApicBase); /* @todo cache this */
4001 GCPhysApicBase &= PAGE_BASE_GC_MASK;
4002 if (GCPhys == GCPhysApicBase + 0x80)
4003 {
4004 Log(("Enable VT-x virtual APIC access filtering\n"));
4005 rc2 = IOMMMIOMapMMIOHCPage(pVM, GCPhysApicBase, pVM->hwaccm.s.vmx.pAPICPhys, X86_PTE_RW | X86_PTE_P);
4006 AssertRC(rc2);
4007 }
4008 }
4009 }
4010 Log(("EPT Page fault %x at %RGp error code %x\n", (uint32_t)exitQualification, GCPhys, errCode));
4011
4012 /* GCPhys contains the guest physical address of the page fault. */
4013 TRPMAssertTrap(pVCpu, X86_XCPT_PF, TRPM_TRAP);
4014 TRPMSetErrorCode(pVCpu, errCode);
4015 TRPMSetFaultAddress(pVCpu, GCPhys);
4016
4017 /* Handle the pagefault trap for the nested shadow table. */
4018 rc = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, errCode, CPUMCTX2CORE(pCtx), GCPhys);
4019
4020 /*
4021 * Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment below, @bugref{6043}.
4022 */
4023 if ( rc == VINF_SUCCESS
4024 || rc == VERR_PAGE_TABLE_NOT_PRESENT
4025 || rc == VERR_PAGE_NOT_PRESENT)
4026 {
4027 /* We've successfully synced our shadow pages, so let's just continue execution. */
4028 Log2(("Shadow page fault at %RGv cr2=%RGp error code %x\n", (RTGCPTR)pCtx->rip, exitQualification , errCode));
4029 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitReasonNPF);
4030
4031 TRPMResetTrap(pVCpu);
4032 goto ResumeExecution;
4033 }
4034
4035#ifdef VBOX_STRICT
4036 if (rc != VINF_EM_RAW_EMULATE_INSTR)
4037 LogFlow(("PGMTrap0eHandlerNestedPaging at %RGv failed with %Rrc\n", (RTGCPTR)pCtx->rip, VBOXSTRICTRC_VAL(rc)));
4038#endif
4039 /* Need to go back to the recompiler to emulate the instruction. */
4040 TRPMResetTrap(pVCpu);
4041 break;
4042 }
4043
4044 case VMX_EXIT_EPT_MISCONFIG:
4045 {
4046 RTGCPHYS GCPhys;
4047
4048 Assert(pVM->hwaccm.s.fNestedPaging);
4049
4050 rc2 = VMXReadVMCS64(VMX_VMCS_EXIT_PHYS_ADDR_FULL, &GCPhys);
4051 AssertRC(rc2);
4052 Log(("VMX_EXIT_EPT_MISCONFIG for %RGp\n", GCPhys));
4053
4054 /* Shortcut for APIC TPR reads and writes. */
4055 if ( (GCPhys & 0xfff) == 0x080
4056 && GCPhys > 0x1000000 /* to skip VGA frame buffer accesses */
4057 && fSetupTPRCaching
4058 && (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC))
4059 {
4060 RTGCPHYS GCPhysApicBase;
4061 PDMApicGetBase(pVM, &GCPhysApicBase); /* @todo cache this */
4062 GCPhysApicBase &= PAGE_BASE_GC_MASK;
4063 if (GCPhys == GCPhysApicBase + 0x80)
4064 {
4065 Log(("Enable VT-x virtual APIC access filtering\n"));
4066 rc2 = IOMMMIOMapMMIOHCPage(pVM, GCPhysApicBase, pVM->hwaccm.s.vmx.pAPICPhys, X86_PTE_RW | X86_PTE_P);
4067 AssertRC(rc2);
4068 }
4069 }
4070
4071 rc = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pCtx), GCPhys, UINT32_MAX);
4072
4073 /*
4074 * If we succeed, resume execution.
4075 * Or, if fail in interpreting the instruction because we couldn't get the guest physical address
4076 * of the page containing the instruction via the guest's page tables (we would invalidate the guest page
4077 * in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
4078 * weird case. See @bugref{6043}.
4079 */
4080 if ( rc == VINF_SUCCESS
4081 || rc == VERR_PAGE_TABLE_NOT_PRESENT
4082 || rc == VERR_PAGE_NOT_PRESENT)
4083 {
4084 Log2(("PGMR0Trap0eHandlerNPMisconfig(,,,%RGp) at %RGv -> resume\n", GCPhys, (RTGCPTR)pCtx->rip));
4085 goto ResumeExecution;
4086 }
4087
4088 Log2(("PGMR0Trap0eHandlerNPMisconfig(,,,%RGp) at %RGv -> %Rrc\n", GCPhys, (RTGCPTR)pCtx->rip, VBOXSTRICTRC_VAL(rc)));
4089 break;
4090 }
4091
4092 case VMX_EXIT_IRQ_WINDOW: /* 7 Interrupt window. */
4093 /* Clear VM-exit on IF=1 change. */
4094 LogFlow(("VMX_EXIT_IRQ_WINDOW %RGv pending=%d IF=%d\n", (RTGCPTR)pCtx->rip,
4095 VMCPU_FF_ISPENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC|VMCPU_FF_INTERRUPT_PIC)), pCtx->eflags.Bits.u1IF));
4096 pVCpu->hwaccm.s.vmx.proc_ctls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_IRQ_WINDOW_EXIT;
4097 rc2 = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
4098 AssertRC(rc2);
4099 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitIrqWindow);
4100 goto ResumeExecution; /* we check for pending guest interrupts there */
4101
4102 case VMX_EXIT_WBINVD: /* 54 Guest software attempted to execute WBINVD. (conditional) */
4103 case VMX_EXIT_INVD: /* 13 Guest software attempted to execute INVD. (unconditional) */
4104 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitInvd);
4105 /* Skip instruction and continue directly. */
4106 pCtx->rip += cbInstr;
4107 /* Continue execution.*/
4108 goto ResumeExecution;
4109
4110 case VMX_EXIT_CPUID: /* 10 Guest software attempted to execute CPUID. */
4111 {
4112 Log2(("VMX: Cpuid %x\n", pCtx->eax));
4113 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitCpuid);
4114 rc = EMInterpretCpuId(pVM, pVCpu, CPUMCTX2CORE(pCtx));
4115 if (rc == VINF_SUCCESS)
4116 {
4117 /* Update EIP and continue execution. */
4118 Assert(cbInstr == 2);
4119 pCtx->rip += cbInstr;
4120 goto ResumeExecution;
4121 }
4122 AssertMsgFailed(("EMU: cpuid failed with %Rrc\n", VBOXSTRICTRC_VAL(rc)));
4123 rc = VINF_EM_RAW_EMULATE_INSTR;
4124 break;
4125 }
4126
4127 case VMX_EXIT_RDPMC: /* 15 Guest software attempted to execute RDPMC. */
4128 {
4129 Log2(("VMX: Rdpmc %x\n", pCtx->ecx));
4130 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitRdpmc);
4131 rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pCtx));
4132 if (rc == VINF_SUCCESS)
4133 {
4134 /* Update EIP and continue execution. */
4135 Assert(cbInstr == 2);
4136 pCtx->rip += cbInstr;
4137 goto ResumeExecution;
4138 }
4139 rc = VINF_EM_RAW_EMULATE_INSTR;
4140 break;
4141 }
4142
4143 case VMX_EXIT_RDTSC: /* 16 Guest software attempted to execute RDTSC. */
4144 {
4145 Log2(("VMX: Rdtsc\n"));
4146 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitRdtsc);
4147 rc = EMInterpretRdtsc(pVM, pVCpu, CPUMCTX2CORE(pCtx));
4148 if (rc == VINF_SUCCESS)
4149 {
4150 /* Update EIP and continue execution. */
4151 Assert(cbInstr == 2);
4152 pCtx->rip += cbInstr;
4153 goto ResumeExecution;
4154 }
4155 rc = VINF_EM_RAW_EMULATE_INSTR;
4156 break;
4157 }
4158
4159 case VMX_EXIT_RDTSCP: /* 51 Guest software attempted to execute RDTSCP. */
4160 {
4161 Log2(("VMX: Rdtscp\n"));
4162 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitRdtscp);
4163 rc = EMInterpretRdtscp(pVM, pVCpu, pCtx);
4164 if (rc == VINF_SUCCESS)
4165 {
4166 /* Update EIP and continue execution. */
4167 Assert(cbInstr == 3);
4168 pCtx->rip += cbInstr;
4169 goto ResumeExecution;
4170 }
4171 rc = VINF_EM_RAW_EMULATE_INSTR;
4172 break;
4173 }
4174
4175 case VMX_EXIT_INVPG: /* 14 Guest software attempted to execute INVPG. */
4176 {
4177 Log2(("VMX: invlpg\n"));
4178 Assert(!pVM->hwaccm.s.fNestedPaging);
4179
4180 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitInvpg);
4181 rc = EMInterpretInvlpg(pVM, pVCpu, CPUMCTX2CORE(pCtx), exitQualification);
4182 if (rc == VINF_SUCCESS)
4183 {
4184 /* Update EIP and continue execution. */
4185 pCtx->rip += cbInstr;
4186 goto ResumeExecution;
4187 }
4188 AssertMsg(rc == VERR_EM_INTERPRETER, ("EMU: invlpg %RGv failed with %Rrc\n", exitQualification, VBOXSTRICTRC_VAL(rc)));
4189 break;
4190 }
4191
4192 case VMX_EXIT_MONITOR: /* 39 Guest software attempted to execute MONITOR. */
4193 {
4194 Log2(("VMX: monitor\n"));
4195
4196 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitMonitor);
4197 rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pCtx));
4198 if (rc == VINF_SUCCESS)
4199 {
4200 /* Update EIP and continue execution. */
4201 pCtx->rip += cbInstr;
4202 goto ResumeExecution;
4203 }
4204 AssertMsg(rc == VERR_EM_INTERPRETER, ("EMU: monitor failed with %Rrc\n", VBOXSTRICTRC_VAL(rc)));
4205 break;
4206 }
4207
4208 case VMX_EXIT_WRMSR: /* 32 WRMSR. Guest software attempted to execute WRMSR. */
4209 /* When an interrupt is pending, we'll let MSR_K8_LSTAR writes fault in our TPR patch code. */
4210 if ( pVM->hwaccm.s.fTPRPatchingActive
4211 && pCtx->ecx == MSR_K8_LSTAR)
4212 {
4213 Assert(!CPUMIsGuestInLongModeEx(pCtx));
4214 if ((pCtx->eax & 0xff) != u8LastTPR)
4215 {
4216 Log(("VMX: Faulting MSR_K8_LSTAR write with new TPR value %x\n", pCtx->eax & 0xff));
4217
4218 /* Our patch code uses LSTAR for TPR caching. */
4219 rc2 = PDMApicSetTPR(pVCpu, pCtx->eax & 0xff);
4220 AssertRC(rc2);
4221 }
4222
4223 /* Skip the instruction and continue. */
4224 pCtx->rip += cbInstr; /* wrmsr = [0F 30] */
4225
4226 /* Only resume if successful. */
4227 goto ResumeExecution;
4228 }
4229 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_GUEST_MSR;
4230 /* no break */
4231 case VMX_EXIT_RDMSR: /* 31 RDMSR. Guest software attempted to execute RDMSR. */
4232 {
4233 STAM_COUNTER_INC((exitReason == VMX_EXIT_RDMSR) ? &pVCpu->hwaccm.s.StatExitRdmsr : &pVCpu->hwaccm.s.StatExitWrmsr);
4234
4235 /*
4236 * Note: The Intel spec. claims there's an REX version of RDMSR that's slightly different,
4237 * so we play safe by completely disassembling the instruction.
4238 */
4239 Log2(("VMX: %s\n", (exitReason == VMX_EXIT_RDMSR) ? "rdmsr" : "wrmsr"));
4240 rc = EMInterpretInstruction(pVCpu, CPUMCTX2CORE(pCtx), 0);
4241 if (rc == VINF_SUCCESS)
4242 {
4243 /* EIP has been updated already. */
4244 /* Only resume if successful. */
4245 goto ResumeExecution;
4246 }
4247 AssertMsg(rc == VERR_EM_INTERPRETER, ("EMU: %s failed with %Rrc\n",
4248 (exitReason == VMX_EXIT_RDMSR) ? "rdmsr" : "wrmsr", VBOXSTRICTRC_VAL(rc)));
4249 break;
4250 }
4251
4252 case VMX_EXIT_CRX_MOVE: /* 28 Control-register accesses. */
4253 {
4254 STAM_PROFILE_ADV_START(&pVCpu->hwaccm.s.StatExit2Sub2, y2);
4255
4256 switch (VMX_EXIT_QUALIFICATION_CRX_ACCESS(exitQualification))
4257 {
4258 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_WRITE:
4259 {
4260 Log2(("VMX: %RGv mov cr%d, x\n", (RTGCPTR)pCtx->rip, VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification)));
4261 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitCRxWrite[VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification)]);
4262 rc = EMInterpretCRxWrite(pVM, pVCpu, CPUMCTX2CORE(pCtx),
4263 VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification),
4264 VMX_EXIT_QUALIFICATION_CRX_GENREG(exitQualification));
4265 switch (VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification))
4266 {
4267 case 0:
4268 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_GUEST_CR0 | HWACCM_CHANGED_GUEST_CR3;
4269 break;
4270 case 2:
4271 break;
4272 case 3:
4273 Assert(!pVM->hwaccm.s.fNestedPaging || !CPUMIsGuestInPagedProtectedModeEx(pCtx));
4274 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_GUEST_CR3;
4275 break;
4276 case 4:
4277 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_GUEST_CR4;
4278 break;
4279 case 8:
4280 /* CR8 contains the APIC TPR */
4281 Assert(!(pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1
4282 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW));
4283 break;
4284
4285 default:
4286 AssertFailed();
4287 break;
4288 }
4289 break;
4290 }
4291
4292 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ:
4293 {
4294 Log2(("VMX: mov x, crx\n"));
4295 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitCRxRead[VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification)]);
4296
4297 Assert( !pVM->hwaccm.s.fNestedPaging
4298 || !CPUMIsGuestInPagedProtectedModeEx(pCtx)
4299 || VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification) != DISCREG_CR3);
4300
4301 /* CR8 reads only cause an exit when the TPR shadow feature isn't present. */
4302 Assert( VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification) != 8
4303 || !(pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW));
4304
4305 rc = EMInterpretCRxRead(pVM, pVCpu, CPUMCTX2CORE(pCtx),
4306 VMX_EXIT_QUALIFICATION_CRX_GENREG(exitQualification),
4307 VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification));
4308 break;
4309 }
4310
4311 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_CLTS:
4312 {
4313 Log2(("VMX: clts\n"));
4314 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitCLTS);
4315 rc = EMInterpretCLTS(pVM, pVCpu);
4316 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_GUEST_CR0;
4317 break;
4318 }
4319
4320 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_LMSW:
4321 {
4322 Log2(("VMX: lmsw %x\n", VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(exitQualification)));
4323 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitLMSW);
4324 rc = EMInterpretLMSW(pVM, pVCpu, CPUMCTX2CORE(pCtx), VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(exitQualification));
4325 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_GUEST_CR0;
4326 break;
4327 }
4328 }
4329
4330 /* Update EIP if no error occurred. */
4331 if (RT_SUCCESS(rc))
4332 pCtx->rip += cbInstr;
4333
4334 if (rc == VINF_SUCCESS)
4335 {
4336 /* Only resume if successful. */
4337 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub2, y2);
4338 goto ResumeExecution;
4339 }
4340 Assert(rc == VERR_EM_INTERPRETER || rc == VINF_PGM_CHANGE_MODE || rc == VINF_PGM_SYNC_CR3);
4341 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub2, y2);
4342 break;
4343 }
4344
4345 case VMX_EXIT_DRX_MOVE: /* 29 Debug-register accesses. */
4346 {
4347 if ( !DBGFIsStepping(pVCpu)
4348 && !CPUMIsHyperDebugStateActive(pVCpu))
4349 {
4350 /* Disable DRx move intercepts. */
4351 pVCpu->hwaccm.s.vmx.proc_ctls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT;
4352 rc2 = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
4353 AssertRC(rc2);
4354
4355 /* Save the host and load the guest debug state. */
4356 rc2 = CPUMR0LoadGuestDebugState(pVM, pVCpu, pCtx, true /* include DR6 */);
4357 AssertRC(rc2);
4358
4359#ifdef LOG_ENABLED
4360 if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(exitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
4361 {
4362 Log(("VMX_EXIT_DRX_MOVE: write DR%d genreg %d\n", VMX_EXIT_QUALIFICATION_DRX_REGISTER(exitQualification),
4363 VMX_EXIT_QUALIFICATION_DRX_GENREG(exitQualification)));
4364 }
4365 else
4366 Log(("VMX_EXIT_DRX_MOVE: read DR%d\n", VMX_EXIT_QUALIFICATION_DRX_REGISTER(exitQualification)));
4367#endif
4368
4369#ifdef VBOX_WITH_STATISTICS
4370 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatDRxContextSwitch);
4371 if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(exitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
4372 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitDRxWrite);
4373 else
4374 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitDRxRead);
4375#endif
4376
4377 goto ResumeExecution;
4378 }
4379
4380 /** @todo clear VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT after the first
4381 * time and restore DRx registers afterwards */
4382 if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(exitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
4383 {
4384 Log2(("VMX: mov DRx%d, genreg%d\n", VMX_EXIT_QUALIFICATION_DRX_REGISTER(exitQualification),
4385 VMX_EXIT_QUALIFICATION_DRX_GENREG(exitQualification)));
4386 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitDRxWrite);
4387 rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pCtx),
4388 VMX_EXIT_QUALIFICATION_DRX_REGISTER(exitQualification),
4389 VMX_EXIT_QUALIFICATION_DRX_GENREG(exitQualification));
4390 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_GUEST_DEBUG;
4391 Log2(("DR7=%08x\n", pCtx->dr[7]));
4392 }
4393 else
4394 {
4395 Log2(("VMX: mov x, DRx\n"));
4396 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitDRxRead);
4397 rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pCtx),
4398 VMX_EXIT_QUALIFICATION_DRX_GENREG(exitQualification),
4399 VMX_EXIT_QUALIFICATION_DRX_REGISTER(exitQualification));
4400 }
4401 /* Update EIP if no error occurred. */
4402 if (RT_SUCCESS(rc))
4403 pCtx->rip += cbInstr;
4404
4405 if (rc == VINF_SUCCESS)
4406 {
4407 /* Only resume if successful. */
4408 goto ResumeExecution;
4409 }
4410 Assert(rc == VERR_EM_INTERPRETER);
4411 break;
4412 }
4413
4414 /* Note: We'll get a #GP if the IO instruction isn't allowed (IOPL or TSS bitmap); no need to double check. */
4415 case VMX_EXIT_PORT_IO: /* 30 I/O instruction. */
4416 {
4417 STAM_PROFILE_ADV_START(&pVCpu->hwaccm.s.StatExit2Sub1, y1);
4418 uint32_t uPort;
4419 uint32_t uIOWidth = VMX_EXIT_QUALIFICATION_IO_WIDTH(exitQualification);
4420 bool fIOWrite = (VMX_EXIT_QUALIFICATION_IO_DIRECTION(exitQualification) == VMX_EXIT_QUALIFICATION_IO_DIRECTION_OUT);
4421
4422 /** @todo necessary to make the distinction? */
4423 if (VMX_EXIT_QUALIFICATION_IO_ENCODING(exitQualification) == VMX_EXIT_QUALIFICATION_IO_ENCODING_DX)
4424 uPort = pCtx->edx & 0xffff;
4425 else
4426 uPort = VMX_EXIT_QUALIFICATION_IO_PORT(exitQualification); /* Immediate encoding. */
4427
4428 if (RT_UNLIKELY(uIOWidth == 2 || uIOWidth >= 4)) /* paranoia */
4429 {
4430 rc = fIOWrite ? VINF_IOM_R3_IOPORT_WRITE : VINF_IOM_R3_IOPORT_READ;
4431 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub1, y1);
4432 break;
4433 }
4434
4435 uint32_t cbSize = g_aIOSize[uIOWidth];
4436 if (VMX_EXIT_QUALIFICATION_IO_STRING(exitQualification))
4437 {
4438 /* ins/outs */
4439 PDISCPUSTATE pDis = &pVCpu->hwaccm.s.DisState;
4440
4441 /* Disassemble manually to deal with segment prefixes. */
4442 /** @todo VMX_VMCS_EXIT_GUEST_LINEAR_ADDR contains the flat pointer operand of the instruction. */
4443 /** @todo VMX_VMCS32_RO_EXIT_INSTR_INFO also contains segment prefix info. */
4444 rc2 = EMInterpretDisasCurrent(pVM, pVCpu, pDis, NULL);
4445 if (RT_SUCCESS(rc))
4446 {
4447 if (fIOWrite)
4448 {
4449 Log2(("IOMInterpretOUTSEx %RGv %x size=%d\n", (RTGCPTR)pCtx->rip, uPort, cbSize));
4450 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitIOStringWrite);
4451 rc = IOMInterpretOUTSEx(pVM, CPUMCTX2CORE(pCtx), uPort, pDis->fPrefix, (DISCPUMODE)pDis->uAddrMode, cbSize);
4452 }
4453 else
4454 {
4455 Log2(("IOMInterpretINSEx %RGv %x size=%d\n", (RTGCPTR)pCtx->rip, uPort, cbSize));
4456 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitIOStringRead);
4457 rc = IOMInterpretINSEx(pVM, CPUMCTX2CORE(pCtx), uPort, pDis->fPrefix, (DISCPUMODE)pDis->uAddrMode, cbSize);
4458 }
4459 }
4460 else
4461 rc = VINF_EM_RAW_EMULATE_INSTR;
4462 }
4463 else
4464 {
4465 /* Normal in/out */
4466 uint32_t uAndVal = g_aIOOpAnd[uIOWidth];
4467
4468 Assert(!VMX_EXIT_QUALIFICATION_IO_REP(exitQualification));
4469
4470 if (fIOWrite)
4471 {
4472 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitIOWrite);
4473 rc = IOMIOPortWrite(pVM, uPort, pCtx->eax & uAndVal, cbSize);
4474 if (rc == VINF_IOM_R3_IOPORT_WRITE)
4475 HWACCMR0SavePendingIOPortWrite(pVCpu, pCtx->rip, pCtx->rip + cbInstr, uPort, uAndVal, cbSize);
4476 }
4477 else
4478 {
4479 uint32_t u32Val = 0;
4480
4481 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitIORead);
4482 rc = IOMIOPortRead(pVM, uPort, &u32Val, cbSize);
4483 if (IOM_SUCCESS(rc))
4484 {
4485 /* Write back to the EAX register. */
4486 pCtx->eax = (pCtx->eax & ~uAndVal) | (u32Val & uAndVal);
4487 }
4488 else
4489 if (rc == VINF_IOM_R3_IOPORT_READ)
4490 HWACCMR0SavePendingIOPortRead(pVCpu, pCtx->rip, pCtx->rip + cbInstr, uPort, uAndVal, cbSize);
4491 }
4492 }
4493
4494 /*
4495 * Handled the I/O return codes.
4496 * (The unhandled cases end up with rc == VINF_EM_RAW_EMULATE_INSTR.)
4497 */
4498 if (IOM_SUCCESS(rc))
4499 {
4500 /* Update EIP and continue execution. */
4501 pCtx->rip += cbInstr;
4502 if (RT_LIKELY(rc == VINF_SUCCESS))
4503 {
4504 /* If any IO breakpoints are armed, then we should check if a debug trap needs to be generated. */
4505 if (pCtx->dr[7] & X86_DR7_ENABLED_MASK)
4506 {
4507 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatDRxIOCheck);
4508 for (unsigned i = 0; i < 4; i++)
4509 {
4510 unsigned uBPLen = g_aIOSize[X86_DR7_GET_LEN(pCtx->dr[7], i)];
4511
4512 if ( (uPort >= pCtx->dr[i] && uPort < pCtx->dr[i] + uBPLen)
4513 && (pCtx->dr[7] & (X86_DR7_L(i) | X86_DR7_G(i)))
4514 && (pCtx->dr[7] & X86_DR7_RW(i, X86_DR7_RW_IO)) == X86_DR7_RW(i, X86_DR7_RW_IO))
4515 {
4516 uint64_t uDR6;
4517
4518 Assert(CPUMIsGuestDebugStateActive(pVCpu));
4519
4520 uDR6 = ASMGetDR6();
4521
4522 /* Clear all breakpoint status flags and set the one we just hit. */
4523 uDR6 &= ~(X86_DR6_B0|X86_DR6_B1|X86_DR6_B2|X86_DR6_B3);
4524 uDR6 |= (uint64_t)RT_BIT(i);
4525
4526 /*
4527 * Note: AMD64 Architecture Programmer's Manual 13.1:
4528 * Bits 15:13 of the DR6 register is never cleared by the processor and must
4529 * be cleared by software after the contents have been read.
4530 */
4531 ASMSetDR6(uDR6);
4532
4533 /* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
4534 pCtx->dr[7] &= ~X86_DR7_GD;
4535
4536 /* Paranoia. */
4537 pCtx->dr[7] &= 0xffffffff; /* upper 32 bits reserved */
4538 pCtx->dr[7] &= ~(RT_BIT(11) | RT_BIT(12) | RT_BIT(14) | RT_BIT(15)); /* must be zero */
4539 pCtx->dr[7] |= 0x400; /* must be one */
4540
4541 /* Resync DR7 */
4542 rc2 = VMXWriteVMCS64(VMX_VMCS64_GUEST_DR7, pCtx->dr[7]);
4543 AssertRC(rc2);
4544
4545 /* Construct inject info. */
4546 intInfo = X86_XCPT_DB;
4547 intInfo |= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
4548 intInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HWEXCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
4549
4550 Log(("Inject IO debug trap at %RGv\n", (RTGCPTR)pCtx->rip));
4551 rc2 = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo),
4552 0 /* cbInstr */, 0 /* errCode */);
4553 AssertRC(rc2);
4554
4555 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub1, y1);
4556 goto ResumeExecution;
4557 }
4558 }
4559 }
4560 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub1, y1);
4561 goto ResumeExecution;
4562 }
4563 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub1, y1);
4564 break;
4565 }
4566
4567#ifdef VBOX_STRICT
4568 if (rc == VINF_IOM_R3_IOPORT_READ)
4569 Assert(!fIOWrite);
4570 else if (rc == VINF_IOM_R3_IOPORT_WRITE)
4571 Assert(fIOWrite);
4572 else
4573 {
4574 AssertMsg( RT_FAILURE(rc)
4575 || rc == VINF_EM_RAW_EMULATE_INSTR
4576 || rc == VINF_EM_RAW_GUEST_TRAP
4577 || rc == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", VBOXSTRICTRC_VAL(rc)));
4578 }
4579#endif
4580 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub1, y1);
4581 break;
4582 }
4583
4584 case VMX_EXIT_TPR: /* 43 TPR below threshold. Guest software executed MOV to CR8. */
4585 LogFlow(("VMX_EXIT_TPR\n"));
4586 /* RIP is already set to the next instruction and the TPR has been synced back. Just resume. */
4587 goto ResumeExecution;
4588
4589 case VMX_EXIT_APIC_ACCESS: /* 44 APIC access. Guest software attempted to access memory at a physical address
4590 on the APIC-access page. */
4591 {
4592 LogFlow(("VMX_EXIT_APIC_ACCESS\n"));
4593 unsigned uAccessType = VMX_EXIT_QUALIFICATION_APIC_ACCESS_TYPE(exitQualification);
4594
4595 switch(uAccessType)
4596 {
4597 case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
4598 case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
4599 {
4600 RTGCPHYS GCPhys;
4601 PDMApicGetBase(pVM, &GCPhys);
4602 GCPhys &= PAGE_BASE_GC_MASK;
4603 GCPhys += VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(exitQualification);
4604
4605 LogFlow(("Apic access at %RGp\n", GCPhys));
4606 rc = IOMMMIOPhysHandler(pVM, (uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ) ? 0 : X86_TRAP_PF_RW,
4607 CPUMCTX2CORE(pCtx), GCPhys);
4608 if (rc == VINF_SUCCESS)
4609 goto ResumeExecution; /* rip already updated */
4610 break;
4611 }
4612
4613 default:
4614 rc = VINF_EM_RAW_EMULATE_INSTR;
4615 break;
4616 }
4617 break;
4618 }
4619
4620 case VMX_EXIT_PREEMPTION_TIMER: /* 52 VMX-preemption timer expired. The preemption timer counted down to zero. */
4621 if (!TMTimerPollBool(pVM, pVCpu))
4622 goto ResumeExecution;
4623 rc = VINF_EM_RAW_TIMER_PENDING;
4624 break;
4625
4626 default:
4627 /* The rest is handled after syncing the entire CPU state. */
4628 break;
4629 }
4630
4631
4632 /*
4633 * Note: The guest state is not entirely synced back at this stage!
4634 */
4635
4636 /* Investigate why there was a VM-exit. (part 2) */
4637 switch (exitReason)
4638 {
4639 case VMX_EXIT_EXCEPTION: /* 0 Exception or non-maskable interrupt (NMI). */
4640 case VMX_EXIT_EXTERNAL_IRQ: /* 1 External interrupt. */
4641 case VMX_EXIT_EPT_VIOLATION:
4642 case VMX_EXIT_EPT_MISCONFIG: /* 49 EPT misconfig is used by the PGM/MMIO optimizations. */
4643 case VMX_EXIT_PREEMPTION_TIMER: /* 52 VMX-preemption timer expired. The preemption timer counted down to zero. */
4644 /* Already handled above. */
4645 break;
4646
4647 case VMX_EXIT_TRIPLE_FAULT: /* 2 Triple fault. */
4648 rc = VINF_EM_RESET; /* Triple fault equals a reset. */
4649 break;
4650
4651 case VMX_EXIT_INIT_SIGNAL: /* 3 INIT signal. */
4652 case VMX_EXIT_SIPI: /* 4 Start-up IPI (SIPI). */
4653 rc = VINF_EM_RAW_INTERRUPT;
4654 AssertFailed(); /* Can't happen. Yet. */
4655 break;
4656
4657 case VMX_EXIT_IO_SMI_IRQ: /* 5 I/O system-management interrupt (SMI). */
4658 case VMX_EXIT_SMI_IRQ: /* 6 Other SMI. */
4659 rc = VINF_EM_RAW_INTERRUPT;
4660 AssertFailed(); /* Can't happen afaik. */
4661 break;
4662
4663 case VMX_EXIT_TASK_SWITCH: /* 9 Task switch: too complicated to emulate, so fall back to the recompiler */
4664 Log(("VMX_EXIT_TASK_SWITCH: exit=%RX64\n", exitQualification));
4665 if ( (VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE(exitQualification) == VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE_IDT)
4666 && pVCpu->hwaccm.s.Event.fPending)
4667 {
4668 /* Caused by an injected interrupt. */
4669 pVCpu->hwaccm.s.Event.fPending = false;
4670
4671 Log(("VMX_EXIT_TASK_SWITCH: reassert trap %d\n", VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVCpu->hwaccm.s.Event.intInfo)));
4672 Assert(!VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(pVCpu->hwaccm.s.Event.intInfo));
4673 rc2 = TRPMAssertTrap(pVCpu, VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVCpu->hwaccm.s.Event.intInfo), TRPM_HARDWARE_INT);
4674 AssertRC(rc2);
4675 }
4676 /* else Exceptions and software interrupts can just be restarted. */
4677 rc = VERR_EM_INTERPRETER;
4678 break;
4679
4680 case VMX_EXIT_HLT: /* 12 Guest software attempted to execute HLT. */
4681 /* Check if external interrupts are pending; if so, don't switch back. */
4682 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitHlt);
4683 pCtx->rip++; /* skip hlt */
4684 if (EMShouldContinueAfterHalt(pVCpu, pCtx))
4685 goto ResumeExecution;
4686
4687 rc = VINF_EM_HALT;
4688 break;
4689
4690 case VMX_EXIT_MWAIT: /* 36 Guest software executed MWAIT. */
4691 Log2(("VMX: mwait\n"));
4692 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitMwait);
4693 rc = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pCtx));
4694 if ( rc == VINF_EM_HALT
4695 || rc == VINF_SUCCESS)
4696 {
4697 /* Update EIP and continue execution. */
4698 pCtx->rip += cbInstr;
4699
4700 /* Check if external interrupts are pending; if so, don't switch back. */
4701 if ( rc == VINF_SUCCESS
4702 || ( rc == VINF_EM_HALT
4703 && EMShouldContinueAfterHalt(pVCpu, pCtx))
4704 )
4705 goto ResumeExecution;
4706 }
4707 AssertMsg(rc == VERR_EM_INTERPRETER || rc == VINF_EM_HALT, ("EMU: mwait failed with %Rrc\n", VBOXSTRICTRC_VAL(rc)));
4708 break;
4709
4710 case VMX_EXIT_RSM: /* 17 Guest software attempted to execute RSM in SMM. */
4711 AssertFailed(); /* can't happen. */
4712 rc = VERR_EM_INTERPRETER;
4713 break;
4714
4715 case VMX_EXIT_MTF: /* 37 Exit due to Monitor Trap Flag. */
4716 LogFlow(("VMX_EXIT_MTF at %RGv\n", (RTGCPTR)pCtx->rip));
4717 pVCpu->hwaccm.s.vmx.proc_ctls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_TRAP_FLAG;
4718 rc2 = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
4719 AssertRC(rc2);
4720 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitMTF);
4721#if 0
4722 DBGFDoneStepping(pVCpu);
4723#endif
4724 rc = VINF_EM_DBG_STOP;
4725 break;
4726
4727 case VMX_EXIT_VMCALL: /* 18 Guest software executed VMCALL. */
4728 case VMX_EXIT_VMCLEAR: /* 19 Guest software executed VMCLEAR. */
4729 case VMX_EXIT_VMLAUNCH: /* 20 Guest software executed VMLAUNCH. */
4730 case VMX_EXIT_VMPTRLD: /* 21 Guest software executed VMPTRLD. */
4731 case VMX_EXIT_VMPTRST: /* 22 Guest software executed VMPTRST. */
4732 case VMX_EXIT_VMREAD: /* 23 Guest software executed VMREAD. */
4733 case VMX_EXIT_VMRESUME: /* 24 Guest software executed VMRESUME. */
4734 case VMX_EXIT_VMWRITE: /* 25 Guest software executed VMWRITE. */
4735 case VMX_EXIT_VMXOFF: /* 26 Guest software executed VMXOFF. */
4736 case VMX_EXIT_VMXON: /* 27 Guest software executed VMXON. */
4737 /** @todo inject #UD immediately */
4738 rc = VERR_EM_INTERPRETER;
4739 break;
4740
4741 case VMX_EXIT_CPUID: /* 10 Guest software attempted to execute CPUID. */
4742 case VMX_EXIT_RDTSC: /* 16 Guest software attempted to execute RDTSC. */
4743 case VMX_EXIT_INVPG: /* 14 Guest software attempted to execute INVPG. */
4744 case VMX_EXIT_CRX_MOVE: /* 28 Control-register accesses. */
4745 case VMX_EXIT_DRX_MOVE: /* 29 Debug-register accesses. */
4746 case VMX_EXIT_PORT_IO: /* 30 I/O instruction. */
4747 case VMX_EXIT_RDPMC: /* 15 Guest software attempted to execute RDPMC. */
4748 case VMX_EXIT_RDTSCP: /* 51 Guest software attempted to execute RDTSCP. */
4749 /* already handled above */
4750 AssertMsg( rc == VINF_PGM_CHANGE_MODE
4751 || rc == VINF_EM_RAW_INTERRUPT
4752 || rc == VERR_EM_INTERPRETER
4753 || rc == VINF_EM_RAW_EMULATE_INSTR
4754 || rc == VINF_PGM_SYNC_CR3
4755 || rc == VINF_IOM_R3_IOPORT_READ
4756 || rc == VINF_IOM_R3_IOPORT_WRITE
4757 || rc == VINF_EM_RAW_GUEST_TRAP
4758 || rc == VINF_TRPM_XCPT_DISPATCHED
4759 || rc == VINF_EM_RESCHEDULE_REM,
4760 ("rc = %d\n", VBOXSTRICTRC_VAL(rc)));
4761 break;
4762
4763 case VMX_EXIT_TPR: /* 43 TPR below threshold. Guest software executed MOV to CR8. */
4764 case VMX_EXIT_RDMSR: /* 31 RDMSR. Guest software attempted to execute RDMSR. */
4765 case VMX_EXIT_WRMSR: /* 32 WRMSR. Guest software attempted to execute WRMSR. */
4766 case VMX_EXIT_PAUSE: /* 40 Guest software attempted to execute PAUSE. */
4767 case VMX_EXIT_MONITOR: /* 39 Guest software attempted to execute MONITOR. */
4768 case VMX_EXIT_APIC_ACCESS: /* 44 APIC access. Guest software attempted to access memory at a physical address
4769 on the APIC-access page. */
4770 {
4771 /*
4772 * If we decided to emulate them here, then we must sync the MSRs that could have been changed (sysenter, FS/GS base)
4773 */
4774 rc = VERR_EM_INTERPRETER;
4775 break;
4776 }
4777
4778 case VMX_EXIT_IRQ_WINDOW: /* 7 Interrupt window. */
4779 Assert(rc == VINF_EM_RAW_INTERRUPT);
4780 break;
4781
4782 case VMX_EXIT_ERR_INVALID_GUEST_STATE: /* 33 VM-entry failure due to invalid guest state. */
4783 {
4784#ifdef VBOX_STRICT
4785 RTCCUINTREG val2 = 0;
4786
4787 Log(("VMX_EXIT_ERR_INVALID_GUEST_STATE\n"));
4788
4789 VMXReadVMCS(VMX_VMCS64_GUEST_RIP, &val2);
4790 Log(("Old eip %RGv new %RGv\n", (RTGCPTR)pCtx->rip, (RTGCPTR)val2));
4791
4792 VMXReadVMCS(VMX_VMCS64_GUEST_CR0, &val2);
4793 Log(("VMX_VMCS_GUEST_CR0 %RX64\n", (uint64_t)val2));
4794
4795 VMXReadVMCS(VMX_VMCS64_GUEST_CR3, &val2);
4796 Log(("VMX_VMCS_GUEST_CR3 %RX64\n", (uint64_t)val2));
4797
4798 VMXReadVMCS(VMX_VMCS64_GUEST_CR4, &val2);
4799 Log(("VMX_VMCS_GUEST_CR4 %RX64\n", (uint64_t)val2));
4800
4801 VMXReadVMCS(VMX_VMCS_GUEST_RFLAGS, &val2);
4802 Log(("VMX_VMCS_GUEST_RFLAGS %08x\n", val2));
4803
4804 VMX_LOG_SELREG(CS, "CS", val2);
4805 VMX_LOG_SELREG(DS, "DS", val2);
4806 VMX_LOG_SELREG(ES, "ES", val2);
4807 VMX_LOG_SELREG(FS, "FS", val2);
4808 VMX_LOG_SELREG(GS, "GS", val2);
4809 VMX_LOG_SELREG(SS, "SS", val2);
4810 VMX_LOG_SELREG(TR, "TR", val2);
4811 VMX_LOG_SELREG(LDTR, "LDTR", val2);
4812
4813 VMXReadVMCS(VMX_VMCS64_GUEST_GDTR_BASE, &val2);
4814 Log(("VMX_VMCS_GUEST_GDTR_BASE %RX64\n", (uint64_t)val2));
4815 VMXReadVMCS(VMX_VMCS64_GUEST_IDTR_BASE, &val2);
4816 Log(("VMX_VMCS_GUEST_IDTR_BASE %RX64\n", (uint64_t)val2));
4817#endif /* VBOX_STRICT */
4818 rc = VERR_VMX_INVALID_GUEST_STATE;
4819 break;
4820 }
4821
4822 case VMX_EXIT_ERR_MSR_LOAD: /* 34 VM-entry failure due to MSR loading. */
4823 case VMX_EXIT_ERR_MACHINE_CHECK: /* 41 VM-entry failure due to machine-check. */
4824 default:
4825 rc = VERR_VMX_UNEXPECTED_EXIT_CODE;
4826 AssertMsgFailed(("Unexpected exit code %d\n", exitReason)); /* Can't happen. */
4827 break;
4828
4829 }
4830
4831end:
4832 /* We now going back to ring-3, so clear the action flag. */
4833 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
4834
4835 /*
4836 * Signal changes for the recompiler.
4837 */
4838 CPUMSetChangedFlags(pVCpu,
4839 CPUM_CHANGED_SYSENTER_MSR
4840 | CPUM_CHANGED_LDTR
4841 | CPUM_CHANGED_GDTR
4842 | CPUM_CHANGED_IDTR
4843 | CPUM_CHANGED_TR
4844 | CPUM_CHANGED_HIDDEN_SEL_REGS);
4845
4846 /*
4847 * If we executed vmlaunch/vmresume and an external IRQ was pending, then we don't have to do a full sync the next time.
4848 */
4849 if ( exitReason == VMX_EXIT_EXTERNAL_IRQ
4850 && !VMX_EXIT_INTERRUPTION_INFO_VALID(intInfo))
4851 {
4852 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatPendingHostIrq);
4853 /* On the next entry we'll only sync the host context. */
4854 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_HOST_CONTEXT;
4855 }
4856 else
4857 {
4858 /* On the next entry we'll sync everything. */
4859 /** @todo we can do better than this */
4860 /* Not in the VINF_PGM_CHANGE_MODE though! */
4861 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_ALL;
4862 }
4863
4864 /* Translate into a less severe return code */
4865 if (rc == VERR_EM_INTERPRETER)
4866 rc = VINF_EM_RAW_EMULATE_INSTR;
4867 else if (rc == VERR_VMX_INVALID_VMCS_PTR)
4868 {
4869 /* Try to extract more information about what might have gone wrong here. */
4870 VMXGetActivateVMCS(&pVCpu->hwaccm.s.vmx.lasterror.u64VMCSPhys);
4871 pVCpu->hwaccm.s.vmx.lasterror.ulVMCSRevision = *(uint32_t *)pVCpu->hwaccm.s.vmx.pvVMCS;
4872 pVCpu->hwaccm.s.vmx.lasterror.idEnteredCpu = pVCpu->hwaccm.s.idEnteredCpu;
4873 pVCpu->hwaccm.s.vmx.lasterror.idCurrentCpu = RTMpCpuId();
4874 }
4875
4876 /* Just set the correct state here instead of trying to catch every goto above. */
4877 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED, VMCPUSTATE_STARTED_EXEC);
4878
4879#ifdef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
4880 /* Restore interrupts if we exited after disabling them. */
4881 if (uOldEFlags != ~(RTCCUINTREG)0)
4882 ASMSetFlags(uOldEFlags);
4883#endif
4884
4885 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2, x);
4886 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit1, x);
4887 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatEntry, x);
4888 Log2(("X"));
4889 return VBOXSTRICTRC_TODO(rc);
4890}
4891
4892
4893/**
4894 * Enters the VT-x session.
4895 *
4896 * @returns VBox status code.
4897 * @param pVM Pointer to the VM.
4898 * @param pVCpu Pointer to the VMCPU.
4899 * @param pCpu Pointer to the CPU info struct.
4900 */
4901VMMR0DECL(int) VMXR0Enter(PVM pVM, PVMCPU pVCpu, PHMGLOBLCPUINFO pCpu)
4902{
4903 Assert(pVM->hwaccm.s.vmx.fSupported);
4904 NOREF(pCpu);
4905
4906 unsigned cr4 = ASMGetCR4();
4907 if (!(cr4 & X86_CR4_VMXE))
4908 {
4909 AssertMsgFailed(("X86_CR4_VMXE should be set!\n"));
4910 return VERR_VMX_X86_CR4_VMXE_CLEARED;
4911 }
4912
4913 /* Activate the VMCS. */
4914 int rc = VMXActivateVMCS(pVCpu->hwaccm.s.vmx.HCPhysVMCS);
4915 if (RT_FAILURE(rc))
4916 return rc;
4917
4918 pVCpu->hwaccm.s.fResumeVM = false;
4919 return VINF_SUCCESS;
4920}
4921
4922
4923/**
4924 * Leaves the VT-x session.
4925 *
4926 * @returns VBox status code.
4927 * @param pVM Pointer to the VM.
4928 * @param pVCpu Pointer to the VMCPU.
4929 * @param pCtx Pointer to the guests CPU context.
4930 */
4931VMMR0DECL(int) VMXR0Leave(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
4932{
4933 Assert(pVM->hwaccm.s.vmx.fSupported);
4934
4935#ifdef DEBUG
4936 if (CPUMIsHyperDebugStateActive(pVCpu))
4937 {
4938 CPUMR0LoadHostDebugState(pVM, pVCpu);
4939 Assert(pVCpu->hwaccm.s.vmx.proc_ctls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT);
4940 }
4941 else
4942#endif
4943
4944 /*
4945 * Save the guest debug state if necessary.
4946 */
4947 if (CPUMIsGuestDebugStateActive(pVCpu))
4948 {
4949 CPUMR0SaveGuestDebugState(pVM, pVCpu, pCtx, true /* save DR6 */);
4950
4951 /* Enable DRx move intercepts again. */
4952 pVCpu->hwaccm.s.vmx.proc_ctls |= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT;
4953 int rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
4954 AssertRC(rc);
4955
4956 /* Resync the debug registers the next time. */
4957 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_GUEST_DEBUG;
4958 }
4959 else
4960 Assert(pVCpu->hwaccm.s.vmx.proc_ctls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT);
4961
4962 /*
4963 * Clear VMCS, marking it inactive, clearing implementation-specific data and writing
4964 * VMCS data back to memory.
4965 */
4966 int rc = VMXClearVMCS(pVCpu->hwaccm.s.vmx.HCPhysVMCS);
4967 AssertRC(rc);
4968
4969 return VINF_SUCCESS;
4970}
4971
4972
4973/**
4974 * Flush the TLB using EPT.
4975 *
4976 * @returns VBox status code.
4977 * @param pVM Pointer to the VM.
4978 * @param pVCpu Pointer to the VMCPU.
4979 * @param enmFlush Type of flush.
4980 */
4981static void hmR0VmxFlushEPT(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_EPT enmFlush)
4982{
4983 uint64_t descriptor[2];
4984
4985 LogFlow(("hmR0VmxFlushEPT %d\n", enmFlush));
4986 Assert(pVM->hwaccm.s.fNestedPaging);
4987 descriptor[0] = pVCpu->hwaccm.s.vmx.GCPhysEPTP;
4988 descriptor[1] = 0; /* MBZ. Intel spec. 33.3 VMX Instructions */
4989 int rc = VMXR0InvEPT(enmFlush, &descriptor[0]);
4990 AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %x %RGv failed with %d\n", enmFlush, pVCpu->hwaccm.s.vmx.GCPhysEPTP, rc));
4991}
4992
4993
4994/**
4995 * Flush the TLB using VPID.
4996 *
4997 * @returns VBox status code.
4998 * @param pVM Pointer to the VM.
4999 * @param pVCpu Pointer to the VMCPU (can be NULL depending on @a
5000 * enmFlush).
5001 * @param enmFlush Type of flush.
5002 * @param GCPtr Virtual address of the page to flush (can be 0 depending
5003 * on @a enmFlush).
5004 */
5005static void hmR0VmxFlushVPID(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_VPID enmFlush, RTGCPTR GCPtr)
5006{
5007#if HC_ARCH_BITS == 32
5008 /*
5009 * If we get a flush in 64-bit guest mode, then force a full TLB flush. invvpid probably takes only 32-bit addresses.
5010 */
5011 if ( CPUMIsGuestInLongMode(pVCpu)
5012 && !VMX_IS_64BIT_HOST_MODE())
5013 {
5014 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
5015 }
5016 else
5017#endif
5018 {
5019 uint64_t descriptor[2];
5020
5021 Assert(pVM->hwaccm.s.vmx.fVPID);
5022 if (enmFlush == VMX_FLUSH_VPID_ALL_CONTEXTS)
5023 {
5024 descriptor[0] = 0;
5025 descriptor[1] = 0;
5026 }
5027 else
5028 {
5029 AssertPtr(pVCpu);
5030 Assert(pVCpu->hwaccm.s.uCurrentASID != 0);
5031 descriptor[0] = pVCpu->hwaccm.s.uCurrentASID;
5032 descriptor[1] = GCPtr;
5033 }
5034 int rc = VMXR0InvVPID(enmFlush, &descriptor[0]); NOREF(rc);
5035 AssertMsg(rc == VINF_SUCCESS,
5036 ("VMXR0InvVPID %x %x %RGv failed with %d\n", enmFlush, pVCpu ? pVCpu->hwaccm.s.uCurrentASID : 0, GCPtr, rc));
5037 }
5038}
5039
5040
5041/**
5042 * Invalidates a guest page by guest virtual address. Only relevant for
5043 * EPT/VPID, otherwise there is nothing really to invalidate.
5044 *
5045 * @returns VBox status code.
5046 * @param pVM Pointer to the VM.
5047 * @param pVCpu Pointer to the VMCPU.
5048 * @param GCVirt Guest virtual address of the page to invalidate.
5049 */
5050VMMR0DECL(int) VMXR0InvalidatePage(PVM pVM, PVMCPU pVCpu, RTGCPTR GCVirt)
5051{
5052 bool fFlushPending = VMCPU_FF_ISSET(pVCpu, VMCPU_FF_TLB_FLUSH);
5053
5054 Log2(("VMXR0InvalidatePage %RGv\n", GCVirt));
5055
5056 if (!fFlushPending)
5057 {
5058 /*
5059 * We must invalidate the guest TLB entry in either case, we cannot ignore it even for the EPT case
5060 * See @bugref{6043} and @bugref{6177}
5061 *
5062 * Set the VMCPU_FF_TLB_FLUSH force flag and flush before VMENTRY in hmR0VmxSetupTLB*() as this
5063 * function maybe called in a loop with individual addresses.
5064 */
5065 if (pVM->hwaccm.s.vmx.fVPID)
5066 {
5067 /* If we can flush just this page do it, otherwise flush as little as possible. */
5068 if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVVPID_CAPS_INDIV_ADDR)
5069 hmR0VmxFlushVPID(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, GCVirt);
5070 else
5071 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
5072 }
5073 else if (pVM->hwaccm.s.fNestedPaging)
5074 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
5075 }
5076
5077 return VINF_SUCCESS;
5078}
5079
5080
5081/**
5082 * Invalidates a guest page by physical address. Only relevant for EPT/VPID,
5083 * otherwise there is nothing really to invalidate.
5084 *
5085 * NOTE: Assumes the current instruction references this physical page though a virtual address!!
5086 *
5087 * @returns VBox status code.
5088 * @param pVM Pointer to the VM.
5089 * @param pVCpu Pointer to the VMCPU.
5090 * @param GCPhys Guest physical address of the page to invalidate.
5091 */
5092VMMR0DECL(int) VMXR0InvalidatePhysPage(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys)
5093{
5094 LogFlow(("VMXR0InvalidatePhysPage %RGp\n", GCPhys));
5095
5096 /*
5097 * We cannot flush a page by guest-physical address. invvpid takes only a linear address
5098 * while invept only flushes by EPT not individual addresses. We update the force flag here
5099 * and flush before VMENTRY in hmR0VmxSetupTLB*(). This function might be called in a loop.
5100 */
5101 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
5102 return VINF_SUCCESS;
5103}
5104
5105
5106/**
5107 * Report world switch error and dump some useful debug info.
5108 *
5109 * @param pVM Pointer to the VM.
5110 * @param pVCpu Pointer to the VMCPU.
5111 * @param rc Return code.
5112 * @param pCtx Pointer to the current guest CPU context (not updated).
5113 */
5114static void hmR0VmxReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, VBOXSTRICTRC rc, PCPUMCTX pCtx)
5115{
5116 NOREF(pVM);
5117
5118 switch (VBOXSTRICTRC_VAL(rc))
5119 {
5120 case VERR_VMX_INVALID_VMXON_PTR:
5121 AssertFailed();
5122 break;
5123
5124 case VERR_VMX_UNABLE_TO_START_VM:
5125 case VERR_VMX_UNABLE_TO_RESUME_VM:
5126 {
5127 int rc2;
5128 RTCCUINTREG exitReason, instrError;
5129
5130 rc2 = VMXReadVMCS(VMX_VMCS32_RO_EXIT_REASON, &exitReason);
5131 rc2 |= VMXReadVMCS(VMX_VMCS32_RO_VM_INSTR_ERROR, &instrError);
5132 AssertRC(rc2);
5133 if (rc2 == VINF_SUCCESS)
5134 {
5135 Log(("Unable to start/resume VM for reason: %x. Instruction error %x\n", (uint32_t)exitReason,
5136 (uint32_t)instrError));
5137 Log(("Current stack %08x\n", &rc2));
5138
5139 pVCpu->hwaccm.s.vmx.lasterror.ulInstrError = instrError;
5140 pVCpu->hwaccm.s.vmx.lasterror.ulExitReason = exitReason;
5141
5142#ifdef VBOX_STRICT
5143 RTGDTR gdtr;
5144 PCX86DESCHC pDesc;
5145 RTCCUINTREG val;
5146
5147 ASMGetGDTR(&gdtr);
5148
5149 VMXReadVMCS(VMX_VMCS64_GUEST_RIP, &val);
5150 Log(("Old eip %RGv new %RGv\n", (RTGCPTR)pCtx->rip, (RTGCPTR)val));
5151 VMXReadVMCS(VMX_VMCS_CTRL_PIN_EXEC_CONTROLS, &val);
5152 Log(("VMX_VMCS_CTRL_PIN_EXEC_CONTROLS %08x\n", val));
5153 VMXReadVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, &val);
5154 Log(("VMX_VMCS_CTRL_PROC_EXEC_CONTROLS %08x\n", val));
5155 VMXReadVMCS(VMX_VMCS_CTRL_ENTRY_CONTROLS, &val);
5156 Log(("VMX_VMCS_CTRL_ENTRY_CONTROLS %08x\n", val));
5157 VMXReadVMCS(VMX_VMCS_CTRL_EXIT_CONTROLS, &val);
5158 Log(("VMX_VMCS_CTRL_EXIT_CONTROLS %08x\n", val));
5159
5160 VMXReadVMCS(VMX_VMCS_HOST_CR0, &val);
5161 Log(("VMX_VMCS_HOST_CR0 %08x\n", val));
5162 VMXReadVMCS(VMX_VMCS_HOST_CR3, &val);
5163 Log(("VMX_VMCS_HOST_CR3 %08x\n", val));
5164 VMXReadVMCS(VMX_VMCS_HOST_CR4, &val);
5165 Log(("VMX_VMCS_HOST_CR4 %08x\n", val));
5166
5167 VMXReadVMCS(VMX_VMCS16_HOST_FIELD_CS, &val);
5168 Log(("VMX_VMCS_HOST_FIELD_CS %08x\n", val));
5169 VMXReadVMCS(VMX_VMCS_GUEST_RFLAGS, &val);
5170 Log(("VMX_VMCS_GUEST_RFLAGS %08x\n", val));
5171
5172 if (val < gdtr.cbGdt)
5173 {
5174 pDesc = (PCX86DESCHC)(gdtr.pGdt + (val & X86_SEL_MASK));
5175 HWACCMR0DumpDescriptor(pDesc, val, "CS: ");
5176 }
5177
5178 VMXReadVMCS(VMX_VMCS16_HOST_FIELD_DS, &val);
5179 Log(("VMX_VMCS_HOST_FIELD_DS %08x\n", val));
5180 if (val < gdtr.cbGdt)
5181 {
5182 pDesc = (PCX86DESCHC)(gdtr.pGdt + (val & X86_SEL_MASK));
5183 HWACCMR0DumpDescriptor(pDesc, val, "DS: ");
5184 }
5185
5186 VMXReadVMCS(VMX_VMCS16_HOST_FIELD_ES, &val);
5187 Log(("VMX_VMCS_HOST_FIELD_ES %08x\n", val));
5188 if (val < gdtr.cbGdt)
5189 {
5190 pDesc = (PCX86DESCHC)(gdtr.pGdt + (val & X86_SEL_MASK));
5191 HWACCMR0DumpDescriptor(pDesc, val, "ES: ");
5192 }
5193
5194 VMXReadVMCS(VMX_VMCS16_HOST_FIELD_FS, &val);
5195 Log(("VMX_VMCS16_HOST_FIELD_FS %08x\n", val));
5196 if (val < gdtr.cbGdt)
5197 {
5198 pDesc = (PCX86DESCHC)(gdtr.pGdt + (val & X86_SEL_MASK));
5199 HWACCMR0DumpDescriptor(pDesc, val, "FS: ");
5200 }
5201
5202 VMXReadVMCS(VMX_VMCS16_HOST_FIELD_GS, &val);
5203 Log(("VMX_VMCS16_HOST_FIELD_GS %08x\n", val));
5204 if (val < gdtr.cbGdt)
5205 {
5206 pDesc = (PCX86DESCHC)(gdtr.pGdt + (val & X86_SEL_MASK));
5207 HWACCMR0DumpDescriptor(pDesc, val, "GS: ");
5208 }
5209
5210 VMXReadVMCS(VMX_VMCS16_HOST_FIELD_SS, &val);
5211 Log(("VMX_VMCS16_HOST_FIELD_SS %08x\n", val));
5212 if (val < gdtr.cbGdt)
5213 {
5214 pDesc = (PCX86DESCHC)(gdtr.pGdt + (val & X86_SEL_MASK));
5215 HWACCMR0DumpDescriptor(pDesc, val, "SS: ");
5216 }
5217
5218 VMXReadVMCS(VMX_VMCS16_HOST_FIELD_TR, &val);
5219 Log(("VMX_VMCS16_HOST_FIELD_TR %08x\n", val));
5220 if (val < gdtr.cbGdt)
5221 {
5222 pDesc = (PCX86DESCHC)(gdtr.pGdt + (val & X86_SEL_MASK));
5223 HWACCMR0DumpDescriptor(pDesc, val, "TR: ");
5224 }
5225
5226 VMXReadVMCS(VMX_VMCS_HOST_TR_BASE, &val);
5227 Log(("VMX_VMCS_HOST_TR_BASE %RHv\n", val));
5228 VMXReadVMCS(VMX_VMCS_HOST_GDTR_BASE, &val);
5229 Log(("VMX_VMCS_HOST_GDTR_BASE %RHv\n", val));
5230 VMXReadVMCS(VMX_VMCS_HOST_IDTR_BASE, &val);
5231 Log(("VMX_VMCS_HOST_IDTR_BASE %RHv\n", val));
5232 VMXReadVMCS(VMX_VMCS32_HOST_SYSENTER_CS, &val);
5233 Log(("VMX_VMCS_HOST_SYSENTER_CS %08x\n", val));
5234 VMXReadVMCS(VMX_VMCS_HOST_SYSENTER_EIP, &val);
5235 Log(("VMX_VMCS_HOST_SYSENTER_EIP %RHv\n", val));
5236 VMXReadVMCS(VMX_VMCS_HOST_SYSENTER_ESP, &val);
5237 Log(("VMX_VMCS_HOST_SYSENTER_ESP %RHv\n", val));
5238 VMXReadVMCS(VMX_VMCS_HOST_RSP, &val);
5239 Log(("VMX_VMCS_HOST_RSP %RHv\n", val));
5240 VMXReadVMCS(VMX_VMCS_HOST_RIP, &val);
5241 Log(("VMX_VMCS_HOST_RIP %RHv\n", val));
5242# if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
5243 if (VMX_IS_64BIT_HOST_MODE())
5244 {
5245 Log(("MSR_K6_EFER = %RX64\n", ASMRdMsr(MSR_K6_EFER)));
5246 Log(("MSR_K6_STAR = %RX64\n", ASMRdMsr(MSR_K6_STAR)));
5247 Log(("MSR_K8_LSTAR = %RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
5248 Log(("MSR_K8_CSTAR = %RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
5249 Log(("MSR_K8_SF_MASK = %RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
5250 Log(("MSR_K8_KERNEL_GS_BASE = %RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
5251 }
5252# endif
5253#endif /* VBOX_STRICT */
5254 }
5255 break;
5256 }
5257
5258 default:
5259 /* impossible */
5260 AssertMsgFailed(("%Rrc (%#x)\n", VBOXSTRICTRC_VAL(rc), VBOXSTRICTRC_VAL(rc)));
5261 break;
5262 }
5263}
5264
5265
5266#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
5267/**
5268 * Prepares for and executes VMLAUNCH (64 bits guest mode).
5269 *
5270 * @returns VBox status code.
5271 * @param fResume Whether to vmlauch/vmresume.
5272 * @param pCtx Pointer to the guest CPU context.
5273 * @param pCache Pointer to the VMCS cache.
5274 * @param pVM Pointer to the VM.
5275 * @param pVCpu Pointer to the VMCPU.
5276 */
5277DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
5278{
5279 uint32_t aParam[6];
5280 PHMGLOBLCPUINFO pCpu;
5281 RTHCPHYS HCPhysCpuPage;
5282 int rc;
5283
5284 pCpu = HWACCMR0GetCurrentCpu();
5285 HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
5286
5287#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5288 pCache->uPos = 1;
5289 pCache->interPD = PGMGetInterPaeCR3(pVM);
5290 pCache->pSwitcher = (uint64_t)pVM->hwaccm.s.pfnHost32ToGuest64R0;
5291#endif
5292
5293#ifdef DEBUG
5294 pCache->TestIn.HCPhysCpuPage= 0;
5295 pCache->TestIn.HCPhysVMCS = 0;
5296 pCache->TestIn.pCache = 0;
5297 pCache->TestOut.HCPhysVMCS = 0;
5298 pCache->TestOut.pCache = 0;
5299 pCache->TestOut.pCtx = 0;
5300 pCache->TestOut.eflags = 0;
5301#endif
5302
5303 aParam[0] = (uint32_t)(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
5304 aParam[1] = (uint32_t)(HCPhysCpuPage >> 32); /* Param 1: VMXON physical address - Hi. */
5305 aParam[2] = (uint32_t)(pVCpu->hwaccm.s.vmx.HCPhysVMCS); /* Param 2: VMCS physical address - Lo. */
5306 aParam[3] = (uint32_t)(pVCpu->hwaccm.s.vmx.HCPhysVMCS >> 32); /* Param 2: VMCS physical address - Hi. */
5307 aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hwaccm.s.vmx.VMCSCache);
5308 aParam[5] = 0;
5309
5310#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5311 pCtx->dr[4] = pVM->hwaccm.s.vmx.pScratchPhys + 16 + 8;
5312 *(uint32_t *)(pVM->hwaccm.s.vmx.pScratch + 16 + 8) = 1;
5313#endif
5314 rc = VMXR0Execute64BitsHandler(pVM, pVCpu, pCtx, pVM->hwaccm.s.pfnVMXGCStartVM64, 6, &aParam[0]);
5315
5316#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5317 Assert(*(uint32_t *)(pVM->hwaccm.s.vmx.pScratch + 16 + 8) == 5);
5318 Assert(pCtx->dr[4] == 10);
5319 *(uint32_t *)(pVM->hwaccm.s.vmx.pScratch + 16 + 8) = 0xff;
5320#endif
5321
5322#ifdef DEBUG
5323 AssertMsg(pCache->TestIn.HCPhysCpuPage== HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
5324 AssertMsg(pCache->TestIn.HCPhysVMCS == pVCpu->hwaccm.s.vmx.HCPhysVMCS, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVMCS,
5325 pVCpu->hwaccm.s.vmx.HCPhysVMCS));
5326 AssertMsg(pCache->TestIn.HCPhysVMCS == pCache->TestOut.HCPhysVMCS, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVMCS,
5327 pCache->TestOut.HCPhysVMCS));
5328 AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
5329 pCache->TestOut.pCache));
5330 AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hwaccm.s.vmx.VMCSCache),
5331 ("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hwaccm.s.vmx.VMCSCache)));
5332 AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
5333 pCache->TestOut.pCtx));
5334 Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
5335#endif
5336 return rc;
5337}
5338
5339
5340# ifdef VBOX_STRICT
5341static bool hmR0VmxIsValidReadField(uint32_t idxField)
5342{
5343 switch (idxField)
5344 {
5345 case VMX_VMCS64_GUEST_RIP:
5346 case VMX_VMCS64_GUEST_RSP:
5347 case VMX_VMCS_GUEST_RFLAGS:
5348 case VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE:
5349 case VMX_VMCS_CTRL_CR0_READ_SHADOW:
5350 case VMX_VMCS64_GUEST_CR0:
5351 case VMX_VMCS_CTRL_CR4_READ_SHADOW:
5352 case VMX_VMCS64_GUEST_CR4:
5353 case VMX_VMCS64_GUEST_DR7:
5354 case VMX_VMCS32_GUEST_SYSENTER_CS:
5355 case VMX_VMCS64_GUEST_SYSENTER_EIP:
5356 case VMX_VMCS64_GUEST_SYSENTER_ESP:
5357 case VMX_VMCS32_GUEST_GDTR_LIMIT:
5358 case VMX_VMCS64_GUEST_GDTR_BASE:
5359 case VMX_VMCS32_GUEST_IDTR_LIMIT:
5360 case VMX_VMCS64_GUEST_IDTR_BASE:
5361 case VMX_VMCS16_GUEST_FIELD_CS:
5362 case VMX_VMCS32_GUEST_CS_LIMIT:
5363 case VMX_VMCS64_GUEST_CS_BASE:
5364 case VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS:
5365 case VMX_VMCS16_GUEST_FIELD_DS:
5366 case VMX_VMCS32_GUEST_DS_LIMIT:
5367 case VMX_VMCS64_GUEST_DS_BASE:
5368 case VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS:
5369 case VMX_VMCS16_GUEST_FIELD_ES:
5370 case VMX_VMCS32_GUEST_ES_LIMIT:
5371 case VMX_VMCS64_GUEST_ES_BASE:
5372 case VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS:
5373 case VMX_VMCS16_GUEST_FIELD_FS:
5374 case VMX_VMCS32_GUEST_FS_LIMIT:
5375 case VMX_VMCS64_GUEST_FS_BASE:
5376 case VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS:
5377 case VMX_VMCS16_GUEST_FIELD_GS:
5378 case VMX_VMCS32_GUEST_GS_LIMIT:
5379 case VMX_VMCS64_GUEST_GS_BASE:
5380 case VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS:
5381 case VMX_VMCS16_GUEST_FIELD_SS:
5382 case VMX_VMCS32_GUEST_SS_LIMIT:
5383 case VMX_VMCS64_GUEST_SS_BASE:
5384 case VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS:
5385 case VMX_VMCS16_GUEST_FIELD_LDTR:
5386 case VMX_VMCS32_GUEST_LDTR_LIMIT:
5387 case VMX_VMCS64_GUEST_LDTR_BASE:
5388 case VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS:
5389 case VMX_VMCS16_GUEST_FIELD_TR:
5390 case VMX_VMCS32_GUEST_TR_LIMIT:
5391 case VMX_VMCS64_GUEST_TR_BASE:
5392 case VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS:
5393 case VMX_VMCS32_RO_EXIT_REASON:
5394 case VMX_VMCS32_RO_VM_INSTR_ERROR:
5395 case VMX_VMCS32_RO_EXIT_INSTR_LENGTH:
5396 case VMX_VMCS32_RO_EXIT_INTERRUPTION_ERRCODE:
5397 case VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO:
5398 case VMX_VMCS32_RO_EXIT_INSTR_INFO:
5399 case VMX_VMCS_RO_EXIT_QUALIFICATION:
5400 case VMX_VMCS32_RO_IDT_INFO:
5401 case VMX_VMCS32_RO_IDT_ERRCODE:
5402 case VMX_VMCS64_GUEST_CR3:
5403 case VMX_VMCS_EXIT_PHYS_ADDR_FULL:
5404 return true;
5405 }
5406 return false;
5407}
5408
5409
5410static bool hmR0VmxIsValidWriteField(uint32_t idxField)
5411{
5412 switch (idxField)
5413 {
5414 case VMX_VMCS64_GUEST_LDTR_BASE:
5415 case VMX_VMCS64_GUEST_TR_BASE:
5416 case VMX_VMCS64_GUEST_GDTR_BASE:
5417 case VMX_VMCS64_GUEST_IDTR_BASE:
5418 case VMX_VMCS64_GUEST_SYSENTER_EIP:
5419 case VMX_VMCS64_GUEST_SYSENTER_ESP:
5420 case VMX_VMCS64_GUEST_CR0:
5421 case VMX_VMCS64_GUEST_CR4:
5422 case VMX_VMCS64_GUEST_CR3:
5423 case VMX_VMCS64_GUEST_DR7:
5424 case VMX_VMCS64_GUEST_RIP:
5425 case VMX_VMCS64_GUEST_RSP:
5426 case VMX_VMCS64_GUEST_CS_BASE:
5427 case VMX_VMCS64_GUEST_DS_BASE:
5428 case VMX_VMCS64_GUEST_ES_BASE:
5429 case VMX_VMCS64_GUEST_FS_BASE:
5430 case VMX_VMCS64_GUEST_GS_BASE:
5431 case VMX_VMCS64_GUEST_SS_BASE:
5432 return true;
5433 }
5434 return false;
5435}
5436# endif /* VBOX_STRICT */
5437
5438
5439/**
5440 * Executes the specified handler in 64-bit mode.
5441 *
5442 * @returns VBox status code.
5443 * @param pVM Pointer to the VM.
5444 * @param pVCpu Pointer to the VMCPU.
5445 * @param pCtx Pointer to the guest CPU context.
5446 * @param pfnHandler Pointer to the RC handler function.
5447 * @param cbParam Number of parameters.
5448 * @param paParam Array of 32-bit parameters.
5449 */
5450VMMR0DECL(int) VMXR0Execute64BitsHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, RTRCPTR pfnHandler, uint32_t cbParam,
5451 uint32_t *paParam)
5452{
5453 int rc, rc2;
5454 PHMGLOBLCPUINFO pCpu;
5455 RTHCPHYS HCPhysCpuPage;
5456 RTHCUINTREG uOldEFlags;
5457
5458 AssertReturn(pVM->hwaccm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
5459 Assert(pfnHandler);
5460 Assert(pVCpu->hwaccm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hwaccm.s.vmx.VMCSCache.Write.aField));
5461 Assert(pVCpu->hwaccm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hwaccm.s.vmx.VMCSCache.Read.aField));
5462
5463#ifdef VBOX_STRICT
5464 for (unsigned i=0;i<pVCpu->hwaccm.s.vmx.VMCSCache.Write.cValidEntries;i++)
5465 Assert(hmR0VmxIsValidWriteField(pVCpu->hwaccm.s.vmx.VMCSCache.Write.aField[i]));
5466
5467 for (unsigned i=0;i<pVCpu->hwaccm.s.vmx.VMCSCache.Read.cValidEntries;i++)
5468 Assert(hmR0VmxIsValidReadField(pVCpu->hwaccm.s.vmx.VMCSCache.Read.aField[i]));
5469#endif
5470
5471 /* Disable interrupts. */
5472 uOldEFlags = ASMIntDisableFlags();
5473
5474#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
5475 RTCPUID idHostCpu = RTMpCpuId();
5476 CPUMR0SetLApic(pVM, idHostCpu);
5477#endif
5478
5479 pCpu = HWACCMR0GetCurrentCpu();
5480 HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
5481
5482 /* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
5483 VMXClearVMCS(pVCpu->hwaccm.s.vmx.HCPhysVMCS);
5484
5485 /* Leave VMX Root Mode. */
5486 VMXDisable();
5487
5488 ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
5489
5490 CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
5491 CPUMSetHyperEIP(pVCpu, pfnHandler);
5492 for (int i=(int)cbParam-1;i>=0;i--)
5493 CPUMPushHyper(pVCpu, paParam[i]);
5494
5495 STAM_PROFILE_ADV_START(&pVCpu->hwaccm.s.StatWorldSwitch3264, z);
5496
5497 /* Call switcher. */
5498 rc = pVM->hwaccm.s.pfnHost32ToGuest64R0(pVM, RT_OFFSETOF(VM, aCpus[pVCpu->idCpu].cpum) - RT_OFFSETOF(VM, cpum));
5499 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatWorldSwitch3264, z);
5500
5501 /* Make sure the VMX instructions don't cause #UD faults. */
5502 ASMSetCR4(ASMGetCR4() | X86_CR4_VMXE);
5503
5504 /* Enter VMX Root Mode */
5505 rc2 = VMXEnable(HCPhysCpuPage);
5506 if (RT_FAILURE(rc2))
5507 {
5508 ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
5509 ASMSetFlags(uOldEFlags);
5510 return VERR_VMX_VMXON_FAILED;
5511 }
5512
5513 rc2 = VMXActivateVMCS(pVCpu->hwaccm.s.vmx.HCPhysVMCS);
5514 AssertRC(rc2);
5515 Assert(!(ASMGetFlags() & X86_EFL_IF));
5516 ASMSetFlags(uOldEFlags);
5517 return rc;
5518}
5519#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL) */
5520
5521
5522#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
5523/**
5524 * Executes VMWRITE.
5525 *
5526 * @returns VBox status code
5527 * @param pVCpu Pointer to the VMCPU.
5528 * @param idxField VMCS field index.
5529 * @param u64Val 16, 32 or 64 bits value.
5530 */
5531VMMR0DECL(int) VMXWriteVMCS64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5532{
5533 int rc;
5534 switch (idxField)
5535 {
5536 case VMX_VMCS_CTRL_TSC_OFFSET_FULL:
5537 case VMX_VMCS_CTRL_IO_BITMAP_A_FULL:
5538 case VMX_VMCS_CTRL_IO_BITMAP_B_FULL:
5539 case VMX_VMCS_CTRL_MSR_BITMAP_FULL:
5540 case VMX_VMCS_CTRL_VMEXIT_MSR_STORE_FULL:
5541 case VMX_VMCS_CTRL_VMEXIT_MSR_LOAD_FULL:
5542 case VMX_VMCS_CTRL_VMENTRY_MSR_LOAD_FULL:
5543 case VMX_VMCS_CTRL_VAPIC_PAGEADDR_FULL:
5544 case VMX_VMCS_CTRL_APIC_ACCESSADDR_FULL:
5545 case VMX_VMCS_GUEST_LINK_PTR_FULL:
5546 case VMX_VMCS_GUEST_PDPTR0_FULL:
5547 case VMX_VMCS_GUEST_PDPTR1_FULL:
5548 case VMX_VMCS_GUEST_PDPTR2_FULL:
5549 case VMX_VMCS_GUEST_PDPTR3_FULL:
5550 case VMX_VMCS_GUEST_DEBUGCTL_FULL:
5551 case VMX_VMCS_GUEST_EFER_FULL:
5552 case VMX_VMCS_CTRL_EPTP_FULL:
5553 /* These fields consist of two parts, which are both writable in 32 bits mode. */
5554 rc = VMXWriteVMCS32(idxField, u64Val);
5555 rc |= VMXWriteVMCS32(idxField + 1, (uint32_t)(u64Val >> 32ULL));
5556 AssertRC(rc);
5557 return rc;
5558
5559 case VMX_VMCS64_GUEST_LDTR_BASE:
5560 case VMX_VMCS64_GUEST_TR_BASE:
5561 case VMX_VMCS64_GUEST_GDTR_BASE:
5562 case VMX_VMCS64_GUEST_IDTR_BASE:
5563 case VMX_VMCS64_GUEST_SYSENTER_EIP:
5564 case VMX_VMCS64_GUEST_SYSENTER_ESP:
5565 case VMX_VMCS64_GUEST_CR0:
5566 case VMX_VMCS64_GUEST_CR4:
5567 case VMX_VMCS64_GUEST_CR3:
5568 case VMX_VMCS64_GUEST_DR7:
5569 case VMX_VMCS64_GUEST_RIP:
5570 case VMX_VMCS64_GUEST_RSP:
5571 case VMX_VMCS64_GUEST_CS_BASE:
5572 case VMX_VMCS64_GUEST_DS_BASE:
5573 case VMX_VMCS64_GUEST_ES_BASE:
5574 case VMX_VMCS64_GUEST_FS_BASE:
5575 case VMX_VMCS64_GUEST_GS_BASE:
5576 case VMX_VMCS64_GUEST_SS_BASE:
5577 /* Queue a 64 bits value as we can't set it in 32 bits host mode. */
5578 if (u64Val >> 32ULL)
5579 rc = VMXWriteCachedVMCSEx(pVCpu, idxField, u64Val);
5580 else
5581 rc = VMXWriteVMCS32(idxField, (uint32_t)u64Val);
5582
5583 return rc;
5584
5585 default:
5586 AssertMsgFailed(("Unexpected field %x\n", idxField));
5587 return VERR_INVALID_PARAMETER;
5588 }
5589}
5590
5591
5592/**
5593 * Cache VMCS writes for performance reasons (Darwin) and for running 64 bits guests on 32 bits hosts.
5594 *
5595 * @param pVCpu Pointer to the VMCPU.
5596 * @param idxField VMCS field index.
5597 * @param u64Val 16, 32 or 64 bits value.
5598 */
5599VMMR0DECL(int) VMXWriteCachedVMCSEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5600{
5601 PVMCSCACHE pCache = &pVCpu->hwaccm.s.vmx.VMCSCache;
5602
5603 AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1,
5604 ("entries=%x\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
5605
5606 /* Make sure there are no duplicates. */
5607 for (unsigned i = 0; i < pCache->Write.cValidEntries; i++)
5608 {
5609 if (pCache->Write.aField[i] == idxField)
5610 {
5611 pCache->Write.aFieldVal[i] = u64Val;
5612 return VINF_SUCCESS;
5613 }
5614 }
5615
5616 pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
5617 pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
5618 pCache->Write.cValidEntries++;
5619 return VINF_SUCCESS;
5620}
5621
5622#endif /* HC_ARCH_BITS == 32 && !VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0 */
5623
Note: See TracBrowser for help on using the repository browser.

© 2025 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette