VirtualBox

source: vbox/trunk/src/VBox/VMM/VMMR0/HWVMXR0.cpp@ 30390

Last change on this file since 30390 was 30390, checked in by vboxsync, 14 years ago

VT-x: when the guest's FPU state is active, then we no longer care about the FPU related bits. Saves a lot of world switches
  • Property svn:eol-style set to native
  • Property svn:keywords set to Id
File size: 194.4 KB
Line 
1/* $Id: HWVMXR0.cpp 30390 2010-06-23 12:49:37Z vboxsync $ */
2/** @file
3 * HWACCM VMX - Host Context Ring 0.
4 */
5
6/*
7 * Copyright (C) 2006-2007 Oracle Corporation
8 *
9 * This file is part of VirtualBox Open Source Edition (OSE), as
10 * available from http://www.virtualbox.org. This file is free software;
11 * you can redistribute it and/or modify it under the terms of the GNU
12 * General Public License (GPL) as published by the Free Software
13 * Foundation, in version 2 as it comes in the "COPYING" file of the
14 * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15 * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16 */
17
18
19/*******************************************************************************
20* Header Files *
21*******************************************************************************/
22#define LOG_GROUP LOG_GROUP_HWACCM
23#include <VBox/hwaccm.h>
24#include <VBox/pgm.h>
25#include <VBox/dbgf.h>
26#include <VBox/selm.h>
27#include <VBox/iom.h>
28#include <VBox/rem.h>
29#include <VBox/tm.h>
30#include "HWACCMInternal.h"
31#include <VBox/vm.h>
32#include <VBox/x86.h>
33#include <VBox/pdmapi.h>
34#include <VBox/err.h>
35#include <VBox/log.h>
36#include <iprt/asm-amd64-x86.h>
37#include <iprt/assert.h>
38#include <iprt/param.h>
39#include <iprt/string.h>
40#include <iprt/time.h>
41#ifdef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
42# include <iprt/thread.h>
43#endif
44#include "HWVMXR0.h"
45
46/*******************************************************************************
47* Defined Constants And Macros *
48*******************************************************************************/
49#if defined(RT_ARCH_AMD64)
50# define VMX_IS_64BIT_HOST_MODE() (true)
51#elif defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
52# define VMX_IS_64BIT_HOST_MODE() (g_fVMXIs64bitHost != 0)
53#else
54# define VMX_IS_64BIT_HOST_MODE() (false)
55#endif
56
57/*******************************************************************************
58* Global Variables *
59*******************************************************************************/
60/* IO operation lookup arrays. */
61static uint32_t const g_aIOSize[4] = {1, 2, 0, 4};
62static uint32_t const g_aIOOpAnd[4] = {0xff, 0xffff, 0, 0xffffffff};
63
64#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
65/** See HWACCMR0A.asm. */
66extern "C" uint32_t g_fVMXIs64bitHost;
67#endif
68
69/*******************************************************************************
70* Local Functions *
71*******************************************************************************/
72static void VMXR0ReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, int rc, PCPUMCTX pCtx);
73static void vmxR0SetupTLBEPT(PVM pVM, PVMCPU pVCpu);
74static void vmxR0SetupTLBVPID(PVM pVM, PVMCPU pVCpu);
75static void vmxR0SetupTLBDummy(PVM pVM, PVMCPU pVCpu);
76static void vmxR0FlushEPT(PVM pVM, PVMCPU pVCpu, VMX_FLUSH enmFlush, RTGCPHYS GCPhys);
77static void vmxR0FlushVPID(PVM pVM, PVMCPU pVCpu, VMX_FLUSH enmFlush, RTGCPTR GCPtr);
78static void vmxR0UpdateExceptionBitmap(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx);
79#ifdef VBOX_STRICT
80static bool vmxR0IsValidReadField(uint32_t idxField);
81static bool vmxR0IsValidWriteField(uint32_t idxField);
82#endif
83static void vmxR0SetMSRPermission(PVMCPU pVCpu, unsigned ulMSR, bool fRead, bool fWrite);
84
85static void VMXR0CheckError(PVM pVM, PVMCPU pVCpu, int rc)
86{
87 if (rc == VERR_VMX_GENERIC)
88 {
89 RTCCUINTREG instrError;
90
91 VMXReadVMCS(VMX_VMCS32_RO_VM_INSTR_ERROR, &instrError);
92 pVCpu->hwaccm.s.vmx.lasterror.ulInstrError = instrError;
93 }
94 pVM->hwaccm.s.lLastError = rc;
95}
96
97/**
98 * Sets up and activates VT-x on the current CPU
99 *
100 * @returns VBox status code.
101 * @param pCpu CPU info struct
102 * @param pVM The VM to operate on. (can be NULL after a resume!!)
103 * @param pvPageCpu Pointer to the global cpu page
104 * @param pPageCpuPhys Physical address of the global cpu page
105 */
106VMMR0DECL(int) VMXR0EnableCpu(PHWACCM_CPUINFO pCpu, PVM pVM, void *pvPageCpu, RTHCPHYS pPageCpuPhys)
107{
108 AssertReturn(pPageCpuPhys, VERR_INVALID_PARAMETER);
109 AssertReturn(pvPageCpu, VERR_INVALID_PARAMETER);
110
111 if (pVM)
112 {
113 /* Set revision dword at the beginning of the VMXON structure. */
114 *(uint32_t *)pvPageCpu = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hwaccm.s.vmx.msr.vmx_basic_info);
115 }
116
117 /** @todo we should unmap the two pages from the virtual address space in order to prevent accidental corruption.
118 * (which can have very bad consequences!!!)
119 */
120
121 if (ASMGetCR4() & X86_CR4_VMXE)
122 return VERR_VMX_IN_VMX_ROOT_MODE;
123
124 /* Make sure the VMX instructions don't cause #UD faults. */
125 ASMSetCR4(ASMGetCR4() | X86_CR4_VMXE);
126
127 /* Enter VMX Root Mode */
128 int rc = VMXEnable(pPageCpuPhys);
129 if (RT_FAILURE(rc))
130 {
131 ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
132 return VERR_VMX_VMXON_FAILED;
133 }
134 return VINF_SUCCESS;
135}
136
137/**
138 * Deactivates VT-x on the current CPU
139 *
140 * @returns VBox status code.
141 * @param pCpu CPU info struct
142 * @param pvPageCpu Pointer to the global cpu page
143 * @param pPageCpuPhys Physical address of the global cpu page
144 */
145VMMR0DECL(int) VMXR0DisableCpu(PHWACCM_CPUINFO pCpu, void *pvPageCpu, RTHCPHYS pPageCpuPhys)
146{
147 AssertReturn(pPageCpuPhys, VERR_INVALID_PARAMETER);
148 AssertReturn(pvPageCpu, VERR_INVALID_PARAMETER);
149
150 /* If we're somehow not in VMX root mode, then we shouldn't dare leaving it. */
151 if (!(ASMGetCR4() & X86_CR4_VMXE))
152 return VERR_VMX_NOT_IN_VMX_ROOT_MODE;
153
154 /* Leave VMX Root Mode. */
155 VMXDisable();
156
157 /* And clear the X86_CR4_VMXE bit */
158 ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
159 return VINF_SUCCESS;
160}
161
162/**
163 * Does Ring-0 per VM VT-x init.
164 *
165 * @returns VBox status code.
166 * @param pVM The VM to operate on.
167 */
168VMMR0DECL(int) VMXR0InitVM(PVM pVM)
169{
170 int rc;
171
172#ifdef LOG_ENABLED
173 SUPR0Printf("VMXR0InitVM %x\n", pVM);
174#endif
175
176 pVM->hwaccm.s.vmx.pMemObjAPIC = NIL_RTR0MEMOBJ;
177
178 if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
179 {
180 /* Allocate one page for the APIC physical page (serves for filtering accesses). */
181 rc = RTR0MemObjAllocCont(&pVM->hwaccm.s.vmx.pMemObjAPIC, 1 << PAGE_SHIFT, true /* executable R0 mapping */);
182 AssertRC(rc);
183 if (RT_FAILURE(rc))
184 return rc;
185
186 pVM->hwaccm.s.vmx.pAPIC = (uint8_t *)RTR0MemObjAddress(pVM->hwaccm.s.vmx.pMemObjAPIC);
187 pVM->hwaccm.s.vmx.pAPICPhys = RTR0MemObjGetPagePhysAddr(pVM->hwaccm.s.vmx.pMemObjAPIC, 0);
188 ASMMemZero32(pVM->hwaccm.s.vmx.pAPIC, PAGE_SIZE);
189 }
190 else
191 {
192 pVM->hwaccm.s.vmx.pMemObjAPIC = 0;
193 pVM->hwaccm.s.vmx.pAPIC = 0;
194 pVM->hwaccm.s.vmx.pAPICPhys = 0;
195 }
196
197#ifdef VBOX_WITH_CRASHDUMP_MAGIC
198 {
199 rc = RTR0MemObjAllocCont(&pVM->hwaccm.s.vmx.pMemObjScratch, 1 << PAGE_SHIFT, true /* executable R0 mapping */);
200 AssertRC(rc);
201 if (RT_FAILURE(rc))
202 return rc;
203
204 pVM->hwaccm.s.vmx.pScratch = (uint8_t *)RTR0MemObjAddress(pVM->hwaccm.s.vmx.pMemObjScratch);
205 pVM->hwaccm.s.vmx.pScratchPhys = RTR0MemObjGetPagePhysAddr(pVM->hwaccm.s.vmx.pMemObjScratch, 0);
206
207 ASMMemZero32(pVM->hwaccm.s.vmx.pScratch, PAGE_SIZE);
208 strcpy((char *)pVM->hwaccm.s.vmx.pScratch, "SCRATCH Magic");
209 *(uint64_t *)(pVM->hwaccm.s.vmx.pScratch + 16) = UINT64_C(0xDEADBEEFDEADBEEF);
210 }
211#endif
212
213 /* Allocate VMCBs for all guest CPUs. */
214 for (VMCPUID i = 0; i < pVM->cCpus; i++)
215 {
216 PVMCPU pVCpu = &pVM->aCpus[i];
217
218 pVCpu->hwaccm.s.vmx.pMemObjVMCS = NIL_RTR0MEMOBJ;
219
220 /* Allocate one page for the VM control structure (VMCS). */
221 rc = RTR0MemObjAllocCont(&pVCpu->hwaccm.s.vmx.pMemObjVMCS, 1 << PAGE_SHIFT, true /* executable R0 mapping */);
222 AssertRC(rc);
223 if (RT_FAILURE(rc))
224 return rc;
225
226 pVCpu->hwaccm.s.vmx.pVMCS = RTR0MemObjAddress(pVCpu->hwaccm.s.vmx.pMemObjVMCS);
227 pVCpu->hwaccm.s.vmx.pVMCSPhys = RTR0MemObjGetPagePhysAddr(pVCpu->hwaccm.s.vmx.pMemObjVMCS, 0);
228 ASMMemZero32(pVCpu->hwaccm.s.vmx.pVMCS, PAGE_SIZE);
229
230 pVCpu->hwaccm.s.vmx.cr0_mask = 0;
231 pVCpu->hwaccm.s.vmx.cr4_mask = 0;
232
233 /* Allocate one page for the virtual APIC page for TPR caching. */
234 rc = RTR0MemObjAllocCont(&pVCpu->hwaccm.s.vmx.pMemObjVAPIC, 1 << PAGE_SHIFT, true /* executable R0 mapping */);
235 AssertRC(rc);
236 if (RT_FAILURE(rc))
237 return rc;
238
239 pVCpu->hwaccm.s.vmx.pVAPIC = (uint8_t *)RTR0MemObjAddress(pVCpu->hwaccm.s.vmx.pMemObjVAPIC);
240 pVCpu->hwaccm.s.vmx.pVAPICPhys = RTR0MemObjGetPagePhysAddr(pVCpu->hwaccm.s.vmx.pMemObjVAPIC, 0);
241 ASMMemZero32(pVCpu->hwaccm.s.vmx.pVAPIC, PAGE_SIZE);
242
243 /* Allocate the MSR bitmap if this feature is supported. */
244 if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
245 {
246 rc = RTR0MemObjAllocCont(&pVCpu->hwaccm.s.vmx.pMemObjMSRBitmap, 1 << PAGE_SHIFT, true /* executable R0 mapping */);
247 AssertRC(rc);
248 if (RT_FAILURE(rc))
249 return rc;
250
251 pVCpu->hwaccm.s.vmx.pMSRBitmap = (uint8_t *)RTR0MemObjAddress(pVCpu->hwaccm.s.vmx.pMemObjMSRBitmap);
252 pVCpu->hwaccm.s.vmx.pMSRBitmapPhys = RTR0MemObjGetPagePhysAddr(pVCpu->hwaccm.s.vmx.pMemObjMSRBitmap, 0);
253 memset(pVCpu->hwaccm.s.vmx.pMSRBitmap, 0xff, PAGE_SIZE);
254 }
255
256#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
257 /* Allocate one page for the guest MSR load area (for preloading guest MSRs during the world switch). */
258 rc = RTR0MemObjAllocCont(&pVCpu->hwaccm.s.vmx.pMemObjGuestMSR, 1 << PAGE_SHIFT, true /* executable R0 mapping */);
259 AssertRC(rc);
260 if (RT_FAILURE(rc))
261 return rc;
262
263 pVCpu->hwaccm.s.vmx.pGuestMSR = (uint8_t *)RTR0MemObjAddress(pVCpu->hwaccm.s.vmx.pMemObjGuestMSR);
264 pVCpu->hwaccm.s.vmx.pGuestMSRPhys = RTR0MemObjGetPagePhysAddr(pVCpu->hwaccm.s.vmx.pMemObjGuestMSR, 0);
265 memset(pVCpu->hwaccm.s.vmx.pGuestMSR, 0, PAGE_SIZE);
266
267 /* Allocate one page for the host MSR load area (for restoring host MSRs after the world switch back). */
268 rc = RTR0MemObjAllocCont(&pVCpu->hwaccm.s.vmx.pMemObjHostMSR, 1 << PAGE_SHIFT, true /* executable R0 mapping */);
269 AssertRC(rc);
270 if (RT_FAILURE(rc))
271 return rc;
272
273 pVCpu->hwaccm.s.vmx.pHostMSR = (uint8_t *)RTR0MemObjAddress(pVCpu->hwaccm.s.vmx.pMemObjHostMSR);
274 pVCpu->hwaccm.s.vmx.pHostMSRPhys = RTR0MemObjGetPagePhysAddr(pVCpu->hwaccm.s.vmx.pMemObjHostMSR, 0);
275 memset(pVCpu->hwaccm.s.vmx.pHostMSR, 0, PAGE_SIZE);
276#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
277
278 /* Current guest paging mode. */
279 pVCpu->hwaccm.s.vmx.enmLastSeenGuestMode = PGMMODE_REAL;
280
281#ifdef LOG_ENABLED
282 SUPR0Printf("VMXR0InitVM %x VMCS=%x (%x)\n", pVM, pVCpu->hwaccm.s.vmx.pVMCS, (uint32_t)pVCpu->hwaccm.s.vmx.pVMCSPhys);
283#endif
284 }
285
286 return VINF_SUCCESS;
287}
288
289/**
290 * Does Ring-0 per VM VT-x termination.
291 *
292 * @returns VBox status code.
293 * @param pVM The VM to operate on.
294 */
295VMMR0DECL(int) VMXR0TermVM(PVM pVM)
296{
297 for (VMCPUID i = 0; i < pVM->cCpus; i++)
298 {
299 PVMCPU pVCpu = &pVM->aCpus[i];
300
301 if (pVCpu->hwaccm.s.vmx.pMemObjVMCS != NIL_RTR0MEMOBJ)
302 {
303 RTR0MemObjFree(pVCpu->hwaccm.s.vmx.pMemObjVMCS, false);
304 pVCpu->hwaccm.s.vmx.pMemObjVMCS = NIL_RTR0MEMOBJ;
305 pVCpu->hwaccm.s.vmx.pVMCS = 0;
306 pVCpu->hwaccm.s.vmx.pVMCSPhys = 0;
307 }
308 if (pVCpu->hwaccm.s.vmx.pMemObjVAPIC != NIL_RTR0MEMOBJ)
309 {
310 RTR0MemObjFree(pVCpu->hwaccm.s.vmx.pMemObjVAPIC, false);
311 pVCpu->hwaccm.s.vmx.pMemObjVAPIC = NIL_RTR0MEMOBJ;
312 pVCpu->hwaccm.s.vmx.pVAPIC = 0;
313 pVCpu->hwaccm.s.vmx.pVAPICPhys = 0;
314 }
315 if (pVCpu->hwaccm.s.vmx.pMemObjMSRBitmap != NIL_RTR0MEMOBJ)
316 {
317 RTR0MemObjFree(pVCpu->hwaccm.s.vmx.pMemObjMSRBitmap, false);
318 pVCpu->hwaccm.s.vmx.pMemObjMSRBitmap = NIL_RTR0MEMOBJ;
319 pVCpu->hwaccm.s.vmx.pMSRBitmap = 0;
320 pVCpu->hwaccm.s.vmx.pMSRBitmapPhys = 0;
321 }
322#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
323 if (pVCpu->hwaccm.s.vmx.pMemObjHostMSR != NIL_RTR0MEMOBJ)
324 {
325 RTR0MemObjFree(pVCpu->hwaccm.s.vmx.pMemObjHostMSR, false);
326 pVCpu->hwaccm.s.vmx.pMemObjHostMSR = NIL_RTR0MEMOBJ;
327 pVCpu->hwaccm.s.vmx.pHostMSR = 0;
328 pVCpu->hwaccm.s.vmx.pHostMSRPhys = 0;
329 }
330 if (pVCpu->hwaccm.s.vmx.pMemObjGuestMSR != NIL_RTR0MEMOBJ)
331 {
332 RTR0MemObjFree(pVCpu->hwaccm.s.vmx.pMemObjGuestMSR, false);
333 pVCpu->hwaccm.s.vmx.pMemObjGuestMSR = NIL_RTR0MEMOBJ;
334 pVCpu->hwaccm.s.vmx.pGuestMSR = 0;
335 pVCpu->hwaccm.s.vmx.pGuestMSRPhys = 0;
336 }
337#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
338 }
339 if (pVM->hwaccm.s.vmx.pMemObjAPIC != NIL_RTR0MEMOBJ)
340 {
341 RTR0MemObjFree(pVM->hwaccm.s.vmx.pMemObjAPIC, false);
342 pVM->hwaccm.s.vmx.pMemObjAPIC = NIL_RTR0MEMOBJ;
343 pVM->hwaccm.s.vmx.pAPIC = 0;
344 pVM->hwaccm.s.vmx.pAPICPhys = 0;
345 }
346#ifdef VBOX_WITH_CRASHDUMP_MAGIC
347 if (pVM->hwaccm.s.vmx.pMemObjScratch != NIL_RTR0MEMOBJ)
348 {
349 ASMMemZero32(pVM->hwaccm.s.vmx.pScratch, PAGE_SIZE);
350 RTR0MemObjFree(pVM->hwaccm.s.vmx.pMemObjScratch, false);
351 pVM->hwaccm.s.vmx.pMemObjScratch = NIL_RTR0MEMOBJ;
352 pVM->hwaccm.s.vmx.pScratch = 0;
353 pVM->hwaccm.s.vmx.pScratchPhys = 0;
354 }
355#endif
356 return VINF_SUCCESS;
357}
358
359/**
360 * Sets up VT-x for the specified VM
361 *
362 * @returns VBox status code.
363 * @param pVM The VM to operate on.
364 */
365VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
366{
367 int rc = VINF_SUCCESS;
368 uint32_t val;
369
370 AssertReturn(pVM, VERR_INVALID_PARAMETER);
371
372 for (VMCPUID i = 0; i < pVM->cCpus; i++)
373 {
374 PVMCPU pVCpu = &pVM->aCpus[i];
375
376 Assert(pVCpu->hwaccm.s.vmx.pVMCS);
377
378 /* Set revision dword at the beginning of the VMCS structure. */
379 *(uint32_t *)pVCpu->hwaccm.s.vmx.pVMCS = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hwaccm.s.vmx.msr.vmx_basic_info);
380
381 /* Clear VM Control Structure. */
382 Log(("pVMCSPhys = %RHp\n", pVCpu->hwaccm.s.vmx.pVMCSPhys));
383 rc = VMXClearVMCS(pVCpu->hwaccm.s.vmx.pVMCSPhys);
384 if (RT_FAILURE(rc))
385 goto vmx_end;
386
387 /* Activate the VM Control Structure. */
388 rc = VMXActivateVMCS(pVCpu->hwaccm.s.vmx.pVMCSPhys);
389 if (RT_FAILURE(rc))
390 goto vmx_end;
391
392 /* VMX_VMCS_CTRL_PIN_EXEC_CONTROLS
393 * Set required bits to one and zero according to the MSR capabilities.
394 */
395 val = pVM->hwaccm.s.vmx.msr.vmx_pin_ctls.n.disallowed0;
396 /* External and non-maskable interrupts cause VM-exits. */
397 val = val | VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_EXT_INT_EXIT | VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_NMI_EXIT;
398 val &= pVM->hwaccm.s.vmx.msr.vmx_pin_ctls.n.allowed1;
399
400 rc = VMXWriteVMCS(VMX_VMCS_CTRL_PIN_EXEC_CONTROLS, val);
401 AssertRC(rc);
402
403 /* VMX_VMCS_CTRL_PROC_EXEC_CONTROLS
404 * Set required bits to one and zero according to the MSR capabilities.
405 */
406 val = pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.disallowed0;
407 /* Program which event cause VM-exits and which features we want to use. */
408 val = val | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_HLT_EXIT
409 | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_TSC_OFFSET
410 | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT
411 | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_UNCOND_IO_EXIT
412 | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDPMC_EXIT
413 | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_EXIT
414 | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MWAIT_EXIT; /* don't execute mwait or else we'll idle inside the guest (host thinks the cpu load is high) */
415
416 /* Without nested paging we should intercept invlpg and cr3 mov instructions. */
417 if (!pVM->hwaccm.s.fNestedPaging)
418 val |= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INVLPG_EXIT
419 | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
420 | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT;
421
422 /* Note: VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MWAIT_EXIT might cause a vmlaunch failure with an invalid control fields error. (combined with some other exit reasons) */
423 if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
424 {
425 /* CR8 reads from the APIC shadow page; writes cause an exit is they lower the TPR below the threshold */
426 val |= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW;
427 Assert(pVM->hwaccm.s.vmx.pAPIC);
428 }
429 else
430 /* Exit on CR8 reads & writes in case the TPR shadow feature isn't present. */
431 val |= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR8_STORE_EXIT | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR8_LOAD_EXIT;
432
433 if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
434 {
435 Assert(pVCpu->hwaccm.s.vmx.pMSRBitmapPhys);
436 val |= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS;
437 }
438
439 /* We will use the secondary control if it's present. */
440 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
441
442 /* Mask away the bits that the CPU doesn't support */
443 /** @todo make sure they don't conflict with the above requirements. */
444 val &= pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1;
445 pVCpu->hwaccm.s.vmx.proc_ctls = val;
446
447 rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, val);
448 AssertRC(rc);
449
450 if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
451 {
452 /* VMX_VMCS_CTRL_PROC_EXEC_CONTROLS2
453 * Set required bits to one and zero according to the MSR capabilities.
454 */
455 val = pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2.n.disallowed0;
456 val |= VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT;
457
458#ifdef HWACCM_VTX_WITH_EPT
459 if (pVM->hwaccm.s.fNestedPaging)
460 val |= VMX_VMCS_CTRL_PROC_EXEC2_EPT;
461#endif /* HWACCM_VTX_WITH_EPT */
462#ifdef HWACCM_VTX_WITH_VPID
463 else
464 if (pVM->hwaccm.s.vmx.fVPID)
465 val |= VMX_VMCS_CTRL_PROC_EXEC2_VPID;
466#endif /* HWACCM_VTX_WITH_VPID */
467
468 if (pVM->hwaccm.s.fHasIoApic)
469 val |= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC;
470
471 if (pVM->hwaccm.s.vmx.fUnrestrictedGuest)
472 val |= VMX_VMCS_CTRL_PROC_EXEC2_REAL_MODE;
473
474 /* Mask away the bits that the CPU doesn't support */
475 /** @todo make sure they don't conflict with the above requirements. */
476 val &= pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2.n.allowed1;
477 pVCpu->hwaccm.s.vmx.proc_ctls2 = val;
478 rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS2, val);
479 AssertRC(rc);
480 }
481
482 /* VMX_VMCS_CTRL_CR3_TARGET_COUNT
483 * Set required bits to one and zero according to the MSR capabilities.
484 */
485 rc = VMXWriteVMCS(VMX_VMCS_CTRL_CR3_TARGET_COUNT, 0);
486 AssertRC(rc);
487
488 /* Forward all exception except #NM & #PF to the guest.
489 * We always need to check pagefaults since our shadow page table can be out of sync.
490 * And we always lazily sync the FPU & XMM state.
491 */
492
493 /** @todo Possible optimization:
494 * Keep the FPU and XMM state current in the EM thread. That way there's no need to
495 * lazily sync anything, but the downside is that we can't use the FPU stack or XMM
496 * registers ourselves of course.
497 *
498 * Note: only possible if the current state is actually ours (X86_CR0_TS flag)
499 */
500
501 /* Don't filter page faults; all of them should cause a switch. */
502 rc = VMXWriteVMCS(VMX_VMCS_CTRL_PAGEFAULT_ERROR_MASK, 0);
503 rc |= VMXWriteVMCS(VMX_VMCS_CTRL_PAGEFAULT_ERROR_MATCH, 0);
504 AssertRC(rc);
505
506 /* Init TSC offset to zero. */
507 rc = VMXWriteVMCS64(VMX_VMCS_CTRL_TSC_OFFSET_FULL, 0);
508 AssertRC(rc);
509
510 rc = VMXWriteVMCS64(VMX_VMCS_CTRL_IO_BITMAP_A_FULL, 0);
511 AssertRC(rc);
512
513 rc = VMXWriteVMCS64(VMX_VMCS_CTRL_IO_BITMAP_B_FULL, 0);
514 AssertRC(rc);
515
516 /* Set the MSR bitmap address. */
517 if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
518 {
519 Assert(pVCpu->hwaccm.s.vmx.pMSRBitmapPhys);
520
521 rc = VMXWriteVMCS64(VMX_VMCS_CTRL_MSR_BITMAP_FULL, pVCpu->hwaccm.s.vmx.pMSRBitmapPhys);
522 AssertRC(rc);
523
524 /* Allow the guest to directly modify these MSRs; they are restored and saved automatically. */
525 vmxR0SetMSRPermission(pVCpu, MSR_IA32_SYSENTER_CS, true, true);
526 vmxR0SetMSRPermission(pVCpu, MSR_IA32_SYSENTER_ESP, true, true);
527 vmxR0SetMSRPermission(pVCpu, MSR_IA32_SYSENTER_EIP, true, true);
528 vmxR0SetMSRPermission(pVCpu, MSR_K8_LSTAR, true, true);
529 vmxR0SetMSRPermission(pVCpu, MSR_K6_STAR, true, true);
530 vmxR0SetMSRPermission(pVCpu, MSR_K8_SF_MASK, true, true);
531 vmxR0SetMSRPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, true, true);
532 vmxR0SetMSRPermission(pVCpu, MSR_K8_GS_BASE, true, true);
533 vmxR0SetMSRPermission(pVCpu, MSR_K8_FS_BASE, true, true);
534 }
535
536#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
537 /* Set the guest & host MSR load/store physical addresses. */
538 Assert(pVCpu->hwaccm.s.vmx.pGuestMSRPhys);
539 rc = VMXWriteVMCS64(VMX_VMCS_CTRL_VMENTRY_MSR_LOAD_FULL, pVCpu->hwaccm.s.vmx.pGuestMSRPhys);
540 AssertRC(rc);
541 rc = VMXWriteVMCS64(VMX_VMCS_CTRL_VMEXIT_MSR_STORE_FULL, pVCpu->hwaccm.s.vmx.pGuestMSRPhys);
542 AssertRC(rc);
543
544 Assert(pVCpu->hwaccm.s.vmx.pHostMSRPhys);
545 rc = VMXWriteVMCS64(VMX_VMCS_CTRL_VMEXIT_MSR_LOAD_FULL, pVCpu->hwaccm.s.vmx.pHostMSRPhys);
546 AssertRC(rc);
547#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
548
549 rc = VMXWriteVMCS(VMX_VMCS_CTRL_ENTRY_MSR_LOAD_COUNT, 0);
550 AssertRC(rc);
551
552 rc = VMXWriteVMCS(VMX_VMCS_CTRL_EXIT_MSR_STORE_COUNT, 0);
553 AssertRC(rc);
554
555 if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
556 {
557 Assert(pVM->hwaccm.s.vmx.pMemObjAPIC);
558 /* Optional */
559 rc = VMXWriteVMCS(VMX_VMCS_CTRL_TPR_THRESHOLD, 0);
560 rc |= VMXWriteVMCS64(VMX_VMCS_CTRL_VAPIC_PAGEADDR_FULL, pVCpu->hwaccm.s.vmx.pVAPICPhys);
561
562 if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
563 rc |= VMXWriteVMCS64(VMX_VMCS_CTRL_APIC_ACCESSADDR_FULL, pVM->hwaccm.s.vmx.pAPICPhys);
564
565 AssertRC(rc);
566 }
567
568 /* Set link pointer to -1. Not currently used. */
569 rc = VMXWriteVMCS64(VMX_VMCS_GUEST_LINK_PTR_FULL, 0xFFFFFFFFFFFFFFFFULL);
570 AssertRC(rc);
571
572 /* Clear VM Control Structure. Marking it inactive, clearing implementation specific data and writing back VMCS data to memory. */
573 rc = VMXClearVMCS(pVCpu->hwaccm.s.vmx.pVMCSPhys);
574 AssertRC(rc);
575
576 /* Configure the VMCS read cache. */
577 PVMCSCACHE pCache = &pVCpu->hwaccm.s.vmx.VMCSCache;
578
579 VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_RIP);
580 VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_RSP);
581 VMXSetupCachedReadVMCS(pCache, VMX_VMCS_GUEST_RFLAGS);
582 VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE);
583 VMXSetupCachedReadVMCS(pCache, VMX_VMCS_CTRL_CR0_READ_SHADOW);
584 VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_CR0);
585 VMXSetupCachedReadVMCS(pCache, VMX_VMCS_CTRL_CR4_READ_SHADOW);
586 VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_CR4);
587 VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_DR7);
588 VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_GUEST_SYSENTER_CS);
589 VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_SYSENTER_EIP);
590 VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_SYSENTER_ESP);
591 VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_GUEST_GDTR_LIMIT);
592 VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_GDTR_BASE);
593 VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_GUEST_IDTR_LIMIT);
594 VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_IDTR_BASE);
595
596 VMX_SETUP_SELREG(ES, pCache);
597 VMX_SETUP_SELREG(SS, pCache);
598 VMX_SETUP_SELREG(CS, pCache);
599 VMX_SETUP_SELREG(DS, pCache);
600 VMX_SETUP_SELREG(FS, pCache);
601 VMX_SETUP_SELREG(GS, pCache);
602 VMX_SETUP_SELREG(LDTR, pCache);
603 VMX_SETUP_SELREG(TR, pCache);
604
605 /* Status code VMCS reads. */
606 VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_EXIT_REASON);
607 VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_VM_INSTR_ERROR);
608 VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_EXIT_INSTR_LENGTH);
609 VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_EXIT_INTERRUPTION_ERRCODE);
610 VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO);
611 VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_EXIT_INSTR_INFO);
612 VMXSetupCachedReadVMCS(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
613 VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_IDT_INFO);
614 VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_IDT_ERRCODE);
615
616 if (pVM->hwaccm.s.fNestedPaging)
617 {
618 VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_CR3);
619 VMXSetupCachedReadVMCS(pCache, VMX_VMCS_EXIT_PHYS_ADDR_FULL);
620 pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
621 }
622 else
623 pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
624 } /* for each VMCPU */
625
626 /* Choose the right TLB setup function. */
627 if (pVM->hwaccm.s.fNestedPaging)
628 {
629 pVM->hwaccm.s.vmx.pfnSetupTaggedTLB = vmxR0SetupTLBEPT;
630
631 /* Default values for flushing. */
632 pVM->hwaccm.s.vmx.enmFlushPage = VMX_FLUSH_ALL_CONTEXTS;
633 pVM->hwaccm.s.vmx.enmFlushContext = VMX_FLUSH_ALL_CONTEXTS;
634
635 /* If the capabilities specify we can do more, then make use of it. */
636 if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVEPT_CAPS_INDIV)
637 pVM->hwaccm.s.vmx.enmFlushPage = VMX_FLUSH_PAGE;
638 else
639 if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVEPT_CAPS_CONTEXT)
640 pVM->hwaccm.s.vmx.enmFlushPage = VMX_FLUSH_SINGLE_CONTEXT;
641
642 if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVEPT_CAPS_CONTEXT)
643 pVM->hwaccm.s.vmx.enmFlushContext = VMX_FLUSH_SINGLE_CONTEXT;
644 }
645#ifdef HWACCM_VTX_WITH_VPID
646 else
647 if (pVM->hwaccm.s.vmx.fVPID)
648 {
649 pVM->hwaccm.s.vmx.pfnSetupTaggedTLB = vmxR0SetupTLBVPID;
650
651 /* Default values for flushing. */
652 pVM->hwaccm.s.vmx.enmFlushPage = VMX_FLUSH_ALL_CONTEXTS;
653 pVM->hwaccm.s.vmx.enmFlushContext = VMX_FLUSH_ALL_CONTEXTS;
654
655 /* If the capabilities specify we can do more, then make use of it. */
656 if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVVPID_CAPS_INDIV)
657 pVM->hwaccm.s.vmx.enmFlushPage = VMX_FLUSH_PAGE;
658 else
659 if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVVPID_CAPS_CONTEXT)
660 pVM->hwaccm.s.vmx.enmFlushPage = VMX_FLUSH_SINGLE_CONTEXT;
661
662 if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVVPID_CAPS_CONTEXT)
663 pVM->hwaccm.s.vmx.enmFlushContext = VMX_FLUSH_SINGLE_CONTEXT;
664 }
665#endif /* HWACCM_VTX_WITH_VPID */
666 else
667 pVM->hwaccm.s.vmx.pfnSetupTaggedTLB = vmxR0SetupTLBDummy;
668
669vmx_end:
670 VMXR0CheckError(pVM, &pVM->aCpus[0], rc);
671 return rc;
672}
673
674/**
675 * Sets the permission bits for the specified MSR
676 *
677 * @param pVCpu The VMCPU to operate on.
678 * @param ulMSR MSR value
679 * @param fRead Reading allowed/disallowed
680 * @param fWrite Writing allowed/disallowed
681 */
682static void vmxR0SetMSRPermission(PVMCPU pVCpu, unsigned ulMSR, bool fRead, bool fWrite)
683{
684 unsigned ulBit;
685 uint8_t *pMSRBitmap = (uint8_t *)pVCpu->hwaccm.s.vmx.pMSRBitmap;
686
687 /* Layout:
688 * 0x000 - 0x3ff - Low MSR read bits
689 * 0x400 - 0x7ff - High MSR read bits
690 * 0x800 - 0xbff - Low MSR write bits
691 * 0xc00 - 0xfff - High MSR write bits
692 */
693 if (ulMSR <= 0x00001FFF)
694 {
695 /* Pentium-compatible MSRs */
696 ulBit = ulMSR;
697 }
698 else
699 if ( ulMSR >= 0xC0000000
700 && ulMSR <= 0xC0001FFF)
701 {
702 /* AMD Sixth Generation x86 Processor MSRs */
703 ulBit = (ulMSR - 0xC0000000);
704 pMSRBitmap += 0x400;
705 }
706 else
707 {
708 AssertFailed();
709 return;
710 }
711
712 Assert(ulBit <= 0x1fff);
713 if (fRead)
714 ASMBitClear(pMSRBitmap, ulBit);
715 else
716 ASMBitSet(pMSRBitmap, ulBit);
717
718 if (fWrite)
719 ASMBitClear(pMSRBitmap + 0x800, ulBit);
720 else
721 ASMBitSet(pMSRBitmap + 0x800, ulBit);
722}
723
724
725/**
726 * Injects an event (trap or external interrupt)
727 *
728 * @returns VBox status code.
729 * @param pVM The VM to operate on.
730 * @param pVCpu The VMCPU to operate on.
731 * @param pCtx CPU Context
732 * @param intInfo VMX interrupt info
733 * @param cbInstr Opcode length of faulting instruction
734 * @param errCode Error code (optional)
735 */
736static int VMXR0InjectEvent(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, uint32_t intInfo, uint32_t cbInstr, uint32_t errCode)
737{
738 int rc;
739 uint32_t iGate = VMX_EXIT_INTERRUPTION_INFO_VECTOR(intInfo);
740
741#ifdef VBOX_WITH_STATISTICS
742 STAM_COUNTER_INC(&pVCpu->hwaccm.s.paStatInjectedIrqsR0[iGate & MASK_INJECT_IRQ_STAT]);
743#endif
744
745#ifdef VBOX_STRICT
746 if (iGate == 0xE)
747 LogFlow(("VMXR0InjectEvent: Injecting interrupt %d at %RGv error code=%08x CR2=%RGv intInfo=%08x\n", iGate, (RTGCPTR)pCtx->rip, errCode, pCtx->cr2, intInfo));
748 else
749 if (iGate < 0x20)
750 LogFlow(("VMXR0InjectEvent: Injecting interrupt %d at %RGv error code=%08x\n", iGate, (RTGCPTR)pCtx->rip, errCode));
751 else
752 {
753 LogFlow(("INJ-EI: %x at %RGv\n", iGate, (RTGCPTR)pCtx->rip));
754 Assert(VMX_EXIT_INTERRUPTION_INFO_TYPE(intInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW || !VMCPU_FF_ISSET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
755 Assert(VMX_EXIT_INTERRUPTION_INFO_TYPE(intInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW || pCtx->eflags.u32 & X86_EFL_IF);
756 }
757#endif
758
759 if ( CPUMIsGuestInRealModeEx(pCtx)
760 && pVM->hwaccm.s.vmx.pRealModeTSS)
761 {
762 RTGCPHYS GCPhysHandler;
763 uint16_t offset, ip;
764 RTSEL sel;
765
766 /* Injecting events doesn't work right with real mode emulation.
767 * (#GP if we try to inject external hardware interrupts)
768 * Inject the interrupt or trap directly instead.
769 *
770 * ASSUMES no access handlers for the bits we read or write below (should be safe).
771 */
772 Log(("Manual interrupt/trap '%x' inject (real mode)\n", iGate));
773
774 /* Check if the interrupt handler is present. */
775 if (iGate * 4 + 3 > pCtx->idtr.cbIdt)
776 {
777 Log(("IDT cbIdt violation\n"));
778 if (iGate != X86_XCPT_DF)
779 {
780 uint32_t intInfo2;
781
782 intInfo2 = (iGate == X86_XCPT_GP) ? (uint32_t)X86_XCPT_DF : iGate;
783 intInfo2 |= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
784 intInfo2 |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
785 intInfo2 |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HWEXCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
786
787 return VMXR0InjectEvent(pVM, pVCpu, pCtx, intInfo2, 0, 0 /* no error code according to the Intel docs */);
788 }
789 Log(("Triple fault -> reset the VM!\n"));
790 return VINF_EM_RESET;
791 }
792 if ( VMX_EXIT_INTERRUPTION_INFO_TYPE(intInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW
793 || iGate == 3 /* Both #BP and #OF point to the instruction after. */
794 || iGate == 4)
795 {
796 ip = pCtx->ip + cbInstr;
797 }
798 else
799 ip = pCtx->ip;
800
801 /* Read the selector:offset pair of the interrupt handler. */
802 GCPhysHandler = (RTGCPHYS)pCtx->idtr.pIdt + iGate * 4;
803 rc = PGMPhysSimpleReadGCPhys(pVM, &offset, GCPhysHandler, sizeof(offset)); AssertRC(rc);
804 rc = PGMPhysSimpleReadGCPhys(pVM, &sel, GCPhysHandler + 2, sizeof(sel)); AssertRC(rc);
805
806 LogFlow(("IDT handler %04X:%04X\n", sel, offset));
807
808 /* Construct the stack frame. */
809 /** @todo should check stack limit. */
810 pCtx->sp -= 2;
811 LogFlow(("ss:sp %04X:%04X eflags=%x\n", pCtx->ss, pCtx->sp, pCtx->eflags.u));
812 rc = PGMPhysSimpleWriteGCPhys(pVM, pCtx->ssHid.u64Base + pCtx->sp, &pCtx->eflags, sizeof(uint16_t)); AssertRC(rc);
813 pCtx->sp -= 2;
814 LogFlow(("ss:sp %04X:%04X cs=%x\n", pCtx->ss, pCtx->sp, pCtx->cs));
815 rc = PGMPhysSimpleWriteGCPhys(pVM, pCtx->ssHid.u64Base + pCtx->sp, &pCtx->cs, sizeof(uint16_t)); AssertRC(rc);
816 pCtx->sp -= 2;
817 LogFlow(("ss:sp %04X:%04X ip=%x\n", pCtx->ss, pCtx->sp, ip));
818 rc = PGMPhysSimpleWriteGCPhys(pVM, pCtx->ssHid.u64Base + pCtx->sp, &ip, sizeof(ip)); AssertRC(rc);
819
820 /* Update the CPU state for executing the handler. */
821 pCtx->rip = offset;
822 pCtx->cs = sel;
823 pCtx->csHid.u64Base = sel << 4;
824 pCtx->eflags.u &= ~(X86_EFL_IF|X86_EFL_TF|X86_EFL_RF|X86_EFL_AC);
825
826 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_GUEST_SEGMENT_REGS;
827 return VINF_SUCCESS;
828 }
829
830 /* Set event injection state. */
831 rc = VMXWriteVMCS(VMX_VMCS_CTRL_ENTRY_IRQ_INFO, intInfo | (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT));
832
833 rc |= VMXWriteVMCS(VMX_VMCS_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
834 rc |= VMXWriteVMCS(VMX_VMCS_CTRL_ENTRY_EXCEPTION_ERRCODE, errCode);
835
836 AssertRC(rc);
837 return rc;
838}
839
840
841/**
842 * Checks for pending guest interrupts and injects them
843 *
844 * @returns VBox status code.
845 * @param pVM The VM to operate on.
846 * @param pVCpu The VMCPU to operate on.
847 * @param pCtx CPU Context
848 */
849static int VMXR0CheckPendingInterrupt(PVM pVM, PVMCPU pVCpu, CPUMCTX *pCtx)
850{
851 int rc;
852
853 /* Dispatch any pending interrupts. (injected before, but a VM exit occurred prematurely) */
854 if (pVCpu->hwaccm.s.Event.fPending)
855 {
856 Log(("CPU%d: Reinjecting event %RX64 %08x at %RGv cr2=%RX64\n", pVCpu->idCpu, pVCpu->hwaccm.s.Event.intInfo, pVCpu->hwaccm.s.Event.errCode, (RTGCPTR)pCtx->rip, pCtx->cr2));
857 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatIntReinject);
858 rc = VMXR0InjectEvent(pVM, pVCpu, pCtx, pVCpu->hwaccm.s.Event.intInfo, 0, pVCpu->hwaccm.s.Event.errCode);
859 AssertRC(rc);
860
861 pVCpu->hwaccm.s.Event.fPending = false;
862 return VINF_SUCCESS;
863 }
864
865 /* If an active trap is already pending, then we must forward it first! */
866 if (!TRPMHasTrap(pVCpu))
867 {
868 if (VMCPU_FF_TESTANDCLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI))
869 {
870 RTGCUINTPTR intInfo;
871
872 Log(("CPU%d: injecting #NMI\n", pVCpu->idCpu));
873
874 intInfo = X86_XCPT_NMI;
875 intInfo |= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
876 intInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
877
878 rc = VMXR0InjectEvent(pVM, pVCpu, pCtx, intInfo, 0, 0);
879 AssertRC(rc);
880
881 return VINF_SUCCESS;
882 }
883
884 /* @todo SMI interrupts. */
885
886 /* When external interrupts are pending, we should exit the VM when IF is set. */
887 if (VMCPU_FF_ISPENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC|VMCPU_FF_INTERRUPT_PIC)))
888 {
889 if (!(pCtx->eflags.u32 & X86_EFL_IF))
890 {
891 if (!(pVCpu->hwaccm.s.vmx.proc_ctls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_IRQ_WINDOW_EXIT))
892 {
893 LogFlow(("Enable irq window exit!\n"));
894 pVCpu->hwaccm.s.vmx.proc_ctls |= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_IRQ_WINDOW_EXIT;
895 rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
896 AssertRC(rc);
897 }
898 /* else nothing to do but wait */
899 }
900 else
901 if (!VMCPU_FF_ISSET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
902 {
903 uint8_t u8Interrupt;
904
905 rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
906 Log(("CPU%d: Dispatch interrupt: u8Interrupt=%x (%d) rc=%Rrc cs:rip=%04X:%RGv\n", pVCpu->idCpu, u8Interrupt, u8Interrupt, rc, pCtx->cs, (RTGCPTR)pCtx->rip));
907 if (RT_SUCCESS(rc))
908 {
909 rc = TRPMAssertTrap(pVCpu, u8Interrupt, TRPM_HARDWARE_INT);
910 AssertRC(rc);
911 }
912 else
913 {
914 /* Can only happen in rare cases where a pending interrupt is cleared behind our back */
915 Assert(!VMCPU_FF_ISPENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC|VMCPU_FF_INTERRUPT_PIC)));
916 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatSwitchGuestIrq);
917 /* Just continue */
918 }
919 }
920 else
921 Log(("Pending interrupt blocked at %RGv by VM_FF_INHIBIT_INTERRUPTS!!\n", (RTGCPTR)pCtx->rip));
922 }
923 }
924
925#ifdef VBOX_STRICT
926 if (TRPMHasTrap(pVCpu))
927 {
928 uint8_t u8Vector;
929 rc = TRPMQueryTrapAll(pVCpu, &u8Vector, 0, 0, 0);
930 AssertRC(rc);
931 }
932#endif
933
934 if ( (pCtx->eflags.u32 & X86_EFL_IF)
935 && (!VMCPU_FF_ISSET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
936 && TRPMHasTrap(pVCpu)
937 )
938 {
939 uint8_t u8Vector;
940 TRPMEVENT enmType;
941 RTGCUINTPTR intInfo;
942 RTGCUINT errCode;
943
944 /* If a new event is pending, then dispatch it now. */
945 rc = TRPMQueryTrapAll(pVCpu, &u8Vector, &enmType, &errCode, 0);
946 AssertRC(rc);
947 Assert(pCtx->eflags.Bits.u1IF == 1 || enmType == TRPM_TRAP);
948 Assert(enmType != TRPM_SOFTWARE_INT);
949
950 /* Clear the pending trap. */
951 rc = TRPMResetTrap(pVCpu);
952 AssertRC(rc);
953
954 intInfo = u8Vector;
955 intInfo |= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
956
957 if (enmType == TRPM_TRAP)
958 {
959 switch (u8Vector) {
960 case 8:
961 case 10:
962 case 11:
963 case 12:
964 case 13:
965 case 14:
966 case 17:
967 /* Valid error codes. */
968 intInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
969 break;
970 default:
971 break;
972 }
973 if (u8Vector == X86_XCPT_BP || u8Vector == X86_XCPT_OF)
974 intInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SWEXCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
975 else
976 intInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HWEXCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
977 }
978 else
979 intInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
980
981 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatIntInject);
982 rc = VMXR0InjectEvent(pVM, pVCpu, pCtx, intInfo, 0, errCode);
983 AssertRC(rc);
984 } /* if (interrupts can be dispatched) */
985
986 return VINF_SUCCESS;
987}
988
989/**
990 * Save the host state
991 *
992 * @returns VBox status code.
993 * @param pVM The VM to operate on.
994 * @param pVCpu The VMCPU to operate on.
995 */
996VMMR0DECL(int) VMXR0SaveHostState(PVM pVM, PVMCPU pVCpu)
997{
998 int rc = VINF_SUCCESS;
999
1000 /*
1001 * Host CPU Context
1002 */
1003 if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_HOST_CONTEXT)
1004 {
1005 RTIDTR idtr;
1006 RTGDTR gdtr;
1007 RTSEL SelTR;
1008 PCX86DESCHC pDesc;
1009 uintptr_t trBase;
1010 RTSEL cs;
1011 RTSEL ss;
1012 uint64_t cr3;
1013
1014 /* Control registers */
1015 rc = VMXWriteVMCS(VMX_VMCS_HOST_CR0, ASMGetCR0());
1016#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
1017 if (VMX_IS_64BIT_HOST_MODE())
1018 {
1019 cr3 = hwaccmR0Get64bitCR3();
1020 rc |= VMXWriteVMCS64(VMX_VMCS_HOST_CR3, cr3);
1021 }
1022 else
1023#endif
1024 {
1025 cr3 = ASMGetCR3();
1026 rc |= VMXWriteVMCS(VMX_VMCS_HOST_CR3, cr3);
1027 }
1028 rc |= VMXWriteVMCS(VMX_VMCS_HOST_CR4, ASMGetCR4());
1029 AssertRC(rc);
1030 Log2(("VMX_VMCS_HOST_CR0 %08x\n", ASMGetCR0()));
1031 Log2(("VMX_VMCS_HOST_CR3 %08RX64\n", cr3));
1032 Log2(("VMX_VMCS_HOST_CR4 %08x\n", ASMGetCR4()));
1033
1034 /* Selector registers. */
1035#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
1036 if (VMX_IS_64BIT_HOST_MODE())
1037 {
1038 cs = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelCS;
1039 ss = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelSS;
1040 }
1041 else
1042 {
1043 /* sysenter loads LDT cs & ss, VMX doesn't like this. Load the GDT ones (safe). */
1044 cs = (RTSEL)(uintptr_t)&SUPR0AbsKernelCS;
1045 ss = (RTSEL)(uintptr_t)&SUPR0AbsKernelSS;
1046 }
1047#else
1048 cs = ASMGetCS();
1049 ss = ASMGetSS();
1050#endif
1051 Assert(!(cs & X86_SEL_LDT)); Assert((cs & X86_SEL_RPL) == 0);
1052 Assert(!(ss & X86_SEL_LDT)); Assert((ss & X86_SEL_RPL) == 0);
1053 rc = VMXWriteVMCS(VMX_VMCS16_HOST_FIELD_CS, cs);
1054 /* Note: VMX is (again) very picky about the RPL of the selectors here; we'll restore them manually. */
1055 rc |= VMXWriteVMCS(VMX_VMCS16_HOST_FIELD_DS, 0);
1056 rc |= VMXWriteVMCS(VMX_VMCS16_HOST_FIELD_ES, 0);
1057#if HC_ARCH_BITS == 32
1058 if (!VMX_IS_64BIT_HOST_MODE())
1059 {
1060 rc |= VMXWriteVMCS(VMX_VMCS16_HOST_FIELD_FS, 0);
1061 rc |= VMXWriteVMCS(VMX_VMCS16_HOST_FIELD_GS, 0);
1062 }
1063#endif
1064 rc |= VMXWriteVMCS(VMX_VMCS16_HOST_FIELD_SS, ss);
1065 SelTR = ASMGetTR();
1066 rc |= VMXWriteVMCS(VMX_VMCS16_HOST_FIELD_TR, SelTR);
1067 AssertRC(rc);
1068 Log2(("VMX_VMCS_HOST_FIELD_CS %08x (%08x)\n", cs, ASMGetSS()));
1069 Log2(("VMX_VMCS_HOST_FIELD_DS 00000000 (%08x)\n", ASMGetDS()));
1070 Log2(("VMX_VMCS_HOST_FIELD_ES 00000000 (%08x)\n", ASMGetES()));
1071 Log2(("VMX_VMCS_HOST_FIELD_FS 00000000 (%08x)\n", ASMGetFS()));
1072 Log2(("VMX_VMCS_HOST_FIELD_GS 00000000 (%08x)\n", ASMGetGS()));
1073 Log2(("VMX_VMCS_HOST_FIELD_SS %08x (%08x)\n", ss, ASMGetSS()));
1074 Log2(("VMX_VMCS_HOST_FIELD_TR %08x\n", ASMGetTR()));
1075
1076 /* GDTR & IDTR */
1077#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
1078 if (VMX_IS_64BIT_HOST_MODE())
1079 {
1080 X86XDTR64 gdtr64, idtr64;
1081 hwaccmR0Get64bitGDTRandIDTR(&gdtr64, &idtr64);
1082 rc = VMXWriteVMCS64(VMX_VMCS_HOST_GDTR_BASE, gdtr64.uAddr);
1083 rc |= VMXWriteVMCS64(VMX_VMCS_HOST_IDTR_BASE, gdtr64.uAddr);
1084 AssertRC(rc);
1085 Log2(("VMX_VMCS_HOST_GDTR_BASE %RX64\n", gdtr64.uAddr));
1086 Log2(("VMX_VMCS_HOST_IDTR_BASE %RX64\n", idtr64.uAddr));
1087 gdtr.cbGdt = gdtr64.cb;
1088 gdtr.pGdt = (uintptr_t)gdtr64.uAddr;
1089 }
1090 else
1091#endif
1092 {
1093 ASMGetGDTR(&gdtr);
1094 rc = VMXWriteVMCS(VMX_VMCS_HOST_GDTR_BASE, gdtr.pGdt);
1095 ASMGetIDTR(&idtr);
1096 rc |= VMXWriteVMCS(VMX_VMCS_HOST_IDTR_BASE, idtr.pIdt);
1097 AssertRC(rc);
1098 Log2(("VMX_VMCS_HOST_GDTR_BASE %RHv\n", gdtr.pGdt));
1099 Log2(("VMX_VMCS_HOST_IDTR_BASE %RHv\n", idtr.pIdt));
1100 }
1101
1102 /* Save the base address of the TR selector. */
1103 if (SelTR > gdtr.cbGdt)
1104 {
1105 AssertMsgFailed(("Invalid TR selector %x. GDTR.cbGdt=%x\n", SelTR, gdtr.cbGdt));
1106 return VERR_VMX_INVALID_HOST_STATE;
1107 }
1108
1109 pDesc = (PCX86DESCHC)(gdtr.pGdt + (SelTR & X86_SEL_MASK));
1110#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
1111 if (VMX_IS_64BIT_HOST_MODE())
1112 {
1113 uint64_t trBase64 = X86DESC64_BASE(*(PX86DESC64)pDesc);
1114 rc = VMXWriteVMCS64(VMX_VMCS_HOST_TR_BASE, trBase64);
1115 Log2(("VMX_VMCS_HOST_TR_BASE %RX64\n", trBase64));
1116 AssertRC(rc);
1117 }
1118 else
1119#endif
1120 {
1121#if HC_ARCH_BITS == 64
1122 trBase = X86DESC64_BASE(*pDesc);
1123#else
1124 trBase = X86DESC_BASE(*pDesc);
1125#endif
1126 rc = VMXWriteVMCS(VMX_VMCS_HOST_TR_BASE, trBase);
1127 AssertRC(rc);
1128 Log2(("VMX_VMCS_HOST_TR_BASE %RHv\n", trBase));
1129 }
1130
1131 /* FS and GS base. */
1132#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
1133 if (VMX_IS_64BIT_HOST_MODE())
1134 {
1135 Log2(("MSR_K8_FS_BASE = %RX64\n", ASMRdMsr(MSR_K8_FS_BASE)));
1136 Log2(("MSR_K8_GS_BASE = %RX64\n", ASMRdMsr(MSR_K8_GS_BASE)));
1137 rc = VMXWriteVMCS64(VMX_VMCS_HOST_FS_BASE, ASMRdMsr(MSR_K8_FS_BASE));
1138 rc |= VMXWriteVMCS64(VMX_VMCS_HOST_GS_BASE, ASMRdMsr(MSR_K8_GS_BASE));
1139 }
1140#endif
1141 AssertRC(rc);
1142
1143 /* Sysenter MSRs. */
1144 /** @todo expensive!! */
1145 rc = VMXWriteVMCS(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
1146 Log2(("VMX_VMCS_HOST_SYSENTER_CS %08x\n", ASMRdMsr_Low(MSR_IA32_SYSENTER_CS)));
1147#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
1148 if (VMX_IS_64BIT_HOST_MODE())
1149 {
1150 Log2(("VMX_VMCS_HOST_SYSENTER_EIP %RX64\n", ASMRdMsr(MSR_IA32_SYSENTER_EIP)));
1151 Log2(("VMX_VMCS_HOST_SYSENTER_ESP %RX64\n", ASMRdMsr(MSR_IA32_SYSENTER_ESP)));
1152 rc |= VMXWriteVMCS64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
1153 rc |= VMXWriteVMCS64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
1154 }
1155 else
1156 {
1157 rc |= VMXWriteVMCS(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
1158 rc |= VMXWriteVMCS(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
1159 Log2(("VMX_VMCS_HOST_SYSENTER_EIP %RX32\n", ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP)));
1160 Log2(("VMX_VMCS_HOST_SYSENTER_ESP %RX32\n", ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP)));
1161 }
1162#elif HC_ARCH_BITS == 32
1163 rc |= VMXWriteVMCS(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
1164 rc |= VMXWriteVMCS(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
1165 Log2(("VMX_VMCS_HOST_SYSENTER_EIP %RX32\n", ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP)));
1166 Log2(("VMX_VMCS_HOST_SYSENTER_ESP %RX32\n", ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP)));
1167#else
1168 Log2(("VMX_VMCS_HOST_SYSENTER_EIP %RX64\n", ASMRdMsr(MSR_IA32_SYSENTER_EIP)));
1169 Log2(("VMX_VMCS_HOST_SYSENTER_ESP %RX64\n", ASMRdMsr(MSR_IA32_SYSENTER_ESP)));
1170 rc |= VMXWriteVMCS64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
1171 rc |= VMXWriteVMCS64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
1172#endif
1173 AssertRC(rc);
1174
1175#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
1176 /* Store all host MSRs in the VM-Exit load area, so they will be reloaded after the world switch back to the host. */
1177 PVMXMSR pMsr = (PVMXMSR)pVCpu->hwaccm.s.vmx.pHostMSR;
1178 unsigned idxMsr = 0;
1179
1180 /* EFER MSR present? */
1181 if (ASMCpuId_EDX(0x80000001) & (X86_CPUID_AMD_FEATURE_EDX_NX|X86_CPUID_AMD_FEATURE_EDX_LONG_MODE))
1182 {
1183 if (ASMCpuId_EDX(0x80000001) & X86_CPUID_AMD_FEATURE_EDX_SEP)
1184 {
1185 pMsr->u32IndexMSR = MSR_K6_STAR;
1186 pMsr->u32Reserved = 0;
1187 pMsr->u64Value = ASMRdMsr(MSR_K6_STAR); /* legacy syscall eip, cs & ss */
1188 pMsr++; idxMsr++;
1189 }
1190
1191 pMsr->u32IndexMSR = MSR_K6_EFER;
1192 pMsr->u32Reserved = 0;
1193# if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
1194 if (CPUMIsGuestInLongMode(pVCpu))
1195 {
1196 /* Must match the efer value in our 64 bits switcher. */
1197 pMsr->u64Value = ASMRdMsr(MSR_K6_EFER) | MSR_K6_EFER_LME | MSR_K6_EFER_SCE | MSR_K6_EFER_NXE;
1198 }
1199 else
1200# endif
1201 pMsr->u64Value = ASMRdMsr(MSR_K6_EFER);
1202 pMsr++; idxMsr++;
1203 }
1204
1205# if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
1206 if (VMX_IS_64BIT_HOST_MODE())
1207 {
1208 pMsr->u32IndexMSR = MSR_K8_LSTAR;
1209 pMsr->u32Reserved = 0;
1210 pMsr->u64Value = ASMRdMsr(MSR_K8_LSTAR); /* 64 bits mode syscall rip */
1211 pMsr++; idxMsr++;
1212 pMsr->u32IndexMSR = MSR_K8_SF_MASK;
1213 pMsr->u32Reserved = 0;
1214 pMsr->u64Value = ASMRdMsr(MSR_K8_SF_MASK); /* syscall flag mask */
1215 pMsr++; idxMsr++;
1216 pMsr->u32IndexMSR = MSR_K8_KERNEL_GS_BASE;
1217 pMsr->u32Reserved = 0;
1218 pMsr->u64Value = ASMRdMsr(MSR_K8_KERNEL_GS_BASE); /* swapgs exchange value */
1219 pMsr++; idxMsr++;
1220 }
1221# endif
1222 rc = VMXWriteVMCS(VMX_VMCS_CTRL_EXIT_MSR_LOAD_COUNT, idxMsr);
1223 AssertRC(rc);
1224#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
1225
1226 pVCpu->hwaccm.s.fContextUseFlags &= ~HWACCM_CHANGED_HOST_CONTEXT;
1227 }
1228 return rc;
1229}
1230
1231/**
1232 * Prefetch the 4 PDPT pointers (PAE and nested paging only)
1233 *
1234 * @param pVM The VM to operate on.
1235 * @param pVCpu The VMCPU to operate on.
1236 * @param pCtx Guest context
1237 */
1238static void vmxR0PrefetchPAEPdptrs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
1239{
1240 if (CPUMIsGuestInPAEModeEx(pCtx))
1241 {
1242 X86PDPE Pdpe;
1243
1244 for (unsigned i=0;i<4;i++)
1245 {
1246 Pdpe = PGMGstGetPaePDPtr(pVCpu, i);
1247 int rc = VMXWriteVMCS64(VMX_VMCS_GUEST_PDPTR0_FULL + i*2, Pdpe.u);
1248 AssertRC(rc);
1249 }
1250 }
1251}
1252
1253/**
1254 * Update the exception bitmap according to the current CPU state
1255 *
1256 * @param pVM The VM to operate on.
1257 * @param pVCpu The VMCPU to operate on.
1258 * @param pCtx Guest context
1259 */
1260static void vmxR0UpdateExceptionBitmap(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
1261{
1262 uint32_t u32TrapMask;
1263 Assert(pCtx);
1264
1265 u32TrapMask = HWACCM_VMX_TRAP_MASK;
1266#ifndef DEBUG
1267 if (pVM->hwaccm.s.fNestedPaging)
1268 u32TrapMask &= ~RT_BIT(X86_XCPT_PF); /* no longer need to intercept #PF. */
1269#endif
1270
1271 /* Also catch floating point exceptions as we need to report them to the guest in a different way. */
1272 if ( CPUMIsGuestFPUStateActive(pVCpu) == true
1273 && !(pCtx->cr0 & X86_CR0_NE)
1274 && !pVCpu->hwaccm.s.fFPUOldStyleOverride)
1275 {
1276 u32TrapMask |= RT_BIT(X86_XCPT_MF);
1277 pVCpu->hwaccm.s.fFPUOldStyleOverride = true;
1278 }
1279
1280#ifdef VBOX_STRICT
1281 Assert(u32TrapMask & RT_BIT(X86_XCPT_GP));
1282#endif
1283
1284 /* Intercept all exceptions in real mode as none of them can be injected directly (#GP otherwise). */
1285 if ( CPUMIsGuestInRealModeEx(pCtx)
1286 && pVM->hwaccm.s.vmx.pRealModeTSS)
1287 u32TrapMask |= HWACCM_VMX_TRAP_MASK_REALMODE;
1288
1289 int rc = VMXWriteVMCS(VMX_VMCS_CTRL_EXCEPTION_BITMAP, u32TrapMask);
1290 AssertRC(rc);
1291}
1292
1293/**
1294 * Loads the guest state
1295 *
1296 * NOTE: Don't do anything here that can cause a jump back to ring 3!!!!!
1297 *
1298 * @returns VBox status code.
1299 * @param pVM The VM to operate on.
1300 * @param pVCpu The VMCPU to operate on.
1301 * @param pCtx Guest context
1302 */
1303VMMR0DECL(int) VMXR0LoadGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
1304{
1305 int rc = VINF_SUCCESS;
1306 RTGCUINTPTR val;
1307 X86EFLAGS eflags;
1308
1309 /* VMX_VMCS_CTRL_ENTRY_CONTROLS
1310 * Set required bits to one and zero according to the MSR capabilities.
1311 */
1312 val = pVM->hwaccm.s.vmx.msr.vmx_entry.n.disallowed0;
1313 /* Load guest debug controls (dr7 & IA32_DEBUGCTL_MSR) (forced to 1 on the 'first' VT-x capable CPUs; this actually includes the newest Nehalem CPUs) */
1314 val |= VMX_VMCS_CTRL_ENTRY_CONTROLS_LOAD_DEBUG;
1315 /* 64 bits guest mode? */
1316 if (CPUMIsGuestInLongModeEx(pCtx))
1317 val |= VMX_VMCS_CTRL_ENTRY_CONTROLS_IA64_MODE;
1318 /* else Must be zero when AMD64 is not available. */
1319
1320 /* Mask away the bits that the CPU doesn't support */
1321 val &= pVM->hwaccm.s.vmx.msr.vmx_entry.n.allowed1;
1322 rc = VMXWriteVMCS(VMX_VMCS_CTRL_ENTRY_CONTROLS, val);
1323 AssertRC(rc);
1324
1325 /* VMX_VMCS_CTRL_EXIT_CONTROLS
1326 * Set required bits to one and zero according to the MSR capabilities.
1327 */
1328 val = pVM->hwaccm.s.vmx.msr.vmx_exit.n.disallowed0;
1329
1330 /* Save debug controls (dr7 & IA32_DEBUGCTL_MSR) (forced to 1 on the 'first' VT-x capable CPUs; this actually includes the newest Nehalem CPUs) */
1331 val |= VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_DEBUG;
1332
1333#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
1334 if (VMX_IS_64BIT_HOST_MODE())
1335 val |= VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_AMD64;
1336 /* else: Must be zero when AMD64 is not available. */
1337#elif HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
1338 if (CPUMIsGuestInLongModeEx(pCtx))
1339 val |= VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_AMD64; /* our switcher goes to long mode */
1340 else
1341 Assert(!(val & VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_AMD64));
1342#endif
1343 val &= pVM->hwaccm.s.vmx.msr.vmx_exit.n.allowed1;
1344 /* Don't acknowledge external interrupts on VM-exit. */
1345 rc = VMXWriteVMCS(VMX_VMCS_CTRL_EXIT_CONTROLS, val);
1346 AssertRC(rc);
1347
1348 /* Guest CPU context: ES, CS, SS, DS, FS, GS. */
1349 if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_SEGMENT_REGS)
1350 {
1351 if (pVM->hwaccm.s.vmx.pRealModeTSS)
1352 {
1353 PGMMODE enmGuestMode = PGMGetGuestMode(pVCpu);
1354 if (pVCpu->hwaccm.s.vmx.enmLastSeenGuestMode != enmGuestMode)
1355 {
1356 /* Correct weird requirements for switching to protected mode. */
1357 if ( pVCpu->hwaccm.s.vmx.enmLastSeenGuestMode == PGMMODE_REAL
1358 && enmGuestMode >= PGMMODE_PROTECTED)
1359 {
1360 /* Flush the recompiler code cache as it's not unlikely
1361 * the guest will rewrite code it will later execute in real
1362 * mode (OpenBSD 4.0 is one such example)
1363 */
1364 REMFlushTBs(pVM);
1365
1366 /* DPL of all hidden selector registers must match the current CPL (0). */
1367 pCtx->csHid.Attr.n.u2Dpl = 0;
1368 pCtx->csHid.Attr.n.u4Type = X86_SEL_TYPE_CODE | X86_SEL_TYPE_RW_ACC;
1369
1370 pCtx->dsHid.Attr.n.u2Dpl = 0;
1371 pCtx->esHid.Attr.n.u2Dpl = 0;
1372 pCtx->fsHid.Attr.n.u2Dpl = 0;
1373 pCtx->gsHid.Attr.n.u2Dpl = 0;
1374 pCtx->ssHid.Attr.n.u2Dpl = 0;
1375
1376 /* The limit must correspond to the 32 bits setting. */
1377 if (!pCtx->csHid.Attr.n.u1DefBig)
1378 pCtx->csHid.u32Limit &= 0xffff;
1379 if (!pCtx->dsHid.Attr.n.u1DefBig)
1380 pCtx->dsHid.u32Limit &= 0xffff;
1381 if (!pCtx->esHid.Attr.n.u1DefBig)
1382 pCtx->esHid.u32Limit &= 0xffff;
1383 if (!pCtx->fsHid.Attr.n.u1DefBig)
1384 pCtx->fsHid.u32Limit &= 0xffff;
1385 if (!pCtx->gsHid.Attr.n.u1DefBig)
1386 pCtx->gsHid.u32Limit &= 0xffff;
1387 if (!pCtx->ssHid.Attr.n.u1DefBig)
1388 pCtx->ssHid.u32Limit &= 0xffff;
1389 }
1390 else
1391 /* Switching from protected mode to real mode. */
1392 if ( pVCpu->hwaccm.s.vmx.enmLastSeenGuestMode >= PGMMODE_PROTECTED
1393 && enmGuestMode == PGMMODE_REAL)
1394 {
1395 /* The limit must also be set to 0xffff. */
1396 pCtx->csHid.u32Limit = 0xffff;
1397 pCtx->dsHid.u32Limit = 0xffff;
1398 pCtx->esHid.u32Limit = 0xffff;
1399 pCtx->fsHid.u32Limit = 0xffff;
1400 pCtx->gsHid.u32Limit = 0xffff;
1401 pCtx->ssHid.u32Limit = 0xffff;
1402
1403 Assert(pCtx->csHid.u64Base <= 0xfffff);
1404 Assert(pCtx->dsHid.u64Base <= 0xfffff);
1405 Assert(pCtx->esHid.u64Base <= 0xfffff);
1406 Assert(pCtx->fsHid.u64Base <= 0xfffff);
1407 Assert(pCtx->gsHid.u64Base <= 0xfffff);
1408 }
1409 pVCpu->hwaccm.s.vmx.enmLastSeenGuestMode = enmGuestMode;
1410 }
1411 else
1412 /* VT-x will fail with a guest invalid state otherwise... (CPU state after a reset) */
1413 if ( CPUMIsGuestInRealModeEx(pCtx)
1414 && pCtx->csHid.u64Base == 0xffff0000)
1415 {
1416 pCtx->csHid.u64Base = 0xf0000;
1417 pCtx->cs = 0xf000;
1418 }
1419 }
1420
1421 VMX_WRITE_SELREG(ES, es);
1422 AssertRC(rc);
1423
1424 VMX_WRITE_SELREG(CS, cs);
1425 AssertRC(rc);
1426
1427 VMX_WRITE_SELREG(SS, ss);
1428 AssertRC(rc);
1429
1430 VMX_WRITE_SELREG(DS, ds);
1431 AssertRC(rc);
1432
1433 VMX_WRITE_SELREG(FS, fs);
1434 AssertRC(rc);
1435
1436 VMX_WRITE_SELREG(GS, gs);
1437 AssertRC(rc);
1438 }
1439
1440 /* Guest CPU context: LDTR. */
1441 if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_LDTR)
1442 {
1443 if (pCtx->ldtr == 0)
1444 {
1445 rc = VMXWriteVMCS(VMX_VMCS16_GUEST_FIELD_LDTR, 0);
1446 rc |= VMXWriteVMCS(VMX_VMCS32_GUEST_LDTR_LIMIT, 0);
1447 rc |= VMXWriteVMCS64(VMX_VMCS64_GUEST_LDTR_BASE, 0);
1448 /* Note: vmlaunch will fail with 0 or just 0x02. No idea why. */
1449 rc |= VMXWriteVMCS(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, 0x82 /* present, LDT */);
1450 }
1451 else
1452 {
1453 rc = VMXWriteVMCS(VMX_VMCS16_GUEST_FIELD_LDTR, pCtx->ldtr);
1454 rc |= VMXWriteVMCS(VMX_VMCS32_GUEST_LDTR_LIMIT, pCtx->ldtrHid.u32Limit);
1455 rc |= VMXWriteVMCS64(VMX_VMCS64_GUEST_LDTR_BASE, pCtx->ldtrHid.u64Base);
1456 rc |= VMXWriteVMCS(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, pCtx->ldtrHid.Attr.u);
1457 }
1458 AssertRC(rc);
1459 }
1460 /* Guest CPU context: TR. */
1461 if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_TR)
1462 {
1463 /* Real mode emulation using v86 mode with CR4.VME (interrupt redirection using the int bitmap in the TSS) */
1464 if ( CPUMIsGuestInRealModeEx(pCtx)
1465 && pVM->hwaccm.s.vmx.pRealModeTSS)
1466 {
1467 RTGCPHYS GCPhys;
1468
1469 /* We convert it here every time as pci regions could be reconfigured. */
1470 rc = PDMVMMDevHeapR3ToGCPhys(pVM, pVM->hwaccm.s.vmx.pRealModeTSS, &GCPhys);
1471 AssertRC(rc);
1472
1473 rc = VMXWriteVMCS(VMX_VMCS16_GUEST_FIELD_TR, 0);
1474 rc |= VMXWriteVMCS(VMX_VMCS32_GUEST_TR_LIMIT, HWACCM_VTX_TSS_SIZE);
1475 rc |= VMXWriteVMCS64(VMX_VMCS64_GUEST_TR_BASE, GCPhys /* phys = virt in this mode */);
1476
1477 X86DESCATTR attr;
1478
1479 attr.u = 0;
1480 attr.n.u1Present = 1;
1481 attr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
1482 val = attr.u;
1483 }
1484 else
1485 {
1486 rc = VMXWriteVMCS(VMX_VMCS16_GUEST_FIELD_TR, pCtx->tr);
1487 rc |= VMXWriteVMCS(VMX_VMCS32_GUEST_TR_LIMIT, pCtx->trHid.u32Limit);
1488 rc |= VMXWriteVMCS64(VMX_VMCS64_GUEST_TR_BASE, pCtx->trHid.u64Base);
1489
1490 val = pCtx->trHid.Attr.u;
1491
1492 /* The TSS selector must be busy. */
1493 if ((val & 0xF) == X86_SEL_TYPE_SYS_286_TSS_AVAIL)
1494 val = (val & ~0xF) | X86_SEL_TYPE_SYS_286_TSS_BUSY;
1495 else
1496 /* Default even if no TR selector has been set (otherwise vmlaunch will fail!) */
1497 val = (val & ~0xF) | X86_SEL_TYPE_SYS_386_TSS_BUSY;
1498
1499 }
1500 rc |= VMXWriteVMCS(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, val);
1501 AssertRC(rc);
1502 }
1503 /* Guest CPU context: GDTR. */
1504 if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_GDTR)
1505 {
1506 rc = VMXWriteVMCS(VMX_VMCS32_GUEST_GDTR_LIMIT, pCtx->gdtr.cbGdt);
1507 rc |= VMXWriteVMCS64(VMX_VMCS64_GUEST_GDTR_BASE, pCtx->gdtr.pGdt);
1508 AssertRC(rc);
1509 }
1510 /* Guest CPU context: IDTR. */
1511 if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_IDTR)
1512 {
1513 rc = VMXWriteVMCS(VMX_VMCS32_GUEST_IDTR_LIMIT, pCtx->idtr.cbIdt);
1514 rc |= VMXWriteVMCS64(VMX_VMCS64_GUEST_IDTR_BASE, pCtx->idtr.pIdt);
1515 AssertRC(rc);
1516 }
1517
1518 /*
1519 * Sysenter MSRs (unconditional)
1520 */
1521 rc = VMXWriteVMCS(VMX_VMCS32_GUEST_SYSENTER_CS, pCtx->SysEnter.cs);
1522 rc |= VMXWriteVMCS64(VMX_VMCS64_GUEST_SYSENTER_EIP, pCtx->SysEnter.eip);
1523 rc |= VMXWriteVMCS64(VMX_VMCS64_GUEST_SYSENTER_ESP, pCtx->SysEnter.esp);
1524 AssertRC(rc);
1525
1526 /* Control registers */
1527 if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_CR0)
1528 {
1529 val = pCtx->cr0;
1530 rc = VMXWriteVMCS(VMX_VMCS_CTRL_CR0_READ_SHADOW, val);
1531 Log2(("Guest CR0-shadow %08x\n", val));
1532 if (CPUMIsGuestFPUStateActive(pVCpu) == false)
1533 {
1534 /* Always use #NM exceptions to load the FPU/XMM state on demand. */
1535 val |= X86_CR0_TS | X86_CR0_ET | X86_CR0_NE | X86_CR0_MP;
1536 }
1537 else
1538 {
1539 /** @todo check if we support the old style mess correctly. */
1540 if (!(val & X86_CR0_NE))
1541 Log(("Forcing X86_CR0_NE!!!\n"));
1542
1543 val |= X86_CR0_NE; /* always turn on the native mechanism to report FPU errors (old style uses interrupts) */
1544 }
1545 /* Note: protected mode & paging are always enabled; we use them for emulating real and protected mode without paging too. */
1546 if (!pVM->hwaccm.s.vmx.fUnrestrictedGuest)
1547 val |= X86_CR0_PE | X86_CR0_PG;
1548
1549 if (pVM->hwaccm.s.fNestedPaging)
1550 {
1551 if (CPUMIsGuestInPagedProtectedModeEx(pCtx))
1552 {
1553 /* Disable cr3 read/write monitoring as we don't need it for EPT. */
1554 pVCpu->hwaccm.s.vmx.proc_ctls &= ~( VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
1555 | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT);
1556 }
1557 else
1558 {
1559 /* Reenable cr3 read/write monitoring as our identity mapped page table is active. */
1560 pVCpu->hwaccm.s.vmx.proc_ctls |= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
1561 | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT;
1562 }
1563 rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
1564 AssertRC(rc);
1565 }
1566 else
1567 {
1568 /* Note: We must also set this as we rely on protecting various pages for which supervisor writes must be caught. */
1569 val |= X86_CR0_WP;
1570 }
1571
1572 /* Always enable caching. */
1573 val &= ~(X86_CR0_CD|X86_CR0_NW);
1574
1575 rc |= VMXWriteVMCS64(VMX_VMCS64_GUEST_CR0, val);
1576 Log2(("Guest CR0 %08x\n", val));
1577 /* CR0 flags owned by the host; if the guests attempts to change them, then
1578 * the VM will exit.
1579 */
1580 val = X86_CR0_PE /* Must monitor this bit (assumptions are made for real mode emulation) */
1581 | X86_CR0_WP /* Must monitor this bit (it must always be enabled). */
1582 | X86_CR0_PG /* Must monitor this bit (assumptions are made for real mode & protected mode without paging emulation) */
1583 | X86_CR0_CD /* Bit not restored during VM-exit! */
1584 | X86_CR0_NW; /* Bit not restored during VM-exit! */
1585
1586 /* When the guest's FPU state is active, then we no longer care about
1587 * the FPU related bits.
1588 */
1589 if (CPUMIsGuestFPUStateActive(pVCpu) == false)
1590 val |= X86_CR0_TS | X86_CR0_ET | X86_CR0_NE | X86_CR0_MP;
1591
1592 pVCpu->hwaccm.s.vmx.cr0_mask = val;
1593
1594 rc |= VMXWriteVMCS(VMX_VMCS_CTRL_CR0_MASK, val);
1595 Log2(("Guest CR0-mask %08x\n", val));
1596 AssertRC(rc);
1597 }
1598 if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_CR4)
1599 {
1600 /* CR4 */
1601 rc = VMXWriteVMCS(VMX_VMCS_CTRL_CR4_READ_SHADOW, pCtx->cr4);
1602 Log2(("Guest CR4-shadow %08x\n", pCtx->cr4));
1603 /* Set the required bits in cr4 too (currently X86_CR4_VMXE). */
1604 val = pCtx->cr4 | (uint32_t)pVM->hwaccm.s.vmx.msr.vmx_cr4_fixed0;
1605
1606 if (!pVM->hwaccm.s.fNestedPaging)
1607 {
1608 switch(pVCpu->hwaccm.s.enmShadowMode)
1609 {
1610 case PGMMODE_REAL: /* Real mode -> emulated using v86 mode */
1611 case PGMMODE_PROTECTED: /* Protected mode, no paging -> emulated using identity mapping. */
1612 case PGMMODE_32_BIT: /* 32-bit paging. */
1613 val &= ~X86_CR4_PAE;
1614 break;
1615
1616 case PGMMODE_PAE: /* PAE paging. */
1617 case PGMMODE_PAE_NX: /* PAE paging with NX enabled. */
1618 /** @todo use normal 32 bits paging */
1619 val |= X86_CR4_PAE;
1620 break;
1621
1622 case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
1623 case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
1624#ifdef VBOX_ENABLE_64_BITS_GUESTS
1625 break;
1626#else
1627 AssertFailed();
1628 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
1629#endif
1630 default: /* shut up gcc */
1631 AssertFailed();
1632 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
1633 }
1634 }
1635 else
1636 if ( !CPUMIsGuestInPagedProtectedModeEx(pCtx)
1637 && !pVM->hwaccm.s.vmx.fUnrestrictedGuest)
1638 {
1639 /* We use 4 MB pages in our identity mapping page table for real and protected mode without paging. */
1640 val |= X86_CR4_PSE;
1641 /* Our identity mapping is a 32 bits page directory. */
1642 val &= ~X86_CR4_PAE;
1643 }
1644
1645 /* Turn off VME if we're in emulated real mode. */
1646 if ( CPUMIsGuestInRealModeEx(pCtx)
1647 && pVM->hwaccm.s.vmx.pRealModeTSS)
1648 val &= ~X86_CR4_VME;
1649
1650 rc |= VMXWriteVMCS64(VMX_VMCS64_GUEST_CR4, val);
1651 Log2(("Guest CR4 %08x\n", val));
1652 /* CR4 flags owned by the host; if the guests attempts to change them, then
1653 * the VM will exit.
1654 */
1655 val = 0
1656 | X86_CR4_VME
1657 | X86_CR4_PAE
1658 | X86_CR4_PGE
1659 | X86_CR4_PSE
1660 | X86_CR4_VMXE;
1661 pVCpu->hwaccm.s.vmx.cr4_mask = val;
1662
1663 rc |= VMXWriteVMCS(VMX_VMCS_CTRL_CR4_MASK, val);
1664 Log2(("Guest CR4-mask %08x\n", val));
1665 AssertRC(rc);
1666 }
1667
1668 if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_CR3)
1669 {
1670 if (pVM->hwaccm.s.fNestedPaging)
1671 {
1672 Assert(PGMGetHyperCR3(pVCpu));
1673 pVCpu->hwaccm.s.vmx.GCPhysEPTP = PGMGetHyperCR3(pVCpu);
1674
1675 Assert(!(pVCpu->hwaccm.s.vmx.GCPhysEPTP & 0xfff));
1676 /** @todo Check the IA32_VMX_EPT_VPID_CAP MSR for other supported memory types. */
1677 pVCpu->hwaccm.s.vmx.GCPhysEPTP |= VMX_EPT_MEMTYPE_WB
1678 | (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
1679
1680 rc = VMXWriteVMCS64(VMX_VMCS_CTRL_EPTP_FULL, pVCpu->hwaccm.s.vmx.GCPhysEPTP);
1681 AssertRC(rc);
1682
1683 if ( !CPUMIsGuestInPagedProtectedModeEx(pCtx)
1684 && !pVM->hwaccm.s.vmx.fUnrestrictedGuest)
1685 {
1686 RTGCPHYS GCPhys;
1687
1688 /* We convert it here every time as pci regions could be reconfigured. */
1689 rc = PDMVMMDevHeapR3ToGCPhys(pVM, pVM->hwaccm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
1690 AssertMsgRC(rc, ("pNonPagingModeEPTPageTable = %RGv\n", pVM->hwaccm.s.vmx.pNonPagingModeEPTPageTable));
1691
1692 /* We use our identity mapping page table here as we need to map guest virtual to guest physical addresses; EPT will
1693 * take care of the translation to host physical addresses.
1694 */
1695 val = GCPhys;
1696 }
1697 else
1698 {
1699 /* Save the real guest CR3 in VMX_VMCS_GUEST_CR3 */
1700 val = pCtx->cr3;
1701 /* Prefetch the four PDPT entries in PAE mode. */
1702 vmxR0PrefetchPAEPdptrs(pVM, pVCpu, pCtx);
1703 }
1704 }
1705 else
1706 {
1707 val = PGMGetHyperCR3(pVCpu);
1708 Assert(val || VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL));
1709 }
1710
1711 /* Save our shadow CR3 register. */
1712 rc = VMXWriteVMCS64(VMX_VMCS64_GUEST_CR3, val);
1713 AssertRC(rc);
1714 }
1715
1716 /* Debug registers. */
1717 if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_DEBUG)
1718 {
1719 pCtx->dr[6] |= X86_DR6_INIT_VAL; /* set all reserved bits to 1. */
1720 pCtx->dr[6] &= ~RT_BIT(12); /* must be zero. */
1721
1722 pCtx->dr[7] &= 0xffffffff; /* upper 32 bits reserved */
1723 pCtx->dr[7] &= ~(RT_BIT(11) | RT_BIT(12) | RT_BIT(14) | RT_BIT(15)); /* must be zero */
1724 pCtx->dr[7] |= 0x400; /* must be one */
1725
1726 /* Resync DR7 */
1727 rc = VMXWriteVMCS64(VMX_VMCS64_GUEST_DR7, pCtx->dr[7]);
1728 AssertRC(rc);
1729
1730#ifdef DEBUG
1731 /* Sync the hypervisor debug state now if any breakpoint is armed. */
1732 if ( CPUMGetHyperDR7(pVCpu) & (X86_DR7_ENABLED_MASK|X86_DR7_GD)
1733 && !CPUMIsHyperDebugStateActive(pVCpu)
1734 && !DBGFIsStepping(pVCpu))
1735 {
1736 /* Save the host and load the hypervisor debug state. */
1737 rc = CPUMR0LoadHyperDebugState(pVM, pVCpu, pCtx, true /* include DR6 */);
1738 AssertRC(rc);
1739
1740 /* DRx intercepts remain enabled. */
1741
1742 /* Override dr7 with the hypervisor value. */
1743 rc = VMXWriteVMCS64(VMX_VMCS64_GUEST_DR7, CPUMGetHyperDR7(pVCpu));
1744 AssertRC(rc);
1745 }
1746 else
1747#endif
1748 /* Sync the debug state now if any breakpoint is armed. */
1749 if ( (pCtx->dr[7] & (X86_DR7_ENABLED_MASK|X86_DR7_GD))
1750 && !CPUMIsGuestDebugStateActive(pVCpu)
1751 && !DBGFIsStepping(pVCpu))
1752 {
1753 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatDRxArmed);
1754
1755 /* Disable drx move intercepts. */
1756 pVCpu->hwaccm.s.vmx.proc_ctls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT;
1757 rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
1758 AssertRC(rc);
1759
1760 /* Save the host and load the guest debug state. */
1761 rc = CPUMR0LoadGuestDebugState(pVM, pVCpu, pCtx, true /* include DR6 */);
1762 AssertRC(rc);
1763 }
1764
1765 /* IA32_DEBUGCTL MSR. */
1766 rc = VMXWriteVMCS64(VMX_VMCS_GUEST_DEBUGCTL_FULL, 0);
1767 AssertRC(rc);
1768
1769 /** @todo do we really ever need this? */
1770 rc |= VMXWriteVMCS(VMX_VMCS_GUEST_DEBUG_EXCEPTIONS, 0);
1771 AssertRC(rc);
1772 }
1773
1774 /* EIP, ESP and EFLAGS */
1775 rc = VMXWriteVMCS64(VMX_VMCS64_GUEST_RIP, pCtx->rip);
1776 rc |= VMXWriteVMCS64(VMX_VMCS64_GUEST_RSP, pCtx->rsp);
1777 AssertRC(rc);
1778
1779 /* Bits 22-31, 15, 5 & 3 must be zero. Bit 1 must be 1. */
1780 eflags = pCtx->eflags;
1781 eflags.u32 &= VMX_EFLAGS_RESERVED_0;
1782 eflags.u32 |= VMX_EFLAGS_RESERVED_1;
1783
1784 /* Real mode emulation using v86 mode. */
1785 if ( CPUMIsGuestInRealModeEx(pCtx)
1786 && pVM->hwaccm.s.vmx.pRealModeTSS)
1787 {
1788 pVCpu->hwaccm.s.vmx.RealMode.eflags = eflags;
1789
1790 eflags.Bits.u1VM = 1;
1791 eflags.Bits.u2IOPL = 0; /* must always be 0 or else certain instructions won't cause faults. */
1792 }
1793 rc = VMXWriteVMCS(VMX_VMCS_GUEST_RFLAGS, eflags.u32);
1794 AssertRC(rc);
1795
1796 if (TMCpuTickCanUseRealTSC(pVCpu, &pVCpu->hwaccm.s.vmx.u64TSCOffset))
1797 {
1798 uint64_t u64CurTSC = ASMReadTSC();
1799 if (u64CurTSC + pVCpu->hwaccm.s.vmx.u64TSCOffset >= TMCpuTickGetLastSeen(pVCpu))
1800 {
1801 /* Note: VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT takes precedence over TSC_OFFSET */
1802 rc = VMXWriteVMCS64(VMX_VMCS_CTRL_TSC_OFFSET_FULL, pVCpu->hwaccm.s.vmx.u64TSCOffset);
1803 AssertRC(rc);
1804
1805 pVCpu->hwaccm.s.vmx.proc_ctls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT;
1806 rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
1807 AssertRC(rc);
1808 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatTSCOffset);
1809 }
1810 else
1811 {
1812 /* Fall back to rdtsc emulation as we would otherwise pass decreasing tsc values to the guest. */
1813 LogFlow(("TSC %RX64 offset %RX64 time=%RX64 last=%RX64 (diff=%RX64, virt_tsc=%RX64)\n", u64CurTSC, pVCpu->hwaccm.s.vmx.u64TSCOffset, u64CurTSC + pVCpu->hwaccm.s.vmx.u64TSCOffset, TMCpuTickGetLastSeen(pVCpu), TMCpuTickGetLastSeen(pVCpu) - u64CurTSC - pVCpu->hwaccm.s.vmx.u64TSCOffset, TMCpuTickGet(pVCpu)));
1814 pVCpu->hwaccm.s.vmx.proc_ctls |= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT;
1815 rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
1816 AssertRC(rc);
1817 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatTSCInterceptOverFlow);
1818 }
1819 }
1820 else
1821 {
1822 pVCpu->hwaccm.s.vmx.proc_ctls |= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT;
1823 rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
1824 AssertRC(rc);
1825 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatTSCIntercept);
1826 }
1827
1828 /* 64 bits guest mode? */
1829 if (CPUMIsGuestInLongModeEx(pCtx))
1830 {
1831#if !defined(VBOX_ENABLE_64_BITS_GUESTS)
1832 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
1833#elif HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
1834 pVCpu->hwaccm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
1835#else
1836# ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
1837 if (!pVM->hwaccm.s.fAllow64BitGuests)
1838 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
1839# endif
1840 pVCpu->hwaccm.s.vmx.pfnStartVM = VMXR0StartVM64;
1841#endif
1842 /* Unconditionally update these as wrmsr might have changed them. */
1843 rc = VMXWriteVMCS64(VMX_VMCS64_GUEST_FS_BASE, pCtx->fsHid.u64Base);
1844 AssertRC(rc);
1845 rc = VMXWriteVMCS64(VMX_VMCS64_GUEST_GS_BASE, pCtx->gsHid.u64Base);
1846 AssertRC(rc);
1847 }
1848 else
1849 {
1850 pVCpu->hwaccm.s.vmx.pfnStartVM = VMXR0StartVM32;
1851 }
1852
1853 vmxR0UpdateExceptionBitmap(pVM, pVCpu, pCtx);
1854
1855#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
1856 /* Store all guest MSRs in the VM-Entry load area, so they will be loaded during the world switch. */
1857 PVMXMSR pMsr = (PVMXMSR)pVCpu->hwaccm.s.vmx.pGuestMSR;
1858 unsigned idxMsr = 0;
1859
1860 uint32_t ulEdx;
1861 uint32_t ulTemp;
1862 CPUMGetGuestCpuId(pVCpu, 0x80000001, &ulTemp, &ulTemp, &ulTemp, &ulEdx);
1863 /* EFER MSR present? */
1864 if (ulEdx & (X86_CPUID_AMD_FEATURE_EDX_NX|X86_CPUID_AMD_FEATURE_EDX_LONG_MODE))
1865 {
1866 pMsr->u32IndexMSR = MSR_K6_EFER;
1867 pMsr->u32Reserved = 0;
1868 pMsr->u64Value = pCtx->msrEFER;
1869 /* VT-x will complain if only MSR_K6_EFER_LME is set. */
1870 if (!CPUMIsGuestInLongModeEx(pCtx))
1871 pMsr->u64Value &= ~(MSR_K6_EFER_LMA|MSR_K6_EFER_LME);
1872 pMsr++; idxMsr++;
1873
1874 if (ulEdx & X86_CPUID_AMD_FEATURE_EDX_LONG_MODE)
1875 {
1876 pMsr->u32IndexMSR = MSR_K8_LSTAR;
1877 pMsr->u32Reserved = 0;
1878 pMsr->u64Value = pCtx->msrLSTAR; /* 64 bits mode syscall rip */
1879 pMsr++; idxMsr++;
1880 pMsr->u32IndexMSR = MSR_K6_STAR;
1881 pMsr->u32Reserved = 0;
1882 pMsr->u64Value = pCtx->msrSTAR; /* legacy syscall eip, cs & ss */
1883 pMsr++; idxMsr++;
1884 pMsr->u32IndexMSR = MSR_K8_SF_MASK;
1885 pMsr->u32Reserved = 0;
1886 pMsr->u64Value = pCtx->msrSFMASK; /* syscall flag mask */
1887 pMsr++; idxMsr++;
1888 pMsr->u32IndexMSR = MSR_K8_KERNEL_GS_BASE;
1889 pMsr->u32Reserved = 0;
1890 pMsr->u64Value = pCtx->msrKERNELGSBASE; /* swapgs exchange value */
1891 pMsr++; idxMsr++;
1892 }
1893 }
1894 pVCpu->hwaccm.s.vmx.cCachedMSRs = idxMsr;
1895
1896 rc = VMXWriteVMCS(VMX_VMCS_CTRL_ENTRY_MSR_LOAD_COUNT, idxMsr);
1897 AssertRC(rc);
1898
1899 rc = VMXWriteVMCS(VMX_VMCS_CTRL_EXIT_MSR_STORE_COUNT, idxMsr);
1900 AssertRC(rc);
1901#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
1902
1903 /* Done. */
1904 pVCpu->hwaccm.s.fContextUseFlags &= ~HWACCM_CHANGED_ALL_GUEST;
1905
1906 return rc;
1907}
1908
1909/**
1910 * Syncs back the guest state
1911 *
1912 * @returns VBox status code.
1913 * @param pVM The VM to operate on.
1914 * @param pVCpu The VMCPU to operate on.
1915 * @param pCtx Guest context
1916 */
1917DECLINLINE(int) VMXR0SaveGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
1918{
1919 RTGCUINTREG val, valShadow;
1920 RTGCUINTPTR uInterruptState;
1921 int rc;
1922
1923 /* Let's first sync back eip, esp, and eflags. */
1924 rc = VMXReadCachedVMCS(VMX_VMCS64_GUEST_RIP, &val);
1925 AssertRC(rc);
1926 pCtx->rip = val;
1927 rc = VMXReadCachedVMCS(VMX_VMCS64_GUEST_RSP, &val);
1928 AssertRC(rc);
1929 pCtx->rsp = val;
1930 rc = VMXReadCachedVMCS(VMX_VMCS_GUEST_RFLAGS, &val);
1931 AssertRC(rc);
1932 pCtx->eflags.u32 = val;
1933
1934 /* Take care of instruction fusing (sti, mov ss) */
1935 rc |= VMXReadCachedVMCS(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &val);
1936 uInterruptState = val;
1937 if (uInterruptState != 0)
1938 {
1939 Assert(uInterruptState <= 2); /* only sti & mov ss */
1940 Log(("uInterruptState %x eip=%RGv\n", (uint32_t)uInterruptState, pCtx->rip));
1941 EMSetInhibitInterruptsPC(pVCpu, pCtx->rip);
1942 }
1943 else
1944 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
1945
1946 /* Control registers. */
1947 VMXReadCachedVMCS(VMX_VMCS_CTRL_CR0_READ_SHADOW, &valShadow);
1948 VMXReadCachedVMCS(VMX_VMCS64_GUEST_CR0, &val);
1949 val = (valShadow & pVCpu->hwaccm.s.vmx.cr0_mask) | (val & ~pVCpu->hwaccm.s.vmx.cr0_mask);
1950 CPUMSetGuestCR0(pVCpu, val);
1951
1952 VMXReadCachedVMCS(VMX_VMCS_CTRL_CR4_READ_SHADOW, &valShadow);
1953 VMXReadCachedVMCS(VMX_VMCS64_GUEST_CR4, &val);
1954 val = (valShadow & pVCpu->hwaccm.s.vmx.cr4_mask) | (val & ~pVCpu->hwaccm.s.vmx.cr4_mask);
1955 CPUMSetGuestCR4(pVCpu, val);
1956
1957 /* Note: no reason to sync back the CRx registers. They can't be changed by the guest. */
1958 /* Note: only in the nested paging case can CR3 & CR4 be changed by the guest. */
1959 if ( pVM->hwaccm.s.fNestedPaging
1960 && CPUMIsGuestInPagedProtectedModeEx(pCtx))
1961 {
1962 PVMCSCACHE pCache = &pVCpu->hwaccm.s.vmx.VMCSCache;
1963
1964 /* Can be updated behind our back in the nested paging case. */
1965 CPUMSetGuestCR2(pVCpu, pCache->cr2);
1966
1967 VMXReadCachedVMCS(VMX_VMCS64_GUEST_CR3, &val);
1968
1969 if (val != pCtx->cr3)
1970 {
1971 CPUMSetGuestCR3(pVCpu, val);
1972 PGMUpdateCR3(pVCpu, val);
1973 }
1974 /* Prefetch the four PDPT entries in PAE mode. */
1975 vmxR0PrefetchPAEPdptrs(pVM, pVCpu, pCtx);
1976 }
1977
1978 /* Sync back DR7 here. */
1979 VMXReadCachedVMCS(VMX_VMCS64_GUEST_DR7, &val);
1980 pCtx->dr[7] = val;
1981
1982 /* Guest CPU context: ES, CS, SS, DS, FS, GS. */
1983 VMX_READ_SELREG(ES, es);
1984 VMX_READ_SELREG(SS, ss);
1985 VMX_READ_SELREG(CS, cs);
1986 VMX_READ_SELREG(DS, ds);
1987 VMX_READ_SELREG(FS, fs);
1988 VMX_READ_SELREG(GS, gs);
1989
1990 /*
1991 * System MSRs
1992 */
1993 VMXReadCachedVMCS(VMX_VMCS32_GUEST_SYSENTER_CS, &val);
1994 pCtx->SysEnter.cs = val;
1995 VMXReadCachedVMCS(VMX_VMCS64_GUEST_SYSENTER_EIP, &val);
1996 pCtx->SysEnter.eip = val;
1997 VMXReadCachedVMCS(VMX_VMCS64_GUEST_SYSENTER_ESP, &val);
1998 pCtx->SysEnter.esp = val;
1999
2000 /* Misc. registers; must sync everything otherwise we can get out of sync when jumping to ring 3. */
2001 VMX_READ_SELREG(LDTR, ldtr);
2002
2003 VMXReadCachedVMCS(VMX_VMCS32_GUEST_GDTR_LIMIT, &val);
2004 pCtx->gdtr.cbGdt = val;
2005 VMXReadCachedVMCS(VMX_VMCS64_GUEST_GDTR_BASE, &val);
2006 pCtx->gdtr.pGdt = val;
2007
2008 VMXReadCachedVMCS(VMX_VMCS32_GUEST_IDTR_LIMIT, &val);
2009 pCtx->idtr.cbIdt = val;
2010 VMXReadCachedVMCS(VMX_VMCS64_GUEST_IDTR_BASE, &val);
2011 pCtx->idtr.pIdt = val;
2012
2013 /* Real mode emulation using v86 mode. */
2014 if ( CPUMIsGuestInRealModeEx(pCtx)
2015 && pVM->hwaccm.s.vmx.pRealModeTSS)
2016 {
2017 /* Hide our emulation flags */
2018 pCtx->eflags.Bits.u1VM = 0;
2019
2020 /* Restore original IOPL setting as we always use 0. */
2021 pCtx->eflags.Bits.u2IOPL = pVCpu->hwaccm.s.vmx.RealMode.eflags.Bits.u2IOPL;
2022
2023 /* Force a TR resync every time in case we switch modes. */
2024 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_GUEST_TR;
2025 }
2026 else
2027 {
2028 /* In real mode we have a fake TSS, so only sync it back when it's supposed to be valid. */
2029 VMX_READ_SELREG(TR, tr);
2030 }
2031
2032#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
2033 /* Save the possibly changed MSRs that we automatically restore and save during a world switch. */
2034 for (unsigned i = 0; i < pVCpu->hwaccm.s.vmx.cCachedMSRs; i++)
2035 {
2036 PVMXMSR pMsr = (PVMXMSR)pVCpu->hwaccm.s.vmx.pGuestMSR;
2037 pMsr += i;
2038
2039 switch (pMsr->u32IndexMSR)
2040 {
2041 case MSR_K8_LSTAR:
2042 pCtx->msrLSTAR = pMsr->u64Value;
2043 break;
2044 case MSR_K6_STAR:
2045 pCtx->msrSTAR = pMsr->u64Value;
2046 break;
2047 case MSR_K8_SF_MASK:
2048 pCtx->msrSFMASK = pMsr->u64Value;
2049 break;
2050 case MSR_K8_KERNEL_GS_BASE:
2051 pCtx->msrKERNELGSBASE = pMsr->u64Value;
2052 break;
2053 case MSR_K6_EFER:
2054 /* EFER can't be changed without causing a VM-exit. */
2055// Assert(pCtx->msrEFER == pMsr->u64Value);
2056 break;
2057 default:
2058 AssertFailed();
2059 return VERR_INTERNAL_ERROR;
2060 }
2061 }
2062#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
2063 return VINF_SUCCESS;
2064}
2065
2066/**
2067 * Dummy placeholder
2068 *
2069 * @param pVM The VM to operate on.
2070 * @param pVCpu The VMCPU to operate on.
2071 */
2072static void vmxR0SetupTLBDummy(PVM pVM, PVMCPU pVCpu)
2073{
2074 NOREF(pVM);
2075 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
2076 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
2077 pVCpu->hwaccm.s.TlbShootdown.cPages = 0;
2078 return;
2079}
2080
2081/**
2082 * Setup the tagged TLB for EPT
2083 *
2084 * @returns VBox status code.
2085 * @param pVM The VM to operate on.
2086 * @param pVCpu The VMCPU to operate on.
2087 */
2088static void vmxR0SetupTLBEPT(PVM pVM, PVMCPU pVCpu)
2089{
2090 PHWACCM_CPUINFO pCpu;
2091
2092 Assert(pVM->hwaccm.s.fNestedPaging);
2093 Assert(!pVM->hwaccm.s.vmx.fVPID);
2094
2095 /* Deal with tagged TLBs if VPID or EPT is supported. */
2096 pCpu = HWACCMR0GetCurrentCpu();
2097 /* Force a TLB flush for the first world switch if the current cpu differs from the one we ran on last. */
2098 /* Note that this can happen both for start and resume due to long jumps back to ring 3. */
2099 if ( pVCpu->hwaccm.s.idLastCpu != pCpu->idCpu
2100 /* if the tlb flush count has changed, another VM has flushed the TLB of this cpu, so we can't use our current ASID anymore. */
2101 || pVCpu->hwaccm.s.cTLBFlushes != pCpu->cTLBFlushes)
2102 {
2103 /* Force a TLB flush on VM entry. */
2104 pVCpu->hwaccm.s.fForceTLBFlush = true;
2105 }
2106 else
2107 Assert(!pCpu->fFlushTLB);
2108
2109 /* Check for tlb shootdown flushes. */
2110 if (VMCPU_FF_TESTANDCLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2111 pVCpu->hwaccm.s.fForceTLBFlush = true;
2112
2113 pVCpu->hwaccm.s.idLastCpu = pCpu->idCpu;
2114 pCpu->fFlushTLB = false;
2115
2116 if (pVCpu->hwaccm.s.fForceTLBFlush)
2117 {
2118 vmxR0FlushEPT(pVM, pVCpu, pVM->hwaccm.s.vmx.enmFlushContext, 0);
2119 }
2120 else
2121 if (VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
2122 {
2123 /* Deal with pending TLB shootdown actions which were queued when we were not executing code. */
2124 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatTlbShootdown);
2125
2126 for (unsigned i=0;i<pVCpu->hwaccm.s.TlbShootdown.cPages;i++)
2127 {
2128 /* aTlbShootdownPages contains physical addresses in this case. */
2129 vmxR0FlushEPT(pVM, pVCpu, pVM->hwaccm.s.vmx.enmFlushPage, pVCpu->hwaccm.s.TlbShootdown.aPages[i]);
2130 }
2131 }
2132 pVCpu->hwaccm.s.TlbShootdown.cPages= 0;
2133 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
2134
2135#ifdef VBOX_WITH_STATISTICS
2136 if (pVCpu->hwaccm.s.fForceTLBFlush)
2137 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatFlushTLBWorldSwitch);
2138 else
2139 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatNoFlushTLBWorldSwitch);
2140#endif
2141}
2142
2143#ifdef HWACCM_VTX_WITH_VPID
2144/**
2145 * Setup the tagged TLB for VPID
2146 *
2147 * @returns VBox status code.
2148 * @param pVM The VM to operate on.
2149 * @param pVCpu The VMCPU to operate on.
2150 */
2151static void vmxR0SetupTLBVPID(PVM pVM, PVMCPU pVCpu)
2152{
2153 PHWACCM_CPUINFO pCpu;
2154
2155 Assert(pVM->hwaccm.s.vmx.fVPID);
2156 Assert(!pVM->hwaccm.s.fNestedPaging);
2157
2158 /* Deal with tagged TLBs if VPID or EPT is supported. */
2159 pCpu = HWACCMR0GetCurrentCpu();
2160 /* Force a TLB flush for the first world switch if the current cpu differs from the one we ran on last. */
2161 /* Note that this can happen both for start and resume due to long jumps back to ring 3. */
2162 if ( pVCpu->hwaccm.s.idLastCpu != pCpu->idCpu
2163 /* if the tlb flush count has changed, another VM has flushed the TLB of this cpu, so we can't use our current ASID anymore. */
2164 || pVCpu->hwaccm.s.cTLBFlushes != pCpu->cTLBFlushes)
2165 {
2166 /* Force a TLB flush on VM entry. */
2167 pVCpu->hwaccm.s.fForceTLBFlush = true;
2168 }
2169 else
2170 Assert(!pCpu->fFlushTLB);
2171
2172 pVCpu->hwaccm.s.idLastCpu = pCpu->idCpu;
2173
2174 /* Check for tlb shootdown flushes. */
2175 if (VMCPU_FF_TESTANDCLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2176 pVCpu->hwaccm.s.fForceTLBFlush = true;
2177
2178 /* Make sure we flush the TLB when required. Switch ASID to achieve the same thing, but without actually flushing the whole TLB (which is expensive). */
2179 if (pVCpu->hwaccm.s.fForceTLBFlush)
2180 {
2181 if ( ++pCpu->uCurrentASID >= pVM->hwaccm.s.uMaxASID
2182 || pCpu->fFlushTLB)
2183 {
2184 pCpu->fFlushTLB = false;
2185 pCpu->uCurrentASID = 1; /* start at 1; host uses 0 */
2186 pCpu->cTLBFlushes++;
2187 vmxR0FlushVPID(pVM, pVCpu, VMX_FLUSH_ALL_CONTEXTS, 0);
2188 }
2189 else
2190 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatFlushASID);
2191
2192 pVCpu->hwaccm.s.fForceTLBFlush = false;
2193 pVCpu->hwaccm.s.cTLBFlushes = pCpu->cTLBFlushes;
2194 pVCpu->hwaccm.s.uCurrentASID = pCpu->uCurrentASID;
2195 }
2196 else
2197 {
2198 Assert(!pCpu->fFlushTLB);
2199 Assert(pVCpu->hwaccm.s.uCurrentASID && pCpu->uCurrentASID);
2200
2201 if (VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
2202 {
2203 /* Deal with pending TLB shootdown actions which were queued when we were not executing code. */
2204 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatTlbShootdown);
2205 for (unsigned i=0;i<pVCpu->hwaccm.s.TlbShootdown.cPages;i++)
2206 vmxR0FlushVPID(pVM, pVCpu, pVM->hwaccm.s.vmx.enmFlushPage, pVCpu->hwaccm.s.TlbShootdown.aPages[i]);
2207 }
2208 }
2209 pVCpu->hwaccm.s.TlbShootdown.cPages = 0;
2210 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
2211
2212 AssertMsg(pVCpu->hwaccm.s.cTLBFlushes == pCpu->cTLBFlushes, ("Flush count mismatch for cpu %d (%x vs %x)\n", pCpu->idCpu, pVCpu->hwaccm.s.cTLBFlushes, pCpu->cTLBFlushes));
2213 AssertMsg(pCpu->uCurrentASID >= 1 && pCpu->uCurrentASID < pVM->hwaccm.s.uMaxASID, ("cpu%d uCurrentASID = %x\n", pCpu->idCpu, pCpu->uCurrentASID));
2214 AssertMsg(pVCpu->hwaccm.s.uCurrentASID >= 1 && pVCpu->hwaccm.s.uCurrentASID < pVM->hwaccm.s.uMaxASID, ("cpu%d VM uCurrentASID = %x\n", pCpu->idCpu, pVCpu->hwaccm.s.uCurrentASID));
2215
2216 int rc = VMXWriteVMCS(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hwaccm.s.uCurrentASID);
2217 AssertRC(rc);
2218
2219 if (pVCpu->hwaccm.s.fForceTLBFlush)
2220 vmxR0FlushVPID(pVM, pVCpu, pVM->hwaccm.s.vmx.enmFlushContext, 0);
2221
2222#ifdef VBOX_WITH_STATISTICS
2223 if (pVCpu->hwaccm.s.fForceTLBFlush)
2224 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatFlushTLBWorldSwitch);
2225 else
2226 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatNoFlushTLBWorldSwitch);
2227#endif
2228}
2229#endif /* HWACCM_VTX_WITH_VPID */
2230
2231/**
2232 * Runs guest code in a VT-x VM.
2233 *
2234 * @returns VBox status code.
2235 * @param pVM The VM to operate on.
2236 * @param pVCpu The VMCPU to operate on.
2237 * @param pCtx Guest context
2238 */
2239VMMR0DECL(int) VMXR0RunGuestCode(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
2240{
2241 int rc = VINF_SUCCESS;
2242 RTGCUINTREG val;
2243 RTGCUINTREG exitReason = (RTGCUINTREG)VMX_EXIT_INVALID;
2244 RTGCUINTREG instrError, cbInstr;
2245 RTGCUINTPTR exitQualification = 0;
2246 RTGCUINTPTR intInfo = 0; /* shut up buggy gcc 4 */
2247 RTGCUINTPTR errCode, instrInfo;
2248 bool fSetupTPRCaching = false;
2249 uint64_t u64OldLSTAR = 0;
2250 uint8_t u8LastTPR = 0;
2251 RTCCUINTREG uOldEFlags = ~(RTCCUINTREG)0;
2252 unsigned cResume = 0;
2253#ifdef VBOX_STRICT
2254 RTCPUID idCpuCheck;
2255 bool fWasInLongMode = false;
2256#endif
2257#ifdef VBOX_HIGH_RES_TIMERS_HACK_IN_RING0
2258 uint64_t u64LastTime = RTTimeMilliTS();
2259#endif
2260#ifdef VBOX_WITH_STATISTICS
2261 bool fStatEntryStarted = true;
2262 bool fStatExit2Started = false;
2263#endif
2264
2265 Assert(!(pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC) || (pVCpu->hwaccm.s.vmx.pVAPIC && pVM->hwaccm.s.vmx.pAPIC));
2266
2267 /* Check if we need to use TPR shadowing. */
2268 if ( CPUMIsGuestInLongModeEx(pCtx)
2269 || ( ((pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC) || pVM->hwaccm.s.fTRPPatchingAllowed)
2270 && pVM->hwaccm.s.fHasIoApic)
2271 )
2272 {
2273 fSetupTPRCaching = true;
2274 }
2275
2276 Log2(("\nE"));
2277
2278 STAM_PROFILE_ADV_START(&pVCpu->hwaccm.s.StatEntry, x);
2279
2280#ifdef VBOX_STRICT
2281 {
2282 RTCCUINTREG val2;
2283
2284 rc = VMXReadVMCS(VMX_VMCS_CTRL_PIN_EXEC_CONTROLS, &val2);
2285 AssertRC(rc);
2286 Log2(("VMX_VMCS_CTRL_PIN_EXEC_CONTROLS = %08x\n", val2));
2287
2288 /* allowed zero */
2289 if ((val2 & pVM->hwaccm.s.vmx.msr.vmx_pin_ctls.n.disallowed0) != pVM->hwaccm.s.vmx.msr.vmx_pin_ctls.n.disallowed0)
2290 Log(("Invalid VMX_VMCS_CTRL_PIN_EXEC_CONTROLS: zero\n"));
2291
2292 /* allowed one */
2293 if ((val2 & ~pVM->hwaccm.s.vmx.msr.vmx_pin_ctls.n.allowed1) != 0)
2294 Log(("Invalid VMX_VMCS_CTRL_PIN_EXEC_CONTROLS: one\n"));
2295
2296 rc = VMXReadVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, &val2);
2297 AssertRC(rc);
2298 Log2(("VMX_VMCS_CTRL_PROC_EXEC_CONTROLS = %08x\n", val2));
2299
2300 /* Must be set according to the MSR, but can be cleared in case of EPT. */
2301 if (pVM->hwaccm.s.fNestedPaging)
2302 val2 |= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INVLPG_EXIT
2303 | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
2304 | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT;
2305
2306 /* allowed zero */
2307 if ((val2 & pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.disallowed0) != pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.disallowed0)
2308 Log(("Invalid VMX_VMCS_CTRL_PROC_EXEC_CONTROLS: zero\n"));
2309
2310 /* allowed one */
2311 if ((val2 & ~pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1) != 0)
2312 Log(("Invalid VMX_VMCS_CTRL_PROC_EXEC_CONTROLS: one\n"));
2313
2314 rc = VMXReadVMCS(VMX_VMCS_CTRL_ENTRY_CONTROLS, &val2);
2315 AssertRC(rc);
2316 Log2(("VMX_VMCS_CTRL_ENTRY_CONTROLS = %08x\n", val2));
2317
2318 /* allowed zero */
2319 if ((val2 & pVM->hwaccm.s.vmx.msr.vmx_entry.n.disallowed0) != pVM->hwaccm.s.vmx.msr.vmx_entry.n.disallowed0)
2320 Log(("Invalid VMX_VMCS_CTRL_ENTRY_CONTROLS: zero\n"));
2321
2322 /* allowed one */
2323 if ((val2 & ~pVM->hwaccm.s.vmx.msr.vmx_entry.n.allowed1) != 0)
2324 Log(("Invalid VMX_VMCS_CTRL_ENTRY_CONTROLS: one\n"));
2325
2326 rc = VMXReadVMCS(VMX_VMCS_CTRL_EXIT_CONTROLS, &val2);
2327 AssertRC(rc);
2328 Log2(("VMX_VMCS_CTRL_EXIT_CONTROLS = %08x\n", val2));
2329
2330 /* allowed zero */
2331 if ((val2 & pVM->hwaccm.s.vmx.msr.vmx_exit.n.disallowed0) != pVM->hwaccm.s.vmx.msr.vmx_exit.n.disallowed0)
2332 Log(("Invalid VMX_VMCS_CTRL_EXIT_CONTROLS: zero\n"));
2333
2334 /* allowed one */
2335 if ((val2 & ~pVM->hwaccm.s.vmx.msr.vmx_exit.n.allowed1) != 0)
2336 Log(("Invalid VMX_VMCS_CTRL_EXIT_CONTROLS: one\n"));
2337 }
2338 fWasInLongMode = CPUMIsGuestInLongModeEx(pCtx);
2339#endif /* VBOX_STRICT */
2340
2341#ifdef VBOX_WITH_CRASHDUMP_MAGIC
2342 pVCpu->hwaccm.s.vmx.VMCSCache.u64TimeEntry = RTTimeNanoTS();
2343#endif
2344
2345 /* We can jump to this point to resume execution after determining that a VM-exit is innocent.
2346 */
2347ResumeExecution:
2348 STAM_STATS({
2349 if (fStatExit2Started) { STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2, y); fStatExit2Started = false; }
2350 if (!fStatEntryStarted) { STAM_PROFILE_ADV_START(&pVCpu->hwaccm.s.StatEntry, x); fStatEntryStarted = true; }
2351 });
2352 AssertMsg(pVCpu->hwaccm.s.idEnteredCpu == RTMpCpuId(),
2353 ("Expected %d, I'm %d; cResume=%d exitReason=%RGv exitQualification=%RGv\n",
2354 (int)pVCpu->hwaccm.s.idEnteredCpu, (int)RTMpCpuId(), cResume, exitReason, exitQualification));
2355 Assert(!HWACCMR0SuspendPending());
2356 /* Not allowed to switch modes without reloading the host state (32->64 switcher)!! */
2357 Assert(fWasInLongMode == CPUMIsGuestInLongModeEx(pCtx));
2358
2359 /* Safety precaution; looping for too long here can have a very bad effect on the host */
2360 if (RT_UNLIKELY(++cResume > pVM->hwaccm.s.cMaxResumeLoops))
2361 {
2362 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitMaxResume);
2363 rc = VINF_EM_RAW_INTERRUPT;
2364 goto end;
2365 }
2366
2367 /* Check for irq inhibition due to instruction fusing (sti, mov ss). */
2368 if (VMCPU_FF_ISSET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
2369 {
2370 Log(("VM_FF_INHIBIT_INTERRUPTS at %RGv successor %RGv\n", (RTGCPTR)pCtx->rip, EMGetInhibitInterruptsPC(pVCpu)));
2371 if (pCtx->rip != EMGetInhibitInterruptsPC(pVCpu))
2372 {
2373 /* Note: we intentionally don't clear VM_FF_INHIBIT_INTERRUPTS here.
2374 * Before we are able to execute this instruction in raw mode (iret to guest code) an external interrupt might
2375 * force a world switch again. Possibly allowing a guest interrupt to be dispatched in the process. This could
2376 * break the guest. Sounds very unlikely, but such timing sensitive problems are not as rare as you might think.
2377 */
2378 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
2379 /* Irq inhibition is no longer active; clear the corresponding VMX state. */
2380 rc = VMXWriteVMCS(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, 0);
2381 AssertRC(rc);
2382 }
2383 }
2384 else
2385 {
2386 /* Irq inhibition is no longer active; clear the corresponding VMX state. */
2387 rc = VMXWriteVMCS(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, 0);
2388 AssertRC(rc);
2389 }
2390
2391#ifdef VBOX_HIGH_RES_TIMERS_HACK_IN_RING0
2392 if (RT_UNLIKELY(cResume & 0xf) == 0)
2393 {
2394 uint64_t u64CurTime = RTTimeMilliTS();
2395
2396 if (RT_UNLIKELY(u64CurTime > u64LastTime))
2397 {
2398 u64LastTime = u64CurTime;
2399 TMTimerPollVoid(pVM, pVCpu);
2400 }
2401 }
2402#endif
2403
2404 /* Check for pending actions that force us to go back to ring 3. */
2405 if ( VM_FF_ISPENDING(pVM, VM_FF_HWACCM_TO_R3_MASK | VM_FF_REQUEST | VM_FF_PGM_POOL_FLUSH_PENDING)
2406 || VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_HWACCM_TO_R3_MASK | VMCPU_FF_PGM_SYNC_CR3 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL | VMCPU_FF_REQUEST))
2407 {
2408 /* Check if a sync operation is pending. */
2409 if (VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
2410 {
2411 rc = PGMSyncCR3(pVCpu, pCtx->cr0, pCtx->cr3, pCtx->cr4, VMCPU_FF_ISSET(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
2412 AssertRC(rc);
2413 if (rc != VINF_SUCCESS)
2414 {
2415 Log(("Pending pool sync is forcing us back to ring 3; rc=%d\n", rc));
2416 goto end;
2417 }
2418 }
2419
2420#ifdef DEBUG
2421 /* Intercept X86_XCPT_DB if stepping is enabled */
2422 if (!DBGFIsStepping(pVCpu))
2423#endif
2424 {
2425 if ( VM_FF_ISPENDING(pVM, VM_FF_HWACCM_TO_R3_MASK)
2426 || VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_HWACCM_TO_R3_MASK))
2427 {
2428 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
2429 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatSwitchToR3);
2430 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatEntry, x);
2431 rc = RT_UNLIKELY(VM_FF_ISPENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
2432 goto end;
2433 }
2434 }
2435
2436 /* Pending request packets might contain actions that need immediate attention, such as pending hardware interrupts. */
2437 if ( VM_FF_ISPENDING(pVM, VM_FF_REQUEST)
2438 || VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_REQUEST))
2439 {
2440 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatEntry, x);
2441 rc = VINF_EM_PENDING_REQUEST;
2442 goto end;
2443 }
2444
2445 /* Check if a pgm pool flush is in progress. */
2446 if (VM_FF_ISPENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
2447 {
2448 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatEntry, x);
2449 rc = VINF_PGM_POOL_FLUSH_PENDING;
2450 goto end;
2451 }
2452 }
2453
2454#ifdef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
2455 /*
2456 * Exit to ring-3 preemption/work is pending.
2457 *
2458 * Interrupts are disabled before the call to make sure we don't miss any interrupt
2459 * that would flag preemption (IPI, timer tick, ++). (Would've been nice to do this
2460 * further down, but VMXR0CheckPendingInterrupt makes that impossible.)
2461 *
2462 * Note! Interrupts must be disabled done *before* we check for TLB flushes; TLB
2463 * shootdowns rely on this.
2464 */
2465 uOldEFlags = ASMIntDisableFlags();
2466 if (RTThreadPreemptIsPending(NIL_RTTHREAD))
2467 {
2468 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitPreemptPending);
2469 rc = VINF_EM_RAW_INTERRUPT;
2470 goto end;
2471 }
2472 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
2473#endif
2474
2475 /* When external interrupts are pending, we should exit the VM when IF is set. */
2476 /* Note! *After* VM_FF_INHIBIT_INTERRUPTS check!!! */
2477 rc = VMXR0CheckPendingInterrupt(pVM, pVCpu, pCtx);
2478 if (RT_FAILURE(rc))
2479 goto end;
2480
2481 /** @todo check timers?? */
2482
2483 /* TPR caching using CR8 is only available in 64 bits mode */
2484 /* Note the 32 bits exception for AMD (X86_CPUID_AMD_FEATURE_ECX_CR8L), but that appears missing in Intel CPUs */
2485 /* Note: we can't do this in LoadGuestState as PDMApicGetTPR can jump back to ring 3 (lock)!!!!! (no longer true) */
2486 /**
2487 * @todo query and update the TPR only when it could have been changed (mmio access & wrmsr (x2apic))
2488 */
2489 if (fSetupTPRCaching)
2490 {
2491 /* TPR caching in CR8 */
2492 bool fPending;
2493
2494 int rc2 = PDMApicGetTPR(pVCpu, &u8LastTPR, &fPending);
2495 AssertRC(rc2);
2496 /* The TPR can be found at offset 0x80 in the APIC mmio page. */
2497 pVCpu->hwaccm.s.vmx.pVAPIC[0x80] = u8LastTPR;
2498
2499 /* Two options here:
2500 * - external interrupt pending, but masked by the TPR value.
2501 * -> a CR8 update that lower the current TPR value should cause an exit
2502 * - no pending interrupts
2503 * -> We don't need to be explicitely notified. There are enough world switches for detecting pending interrupts.
2504 */
2505 rc = VMXWriteVMCS(VMX_VMCS_CTRL_TPR_THRESHOLD, (fPending) ? (u8LastTPR >> 4) : 0); /* cr8 bits 3-0 correspond to bits 7-4 of the task priority mmio register. */
2506 AssertRC(rc);
2507
2508 if (pVM->hwaccm.s.fTPRPatchingActive)
2509 {
2510 Assert(!CPUMIsGuestInLongModeEx(pCtx));
2511 /* Our patch code uses LSTAR for TPR caching. */
2512 pCtx->msrLSTAR = u8LastTPR;
2513
2514 if (fPending)
2515 {
2516 /* A TPR change could activate a pending interrupt, so catch lstar writes. */
2517 vmxR0SetMSRPermission(pVCpu, MSR_K8_LSTAR, true, false);
2518 }
2519 else
2520 {
2521 /* No interrupts are pending, so we don't need to be explicitely notified.
2522 * There are enough world switches for detecting pending interrupts.
2523 */
2524 vmxR0SetMSRPermission(pVCpu, MSR_K8_LSTAR, true, true);
2525 }
2526 }
2527 }
2528
2529#if defined(HWACCM_VTX_WITH_EPT) && defined(LOG_ENABLED)
2530 if ( pVM->hwaccm.s.fNestedPaging
2531# ifdef HWACCM_VTX_WITH_VPID
2532 || pVM->hwaccm.s.vmx.fVPID
2533# endif /* HWACCM_VTX_WITH_VPID */
2534 )
2535 {
2536 PHWACCM_CPUINFO pCpu;
2537
2538 pCpu = HWACCMR0GetCurrentCpu();
2539 if ( pVCpu->hwaccm.s.idLastCpu != pCpu->idCpu
2540 || pVCpu->hwaccm.s.cTLBFlushes != pCpu->cTLBFlushes)
2541 {
2542 if (pVCpu->hwaccm.s.idLastCpu != pCpu->idCpu)
2543 LogFlow(("Force TLB flush due to rescheduling to a different cpu (%d vs %d)\n", pVCpu->hwaccm.s.idLastCpu, pCpu->idCpu));
2544 else
2545 LogFlow(("Force TLB flush due to changed TLB flush count (%x vs %x)\n", pVCpu->hwaccm.s.cTLBFlushes, pCpu->cTLBFlushes));
2546 }
2547 if (pCpu->fFlushTLB)
2548 LogFlow(("Force TLB flush: first time cpu %d is used -> flush\n", pCpu->idCpu));
2549 else
2550 if (pVCpu->hwaccm.s.fForceTLBFlush)
2551 LogFlow(("Manual TLB flush\n"));
2552 }
2553#endif
2554#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
2555 PGMDynMapFlushAutoSet(pVCpu);
2556#endif
2557
2558 /*
2559 * NOTE: DO NOT DO ANYTHING AFTER THIS POINT THAT MIGHT JUMP BACK TO RING 3!
2560 * (until the actual world switch)
2561 */
2562#ifdef VBOX_STRICT
2563 idCpuCheck = RTMpCpuId();
2564#endif
2565#ifdef LOG_ENABLED
2566 VMMR0LogFlushDisable(pVCpu);
2567#endif
2568 /* Save the host state first. */
2569 rc = VMXR0SaveHostState(pVM, pVCpu);
2570 if (RT_UNLIKELY(rc != VINF_SUCCESS))
2571 {
2572 VMMR0LogFlushEnable(pVCpu);
2573 goto end;
2574 }
2575 /* Load the guest state */
2576 rc = VMXR0LoadGuestState(pVM, pVCpu, pCtx);
2577 if (RT_UNLIKELY(rc != VINF_SUCCESS))
2578 {
2579 VMMR0LogFlushEnable(pVCpu);
2580 goto end;
2581 }
2582
2583#ifndef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
2584 /* Disable interrupts to make sure a poke will interrupt execution.
2585 * This must be done *before* we check for TLB flushes; TLB shootdowns rely on this.
2586 */
2587 uOldEFlags = ASMIntDisableFlags();
2588 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
2589#endif
2590
2591 /* Non-register state Guest Context */
2592 /** @todo change me according to cpu state */
2593 rc = VMXWriteVMCS(VMX_VMCS32_GUEST_ACTIVITY_STATE, VMX_CMS_GUEST_ACTIVITY_ACTIVE);
2594 AssertRC(rc);
2595
2596 /** Set TLB flush state as checked until we return from the world switch. */
2597 ASMAtomicWriteU8(&pVCpu->hwaccm.s.fCheckedTLBFlush, true);
2598 /* Deal with tagged TLB setup and invalidation. */
2599 pVM->hwaccm.s.vmx.pfnSetupTaggedTLB(pVM, pVCpu);
2600
2601 STAM_STATS({ STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatEntry, x); fStatEntryStarted = false; });
2602
2603 /* Manual save and restore:
2604 * - General purpose registers except RIP, RSP
2605 *
2606 * Trashed:
2607 * - CR2 (we don't care)
2608 * - LDTR (reset to 0)
2609 * - DRx (presumably not changed at all)
2610 * - DR7 (reset to 0x400)
2611 * - EFLAGS (reset to RT_BIT(1); not relevant)
2612 *
2613 */
2614
2615 /* All done! Let's start VM execution. */
2616 STAM_PROFILE_ADV_START(&pVCpu->hwaccm.s.StatInGC, z);
2617 Assert(idCpuCheck == RTMpCpuId());
2618
2619#ifdef VBOX_WITH_CRASHDUMP_MAGIC
2620 pVCpu->hwaccm.s.vmx.VMCSCache.cResume = cResume;
2621 pVCpu->hwaccm.s.vmx.VMCSCache.u64TimeSwitch = RTTimeNanoTS();
2622#endif
2623
2624 /* Save the current TPR value in the LSTAR msr so our patches can access it. */
2625 if (pVM->hwaccm.s.fTPRPatchingActive)
2626 {
2627 Assert(pVM->hwaccm.s.fTPRPatchingActive);
2628 u64OldLSTAR = ASMRdMsr(MSR_K8_LSTAR);
2629 ASMWrMsr(MSR_K8_LSTAR, u8LastTPR);
2630 }
2631
2632 TMNotifyStartOfExecution(pVCpu);
2633#ifdef VBOX_WITH_KERNEL_USING_XMM
2634 rc = hwaccmR0VMXStartVMWrapXMM(pVCpu->hwaccm.s.fResumeVM, pCtx, &pVCpu->hwaccm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hwaccm.s.vmx.pfnStartVM);
2635#else
2636 rc = pVCpu->hwaccm.s.vmx.pfnStartVM(pVCpu->hwaccm.s.fResumeVM, pCtx, &pVCpu->hwaccm.s.vmx.VMCSCache, pVM, pVCpu);
2637#endif
2638 ASMAtomicWriteU8(&pVCpu->hwaccm.s.fCheckedTLBFlush, false);
2639 ASMAtomicIncU32(&pVCpu->hwaccm.s.cWorldSwitchExit);
2640 /* Possibly the last TSC value seen by the guest (too high) (only when we're in tsc offset mode). */
2641 if (!(pVCpu->hwaccm.s.vmx.proc_ctls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT))
2642 TMCpuTickSetLastSeen(pVCpu, ASMReadTSC() + pVCpu->hwaccm.s.vmx.u64TSCOffset - 0x400 /* guestimate of world switch overhead in clock ticks */);
2643
2644 TMNotifyEndOfExecution(pVCpu);
2645 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED);
2646 Assert(!(ASMGetFlags() & X86_EFL_IF));
2647
2648 /* Restore the host LSTAR msr if the guest could have changed it. */
2649 if (pVM->hwaccm.s.fTPRPatchingActive)
2650 {
2651 Assert(pVM->hwaccm.s.fTPRPatchingActive);
2652 pVCpu->hwaccm.s.vmx.pVAPIC[0x80] = pCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR);
2653 ASMWrMsr(MSR_K8_LSTAR, u64OldLSTAR);
2654 }
2655
2656 ASMSetFlags(uOldEFlags);
2657#ifdef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
2658 uOldEFlags = ~(RTCCUINTREG)0;
2659#endif
2660
2661 AssertMsg(!pVCpu->hwaccm.s.vmx.VMCSCache.Write.cValidEntries, ("pVCpu->hwaccm.s.vmx.VMCSCache.Write.cValidEntries=%d\n", pVCpu->hwaccm.s.vmx.VMCSCache.Write.cValidEntries));
2662
2663 /* In case we execute a goto ResumeExecution later on. */
2664 pVCpu->hwaccm.s.fResumeVM = true;
2665 pVCpu->hwaccm.s.fForceTLBFlush = false;
2666
2667 /*
2668 * !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
2669 * IMPORTANT: WE CAN'T DO ANY LOGGING OR OPERATIONS THAT CAN DO A LONGJMP BACK TO RING 3 *BEFORE* WE'VE SYNCED BACK (MOST OF) THE GUEST STATE
2670 * !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
2671 */
2672 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatInGC, z);
2673
2674 if (RT_UNLIKELY(rc != VINF_SUCCESS))
2675 {
2676 VMXR0ReportWorldSwitchError(pVM, pVCpu, rc, pCtx);
2677 VMMR0LogFlushEnable(pVCpu);
2678 goto end;
2679 }
2680
2681 STAM_PROFILE_ADV_START(&pVCpu->hwaccm.s.StatExit1, v);
2682 /* Success. Query the guest state and figure out what has happened. */
2683
2684 /* Investigate why there was a VM-exit. */
2685 rc = VMXReadCachedVMCS(VMX_VMCS32_RO_EXIT_REASON, &exitReason);
2686 STAM_COUNTER_INC(&pVCpu->hwaccm.s.paStatExitReasonR0[exitReason & MASK_EXITREASON_STAT]);
2687
2688 exitReason &= 0xffff; /* bit 0-15 contain the exit code. */
2689 rc |= VMXReadCachedVMCS(VMX_VMCS32_RO_VM_INSTR_ERROR, &instrError);
2690 rc |= VMXReadCachedVMCS(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &cbInstr);
2691 rc |= VMXReadCachedVMCS(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &intInfo);
2692 /* might not be valid; depends on VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID. */
2693 rc |= VMXReadCachedVMCS(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERRCODE, &errCode);
2694 rc |= VMXReadCachedVMCS(VMX_VMCS32_RO_EXIT_INSTR_INFO, &instrInfo);
2695 rc |= VMXReadCachedVMCS(VMX_VMCS_RO_EXIT_QUALIFICATION, &exitQualification);
2696 AssertRC(rc);
2697
2698 /* Sync back the guest state */
2699 rc = VMXR0SaveGuestState(pVM, pVCpu, pCtx);
2700 AssertRC(rc);
2701
2702 /* Note! NOW IT'S SAFE FOR LOGGING! */
2703 VMMR0LogFlushEnable(pVCpu);
2704 Log2(("Raw exit reason %08x\n", exitReason));
2705
2706 /* Check if an injected event was interrupted prematurely. */
2707 rc = VMXReadCachedVMCS(VMX_VMCS32_RO_IDT_INFO, &val);
2708 AssertRC(rc);
2709 pVCpu->hwaccm.s.Event.intInfo = VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(val);
2710 if ( VMX_EXIT_INTERRUPTION_INFO_VALID(pVCpu->hwaccm.s.Event.intInfo)
2711 /* Ignore 'int xx' as they'll be restarted anyway. */
2712 && VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hwaccm.s.Event.intInfo) != VMX_EXIT_INTERRUPTION_INFO_TYPE_SW
2713 /* Ignore software exceptions (such as int3) as they'll reoccur when we restart the instruction anyway. */
2714 && VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hwaccm.s.Event.intInfo) != VMX_EXIT_INTERRUPTION_INFO_TYPE_SWEXCPT)
2715 {
2716 Assert(!pVCpu->hwaccm.s.Event.fPending);
2717 pVCpu->hwaccm.s.Event.fPending = true;
2718 /* Error code present? */
2719 if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(pVCpu->hwaccm.s.Event.intInfo))
2720 {
2721 rc = VMXReadCachedVMCS(VMX_VMCS32_RO_IDT_ERRCODE, &val);
2722 AssertRC(rc);
2723 pVCpu->hwaccm.s.Event.errCode = val;
2724 Log(("Pending inject %RX64 at %RGv exit=%08x intInfo=%08x exitQualification=%RGv pending error=%RX64\n", pVCpu->hwaccm.s.Event.intInfo, (RTGCPTR)pCtx->rip, exitReason, intInfo, exitQualification, val));
2725 }
2726 else
2727 {
2728 Log(("Pending inject %RX64 at %RGv exit=%08x intInfo=%08x exitQualification=%RGv\n", pVCpu->hwaccm.s.Event.intInfo, (RTGCPTR)pCtx->rip, exitReason, intInfo, exitQualification));
2729 pVCpu->hwaccm.s.Event.errCode = 0;
2730 }
2731 }
2732#ifdef VBOX_STRICT
2733 else
2734 if ( VMX_EXIT_INTERRUPTION_INFO_VALID(pVCpu->hwaccm.s.Event.intInfo)
2735 /* Ignore software exceptions (such as int3) as they're reoccur when we restart the instruction anyway. */
2736 && VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hwaccm.s.Event.intInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_SWEXCPT)
2737 {
2738 Log(("Ignore pending inject %RX64 at %RGv exit=%08x intInfo=%08x exitQualification=%RGv\n", pVCpu->hwaccm.s.Event.intInfo, (RTGCPTR)pCtx->rip, exitReason, intInfo, exitQualification));
2739 }
2740
2741 if (exitReason == VMX_EXIT_ERR_INVALID_GUEST_STATE)
2742 HWACCMDumpRegs(pVM, pVCpu, pCtx);
2743#endif
2744
2745 Log2(("E%d: New EIP=%x:%RGv\n", (uint32_t)exitReason, pCtx->cs, (RTGCPTR)pCtx->rip));
2746 Log2(("Exit reason %d, exitQualification %RGv\n", (uint32_t)exitReason, exitQualification));
2747 Log2(("instrInfo=%d instrError=%d instr length=%d\n", (uint32_t)instrInfo, (uint32_t)instrError, (uint32_t)cbInstr));
2748 Log2(("Interruption error code %d\n", (uint32_t)errCode));
2749 Log2(("IntInfo = %08x\n", (uint32_t)intInfo));
2750
2751 /* Sync back the TPR if it was changed. */
2752 if ( fSetupTPRCaching
2753 && u8LastTPR != pVCpu->hwaccm.s.vmx.pVAPIC[0x80])
2754 {
2755 rc = PDMApicSetTPR(pVCpu, pVCpu->hwaccm.s.vmx.pVAPIC[0x80]);
2756 AssertRC(rc);
2757 }
2758
2759 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit1, v);
2760 STAM_STATS({ STAM_PROFILE_ADV_START(&pVCpu->hwaccm.s.StatExit2, y); fStatExit2Started = true; });
2761
2762 /* Some cases don't need a complete resync of the guest CPU state; handle them here. */
2763 switch (exitReason)
2764 {
2765 case VMX_EXIT_EXCEPTION: /* 0 Exception or non-maskable interrupt (NMI). */
2766 case VMX_EXIT_EXTERNAL_IRQ: /* 1 External interrupt. */
2767 {
2768 uint32_t vector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(intInfo);
2769
2770 if (!VMX_EXIT_INTERRUPTION_INFO_VALID(intInfo))
2771 {
2772 Assert(exitReason == VMX_EXIT_EXTERNAL_IRQ);
2773#if 0 //def VBOX_WITH_VMMR0_DISABLE_PREEMPTION
2774 if ( RTThreadPreemptIsPendingTrusty()
2775 && !RTThreadPreemptIsPending(NIL_RTTHREAD))
2776 {
2777 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
2778 goto ResumeExecution;
2779 }
2780#endif
2781 /* External interrupt; leave to allow it to be dispatched again. */
2782 rc = VINF_EM_RAW_INTERRUPT;
2783 break;
2784 }
2785 STAM_PROFILE_ADV_START(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
2786 switch (VMX_EXIT_INTERRUPTION_INFO_TYPE(intInfo))
2787 {
2788 case VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI: /* Non-maskable interrupt. */
2789 /* External interrupt; leave to allow it to be dispatched again. */
2790 rc = VINF_EM_RAW_INTERRUPT;
2791 break;
2792
2793 case VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT: /* External hardware interrupt. */
2794 AssertFailed(); /* can't come here; fails the first check. */
2795 break;
2796
2797 case VMX_EXIT_INTERRUPTION_INFO_TYPE_DBEXCPT: /* Unknown why we get this type for #DB */
2798 case VMX_EXIT_INTERRUPTION_INFO_TYPE_SWEXCPT: /* Software exception. (#BP or #OF) */
2799 Assert(vector == 1 || vector == 3 || vector == 4);
2800 /* no break */
2801 case VMX_EXIT_INTERRUPTION_INFO_TYPE_HWEXCPT: /* Hardware exception. */
2802 Log2(("Hardware/software interrupt %d\n", vector));
2803 switch (vector)
2804 {
2805 case X86_XCPT_NM:
2806 {
2807 Log(("#NM fault at %RGv error code %x\n", (RTGCPTR)pCtx->rip, errCode));
2808
2809 /** @todo don't intercept #NM exceptions anymore when we've activated the guest FPU state. */
2810 /* If we sync the FPU/XMM state on-demand, then we can continue execution as if nothing has happened. */
2811 rc = CPUMR0LoadGuestFPU(pVM, pVCpu, pCtx);
2812 if (rc == VINF_SUCCESS)
2813 {
2814 Assert(CPUMIsGuestFPUStateActive(pVCpu));
2815
2816 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitShadowNM);
2817
2818 /* Continue execution. */
2819 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_GUEST_CR0;
2820
2821 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
2822 goto ResumeExecution;
2823 }
2824
2825 Log(("Forward #NM fault to the guest\n"));
2826 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestNM);
2827 rc = VMXR0InjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), cbInstr, 0);
2828 AssertRC(rc);
2829 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
2830 goto ResumeExecution;
2831 }
2832
2833 case X86_XCPT_PF: /* Page fault */
2834 {
2835#ifdef DEBUG
2836 if (pVM->hwaccm.s.fNestedPaging)
2837 { /* A genuine pagefault.
2838 * Forward the trap to the guest by injecting the exception and resuming execution.
2839 */
2840 Log(("Guest page fault at %RGv cr2=%RGv error code %x rsp=%RGv\n", (RTGCPTR)pCtx->rip, exitQualification, errCode, (RTGCPTR)pCtx->rsp));
2841
2842 Assert(CPUMIsGuestInPagedProtectedModeEx(pCtx));
2843
2844 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestPF);
2845
2846 /* Now we must update CR2. */
2847 pCtx->cr2 = exitQualification;
2848 rc = VMXR0InjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), cbInstr, errCode);
2849 AssertRC(rc);
2850
2851 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
2852 goto ResumeExecution;
2853 }
2854#endif
2855 Assert(!pVM->hwaccm.s.fNestedPaging);
2856
2857#ifdef VBOX_HWACCM_WITH_GUEST_PATCHING
2858 /* Shortcut for APIC TPR reads and writes; 32 bits guests only */
2859 if ( pVM->hwaccm.s.fTRPPatchingAllowed
2860 && pVM->hwaccm.s.pGuestPatchMem
2861 && (exitQualification & 0xfff) == 0x080
2862 && !(errCode & X86_TRAP_PF_P) /* not present */
2863 && CPUMGetGuestCPL(pVCpu, CPUMCTX2CORE(pCtx)) == 0
2864 && !CPUMIsGuestInLongModeEx(pCtx)
2865 && pVM->hwaccm.s.cPatches < RT_ELEMENTS(pVM->hwaccm.s.aPatches))
2866 {
2867 RTGCPHYS GCPhysApicBase, GCPhys;
2868 PDMApicGetBase(pVM, &GCPhysApicBase); /* @todo cache this */
2869 GCPhysApicBase &= PAGE_BASE_GC_MASK;
2870
2871 rc = PGMGstGetPage(pVCpu, (RTGCPTR)exitQualification, NULL, &GCPhys);
2872 if ( rc == VINF_SUCCESS
2873 && GCPhys == GCPhysApicBase)
2874 {
2875 /* Only attempt to patch the instruction once. */
2876 PHWACCMTPRPATCH pPatch = (PHWACCMTPRPATCH)RTAvloU32Get(&pVM->hwaccm.s.PatchTree, (AVLOU32KEY)pCtx->eip);
2877 if (!pPatch)
2878 {
2879 rc = VINF_EM_HWACCM_PATCH_TPR_INSTR;
2880 break;
2881 }
2882 }
2883 }
2884#endif
2885
2886 Log2(("Page fault at %RGv error code %x\n", exitQualification, errCode));
2887 /* Exit qualification contains the linear address of the page fault. */
2888 TRPMAssertTrap(pVCpu, X86_XCPT_PF, TRPM_TRAP);
2889 TRPMSetErrorCode(pVCpu, errCode);
2890 TRPMSetFaultAddress(pVCpu, exitQualification);
2891
2892 /* Shortcut for APIC TPR reads and writes. */
2893 if ( (exitQualification & 0xfff) == 0x080
2894 && !(errCode & X86_TRAP_PF_P) /* not present */
2895 && fSetupTPRCaching
2896 && (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC))
2897 {
2898 RTGCPHYS GCPhysApicBase, GCPhys;
2899 PDMApicGetBase(pVM, &GCPhysApicBase); /* @todo cache this */
2900 GCPhysApicBase &= PAGE_BASE_GC_MASK;
2901
2902 rc = PGMGstGetPage(pVCpu, (RTGCPTR)exitQualification, NULL, &GCPhys);
2903 if ( rc == VINF_SUCCESS
2904 && GCPhys == GCPhysApicBase)
2905 {
2906 Log(("Enable VT-x virtual APIC access filtering\n"));
2907 rc = IOMMMIOMapMMIOHCPage(pVM, GCPhysApicBase, pVM->hwaccm.s.vmx.pAPICPhys, X86_PTE_RW | X86_PTE_P);
2908 AssertRC(rc);
2909 }
2910 }
2911
2912 /* Forward it to our trap handler first, in case our shadow pages are out of sync. */
2913 rc = PGMTrap0eHandler(pVCpu, errCode, CPUMCTX2CORE(pCtx), (RTGCPTR)exitQualification);
2914 Log2(("PGMTrap0eHandler %RGv returned %Rrc\n", (RTGCPTR)pCtx->rip, rc));
2915
2916 if (rc == VINF_SUCCESS)
2917 { /* We've successfully synced our shadow pages, so let's just continue execution. */
2918 Log2(("Shadow page fault at %RGv cr2=%RGv error code %x\n", (RTGCPTR)pCtx->rip, exitQualification ,errCode));
2919 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitShadowPF);
2920
2921 TRPMResetTrap(pVCpu);
2922 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
2923 goto ResumeExecution;
2924 }
2925 else
2926 if (rc == VINF_EM_RAW_GUEST_TRAP)
2927 { /* A genuine pagefault.
2928 * Forward the trap to the guest by injecting the exception and resuming execution.
2929 */
2930 Log2(("Forward page fault to the guest\n"));
2931
2932 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestPF);
2933 /* The error code might have been changed. */
2934 errCode = TRPMGetErrorCode(pVCpu);
2935
2936 TRPMResetTrap(pVCpu);
2937
2938 /* Now we must update CR2. */
2939 pCtx->cr2 = exitQualification;
2940 rc = VMXR0InjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), cbInstr, errCode);
2941 AssertRC(rc);
2942
2943 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
2944 goto ResumeExecution;
2945 }
2946#ifdef VBOX_STRICT
2947 if (rc != VINF_EM_RAW_EMULATE_INSTR && rc != VINF_EM_RAW_EMULATE_IO_BLOCK)
2948 Log2(("PGMTrap0eHandler failed with %d\n", rc));
2949#endif
2950 /* Need to go back to the recompiler to emulate the instruction. */
2951 TRPMResetTrap(pVCpu);
2952 break;
2953 }
2954
2955 case X86_XCPT_MF: /* Floating point exception. */
2956 {
2957 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestMF);
2958 if (!(pCtx->cr0 & X86_CR0_NE))
2959 {
2960 /* old style FPU error reporting needs some extra work. */
2961 /** @todo don't fall back to the recompiler, but do it manually. */
2962 rc = VINF_EM_RAW_EMULATE_INSTR;
2963 break;
2964 }
2965 Log(("Trap %x at %04X:%RGv\n", vector, pCtx->cs, (RTGCPTR)pCtx->rip));
2966 rc = VMXR0InjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), cbInstr, errCode);
2967 AssertRC(rc);
2968
2969 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
2970 goto ResumeExecution;
2971 }
2972
2973 case X86_XCPT_DB: /* Debug exception. */
2974 {
2975 uint64_t uDR6;
2976
2977 /* DR6, DR7.GD and IA32_DEBUGCTL.LBR are not updated yet.
2978 *
2979 * Exit qualification bits:
2980 * 3:0 B0-B3 which breakpoint condition was met
2981 * 12:4 Reserved (0)
2982 * 13 BD - debug register access detected
2983 * 14 BS - single step execution or branch taken
2984 * 63:15 Reserved (0)
2985 */
2986 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestDB);
2987
2988 /* Note that we don't support guest and host-initiated debugging at the same time. */
2989
2990 uDR6 = X86_DR6_INIT_VAL;
2991 uDR6 |= (exitQualification & (X86_DR6_B0|X86_DR6_B1|X86_DR6_B2|X86_DR6_B3|X86_DR6_BD|X86_DR6_BS));
2992 rc = DBGFRZTrap01Handler(pVM, pVCpu, CPUMCTX2CORE(pCtx), uDR6);
2993 if (rc == VINF_EM_RAW_GUEST_TRAP)
2994 {
2995 /* Update DR6 here. */
2996 pCtx->dr[6] = uDR6;
2997
2998 /* Resync DR6 if the debug state is active. */
2999 if (CPUMIsGuestDebugStateActive(pVCpu))
3000 ASMSetDR6(pCtx->dr[6]);
3001
3002 /* X86_DR7_GD will be cleared if drx accesses should be trapped inside the guest. */
3003 pCtx->dr[7] &= ~X86_DR7_GD;
3004
3005 /* Paranoia. */
3006 pCtx->dr[7] &= 0xffffffff; /* upper 32 bits reserved */
3007 pCtx->dr[7] &= ~(RT_BIT(11) | RT_BIT(12) | RT_BIT(14) | RT_BIT(15)); /* must be zero */
3008 pCtx->dr[7] |= 0x400; /* must be one */
3009
3010 /* Resync DR7 */
3011 rc = VMXWriteVMCS64(VMX_VMCS64_GUEST_DR7, pCtx->dr[7]);
3012 AssertRC(rc);
3013
3014 Log(("Trap %x (debug) at %RGv exit qualification %RX64 dr6=%x dr7=%x\n", vector, (RTGCPTR)pCtx->rip, exitQualification, (uint32_t)pCtx->dr[6], (uint32_t)pCtx->dr[7]));
3015 rc = VMXR0InjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), cbInstr, errCode);
3016 AssertRC(rc);
3017
3018 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3019 goto ResumeExecution;
3020 }
3021 /* Return to ring 3 to deal with the debug exit code. */
3022 Log(("Debugger hardware BP at %04x:%RGv (rc=%Rrc)\n", pCtx->cs, pCtx->rip, rc));
3023 break;
3024 }
3025
3026 case X86_XCPT_BP: /* Breakpoint. */
3027 {
3028 rc = DBGFRZTrap03Handler(pVM, pVCpu, CPUMCTX2CORE(pCtx));
3029 if (rc == VINF_EM_RAW_GUEST_TRAP)
3030 {
3031 Log(("Guest #BP at %04x:%RGv\n", pCtx->cs, pCtx->rip));
3032 rc = VMXR0InjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), cbInstr, errCode);
3033 AssertRC(rc);
3034 goto ResumeExecution;
3035 }
3036 if (rc == VINF_SUCCESS)
3037 goto ResumeExecution;
3038 Log(("Debugger BP at %04x:%RGv (rc=%Rrc)\n", pCtx->cs, pCtx->rip, rc));
3039 break;
3040 }
3041
3042 case X86_XCPT_GP: /* General protection failure exception.*/
3043 {
3044 uint32_t cbOp;
3045 uint32_t cbSize;
3046 PDISCPUSTATE pDis = &pVCpu->hwaccm.s.DisState;
3047
3048 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestGP);
3049#ifdef VBOX_STRICT
3050 if ( !CPUMIsGuestInRealModeEx(pCtx)
3051 || !pVM->hwaccm.s.vmx.pRealModeTSS)
3052 {
3053 Log(("Trap %x at %04X:%RGv errorCode=%x\n", vector, pCtx->cs, (RTGCPTR)pCtx->rip, errCode));
3054 rc = VMXR0InjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), cbInstr, errCode);
3055 AssertRC(rc);
3056 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3057 goto ResumeExecution;
3058 }
3059#endif
3060 Assert(CPUMIsGuestInRealModeEx(pCtx));
3061
3062 LogFlow(("Real mode X86_XCPT_GP instruction emulation at %x:%RGv\n", pCtx->cs, (RTGCPTR)pCtx->rip));
3063
3064 rc = EMInterpretDisasOne(pVM, pVCpu, CPUMCTX2CORE(pCtx), pDis, &cbOp);
3065 if (RT_SUCCESS(rc))
3066 {
3067 bool fUpdateRIP = true;
3068
3069 Assert(cbOp == pDis->opsize);
3070 switch (pDis->pCurInstr->opcode)
3071 {
3072 case OP_CLI:
3073 pCtx->eflags.Bits.u1IF = 0;
3074 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitCli);
3075 break;
3076
3077 case OP_STI:
3078 pCtx->eflags.Bits.u1IF = 1;
3079 EMSetInhibitInterruptsPC(pVCpu, pCtx->rip + pDis->opsize);
3080 Assert(VMCPU_FF_ISSET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
3081 rc = VMXWriteVMCS(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
3082 AssertRC(rc);
3083 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitSti);
3084 break;
3085
3086 case OP_HLT:
3087 fUpdateRIP = false;
3088 rc = VINF_EM_HALT;
3089 pCtx->rip += pDis->opsize;
3090 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitHlt);
3091 break;
3092
3093 case OP_POPF:
3094 {
3095 RTGCPTR GCPtrStack;
3096 uint32_t cbParm;
3097 uint32_t uMask;
3098 X86EFLAGS eflags;
3099
3100 if (pDis->prefix & PREFIX_OPSIZE)
3101 {
3102 cbParm = 4;
3103 uMask = 0xffffffff;
3104 }
3105 else
3106 {
3107 cbParm = 2;
3108 uMask = 0xffff;
3109 }
3110
3111 rc = SELMToFlatEx(pVM, DIS_SELREG_SS, CPUMCTX2CORE(pCtx), pCtx->esp & uMask, 0, &GCPtrStack);
3112 if (RT_FAILURE(rc))
3113 {
3114 rc = VERR_EM_INTERPRETER;
3115 break;
3116 }
3117 eflags.u = 0;
3118 rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &eflags.u, cbParm);
3119 if (RT_FAILURE(rc))
3120 {
3121 rc = VERR_EM_INTERPRETER;
3122 break;
3123 }
3124 LogFlow(("POPF %x -> %RGv mask=%x\n", eflags.u, pCtx->rsp, uMask));
3125 pCtx->eflags.u = (pCtx->eflags.u & ~(X86_EFL_POPF_BITS & uMask)) | (eflags.u & X86_EFL_POPF_BITS & uMask);
3126 /* RF cleared when popped in real mode; see pushf description in AMD manual. */
3127 pCtx->eflags.Bits.u1RF = 0;
3128 pCtx->esp += cbParm;
3129 pCtx->esp &= uMask;
3130
3131 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitPopf);
3132 break;
3133 }
3134
3135 case OP_PUSHF:
3136 {
3137 RTGCPTR GCPtrStack;
3138 uint32_t cbParm;
3139 uint32_t uMask;
3140 X86EFLAGS eflags;
3141
3142 if (pDis->prefix & PREFIX_OPSIZE)
3143 {
3144 cbParm = 4;
3145 uMask = 0xffffffff;
3146 }
3147 else
3148 {
3149 cbParm = 2;
3150 uMask = 0xffff;
3151 }
3152
3153 rc = SELMToFlatEx(pVM, DIS_SELREG_SS, CPUMCTX2CORE(pCtx), (pCtx->esp - cbParm) & uMask, 0, &GCPtrStack);
3154 if (RT_FAILURE(rc))
3155 {
3156 rc = VERR_EM_INTERPRETER;
3157 break;
3158 }
3159 eflags = pCtx->eflags;
3160 /* RF & VM cleared when pushed in real mode; see pushf description in AMD manual. */
3161 eflags.Bits.u1RF = 0;
3162 eflags.Bits.u1VM = 0;
3163
3164 rc = PGMPhysWrite(pVM, (RTGCPHYS)GCPtrStack, &eflags.u, cbParm);
3165 if (RT_FAILURE(rc))
3166 {
3167 rc = VERR_EM_INTERPRETER;
3168 break;
3169 }
3170 LogFlow(("PUSHF %x -> %RGv\n", eflags.u, GCPtrStack));
3171 pCtx->esp -= cbParm;
3172 pCtx->esp &= uMask;
3173 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitPushf);
3174 break;
3175 }
3176
3177 case OP_IRET:
3178 {
3179 RTGCPTR GCPtrStack;
3180 uint32_t uMask = 0xffff;
3181 uint16_t aIretFrame[3];
3182
3183 if (pDis->prefix & (PREFIX_OPSIZE | PREFIX_ADDRSIZE))
3184 {
3185 rc = VERR_EM_INTERPRETER;
3186 break;
3187 }
3188
3189 rc = SELMToFlatEx(pVM, DIS_SELREG_SS, CPUMCTX2CORE(pCtx), pCtx->esp & uMask, 0, &GCPtrStack);
3190 if (RT_FAILURE(rc))
3191 {
3192 rc = VERR_EM_INTERPRETER;
3193 break;
3194 }
3195 rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &aIretFrame[0], sizeof(aIretFrame));
3196 if (RT_FAILURE(rc))
3197 {
3198 rc = VERR_EM_INTERPRETER;
3199 break;
3200 }
3201 pCtx->ip = aIretFrame[0];
3202 pCtx->cs = aIretFrame[1];
3203 pCtx->csHid.u64Base = pCtx->cs << 4;
3204 pCtx->eflags.u = (pCtx->eflags.u & ~(X86_EFL_POPF_BITS & uMask)) | (aIretFrame[2] & X86_EFL_POPF_BITS & uMask);
3205 pCtx->sp += sizeof(aIretFrame);
3206
3207 LogFlow(("iret to %04x:%x\n", pCtx->cs, pCtx->ip));
3208 fUpdateRIP = false;
3209 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitIret);
3210 break;
3211 }
3212
3213 case OP_INT:
3214 {
3215 uint32_t intInfo2;
3216
3217 LogFlow(("Realmode: INT %x\n", pDis->param1.parval & 0xff));
3218 intInfo2 = pDis->param1.parval & 0xff;
3219 intInfo2 |= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
3220 intInfo2 |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
3221
3222 rc = VMXR0InjectEvent(pVM, pVCpu, pCtx, intInfo2, cbOp, 0);
3223 AssertRC(rc);
3224 fUpdateRIP = false;
3225 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitInt);
3226 break;
3227 }
3228
3229 case OP_INTO:
3230 {
3231 if (pCtx->eflags.Bits.u1OF)
3232 {
3233 uint32_t intInfo2;
3234
3235 LogFlow(("Realmode: INTO\n"));
3236 intInfo2 = X86_XCPT_OF;
3237 intInfo2 |= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
3238 intInfo2 |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
3239
3240 rc = VMXR0InjectEvent(pVM, pVCpu, pCtx, intInfo2, cbOp, 0);
3241 AssertRC(rc);
3242 fUpdateRIP = false;
3243 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitInt);
3244 }
3245 break;
3246 }
3247
3248 case OP_INT3:
3249 {
3250 uint32_t intInfo2;
3251
3252 LogFlow(("Realmode: INT 3\n"));
3253 intInfo2 = 3;
3254 intInfo2 |= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
3255 intInfo2 |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
3256
3257 rc = VMXR0InjectEvent(pVM, pVCpu, pCtx, intInfo2, cbOp, 0);
3258 AssertRC(rc);
3259 fUpdateRIP = false;
3260 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitInt);
3261 break;
3262 }
3263
3264 default:
3265 rc = EMInterpretInstructionCPU(pVM, pVCpu, pDis, CPUMCTX2CORE(pCtx), 0, &cbSize);
3266 break;
3267 }
3268
3269 if (rc == VINF_SUCCESS)
3270 {
3271 if (fUpdateRIP)
3272 pCtx->rip += cbOp; /* Move on to the next instruction. */
3273
3274 /* lidt, lgdt can end up here. In the future crx changes as well. Just reload the whole context to be done with it. */
3275 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_ALL;
3276
3277 /* Only resume if successful. */
3278 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3279 goto ResumeExecution;
3280 }
3281 }
3282 else
3283 rc = VERR_EM_INTERPRETER;
3284
3285 AssertMsg(rc == VERR_EM_INTERPRETER || rc == VINF_PGM_CHANGE_MODE || rc == VINF_EM_HALT, ("Unexpected rc=%Rrc\n", rc));
3286 break;
3287 }
3288
3289#ifdef VBOX_STRICT
3290 case X86_XCPT_XF: /* SIMD exception. */
3291 case X86_XCPT_DE: /* Divide error. */
3292 case X86_XCPT_UD: /* Unknown opcode exception. */
3293 case X86_XCPT_SS: /* Stack segment exception. */
3294 case X86_XCPT_NP: /* Segment not present exception. */
3295 {
3296 switch(vector)
3297 {
3298 case X86_XCPT_DE:
3299 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestDE);
3300 break;
3301 case X86_XCPT_UD:
3302 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestUD);
3303 break;
3304 case X86_XCPT_SS:
3305 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestSS);
3306 break;
3307 case X86_XCPT_NP:
3308 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestNP);
3309 break;
3310 }
3311
3312 Log(("Trap %x at %04X:%RGv\n", vector, pCtx->cs, (RTGCPTR)pCtx->rip));
3313 rc = VMXR0InjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), cbInstr, errCode);
3314 AssertRC(rc);
3315
3316 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3317 goto ResumeExecution;
3318 }
3319#endif
3320 default:
3321 if ( CPUMIsGuestInRealModeEx(pCtx)
3322 && pVM->hwaccm.s.vmx.pRealModeTSS)
3323 {
3324 Log(("Real Mode Trap %x at %04x:%04X error code %x\n", vector, pCtx->cs, pCtx->eip, errCode));
3325 rc = VMXR0InjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), cbInstr, errCode);
3326 AssertRC(rc);
3327
3328 /* Go back to ring 3 in case of a triple fault. */
3329 if ( vector == X86_XCPT_DF
3330 && rc == VINF_EM_RESET)
3331 break;
3332
3333 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3334 goto ResumeExecution;
3335 }
3336 AssertMsgFailed(("Unexpected vm-exit caused by exception %x\n", vector));
3337 rc = VERR_VMX_UNEXPECTED_EXCEPTION;
3338 break;
3339 } /* switch (vector) */
3340
3341 break;
3342
3343 default:
3344 rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_CODE;
3345 AssertMsgFailed(("Unexpected interuption code %x\n", intInfo));
3346 break;
3347 }
3348
3349 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3350 break;
3351 }
3352
3353 case VMX_EXIT_EPT_VIOLATION: /* 48 EPT violation. An attempt to access memory with a guest-physical address was disallowed by the configuration of the EPT paging structures. */
3354 {
3355 RTGCPHYS GCPhys;
3356
3357 Assert(pVM->hwaccm.s.fNestedPaging);
3358
3359 rc = VMXReadVMCS64(VMX_VMCS_EXIT_PHYS_ADDR_FULL, &GCPhys);
3360 AssertRC(rc);
3361 Assert(((exitQualification >> 7) & 3) != 2);
3362
3363 /* Determine the kind of violation. */
3364 errCode = 0;
3365 if (exitQualification & VMX_EXIT_QUALIFICATION_EPT_INSTR_FETCH)
3366 errCode |= X86_TRAP_PF_ID;
3367
3368 if (exitQualification & VMX_EXIT_QUALIFICATION_EPT_DATA_WRITE)
3369 errCode |= X86_TRAP_PF_RW;
3370
3371 /* If the page is present, then it's a page level protection fault. */
3372 if (exitQualification & VMX_EXIT_QUALIFICATION_EPT_ENTRY_PRESENT)
3373 {
3374 errCode |= X86_TRAP_PF_P;
3375 }
3376 else
3377 {
3378 /* Shortcut for APIC TPR reads and writes. */
3379 if ( (GCPhys & 0xfff) == 0x080
3380 && GCPhys > 0x1000000 /* to skip VGA frame buffer accesses */
3381 && fSetupTPRCaching
3382 && (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC))
3383 {
3384 RTGCPHYS GCPhysApicBase;
3385 PDMApicGetBase(pVM, &GCPhysApicBase); /* @todo cache this */
3386 GCPhysApicBase &= PAGE_BASE_GC_MASK;
3387 if (GCPhys == GCPhysApicBase + 0x80)
3388 {
3389 Log(("Enable VT-x virtual APIC access filtering\n"));
3390 rc = IOMMMIOMapMMIOHCPage(pVM, GCPhysApicBase, pVM->hwaccm.s.vmx.pAPICPhys, X86_PTE_RW | X86_PTE_P);
3391 AssertRC(rc);
3392 }
3393 }
3394 }
3395 Log(("EPT Page fault %x at %RGp error code %x\n", (uint32_t)exitQualification, GCPhys, errCode));
3396
3397 /* GCPhys contains the guest physical address of the page fault. */
3398 TRPMAssertTrap(pVCpu, X86_XCPT_PF, TRPM_TRAP);
3399 TRPMSetErrorCode(pVCpu, errCode);
3400 TRPMSetFaultAddress(pVCpu, GCPhys);
3401
3402 /* Handle the pagefault trap for the nested shadow table. */
3403 rc = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, errCode, CPUMCTX2CORE(pCtx), GCPhys);
3404 Log2(("PGMR0Trap0eHandlerNestedPaging %RGv returned %Rrc\n", (RTGCPTR)pCtx->rip, rc));
3405 if (rc == VINF_SUCCESS)
3406 { /* We've successfully synced our shadow pages, so let's just continue execution. */
3407 Log2(("Shadow page fault at %RGv cr2=%RGp error code %x\n", (RTGCPTR)pCtx->rip, exitQualification , errCode));
3408 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitReasonNPF);
3409
3410 TRPMResetTrap(pVCpu);
3411 goto ResumeExecution;
3412 }
3413
3414#ifdef VBOX_STRICT
3415 if (rc != VINF_EM_RAW_EMULATE_INSTR)
3416 LogFlow(("PGMTrap0eHandlerNestedPaging failed with %d\n", rc));
3417#endif
3418 /* Need to go back to the recompiler to emulate the instruction. */
3419 TRPMResetTrap(pVCpu);
3420 break;
3421 }
3422
3423 case VMX_EXIT_EPT_MISCONFIG:
3424 {
3425 RTGCPHYS GCPhys;
3426
3427 Assert(pVM->hwaccm.s.fNestedPaging);
3428
3429 rc = VMXReadVMCS64(VMX_VMCS_EXIT_PHYS_ADDR_FULL, &GCPhys);
3430 AssertRC(rc);
3431
3432 Log(("VMX_EXIT_EPT_MISCONFIG for %RGp\n", GCPhys));
3433 break;
3434 }
3435
3436 case VMX_EXIT_IRQ_WINDOW: /* 7 Interrupt window. */
3437 /* Clear VM-exit on IF=1 change. */
3438 LogFlow(("VMX_EXIT_IRQ_WINDOW %RGv pending=%d IF=%d\n", (RTGCPTR)pCtx->rip, VMCPU_FF_ISPENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC|VMCPU_FF_INTERRUPT_PIC)), pCtx->eflags.Bits.u1IF));
3439 pVCpu->hwaccm.s.vmx.proc_ctls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_IRQ_WINDOW_EXIT;
3440 rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
3441 AssertRC(rc);
3442 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitIrqWindow);
3443 goto ResumeExecution; /* we check for pending guest interrupts there */
3444
3445 case VMX_EXIT_WBINVD: /* 54 Guest software attempted to execute WBINVD. (conditional) */
3446 case VMX_EXIT_INVD: /* 13 Guest software attempted to execute INVD. (unconditional) */
3447 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitInvd);
3448 /* Skip instruction and continue directly. */
3449 pCtx->rip += cbInstr;
3450 /* Continue execution.*/
3451 goto ResumeExecution;
3452
3453 case VMX_EXIT_CPUID: /* 10 Guest software attempted to execute CPUID. */
3454 {
3455 Log2(("VMX: Cpuid %x\n", pCtx->eax));
3456 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitCpuid);
3457 rc = EMInterpretCpuId(pVM, pVCpu, CPUMCTX2CORE(pCtx));
3458 if (rc == VINF_SUCCESS)
3459 {
3460 /* Update EIP and continue execution. */
3461 Assert(cbInstr == 2);
3462 pCtx->rip += cbInstr;
3463 goto ResumeExecution;
3464 }
3465 AssertMsgFailed(("EMU: cpuid failed with %Rrc\n", rc));
3466 rc = VINF_EM_RAW_EMULATE_INSTR;
3467 break;
3468 }
3469
3470 case VMX_EXIT_RDPMC: /* 15 Guest software attempted to execute RDPMC. */
3471 {
3472 Log2(("VMX: Rdpmc %x\n", pCtx->ecx));
3473 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitRdpmc);
3474 rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pCtx));
3475 if (rc == VINF_SUCCESS)
3476 {
3477 /* Update EIP and continue execution. */
3478 Assert(cbInstr == 2);
3479 pCtx->rip += cbInstr;
3480 goto ResumeExecution;
3481 }
3482 rc = VINF_EM_RAW_EMULATE_INSTR;
3483 break;
3484 }
3485
3486 case VMX_EXIT_RDTSC: /* 16 Guest software attempted to execute RDTSC. */
3487 {
3488 Log2(("VMX: Rdtsc\n"));
3489 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitRdtsc);
3490 rc = EMInterpretRdtsc(pVM, pVCpu, CPUMCTX2CORE(pCtx));
3491 if (rc == VINF_SUCCESS)
3492 {
3493 /* Update EIP and continue execution. */
3494 Assert(cbInstr == 2);
3495 pCtx->rip += cbInstr;
3496 goto ResumeExecution;
3497 }
3498 rc = VINF_EM_RAW_EMULATE_INSTR;
3499 break;
3500 }
3501
3502 case VMX_EXIT_INVPG: /* 14 Guest software attempted to execute INVPG. */
3503 {
3504 Log2(("VMX: invlpg\n"));
3505 Assert(!pVM->hwaccm.s.fNestedPaging);
3506
3507 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitInvpg);
3508 rc = EMInterpretInvlpg(pVM, pVCpu, CPUMCTX2CORE(pCtx), exitQualification);
3509 if (rc == VINF_SUCCESS)
3510 {
3511 /* Update EIP and continue execution. */
3512 pCtx->rip += cbInstr;
3513 goto ResumeExecution;
3514 }
3515 AssertMsg(rc == VERR_EM_INTERPRETER, ("EMU: invlpg %RGv failed with %Rrc\n", exitQualification, rc));
3516 break;
3517 }
3518
3519 case VMX_EXIT_MONITOR: /* 39 Guest software attempted to execute MONITOR. */
3520 {
3521 Log2(("VMX: monitor\n"));
3522
3523 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitMonitor);
3524 rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pCtx));
3525 if (rc == VINF_SUCCESS)
3526 {
3527 /* Update EIP and continue execution. */
3528 pCtx->rip += cbInstr;
3529 goto ResumeExecution;
3530 }
3531 AssertMsg(rc == VERR_EM_INTERPRETER, ("EMU: monitor failed with %Rrc\n", rc));
3532 break;
3533 }
3534
3535 case VMX_EXIT_WRMSR: /* 32 WRMSR. Guest software attempted to execute WRMSR. */
3536 /* When an interrupt is pending, we'll let MSR_K8_LSTAR writes fault in our TPR patch code. */
3537 if ( pVM->hwaccm.s.fTPRPatchingActive
3538 && pCtx->ecx == MSR_K8_LSTAR)
3539 {
3540 Assert(!CPUMIsGuestInLongModeEx(pCtx));
3541 if ((pCtx->eax & 0xff) != u8LastTPR)
3542 {
3543 Log(("VMX: Faulting MSR_K8_LSTAR write with new TPR value %x\n", pCtx->eax & 0xff));
3544
3545 /* Our patch code uses LSTAR for TPR caching. */
3546 rc = PDMApicSetTPR(pVCpu, pCtx->eax & 0xff);
3547 AssertRC(rc);
3548 }
3549
3550 /* Skip the instruction and continue. */
3551 pCtx->rip += cbInstr; /* wrmsr = [0F 30] */
3552
3553 /* Only resume if successful. */
3554 goto ResumeExecution;
3555 }
3556 /* no break */
3557 case VMX_EXIT_RDMSR: /* 31 RDMSR. Guest software attempted to execute RDMSR. */
3558 {
3559 uint32_t cbSize;
3560
3561 STAM_COUNTER_INC((exitReason == VMX_EXIT_RDMSR) ? &pVCpu->hwaccm.s.StatExitRdmsr : &pVCpu->hwaccm.s.StatExitWrmsr);
3562
3563 /* Note: the intel manual claims there's a REX version of RDMSR that's slightly different, so we play safe by completely disassembling the instruction. */
3564 Log2(("VMX: %s\n", (exitReason == VMX_EXIT_RDMSR) ? "rdmsr" : "wrmsr"));
3565 rc = EMInterpretInstruction(pVM, pVCpu, CPUMCTX2CORE(pCtx), 0, &cbSize);
3566 if (rc == VINF_SUCCESS)
3567 {
3568 /* EIP has been updated already. */
3569
3570 /* Only resume if successful. */
3571 goto ResumeExecution;
3572 }
3573 AssertMsg(rc == VERR_EM_INTERPRETER, ("EMU: %s failed with %Rrc\n", (exitReason == VMX_EXIT_RDMSR) ? "rdmsr" : "wrmsr", rc));
3574 break;
3575 }
3576
3577 case VMX_EXIT_CRX_MOVE: /* 28 Control-register accesses. */
3578 {
3579 STAM_PROFILE_ADV_START(&pVCpu->hwaccm.s.StatExit2Sub2, y2);
3580
3581 switch (VMX_EXIT_QUALIFICATION_CRX_ACCESS(exitQualification))
3582 {
3583 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_WRITE:
3584 Log2(("VMX: %RGv mov cr%d, x\n", (RTGCPTR)pCtx->rip, VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification)));
3585 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitCRxWrite[VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification)]);
3586 rc = EMInterpretCRxWrite(pVM, pVCpu, CPUMCTX2CORE(pCtx),
3587 VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification),
3588 VMX_EXIT_QUALIFICATION_CRX_GENREG(exitQualification));
3589
3590 switch (VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification))
3591 {
3592 case 0:
3593 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_GUEST_CR0 | HWACCM_CHANGED_GUEST_CR3;
3594 break;
3595 case 2:
3596 break;
3597 case 3:
3598 Assert(!pVM->hwaccm.s.fNestedPaging || !CPUMIsGuestInPagedProtectedModeEx(pCtx));
3599 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_GUEST_CR3;
3600 break;
3601 case 4:
3602 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_GUEST_CR4;
3603 break;
3604 case 8:
3605 /* CR8 contains the APIC TPR */
3606 Assert(!(pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW));
3607 break;
3608
3609 default:
3610 AssertFailed();
3611 break;
3612 }
3613 break;
3614
3615 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ:
3616 Log2(("VMX: mov x, crx\n"));
3617 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitCRxRead[VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification)]);
3618
3619 Assert(!pVM->hwaccm.s.fNestedPaging || !CPUMIsGuestInPagedProtectedModeEx(pCtx) || VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification) != USE_REG_CR3);
3620
3621 /* CR8 reads only cause an exit when the TPR shadow feature isn't present. */
3622 Assert(VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification) != 8 || !(pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW));
3623
3624 rc = EMInterpretCRxRead(pVM, pVCpu, CPUMCTX2CORE(pCtx),
3625 VMX_EXIT_QUALIFICATION_CRX_GENREG(exitQualification),
3626 VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification));
3627 break;
3628
3629 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_CLTS:
3630 Log2(("VMX: clts\n"));
3631 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitCLTS);
3632 rc = EMInterpretCLTS(pVM, pVCpu);
3633 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_GUEST_CR0;
3634 break;
3635
3636 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_LMSW:
3637 Log2(("VMX: lmsw %x\n", VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(exitQualification)));
3638 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitLMSW);
3639 rc = EMInterpretLMSW(pVM, pVCpu, CPUMCTX2CORE(pCtx), VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(exitQualification));
3640 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_GUEST_CR0;
3641 break;
3642 }
3643
3644 /* Update EIP if no error occurred. */
3645 if (RT_SUCCESS(rc))
3646 pCtx->rip += cbInstr;
3647
3648 if (rc == VINF_SUCCESS)
3649 {
3650 /* Only resume if successful. */
3651 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub2, y2);
3652 goto ResumeExecution;
3653 }
3654 Assert(rc == VERR_EM_INTERPRETER || rc == VINF_PGM_CHANGE_MODE || rc == VINF_PGM_SYNC_CR3);
3655 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub2, y2);
3656 break;
3657 }
3658
3659 case VMX_EXIT_DRX_MOVE: /* 29 Debug-register accesses. */
3660 {
3661 if ( !DBGFIsStepping(pVCpu)
3662 && !CPUMIsHyperDebugStateActive(pVCpu))
3663 {
3664 /* Disable drx move intercepts. */
3665 pVCpu->hwaccm.s.vmx.proc_ctls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT;
3666 rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
3667 AssertRC(rc);
3668
3669 /* Save the host and load the guest debug state. */
3670 rc = CPUMR0LoadGuestDebugState(pVM, pVCpu, pCtx, true /* include DR6 */);
3671 AssertRC(rc);
3672
3673#ifdef LOG_ENABLED
3674 if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(exitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
3675 Log(("VMX_EXIT_DRX_MOVE: write DR%d genreg %d\n", VMX_EXIT_QUALIFICATION_DRX_REGISTER(exitQualification), VMX_EXIT_QUALIFICATION_DRX_GENREG(exitQualification)));
3676 else
3677 Log(("VMX_EXIT_DRX_MOVE: read DR%d\n", VMX_EXIT_QUALIFICATION_DRX_REGISTER(exitQualification)));
3678#endif
3679
3680#ifdef VBOX_WITH_STATISTICS
3681 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatDRxContextSwitch);
3682 if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(exitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
3683 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitDRxWrite);
3684 else
3685 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitDRxRead);
3686#endif
3687
3688 goto ResumeExecution;
3689 }
3690
3691 /** @todo clear VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT after the first time and restore drx registers afterwards */
3692 if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(exitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
3693 {
3694 Log2(("VMX: mov drx%d, genreg%d\n", VMX_EXIT_QUALIFICATION_DRX_REGISTER(exitQualification), VMX_EXIT_QUALIFICATION_DRX_GENREG(exitQualification)));
3695 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitDRxWrite);
3696 rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pCtx),
3697 VMX_EXIT_QUALIFICATION_DRX_REGISTER(exitQualification),
3698 VMX_EXIT_QUALIFICATION_DRX_GENREG(exitQualification));
3699 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_GUEST_DEBUG;
3700 Log2(("DR7=%08x\n", pCtx->dr[7]));
3701 }
3702 else
3703 {
3704 Log2(("VMX: mov x, drx\n"));
3705 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitDRxRead);
3706 rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pCtx),
3707 VMX_EXIT_QUALIFICATION_DRX_GENREG(exitQualification),
3708 VMX_EXIT_QUALIFICATION_DRX_REGISTER(exitQualification));
3709 }
3710 /* Update EIP if no error occurred. */
3711 if (RT_SUCCESS(rc))
3712 pCtx->rip += cbInstr;
3713
3714 if (rc == VINF_SUCCESS)
3715 {
3716 /* Only resume if successful. */
3717 goto ResumeExecution;
3718 }
3719 Assert(rc == VERR_EM_INTERPRETER);
3720 break;
3721 }
3722
3723 /* Note: We'll get a #GP if the IO instruction isn't allowed (IOPL or TSS bitmap); no need to double check. */
3724 case VMX_EXIT_PORT_IO: /* 30 I/O instruction. */
3725 {
3726 STAM_PROFILE_ADV_START(&pVCpu->hwaccm.s.StatExit2Sub1, y1);
3727 uint32_t uIOWidth = VMX_EXIT_QUALIFICATION_IO_WIDTH(exitQualification);
3728 uint32_t uPort;
3729 bool fIOWrite = (VMX_EXIT_QUALIFICATION_IO_DIRECTION(exitQualification) == VMX_EXIT_QUALIFICATION_IO_DIRECTION_OUT);
3730
3731 /** @todo necessary to make the distinction? */
3732 if (VMX_EXIT_QUALIFICATION_IO_ENCODING(exitQualification) == VMX_EXIT_QUALIFICATION_IO_ENCODING_DX)
3733 {
3734 uPort = pCtx->edx & 0xffff;
3735 }
3736 else
3737 uPort = VMX_EXIT_QUALIFICATION_IO_PORT(exitQualification); /* Immediate encoding. */
3738
3739 /* paranoia */
3740 if (RT_UNLIKELY(uIOWidth == 2 || uIOWidth >= 4))
3741 {
3742 rc = fIOWrite ? VINF_IOM_HC_IOPORT_WRITE : VINF_IOM_HC_IOPORT_READ;
3743 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub1, y1);
3744 break;
3745 }
3746
3747 uint32_t cbSize = g_aIOSize[uIOWidth];
3748
3749 if (VMX_EXIT_QUALIFICATION_IO_STRING(exitQualification))
3750 {
3751 /* ins/outs */
3752 PDISCPUSTATE pDis = &pVCpu->hwaccm.s.DisState;
3753
3754 /* Disassemble manually to deal with segment prefixes. */
3755 /** @todo VMX_VMCS_EXIT_GUEST_LINEAR_ADDR contains the flat pointer operand of the instruction. */
3756 /** @todo VMX_VMCS32_RO_EXIT_INSTR_INFO also contains segment prefix info. */
3757 rc = EMInterpretDisasOne(pVM, pVCpu, CPUMCTX2CORE(pCtx), pDis, NULL);
3758 if (rc == VINF_SUCCESS)
3759 {
3760 if (fIOWrite)
3761 {
3762 Log2(("IOMInterpretOUTSEx %RGv %x size=%d\n", (RTGCPTR)pCtx->rip, uPort, cbSize));
3763 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitIOStringWrite);
3764 rc = VBOXSTRICTRC_TODO(IOMInterpretOUTSEx(pVM, CPUMCTX2CORE(pCtx), uPort, pDis->prefix, cbSize));
3765 }
3766 else
3767 {
3768 Log2(("IOMInterpretINSEx %RGv %x size=%d\n", (RTGCPTR)pCtx->rip, uPort, cbSize));
3769 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitIOStringRead);
3770 rc = VBOXSTRICTRC_TODO(IOMInterpretINSEx(pVM, CPUMCTX2CORE(pCtx), uPort, pDis->prefix, cbSize));
3771 }
3772 }
3773 else
3774 rc = VINF_EM_RAW_EMULATE_INSTR;
3775 }
3776 else
3777 {
3778 /* normal in/out */
3779 uint32_t uAndVal = g_aIOOpAnd[uIOWidth];
3780
3781 Assert(!VMX_EXIT_QUALIFICATION_IO_REP(exitQualification));
3782
3783 if (fIOWrite)
3784 {
3785 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitIOWrite);
3786 rc = VBOXSTRICTRC_TODO(IOMIOPortWrite(pVM, uPort, pCtx->eax & uAndVal, cbSize));
3787 if (rc == VINF_IOM_HC_IOPORT_WRITE)
3788 HWACCMR0SavePendingIOPortWrite(pVCpu, pCtx->rip, pCtx->rip + cbInstr, uPort, uAndVal, cbSize);
3789 }
3790 else
3791 {
3792 uint32_t u32Val = 0;
3793
3794 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitIORead);
3795 rc = VBOXSTRICTRC_TODO(IOMIOPortRead(pVM, uPort, &u32Val, cbSize));
3796 if (IOM_SUCCESS(rc))
3797 {
3798 /* Write back to the EAX register. */
3799 pCtx->eax = (pCtx->eax & ~uAndVal) | (u32Val & uAndVal);
3800 }
3801 else
3802 if (rc == VINF_IOM_HC_IOPORT_READ)
3803 HWACCMR0SavePendingIOPortRead(pVCpu, pCtx->rip, pCtx->rip + cbInstr, uPort, uAndVal, cbSize);
3804 }
3805 }
3806 /*
3807 * Handled the I/O return codes.
3808 * (The unhandled cases end up with rc == VINF_EM_RAW_EMULATE_INSTR.)
3809 */
3810 if (IOM_SUCCESS(rc))
3811 {
3812 /* Update EIP and continue execution. */
3813 pCtx->rip += cbInstr;
3814 if (RT_LIKELY(rc == VINF_SUCCESS))
3815 {
3816 /* If any IO breakpoints are armed, then we should check if a debug trap needs to be generated. */
3817 if (pCtx->dr[7] & X86_DR7_ENABLED_MASK)
3818 {
3819 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatDRxIOCheck);
3820 for (unsigned i=0;i<4;i++)
3821 {
3822 unsigned uBPLen = g_aIOSize[X86_DR7_GET_LEN(pCtx->dr[7], i)];
3823
3824 if ( (uPort >= pCtx->dr[i] && uPort < pCtx->dr[i] + uBPLen)
3825 && (pCtx->dr[7] & (X86_DR7_L(i) | X86_DR7_G(i)))
3826 && (pCtx->dr[7] & X86_DR7_RW(i, X86_DR7_RW_IO)) == X86_DR7_RW(i, X86_DR7_RW_IO))
3827 {
3828 uint64_t uDR6;
3829
3830 Assert(CPUMIsGuestDebugStateActive(pVCpu));
3831
3832 uDR6 = ASMGetDR6();
3833
3834 /* Clear all breakpoint status flags and set the one we just hit. */
3835 uDR6 &= ~(X86_DR6_B0|X86_DR6_B1|X86_DR6_B2|X86_DR6_B3);
3836 uDR6 |= (uint64_t)RT_BIT(i);
3837
3838 /* Note: AMD64 Architecture Programmer's Manual 13.1:
3839 * Bits 15:13 of the DR6 register is never cleared by the processor and must be cleared by software after
3840 * the contents have been read.
3841 */
3842 ASMSetDR6(uDR6);
3843
3844 /* X86_DR7_GD will be cleared if drx accesses should be trapped inside the guest. */
3845 pCtx->dr[7] &= ~X86_DR7_GD;
3846
3847 /* Paranoia. */
3848 pCtx->dr[7] &= 0xffffffff; /* upper 32 bits reserved */
3849 pCtx->dr[7] &= ~(RT_BIT(11) | RT_BIT(12) | RT_BIT(14) | RT_BIT(15)); /* must be zero */
3850 pCtx->dr[7] |= 0x400; /* must be one */
3851
3852 /* Resync DR7 */
3853 rc = VMXWriteVMCS64(VMX_VMCS64_GUEST_DR7, pCtx->dr[7]);
3854 AssertRC(rc);
3855
3856 /* Construct inject info. */
3857 intInfo = X86_XCPT_DB;
3858 intInfo |= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
3859 intInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HWEXCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
3860
3861 Log(("Inject IO debug trap at %RGv\n", (RTGCPTR)pCtx->rip));
3862 rc = VMXR0InjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), 0, 0);
3863 AssertRC(rc);
3864
3865 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub1, y1);
3866 goto ResumeExecution;
3867 }
3868 }
3869 }
3870
3871 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub1, y1);
3872 goto ResumeExecution;
3873 }
3874 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub1, y1);
3875 break;
3876 }
3877
3878#ifdef VBOX_STRICT
3879 if (rc == VINF_IOM_HC_IOPORT_READ)
3880 Assert(!fIOWrite);
3881 else if (rc == VINF_IOM_HC_IOPORT_WRITE)
3882 Assert(fIOWrite);
3883 else
3884 AssertMsg(RT_FAILURE(rc) || rc == VINF_EM_RAW_EMULATE_INSTR || rc == VINF_EM_RAW_GUEST_TRAP || rc == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", rc));
3885#endif
3886 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub1, y1);
3887 break;
3888 }
3889
3890 case VMX_EXIT_TPR: /* 43 TPR below threshold. Guest software executed MOV to CR8. */
3891 LogFlow(("VMX_EXIT_TPR\n"));
3892 /* RIP is already set to the next instruction and the TPR has been synced back. Just resume. */
3893 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub1, y1);
3894 goto ResumeExecution;
3895
3896 case VMX_EXIT_APIC_ACCESS: /* 44 APIC access. Guest software attempted to access memory at a physical address on the APIC-access page. */
3897 {
3898 LogFlow(("VMX_EXIT_APIC_ACCESS\n"));
3899 unsigned uAccessType = VMX_EXIT_QUALIFICATION_APIC_ACCESS_TYPE(exitQualification);
3900
3901 switch(uAccessType)
3902 {
3903 case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
3904 case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
3905 {
3906 RTGCPHYS GCPhys;
3907 PDMApicGetBase(pVM, &GCPhys);
3908 GCPhys &= PAGE_BASE_GC_MASK;
3909 GCPhys += VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(exitQualification);
3910
3911 LogFlow(("Apic access at %RGp\n", GCPhys));
3912 rc = VBOXSTRICTRC_TODO(IOMMMIOPhysHandler(pVM, (uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ) ? 0 : X86_TRAP_PF_RW, CPUMCTX2CORE(pCtx), GCPhys));
3913 if (rc == VINF_SUCCESS)
3914 {
3915 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub1, y1);
3916 goto ResumeExecution; /* rip already updated */
3917 }
3918 break;
3919 }
3920
3921 default:
3922 rc = VINF_EM_RAW_EMULATE_INSTR;
3923 break;
3924 }
3925 break;
3926 }
3927
3928 case VMX_EXIT_PREEMPTION_TIMER: /* 52 VMX-preemption timer expired. The preemption timer counted down to zero. */
3929 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub1, y1);
3930 goto ResumeExecution;
3931
3932 default:
3933 /* The rest is handled after syncing the entire CPU state. */
3934 break;
3935 }
3936
3937 /* Note: the guest state isn't entirely synced back at this stage. */
3938
3939 /* Investigate why there was a VM-exit. (part 2) */
3940 switch (exitReason)
3941 {
3942 case VMX_EXIT_EXCEPTION: /* 0 Exception or non-maskable interrupt (NMI). */
3943 case VMX_EXIT_EXTERNAL_IRQ: /* 1 External interrupt. */
3944 case VMX_EXIT_EPT_VIOLATION:
3945 case VMX_EXIT_PREEMPTION_TIMER: /* 52 VMX-preemption timer expired. The preemption timer counted down to zero. */
3946 /* Already handled above. */
3947 break;
3948
3949 case VMX_EXIT_TRIPLE_FAULT: /* 2 Triple fault. */
3950 rc = VINF_EM_RESET; /* Triple fault equals a reset. */
3951 break;
3952
3953 case VMX_EXIT_INIT_SIGNAL: /* 3 INIT signal. */
3954 case VMX_EXIT_SIPI: /* 4 Start-up IPI (SIPI). */
3955 rc = VINF_EM_RAW_INTERRUPT;
3956 AssertFailed(); /* Can't happen. Yet. */
3957 break;
3958
3959 case VMX_EXIT_IO_SMI_IRQ: /* 5 I/O system-management interrupt (SMI). */
3960 case VMX_EXIT_SMI_IRQ: /* 6 Other SMI. */
3961 rc = VINF_EM_RAW_INTERRUPT;
3962 AssertFailed(); /* Can't happen afaik. */
3963 break;
3964
3965 case VMX_EXIT_TASK_SWITCH: /* 9 Task switch: too complicated to emulate, so fall back to the recompiler */
3966 Log(("VMX_EXIT_TASK_SWITCH: exit=%RX64\n", exitQualification));
3967 if ( (VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE(exitQualification) == VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE_IDT)
3968 && pVCpu->hwaccm.s.Event.fPending)
3969 {
3970 /* Caused by an injected interrupt. */
3971 pVCpu->hwaccm.s.Event.fPending = false;
3972
3973 Log(("VMX_EXIT_TASK_SWITCH: reassert trap %d\n", VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVCpu->hwaccm.s.Event.intInfo)));
3974 Assert(!VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(pVCpu->hwaccm.s.Event.intInfo));
3975 rc = TRPMAssertTrap(pVCpu, VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVCpu->hwaccm.s.Event.intInfo), TRPM_HARDWARE_INT);
3976 AssertRC(rc);
3977 }
3978 /* else Exceptions and software interrupts can just be restarted. */
3979 rc = VERR_EM_INTERPRETER;
3980 break;
3981
3982 case VMX_EXIT_HLT: /* 12 Guest software attempted to execute HLT. */
3983 /** Check if external interrupts are pending; if so, don't switch back. */
3984 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitHlt);
3985 pCtx->rip++; /* skip hlt */
3986 if (EMShouldContinueAfterHalt(pVCpu, pCtx))
3987 goto ResumeExecution;
3988
3989 rc = VINF_EM_HALT;
3990 break;
3991
3992 case VMX_EXIT_MWAIT: /* 36 Guest software executed MWAIT. */
3993 Log2(("VMX: mwait\n"));
3994 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitMwait);
3995 rc = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pCtx));
3996 if ( rc == VINF_EM_HALT
3997 || rc == VINF_SUCCESS)
3998 {
3999 /* Update EIP and continue execution. */
4000 pCtx->rip += cbInstr;
4001
4002 /** Check if external interrupts are pending; if so, don't switch back. */
4003 if ( rc == VINF_SUCCESS
4004 || ( rc == VINF_EM_HALT
4005 && EMShouldContinueAfterHalt(pVCpu, pCtx))
4006 )
4007 goto ResumeExecution;
4008 }
4009 AssertMsg(rc == VERR_EM_INTERPRETER || rc == VINF_EM_HALT, ("EMU: mwait failed with %Rrc\n", rc));
4010 break;
4011
4012 case VMX_EXIT_RSM: /* 17 Guest software attempted to execute RSM in SMM. */
4013 AssertFailed(); /* can't happen. */
4014 rc = VERR_EM_INTERPRETER;
4015 break;
4016
4017 case VMX_EXIT_VMCALL: /* 18 Guest software executed VMCALL. */
4018 case VMX_EXIT_VMCLEAR: /* 19 Guest software executed VMCLEAR. */
4019 case VMX_EXIT_VMLAUNCH: /* 20 Guest software executed VMLAUNCH. */
4020 case VMX_EXIT_VMPTRLD: /* 21 Guest software executed VMPTRLD. */
4021 case VMX_EXIT_VMPTRST: /* 22 Guest software executed VMPTRST. */
4022 case VMX_EXIT_VMREAD: /* 23 Guest software executed VMREAD. */
4023 case VMX_EXIT_VMRESUME: /* 24 Guest software executed VMRESUME. */
4024 case VMX_EXIT_VMWRITE: /* 25 Guest software executed VMWRITE. */
4025 case VMX_EXIT_VMXOFF: /* 26 Guest software executed VMXOFF. */
4026 case VMX_EXIT_VMXON: /* 27 Guest software executed VMXON. */
4027 /** @todo inject #UD immediately */
4028 rc = VERR_EM_INTERPRETER;
4029 break;
4030
4031 case VMX_EXIT_CPUID: /* 10 Guest software attempted to execute CPUID. */
4032 case VMX_EXIT_RDTSC: /* 16 Guest software attempted to execute RDTSC. */
4033 case VMX_EXIT_INVPG: /* 14 Guest software attempted to execute INVPG. */
4034 case VMX_EXIT_CRX_MOVE: /* 28 Control-register accesses. */
4035 case VMX_EXIT_DRX_MOVE: /* 29 Debug-register accesses. */
4036 case VMX_EXIT_PORT_IO: /* 30 I/O instruction. */
4037 case VMX_EXIT_RDPMC: /* 15 Guest software attempted to execute RDPMC. */
4038 /* already handled above */
4039 AssertMsg( rc == VINF_PGM_CHANGE_MODE
4040 || rc == VINF_EM_RAW_INTERRUPT
4041 || rc == VERR_EM_INTERPRETER
4042 || rc == VINF_EM_RAW_EMULATE_INSTR
4043 || rc == VINF_PGM_SYNC_CR3
4044 || rc == VINF_IOM_HC_IOPORT_READ
4045 || rc == VINF_IOM_HC_IOPORT_WRITE
4046 || rc == VINF_EM_RAW_GUEST_TRAP
4047 || rc == VINF_TRPM_XCPT_DISPATCHED
4048 || rc == VINF_EM_RESCHEDULE_REM,
4049 ("rc = %d\n", rc));
4050 break;
4051
4052 case VMX_EXIT_TPR: /* 43 TPR below threshold. Guest software executed MOV to CR8. */
4053 case VMX_EXIT_APIC_ACCESS: /* 44 APIC access. Guest software attempted to access memory at a physical address on the APIC-access page. */
4054 case VMX_EXIT_RDMSR: /* 31 RDMSR. Guest software attempted to execute RDMSR. */
4055 case VMX_EXIT_WRMSR: /* 32 WRMSR. Guest software attempted to execute WRMSR. */
4056 case VMX_EXIT_PAUSE: /* 40 Guest software attempted to execute PAUSE. */
4057 case VMX_EXIT_MONITOR: /* 39 Guest software attempted to execute MONITOR. */
4058 /* Note: If we decide to emulate them here, then we must sync the MSRs that could have been changed (sysenter, fs/gs base)!!! */
4059 rc = VERR_EM_INTERPRETER;
4060 break;
4061
4062 case VMX_EXIT_IRQ_WINDOW: /* 7 Interrupt window. */
4063 Assert(rc == VINF_EM_RAW_INTERRUPT);
4064 break;
4065
4066 case VMX_EXIT_ERR_INVALID_GUEST_STATE: /* 33 VM-entry failure due to invalid guest state. */
4067 {
4068#ifdef VBOX_STRICT
4069 RTCCUINTREG val2 = 0;
4070
4071 Log(("VMX_EXIT_ERR_INVALID_GUEST_STATE\n"));
4072
4073 VMXReadVMCS(VMX_VMCS64_GUEST_RIP, &val2);
4074 Log(("Old eip %RGv new %RGv\n", (RTGCPTR)pCtx->rip, (RTGCPTR)val2));
4075
4076 VMXReadVMCS(VMX_VMCS64_GUEST_CR0, &val2);
4077 Log(("VMX_VMCS_GUEST_CR0 %RX64\n", (uint64_t)val2));
4078
4079 VMXReadVMCS(VMX_VMCS64_GUEST_CR3, &val2);
4080 Log(("VMX_VMCS_GUEST_CR3 %RX64\n", (uint64_t)val2));
4081
4082 VMXReadVMCS(VMX_VMCS64_GUEST_CR4, &val2);
4083 Log(("VMX_VMCS_GUEST_CR4 %RX64\n", (uint64_t)val2));
4084
4085 VMXReadVMCS(VMX_VMCS_GUEST_RFLAGS, &val2);
4086 Log(("VMX_VMCS_GUEST_RFLAGS %08x\n", val2));
4087
4088 VMX_LOG_SELREG(CS, "CS", val2);
4089 VMX_LOG_SELREG(DS, "DS", val2);
4090 VMX_LOG_SELREG(ES, "ES", val2);
4091 VMX_LOG_SELREG(FS, "FS", val2);
4092 VMX_LOG_SELREG(GS, "GS", val2);
4093 VMX_LOG_SELREG(SS, "SS", val2);
4094 VMX_LOG_SELREG(TR, "TR", val2);
4095 VMX_LOG_SELREG(LDTR, "LDTR", val2);
4096
4097 VMXReadVMCS(VMX_VMCS64_GUEST_GDTR_BASE, &val2);
4098 Log(("VMX_VMCS_GUEST_GDTR_BASE %RX64\n", (uint64_t)val2));
4099 VMXReadVMCS(VMX_VMCS64_GUEST_IDTR_BASE, &val2);
4100 Log(("VMX_VMCS_GUEST_IDTR_BASE %RX64\n", (uint64_t)val2));
4101#endif /* VBOX_STRICT */
4102 rc = VERR_VMX_INVALID_GUEST_STATE;
4103 break;
4104 }
4105
4106 case VMX_EXIT_ERR_MSR_LOAD: /* 34 VM-entry failure due to MSR loading. */
4107 case VMX_EXIT_ERR_MACHINE_CHECK: /* 41 VM-entry failure due to machine-check. */
4108 default:
4109 rc = VERR_VMX_UNEXPECTED_EXIT_CODE;
4110 AssertMsgFailed(("Unexpected exit code %d\n", exitReason)); /* Can't happen. */
4111 break;
4112
4113 }
4114end:
4115
4116 /* Signal changes for the recompiler. */
4117 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR | CPUM_CHANGED_LDTR | CPUM_CHANGED_GDTR | CPUM_CHANGED_IDTR | CPUM_CHANGED_TR | CPUM_CHANGED_HIDDEN_SEL_REGS);
4118
4119 /* If we executed vmlaunch/vmresume and an external irq was pending, then we don't have to do a full sync the next time. */
4120 if ( exitReason == VMX_EXIT_EXTERNAL_IRQ
4121 && !VMX_EXIT_INTERRUPTION_INFO_VALID(intInfo))
4122 {
4123 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatPendingHostIrq);
4124 /* On the next entry we'll only sync the host context. */
4125 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_HOST_CONTEXT;
4126 }
4127 else
4128 {
4129 /* On the next entry we'll sync everything. */
4130 /** @todo we can do better than this */
4131 /* Not in the VINF_PGM_CHANGE_MODE though! */
4132 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_ALL;
4133 }
4134
4135 /* translate into a less severe return code */
4136 if (rc == VERR_EM_INTERPRETER)
4137 rc = VINF_EM_RAW_EMULATE_INSTR;
4138 else
4139 /* Try to extract more information about what might have gone wrong here. */
4140 if (rc == VERR_VMX_INVALID_VMCS_PTR)
4141 {
4142 VMXGetActivateVMCS(&pVCpu->hwaccm.s.vmx.lasterror.u64VMCSPhys);
4143 pVCpu->hwaccm.s.vmx.lasterror.ulVMCSRevision = *(uint32_t *)pVCpu->hwaccm.s.vmx.pVMCS;
4144 pVCpu->hwaccm.s.vmx.lasterror.idEnteredCpu = pVCpu->hwaccm.s.idEnteredCpu;
4145 pVCpu->hwaccm.s.vmx.lasterror.idCurrentCpu = RTMpCpuId();
4146 }
4147
4148 /* Just set the correct state here instead of trying to catch every goto above. */
4149 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED, VMCPUSTATE_STARTED_EXEC);
4150
4151#ifdef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
4152 /* Restore interrupts if we exitted after disabling them. */
4153 if (uOldEFlags != ~(RTCCUINTREG)0)
4154 ASMSetFlags(uOldEFlags);
4155#endif
4156
4157 STAM_STATS({
4158 if (fStatExit2Started) STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2, y);
4159 else if (fStatEntryStarted) STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatEntry, x);
4160 });
4161 Log2(("X"));
4162 return rc;
4163}
4164
4165
4166/**
4167 * Enters the VT-x session
4168 *
4169 * @returns VBox status code.
4170 * @param pVM The VM to operate on.
4171 * @param pVCpu The VMCPU to operate on.
4172 * @param pCpu CPU info struct
4173 */
4174VMMR0DECL(int) VMXR0Enter(PVM pVM, PVMCPU pVCpu, PHWACCM_CPUINFO pCpu)
4175{
4176 Assert(pVM->hwaccm.s.vmx.fSupported);
4177
4178 unsigned cr4 = ASMGetCR4();
4179 if (!(cr4 & X86_CR4_VMXE))
4180 {
4181 AssertMsgFailed(("X86_CR4_VMXE should be set!\n"));
4182 return VERR_VMX_X86_CR4_VMXE_CLEARED;
4183 }
4184
4185 /* Activate the VM Control Structure. */
4186 int rc = VMXActivateVMCS(pVCpu->hwaccm.s.vmx.pVMCSPhys);
4187 if (RT_FAILURE(rc))
4188 return rc;
4189
4190 pVCpu->hwaccm.s.fResumeVM = false;
4191 return VINF_SUCCESS;
4192}
4193
4194
4195/**
4196 * Leaves the VT-x session
4197 *
4198 * @returns VBox status code.
4199 * @param pVM The VM to operate on.
4200 * @param pVCpu The VMCPU to operate on.
4201 * @param pCtx CPU context
4202 */
4203VMMR0DECL(int) VMXR0Leave(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
4204{
4205 Assert(pVM->hwaccm.s.vmx.fSupported);
4206
4207#ifdef DEBUG
4208 if (CPUMIsHyperDebugStateActive(pVCpu))
4209 {
4210 CPUMR0LoadHostDebugState(pVM, pVCpu);
4211 Assert(pVCpu->hwaccm.s.vmx.proc_ctls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT);
4212 }
4213 else
4214#endif
4215 /* Save the guest debug state if necessary. */
4216 if (CPUMIsGuestDebugStateActive(pVCpu))
4217 {
4218 CPUMR0SaveGuestDebugState(pVM, pVCpu, pCtx, true /* save DR6 */);
4219
4220 /* Enable drx move intercepts again. */
4221 pVCpu->hwaccm.s.vmx.proc_ctls |= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT;
4222 int rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
4223 AssertRC(rc);
4224
4225 /* Resync the debug registers the next time. */
4226 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_GUEST_DEBUG;
4227 }
4228 else
4229 Assert(pVCpu->hwaccm.s.vmx.proc_ctls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT);
4230
4231 /* Clear VM Control Structure. Marking it inactive, clearing implementation specific data and writing back VMCS data to memory. */
4232 int rc = VMXClearVMCS(pVCpu->hwaccm.s.vmx.pVMCSPhys);
4233 AssertRC(rc);
4234
4235 return VINF_SUCCESS;
4236}
4237
4238/**
4239 * Flush the TLB (EPT)
4240 *
4241 * @returns VBox status code.
4242 * @param pVM The VM to operate on.
4243 * @param pVCpu The VM CPU to operate on.
4244 * @param enmFlush Type of flush
4245 * @param GCPhys Physical address of the page to flush
4246 */
4247static void vmxR0FlushEPT(PVM pVM, PVMCPU pVCpu, VMX_FLUSH enmFlush, RTGCPHYS GCPhys)
4248{
4249 uint64_t descriptor[2];
4250
4251 LogFlow(("vmxR0FlushEPT %d %RGv\n", enmFlush, GCPhys));
4252 Assert(pVM->hwaccm.s.fNestedPaging);
4253 descriptor[0] = pVCpu->hwaccm.s.vmx.GCPhysEPTP;
4254 descriptor[1] = GCPhys;
4255 int rc = VMXR0InvEPT(enmFlush, &descriptor[0]);
4256 AssertRC(rc);
4257}
4258
4259#ifdef HWACCM_VTX_WITH_VPID
4260/**
4261 * Flush the TLB (EPT)
4262 *
4263 * @returns VBox status code.
4264 * @param pVM The VM to operate on.
4265 * @param pVCpu The VM CPU to operate on.
4266 * @param enmFlush Type of flush
4267 * @param GCPtr Virtual address of the page to flush
4268 */
4269static void vmxR0FlushVPID(PVM pVM, PVMCPU pVCpu, VMX_FLUSH enmFlush, RTGCPTR GCPtr)
4270{
4271#if HC_ARCH_BITS == 32
4272 /* If we get a flush in 64 bits guest mode, then force a full TLB flush. Invvpid probably takes only 32 bits addresses. (@todo) */
4273 if ( CPUMIsGuestInLongMode(pVCpu)
4274 && !VMX_IS_64BIT_HOST_MODE())
4275 {
4276 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
4277 }
4278 else
4279#endif
4280 {
4281 uint64_t descriptor[2];
4282
4283 Assert(pVM->hwaccm.s.vmx.fVPID);
4284 descriptor[0] = pVCpu->hwaccm.s.uCurrentASID;
4285 descriptor[1] = GCPtr;
4286 int rc = VMXR0InvVPID(enmFlush, &descriptor[0]);
4287 AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvVPID %x %x %RGv failed with %d\n", enmFlush, pVCpu->hwaccm.s.uCurrentASID, GCPtr, rc));
4288 }
4289}
4290#endif /* HWACCM_VTX_WITH_VPID */
4291
4292/**
4293 * Invalidates a guest page
4294 *
4295 * @returns VBox status code.
4296 * @param pVM The VM to operate on.
4297 * @param pVCpu The VM CPU to operate on.
4298 * @param GCVirt Page to invalidate
4299 */
4300VMMR0DECL(int) VMXR0InvalidatePage(PVM pVM, PVMCPU pVCpu, RTGCPTR GCVirt)
4301{
4302 bool fFlushPending = VMCPU_FF_ISSET(pVCpu, VMCPU_FF_TLB_FLUSH);
4303
4304 Log2(("VMXR0InvalidatePage %RGv\n", GCVirt));
4305
4306 /* Only relevant if we want to use VPID.
4307 * In the nested paging case we still see such calls, but
4308 * can safely ignore them. (e.g. after cr3 updates)
4309 */
4310#ifdef HWACCM_VTX_WITH_VPID
4311 /* Skip it if a TLB flush is already pending. */
4312 if ( !fFlushPending
4313 && pVM->hwaccm.s.vmx.fVPID)
4314 vmxR0FlushVPID(pVM, pVCpu, pVM->hwaccm.s.vmx.enmFlushPage, GCVirt);
4315#endif /* HWACCM_VTX_WITH_VPID */
4316
4317 return VINF_SUCCESS;
4318}
4319
4320/**
4321 * Invalidates a guest page by physical address
4322 *
4323 * NOTE: Assumes the current instruction references this physical page though a virtual address!!
4324 *
4325 * @returns VBox status code.
4326 * @param pVM The VM to operate on.
4327 * @param pVCpu The VM CPU to operate on.
4328 * @param GCPhys Page to invalidate
4329 */
4330VMMR0DECL(int) VMXR0InvalidatePhysPage(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys)
4331{
4332 bool fFlushPending = VMCPU_FF_ISSET(pVCpu, VMCPU_FF_TLB_FLUSH);
4333
4334 Assert(pVM->hwaccm.s.fNestedPaging);
4335
4336 LogFlow(("VMXR0InvalidatePhysPage %RGp\n", GCPhys));
4337
4338 /* Skip it if a TLB flush is already pending. */
4339 if (!fFlushPending)
4340 vmxR0FlushEPT(pVM, pVCpu, pVM->hwaccm.s.vmx.enmFlushPage, GCPhys);
4341
4342 return VINF_SUCCESS;
4343}
4344
4345/**
4346 * Report world switch error and dump some useful debug info
4347 *
4348 * @param pVM The VM to operate on.
4349 * @param pVCpu The VMCPU to operate on.
4350 * @param rc Return code
4351 * @param pCtx Current CPU context (not updated)
4352 */
4353static void VMXR0ReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, int rc, PCPUMCTX pCtx)
4354{
4355 switch (rc)
4356 {
4357 case VERR_VMX_INVALID_VMXON_PTR:
4358 AssertFailed();
4359 break;
4360
4361 case VERR_VMX_UNABLE_TO_START_VM:
4362 case VERR_VMX_UNABLE_TO_RESUME_VM:
4363 {
4364 int rc2;
4365 RTCCUINTREG exitReason, instrError;
4366
4367 rc2 = VMXReadVMCS(VMX_VMCS32_RO_EXIT_REASON, &exitReason);
4368 rc2 |= VMXReadVMCS(VMX_VMCS32_RO_VM_INSTR_ERROR, &instrError);
4369 AssertRC(rc2);
4370 if (rc2 == VINF_SUCCESS)
4371 {
4372 Log(("Unable to start/resume VM for reason: %x. Instruction error %x\n", (uint32_t)exitReason, (uint32_t)instrError));
4373 Log(("Current stack %08x\n", &rc2));
4374
4375 pVCpu->hwaccm.s.vmx.lasterror.ulInstrError = instrError;
4376 pVCpu->hwaccm.s.vmx.lasterror.ulExitReason = exitReason;
4377
4378#ifdef VBOX_STRICT
4379 RTGDTR gdtr;
4380 PCX86DESCHC pDesc;
4381 RTCCUINTREG val;
4382
4383 ASMGetGDTR(&gdtr);
4384
4385 VMXReadVMCS(VMX_VMCS64_GUEST_RIP, &val);
4386 Log(("Old eip %RGv new %RGv\n", (RTGCPTR)pCtx->rip, (RTGCPTR)val));
4387 VMXReadVMCS(VMX_VMCS_CTRL_PIN_EXEC_CONTROLS, &val);
4388 Log(("VMX_VMCS_CTRL_PIN_EXEC_CONTROLS %08x\n", val));
4389 VMXReadVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, &val);
4390 Log(("VMX_VMCS_CTRL_PROC_EXEC_CONTROLS %08x\n", val));
4391 VMXReadVMCS(VMX_VMCS_CTRL_ENTRY_CONTROLS, &val);
4392 Log(("VMX_VMCS_CTRL_ENTRY_CONTROLS %08x\n", val));
4393 VMXReadVMCS(VMX_VMCS_CTRL_EXIT_CONTROLS, &val);
4394 Log(("VMX_VMCS_CTRL_EXIT_CONTROLS %08x\n", val));
4395
4396 VMXReadVMCS(VMX_VMCS_HOST_CR0, &val);
4397 Log(("VMX_VMCS_HOST_CR0 %08x\n", val));
4398
4399 VMXReadVMCS(VMX_VMCS_HOST_CR3, &val);
4400 Log(("VMX_VMCS_HOST_CR3 %08x\n", val));
4401
4402 VMXReadVMCS(VMX_VMCS_HOST_CR4, &val);
4403 Log(("VMX_VMCS_HOST_CR4 %08x\n", val));
4404
4405 VMXReadVMCS(VMX_VMCS16_HOST_FIELD_CS, &val);
4406 Log(("VMX_VMCS_HOST_FIELD_CS %08x\n", val));
4407
4408 VMXReadVMCS(VMX_VMCS_GUEST_RFLAGS, &val);
4409 Log(("VMX_VMCS_GUEST_RFLAGS %08x\n", val));
4410
4411 if (val < gdtr.cbGdt)
4412 {
4413 pDesc = (PCX86DESCHC)(gdtr.pGdt + (val & X86_SEL_MASK));
4414 HWACCMR0DumpDescriptor(pDesc, val, "CS: ");
4415 }
4416
4417 VMXReadVMCS(VMX_VMCS16_HOST_FIELD_DS, &val);
4418 Log(("VMX_VMCS_HOST_FIELD_DS %08x\n", val));
4419 if (val < gdtr.cbGdt)
4420 {
4421 pDesc = (PCX86DESCHC)(gdtr.pGdt + (val & X86_SEL_MASK));
4422 HWACCMR0DumpDescriptor(pDesc, val, "DS: ");
4423 }
4424
4425 VMXReadVMCS(VMX_VMCS16_HOST_FIELD_ES, &val);
4426 Log(("VMX_VMCS_HOST_FIELD_ES %08x\n", val));
4427 if (val < gdtr.cbGdt)
4428 {
4429 pDesc = (PCX86DESCHC)(gdtr.pGdt + (val & X86_SEL_MASK));
4430 HWACCMR0DumpDescriptor(pDesc, val, "ES: ");
4431 }
4432
4433 VMXReadVMCS(VMX_VMCS16_HOST_FIELD_FS, &val);
4434 Log(("VMX_VMCS16_HOST_FIELD_FS %08x\n", val));
4435 if (val < gdtr.cbGdt)
4436 {
4437 pDesc = (PCX86DESCHC)(gdtr.pGdt + (val & X86_SEL_MASK));
4438 HWACCMR0DumpDescriptor(pDesc, val, "FS: ");
4439 }
4440
4441 VMXReadVMCS(VMX_VMCS16_HOST_FIELD_GS, &val);
4442 Log(("VMX_VMCS16_HOST_FIELD_GS %08x\n", val));
4443 if (val < gdtr.cbGdt)
4444 {
4445 pDesc = (PCX86DESCHC)(gdtr.pGdt + (val & X86_SEL_MASK));
4446 HWACCMR0DumpDescriptor(pDesc, val, "GS: ");
4447 }
4448
4449 VMXReadVMCS(VMX_VMCS16_HOST_FIELD_SS, &val);
4450 Log(("VMX_VMCS16_HOST_FIELD_SS %08x\n", val));
4451 if (val < gdtr.cbGdt)
4452 {
4453 pDesc = (PCX86DESCHC)(gdtr.pGdt + (val & X86_SEL_MASK));
4454 HWACCMR0DumpDescriptor(pDesc, val, "SS: ");
4455 }
4456
4457 VMXReadVMCS(VMX_VMCS16_HOST_FIELD_TR, &val);
4458 Log(("VMX_VMCS16_HOST_FIELD_TR %08x\n", val));
4459 if (val < gdtr.cbGdt)
4460 {
4461 pDesc = (PCX86DESCHC)(gdtr.pGdt + (val & X86_SEL_MASK));
4462 HWACCMR0DumpDescriptor(pDesc, val, "TR: ");
4463 }
4464
4465 VMXReadVMCS(VMX_VMCS_HOST_TR_BASE, &val);
4466 Log(("VMX_VMCS_HOST_TR_BASE %RHv\n", val));
4467
4468 VMXReadVMCS(VMX_VMCS_HOST_GDTR_BASE, &val);
4469 Log(("VMX_VMCS_HOST_GDTR_BASE %RHv\n", val));
4470 VMXReadVMCS(VMX_VMCS_HOST_IDTR_BASE, &val);
4471 Log(("VMX_VMCS_HOST_IDTR_BASE %RHv\n", val));
4472
4473 VMXReadVMCS(VMX_VMCS32_HOST_SYSENTER_CS, &val);
4474 Log(("VMX_VMCS_HOST_SYSENTER_CS %08x\n", val));
4475
4476 VMXReadVMCS(VMX_VMCS_HOST_SYSENTER_EIP, &val);
4477 Log(("VMX_VMCS_HOST_SYSENTER_EIP %RHv\n", val));
4478
4479 VMXReadVMCS(VMX_VMCS_HOST_SYSENTER_ESP, &val);
4480 Log(("VMX_VMCS_HOST_SYSENTER_ESP %RHv\n", val));
4481
4482 VMXReadVMCS(VMX_VMCS_HOST_RSP, &val);
4483 Log(("VMX_VMCS_HOST_RSP %RHv\n", val));
4484 VMXReadVMCS(VMX_VMCS_HOST_RIP, &val);
4485 Log(("VMX_VMCS_HOST_RIP %RHv\n", val));
4486
4487# if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4488 if (VMX_IS_64BIT_HOST_MODE())
4489 {
4490 Log(("MSR_K6_EFER = %RX64\n", ASMRdMsr(MSR_K6_EFER)));
4491 Log(("MSR_K6_STAR = %RX64\n", ASMRdMsr(MSR_K6_STAR)));
4492 Log(("MSR_K8_LSTAR = %RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
4493 Log(("MSR_K8_CSTAR = %RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
4494 Log(("MSR_K8_SF_MASK = %RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
4495 }
4496# endif
4497#endif /* VBOX_STRICT */
4498 }
4499 break;
4500 }
4501
4502 default:
4503 /* impossible */
4504 AssertMsgFailed(("%Rrc (%#x)\n", rc, rc));
4505 break;
4506 }
4507}
4508
4509#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4510/**
4511 * Prepares for and executes VMLAUNCH (64 bits guest mode)
4512 *
4513 * @returns VBox status code
4514 * @param fResume vmlauch/vmresume
4515 * @param pCtx Guest context
4516 * @param pCache VMCS cache
4517 * @param pVM The VM to operate on.
4518 * @param pVCpu The VMCPU to operate on.
4519 */
4520DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
4521{
4522 uint32_t aParam[6];
4523 PHWACCM_CPUINFO pCpu;
4524 RTHCPHYS pPageCpuPhys;
4525 int rc;
4526
4527 pCpu = HWACCMR0GetCurrentCpu();
4528 pPageCpuPhys = RTR0MemObjGetPagePhysAddr(pCpu->pMemObj, 0);
4529
4530#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4531 pCache->uPos = 1;
4532 pCache->interPD = PGMGetInterPaeCR3(pVM);
4533 pCache->pSwitcher = (uint64_t)pVM->hwaccm.s.pfnHost32ToGuest64R0;
4534#endif
4535
4536#ifdef DEBUG
4537 pCache->TestIn.pPageCpuPhys = 0;
4538 pCache->TestIn.pVMCSPhys = 0;
4539 pCache->TestIn.pCache = 0;
4540 pCache->TestOut.pVMCSPhys = 0;
4541 pCache->TestOut.pCache = 0;
4542 pCache->TestOut.pCtx = 0;
4543 pCache->TestOut.eflags = 0;
4544#endif
4545
4546 aParam[0] = (uint32_t)(pPageCpuPhys); /* Param 1: VMXON physical address - Lo. */
4547 aParam[1] = (uint32_t)(pPageCpuPhys >> 32); /* Param 1: VMXON physical address - Hi. */
4548 aParam[2] = (uint32_t)(pVCpu->hwaccm.s.vmx.pVMCSPhys); /* Param 2: VMCS physical address - Lo. */
4549 aParam[3] = (uint32_t)(pVCpu->hwaccm.s.vmx.pVMCSPhys >> 32); /* Param 2: VMCS physical address - Hi. */
4550 aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hwaccm.s.vmx.VMCSCache);
4551 aParam[5] = 0;
4552
4553#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4554 pCtx->dr[4] = pVM->hwaccm.s.vmx.pScratchPhys + 16 + 8;
4555 *(uint32_t *)(pVM->hwaccm.s.vmx.pScratch + 16 + 8) = 1;
4556#endif
4557 rc = VMXR0Execute64BitsHandler(pVM, pVCpu, pCtx, pVM->hwaccm.s.pfnVMXGCStartVM64, 6, &aParam[0]);
4558
4559#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4560 Assert(*(uint32_t *)(pVM->hwaccm.s.vmx.pScratch + 16 + 8) == 5);
4561 Assert(pCtx->dr[4] == 10);
4562 *(uint32_t *)(pVM->hwaccm.s.vmx.pScratch + 16 + 8) = 0xff;
4563#endif
4564
4565#ifdef DEBUG
4566 AssertMsg(pCache->TestIn.pPageCpuPhys == pPageCpuPhys, ("%RHp vs %RHp\n", pCache->TestIn.pPageCpuPhys, pPageCpuPhys));
4567 AssertMsg(pCache->TestIn.pVMCSPhys == pVCpu->hwaccm.s.vmx.pVMCSPhys, ("%RHp vs %RHp\n", pCache->TestIn.pVMCSPhys, pVCpu->hwaccm.s.vmx.pVMCSPhys));
4568 AssertMsg(pCache->TestIn.pVMCSPhys == pCache->TestOut.pVMCSPhys, ("%RHp vs %RHp\n", pCache->TestIn.pVMCSPhys, pCache->TestOut.pVMCSPhys));
4569 AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache, pCache->TestOut.pCache));
4570 AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hwaccm.s.vmx.VMCSCache), ("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hwaccm.s.vmx.VMCSCache)));
4571 AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx, pCache->TestOut.pCtx));
4572 Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
4573#endif
4574 return rc;
4575}
4576
4577/**
4578 * Executes the specified handler in 64 mode
4579 *
4580 * @returns VBox status code.
4581 * @param pVM The VM to operate on.
4582 * @param pVCpu The VMCPU to operate on.
4583 * @param pCtx Guest context
4584 * @param pfnHandler RC handler
4585 * @param cbParam Number of parameters
4586 * @param paParam Array of 32 bits parameters
4587 */
4588VMMR0DECL(int) VMXR0Execute64BitsHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, RTRCPTR pfnHandler, uint32_t cbParam, uint32_t *paParam)
4589{
4590 int rc, rc2;
4591 PHWACCM_CPUINFO pCpu;
4592 RTHCPHYS pPageCpuPhys;
4593 RTHCUINTREG uOldEFlags;
4594
4595 AssertReturn(pVM->hwaccm.s.pfnHost32ToGuest64R0, VERR_INTERNAL_ERROR);
4596 Assert(pfnHandler);
4597 Assert(pVCpu->hwaccm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hwaccm.s.vmx.VMCSCache.Write.aField));
4598 Assert(pVCpu->hwaccm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hwaccm.s.vmx.VMCSCache.Read.aField));
4599
4600#ifdef VBOX_STRICT
4601 for (unsigned i=0;i<pVCpu->hwaccm.s.vmx.VMCSCache.Write.cValidEntries;i++)
4602 Assert(vmxR0IsValidWriteField(pVCpu->hwaccm.s.vmx.VMCSCache.Write.aField[i]));
4603
4604 for (unsigned i=0;i<pVCpu->hwaccm.s.vmx.VMCSCache.Read.cValidEntries;i++)
4605 Assert(vmxR0IsValidReadField(pVCpu->hwaccm.s.vmx.VMCSCache.Read.aField[i]));
4606#endif
4607
4608 /* Disable interrupts. */
4609 uOldEFlags = ASMIntDisableFlags();
4610
4611 pCpu = HWACCMR0GetCurrentCpu();
4612 pPageCpuPhys = RTR0MemObjGetPagePhysAddr(pCpu->pMemObj, 0);
4613
4614 /* Clear VM Control Structure. Marking it inactive, clearing implementation specific data and writing back VMCS data to memory. */
4615 VMXClearVMCS(pVCpu->hwaccm.s.vmx.pVMCSPhys);
4616
4617 /* Leave VMX Root Mode. */
4618 VMXDisable();
4619
4620 ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
4621
4622 CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
4623 CPUMSetHyperEIP(pVCpu, pfnHandler);
4624 for (int i=(int)cbParam-1;i>=0;i--)
4625 CPUMPushHyper(pVCpu, paParam[i]);
4626
4627 STAM_PROFILE_ADV_START(&pVCpu->hwaccm.s.StatWorldSwitch3264, z);
4628 /* Call switcher. */
4629 rc = pVM->hwaccm.s.pfnHost32ToGuest64R0(pVM, RT_OFFSETOF(VM, aCpus[pVCpu->idCpu].cpum) - RT_OFFSETOF(VM, cpum));
4630 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatWorldSwitch3264, z);
4631
4632 /* Make sure the VMX instructions don't cause #UD faults. */
4633 ASMSetCR4(ASMGetCR4() | X86_CR4_VMXE);
4634
4635 /* Enter VMX Root Mode */
4636 rc2 = VMXEnable(pPageCpuPhys);
4637 if (RT_FAILURE(rc2))
4638 {
4639 ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
4640 ASMSetFlags(uOldEFlags);
4641 return VERR_VMX_VMXON_FAILED;
4642 }
4643
4644 rc2 = VMXActivateVMCS(pVCpu->hwaccm.s.vmx.pVMCSPhys);
4645 AssertRC(rc2);
4646 Assert(!(ASMGetFlags() & X86_EFL_IF));
4647 ASMSetFlags(uOldEFlags);
4648 return rc;
4649}
4650
4651#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL) */
4652
4653
4654#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
4655/**
4656 * Executes VMWRITE
4657 *
4658 * @returns VBox status code
4659 * @param pVCpu The VMCPU to operate on.
4660 * @param idxField VMCS index
4661 * @param u64Val 16, 32 or 64 bits value
4662 */
4663VMMR0DECL(int) VMXWriteVMCS64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
4664{
4665 int rc;
4666
4667 switch (idxField)
4668 {
4669 case VMX_VMCS_CTRL_TSC_OFFSET_FULL:
4670 case VMX_VMCS_CTRL_IO_BITMAP_A_FULL:
4671 case VMX_VMCS_CTRL_IO_BITMAP_B_FULL:
4672 case VMX_VMCS_CTRL_MSR_BITMAP_FULL:
4673 case VMX_VMCS_CTRL_VMEXIT_MSR_STORE_FULL:
4674 case VMX_VMCS_CTRL_VMEXIT_MSR_LOAD_FULL:
4675 case VMX_VMCS_CTRL_VMENTRY_MSR_LOAD_FULL:
4676 case VMX_VMCS_CTRL_VAPIC_PAGEADDR_FULL:
4677 case VMX_VMCS_CTRL_APIC_ACCESSADDR_FULL:
4678 case VMX_VMCS_GUEST_LINK_PTR_FULL:
4679 case VMX_VMCS_GUEST_PDPTR0_FULL:
4680 case VMX_VMCS_GUEST_PDPTR1_FULL:
4681 case VMX_VMCS_GUEST_PDPTR2_FULL:
4682 case VMX_VMCS_GUEST_PDPTR3_FULL:
4683 case VMX_VMCS_GUEST_DEBUGCTL_FULL:
4684 case VMX_VMCS_GUEST_EFER_FULL:
4685 case VMX_VMCS_CTRL_EPTP_FULL:
4686 /* These fields consist of two parts, which are both writable in 32 bits mode. */
4687 rc = VMXWriteVMCS32(idxField, u64Val);
4688 rc |= VMXWriteVMCS32(idxField + 1, (uint32_t)(u64Val >> 32ULL));
4689 AssertRC(rc);
4690 return rc;
4691
4692 case VMX_VMCS64_GUEST_LDTR_BASE:
4693 case VMX_VMCS64_GUEST_TR_BASE:
4694 case VMX_VMCS64_GUEST_GDTR_BASE:
4695 case VMX_VMCS64_GUEST_IDTR_BASE:
4696 case VMX_VMCS64_GUEST_SYSENTER_EIP:
4697 case VMX_VMCS64_GUEST_SYSENTER_ESP:
4698 case VMX_VMCS64_GUEST_CR0:
4699 case VMX_VMCS64_GUEST_CR4:
4700 case VMX_VMCS64_GUEST_CR3:
4701 case VMX_VMCS64_GUEST_DR7:
4702 case VMX_VMCS64_GUEST_RIP:
4703 case VMX_VMCS64_GUEST_RSP:
4704 case VMX_VMCS64_GUEST_CS_BASE:
4705 case VMX_VMCS64_GUEST_DS_BASE:
4706 case VMX_VMCS64_GUEST_ES_BASE:
4707 case VMX_VMCS64_GUEST_FS_BASE:
4708 case VMX_VMCS64_GUEST_GS_BASE:
4709 case VMX_VMCS64_GUEST_SS_BASE:
4710 /* Queue a 64 bits value as we can't set it in 32 bits host mode. */
4711 if (u64Val >> 32ULL)
4712 rc = VMXWriteCachedVMCSEx(pVCpu, idxField, u64Val);
4713 else
4714 rc = VMXWriteVMCS32(idxField, (uint32_t)u64Val);
4715
4716 return rc;
4717
4718 default:
4719 AssertMsgFailed(("Unexpected field %x\n", idxField));
4720 return VERR_INVALID_PARAMETER;
4721 }
4722}
4723
4724/**
4725 * Cache VMCS writes for performance reasons (Darwin) and for running 64 bits guests on 32 bits hosts.
4726 *
4727 * @param pVCpu The VMCPU to operate on.
4728 * @param idxField VMCS field
4729 * @param u64Val Value
4730 */
4731VMMR0DECL(int) VMXWriteCachedVMCSEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
4732{
4733 PVMCSCACHE pCache = &pVCpu->hwaccm.s.vmx.VMCSCache;
4734
4735 AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1, ("entries=%x\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
4736
4737 /* Make sure there are no duplicates. */
4738 for (unsigned i=0;i<pCache->Write.cValidEntries;i++)
4739 {
4740 if (pCache->Write.aField[i] == idxField)
4741 {
4742 pCache->Write.aFieldVal[i] = u64Val;
4743 return VINF_SUCCESS;
4744 }
4745 }
4746
4747 pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
4748 pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
4749 pCache->Write.cValidEntries++;
4750 return VINF_SUCCESS;
4751}
4752
4753#endif /* HC_ARCH_BITS == 32 && !VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0 */
4754
4755#ifdef VBOX_STRICT
4756static bool vmxR0IsValidReadField(uint32_t idxField)
4757{
4758 switch(idxField)
4759 {
4760 case VMX_VMCS64_GUEST_RIP:
4761 case VMX_VMCS64_GUEST_RSP:
4762 case VMX_VMCS_GUEST_RFLAGS:
4763 case VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE:
4764 case VMX_VMCS_CTRL_CR0_READ_SHADOW:
4765 case VMX_VMCS64_GUEST_CR0:
4766 case VMX_VMCS_CTRL_CR4_READ_SHADOW:
4767 case VMX_VMCS64_GUEST_CR4:
4768 case VMX_VMCS64_GUEST_DR7:
4769 case VMX_VMCS32_GUEST_SYSENTER_CS:
4770 case VMX_VMCS64_GUEST_SYSENTER_EIP:
4771 case VMX_VMCS64_GUEST_SYSENTER_ESP:
4772 case VMX_VMCS32_GUEST_GDTR_LIMIT:
4773 case VMX_VMCS64_GUEST_GDTR_BASE:
4774 case VMX_VMCS32_GUEST_IDTR_LIMIT:
4775 case VMX_VMCS64_GUEST_IDTR_BASE:
4776 case VMX_VMCS16_GUEST_FIELD_CS:
4777 case VMX_VMCS32_GUEST_CS_LIMIT:
4778 case VMX_VMCS64_GUEST_CS_BASE:
4779 case VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS:
4780 case VMX_VMCS16_GUEST_FIELD_DS:
4781 case VMX_VMCS32_GUEST_DS_LIMIT:
4782 case VMX_VMCS64_GUEST_DS_BASE:
4783 case VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS:
4784 case VMX_VMCS16_GUEST_FIELD_ES:
4785 case VMX_VMCS32_GUEST_ES_LIMIT:
4786 case VMX_VMCS64_GUEST_ES_BASE:
4787 case VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS:
4788 case VMX_VMCS16_GUEST_FIELD_FS:
4789 case VMX_VMCS32_GUEST_FS_LIMIT:
4790 case VMX_VMCS64_GUEST_FS_BASE:
4791 case VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS:
4792 case VMX_VMCS16_GUEST_FIELD_GS:
4793 case VMX_VMCS32_GUEST_GS_LIMIT:
4794 case VMX_VMCS64_GUEST_GS_BASE:
4795 case VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS:
4796 case VMX_VMCS16_GUEST_FIELD_SS:
4797 case VMX_VMCS32_GUEST_SS_LIMIT:
4798 case VMX_VMCS64_GUEST_SS_BASE:
4799 case VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS:
4800 case VMX_VMCS16_GUEST_FIELD_LDTR:
4801 case VMX_VMCS32_GUEST_LDTR_LIMIT:
4802 case VMX_VMCS64_GUEST_LDTR_BASE:
4803 case VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS:
4804 case VMX_VMCS16_GUEST_FIELD_TR:
4805 case VMX_VMCS32_GUEST_TR_LIMIT:
4806 case VMX_VMCS64_GUEST_TR_BASE:
4807 case VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS:
4808 case VMX_VMCS32_RO_EXIT_REASON:
4809 case VMX_VMCS32_RO_VM_INSTR_ERROR:
4810 case VMX_VMCS32_RO_EXIT_INSTR_LENGTH:
4811 case VMX_VMCS32_RO_EXIT_INTERRUPTION_ERRCODE:
4812 case VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO:
4813 case VMX_VMCS32_RO_EXIT_INSTR_INFO:
4814 case VMX_VMCS_RO_EXIT_QUALIFICATION:
4815 case VMX_VMCS32_RO_IDT_INFO:
4816 case VMX_VMCS32_RO_IDT_ERRCODE:
4817 case VMX_VMCS64_GUEST_CR3:
4818 case VMX_VMCS_EXIT_PHYS_ADDR_FULL:
4819 return true;
4820 }
4821 return false;
4822}
4823
4824static bool vmxR0IsValidWriteField(uint32_t idxField)
4825{
4826 switch(idxField)
4827 {
4828 case VMX_VMCS64_GUEST_LDTR_BASE:
4829 case VMX_VMCS64_GUEST_TR_BASE:
4830 case VMX_VMCS64_GUEST_GDTR_BASE:
4831 case VMX_VMCS64_GUEST_IDTR_BASE:
4832 case VMX_VMCS64_GUEST_SYSENTER_EIP:
4833 case VMX_VMCS64_GUEST_SYSENTER_ESP:
4834 case VMX_VMCS64_GUEST_CR0:
4835 case VMX_VMCS64_GUEST_CR4:
4836 case VMX_VMCS64_GUEST_CR3:
4837 case VMX_VMCS64_GUEST_DR7:
4838 case VMX_VMCS64_GUEST_RIP:
4839 case VMX_VMCS64_GUEST_RSP:
4840 case VMX_VMCS64_GUEST_CS_BASE:
4841 case VMX_VMCS64_GUEST_DS_BASE:
4842 case VMX_VMCS64_GUEST_ES_BASE:
4843 case VMX_VMCS64_GUEST_FS_BASE:
4844 case VMX_VMCS64_GUEST_GS_BASE:
4845 case VMX_VMCS64_GUEST_SS_BASE:
4846 return true;
4847 }
4848 return false;
4849}
4850
4851#endif
Note: See TracBrowser for help on using the repository browser.

© 2024 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette