VirtualBox

source: vbox/trunk/src/VBox/VMM/VMMR0/HWVMXR0.cpp@ 21951

Last change on this file since 21951 was 21951, checked in by vboxsync, 15 years ago

Comment updates
  • Property svn:eol-style set to native
  • Property svn:keywords set to Id
File size: 177.5 KB
Line 
1/* $Id: HWVMXR0.cpp 21951 2009-08-04 09:03:29Z vboxsync $ */
2/** @file
3 * HWACCM VMX - Host Context Ring 0.
4 */
5
6/*
7 * Copyright (C) 2006-2007 Sun Microsystems, Inc.
8 *
9 * This file is part of VirtualBox Open Source Edition (OSE), as
10 * available from http://www.virtualbox.org. This file is free software;
11 * you can redistribute it and/or modify it under the terms of the GNU
12 * General Public License (GPL) as published by the Free Software
13 * Foundation, in version 2 as it comes in the "COPYING" file of the
14 * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15 * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16 *
17 * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa
18 * Clara, CA 95054 USA or visit http://www.sun.com if you need
19 * additional information or have any questions.
20 */
21
22
23/*******************************************************************************
24* Header Files *
25*******************************************************************************/
26#define LOG_GROUP LOG_GROUP_HWACCM
27#include <VBox/hwaccm.h>
28#include "HWACCMInternal.h"
29#include <VBox/vm.h>
30#include <VBox/x86.h>
31#include <VBox/pgm.h>
32#include <VBox/pdm.h>
33#include <VBox/err.h>
34#include <VBox/log.h>
35#include <VBox/selm.h>
36#include <VBox/iom.h>
37#include <VBox/rem.h>
38#include <iprt/asm.h>
39#include <iprt/assert.h>
40#include <iprt/param.h>
41#include <iprt/string.h>
42#include <iprt/time.h>
43#ifdef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
44# include <iprt/thread.h>
45#endif
46#include "HWVMXR0.h"
47
48/*******************************************************************************
49* Defined Constants And Macros *
50*******************************************************************************/
51#if defined(RT_ARCH_AMD64)
52# define VMX_IS_64BIT_HOST_MODE() (true)
53#elif defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
54# define VMX_IS_64BIT_HOST_MODE() (g_fVMXIs64bitHost != 0)
55#else
56# define VMX_IS_64BIT_HOST_MODE() (false)
57#endif
58
59/*******************************************************************************
60* Global Variables *
61*******************************************************************************/
62/* IO operation lookup arrays. */
63static uint32_t const g_aIOSize[4] = {1, 2, 0, 4};
64static uint32_t const g_aIOOpAnd[4] = {0xff, 0xffff, 0, 0xffffffff};
65
66#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
67/** See HWACCMR0A.asm. */
68extern "C" uint32_t g_fVMXIs64bitHost;
69#endif
70
71/*******************************************************************************
72* Local Functions *
73*******************************************************************************/
74static void VMXR0ReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, int rc, PCPUMCTX pCtx);
75static void vmxR0SetupTLBEPT(PVM pVM, PVMCPU pVCpu);
76static void vmxR0SetupTLBVPID(PVM pVM, PVMCPU pVCpu);
77static void vmxR0SetupTLBDummy(PVM pVM, PVMCPU pVCpu);
78static void vmxR0FlushEPT(PVM pVM, PVMCPU pVCpu, VMX_FLUSH enmFlush, RTGCPHYS GCPhys);
79static void vmxR0FlushVPID(PVM pVM, PVMCPU pVCpu, VMX_FLUSH enmFlush, RTGCPTR GCPtr);
80static void vmxR0UpdateExceptionBitmap(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx);
81#ifdef VBOX_STRICT
82static bool vmxR0IsValidReadField(uint32_t idxField);
83static bool vmxR0IsValidWriteField(uint32_t idxField);
84#endif
85
86static void VMXR0CheckError(PVM pVM, PVMCPU pVCpu, int rc)
87{
88 if (rc == VERR_VMX_GENERIC)
89 {
90 RTCCUINTREG instrError;
91
92 VMXReadVMCS(VMX_VMCS32_RO_VM_INSTR_ERROR, &instrError);
93 pVCpu->hwaccm.s.vmx.lasterror.ulInstrError = instrError;
94 }
95 pVM->hwaccm.s.lLastError = rc;
96}
97
98/**
99 * Sets up and activates VT-x on the current CPU
100 *
101 * @returns VBox status code.
102 * @param pCpu CPU info struct
103 * @param pVM The VM to operate on. (can be NULL after a resume!!)
104 * @param pvPageCpu Pointer to the global cpu page
105 * @param pPageCpuPhys Physical address of the global cpu page
106 */
107VMMR0DECL(int) VMXR0EnableCpu(PHWACCM_CPUINFO pCpu, PVM pVM, void *pvPageCpu, RTHCPHYS pPageCpuPhys)
108{
109 AssertReturn(pPageCpuPhys, VERR_INVALID_PARAMETER);
110 AssertReturn(pvPageCpu, VERR_INVALID_PARAMETER);
111
112#if defined(LOG_ENABLED) && !defined(DEBUG_bird)
113 SUPR0Printf("VMXR0EnableCpu cpu %d page (%x) %x\n", pCpu->idCpu, pvPageCpu, (uint32_t)pPageCpuPhys);
114#endif
115 if (pVM)
116 {
117 /* Set revision dword at the beginning of the VMXON structure. */
118 *(uint32_t *)pvPageCpu = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hwaccm.s.vmx.msr.vmx_basic_info);
119 }
120
121 /** @todo we should unmap the two pages from the virtual address space in order to prevent accidental corruption.
122 * (which can have very bad consequences!!!)
123 */
124
125 /* Make sure the VMX instructions don't cause #UD faults. */
126 ASMSetCR4(ASMGetCR4() | X86_CR4_VMXE);
127
128 /* Enter VMX Root Mode */
129 int rc = VMXEnable(pPageCpuPhys);
130 if (RT_FAILURE(rc))
131 {
132 if (pVM)
133 VMXR0CheckError(pVM, &pVM->aCpus[0], rc);
134 ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
135 return VERR_VMX_VMXON_FAILED;
136 }
137 return VINF_SUCCESS;
138}
139
140/**
141 * Deactivates VT-x on the current CPU
142 *
143 * @returns VBox status code.
144 * @param pCpu CPU info struct
145 * @param pvPageCpu Pointer to the global cpu page
146 * @param pPageCpuPhys Physical address of the global cpu page
147 */
148VMMR0DECL(int) VMXR0DisableCpu(PHWACCM_CPUINFO pCpu, void *pvPageCpu, RTHCPHYS pPageCpuPhys)
149{
150 AssertReturn(pPageCpuPhys, VERR_INVALID_PARAMETER);
151 AssertReturn(pvPageCpu, VERR_INVALID_PARAMETER);
152
153 /* Leave VMX Root Mode. */
154 VMXDisable();
155
156 /* And clear the X86_CR4_VMXE bit */
157 ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
158
159#if defined(LOG_ENABLED) && !defined(DEBUG_bird)
160 SUPR0Printf("VMXR0DisableCpu cpu %d\n", pCpu->idCpu);
161#endif
162 return VINF_SUCCESS;
163}
164
165/**
166 * Does Ring-0 per VM VT-x init.
167 *
168 * @returns VBox status code.
169 * @param pVM The VM to operate on.
170 */
171VMMR0DECL(int) VMXR0InitVM(PVM pVM)
172{
173 int rc;
174
175#ifdef LOG_ENABLED
176 SUPR0Printf("VMXR0InitVM %x\n", pVM);
177#endif
178
179 pVM->hwaccm.s.vmx.pMemObjAPIC = NIL_RTR0MEMOBJ;
180
181 if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
182 {
183 /* Allocate one page for the APIC physical page (serves for filtering accesses). */
184 rc = RTR0MemObjAllocCont(&pVM->hwaccm.s.vmx.pMemObjAPIC, 1 << PAGE_SHIFT, true /* executable R0 mapping */);
185 AssertRC(rc);
186 if (RT_FAILURE(rc))
187 return rc;
188
189 pVM->hwaccm.s.vmx.pAPIC = (uint8_t *)RTR0MemObjAddress(pVM->hwaccm.s.vmx.pMemObjAPIC);
190 pVM->hwaccm.s.vmx.pAPICPhys = RTR0MemObjGetPagePhysAddr(pVM->hwaccm.s.vmx.pMemObjAPIC, 0);
191 ASMMemZero32(pVM->hwaccm.s.vmx.pAPIC, PAGE_SIZE);
192 }
193 else
194 {
195 pVM->hwaccm.s.vmx.pMemObjAPIC = 0;
196 pVM->hwaccm.s.vmx.pAPIC = 0;
197 pVM->hwaccm.s.vmx.pAPICPhys = 0;
198 }
199
200 /* Allocate the MSR bitmap if this feature is supported. */
201 if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
202 {
203 rc = RTR0MemObjAllocCont(&pVM->hwaccm.s.vmx.pMemObjMSRBitmap, 1 << PAGE_SHIFT, true /* executable R0 mapping */);
204 AssertRC(rc);
205 if (RT_FAILURE(rc))
206 return rc;
207
208 pVM->hwaccm.s.vmx.pMSRBitmap = (uint8_t *)RTR0MemObjAddress(pVM->hwaccm.s.vmx.pMemObjMSRBitmap);
209 pVM->hwaccm.s.vmx.pMSRBitmapPhys = RTR0MemObjGetPagePhysAddr(pVM->hwaccm.s.vmx.pMemObjMSRBitmap, 0);
210 memset(pVM->hwaccm.s.vmx.pMSRBitmap, 0xff, PAGE_SIZE);
211 }
212
213#ifdef VBOX_WITH_CRASHDUMP_MAGIC
214 {
215 rc = RTR0MemObjAllocCont(&pVM->hwaccm.s.vmx.pMemObjScratch, 1 << PAGE_SHIFT, true /* executable R0 mapping */);
216 AssertRC(rc);
217 if (RT_FAILURE(rc))
218 return rc;
219
220 pVM->hwaccm.s.vmx.pScratch = (uint8_t *)RTR0MemObjAddress(pVM->hwaccm.s.vmx.pMemObjScratch);
221 pVM->hwaccm.s.vmx.pScratchPhys = RTR0MemObjGetPagePhysAddr(pVM->hwaccm.s.vmx.pMemObjScratch, 0);
222
223 ASMMemZero32(pVM->hwaccm.s.vmx.pScratch, PAGE_SIZE);
224 strcpy((char *)pVM->hwaccm.s.vmx.pScratch, "SCRATCH Magic");
225 *(uint64_t *)(pVM->hwaccm.s.vmx.pScratch + 16) = UINT64_C(0xDEADBEEFDEADBEEF);
226 }
227#endif
228
229 /* Allocate VMCBs for all guest CPUs. */
230 for (unsigned i=0;i<pVM->cCPUs;i++)
231 {
232 PVMCPU pVCpu = &pVM->aCpus[i];
233
234 pVCpu->hwaccm.s.vmx.pMemObjVMCS = NIL_RTR0MEMOBJ;
235
236 /* Allocate one page for the VM control structure (VMCS). */
237 rc = RTR0MemObjAllocCont(&pVCpu->hwaccm.s.vmx.pMemObjVMCS, 1 << PAGE_SHIFT, true /* executable R0 mapping */);
238 AssertRC(rc);
239 if (RT_FAILURE(rc))
240 return rc;
241
242 pVCpu->hwaccm.s.vmx.pVMCS = RTR0MemObjAddress(pVCpu->hwaccm.s.vmx.pMemObjVMCS);
243 pVCpu->hwaccm.s.vmx.pVMCSPhys = RTR0MemObjGetPagePhysAddr(pVCpu->hwaccm.s.vmx.pMemObjVMCS, 0);
244 ASMMemZero32(pVCpu->hwaccm.s.vmx.pVMCS, PAGE_SIZE);
245
246 pVCpu->hwaccm.s.vmx.cr0_mask = 0;
247 pVCpu->hwaccm.s.vmx.cr4_mask = 0;
248
249 /* Allocate one page for the virtual APIC page for TPR caching. */
250 rc = RTR0MemObjAllocCont(&pVCpu->hwaccm.s.vmx.pMemObjVAPIC, 1 << PAGE_SHIFT, true /* executable R0 mapping */);
251 AssertRC(rc);
252 if (RT_FAILURE(rc))
253 return rc;
254
255 pVCpu->hwaccm.s.vmx.pVAPIC = (uint8_t *)RTR0MemObjAddress(pVCpu->hwaccm.s.vmx.pMemObjVAPIC);
256 pVCpu->hwaccm.s.vmx.pVAPICPhys = RTR0MemObjGetPagePhysAddr(pVCpu->hwaccm.s.vmx.pMemObjVAPIC, 0);
257 ASMMemZero32(pVCpu->hwaccm.s.vmx.pVAPIC, PAGE_SIZE);
258
259 /* Current guest paging mode. */
260 pVCpu->hwaccm.s.vmx.enmLastSeenGuestMode = PGMMODE_REAL;
261
262#ifdef LOG_ENABLED
263 SUPR0Printf("VMXR0InitVM %x VMCS=%x (%x)\n", pVM, pVCpu->hwaccm.s.vmx.pVMCS, (uint32_t)pVCpu->hwaccm.s.vmx.pVMCSPhys);
264#endif
265 }
266
267 return VINF_SUCCESS;
268}
269
270/**
271 * Does Ring-0 per VM VT-x termination.
272 *
273 * @returns VBox status code.
274 * @param pVM The VM to operate on.
275 */
276VMMR0DECL(int) VMXR0TermVM(PVM pVM)
277{
278 for (unsigned i=0;i<pVM->cCPUs;i++)
279 {
280 PVMCPU pVCpu = &pVM->aCpus[i];
281
282 if (pVCpu->hwaccm.s.vmx.pMemObjVMCS != NIL_RTR0MEMOBJ)
283 {
284 RTR0MemObjFree(pVCpu->hwaccm.s.vmx.pMemObjVMCS, false);
285 pVCpu->hwaccm.s.vmx.pMemObjVMCS = NIL_RTR0MEMOBJ;
286 pVCpu->hwaccm.s.vmx.pVMCS = 0;
287 pVCpu->hwaccm.s.vmx.pVMCSPhys = 0;
288 }
289 if (pVCpu->hwaccm.s.vmx.pMemObjVAPIC != NIL_RTR0MEMOBJ)
290 {
291 RTR0MemObjFree(pVCpu->hwaccm.s.vmx.pMemObjVAPIC, false);
292 pVCpu->hwaccm.s.vmx.pMemObjVAPIC = NIL_RTR0MEMOBJ;
293 pVCpu->hwaccm.s.vmx.pVAPIC = 0;
294 pVCpu->hwaccm.s.vmx.pVAPICPhys = 0;
295 }
296 }
297 if (pVM->hwaccm.s.vmx.pMemObjAPIC != NIL_RTR0MEMOBJ)
298 {
299 RTR0MemObjFree(pVM->hwaccm.s.vmx.pMemObjAPIC, false);
300 pVM->hwaccm.s.vmx.pMemObjAPIC = NIL_RTR0MEMOBJ;
301 pVM->hwaccm.s.vmx.pAPIC = 0;
302 pVM->hwaccm.s.vmx.pAPICPhys = 0;
303 }
304 if (pVM->hwaccm.s.vmx.pMemObjMSRBitmap != NIL_RTR0MEMOBJ)
305 {
306 RTR0MemObjFree(pVM->hwaccm.s.vmx.pMemObjMSRBitmap, false);
307 pVM->hwaccm.s.vmx.pMemObjMSRBitmap = NIL_RTR0MEMOBJ;
308 pVM->hwaccm.s.vmx.pMSRBitmap = 0;
309 pVM->hwaccm.s.vmx.pMSRBitmapPhys = 0;
310 }
311#ifdef VBOX_WITH_CRASHDUMP_MAGIC
312 if (pVM->hwaccm.s.vmx.pMemObjScratch != NIL_RTR0MEMOBJ)
313 {
314 ASMMemZero32(pVM->hwaccm.s.vmx.pScratch, PAGE_SIZE);
315 RTR0MemObjFree(pVM->hwaccm.s.vmx.pMemObjScratch, false);
316 pVM->hwaccm.s.vmx.pMemObjScratch = NIL_RTR0MEMOBJ;
317 pVM->hwaccm.s.vmx.pScratch = 0;
318 pVM->hwaccm.s.vmx.pScratchPhys = 0;
319 }
320#endif
321 return VINF_SUCCESS;
322}
323
324/**
325 * Sets up VT-x for the specified VM
326 *
327 * @returns VBox status code.
328 * @param pVM The VM to operate on.
329 */
330VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
331{
332 int rc = VINF_SUCCESS;
333 uint32_t val;
334
335 AssertReturn(pVM, VERR_INVALID_PARAMETER);
336
337 for (unsigned i=0;i<pVM->cCPUs;i++)
338 {
339 PVMCPU pVCpu = &pVM->aCpus[i];
340
341 Assert(pVCpu->hwaccm.s.vmx.pVMCS);
342
343 /* Set revision dword at the beginning of the VMCS structure. */
344 *(uint32_t *)pVCpu->hwaccm.s.vmx.pVMCS = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hwaccm.s.vmx.msr.vmx_basic_info);
345
346 /* Clear VM Control Structure. */
347 Log(("pVMCSPhys = %RHp\n", pVCpu->hwaccm.s.vmx.pVMCSPhys));
348 rc = VMXClearVMCS(pVCpu->hwaccm.s.vmx.pVMCSPhys);
349 if (RT_FAILURE(rc))
350 goto vmx_end;
351
352 /* Activate the VM Control Structure. */
353 rc = VMXActivateVMCS(pVCpu->hwaccm.s.vmx.pVMCSPhys);
354 if (RT_FAILURE(rc))
355 goto vmx_end;
356
357 /* VMX_VMCS_CTRL_PIN_EXEC_CONTROLS
358 * Set required bits to one and zero according to the MSR capabilities.
359 */
360 val = pVM->hwaccm.s.vmx.msr.vmx_pin_ctls.n.disallowed0;
361 /* External and non-maskable interrupts cause VM-exits. */
362 val = val | VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_EXT_INT_EXIT | VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_NMI_EXIT;
363 val &= pVM->hwaccm.s.vmx.msr.vmx_pin_ctls.n.allowed1;
364
365 rc = VMXWriteVMCS(VMX_VMCS_CTRL_PIN_EXEC_CONTROLS, val);
366 AssertRC(rc);
367
368 /* VMX_VMCS_CTRL_PROC_EXEC_CONTROLS
369 * Set required bits to one and zero according to the MSR capabilities.
370 */
371 val = pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.disallowed0;
372 /* Program which event cause VM-exits and which features we want to use. */
373 val = val | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_HLT_EXIT
374 | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_TSC_OFFSET
375 | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT
376 | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_UNCOND_IO_EXIT
377 | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDPMC_EXIT
378 | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MWAIT_EXIT; /* don't execute mwait or else we'll idle inside the guest (host thinks the cpu load is high) */
379
380 /* Without nested paging we should intercept invlpg and cr3 mov instructions. */
381 if (!pVM->hwaccm.s.fNestedPaging)
382 val |= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INVLPG_EXIT
383 | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
384 | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT;
385
386 /* Note: VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MWAIT_EXIT might cause a vmlaunch failure with an invalid control fields error. (combined with some other exit reasons) */
387 if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
388 {
389 /* CR8 reads from the APIC shadow page; writes cause an exit is they lower the TPR below the threshold */
390 val |= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW;
391 Assert(pVM->hwaccm.s.vmx.pAPIC);
392 }
393 else
394 /* Exit on CR8 reads & writes in case the TPR shadow feature isn't present. */
395 val |= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR8_STORE_EXIT | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR8_LOAD_EXIT;
396
397#ifdef VBOX_WITH_VTX_MSR_BITMAPS
398 if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
399 {
400 Assert(pVM->hwaccm.s.vmx.pMSRBitmapPhys);
401 val |= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS;
402 }
403#endif
404
405 /* We will use the secondary control if it's present. */
406 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
407
408 /* Mask away the bits that the CPU doesn't support */
409 /** @todo make sure they don't conflict with the above requirements. */
410 val &= pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1;
411 pVCpu->hwaccm.s.vmx.proc_ctls = val;
412
413 rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, val);
414 AssertRC(rc);
415
416 if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
417 {
418 /* VMX_VMCS_CTRL_PROC_EXEC_CONTROLS2
419 * Set required bits to one and zero according to the MSR capabilities.
420 */
421 val = pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2.n.disallowed0;
422 val |= VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT;
423
424#ifdef HWACCM_VTX_WITH_EPT
425 if (pVM->hwaccm.s.fNestedPaging)
426 val |= VMX_VMCS_CTRL_PROC_EXEC2_EPT;
427#endif /* HWACCM_VTX_WITH_EPT */
428#ifdef HWACCM_VTX_WITH_VPID
429 else
430 if (pVM->hwaccm.s.vmx.fVPID)
431 val |= VMX_VMCS_CTRL_PROC_EXEC2_VPID;
432#endif /* HWACCM_VTX_WITH_VPID */
433
434 if (pVM->hwaccm.s.fHasIoApic)
435 val |= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC;
436
437 /* Mask away the bits that the CPU doesn't support */
438 /** @todo make sure they don't conflict with the above requirements. */
439 val &= pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2.n.allowed1;
440 pVCpu->hwaccm.s.vmx.proc_ctls2 = val;
441 rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS2, val);
442 AssertRC(rc);
443 }
444
445 /* VMX_VMCS_CTRL_CR3_TARGET_COUNT
446 * Set required bits to one and zero according to the MSR capabilities.
447 */
448 rc = VMXWriteVMCS(VMX_VMCS_CTRL_CR3_TARGET_COUNT, 0);
449 AssertRC(rc);
450
451 /* Forward all exception except #NM & #PF to the guest.
452 * We always need to check pagefaults since our shadow page table can be out of sync.
453 * And we always lazily sync the FPU & XMM state.
454 */
455
456 /** @todo Possible optimization:
457 * Keep the FPU and XMM state current in the EM thread. That way there's no need to
458 * lazily sync anything, but the downside is that we can't use the FPU stack or XMM
459 * registers ourselves of course.
460 *
461 * Note: only possible if the current state is actually ours (X86_CR0_TS flag)
462 */
463
464 /* Don't filter page faults; all of them should cause a switch. */
465 rc = VMXWriteVMCS(VMX_VMCS_CTRL_PAGEFAULT_ERROR_MASK, 0);
466 rc |= VMXWriteVMCS(VMX_VMCS_CTRL_PAGEFAULT_ERROR_MATCH, 0);
467 AssertRC(rc);
468
469 /* Init TSC offset to zero. */
470 rc = VMXWriteVMCS64(VMX_VMCS_CTRL_TSC_OFFSET_FULL, 0);
471 AssertRC(rc);
472
473 rc = VMXWriteVMCS64(VMX_VMCS_CTRL_IO_BITMAP_A_FULL, 0);
474 AssertRC(rc);
475
476 rc = VMXWriteVMCS64(VMX_VMCS_CTRL_IO_BITMAP_B_FULL, 0);
477 AssertRC(rc);
478
479 /* Set the MSR bitmap address. */
480 if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
481 {
482 /* Optional */
483 rc = VMXWriteVMCS64(VMX_VMCS_CTRL_MSR_BITMAP_FULL, pVM->hwaccm.s.vmx.pMSRBitmapPhys);
484 AssertRC(rc);
485 }
486
487 /* Clear MSR controls. */
488 rc = VMXWriteVMCS64(VMX_VMCS_CTRL_VMEXIT_MSR_STORE_FULL, 0);
489 rc |= VMXWriteVMCS64(VMX_VMCS_CTRL_VMEXIT_MSR_LOAD_FULL, 0);
490 rc |= VMXWriteVMCS64(VMX_VMCS_CTRL_VMENTRY_MSR_LOAD_FULL, 0);
491 rc |= VMXWriteVMCS(VMX_VMCS_CTRL_EXIT_MSR_STORE_COUNT, 0);
492 rc |= VMXWriteVMCS(VMX_VMCS_CTRL_EXIT_MSR_LOAD_COUNT, 0);
493 AssertRC(rc);
494
495 if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
496 {
497 Assert(pVM->hwaccm.s.vmx.pMemObjAPIC);
498 /* Optional */
499 rc = VMXWriteVMCS(VMX_VMCS_CTRL_TPR_THRESHOLD, 0);
500 rc |= VMXWriteVMCS64(VMX_VMCS_CTRL_VAPIC_PAGEADDR_FULL, pVCpu->hwaccm.s.vmx.pVAPICPhys);
501
502 if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
503 rc |= VMXWriteVMCS64(VMX_VMCS_CTRL_APIC_ACCESSADDR_FULL, pVM->hwaccm.s.vmx.pAPICPhys);
504
505 AssertRC(rc);
506 }
507
508 /* Set link pointer to -1. Not currently used. */
509 rc = VMXWriteVMCS64(VMX_VMCS_GUEST_LINK_PTR_FULL, 0xFFFFFFFFFFFFFFFFULL);
510 AssertRC(rc);
511
512 /* Clear VM Control Structure. Marking it inactive, clearing implementation specific data and writing back VMCS data to memory. */
513 rc = VMXClearVMCS(pVCpu->hwaccm.s.vmx.pVMCSPhys);
514 AssertRC(rc);
515
516 /* Configure the VMCS read cache. */
517 PVMCSCACHE pCache = &pVCpu->hwaccm.s.vmx.VMCSCache;
518
519 VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_RIP);
520 VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_RSP);
521 VMXSetupCachedReadVMCS(pCache, VMX_VMCS_GUEST_RFLAGS);
522 VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE);
523 VMXSetupCachedReadVMCS(pCache, VMX_VMCS_CTRL_CR0_READ_SHADOW);
524 VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_CR0);
525 VMXSetupCachedReadVMCS(pCache, VMX_VMCS_CTRL_CR4_READ_SHADOW);
526 VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_CR4);
527 VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_DR7);
528 VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_GUEST_SYSENTER_CS);
529 VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_SYSENTER_EIP);
530 VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_SYSENTER_ESP);
531 VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_GUEST_GDTR_LIMIT);
532 VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_GDTR_BASE);
533 VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_GUEST_IDTR_LIMIT);
534 VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_IDTR_BASE);
535
536 VMX_SETUP_SELREG(ES, pCache);
537 VMX_SETUP_SELREG(SS, pCache);
538 VMX_SETUP_SELREG(CS, pCache);
539 VMX_SETUP_SELREG(DS, pCache);
540 VMX_SETUP_SELREG(FS, pCache);
541 VMX_SETUP_SELREG(GS, pCache);
542 VMX_SETUP_SELREG(LDTR, pCache);
543 VMX_SETUP_SELREG(TR, pCache);
544
545 /* Status code VMCS reads. */
546 VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_EXIT_REASON);
547 VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_VM_INSTR_ERROR);
548 VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_EXIT_INSTR_LENGTH);
549 VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_EXIT_INTERRUPTION_ERRCODE);
550 VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO);
551 VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_EXIT_INSTR_INFO);
552 VMXSetupCachedReadVMCS(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
553 VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_IDT_INFO);
554 VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_IDT_ERRCODE);
555
556 if (pVM->hwaccm.s.fNestedPaging)
557 {
558 VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_CR3);
559 VMXSetupCachedReadVMCS(pCache, VMX_VMCS_EXIT_PHYS_ADDR_FULL);
560 pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
561 }
562 else
563 pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
564 } /* for each VMCPU */
565
566 /* Choose the right TLB setup function. */
567 if (pVM->hwaccm.s.fNestedPaging)
568 {
569 pVM->hwaccm.s.vmx.pfnSetupTaggedTLB = vmxR0SetupTLBEPT;
570
571 /* Default values for flushing. */
572 pVM->hwaccm.s.vmx.enmFlushPage = VMX_FLUSH_ALL_CONTEXTS;
573 pVM->hwaccm.s.vmx.enmFlushContext = VMX_FLUSH_ALL_CONTEXTS;
574
575 /* If the capabilities specify we can do more, then make use of it. */
576 if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVEPT_CAPS_INDIV)
577 pVM->hwaccm.s.vmx.enmFlushPage = VMX_FLUSH_PAGE;
578 else
579 if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVEPT_CAPS_CONTEXT)
580 pVM->hwaccm.s.vmx.enmFlushPage = VMX_FLUSH_SINGLE_CONTEXT;
581
582 if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVEPT_CAPS_CONTEXT)
583 pVM->hwaccm.s.vmx.enmFlushContext = VMX_FLUSH_SINGLE_CONTEXT;
584 }
585#ifdef HWACCM_VTX_WITH_VPID
586 else
587 if (pVM->hwaccm.s.vmx.fVPID)
588 {
589 pVM->hwaccm.s.vmx.pfnSetupTaggedTLB = vmxR0SetupTLBVPID;
590
591 /* Default values for flushing. */
592 pVM->hwaccm.s.vmx.enmFlushPage = VMX_FLUSH_ALL_CONTEXTS;
593 pVM->hwaccm.s.vmx.enmFlushContext = VMX_FLUSH_ALL_CONTEXTS;
594
595 /* If the capabilities specify we can do more, then make use of it. */
596 if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVVPID_CAPS_INDIV)
597 pVM->hwaccm.s.vmx.enmFlushPage = VMX_FLUSH_PAGE;
598 else
599 if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVVPID_CAPS_CONTEXT)
600 pVM->hwaccm.s.vmx.enmFlushPage = VMX_FLUSH_SINGLE_CONTEXT;
601
602 if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVVPID_CAPS_CONTEXT)
603 pVM->hwaccm.s.vmx.enmFlushContext = VMX_FLUSH_SINGLE_CONTEXT;
604 }
605#endif /* HWACCM_VTX_WITH_VPID */
606 else
607 pVM->hwaccm.s.vmx.pfnSetupTaggedTLB = vmxR0SetupTLBDummy;
608
609vmx_end:
610 VMXR0CheckError(pVM, &pVM->aCpus[0], rc);
611 return rc;
612}
613
614
615/**
616 * Injects an event (trap or external interrupt)
617 *
618 * @returns VBox status code.
619 * @param pVM The VM to operate on.
620 * @param pVCpu The VMCPU to operate on.
621 * @param pCtx CPU Context
622 * @param intInfo VMX interrupt info
623 * @param cbInstr Opcode length of faulting instruction
624 * @param errCode Error code (optional)
625 */
626static int VMXR0InjectEvent(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, uint32_t intInfo, uint32_t cbInstr, uint32_t errCode)
627{
628 int rc;
629 uint32_t iGate = VMX_EXIT_INTERRUPTION_INFO_VECTOR(intInfo);
630
631#ifdef VBOX_WITH_STATISTICS
632 STAM_COUNTER_INC(&pVCpu->hwaccm.s.paStatInjectedIrqsR0[iGate & MASK_INJECT_IRQ_STAT]);
633#endif
634
635#ifdef VBOX_STRICT
636 if (iGate == 0xE)
637 LogFlow(("VMXR0InjectEvent: Injecting interrupt %d at %RGv error code=%08x CR2=%RGv intInfo=%08x\n", iGate, (RTGCPTR)pCtx->rip, errCode, pCtx->cr2, intInfo));
638 else
639 if (iGate < 0x20)
640 LogFlow(("VMXR0InjectEvent: Injecting interrupt %d at %RGv error code=%08x\n", iGate, (RTGCPTR)pCtx->rip, errCode));
641 else
642 {
643 LogFlow(("INJ-EI: %x at %RGv\n", iGate, (RTGCPTR)pCtx->rip));
644 Assert(VMX_EXIT_INTERRUPTION_INFO_TYPE(intInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW || !VMCPU_FF_ISSET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
645 Assert(VMX_EXIT_INTERRUPTION_INFO_TYPE(intInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW || pCtx->eflags.u32 & X86_EFL_IF);
646 }
647#endif
648
649#ifdef HWACCM_VMX_EMULATE_REALMODE
650 if (CPUMIsGuestInRealModeEx(pCtx))
651 {
652 RTGCPHYS GCPhysHandler;
653 uint16_t offset, ip;
654 RTSEL sel;
655
656 /* Injecting events doesn't work right with real mode emulation.
657 * (#GP if we try to inject external hardware interrupts)
658 * Inject the interrupt or trap directly instead.
659 *
660 * ASSUMES no access handlers for the bits we read or write below (should be safe).
661 */
662 Log(("Manual interrupt/trap '%x' inject (real mode)\n", iGate));
663
664 /* Check if the interrupt handler is present. */
665 if (iGate * 4 + 3 > pCtx->idtr.cbIdt)
666 {
667 Log(("IDT cbIdt violation\n"));
668 if (iGate != X86_XCPT_DF)
669 {
670 RTGCUINTPTR intInfo;
671
672 intInfo = (iGate == X86_XCPT_GP) ? (uint32_t)X86_XCPT_DF : iGate;
673 intInfo |= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
674 intInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
675 intInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HWEXCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
676
677 return VMXR0InjectEvent(pVM, pVCpu, pCtx, intInfo, 0, 0 /* no error code according to the Intel docs */);
678 }
679 Log(("Triple fault -> reset the VM!\n"));
680 return VINF_EM_RESET;
681 }
682 if ( VMX_EXIT_INTERRUPTION_INFO_TYPE(intInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW
683 || iGate == 3 /* Both #BP and #OF point to the instruction after. */
684 || iGate == 4)
685 {
686 ip = pCtx->ip + cbInstr;
687 }
688 else
689 ip = pCtx->ip;
690
691 /* Read the selector:offset pair of the interrupt handler. */
692 GCPhysHandler = (RTGCPHYS)pCtx->idtr.pIdt + iGate * 4;
693 rc = PGMPhysSimpleReadGCPhys(pVM, &offset, GCPhysHandler, sizeof(offset)); AssertRC(rc);
694 rc = PGMPhysSimpleReadGCPhys(pVM, &sel, GCPhysHandler + 2, sizeof(sel)); AssertRC(rc);
695
696 LogFlow(("IDT handler %04X:%04X\n", sel, offset));
697
698 /* Construct the stack frame. */
699 /** @todo should check stack limit. */
700 pCtx->sp -= 2;
701 LogFlow(("ss:sp %04X:%04X eflags=%x\n", pCtx->ss, pCtx->sp, pCtx->eflags.u));
702 rc = PGMPhysSimpleWriteGCPhys(pVM, pCtx->ssHid.u64Base + pCtx->sp, &pCtx->eflags, sizeof(uint16_t)); AssertRC(rc);
703 pCtx->sp -= 2;
704 LogFlow(("ss:sp %04X:%04X cs=%x\n", pCtx->ss, pCtx->sp, pCtx->cs));
705 rc = PGMPhysSimpleWriteGCPhys(pVM, pCtx->ssHid.u64Base + pCtx->sp, &pCtx->cs, sizeof(uint16_t)); AssertRC(rc);
706 pCtx->sp -= 2;
707 LogFlow(("ss:sp %04X:%04X ip=%x\n", pCtx->ss, pCtx->sp, ip));
708 rc = PGMPhysSimpleWriteGCPhys(pVM, pCtx->ssHid.u64Base + pCtx->sp, &ip, sizeof(ip)); AssertRC(rc);
709
710 /* Update the CPU state for executing the handler. */
711 pCtx->rip = offset;
712 pCtx->cs = sel;
713 pCtx->csHid.u64Base = sel << 4;
714 pCtx->eflags.u &= ~(X86_EFL_IF|X86_EFL_TF|X86_EFL_RF|X86_EFL_AC);
715
716 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_GUEST_SEGMENT_REGS;
717 return VINF_SUCCESS;
718 }
719#endif /* HWACCM_VMX_EMULATE_REALMODE */
720
721 /* Set event injection state. */
722 rc = VMXWriteVMCS(VMX_VMCS_CTRL_ENTRY_IRQ_INFO, intInfo | (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT));
723
724 rc |= VMXWriteVMCS(VMX_VMCS_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
725 rc |= VMXWriteVMCS(VMX_VMCS_CTRL_ENTRY_EXCEPTION_ERRCODE, errCode);
726
727 AssertRC(rc);
728 return rc;
729}
730
731
732/**
733 * Checks for pending guest interrupts and injects them
734 *
735 * @returns VBox status code.
736 * @param pVM The VM to operate on.
737 * @param pVCpu The VMCPU to operate on.
738 * @param pCtx CPU Context
739 */
740static int VMXR0CheckPendingInterrupt(PVM pVM, PVMCPU pVCpu, CPUMCTX *pCtx)
741{
742 int rc;
743
744 /* Dispatch any pending interrupts. (injected before, but a VM exit occurred prematurely) */
745 if (pVCpu->hwaccm.s.Event.fPending)
746 {
747 Log(("CPU%d: Reinjecting event %RX64 %08x at %RGv cr2=%RX64\n", pVCpu->idCpu, pVCpu->hwaccm.s.Event.intInfo, pVCpu->hwaccm.s.Event.errCode, (RTGCPTR)pCtx->rip, pCtx->cr2));
748 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatIntReinject);
749 rc = VMXR0InjectEvent(pVM, pVCpu, pCtx, pVCpu->hwaccm.s.Event.intInfo, 0, pVCpu->hwaccm.s.Event.errCode);
750 AssertRC(rc);
751
752 pVCpu->hwaccm.s.Event.fPending = false;
753 return VINF_SUCCESS;
754 }
755
756 /* If an active trap is already pending, then we must forward it first! */
757 if (!TRPMHasTrap(pVCpu))
758 {
759 if (VMCPU_FF_TESTANDCLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI))
760 {
761 RTGCUINTPTR intInfo;
762
763 Log(("CPU%d: injecting #NMI\n", pVCpu->idCpu));
764
765 intInfo = X86_XCPT_NMI;
766 intInfo |= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
767 intInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
768
769 rc = VMXR0InjectEvent(pVM, pVCpu, pCtx, intInfo, 0, 0);
770 AssertRC(rc);
771
772 return VINF_SUCCESS;
773 }
774
775 /* @todo SMI interrupts. */
776
777 /* When external interrupts are pending, we should exit the VM when IF is set. */
778 if (VMCPU_FF_ISPENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC|VMCPU_FF_INTERRUPT_PIC)))
779 {
780 if (!(pCtx->eflags.u32 & X86_EFL_IF))
781 {
782 if (!(pVCpu->hwaccm.s.vmx.proc_ctls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_IRQ_WINDOW_EXIT))
783 {
784 LogFlow(("Enable irq window exit!\n"));
785 pVCpu->hwaccm.s.vmx.proc_ctls |= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_IRQ_WINDOW_EXIT;
786 rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
787 AssertRC(rc);
788 }
789 /* else nothing to do but wait */
790 }
791 else
792 if (!VMCPU_FF_ISSET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
793 {
794 uint8_t u8Interrupt;
795
796 rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
797 Log(("CPU%d: Dispatch interrupt: u8Interrupt=%x (%d) rc=%Rrc cs:rip=%04X:%RGv\n", pVCpu->idCpu, u8Interrupt, u8Interrupt, rc, pCtx->cs, (RTGCPTR)pCtx->rip));
798 if (RT_SUCCESS(rc))
799 {
800 rc = TRPMAssertTrap(pVCpu, u8Interrupt, TRPM_HARDWARE_INT);
801 AssertRC(rc);
802 }
803 else
804 {
805 /* Can only happen in rare cases where a pending interrupt is cleared behind our back */
806 Assert(!VMCPU_FF_ISPENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC|VMCPU_FF_INTERRUPT_PIC)));
807 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatSwitchGuestIrq);
808 /* Just continue */
809 }
810 }
811 else
812 Log(("Pending interrupt blocked at %RGv by VM_FF_INHIBIT_INTERRUPTS!!\n", (RTGCPTR)pCtx->rip));
813 }
814 }
815
816#ifdef VBOX_STRICT
817 if (TRPMHasTrap(pVCpu))
818 {
819 uint8_t u8Vector;
820 rc = TRPMQueryTrapAll(pVCpu, &u8Vector, 0, 0, 0);
821 AssertRC(rc);
822 }
823#endif
824
825 if ( (pCtx->eflags.u32 & X86_EFL_IF)
826 && (!VMCPU_FF_ISSET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
827 && TRPMHasTrap(pVCpu)
828 )
829 {
830 uint8_t u8Vector;
831 int rc;
832 TRPMEVENT enmType;
833 RTGCUINTPTR intInfo;
834 RTGCUINT errCode;
835
836 /* If a new event is pending, then dispatch it now. */
837 rc = TRPMQueryTrapAll(pVCpu, &u8Vector, &enmType, &errCode, 0);
838 AssertRC(rc);
839 Assert(pCtx->eflags.Bits.u1IF == 1 || enmType == TRPM_TRAP);
840 Assert(enmType != TRPM_SOFTWARE_INT);
841
842 /* Clear the pending trap. */
843 rc = TRPMResetTrap(pVCpu);
844 AssertRC(rc);
845
846 intInfo = u8Vector;
847 intInfo |= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
848
849 if (enmType == TRPM_TRAP)
850 {
851 switch (u8Vector) {
852 case 8:
853 case 10:
854 case 11:
855 case 12:
856 case 13:
857 case 14:
858 case 17:
859 /* Valid error codes. */
860 intInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
861 break;
862 default:
863 break;
864 }
865 if (u8Vector == X86_XCPT_BP || u8Vector == X86_XCPT_OF)
866 intInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SWEXCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
867 else
868 intInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HWEXCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
869 }
870 else
871 intInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
872
873 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatIntInject);
874 rc = VMXR0InjectEvent(pVM, pVCpu, pCtx, intInfo, 0, errCode);
875 AssertRC(rc);
876 } /* if (interrupts can be dispatched) */
877
878 return VINF_SUCCESS;
879}
880
881/**
882 * Save the host state
883 *
884 * @returns VBox status code.
885 * @param pVM The VM to operate on.
886 * @param pVCpu The VMCPU to operate on.
887 */
888VMMR0DECL(int) VMXR0SaveHostState(PVM pVM, PVMCPU pVCpu)
889{
890 int rc = VINF_SUCCESS;
891
892 /*
893 * Host CPU Context
894 */
895 if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_HOST_CONTEXT)
896 {
897 RTIDTR idtr;
898 RTGDTR gdtr;
899 RTSEL SelTR;
900 PX86DESCHC pDesc;
901 uintptr_t trBase;
902 RTSEL cs;
903 RTSEL ss;
904 uint64_t cr3;
905
906 /* Control registers */
907 rc = VMXWriteVMCS(VMX_VMCS_HOST_CR0, ASMGetCR0());
908#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
909 if (VMX_IS_64BIT_HOST_MODE())
910 {
911 cr3 = hwaccmR0Get64bitCR3();
912 rc |= VMXWriteVMCS64(VMX_VMCS_HOST_CR3, cr3);
913 }
914 else
915#endif
916 {
917 cr3 = ASMGetCR3();
918 rc |= VMXWriteVMCS(VMX_VMCS_HOST_CR3, cr3);
919 }
920 rc |= VMXWriteVMCS(VMX_VMCS_HOST_CR4, ASMGetCR4());
921 AssertRC(rc);
922 Log2(("VMX_VMCS_HOST_CR0 %08x\n", ASMGetCR0()));
923 Log2(("VMX_VMCS_HOST_CR3 %08RX64\n", cr3));
924 Log2(("VMX_VMCS_HOST_CR4 %08x\n", ASMGetCR4()));
925
926 /* Selector registers. */
927#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
928 if (VMX_IS_64BIT_HOST_MODE())
929 {
930 cs = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelCS;
931 ss = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelSS;
932 }
933 else
934 {
935 /* sysenter loads LDT cs & ss, VMX doesn't like this. Load the GDT ones (safe). */
936 cs = (RTSEL)(uintptr_t)&SUPR0AbsKernelCS;
937 ss = (RTSEL)(uintptr_t)&SUPR0AbsKernelSS;
938 }
939#else
940 cs = ASMGetCS();
941 ss = ASMGetSS();
942#endif
943 Assert(!(cs & X86_SEL_LDT)); Assert((cs & X86_SEL_RPL) == 0);
944 Assert(!(ss & X86_SEL_LDT)); Assert((ss & X86_SEL_RPL) == 0);
945 rc = VMXWriteVMCS(VMX_VMCS16_HOST_FIELD_CS, cs);
946 /* Note: VMX is (again) very picky about the RPL of the selectors here; we'll restore them manually. */
947 rc |= VMXWriteVMCS(VMX_VMCS16_HOST_FIELD_DS, 0);
948 rc |= VMXWriteVMCS(VMX_VMCS16_HOST_FIELD_ES, 0);
949#if HC_ARCH_BITS == 32
950 if (!VMX_IS_64BIT_HOST_MODE())
951 {
952 rc |= VMXWriteVMCS(VMX_VMCS16_HOST_FIELD_FS, 0);
953 rc |= VMXWriteVMCS(VMX_VMCS16_HOST_FIELD_GS, 0);
954 }
955#endif
956 rc |= VMXWriteVMCS(VMX_VMCS16_HOST_FIELD_SS, ss);
957 SelTR = ASMGetTR();
958 rc |= VMXWriteVMCS(VMX_VMCS16_HOST_FIELD_TR, SelTR);
959 AssertRC(rc);
960 Log2(("VMX_VMCS_HOST_FIELD_CS %08x (%08x)\n", cs, ASMGetSS()));
961 Log2(("VMX_VMCS_HOST_FIELD_DS 00000000 (%08x)\n", ASMGetDS()));
962 Log2(("VMX_VMCS_HOST_FIELD_ES 00000000 (%08x)\n", ASMGetES()));
963 Log2(("VMX_VMCS_HOST_FIELD_FS 00000000 (%08x)\n", ASMGetFS()));
964 Log2(("VMX_VMCS_HOST_FIELD_GS 00000000 (%08x)\n", ASMGetGS()));
965 Log2(("VMX_VMCS_HOST_FIELD_SS %08x (%08x)\n", ss, ASMGetSS()));
966 Log2(("VMX_VMCS_HOST_FIELD_TR %08x\n", ASMGetTR()));
967
968 /* GDTR & IDTR */
969#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
970 if (VMX_IS_64BIT_HOST_MODE())
971 {
972 X86XDTR64 gdtr64, idtr64;
973 hwaccmR0Get64bitGDTRandIDTR(&gdtr64, &idtr64);
974 rc = VMXWriteVMCS64(VMX_VMCS_HOST_GDTR_BASE, gdtr64.uAddr);
975 rc |= VMXWriteVMCS64(VMX_VMCS_HOST_IDTR_BASE, gdtr64.uAddr);
976 AssertRC(rc);
977 Log2(("VMX_VMCS_HOST_GDTR_BASE %RX64\n", gdtr64.uAddr));
978 Log2(("VMX_VMCS_HOST_IDTR_BASE %RX64\n", idtr64.uAddr));
979 gdtr.cbGdt = gdtr64.cb;
980 gdtr.pGdt = (uintptr_t)gdtr64.uAddr;
981 }
982 else
983#endif
984 {
985 ASMGetGDTR(&gdtr);
986 rc = VMXWriteVMCS(VMX_VMCS_HOST_GDTR_BASE, gdtr.pGdt);
987 ASMGetIDTR(&idtr);
988 rc |= VMXWriteVMCS(VMX_VMCS_HOST_IDTR_BASE, idtr.pIdt);
989 AssertRC(rc);
990 Log2(("VMX_VMCS_HOST_GDTR_BASE %RHv\n", gdtr.pGdt));
991 Log2(("VMX_VMCS_HOST_IDTR_BASE %RHv\n", idtr.pIdt));
992 }
993
994
995 /* Save the base address of the TR selector. */
996 if (SelTR > gdtr.cbGdt)
997 {
998 AssertMsgFailed(("Invalid TR selector %x. GDTR.cbGdt=%x\n", SelTR, gdtr.cbGdt));
999 return VERR_VMX_INVALID_HOST_STATE;
1000 }
1001
1002#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
1003 if (VMX_IS_64BIT_HOST_MODE())
1004 {
1005 pDesc = &((PX86DESCHC)gdtr.pGdt)[SelTR >> X86_SEL_SHIFT_HC]; /// ????
1006 uint64_t trBase64 = X86DESC64_BASE(*(PX86DESC64)pDesc);
1007 rc = VMXWriteVMCS64(VMX_VMCS_HOST_TR_BASE, trBase64);
1008 Log2(("VMX_VMCS_HOST_TR_BASE %RX64\n", trBase64));
1009 AssertRC(rc);
1010 }
1011 else
1012#endif
1013 {
1014 pDesc = &((PX86DESCHC)gdtr.pGdt)[SelTR >> X86_SEL_SHIFT_HC];
1015#if HC_ARCH_BITS == 64
1016 trBase = X86DESC64_BASE(*pDesc);
1017#else
1018 trBase = X86DESC_BASE(*pDesc);
1019#endif
1020 rc = VMXWriteVMCS(VMX_VMCS_HOST_TR_BASE, trBase);
1021 AssertRC(rc);
1022 Log2(("VMX_VMCS_HOST_TR_BASE %RHv\n", trBase));
1023 }
1024
1025 /* FS and GS base. */
1026#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
1027 if (VMX_IS_64BIT_HOST_MODE())
1028 {
1029 Log2(("MSR_K8_FS_BASE = %RX64\n", ASMRdMsr(MSR_K8_FS_BASE)));
1030 Log2(("MSR_K8_GS_BASE = %RX64\n", ASMRdMsr(MSR_K8_GS_BASE)));
1031 rc = VMXWriteVMCS64(VMX_VMCS_HOST_FS_BASE, ASMRdMsr(MSR_K8_FS_BASE));
1032 rc |= VMXWriteVMCS64(VMX_VMCS_HOST_GS_BASE, ASMRdMsr(MSR_K8_GS_BASE));
1033 }
1034#endif
1035 AssertRC(rc);
1036
1037 /* Sysenter MSRs. */
1038 /** @todo expensive!! */
1039 rc = VMXWriteVMCS(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
1040 Log2(("VMX_VMCS_HOST_SYSENTER_CS %08x\n", ASMRdMsr_Low(MSR_IA32_SYSENTER_CS)));
1041#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
1042 if (VMX_IS_64BIT_HOST_MODE())
1043 {
1044 Log2(("VMX_VMCS_HOST_SYSENTER_EIP %RX64\n", ASMRdMsr(MSR_IA32_SYSENTER_EIP)));
1045 Log2(("VMX_VMCS_HOST_SYSENTER_ESP %RX64\n", ASMRdMsr(MSR_IA32_SYSENTER_ESP)));
1046 rc |= VMXWriteVMCS64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
1047 rc |= VMXWriteVMCS64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
1048 }
1049 else
1050 {
1051 rc |= VMXWriteVMCS(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
1052 rc |= VMXWriteVMCS(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
1053 Log2(("VMX_VMCS_HOST_SYSENTER_EIP %RX32\n", ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP)));
1054 Log2(("VMX_VMCS_HOST_SYSENTER_ESP %RX32\n", ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP)));
1055 }
1056#elif HC_ARCH_BITS == 32
1057 rc |= VMXWriteVMCS(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
1058 rc |= VMXWriteVMCS(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
1059 Log2(("VMX_VMCS_HOST_SYSENTER_EIP %RX32\n", ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP)));
1060 Log2(("VMX_VMCS_HOST_SYSENTER_ESP %RX32\n", ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP)));
1061#else
1062 Log2(("VMX_VMCS_HOST_SYSENTER_EIP %RX64\n", ASMRdMsr(MSR_IA32_SYSENTER_EIP)));
1063 Log2(("VMX_VMCS_HOST_SYSENTER_ESP %RX64\n", ASMRdMsr(MSR_IA32_SYSENTER_ESP)));
1064 rc |= VMXWriteVMCS64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
1065 rc |= VMXWriteVMCS64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
1066#endif
1067 AssertRC(rc);
1068
1069#if 0 /* @todo deal with 32/64 */
1070 /* Restore the host EFER - on CPUs that support it. */
1071 if (pVM->hwaccm.s.vmx.msr.vmx_exit.n.allowed1 & VMX_VMCS_CTRL_EXIT_CONTROLS_LOAD_HOST_EFER_MSR)
1072 {
1073 uint64_t msrEFER = ASMRdMsr(MSR_IA32_EFER);
1074 rc = VMXWriteVMCS64(VMX_VMCS_HOST_FIELD_EFER_FULL, msrEFER);
1075 AssertRC(rc);
1076 }
1077#endif
1078 pVCpu->hwaccm.s.fContextUseFlags &= ~HWACCM_CHANGED_HOST_CONTEXT;
1079 }
1080 return rc;
1081}
1082
1083/**
1084 * Prefetch the 4 PDPT pointers (PAE and nested paging only)
1085 *
1086 * @param pVM The VM to operate on.
1087 * @param pVCpu The VMCPU to operate on.
1088 * @param pCtx Guest context
1089 */
1090static void vmxR0PrefetchPAEPdptrs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
1091{
1092 if (CPUMIsGuestInPAEModeEx(pCtx))
1093 {
1094 X86PDPE Pdpe;
1095
1096 for (unsigned i=0;i<4;i++)
1097 {
1098 Pdpe = PGMGstGetPaePDPtr(pVCpu, i);
1099 int rc = VMXWriteVMCS64(VMX_VMCS_GUEST_PDPTR0_FULL + i*2, Pdpe.u);
1100 AssertRC(rc);
1101 }
1102 }
1103}
1104
1105/**
1106 * Update the exception bitmap according to the current CPU state
1107 *
1108 * @param pVM The VM to operate on.
1109 * @param pVCpu The VMCPU to operate on.
1110 * @param pCtx Guest context
1111 */
1112static void vmxR0UpdateExceptionBitmap(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
1113{
1114 uint32_t u32TrapMask;
1115 Assert(pCtx);
1116
1117 u32TrapMask = HWACCM_VMX_TRAP_MASK;
1118#ifndef DEBUG
1119 if (pVM->hwaccm.s.fNestedPaging)
1120 u32TrapMask &= ~RT_BIT(X86_XCPT_PF); /* no longer need to intercept #PF. */
1121#endif
1122
1123 /* Also catch floating point exceptions as we need to report them to the guest in a different way. */
1124 if ( CPUMIsGuestFPUStateActive(pVCpu) == true
1125 && !(pCtx->cr0 & X86_CR0_NE)
1126 && !pVCpu->hwaccm.s.fFPUOldStyleOverride)
1127 {
1128 u32TrapMask |= RT_BIT(X86_XCPT_MF);
1129 pVCpu->hwaccm.s.fFPUOldStyleOverride = true;
1130 }
1131
1132#ifdef DEBUG /* till after branching, enable it by default then. */
1133 /* Intercept X86_XCPT_DB if stepping is enabled */
1134 if ( DBGFIsStepping(pVCpu)
1135 || CPUMIsHyperDebugStateActive(pVCpu))
1136 u32TrapMask |= RT_BIT(X86_XCPT_DB);
1137 /** @todo Don't trap it unless the debugger has armed breakpoints. */
1138 u32TrapMask |= RT_BIT(X86_XCPT_BP);
1139#endif
1140
1141#ifdef VBOX_STRICT
1142 Assert(u32TrapMask & RT_BIT(X86_XCPT_GP));
1143#endif
1144
1145# ifdef HWACCM_VMX_EMULATE_REALMODE
1146 /* Intercept all exceptions in real mode as none of them can be injected directly (#GP otherwise). */
1147 if (CPUMIsGuestInRealModeEx(pCtx) && pVM->hwaccm.s.vmx.pRealModeTSS)
1148 u32TrapMask |= HWACCM_VMX_TRAP_MASK_REALMODE;
1149# endif /* HWACCM_VMX_EMULATE_REALMODE */
1150
1151 int rc = VMXWriteVMCS(VMX_VMCS_CTRL_EXCEPTION_BITMAP, u32TrapMask);
1152 AssertRC(rc);
1153}
1154
1155/**
1156 * Loads the guest state
1157 *
1158 * NOTE: Don't do anything here that can cause a jump back to ring 3!!!!!
1159 *
1160 * @returns VBox status code.
1161 * @param pVM The VM to operate on.
1162 * @param pVCpu The VMCPU to operate on.
1163 * @param pCtx Guest context
1164 */
1165VMMR0DECL(int) VMXR0LoadGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
1166{
1167 int rc = VINF_SUCCESS;
1168 RTGCUINTPTR val;
1169 X86EFLAGS eflags;
1170
1171 /* VMX_VMCS_CTRL_ENTRY_CONTROLS
1172 * Set required bits to one and zero according to the MSR capabilities.
1173 */
1174 val = pVM->hwaccm.s.vmx.msr.vmx_entry.n.disallowed0;
1175 /* Load guest debug controls (dr7 & IA32_DEBUGCTL_MSR) (forced to 1 on the 'first' VT-x capable CPUs; this actually includes the newest Nehalem CPUs) */
1176 val |= VMX_VMCS_CTRL_ENTRY_CONTROLS_LOAD_DEBUG;
1177#if 0 /* @todo deal with 32/64 */
1178 /* Required for the EFER write below, not supported on all CPUs. */
1179 val |= VMX_VMCS_CTRL_ENTRY_CONTROLS_LOAD_GUEST_EFER_MSR;
1180#endif
1181 /* 64 bits guest mode? */
1182 if (CPUMIsGuestInLongModeEx(pCtx))
1183 val |= VMX_VMCS_CTRL_ENTRY_CONTROLS_IA64_MODE;
1184 /* else Must be zero when AMD64 is not available. */
1185
1186 /* Mask away the bits that the CPU doesn't support */
1187 val &= pVM->hwaccm.s.vmx.msr.vmx_entry.n.allowed1;
1188 rc = VMXWriteVMCS(VMX_VMCS_CTRL_ENTRY_CONTROLS, val);
1189 AssertRC(rc);
1190
1191 /* VMX_VMCS_CTRL_EXIT_CONTROLS
1192 * Set required bits to one and zero according to the MSR capabilities.
1193 */
1194 val = pVM->hwaccm.s.vmx.msr.vmx_exit.n.disallowed0;
1195
1196 /* Save debug controls (dr7 & IA32_DEBUGCTL_MSR) (forced to 1 on the 'first' VT-x capable CPUs; this actually includes the newest Nehalem CPUs) */
1197#if 0 /* @todo deal with 32/64 */
1198 val |= VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_DEBUG | VMX_VMCS_CTRL_EXIT_CONTROLS_LOAD_HOST_EFER_MSR;
1199#else
1200 val |= VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_DEBUG;
1201#endif
1202
1203#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
1204 if (VMX_IS_64BIT_HOST_MODE())
1205 val |= VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_AMD64;
1206 /* else: Must be zero when AMD64 is not available. */
1207#elif HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
1208 if (CPUMIsGuestInLongModeEx(pCtx))
1209 val |= VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_AMD64; /* our switcher goes to long mode */
1210 else
1211 Assert(!(val & VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_AMD64));
1212#endif
1213 val &= pVM->hwaccm.s.vmx.msr.vmx_exit.n.allowed1;
1214 /* Don't acknowledge external interrupts on VM-exit. */
1215 rc = VMXWriteVMCS(VMX_VMCS_CTRL_EXIT_CONTROLS, val);
1216 AssertRC(rc);
1217
1218 /* Guest CPU context: ES, CS, SS, DS, FS, GS. */
1219 if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_SEGMENT_REGS)
1220 {
1221#ifdef HWACCM_VMX_EMULATE_REALMODE
1222 if (pVM->hwaccm.s.vmx.pRealModeTSS)
1223 {
1224 PGMMODE enmGuestMode = PGMGetGuestMode(pVCpu);
1225 if (pVCpu->hwaccm.s.vmx.enmLastSeenGuestMode != enmGuestMode)
1226 {
1227 /* Correct weird requirements for switching to protected mode. */
1228 if ( pVCpu->hwaccm.s.vmx.enmLastSeenGuestMode == PGMMODE_REAL
1229 && enmGuestMode >= PGMMODE_PROTECTED)
1230 {
1231 /* Flush the recompiler code cache as it's not unlikely
1232 * the guest will rewrite code it will later execute in real
1233 * mode (OpenBSD 4.0 is one such example)
1234 */
1235 REMFlushTBs(pVM);
1236
1237 /* DPL of all hidden selector registers must match the current CPL (0). */
1238 pCtx->csHid.Attr.n.u2Dpl = 0;
1239 pCtx->csHid.Attr.n.u4Type = X86_SEL_TYPE_CODE | X86_SEL_TYPE_RW_ACC;
1240
1241 pCtx->dsHid.Attr.n.u2Dpl = 0;
1242 pCtx->esHid.Attr.n.u2Dpl = 0;
1243 pCtx->fsHid.Attr.n.u2Dpl = 0;
1244 pCtx->gsHid.Attr.n.u2Dpl = 0;
1245 pCtx->ssHid.Attr.n.u2Dpl = 0;
1246
1247 /* The limit must correspond to the 32 bits setting. */
1248 if (!pCtx->csHid.Attr.n.u1DefBig)
1249 pCtx->csHid.u32Limit &= 0xffff;
1250 if (!pCtx->dsHid.Attr.n.u1DefBig)
1251 pCtx->dsHid.u32Limit &= 0xffff;
1252 if (!pCtx->esHid.Attr.n.u1DefBig)
1253 pCtx->esHid.u32Limit &= 0xffff;
1254 if (!pCtx->fsHid.Attr.n.u1DefBig)
1255 pCtx->fsHid.u32Limit &= 0xffff;
1256 if (!pCtx->gsHid.Attr.n.u1DefBig)
1257 pCtx->gsHid.u32Limit &= 0xffff;
1258 if (!pCtx->ssHid.Attr.n.u1DefBig)
1259 pCtx->ssHid.u32Limit &= 0xffff;
1260 }
1261 else
1262 /* Switching from protected mode to real mode. */
1263 if ( pVCpu->hwaccm.s.vmx.enmLastSeenGuestMode >= PGMMODE_PROTECTED
1264 && enmGuestMode == PGMMODE_REAL)
1265 {
1266 /* The limit must also be set to 0xffff. */
1267 pCtx->csHid.u32Limit = 0xffff;
1268 pCtx->dsHid.u32Limit = 0xffff;
1269 pCtx->esHid.u32Limit = 0xffff;
1270 pCtx->fsHid.u32Limit = 0xffff;
1271 pCtx->gsHid.u32Limit = 0xffff;
1272 pCtx->ssHid.u32Limit = 0xffff;
1273
1274 Assert(pCtx->csHid.u64Base <= 0xfffff);
1275 Assert(pCtx->dsHid.u64Base <= 0xfffff);
1276 Assert(pCtx->esHid.u64Base <= 0xfffff);
1277 Assert(pCtx->fsHid.u64Base <= 0xfffff);
1278 Assert(pCtx->gsHid.u64Base <= 0xfffff);
1279 }
1280 pVCpu->hwaccm.s.vmx.enmLastSeenGuestMode = enmGuestMode;
1281 }
1282 else
1283 /* VT-x will fail with a guest invalid state otherwise... (CPU state after a reset) */
1284 if ( CPUMIsGuestInRealModeEx(pCtx)
1285 && pCtx->csHid.u64Base == 0xffff0000)
1286 {
1287 pCtx->csHid.u64Base = 0xf0000;
1288 pCtx->cs = 0xf000;
1289 }
1290 }
1291#endif /* HWACCM_VMX_EMULATE_REALMODE */
1292
1293 VMX_WRITE_SELREG(ES, es);
1294 AssertRC(rc);
1295
1296 VMX_WRITE_SELREG(CS, cs);
1297 AssertRC(rc);
1298
1299 VMX_WRITE_SELREG(SS, ss);
1300 AssertRC(rc);
1301
1302 VMX_WRITE_SELREG(DS, ds);
1303 AssertRC(rc);
1304
1305 /* The base values in the hidden fs & gs registers are not in sync with the msrs; they are cut to 32 bits. */
1306 VMX_WRITE_SELREG(FS, fs);
1307 AssertRC(rc);
1308
1309 VMX_WRITE_SELREG(GS, gs);
1310 AssertRC(rc);
1311 }
1312
1313 /* Guest CPU context: LDTR. */
1314 if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_LDTR)
1315 {
1316 if (pCtx->ldtr == 0)
1317 {
1318 rc = VMXWriteVMCS(VMX_VMCS16_GUEST_FIELD_LDTR, 0);
1319 rc |= VMXWriteVMCS(VMX_VMCS32_GUEST_LDTR_LIMIT, 0);
1320 rc |= VMXWriteVMCS64(VMX_VMCS64_GUEST_LDTR_BASE, 0);
1321 /* Note: vmlaunch will fail with 0 or just 0x02. No idea why. */
1322 rc |= VMXWriteVMCS(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, 0x82 /* present, LDT */);
1323 }
1324 else
1325 {
1326 rc = VMXWriteVMCS(VMX_VMCS16_GUEST_FIELD_LDTR, pCtx->ldtr);
1327 rc |= VMXWriteVMCS(VMX_VMCS32_GUEST_LDTR_LIMIT, pCtx->ldtrHid.u32Limit);
1328 rc |= VMXWriteVMCS64(VMX_VMCS64_GUEST_LDTR_BASE, pCtx->ldtrHid.u64Base);
1329 rc |= VMXWriteVMCS(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, pCtx->ldtrHid.Attr.u);
1330 }
1331 AssertRC(rc);
1332 }
1333 /* Guest CPU context: TR. */
1334 if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_TR)
1335 {
1336#ifdef HWACCM_VMX_EMULATE_REALMODE
1337 /* Real mode emulation using v86 mode with CR4.VME (interrupt redirection using the int bitmap in the TSS) */
1338 if (CPUMIsGuestInRealModeEx(pCtx))
1339 {
1340 RTGCPHYS GCPhys;
1341
1342 /* We convert it here every time as pci regions could be reconfigured. */
1343 rc = PDMVMMDevHeapR3ToGCPhys(pVM, pVM->hwaccm.s.vmx.pRealModeTSS, &GCPhys);
1344 AssertRC(rc);
1345
1346 rc = VMXWriteVMCS(VMX_VMCS16_GUEST_FIELD_TR, 0);
1347 rc |= VMXWriteVMCS(VMX_VMCS32_GUEST_TR_LIMIT, HWACCM_VTX_TSS_SIZE);
1348 rc |= VMXWriteVMCS64(VMX_VMCS64_GUEST_TR_BASE, GCPhys /* phys = virt in this mode */);
1349
1350 X86DESCATTR attr;
1351
1352 attr.u = 0;
1353 attr.n.u1Present = 1;
1354 attr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
1355 val = attr.u;
1356 }
1357 else
1358#endif /* HWACCM_VMX_EMULATE_REALMODE */
1359 {
1360 rc = VMXWriteVMCS(VMX_VMCS16_GUEST_FIELD_TR, pCtx->tr);
1361 rc |= VMXWriteVMCS(VMX_VMCS32_GUEST_TR_LIMIT, pCtx->trHid.u32Limit);
1362 rc |= VMXWriteVMCS64(VMX_VMCS64_GUEST_TR_BASE, pCtx->trHid.u64Base);
1363
1364 val = pCtx->trHid.Attr.u;
1365
1366 /* The TSS selector must be busy. */
1367 if ((val & 0xF) == X86_SEL_TYPE_SYS_286_TSS_AVAIL)
1368 val = (val & ~0xF) | X86_SEL_TYPE_SYS_286_TSS_BUSY;
1369 else
1370 /* Default even if no TR selector has been set (otherwise vmlaunch will fail!) */
1371 val = (val & ~0xF) | X86_SEL_TYPE_SYS_386_TSS_BUSY;
1372
1373 }
1374 rc |= VMXWriteVMCS(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, val);
1375 AssertRC(rc);
1376 }
1377 /* Guest CPU context: GDTR. */
1378 if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_GDTR)
1379 {
1380 rc = VMXWriteVMCS(VMX_VMCS32_GUEST_GDTR_LIMIT, pCtx->gdtr.cbGdt);
1381 rc |= VMXWriteVMCS64(VMX_VMCS64_GUEST_GDTR_BASE, pCtx->gdtr.pGdt);
1382 AssertRC(rc);
1383 }
1384 /* Guest CPU context: IDTR. */
1385 if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_IDTR)
1386 {
1387 rc = VMXWriteVMCS(VMX_VMCS32_GUEST_IDTR_LIMIT, pCtx->idtr.cbIdt);
1388 rc |= VMXWriteVMCS64(VMX_VMCS64_GUEST_IDTR_BASE, pCtx->idtr.pIdt);
1389 AssertRC(rc);
1390 }
1391
1392 /*
1393 * Sysenter MSRs (unconditional)
1394 */
1395 rc = VMXWriteVMCS(VMX_VMCS32_GUEST_SYSENTER_CS, pCtx->SysEnter.cs);
1396 rc |= VMXWriteVMCS64(VMX_VMCS64_GUEST_SYSENTER_EIP, pCtx->SysEnter.eip);
1397 rc |= VMXWriteVMCS64(VMX_VMCS64_GUEST_SYSENTER_ESP, pCtx->SysEnter.esp);
1398 AssertRC(rc);
1399
1400 /* Control registers */
1401 if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_CR0)
1402 {
1403 val = pCtx->cr0;
1404 rc = VMXWriteVMCS(VMX_VMCS_CTRL_CR0_READ_SHADOW, val);
1405 Log2(("Guest CR0-shadow %08x\n", val));
1406 if (CPUMIsGuestFPUStateActive(pVCpu) == false)
1407 {
1408 /* Always use #NM exceptions to load the FPU/XMM state on demand. */
1409 val |= X86_CR0_TS | X86_CR0_ET | X86_CR0_NE | X86_CR0_MP;
1410 }
1411 else
1412 {
1413 /** @todo check if we support the old style mess correctly. */
1414 if (!(val & X86_CR0_NE))
1415 Log(("Forcing X86_CR0_NE!!!\n"));
1416
1417 val |= X86_CR0_NE; /* always turn on the native mechanism to report FPU errors (old style uses interrupts) */
1418 }
1419 /* Note: protected mode & paging are always enabled; we use them for emulating real and protected mode without paging too. */
1420 val |= X86_CR0_PE | X86_CR0_PG;
1421 if (pVM->hwaccm.s.fNestedPaging)
1422 {
1423 if (CPUMIsGuestInPagedProtectedModeEx(pCtx))
1424 {
1425 /* Disable cr3 read/write monitoring as we don't need it for EPT. */
1426 pVCpu->hwaccm.s.vmx.proc_ctls &= ~( VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
1427 | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT);
1428 }
1429 else
1430 {
1431 /* Reenable cr3 read/write monitoring as our identity mapped page table is active. */
1432 pVCpu->hwaccm.s.vmx.proc_ctls |= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
1433 | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT;
1434 }
1435 rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
1436 AssertRC(rc);
1437 }
1438 else
1439 {
1440 /* Note: We must also set this as we rely on protecting various pages for which supervisor writes must be caught. */
1441 val |= X86_CR0_WP;
1442 }
1443
1444 /* Always enable caching. */
1445 val &= ~(X86_CR0_CD|X86_CR0_NW);
1446
1447 rc |= VMXWriteVMCS64(VMX_VMCS64_GUEST_CR0, val);
1448 Log2(("Guest CR0 %08x\n", val));
1449 /* CR0 flags owned by the host; if the guests attempts to change them, then
1450 * the VM will exit.
1451 */
1452 val = X86_CR0_PE /* Must monitor this bit (assumptions are made for real mode emulation) */
1453 | X86_CR0_WP /* Must monitor this bit (it must always be enabled). */
1454 | X86_CR0_PG /* Must monitor this bit (assumptions are made for real mode & protected mode without paging emulation) */
1455 | X86_CR0_TS
1456 | X86_CR0_ET /* Bit not restored during VM-exit! */
1457 | X86_CR0_CD /* Bit not restored during VM-exit! */
1458 | X86_CR0_NW /* Bit not restored during VM-exit! */
1459 | X86_CR0_NE
1460 | X86_CR0_MP;
1461 pVCpu->hwaccm.s.vmx.cr0_mask = val;
1462
1463 rc |= VMXWriteVMCS(VMX_VMCS_CTRL_CR0_MASK, val);
1464 Log2(("Guest CR0-mask %08x\n", val));
1465 AssertRC(rc);
1466 }
1467 if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_CR4)
1468 {
1469 /* CR4 */
1470 rc = VMXWriteVMCS(VMX_VMCS_CTRL_CR4_READ_SHADOW, pCtx->cr4);
1471 Log2(("Guest CR4-shadow %08x\n", pCtx->cr4));
1472 /* Set the required bits in cr4 too (currently X86_CR4_VMXE). */
1473 val = pCtx->cr4 | (uint32_t)pVM->hwaccm.s.vmx.msr.vmx_cr4_fixed0;
1474
1475 if (!pVM->hwaccm.s.fNestedPaging)
1476 {
1477 switch(pVCpu->hwaccm.s.enmShadowMode)
1478 {
1479 case PGMMODE_REAL: /* Real mode -> emulated using v86 mode */
1480 case PGMMODE_PROTECTED: /* Protected mode, no paging -> emulated using identity mapping. */
1481 case PGMMODE_32_BIT: /* 32-bit paging. */
1482 val &= ~X86_CR4_PAE;
1483 break;
1484
1485 case PGMMODE_PAE: /* PAE paging. */
1486 case PGMMODE_PAE_NX: /* PAE paging with NX enabled. */
1487 /** @todo use normal 32 bits paging */
1488 val |= X86_CR4_PAE;
1489 break;
1490
1491 case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
1492 case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
1493#ifdef VBOX_ENABLE_64_BITS_GUESTS
1494 break;
1495#else
1496 AssertFailed();
1497 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
1498#endif
1499 default: /* shut up gcc */
1500 AssertFailed();
1501 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
1502 }
1503 }
1504 else
1505 if (!CPUMIsGuestInPagedProtectedModeEx(pCtx))
1506 {
1507 /* We use 4 MB pages in our identity mapping page table for real and protected mode without paging. */
1508 val |= X86_CR4_PSE;
1509 /* Our identity mapping is a 32 bits page directory. */
1510 val &= ~X86_CR4_PAE;
1511 }
1512
1513 /* Turn off VME if we're in emulated real mode. */
1514 if (CPUMIsGuestInRealModeEx(pCtx))
1515 val &= ~X86_CR4_VME;
1516
1517 rc |= VMXWriteVMCS64(VMX_VMCS64_GUEST_CR4, val);
1518 Log2(("Guest CR4 %08x\n", val));
1519 /* CR4 flags owned by the host; if the guests attempts to change them, then
1520 * the VM will exit.
1521 */
1522 val = 0
1523 | X86_CR4_VME
1524 | X86_CR4_PAE
1525 | X86_CR4_PGE
1526 | X86_CR4_PSE
1527 | X86_CR4_VMXE;
1528 pVCpu->hwaccm.s.vmx.cr4_mask = val;
1529
1530 rc |= VMXWriteVMCS(VMX_VMCS_CTRL_CR4_MASK, val);
1531 Log2(("Guest CR4-mask %08x\n", val));
1532 AssertRC(rc);
1533 }
1534
1535 if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_CR3)
1536 {
1537 if (pVM->hwaccm.s.fNestedPaging)
1538 {
1539 Assert(PGMGetHyperCR3(pVCpu));
1540 pVCpu->hwaccm.s.vmx.GCPhysEPTP = PGMGetHyperCR3(pVCpu);
1541
1542 Assert(!(pVCpu->hwaccm.s.vmx.GCPhysEPTP & 0xfff));
1543 /** @todo Check the IA32_VMX_EPT_VPID_CAP MSR for other supported memory types. */
1544 pVCpu->hwaccm.s.vmx.GCPhysEPTP |= VMX_EPT_MEMTYPE_WB
1545 | (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
1546
1547 rc = VMXWriteVMCS64(VMX_VMCS_CTRL_EPTP_FULL, pVCpu->hwaccm.s.vmx.GCPhysEPTP);
1548 AssertRC(rc);
1549
1550 if (!CPUMIsGuestInPagedProtectedModeEx(pCtx))
1551 {
1552 RTGCPHYS GCPhys;
1553
1554 /* We convert it here every time as pci regions could be reconfigured. */
1555 rc = PDMVMMDevHeapR3ToGCPhys(pVM, pVM->hwaccm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
1556 AssertRC(rc);
1557
1558 /* We use our identity mapping page table here as we need to map guest virtual to guest physical addresses; EPT will
1559 * take care of the translation to host physical addresses.
1560 */
1561 val = GCPhys;
1562 }
1563 else
1564 {
1565 /* Save the real guest CR3 in VMX_VMCS_GUEST_CR3 */
1566 val = pCtx->cr3;
1567 /* Prefetch the four PDPT entries in PAE mode. */
1568 vmxR0PrefetchPAEPdptrs(pVM, pVCpu, pCtx);
1569 }
1570 }
1571 else
1572 {
1573 val = PGMGetHyperCR3(pVCpu);
1574 Assert(val || VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL));
1575 }
1576
1577 /* Save our shadow CR3 register. */
1578 rc = VMXWriteVMCS64(VMX_VMCS64_GUEST_CR3, val);
1579 AssertRC(rc);
1580 }
1581
1582 /* Debug registers. */
1583 if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_DEBUG)
1584 {
1585 pCtx->dr[6] |= X86_DR6_INIT_VAL; /* set all reserved bits to 1. */
1586 pCtx->dr[6] &= ~RT_BIT(12); /* must be zero. */
1587
1588 pCtx->dr[7] &= 0xffffffff; /* upper 32 bits reserved */
1589 pCtx->dr[7] &= ~(RT_BIT(11) | RT_BIT(12) | RT_BIT(14) | RT_BIT(15)); /* must be zero */
1590 pCtx->dr[7] |= 0x400; /* must be one */
1591
1592 /* Resync DR7 */
1593 rc = VMXWriteVMCS64(VMX_VMCS64_GUEST_DR7, pCtx->dr[7]);
1594 AssertRC(rc);
1595
1596#ifdef DEBUG
1597 /* Sync the hypervisor debug state now if any breakpoint is armed. */
1598 if ( CPUMGetHyperDR7(pVCpu) & (X86_DR7_ENABLED_MASK|X86_DR7_GD)
1599 && !CPUMIsHyperDebugStateActive(pVCpu)
1600 && !DBGFIsStepping(pVCpu))
1601 {
1602 /* Save the host and load the hypervisor debug state. */
1603 rc = CPUMR0LoadHyperDebugState(pVM, pVCpu, pCtx, true /* include DR6 */);
1604 AssertRC(rc);
1605
1606 /* DRx intercepts remain enabled. */
1607
1608 /* Override dr7 with the hypervisor value. */
1609 rc = VMXWriteVMCS64(VMX_VMCS64_GUEST_DR7, CPUMGetHyperDR7(pVCpu));
1610 AssertRC(rc);
1611 }
1612 else
1613#endif
1614 /* Sync the debug state now if any breakpoint is armed. */
1615 if ( (pCtx->dr[7] & (X86_DR7_ENABLED_MASK|X86_DR7_GD))
1616 && !CPUMIsGuestDebugStateActive(pVCpu)
1617 && !DBGFIsStepping(pVCpu))
1618 {
1619 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatDRxArmed);
1620
1621 /* Disable drx move intercepts. */
1622 pVCpu->hwaccm.s.vmx.proc_ctls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT;
1623 rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
1624 AssertRC(rc);
1625
1626 /* Save the host and load the guest debug state. */
1627 rc = CPUMR0LoadGuestDebugState(pVM, pVCpu, pCtx, true /* include DR6 */);
1628 AssertRC(rc);
1629 }
1630
1631 /* IA32_DEBUGCTL MSR. */
1632 rc = VMXWriteVMCS64(VMX_VMCS_GUEST_DEBUGCTL_FULL, 0);
1633 AssertRC(rc);
1634
1635 /** @todo do we really ever need this? */
1636 rc |= VMXWriteVMCS(VMX_VMCS_GUEST_DEBUG_EXCEPTIONS, 0);
1637 AssertRC(rc);
1638 }
1639
1640 /* EIP, ESP and EFLAGS */
1641 rc = VMXWriteVMCS64(VMX_VMCS64_GUEST_RIP, pCtx->rip);
1642 rc |= VMXWriteVMCS64(VMX_VMCS64_GUEST_RSP, pCtx->rsp);
1643 AssertRC(rc);
1644
1645 /* Bits 22-31, 15, 5 & 3 must be zero. Bit 1 must be 1. */
1646 eflags = pCtx->eflags;
1647 eflags.u32 &= VMX_EFLAGS_RESERVED_0;
1648 eflags.u32 |= VMX_EFLAGS_RESERVED_1;
1649
1650#ifdef HWACCM_VMX_EMULATE_REALMODE
1651 /* Real mode emulation using v86 mode. */
1652 if (CPUMIsGuestInRealModeEx(pCtx))
1653 {
1654 pVCpu->hwaccm.s.vmx.RealMode.eflags = eflags;
1655
1656 eflags.Bits.u1VM = 1;
1657 eflags.Bits.u2IOPL = 0; /* must always be 0 or else certain instructions won't cause faults. */
1658 }
1659#endif /* HWACCM_VMX_EMULATE_REALMODE */
1660 rc = VMXWriteVMCS(VMX_VMCS_GUEST_RFLAGS, eflags.u32);
1661 AssertRC(rc);
1662
1663 /* TSC offset. */
1664 uint64_t u64TSCOffset;
1665
1666 if (TMCpuTickCanUseRealTSC(pVCpu, &u64TSCOffset))
1667 {
1668 /* Note: VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT takes precedence over TSC_OFFSET */
1669 rc = VMXWriteVMCS64(VMX_VMCS_CTRL_TSC_OFFSET_FULL, u64TSCOffset);
1670 AssertRC(rc);
1671
1672 pVCpu->hwaccm.s.vmx.proc_ctls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT;
1673 rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
1674 AssertRC(rc);
1675 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatTSCOffset);
1676 }
1677 else
1678 {
1679 pVCpu->hwaccm.s.vmx.proc_ctls |= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT;
1680 rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
1681 AssertRC(rc);
1682 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatTSCIntercept);
1683 }
1684
1685 /* 64 bits guest mode? */
1686 if (CPUMIsGuestInLongModeEx(pCtx))
1687 {
1688#if !defined(VBOX_ENABLE_64_BITS_GUESTS)
1689 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
1690#elif HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
1691 pVCpu->hwaccm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
1692#else
1693# ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
1694 if (!pVM->hwaccm.s.fAllow64BitGuests)
1695 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
1696# endif
1697 pVCpu->hwaccm.s.vmx.pfnStartVM = VMXR0StartVM64;
1698#endif
1699 /* Unconditionally update these as wrmsr might have changed them. */
1700 rc = VMXWriteVMCS64(VMX_VMCS64_GUEST_FS_BASE, pCtx->fsHid.u64Base);
1701 AssertRC(rc);
1702 rc = VMXWriteVMCS64(VMX_VMCS64_GUEST_GS_BASE, pCtx->gsHid.u64Base);
1703 AssertRC(rc);
1704 }
1705 else
1706 {
1707 pVCpu->hwaccm.s.vmx.pfnStartVM = VMXR0StartVM32;
1708 }
1709
1710#if 0 /* @todo deal with 32/64 */
1711 /* Unconditionally update the guest EFER - on CPUs that supports it. */
1712 if (pVM->hwaccm.s.vmx.msr.vmx_entry.n.allowed1 & VMX_VMCS_CTRL_ENTRY_CONTROLS_LOAD_GUEST_EFER_MSR)
1713 {
1714 rc = VMXWriteVMCS64(VMX_VMCS_GUEST_EFER_FULL, pCtx->msrEFER);
1715 AssertRC(rc);
1716 }
1717#endif
1718
1719 vmxR0UpdateExceptionBitmap(pVM, pVCpu, pCtx);
1720
1721 /* Done. */
1722 pVCpu->hwaccm.s.fContextUseFlags &= ~HWACCM_CHANGED_ALL_GUEST;
1723
1724 return rc;
1725}
1726
1727/**
1728 * Syncs back the guest state
1729 *
1730 * @returns VBox status code.
1731 * @param pVM The VM to operate on.
1732 * @param pVCpu The VMCPU to operate on.
1733 * @param pCtx Guest context
1734 */
1735DECLINLINE(int) VMXR0SaveGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
1736{
1737 RTGCUINTREG val, valShadow;
1738 RTGCUINTPTR uInterruptState;
1739 int rc;
1740
1741 /* Let's first sync back eip, esp, and eflags. */
1742 rc = VMXReadCachedVMCS(VMX_VMCS64_GUEST_RIP, &val);
1743 AssertRC(rc);
1744 pCtx->rip = val;
1745 rc = VMXReadCachedVMCS(VMX_VMCS64_GUEST_RSP, &val);
1746 AssertRC(rc);
1747 pCtx->rsp = val;
1748 rc = VMXReadCachedVMCS(VMX_VMCS_GUEST_RFLAGS, &val);
1749 AssertRC(rc);
1750 pCtx->eflags.u32 = val;
1751
1752 /* Take care of instruction fusing (sti, mov ss) */
1753 rc |= VMXReadCachedVMCS(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &val);
1754 uInterruptState = val;
1755 if (uInterruptState != 0)
1756 {
1757 Assert(uInterruptState <= 2); /* only sti & mov ss */
1758 Log(("uInterruptState %x eip=%RGv\n", (uint32_t)uInterruptState, pCtx->rip));
1759 EMSetInhibitInterruptsPC(pVCpu, pCtx->rip);
1760 }
1761 else
1762 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
1763
1764 /* Control registers. */
1765 VMXReadCachedVMCS(VMX_VMCS_CTRL_CR0_READ_SHADOW, &valShadow);
1766 VMXReadCachedVMCS(VMX_VMCS64_GUEST_CR0, &val);
1767 val = (valShadow & pVCpu->hwaccm.s.vmx.cr0_mask) | (val & ~pVCpu->hwaccm.s.vmx.cr0_mask);
1768 CPUMSetGuestCR0(pVCpu, val);
1769
1770 VMXReadCachedVMCS(VMX_VMCS_CTRL_CR4_READ_SHADOW, &valShadow);
1771 VMXReadCachedVMCS(VMX_VMCS64_GUEST_CR4, &val);
1772 val = (valShadow & pVCpu->hwaccm.s.vmx.cr4_mask) | (val & ~pVCpu->hwaccm.s.vmx.cr4_mask);
1773 CPUMSetGuestCR4(pVCpu, val);
1774
1775 /* Note: no reason to sync back the CRx registers. They can't be changed by the guest. */
1776 /* Note: only in the nested paging case can CR3 & CR4 be changed by the guest. */
1777 if ( pVM->hwaccm.s.fNestedPaging
1778 && CPUMIsGuestInPagedProtectedModeEx(pCtx))
1779 {
1780 PVMCSCACHE pCache = &pVCpu->hwaccm.s.vmx.VMCSCache;
1781
1782 /* Can be updated behind our back in the nested paging case. */
1783 CPUMSetGuestCR2(pVCpu, pCache->cr2);
1784
1785 VMXReadCachedVMCS(VMX_VMCS64_GUEST_CR3, &val);
1786
1787 if (val != pCtx->cr3)
1788 {
1789 CPUMSetGuestCR3(pVCpu, val);
1790 PGMUpdateCR3(pVCpu, val);
1791 }
1792 /* Prefetch the four PDPT entries in PAE mode. */
1793 vmxR0PrefetchPAEPdptrs(pVM, pVCpu, pCtx);
1794 }
1795
1796 /* Sync back DR7 here. */
1797 VMXReadCachedVMCS(VMX_VMCS64_GUEST_DR7, &val);
1798 pCtx->dr[7] = val;
1799
1800 /* Guest CPU context: ES, CS, SS, DS, FS, GS. */
1801 VMX_READ_SELREG(ES, es);
1802 VMX_READ_SELREG(SS, ss);
1803 VMX_READ_SELREG(CS, cs);
1804 VMX_READ_SELREG(DS, ds);
1805 VMX_READ_SELREG(FS, fs);
1806 VMX_READ_SELREG(GS, gs);
1807
1808 /*
1809 * System MSRs
1810 */
1811 VMXReadCachedVMCS(VMX_VMCS32_GUEST_SYSENTER_CS, &val);
1812 pCtx->SysEnter.cs = val;
1813 VMXReadCachedVMCS(VMX_VMCS64_GUEST_SYSENTER_EIP, &val);
1814 pCtx->SysEnter.eip = val;
1815 VMXReadCachedVMCS(VMX_VMCS64_GUEST_SYSENTER_ESP, &val);
1816 pCtx->SysEnter.esp = val;
1817
1818 /* Misc. registers; must sync everything otherwise we can get out of sync when jumping to ring 3. */
1819 VMX_READ_SELREG(LDTR, ldtr);
1820
1821 VMXReadCachedVMCS(VMX_VMCS32_GUEST_GDTR_LIMIT, &val);
1822 pCtx->gdtr.cbGdt = val;
1823 VMXReadCachedVMCS(VMX_VMCS64_GUEST_GDTR_BASE, &val);
1824 pCtx->gdtr.pGdt = val;
1825
1826 VMXReadCachedVMCS(VMX_VMCS32_GUEST_IDTR_LIMIT, &val);
1827 pCtx->idtr.cbIdt = val;
1828 VMXReadCachedVMCS(VMX_VMCS64_GUEST_IDTR_BASE, &val);
1829 pCtx->idtr.pIdt = val;
1830
1831#ifdef HWACCM_VMX_EMULATE_REALMODE
1832 /* Real mode emulation using v86 mode. */
1833 if (CPUMIsGuestInRealModeEx(pCtx))
1834 {
1835 /* Hide our emulation flags */
1836 pCtx->eflags.Bits.u1VM = 0;
1837
1838 /* Restore original IOPL setting as we always use 0. */
1839 pCtx->eflags.Bits.u2IOPL = pVCpu->hwaccm.s.vmx.RealMode.eflags.Bits.u2IOPL;
1840
1841 /* Force a TR resync every time in case we switch modes. */
1842 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_GUEST_TR;
1843 }
1844 else
1845#endif /* HWACCM_VMX_EMULATE_REALMODE */
1846 {
1847 /* In real mode we have a fake TSS, so only sync it back when it's supposed to be valid. */
1848 VMX_READ_SELREG(TR, tr);
1849 }
1850 return VINF_SUCCESS;
1851}
1852
1853/**
1854 * Dummy placeholder
1855 *
1856 * @param pVM The VM to operate on.
1857 * @param pVCpu The VMCPU to operate on.
1858 */
1859static void vmxR0SetupTLBDummy(PVM pVM, PVMCPU pVCpu)
1860{
1861 NOREF(pVM);
1862 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
1863 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1864 pVCpu->hwaccm.s.TlbShootdown.cPages = 0;
1865 return;
1866}
1867
1868/**
1869 * Setup the tagged TLB for EPT
1870 *
1871 * @returns VBox status code.
1872 * @param pVM The VM to operate on.
1873 * @param pVCpu The VMCPU to operate on.
1874 */
1875static void vmxR0SetupTLBEPT(PVM pVM, PVMCPU pVCpu)
1876{
1877 PHWACCM_CPUINFO pCpu;
1878
1879 Assert(pVM->hwaccm.s.fNestedPaging);
1880 Assert(!pVM->hwaccm.s.vmx.fVPID);
1881
1882 /* Deal with tagged TLBs if VPID or EPT is supported. */
1883 pCpu = HWACCMR0GetCurrentCpu();
1884 /* Force a TLB flush for the first world switch if the current cpu differs from the one we ran on last. */
1885 /* Note that this can happen both for start and resume due to long jumps back to ring 3. */
1886 if ( pVCpu->hwaccm.s.idLastCpu != pCpu->idCpu
1887 /* if the tlb flush count has changed, another VM has flushed the TLB of this cpu, so we can't use our current ASID anymore. */
1888 || pVCpu->hwaccm.s.cTLBFlushes != pCpu->cTLBFlushes)
1889 {
1890 /* Force a TLB flush on VM entry. */
1891 pVCpu->hwaccm.s.fForceTLBFlush = true;
1892 }
1893 else
1894 Assert(!pCpu->fFlushTLB);
1895
1896 /* Check for tlb shootdown flushes. */
1897 if (VMCPU_FF_TESTANDCLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1898 pVCpu->hwaccm.s.fForceTLBFlush = true;
1899
1900 pVCpu->hwaccm.s.idLastCpu = pCpu->idCpu;
1901 pCpu->fFlushTLB = false;
1902
1903 if (pVCpu->hwaccm.s.fForceTLBFlush)
1904 {
1905 vmxR0FlushEPT(pVM, pVCpu, pVM->hwaccm.s.vmx.enmFlushContext, 0);
1906 }
1907 else
1908 if (VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1909 {
1910 /* Deal with pending TLB shootdown actions which were queued when we were not executing code. */
1911 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatTlbShootdown);
1912
1913 for (unsigned i=0;i<pVCpu->hwaccm.s.TlbShootdown.cPages;i++)
1914 {
1915 /* aTlbShootdownPages contains physical addresses in this case. */
1916 vmxR0FlushEPT(pVM, pVCpu, pVM->hwaccm.s.vmx.enmFlushContext, pVCpu->hwaccm.s.TlbShootdown.aPages[i]);
1917 }
1918 }
1919 pVCpu->hwaccm.s.TlbShootdown.cPages= 0;
1920 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1921
1922#ifdef VBOX_WITH_STATISTICS
1923 if (pVCpu->hwaccm.s.fForceTLBFlush)
1924 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatFlushTLBWorldSwitch);
1925 else
1926 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatNoFlushTLBWorldSwitch);
1927#endif
1928}
1929
1930#ifdef HWACCM_VTX_WITH_VPID
1931/**
1932 * Setup the tagged TLB for VPID
1933 *
1934 * @returns VBox status code.
1935 * @param pVM The VM to operate on.
1936 * @param pVCpu The VMCPU to operate on.
1937 */
1938static void vmxR0SetupTLBVPID(PVM pVM, PVMCPU pVCpu)
1939{
1940 PHWACCM_CPUINFO pCpu;
1941
1942 Assert(pVM->hwaccm.s.vmx.fVPID);
1943 Assert(!pVM->hwaccm.s.fNestedPaging);
1944
1945 /* Deal with tagged TLBs if VPID or EPT is supported. */
1946 pCpu = HWACCMR0GetCurrentCpu();
1947 /* Force a TLB flush for the first world switch if the current cpu differs from the one we ran on last. */
1948 /* Note that this can happen both for start and resume due to long jumps back to ring 3. */
1949 if ( pVCpu->hwaccm.s.idLastCpu != pCpu->idCpu
1950 /* if the tlb flush count has changed, another VM has flushed the TLB of this cpu, so we can't use our current ASID anymore. */
1951 || pVCpu->hwaccm.s.cTLBFlushes != pCpu->cTLBFlushes)
1952 {
1953 /* Force a TLB flush on VM entry. */
1954 pVCpu->hwaccm.s.fForceTLBFlush = true;
1955 }
1956 else
1957 Assert(!pCpu->fFlushTLB);
1958
1959 pVCpu->hwaccm.s.idLastCpu = pCpu->idCpu;
1960
1961 /* Check for tlb shootdown flushes. */
1962 if (VMCPU_FF_TESTANDCLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1963 pVCpu->hwaccm.s.fForceTLBFlush = true;
1964
1965 /* Make sure we flush the TLB when required. Switch ASID to achieve the same thing, but without actually flushing the whole TLB (which is expensive). */
1966 if (pVCpu->hwaccm.s.fForceTLBFlush)
1967 {
1968 if ( ++pCpu->uCurrentASID >= pVM->hwaccm.s.uMaxASID
1969 || pCpu->fFlushTLB)
1970 {
1971 pCpu->fFlushTLB = false;
1972 pCpu->uCurrentASID = 1; /* start at 1; host uses 0 */
1973 pCpu->cTLBFlushes++;
1974 }
1975 else
1976 {
1977 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatFlushASID);
1978 pVCpu->hwaccm.s.fForceTLBFlush = false;
1979 }
1980
1981 pVCpu->hwaccm.s.cTLBFlushes = pCpu->cTLBFlushes;
1982 pVCpu->hwaccm.s.uCurrentASID = pCpu->uCurrentASID;
1983 }
1984 else
1985 {
1986 Assert(!pCpu->fFlushTLB);
1987 Assert(pVCpu->hwaccm.s.uCurrentASID && pCpu->uCurrentASID);
1988
1989 if (VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1990 {
1991 /* Deal with pending TLB shootdown actions which were queued when we were not executing code. */
1992 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatTlbShootdown);
1993 for (unsigned i=0;i<pVCpu->hwaccm.s.TlbShootdown.cPages;i++)
1994 vmxR0FlushVPID(pVM, pVCpu, pVM->hwaccm.s.vmx.enmFlushContext, pVCpu->hwaccm.s.TlbShootdown.aPages[i]);
1995 }
1996 }
1997 pVCpu->hwaccm.s.TlbShootdown.cPages = 0;
1998 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1999
2000 AssertMsg(pVCpu->hwaccm.s.cTLBFlushes == pCpu->cTLBFlushes, ("Flush count mismatch for cpu %d (%x vs %x)\n", pCpu->idCpu, pVCpu->hwaccm.s.cTLBFlushes, pCpu->cTLBFlushes));
2001 AssertMsg(pCpu->uCurrentASID >= 1 && pCpu->uCurrentASID < pVM->hwaccm.s.uMaxASID, ("cpu%d uCurrentASID = %x\n", pCpu->idCpu, pCpu->uCurrentASID));
2002 AssertMsg(pVCpu->hwaccm.s.uCurrentASID >= 1 && pVCpu->hwaccm.s.uCurrentASID < pVM->hwaccm.s.uMaxASID, ("cpu%d VM uCurrentASID = %x\n", pCpu->idCpu, pVCpu->hwaccm.s.uCurrentASID));
2003
2004 int rc = VMXWriteVMCS(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hwaccm.s.uCurrentASID);
2005 AssertRC(rc);
2006
2007 if (pVCpu->hwaccm.s.fForceTLBFlush)
2008 vmxR0FlushVPID(pVM, pVCpu, pVM->hwaccm.s.vmx.enmFlushContext, 0);
2009
2010#ifdef VBOX_WITH_STATISTICS
2011 if (pVCpu->hwaccm.s.fForceTLBFlush)
2012 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatFlushTLBWorldSwitch);
2013 else
2014 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatNoFlushTLBWorldSwitch);
2015#endif
2016}
2017#endif /* HWACCM_VTX_WITH_VPID */
2018
2019/**
2020 * Runs guest code in a VT-x VM.
2021 *
2022 * @returns VBox status code.
2023 * @param pVM The VM to operate on.
2024 * @param pVCpu The VMCPU to operate on.
2025 * @param pCtx Guest context
2026 */
2027VMMR0DECL(int) VMXR0RunGuestCode(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
2028{
2029 int rc = VINF_SUCCESS;
2030 RTGCUINTREG val;
2031 RTGCUINTREG exitReason = VMX_EXIT_INVALID;
2032 RTGCUINTREG instrError, cbInstr;
2033 RTGCUINTPTR exitQualification = 0;
2034 RTGCUINTPTR intInfo = 0; /* shut up buggy gcc 4 */
2035 RTGCUINTPTR errCode, instrInfo;
2036 bool fSetupTPRCaching = false;
2037 uint8_t u8LastTPR = 0;
2038 PHWACCM_CPUINFO pCpu = 0;
2039 RTCCUINTREG uOldEFlags = ~(RTCCUINTREG)0;
2040 unsigned cResume = 0;
2041#ifdef VBOX_STRICT
2042 RTCPUID idCpuCheck;
2043#endif
2044#ifdef VBOX_HIGH_RES_TIMERS_HACK_IN_RING0
2045 uint64_t u64LastTime = RTTimeMilliTS();
2046#endif
2047#ifdef VBOX_WITH_STATISTICS
2048 bool fStatEntryStarted = true;
2049 bool fStatExit2Started = false;
2050#endif
2051
2052 Assert(!(pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC) || (pVCpu->hwaccm.s.vmx.pVAPIC && pVM->hwaccm.s.vmx.pAPIC));
2053
2054 /* Check if we need to use TPR shadowing. */
2055 if ( CPUMIsGuestInLongModeEx(pCtx)
2056 || ( (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
2057 && pVM->hwaccm.s.fHasIoApic)
2058 )
2059 {
2060 fSetupTPRCaching = true;
2061 }
2062
2063 Log2(("\nE"));
2064
2065 STAM_PROFILE_ADV_START(&pVCpu->hwaccm.s.StatEntry, x);
2066
2067#ifdef VBOX_STRICT
2068 {
2069 RTCCUINTREG val;
2070
2071 rc = VMXReadVMCS(VMX_VMCS_CTRL_PIN_EXEC_CONTROLS, &val);
2072 AssertRC(rc);
2073 Log2(("VMX_VMCS_CTRL_PIN_EXEC_CONTROLS = %08x\n", val));
2074
2075 /* allowed zero */
2076 if ((val & pVM->hwaccm.s.vmx.msr.vmx_pin_ctls.n.disallowed0) != pVM->hwaccm.s.vmx.msr.vmx_pin_ctls.n.disallowed0)
2077 Log(("Invalid VMX_VMCS_CTRL_PIN_EXEC_CONTROLS: zero\n"));
2078
2079 /* allowed one */
2080 if ((val & ~pVM->hwaccm.s.vmx.msr.vmx_pin_ctls.n.allowed1) != 0)
2081 Log(("Invalid VMX_VMCS_CTRL_PIN_EXEC_CONTROLS: one\n"));
2082
2083 rc = VMXReadVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, &val);
2084 AssertRC(rc);
2085 Log2(("VMX_VMCS_CTRL_PROC_EXEC_CONTROLS = %08x\n", val));
2086
2087 /* Must be set according to the MSR, but can be cleared in case of EPT. */
2088 if (pVM->hwaccm.s.fNestedPaging)
2089 val |= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INVLPG_EXIT
2090 | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
2091 | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT;
2092
2093 /* allowed zero */
2094 if ((val & pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.disallowed0) != pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.disallowed0)
2095 Log(("Invalid VMX_VMCS_CTRL_PROC_EXEC_CONTROLS: zero\n"));
2096
2097 /* allowed one */
2098 if ((val & ~pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1) != 0)
2099 Log(("Invalid VMX_VMCS_CTRL_PROC_EXEC_CONTROLS: one\n"));
2100
2101 rc = VMXReadVMCS(VMX_VMCS_CTRL_ENTRY_CONTROLS, &val);
2102 AssertRC(rc);
2103 Log2(("VMX_VMCS_CTRL_ENTRY_CONTROLS = %08x\n", val));
2104
2105 /* allowed zero */
2106 if ((val & pVM->hwaccm.s.vmx.msr.vmx_entry.n.disallowed0) != pVM->hwaccm.s.vmx.msr.vmx_entry.n.disallowed0)
2107 Log(("Invalid VMX_VMCS_CTRL_ENTRY_CONTROLS: zero\n"));
2108
2109 /* allowed one */
2110 if ((val & ~pVM->hwaccm.s.vmx.msr.vmx_entry.n.allowed1) != 0)
2111 Log(("Invalid VMX_VMCS_CTRL_ENTRY_CONTROLS: one\n"));
2112
2113 rc = VMXReadVMCS(VMX_VMCS_CTRL_EXIT_CONTROLS, &val);
2114 AssertRC(rc);
2115 Log2(("VMX_VMCS_CTRL_EXIT_CONTROLS = %08x\n", val));
2116
2117 /* allowed zero */
2118 if ((val & pVM->hwaccm.s.vmx.msr.vmx_exit.n.disallowed0) != pVM->hwaccm.s.vmx.msr.vmx_exit.n.disallowed0)
2119 Log(("Invalid VMX_VMCS_CTRL_EXIT_CONTROLS: zero\n"));
2120
2121 /* allowed one */
2122 if ((val & ~pVM->hwaccm.s.vmx.msr.vmx_exit.n.allowed1) != 0)
2123 Log(("Invalid VMX_VMCS_CTRL_EXIT_CONTROLS: one\n"));
2124 }
2125#endif
2126
2127#ifdef VBOX_WITH_CRASHDUMP_MAGIC
2128 pVCpu->hwaccm.s.vmx.VMCSCache.u64TimeEntry = RTTimeNanoTS();
2129#endif
2130
2131 /* We can jump to this point to resume execution after determining that a VM-exit is innocent.
2132 */
2133ResumeExecution:
2134 STAM_STATS({
2135 if (fStatExit2Started) { STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2, y); fStatExit2Started = false; }
2136 if (!fStatEntryStarted) { STAM_PROFILE_ADV_START(&pVCpu->hwaccm.s.StatEntry, x); fStatEntryStarted = true; }
2137 });
2138 AssertMsg(pVCpu->hwaccm.s.idEnteredCpu == RTMpCpuId(),
2139 ("Expected %d, I'm %d; cResume=%d exitReason=%RGv exitQualification=%RGv\n",
2140 (int)pVCpu->hwaccm.s.idEnteredCpu, (int)RTMpCpuId(), cResume, exitReason, exitQualification));
2141 Assert(!HWACCMR0SuspendPending());
2142
2143 /* Safety precaution; looping for too long here can have a very bad effect on the host */
2144 if (RT_UNLIKELY(++cResume > pVM->hwaccm.s.cMaxResumeLoops))
2145 {
2146 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitMaxResume);
2147 rc = VINF_EM_RAW_INTERRUPT;
2148 goto end;
2149 }
2150
2151 /* Check for irq inhibition due to instruction fusing (sti, mov ss). */
2152 if (VMCPU_FF_ISSET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
2153 {
2154 Log(("VM_FF_INHIBIT_INTERRUPTS at %RGv successor %RGv\n", (RTGCPTR)pCtx->rip, EMGetInhibitInterruptsPC(pVCpu)));
2155 if (pCtx->rip != EMGetInhibitInterruptsPC(pVCpu))
2156 {
2157 /* Note: we intentionally don't clear VM_FF_INHIBIT_INTERRUPTS here.
2158 * Before we are able to execute this instruction in raw mode (iret to guest code) an external interrupt might
2159 * force a world switch again. Possibly allowing a guest interrupt to be dispatched in the process. This could
2160 * break the guest. Sounds very unlikely, but such timing sensitive problems are not as rare as you might think.
2161 */
2162 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
2163 /* Irq inhibition is no longer active; clear the corresponding VMX state. */
2164 rc = VMXWriteVMCS(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, 0);
2165 AssertRC(rc);
2166 }
2167 }
2168 else
2169 {
2170 /* Irq inhibition is no longer active; clear the corresponding VMX state. */
2171 rc = VMXWriteVMCS(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, 0);
2172 AssertRC(rc);
2173 }
2174
2175#ifdef VBOX_HIGH_RES_TIMERS_HACK_IN_RING0
2176 if (RT_UNLIKELY(cResume & 0xf) == 0)
2177 {
2178 uint64_t u64CurTime = RTTimeMilliTS();
2179
2180 if (RT_UNLIKELY(u64CurTime > u64LastTime))
2181 {
2182 u64LastTime = u64CurTime;
2183 TMTimerPollVoid(pVM, pVCpu);
2184 }
2185 }
2186#endif
2187
2188 /* Check for pending actions that force us to go back to ring 3. */
2189#ifdef DEBUG
2190 /* Intercept X86_XCPT_DB if stepping is enabled */
2191 if (!DBGFIsStepping(pVCpu))
2192#endif
2193 {
2194 if ( VM_FF_ISPENDING(pVM, VM_FF_HWACCM_TO_R3_MASK)
2195 || VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_HWACCM_TO_R3_MASK))
2196 {
2197 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
2198 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatSwitchToR3);
2199 rc = RT_UNLIKELY(VM_FF_ISPENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
2200 goto end;
2201 }
2202 }
2203
2204 /* Pending request packets might contain actions that need immediate attention, such as pending hardware interrupts. */
2205 if ( VM_FF_ISPENDING(pVM, VM_FF_REQUEST)
2206 || VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_REQUEST))
2207 {
2208 rc = VINF_EM_PENDING_REQUEST;
2209 goto end;
2210 }
2211
2212#ifdef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
2213 /*
2214 * Exit to ring-3 preemption/work is pending.
2215 *
2216 * Interrupts are disabled before the call to make sure we don't miss any interrupt
2217 * that would flag preemption (IPI, timer tick, ++). (Would've been nice to do this
2218 * further down, but VMXR0CheckPendingInterrupt makes that impossible.)
2219 *
2220 * Note! Interrupts must be disabled done *before* we check for TLB flushes; TLB
2221 * shootdowns rely on this.
2222 */
2223 uOldEFlags = ASMIntDisableFlags();
2224 if (RTThreadPreemptIsPending(NIL_RTTHREAD))
2225 {
2226 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitPreemptPending);
2227 rc = VINF_EM_RAW_INTERRUPT;
2228 goto end;
2229 }
2230 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
2231#endif
2232
2233 /* When external interrupts are pending, we should exit the VM when IF is set. */
2234 /* Note! *After* VM_FF_INHIBIT_INTERRUPTS check!!! */
2235 rc = VMXR0CheckPendingInterrupt(pVM, pVCpu, pCtx);
2236 if (RT_FAILURE(rc))
2237 goto end;
2238
2239 /** @todo check timers?? */
2240
2241 /* TPR caching using CR8 is only available in 64 bits mode */
2242 /* Note the 32 bits exception for AMD (X86_CPUID_AMD_FEATURE_ECX_CR8L), but that appears missing in Intel CPUs */
2243 /* Note: we can't do this in LoadGuestState as PDMApicGetTPR can jump back to ring 3 (lock)!!!!! (no longer true) */
2244 /**
2245 * @todo query and update the TPR only when it could have been changed (mmio access & wrmsr (x2apic))
2246 */
2247 if (fSetupTPRCaching)
2248 {
2249 /* TPR caching in CR8 */
2250 bool fPending;
2251
2252 int rc = PDMApicGetTPR(pVCpu, &u8LastTPR, &fPending);
2253 AssertRC(rc);
2254 /* The TPR can be found at offset 0x80 in the APIC mmio page. */
2255 pVCpu->hwaccm.s.vmx.pVAPIC[0x80] = u8LastTPR;
2256
2257 /* Two options here:
2258 * - external interrupt pending, but masked by the TPR value.
2259 * -> a CR8 update that lower the current TPR value should cause an exit
2260 * - no pending interrupts
2261 * -> We don't need to be explicitely notified. There are enough world switches for detecting pending interrupts.
2262 */
2263 rc = VMXWriteVMCS(VMX_VMCS_CTRL_TPR_THRESHOLD, (fPending) ? (u8LastTPR >> 4) : 0); /* cr8 bits 3-0 correspond to bits 7-4 of the task priority mmio register. */
2264 AssertRC(rc);
2265 }
2266
2267#if defined(HWACCM_VTX_WITH_EPT) && defined(LOG_ENABLED)
2268 if ( pVM->hwaccm.s.fNestedPaging
2269# ifdef HWACCM_VTX_WITH_VPID
2270 || pVM->hwaccm.s.vmx.fVPID
2271# endif /* HWACCM_VTX_WITH_VPID */
2272 )
2273 {
2274 pCpu = HWACCMR0GetCurrentCpu();
2275 if ( pVCpu->hwaccm.s.idLastCpu != pCpu->idCpu
2276 || pVCpu->hwaccm.s.cTLBFlushes != pCpu->cTLBFlushes)
2277 {
2278 if (pVCpu->hwaccm.s.idLastCpu != pCpu->idCpu)
2279 Log(("Force TLB flush due to rescheduling to a different cpu (%d vs %d)\n", pVCpu->hwaccm.s.idLastCpu, pCpu->idCpu));
2280 else
2281 Log(("Force TLB flush due to changed TLB flush count (%x vs %x)\n", pVCpu->hwaccm.s.cTLBFlushes, pCpu->cTLBFlushes));
2282 }
2283 if (pCpu->fFlushTLB)
2284 Log(("Force TLB flush: first time cpu %d is used -> flush\n", pCpu->idCpu));
2285 else
2286 if (pVCpu->hwaccm.s.fForceTLBFlush)
2287 LogFlow(("Manual TLB flush\n"));
2288 }
2289#endif
2290#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
2291 PGMDynMapFlushAutoSet(pVCpu);
2292#endif
2293
2294 /*
2295 * NOTE: DO NOT DO ANYTHING AFTER THIS POINT THAT MIGHT JUMP BACK TO RING 3!
2296 * (until the actual world switch)
2297 */
2298#ifdef VBOX_STRICT
2299 idCpuCheck = RTMpCpuId();
2300#endif
2301#ifdef LOG_ENABLED
2302 VMMR0LogFlushDisable(pVCpu);
2303#endif
2304 /* Save the host state first. */
2305 rc = VMXR0SaveHostState(pVM, pVCpu);
2306 if (RT_UNLIKELY(rc != VINF_SUCCESS))
2307 {
2308 VMMR0LogFlushEnable(pVCpu);
2309 goto end;
2310 }
2311 /* Load the guest state */
2312 rc = VMXR0LoadGuestState(pVM, pVCpu, pCtx);
2313 if (RT_UNLIKELY(rc != VINF_SUCCESS))
2314 {
2315 VMMR0LogFlushEnable(pVCpu);
2316 goto end;
2317 }
2318
2319#ifndef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
2320 /* Disable interrupts to make sure a poke will interrupt execution.
2321 * This must be done *before* we check for TLB flushes; TLB shootdowns rely on this.
2322 */
2323 uOldEFlags = ASMIntDisableFlags();
2324 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
2325#endif
2326
2327 /* Deal with tagged TLB setup and invalidation. */
2328 pVM->hwaccm.s.vmx.pfnSetupTaggedTLB(pVM, pVCpu);
2329
2330 /* Non-register state Guest Context */
2331 /** @todo change me according to cpu state */
2332 rc = VMXWriteVMCS(VMX_VMCS32_GUEST_ACTIVITY_STATE, VMX_CMS_GUEST_ACTIVITY_ACTIVE);
2333 AssertRC(rc);
2334
2335 STAM_STATS({ STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatEntry, x); fStatEntryStarted = false; });
2336
2337 /* Manual save and restore:
2338 * - General purpose registers except RIP, RSP
2339 *
2340 * Trashed:
2341 * - CR2 (we don't care)
2342 * - LDTR (reset to 0)
2343 * - DRx (presumably not changed at all)
2344 * - DR7 (reset to 0x400)
2345 * - EFLAGS (reset to RT_BIT(1); not relevant)
2346 *
2347 */
2348
2349
2350 /* All done! Let's start VM execution. */
2351 STAM_PROFILE_ADV_START(&pVCpu->hwaccm.s.StatInGC, z);
2352 Assert(idCpuCheck == RTMpCpuId());
2353
2354#ifdef VBOX_WITH_CRASHDUMP_MAGIC
2355 pVCpu->hwaccm.s.vmx.VMCSCache.cResume = cResume;
2356 pVCpu->hwaccm.s.vmx.VMCSCache.u64TimeSwitch = RTTimeNanoTS();
2357#endif
2358
2359 TMNotifyStartOfExecution(pVCpu);
2360#ifdef VBOX_WITH_KERNEL_USING_XMM
2361 rc = hwaccmR0VMXStartVMWrapXMM(pVCpu->hwaccm.s.fResumeVM, pCtx, &pVCpu->hwaccm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hwaccm.s.vmx.pfnStartVM);
2362#else
2363 rc = pVCpu->hwaccm.s.vmx.pfnStartVM(pVCpu->hwaccm.s.fResumeVM, pCtx, &pVCpu->hwaccm.s.vmx.VMCSCache, pVM, pVCpu);
2364#endif
2365 TMNotifyEndOfExecution(pVCpu);
2366 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED);
2367 Assert(!(ASMGetFlags() & X86_EFL_IF));
2368 ASMSetFlags(uOldEFlags);
2369#ifndef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
2370 uOldEFlags = ~(RTCCUINTREG)0;
2371#endif
2372
2373 AssertMsg(!pVCpu->hwaccm.s.vmx.VMCSCache.Write.cValidEntries, ("pVCpu->hwaccm.s.vmx.VMCSCache.Write.cValidEntries=%d\n", pVCpu->hwaccm.s.vmx.VMCSCache.Write.cValidEntries));
2374
2375 /* In case we execute a goto ResumeExecution later on. */
2376 pVCpu->hwaccm.s.fResumeVM = true;
2377 pVCpu->hwaccm.s.fForceTLBFlush = false;
2378
2379 /*
2380 * !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
2381 * IMPORTANT: WE CAN'T DO ANY LOGGING OR OPERATIONS THAT CAN DO A LONGJMP BACK TO RING 3 *BEFORE* WE'VE SYNCED BACK (MOST OF) THE GUEST STATE
2382 * !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
2383 */
2384 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatInGC, z);
2385 STAM_PROFILE_ADV_START(&pVCpu->hwaccm.s.StatExit1, v);
2386
2387 if (RT_UNLIKELY(rc != VINF_SUCCESS))
2388 {
2389 VMXR0ReportWorldSwitchError(pVM, pVCpu, rc, pCtx);
2390 VMMR0LogFlushEnable(pVCpu);
2391 goto end;
2392 }
2393
2394 /* Success. Query the guest state and figure out what has happened. */
2395
2396 /* Investigate why there was a VM-exit. */
2397 rc = VMXReadCachedVMCS(VMX_VMCS32_RO_EXIT_REASON, &exitReason);
2398 STAM_COUNTER_INC(&pVCpu->hwaccm.s.paStatExitReasonR0[exitReason & MASK_EXITREASON_STAT]);
2399
2400 exitReason &= 0xffff; /* bit 0-15 contain the exit code. */
2401 rc |= VMXReadCachedVMCS(VMX_VMCS32_RO_VM_INSTR_ERROR, &instrError);
2402 rc |= VMXReadCachedVMCS(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &cbInstr);
2403 rc |= VMXReadCachedVMCS(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &intInfo);
2404 /* might not be valid; depends on VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID. */
2405 rc |= VMXReadCachedVMCS(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERRCODE, &errCode);
2406 rc |= VMXReadCachedVMCS(VMX_VMCS32_RO_EXIT_INSTR_INFO, &instrInfo);
2407 rc |= VMXReadCachedVMCS(VMX_VMCS_RO_EXIT_QUALIFICATION, &exitQualification);
2408 AssertRC(rc);
2409
2410 /* Sync back the guest state */
2411 rc = VMXR0SaveGuestState(pVM, pVCpu, pCtx);
2412 AssertRC(rc);
2413
2414 /* Note! NOW IT'S SAFE FOR LOGGING! */
2415 VMMR0LogFlushEnable(pVCpu);
2416 Log2(("Raw exit reason %08x\n", exitReason));
2417
2418 /* Check if an injected event was interrupted prematurely. */
2419 rc = VMXReadCachedVMCS(VMX_VMCS32_RO_IDT_INFO, &val);
2420 AssertRC(rc);
2421 pVCpu->hwaccm.s.Event.intInfo = VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(val);
2422 if ( VMX_EXIT_INTERRUPTION_INFO_VALID(pVCpu->hwaccm.s.Event.intInfo)
2423 /* Ignore 'int xx' as they'll be restarted anyway. */
2424 && VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hwaccm.s.Event.intInfo) != VMX_EXIT_INTERRUPTION_INFO_TYPE_SW
2425 /* Ignore software exceptions (such as int3) as they'll reoccur when we restart the instruction anyway. */
2426 && VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hwaccm.s.Event.intInfo) != VMX_EXIT_INTERRUPTION_INFO_TYPE_SWEXCPT)
2427 {
2428 Assert(!pVCpu->hwaccm.s.Event.fPending);
2429 pVCpu->hwaccm.s.Event.fPending = true;
2430 /* Error code present? */
2431 if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(pVCpu->hwaccm.s.Event.intInfo))
2432 {
2433 rc = VMXReadCachedVMCS(VMX_VMCS32_RO_IDT_ERRCODE, &val);
2434 AssertRC(rc);
2435 pVCpu->hwaccm.s.Event.errCode = val;
2436 Log(("Pending inject %RX64 at %RGv exit=%08x intInfo=%08x exitQualification=%RGv pending error=%RX64\n", pVCpu->hwaccm.s.Event.intInfo, (RTGCPTR)pCtx->rip, exitReason, intInfo, exitQualification, val));
2437 }
2438 else
2439 {
2440 Log(("Pending inject %RX64 at %RGv exit=%08x intInfo=%08x exitQualification=%RGv\n", pVCpu->hwaccm.s.Event.intInfo, (RTGCPTR)pCtx->rip, exitReason, intInfo, exitQualification));
2441 pVCpu->hwaccm.s.Event.errCode = 0;
2442 }
2443 }
2444#ifdef VBOX_STRICT
2445 else
2446 if ( VMX_EXIT_INTERRUPTION_INFO_VALID(pVCpu->hwaccm.s.Event.intInfo)
2447 /* Ignore software exceptions (such as int3) as they're reoccur when we restart the instruction anyway. */
2448 && VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hwaccm.s.Event.intInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_SWEXCPT)
2449 {
2450 Log(("Ignore pending inject %RX64 at %RGv exit=%08x intInfo=%08x exitQualification=%RGv\n", pVCpu->hwaccm.s.Event.intInfo, (RTGCPTR)pCtx->rip, exitReason, intInfo, exitQualification));
2451 }
2452
2453 if (exitReason == VMX_EXIT_ERR_INVALID_GUEST_STATE)
2454 HWACCMDumpRegs(pVM, pVCpu, pCtx);
2455#endif
2456
2457 Log2(("E%d: New EIP=%RGv\n", exitReason, (RTGCPTR)pCtx->rip));
2458 Log2(("Exit reason %d, exitQualification %RGv\n", (uint32_t)exitReason, exitQualification));
2459 Log2(("instrInfo=%d instrError=%d instr length=%d\n", (uint32_t)instrInfo, (uint32_t)instrError, (uint32_t)cbInstr));
2460 Log2(("Interruption error code %d\n", (uint32_t)errCode));
2461 Log2(("IntInfo = %08x\n", (uint32_t)intInfo));
2462
2463 /* Sync back the TPR if it was changed. */
2464 if ( fSetupTPRCaching
2465 && u8LastTPR != pVCpu->hwaccm.s.vmx.pVAPIC[0x80])
2466 {
2467 rc = PDMApicSetTPR(pVCpu, pVCpu->hwaccm.s.vmx.pVAPIC[0x80]);
2468 AssertRC(rc);
2469 }
2470
2471 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit1, v);
2472 STAM_STATS({ STAM_PROFILE_ADV_START(&pVCpu->hwaccm.s.StatExit2, y); fStatExit2Started = true; });
2473
2474 /* Some cases don't need a complete resync of the guest CPU state; handle them here. */
2475 switch (exitReason)
2476 {
2477 case VMX_EXIT_EXCEPTION: /* 0 Exception or non-maskable interrupt (NMI). */
2478 case VMX_EXIT_EXTERNAL_IRQ: /* 1 External interrupt. */
2479 {
2480 uint32_t vector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(intInfo);
2481
2482 if (!VMX_EXIT_INTERRUPTION_INFO_VALID(intInfo))
2483 {
2484 Assert(exitReason == VMX_EXIT_EXTERNAL_IRQ);
2485 /* External interrupt; leave to allow it to be dispatched again. */
2486 rc = VINF_EM_RAW_INTERRUPT;
2487 break;
2488 }
2489 STAM_PROFILE_ADV_START(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
2490 switch (VMX_EXIT_INTERRUPTION_INFO_TYPE(intInfo))
2491 {
2492 case VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI: /* Non-maskable interrupt. */
2493 /* External interrupt; leave to allow it to be dispatched again. */
2494 rc = VINF_EM_RAW_INTERRUPT;
2495 break;
2496
2497 case VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT: /* External hardware interrupt. */
2498 AssertFailed(); /* can't come here; fails the first check. */
2499 break;
2500
2501 case VMX_EXIT_INTERRUPTION_INFO_TYPE_DBEXCPT: /* Unknown why we get this type for #DB */
2502 case VMX_EXIT_INTERRUPTION_INFO_TYPE_SWEXCPT: /* Software exception. (#BP or #OF) */
2503 Assert(vector == 1 || vector == 3 || vector == 4);
2504 /* no break */
2505 case VMX_EXIT_INTERRUPTION_INFO_TYPE_HWEXCPT: /* Hardware exception. */
2506 Log2(("Hardware/software interrupt %d\n", vector));
2507 switch (vector)
2508 {
2509 case X86_XCPT_NM:
2510 {
2511 Log(("#NM fault at %RGv error code %x\n", (RTGCPTR)pCtx->rip, errCode));
2512
2513 /** @todo don't intercept #NM exceptions anymore when we've activated the guest FPU state. */
2514 /* If we sync the FPU/XMM state on-demand, then we can continue execution as if nothing has happened. */
2515 rc = CPUMR0LoadGuestFPU(pVM, pVCpu, pCtx);
2516 if (rc == VINF_SUCCESS)
2517 {
2518 Assert(CPUMIsGuestFPUStateActive(pVCpu));
2519
2520 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitShadowNM);
2521
2522 /* Continue execution. */
2523 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_GUEST_CR0;
2524
2525 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
2526 goto ResumeExecution;
2527 }
2528
2529 Log(("Forward #NM fault to the guest\n"));
2530 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestNM);
2531 rc = VMXR0InjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), cbInstr, 0);
2532 AssertRC(rc);
2533 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
2534 goto ResumeExecution;
2535 }
2536
2537 case X86_XCPT_PF: /* Page fault */
2538 {
2539#ifdef DEBUG
2540 if (pVM->hwaccm.s.fNestedPaging)
2541 { /* A genuine pagefault.
2542 * Forward the trap to the guest by injecting the exception and resuming execution.
2543 */
2544 Log(("Guest page fault at %RGv cr2=%RGv error code %x rsp=%RGv\n", (RTGCPTR)pCtx->rip, exitQualification, errCode, (RTGCPTR)pCtx->rsp));
2545
2546 Assert(CPUMIsGuestInPagedProtectedModeEx(pCtx));
2547
2548 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestPF);
2549
2550 /* Now we must update CR2. */
2551 pCtx->cr2 = exitQualification;
2552 rc = VMXR0InjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), cbInstr, errCode);
2553 AssertRC(rc);
2554
2555 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
2556 goto ResumeExecution;
2557 }
2558#endif
2559 Assert(!pVM->hwaccm.s.fNestedPaging);
2560
2561 Log2(("Page fault at %RGv error code %x\n", exitQualification, errCode));
2562 /* Exit qualification contains the linear address of the page fault. */
2563 TRPMAssertTrap(pVCpu, X86_XCPT_PF, TRPM_TRAP);
2564 TRPMSetErrorCode(pVCpu, errCode);
2565 TRPMSetFaultAddress(pVCpu, exitQualification);
2566
2567 /* Shortcut for APIC TPR reads and writes. */
2568 if ( (exitQualification & 0xfff) == 0x080
2569 && !(errCode & X86_TRAP_PF_P) /* not present */
2570 && fSetupTPRCaching
2571 && (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC))
2572 {
2573 RTGCPHYS GCPhysApicBase, GCPhys;
2574 PDMApicGetBase(pVM, &GCPhysApicBase); /* @todo cache this */
2575 GCPhysApicBase &= PAGE_BASE_GC_MASK;
2576
2577 rc = PGMGstGetPage(pVCpu, (RTGCPTR)exitQualification, NULL, &GCPhys);
2578 if ( rc == VINF_SUCCESS
2579 && GCPhys == GCPhysApicBase)
2580 {
2581 Log(("Enable VT-x virtual APIC access filtering\n"));
2582 rc = IOMMMIOMapMMIOHCPage(pVM, GCPhysApicBase, pVM->hwaccm.s.vmx.pAPICPhys, X86_PTE_RW | X86_PTE_P);
2583 AssertRC(rc);
2584 }
2585 }
2586
2587 /* Forward it to our trap handler first, in case our shadow pages are out of sync. */
2588 rc = PGMTrap0eHandler(pVCpu, errCode, CPUMCTX2CORE(pCtx), (RTGCPTR)exitQualification);
2589 Log2(("PGMTrap0eHandler %RGv returned %Rrc\n", (RTGCPTR)pCtx->rip, rc));
2590 if (rc == VINF_SUCCESS)
2591 { /* We've successfully synced our shadow pages, so let's just continue execution. */
2592 Log2(("Shadow page fault at %RGv cr2=%RGv error code %x\n", (RTGCPTR)pCtx->rip, exitQualification ,errCode));
2593 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitShadowPF);
2594
2595 TRPMResetTrap(pVCpu);
2596
2597 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
2598 goto ResumeExecution;
2599 }
2600 else
2601 if (rc == VINF_EM_RAW_GUEST_TRAP)
2602 { /* A genuine pagefault.
2603 * Forward the trap to the guest by injecting the exception and resuming execution.
2604 */
2605 Log2(("Forward page fault to the guest\n"));
2606
2607 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestPF);
2608 /* The error code might have been changed. */
2609 errCode = TRPMGetErrorCode(pVCpu);
2610
2611 TRPMResetTrap(pVCpu);
2612
2613 /* Now we must update CR2. */
2614 pCtx->cr2 = exitQualification;
2615 rc = VMXR0InjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), cbInstr, errCode);
2616 AssertRC(rc);
2617
2618 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
2619 goto ResumeExecution;
2620 }
2621#ifdef VBOX_STRICT
2622 if (rc != VINF_EM_RAW_EMULATE_INSTR && rc != VINF_EM_RAW_EMULATE_IO_BLOCK)
2623 Log2(("PGMTrap0eHandler failed with %d\n", rc));
2624#endif
2625 /* Need to go back to the recompiler to emulate the instruction. */
2626 TRPMResetTrap(pVCpu);
2627 break;
2628 }
2629
2630 case X86_XCPT_MF: /* Floating point exception. */
2631 {
2632 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestMF);
2633 if (!(pCtx->cr0 & X86_CR0_NE))
2634 {
2635 /* old style FPU error reporting needs some extra work. */
2636 /** @todo don't fall back to the recompiler, but do it manually. */
2637 rc = VINF_EM_RAW_EMULATE_INSTR;
2638 break;
2639 }
2640 Log(("Trap %x at %04X:%RGv\n", vector, pCtx->cs, (RTGCPTR)pCtx->rip));
2641 rc = VMXR0InjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), cbInstr, errCode);
2642 AssertRC(rc);
2643
2644 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
2645 goto ResumeExecution;
2646 }
2647
2648 case X86_XCPT_DB: /* Debug exception. */
2649 {
2650 uint64_t uDR6;
2651
2652 /* DR6, DR7.GD and IA32_DEBUGCTL.LBR are not updated yet.
2653 *
2654 * Exit qualification bits:
2655 * 3:0 B0-B3 which breakpoint condition was met
2656 * 12:4 Reserved (0)
2657 * 13 BD - debug register access detected
2658 * 14 BS - single step execution or branch taken
2659 * 63:15 Reserved (0)
2660 */
2661 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestDB);
2662
2663 /* Note that we don't support guest and host-initiated debugging at the same time. */
2664 Assert(DBGFIsStepping(pVCpu) || CPUMIsGuestInRealModeEx(pCtx) || CPUMIsHyperDebugStateActive(pVCpu));
2665
2666 uDR6 = X86_DR6_INIT_VAL;
2667 uDR6 |= (exitQualification & (X86_DR6_B0|X86_DR6_B1|X86_DR6_B2|X86_DR6_B3|X86_DR6_BD|X86_DR6_BS));
2668 rc = DBGFRZTrap01Handler(pVM, pVCpu, CPUMCTX2CORE(pCtx), uDR6);
2669 if (rc == VINF_EM_RAW_GUEST_TRAP)
2670 {
2671 /** @todo this isn't working, but we'll never get here normally. */
2672
2673 /* Update DR6 here. */
2674 pCtx->dr[6] = uDR6;
2675
2676 /* X86_DR7_GD will be cleared if drx accesses should be trapped inside the guest. */
2677 pCtx->dr[7] &= ~X86_DR7_GD;
2678
2679 /* Paranoia. */
2680 pCtx->dr[7] &= 0xffffffff; /* upper 32 bits reserved */
2681 pCtx->dr[7] &= ~(RT_BIT(11) | RT_BIT(12) | RT_BIT(14) | RT_BIT(15)); /* must be zero */
2682 pCtx->dr[7] |= 0x400; /* must be one */
2683
2684 /* Resync DR7 */
2685 rc = VMXWriteVMCS64(VMX_VMCS64_GUEST_DR7, pCtx->dr[7]);
2686 AssertRC(rc);
2687
2688 Log(("Trap %x (debug) at %RGv exit qualification %RX64\n", vector, (RTGCPTR)pCtx->rip, exitQualification));
2689 rc = VMXR0InjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), cbInstr, errCode);
2690 AssertRC(rc);
2691
2692 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
2693 goto ResumeExecution;
2694 }
2695 /* Return to ring 3 to deal with the debug exit code. */
2696 Log(("Debugger hardware BP at %04x:%RGv (rc=%Rrc)\n", pCtx->cs, pCtx->rip, rc));
2697 break;
2698 }
2699
2700 case X86_XCPT_BP: /* Breakpoint. */
2701 {
2702 rc = DBGFRZTrap03Handler(pVM, pVCpu, CPUMCTX2CORE(pCtx));
2703 if (rc == VINF_EM_RAW_GUEST_TRAP)
2704 {
2705 Log(("Guest #BP at %04x:%RGv\n", pCtx->cs, pCtx->rip));
2706 rc = VMXR0InjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), cbInstr, errCode);
2707 AssertRC(rc);
2708 goto ResumeExecution;
2709 }
2710 if (rc == VINF_SUCCESS)
2711 goto ResumeExecution;
2712 Log(("Debugger BP at %04x:%RGv (rc=%Rrc)\n", pCtx->cs, pCtx->rip, rc));
2713 break;
2714 }
2715
2716 case X86_XCPT_GP: /* General protection failure exception.*/
2717 {
2718 uint32_t cbOp;
2719 uint32_t cbSize;
2720 PDISCPUSTATE pDis = &pVCpu->hwaccm.s.DisState;
2721
2722 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestGP);
2723#ifdef VBOX_STRICT
2724 if (!CPUMIsGuestInRealModeEx(pCtx))
2725 {
2726 Log(("Trap %x at %04X:%RGv errorCode=%x\n", vector, pCtx->cs, (RTGCPTR)pCtx->rip, errCode));
2727 rc = VMXR0InjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), cbInstr, errCode);
2728 AssertRC(rc);
2729 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
2730 goto ResumeExecution;
2731 }
2732#endif
2733 Assert(CPUMIsGuestInRealModeEx(pCtx));
2734
2735 LogFlow(("Real mode X86_XCPT_GP instruction emulation at %RGv\n", (RTGCPTR)pCtx->rip));
2736
2737 rc = EMInterpretDisasOne(pVM, pVCpu, CPUMCTX2CORE(pCtx), pDis, &cbOp);
2738 if (RT_SUCCESS(rc))
2739 {
2740 bool fUpdateRIP = true;
2741
2742 Assert(cbOp == pDis->opsize);
2743 switch (pDis->pCurInstr->opcode)
2744 {
2745 case OP_CLI:
2746 pCtx->eflags.Bits.u1IF = 0;
2747 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitCli);
2748 break;
2749
2750 case OP_STI:
2751 pCtx->eflags.Bits.u1IF = 1;
2752 EMSetInhibitInterruptsPC(pVCpu, pCtx->rip + pDis->opsize);
2753 Assert(VMCPU_FF_ISSET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
2754 rc = VMXWriteVMCS(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
2755 AssertRC(rc);
2756 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitSti);
2757 break;
2758
2759 case OP_HLT:
2760 fUpdateRIP = false;
2761 rc = VINF_EM_HALT;
2762 pCtx->rip += pDis->opsize;
2763 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitHlt);
2764 break;
2765
2766 case OP_POPF:
2767 {
2768 RTGCPTR GCPtrStack;
2769 uint32_t cbParm;
2770 uint32_t uMask;
2771 X86EFLAGS eflags;
2772
2773 if (pDis->prefix & PREFIX_OPSIZE)
2774 {
2775 cbParm = 4;
2776 uMask = 0xffffffff;
2777 }
2778 else
2779 {
2780 cbParm = 2;
2781 uMask = 0xffff;
2782 }
2783
2784 rc = SELMToFlatEx(pVM, DIS_SELREG_SS, CPUMCTX2CORE(pCtx), pCtx->esp & uMask, 0, &GCPtrStack);
2785 if (RT_FAILURE(rc))
2786 {
2787 rc = VERR_EM_INTERPRETER;
2788 break;
2789 }
2790 eflags.u = 0;
2791 rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &eflags.u, cbParm);
2792 if (RT_FAILURE(rc))
2793 {
2794 rc = VERR_EM_INTERPRETER;
2795 break;
2796 }
2797 LogFlow(("POPF %x -> %RGv mask=%x\n", eflags.u, pCtx->rsp, uMask));
2798 pCtx->eflags.u = (pCtx->eflags.u & ~(X86_EFL_POPF_BITS & uMask)) | (eflags.u & X86_EFL_POPF_BITS & uMask);
2799 /* RF cleared when popped in real mode; see pushf description in AMD manual. */
2800 pCtx->eflags.Bits.u1RF = 0;
2801 pCtx->esp += cbParm;
2802 pCtx->esp &= uMask;
2803
2804 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitPopf);
2805 break;
2806 }
2807
2808 case OP_PUSHF:
2809 {
2810 RTGCPTR GCPtrStack;
2811 uint32_t cbParm;
2812 uint32_t uMask;
2813 X86EFLAGS eflags;
2814
2815 if (pDis->prefix & PREFIX_OPSIZE)
2816 {
2817 cbParm = 4;
2818 uMask = 0xffffffff;
2819 }
2820 else
2821 {
2822 cbParm = 2;
2823 uMask = 0xffff;
2824 }
2825
2826 rc = SELMToFlatEx(pVM, DIS_SELREG_SS, CPUMCTX2CORE(pCtx), (pCtx->esp - cbParm) & uMask, 0, &GCPtrStack);
2827 if (RT_FAILURE(rc))
2828 {
2829 rc = VERR_EM_INTERPRETER;
2830 break;
2831 }
2832 eflags = pCtx->eflags;
2833 /* RF & VM cleared when pushed in real mode; see pushf description in AMD manual. */
2834 eflags.Bits.u1RF = 0;
2835 eflags.Bits.u1VM = 0;
2836
2837 rc = PGMPhysWrite(pVM, (RTGCPHYS)GCPtrStack, &eflags.u, cbParm);
2838 if (RT_FAILURE(rc))
2839 {
2840 rc = VERR_EM_INTERPRETER;
2841 break;
2842 }
2843 LogFlow(("PUSHF %x -> %RGv\n", eflags.u, GCPtrStack));
2844 pCtx->esp -= cbParm;
2845 pCtx->esp &= uMask;
2846 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitPushf);
2847 break;
2848 }
2849
2850 case OP_IRET:
2851 {
2852 RTGCPTR GCPtrStack;
2853 uint32_t uMask = 0xffff;
2854 uint16_t aIretFrame[3];
2855
2856 if (pDis->prefix & (PREFIX_OPSIZE | PREFIX_ADDRSIZE))
2857 {
2858 rc = VERR_EM_INTERPRETER;
2859 break;
2860 }
2861
2862 rc = SELMToFlatEx(pVM, DIS_SELREG_SS, CPUMCTX2CORE(pCtx), pCtx->esp & uMask, 0, &GCPtrStack);
2863 if (RT_FAILURE(rc))
2864 {
2865 rc = VERR_EM_INTERPRETER;
2866 break;
2867 }
2868 rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &aIretFrame[0], sizeof(aIretFrame));
2869 if (RT_FAILURE(rc))
2870 {
2871 rc = VERR_EM_INTERPRETER;
2872 break;
2873 }
2874 pCtx->ip = aIretFrame[0];
2875 pCtx->cs = aIretFrame[1];
2876 pCtx->csHid.u64Base = pCtx->cs << 4;
2877 pCtx->eflags.u = (pCtx->eflags.u & ~(X86_EFL_POPF_BITS & uMask)) | (aIretFrame[2] & X86_EFL_POPF_BITS & uMask);
2878 pCtx->sp += sizeof(aIretFrame);
2879
2880 LogFlow(("iret to %04x:%x\n", pCtx->cs, pCtx->ip));
2881 fUpdateRIP = false;
2882 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitIret);
2883 break;
2884 }
2885
2886 case OP_INT:
2887 {
2888 RTGCUINTPTR intInfo;
2889
2890 LogFlow(("Realmode: INT %x\n", pDis->param1.parval & 0xff));
2891 intInfo = pDis->param1.parval & 0xff;
2892 intInfo |= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
2893 intInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
2894
2895 rc = VMXR0InjectEvent(pVM, pVCpu, pCtx, intInfo, cbOp, 0);
2896 AssertRC(rc);
2897 fUpdateRIP = false;
2898 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitInt);
2899 break;
2900 }
2901
2902 case OP_INTO:
2903 {
2904 if (pCtx->eflags.Bits.u1OF)
2905 {
2906 RTGCUINTPTR intInfo;
2907
2908 LogFlow(("Realmode: INTO\n"));
2909 intInfo = X86_XCPT_OF;
2910 intInfo |= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
2911 intInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
2912
2913 rc = VMXR0InjectEvent(pVM, pVCpu, pCtx, intInfo, cbOp, 0);
2914 AssertRC(rc);
2915 fUpdateRIP = false;
2916 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitInt);
2917 }
2918 break;
2919 }
2920
2921 case OP_INT3:
2922 {
2923 RTGCUINTPTR intInfo;
2924
2925 LogFlow(("Realmode: INT 3\n"));
2926 intInfo = 3;
2927 intInfo |= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
2928 intInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
2929
2930 rc = VMXR0InjectEvent(pVM, pVCpu, pCtx, intInfo, cbOp, 0);
2931 AssertRC(rc);
2932 fUpdateRIP = false;
2933 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitInt);
2934 break;
2935 }
2936
2937 default:
2938 rc = EMInterpretInstructionCPU(pVM, pVCpu, pDis, CPUMCTX2CORE(pCtx), 0, &cbSize);
2939 break;
2940 }
2941
2942 if (rc == VINF_SUCCESS)
2943 {
2944 if (fUpdateRIP)
2945 pCtx->rip += cbOp; /* Move on to the next instruction. */
2946
2947 /* lidt, lgdt can end up here. In the future crx changes as well. Just reload the whole context to be done with it. */
2948 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_ALL;
2949
2950 /* Only resume if successful. */
2951 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
2952 goto ResumeExecution;
2953 }
2954 }
2955 else
2956 rc = VERR_EM_INTERPRETER;
2957
2958 AssertMsg(rc == VERR_EM_INTERPRETER || rc == VINF_PGM_CHANGE_MODE || rc == VINF_EM_HALT, ("Unexpected rc=%Rrc\n", rc));
2959 break;
2960 }
2961
2962#ifdef VBOX_STRICT
2963 case X86_XCPT_DE: /* Divide error. */
2964 case X86_XCPT_UD: /* Unknown opcode exception. */
2965 case X86_XCPT_SS: /* Stack segment exception. */
2966 case X86_XCPT_NP: /* Segment not present exception. */
2967 {
2968 switch(vector)
2969 {
2970 case X86_XCPT_DE:
2971 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestDE);
2972 break;
2973 case X86_XCPT_UD:
2974 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestUD);
2975 break;
2976 case X86_XCPT_SS:
2977 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestSS);
2978 break;
2979 case X86_XCPT_NP:
2980 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestNP);
2981 break;
2982 }
2983
2984 Log(("Trap %x at %04X:%RGv\n", vector, pCtx->cs, (RTGCPTR)pCtx->rip));
2985 rc = VMXR0InjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), cbInstr, errCode);
2986 AssertRC(rc);
2987
2988 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
2989 goto ResumeExecution;
2990 }
2991#endif
2992 default:
2993#ifdef HWACCM_VMX_EMULATE_REALMODE
2994 if (CPUMIsGuestInRealModeEx(pCtx))
2995 {
2996 Log(("Real Mode Trap %x at %04x:%04X error code %x\n", vector, pCtx->cs, pCtx->eip, errCode));
2997 rc = VMXR0InjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), cbInstr, errCode);
2998 AssertRC(rc);
2999
3000 /* Go back to ring 3 in case of a triple fault. */
3001 if ( vector == X86_XCPT_DF
3002 && rc == VINF_EM_RESET)
3003 break;
3004
3005 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3006 goto ResumeExecution;
3007 }
3008#endif
3009 AssertMsgFailed(("Unexpected vm-exit caused by exception %x\n", vector));
3010 rc = VERR_VMX_UNEXPECTED_EXCEPTION;
3011 break;
3012 } /* switch (vector) */
3013
3014 break;
3015
3016 default:
3017 rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_CODE;
3018 AssertMsgFailed(("Unexpected interuption code %x\n", intInfo));
3019 break;
3020 }
3021
3022 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3023 break;
3024 }
3025
3026 case VMX_EXIT_EPT_VIOLATION: /* 48 EPT violation. An attempt to access memory with a guest-physical address was disallowed by the configuration of the EPT paging structures. */
3027 {
3028 RTGCPHYS GCPhys;
3029
3030 Assert(pVM->hwaccm.s.fNestedPaging);
3031
3032 rc = VMXReadVMCS64(VMX_VMCS_EXIT_PHYS_ADDR_FULL, &GCPhys);
3033 AssertRC(rc);
3034 Assert(((exitQualification >> 7) & 3) != 2);
3035
3036 /* Determine the kind of violation. */
3037 errCode = 0;
3038 if (exitQualification & VMX_EXIT_QUALIFICATION_EPT_INSTR_FETCH)
3039 errCode |= X86_TRAP_PF_ID;
3040
3041 if (exitQualification & VMX_EXIT_QUALIFICATION_EPT_DATA_WRITE)
3042 errCode |= X86_TRAP_PF_RW;
3043
3044 /* If the page is present, then it's a page level protection fault. */
3045 if (exitQualification & VMX_EXIT_QUALIFICATION_EPT_ENTRY_PRESENT)
3046 {
3047 errCode |= X86_TRAP_PF_P;
3048 }
3049 else {
3050 /* Shortcut for APIC TPR reads and writes. */
3051 if ( (GCPhys & 0xfff) == 0x080
3052 && GCPhys > 0x1000000 /* to skip VGA frame buffer accesses */
3053 && fSetupTPRCaching
3054 && (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC))
3055 {
3056 RTGCPHYS GCPhysApicBase;
3057 PDMApicGetBase(pVM, &GCPhysApicBase); /* @todo cache this */
3058 GCPhysApicBase &= PAGE_BASE_GC_MASK;
3059 if (GCPhys == GCPhysApicBase + 0x80)
3060 {
3061 Log(("Enable VT-x virtual APIC access filtering\n"));
3062 rc = IOMMMIOMapMMIOHCPage(pVM, GCPhysApicBase, pVM->hwaccm.s.vmx.pAPICPhys, X86_PTE_RW | X86_PTE_P);
3063 AssertRC(rc);
3064 }
3065 }
3066 }
3067 Log(("EPT Page fault %x at %RGp error code %x\n", (uint32_t)exitQualification, GCPhys, errCode));
3068
3069 /* GCPhys contains the guest physical address of the page fault. */
3070 TRPMAssertTrap(pVCpu, X86_XCPT_PF, TRPM_TRAP);
3071 TRPMSetErrorCode(pVCpu, errCode);
3072 TRPMSetFaultAddress(pVCpu, GCPhys);
3073
3074 /* Handle the pagefault trap for the nested shadow table. */
3075 rc = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, errCode, CPUMCTX2CORE(pCtx), GCPhys);
3076 Log2(("PGMR0Trap0eHandlerNestedPaging %RGv returned %Rrc\n", (RTGCPTR)pCtx->rip, rc));
3077 if (rc == VINF_SUCCESS)
3078 { /* We've successfully synced our shadow pages, so let's just continue execution. */
3079 Log2(("Shadow page fault at %RGv cr2=%RGp error code %x\n", (RTGCPTR)pCtx->rip, exitQualification , errCode));
3080 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitReasonNPF);
3081
3082 TRPMResetTrap(pVCpu);
3083
3084 goto ResumeExecution;
3085 }
3086
3087#ifdef VBOX_STRICT
3088 if (rc != VINF_EM_RAW_EMULATE_INSTR)
3089 LogFlow(("PGMTrap0eHandlerNestedPaging failed with %d\n", rc));
3090#endif
3091 /* Need to go back to the recompiler to emulate the instruction. */
3092 TRPMResetTrap(pVCpu);
3093 break;
3094 }
3095
3096 case VMX_EXIT_EPT_MISCONFIG:
3097 {
3098 RTGCPHYS GCPhys;
3099
3100 Assert(pVM->hwaccm.s.fNestedPaging);
3101
3102 rc = VMXReadVMCS64(VMX_VMCS_EXIT_PHYS_ADDR_FULL, &GCPhys);
3103 AssertRC(rc);
3104
3105 Log(("VMX_EXIT_EPT_MISCONFIG for %VGp\n", GCPhys));
3106 break;
3107 }
3108
3109 case VMX_EXIT_IRQ_WINDOW: /* 7 Interrupt window. */
3110 /* Clear VM-exit on IF=1 change. */
3111 LogFlow(("VMX_EXIT_IRQ_WINDOW %RGv pending=%d IF=%d\n", (RTGCPTR)pCtx->rip, VMCPU_FF_ISPENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC|VMCPU_FF_INTERRUPT_PIC)), pCtx->eflags.Bits.u1IF));
3112 pVCpu->hwaccm.s.vmx.proc_ctls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_IRQ_WINDOW_EXIT;
3113 rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
3114 AssertRC(rc);
3115 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitIrqWindow);
3116 goto ResumeExecution; /* we check for pending guest interrupts there */
3117
3118 case VMX_EXIT_WBINVD: /* 54 Guest software attempted to execute WBINVD. (conditional) */
3119 case VMX_EXIT_INVD: /* 13 Guest software attempted to execute INVD. (unconditional) */
3120 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitInvd);
3121 /* Skip instruction and continue directly. */
3122 pCtx->rip += cbInstr;
3123 /* Continue execution.*/
3124 goto ResumeExecution;
3125
3126 case VMX_EXIT_CPUID: /* 10 Guest software attempted to execute CPUID. */
3127 {
3128 Log2(("VMX: Cpuid %x\n", pCtx->eax));
3129 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitCpuid);
3130 rc = EMInterpretCpuId(pVM, pVCpu, CPUMCTX2CORE(pCtx));
3131 if (rc == VINF_SUCCESS)
3132 {
3133 /* Update EIP and continue execution. */
3134 Assert(cbInstr == 2);
3135 pCtx->rip += cbInstr;
3136 goto ResumeExecution;
3137 }
3138 AssertMsgFailed(("EMU: cpuid failed with %Rrc\n", rc));
3139 rc = VINF_EM_RAW_EMULATE_INSTR;
3140 break;
3141 }
3142
3143 case VMX_EXIT_RDPMC: /* 15 Guest software attempted to execute RDPMC. */
3144 {
3145 Log2(("VMX: Rdpmc %x\n", pCtx->ecx));
3146 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitRdpmc);
3147 rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pCtx));
3148 if (rc == VINF_SUCCESS)
3149 {
3150 /* Update EIP and continue execution. */
3151 Assert(cbInstr == 2);
3152 pCtx->rip += cbInstr;
3153 goto ResumeExecution;
3154 }
3155 rc = VINF_EM_RAW_EMULATE_INSTR;
3156 break;
3157 }
3158
3159 case VMX_EXIT_RDTSC: /* 16 Guest software attempted to execute RDTSC. */
3160 {
3161 Log2(("VMX: Rdtsc\n"));
3162 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitRdtsc);
3163 rc = EMInterpretRdtsc(pVM, pVCpu, CPUMCTX2CORE(pCtx));
3164 if (rc == VINF_SUCCESS)
3165 {
3166 /* Update EIP and continue execution. */
3167 Assert(cbInstr == 2);
3168 pCtx->rip += cbInstr;
3169 goto ResumeExecution;
3170 }
3171 rc = VINF_EM_RAW_EMULATE_INSTR;
3172 break;
3173 }
3174
3175 case VMX_EXIT_INVPG: /* 14 Guest software attempted to execute INVPG. */
3176 {
3177 Log2(("VMX: invlpg\n"));
3178 Assert(!pVM->hwaccm.s.fNestedPaging);
3179
3180 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitInvpg);
3181 rc = EMInterpretInvlpg(pVM, pVCpu, CPUMCTX2CORE(pCtx), exitQualification);
3182 if (rc == VINF_SUCCESS)
3183 {
3184 /* Update EIP and continue execution. */
3185 pCtx->rip += cbInstr;
3186 goto ResumeExecution;
3187 }
3188 AssertMsg(rc == VERR_EM_INTERPRETER, ("EMU: invlpg %RGv failed with %Rrc\n", exitQualification, rc));
3189 break;
3190 }
3191
3192 case VMX_EXIT_RDMSR: /* 31 RDMSR. Guest software attempted to execute RDMSR. */
3193 case VMX_EXIT_WRMSR: /* 32 WRMSR. Guest software attempted to execute WRMSR. */
3194 {
3195 uint32_t cbSize;
3196
3197 STAM_COUNTER_INC((exitReason == VMX_EXIT_RDMSR) ? &pVCpu->hwaccm.s.StatExitRdmsr : &pVCpu->hwaccm.s.StatExitWrmsr);
3198
3199 /* Note: the intel manual claims there's a REX version of RDMSR that's slightly different, so we play safe by completely disassembling the instruction. */
3200 Log2(("VMX: %s\n", (exitReason == VMX_EXIT_RDMSR) ? "rdmsr" : "wrmsr"));
3201 rc = EMInterpretInstruction(pVM, pVCpu, CPUMCTX2CORE(pCtx), 0, &cbSize);
3202 if (rc == VINF_SUCCESS)
3203 {
3204 /* EIP has been updated already. */
3205
3206 /* Only resume if successful. */
3207 goto ResumeExecution;
3208 }
3209 AssertMsg(rc == VERR_EM_INTERPRETER, ("EMU: %s failed with %Rrc\n", (exitReason == VMX_EXIT_RDMSR) ? "rdmsr" : "wrmsr", rc));
3210 break;
3211 }
3212
3213 case VMX_EXIT_CRX_MOVE: /* 28 Control-register accesses. */
3214 {
3215 STAM_PROFILE_ADV_START(&pVCpu->hwaccm.s.StatExit2Sub2, y2);
3216
3217 switch (VMX_EXIT_QUALIFICATION_CRX_ACCESS(exitQualification))
3218 {
3219 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_WRITE:
3220 Log2(("VMX: %RGv mov cr%d, x\n", (RTGCPTR)pCtx->rip, VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification)));
3221 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitCRxWrite[VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification)]);
3222 rc = EMInterpretCRxWrite(pVM, pVCpu, CPUMCTX2CORE(pCtx),
3223 VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification),
3224 VMX_EXIT_QUALIFICATION_CRX_GENREG(exitQualification));
3225
3226 switch (VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification))
3227 {
3228 case 0:
3229 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_GUEST_CR0 | HWACCM_CHANGED_GUEST_CR3;
3230 break;
3231 case 2:
3232 break;
3233 case 3:
3234 Assert(!pVM->hwaccm.s.fNestedPaging || !CPUMIsGuestInPagedProtectedModeEx(pCtx));
3235 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_GUEST_CR3;
3236 break;
3237 case 4:
3238 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_GUEST_CR4;
3239 break;
3240 case 8:
3241 /* CR8 contains the APIC TPR */
3242 Assert(!(pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW));
3243 break;
3244
3245 default:
3246 AssertFailed();
3247 break;
3248 }
3249 /* Check if a sync operation is pending. */
3250 if ( rc == VINF_SUCCESS /* don't bother if we are going to ring 3 anyway */
3251 && VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
3252 {
3253 rc = PGMSyncCR3(pVCpu, pCtx->cr0, pCtx->cr3, pCtx->cr4, VMCPU_FF_ISSET(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
3254 AssertRC(rc);
3255 }
3256 break;
3257
3258 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ:
3259 Log2(("VMX: mov x, crx\n"));
3260 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitCRxRead[VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification)]);
3261
3262 Assert(!pVM->hwaccm.s.fNestedPaging || !CPUMIsGuestInPagedProtectedModeEx(pCtx) || VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification) != USE_REG_CR3);
3263
3264 /* CR8 reads only cause an exit when the TPR shadow feature isn't present. */
3265 Assert(VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification) != 8 || !(pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW));
3266
3267 rc = EMInterpretCRxRead(pVM, pVCpu, CPUMCTX2CORE(pCtx),
3268 VMX_EXIT_QUALIFICATION_CRX_GENREG(exitQualification),
3269 VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification));
3270 break;
3271
3272 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_CLTS:
3273 Log2(("VMX: clts\n"));
3274 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitCLTS);
3275 rc = EMInterpretCLTS(pVM, pVCpu);
3276 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_GUEST_CR0;
3277 break;
3278
3279 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_LMSW:
3280 Log2(("VMX: lmsw %x\n", VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(exitQualification)));
3281 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitLMSW);
3282 rc = EMInterpretLMSW(pVM, pVCpu, CPUMCTX2CORE(pCtx), VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(exitQualification));
3283 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_GUEST_CR0;
3284 break;
3285 }
3286
3287 /* Update EIP if no error occurred. */
3288 if (RT_SUCCESS(rc))
3289 pCtx->rip += cbInstr;
3290
3291 if (rc == VINF_SUCCESS)
3292 {
3293 /* Only resume if successful. */
3294 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub2, y2);
3295 goto ResumeExecution;
3296 }
3297 Assert(rc == VERR_EM_INTERPRETER || rc == VINF_PGM_CHANGE_MODE || rc == VINF_PGM_SYNC_CR3);
3298 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub2, y2);
3299 break;
3300 }
3301
3302 case VMX_EXIT_DRX_MOVE: /* 29 Debug-register accesses. */
3303 {
3304 if ( !DBGFIsStepping(pVCpu)
3305 && !CPUMIsHyperDebugStateActive(pVCpu))
3306 {
3307 /* Disable drx move intercepts. */
3308 pVCpu->hwaccm.s.vmx.proc_ctls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT;
3309 rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
3310 AssertRC(rc);
3311
3312 /* Save the host and load the guest debug state. */
3313 rc = CPUMR0LoadGuestDebugState(pVM, pVCpu, pCtx, true /* include DR6 */);
3314 AssertRC(rc);
3315
3316#ifdef VBOX_WITH_STATISTICS
3317 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatDRxContextSwitch);
3318 if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(exitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
3319 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitDRxWrite);
3320 else
3321 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitDRxRead);
3322#endif
3323
3324 goto ResumeExecution;
3325 }
3326
3327 /** @todo clear VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT after the first time and restore drx registers afterwards */
3328 if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(exitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
3329 {
3330 Log2(("VMX: mov drx%d, genreg%d\n", VMX_EXIT_QUALIFICATION_DRX_REGISTER(exitQualification), VMX_EXIT_QUALIFICATION_DRX_GENREG(exitQualification)));
3331 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitDRxWrite);
3332 rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pCtx),
3333 VMX_EXIT_QUALIFICATION_DRX_REGISTER(exitQualification),
3334 VMX_EXIT_QUALIFICATION_DRX_GENREG(exitQualification));
3335 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_GUEST_DEBUG;
3336 Log2(("DR7=%08x\n", pCtx->dr[7]));
3337 }
3338 else
3339 {
3340 Log2(("VMX: mov x, drx\n"));
3341 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitDRxRead);
3342 rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pCtx),
3343 VMX_EXIT_QUALIFICATION_DRX_GENREG(exitQualification),
3344 VMX_EXIT_QUALIFICATION_DRX_REGISTER(exitQualification));
3345 }
3346 /* Update EIP if no error occurred. */
3347 if (RT_SUCCESS(rc))
3348 pCtx->rip += cbInstr;
3349
3350 if (rc == VINF_SUCCESS)
3351 {
3352 /* Only resume if successful. */
3353 goto ResumeExecution;
3354 }
3355 Assert(rc == VERR_EM_INTERPRETER);
3356 break;
3357 }
3358
3359 /* Note: We'll get a #GP if the IO instruction isn't allowed (IOPL or TSS bitmap); no need to double check. */
3360 case VMX_EXIT_PORT_IO: /* 30 I/O instruction. */
3361 {
3362 STAM_PROFILE_ADV_START(&pVCpu->hwaccm.s.StatExit2Sub1, y1);
3363 uint32_t uIOWidth = VMX_EXIT_QUALIFICATION_IO_WIDTH(exitQualification);
3364 uint32_t uPort;
3365 bool fIOWrite = (VMX_EXIT_QUALIFICATION_IO_DIRECTION(exitQualification) == VMX_EXIT_QUALIFICATION_IO_DIRECTION_OUT);
3366
3367 /** @todo necessary to make the distinction? */
3368 if (VMX_EXIT_QUALIFICATION_IO_ENCODING(exitQualification) == VMX_EXIT_QUALIFICATION_IO_ENCODING_DX)
3369 {
3370 uPort = pCtx->edx & 0xffff;
3371 }
3372 else
3373 uPort = VMX_EXIT_QUALIFICATION_IO_PORT(exitQualification); /* Immediate encoding. */
3374
3375 /* paranoia */
3376 if (RT_UNLIKELY(uIOWidth == 2 || uIOWidth >= 4))
3377 {
3378 rc = fIOWrite ? VINF_IOM_HC_IOPORT_WRITE : VINF_IOM_HC_IOPORT_READ;
3379 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub1, y1);
3380 break;
3381 }
3382
3383 uint32_t cbSize = g_aIOSize[uIOWidth];
3384
3385 if (VMX_EXIT_QUALIFICATION_IO_STRING(exitQualification))
3386 {
3387 /* ins/outs */
3388 PDISCPUSTATE pDis = &pVCpu->hwaccm.s.DisState;
3389
3390 /* Disassemble manually to deal with segment prefixes. */
3391 /** @todo VMX_VMCS_EXIT_GUEST_LINEAR_ADDR contains the flat pointer operand of the instruction. */
3392 /** @todo VMX_VMCS32_RO_EXIT_INSTR_INFO also contains segment prefix info. */
3393 rc = EMInterpretDisasOne(pVM, pVCpu, CPUMCTX2CORE(pCtx), pDis, NULL);
3394 if (rc == VINF_SUCCESS)
3395 {
3396 if (fIOWrite)
3397 {
3398 Log2(("IOMInterpretOUTSEx %RGv %x size=%d\n", (RTGCPTR)pCtx->rip, uPort, cbSize));
3399 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitIOStringWrite);
3400 rc = IOMInterpretOUTSEx(pVM, CPUMCTX2CORE(pCtx), uPort, pDis->prefix, cbSize);
3401 }
3402 else
3403 {
3404 Log2(("IOMInterpretINSEx %RGv %x size=%d\n", (RTGCPTR)pCtx->rip, uPort, cbSize));
3405 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitIOStringRead);
3406 rc = IOMInterpretINSEx(pVM, CPUMCTX2CORE(pCtx), uPort, pDis->prefix, cbSize);
3407 }
3408 }
3409 else
3410 rc = VINF_EM_RAW_EMULATE_INSTR;
3411 }
3412 else
3413 {
3414 /* normal in/out */
3415 uint32_t uAndVal = g_aIOOpAnd[uIOWidth];
3416
3417 Assert(!VMX_EXIT_QUALIFICATION_IO_REP(exitQualification));
3418
3419 if (fIOWrite)
3420 {
3421 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitIOWrite);
3422 rc = IOMIOPortWrite(pVM, uPort, pCtx->eax & uAndVal, cbSize);
3423 if (rc == VINF_IOM_HC_IOPORT_WRITE)
3424 HWACCMR0SavePendingIOPortWrite(pVCpu, pCtx->rip, pCtx->rip + cbInstr, uPort, uAndVal, cbSize);
3425 }
3426 else
3427 {
3428 uint32_t u32Val = 0;
3429
3430 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitIORead);
3431 rc = IOMIOPortRead(pVM, uPort, &u32Val, cbSize);
3432 if (IOM_SUCCESS(rc))
3433 {
3434 /* Write back to the EAX register. */
3435 pCtx->eax = (pCtx->eax & ~uAndVal) | (u32Val & uAndVal);
3436 }
3437 else
3438 if (rc == VINF_IOM_HC_IOPORT_READ)
3439 HWACCMR0SavePendingIOPortRead(pVCpu, pCtx->rip, pCtx->rip + cbInstr, uPort, uAndVal, cbSize);
3440 }
3441 }
3442 /*
3443 * Handled the I/O return codes.
3444 * (The unhandled cases end up with rc == VINF_EM_RAW_EMULATE_INSTR.)
3445 */
3446 if (IOM_SUCCESS(rc))
3447 {
3448 /* Update EIP and continue execution. */
3449 pCtx->rip += cbInstr;
3450 if (RT_LIKELY(rc == VINF_SUCCESS))
3451 {
3452 /* If any IO breakpoints are armed, then we should check if a debug trap needs to be generated. */
3453 if (pCtx->dr[7] & X86_DR7_ENABLED_MASK)
3454 {
3455 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatDRxIOCheck);
3456 for (unsigned i=0;i<4;i++)
3457 {
3458 unsigned uBPLen = g_aIOSize[X86_DR7_GET_LEN(pCtx->dr[7], i)];
3459
3460 if ( (uPort >= pCtx->dr[i] && uPort < pCtx->dr[i] + uBPLen)
3461 && (pCtx->dr[7] & (X86_DR7_L(i) | X86_DR7_G(i)))
3462 && (pCtx->dr[7] & X86_DR7_RW(i, X86_DR7_RW_IO)) == X86_DR7_RW(i, X86_DR7_RW_IO))
3463 {
3464 uint64_t uDR6;
3465
3466 Assert(CPUMIsGuestDebugStateActive(pVCpu));
3467
3468 uDR6 = ASMGetDR6();
3469
3470 /* Clear all breakpoint status flags and set the one we just hit. */
3471 uDR6 &= ~(X86_DR6_B0|X86_DR6_B1|X86_DR6_B2|X86_DR6_B3);
3472 uDR6 |= (uint64_t)RT_BIT(i);
3473
3474 /* Note: AMD64 Architecture Programmer's Manual 13.1:
3475 * Bits 15:13 of the DR6 register is never cleared by the processor and must be cleared by software after
3476 * the contents have been read.
3477 */
3478 ASMSetDR6(uDR6);
3479
3480 /* X86_DR7_GD will be cleared if drx accesses should be trapped inside the guest. */
3481 pCtx->dr[7] &= ~X86_DR7_GD;
3482
3483 /* Paranoia. */
3484 pCtx->dr[7] &= 0xffffffff; /* upper 32 bits reserved */
3485 pCtx->dr[7] &= ~(RT_BIT(11) | RT_BIT(12) | RT_BIT(14) | RT_BIT(15)); /* must be zero */
3486 pCtx->dr[7] |= 0x400; /* must be one */
3487
3488 /* Resync DR7 */
3489 rc = VMXWriteVMCS64(VMX_VMCS64_GUEST_DR7, pCtx->dr[7]);
3490 AssertRC(rc);
3491
3492 /* Construct inject info. */
3493 intInfo = X86_XCPT_DB;
3494 intInfo |= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
3495 intInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HWEXCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
3496
3497 Log(("Inject IO debug trap at %RGv\n", (RTGCPTR)pCtx->rip));
3498 rc = VMXR0InjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), 0, 0);
3499 AssertRC(rc);
3500
3501 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub1, y1);
3502 goto ResumeExecution;
3503 }
3504 }
3505 }
3506
3507 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub1, y1);
3508 goto ResumeExecution;
3509 }
3510 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub1, y1);
3511 break;
3512 }
3513
3514#ifdef VBOX_STRICT
3515 if (rc == VINF_IOM_HC_IOPORT_READ)
3516 Assert(!fIOWrite);
3517 else if (rc == VINF_IOM_HC_IOPORT_WRITE)
3518 Assert(fIOWrite);
3519 else
3520 AssertMsg(RT_FAILURE(rc) || rc == VINF_EM_RAW_EMULATE_INSTR || rc == VINF_EM_RAW_GUEST_TRAP || rc == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", rc));
3521#endif
3522 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub1, y1);
3523 break;
3524 }
3525
3526 case VMX_EXIT_TPR: /* 43 TPR below threshold. Guest software executed MOV to CR8. */
3527 LogFlow(("VMX_EXIT_TPR\n"));
3528 /* RIP is already set to the next instruction and the TPR has been synced back. Just resume. */
3529 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub1, y1);
3530 goto ResumeExecution;
3531
3532 case VMX_EXIT_APIC_ACCESS: /* 44 APIC access. Guest software attempted to access memory at a physical address on the APIC-access page. */
3533 {
3534 LogFlow(("VMX_EXIT_APIC_ACCESS\n"));
3535 unsigned uAccessType = VMX_EXIT_QUALIFICATION_APIC_ACCESS_TYPE(exitQualification);
3536
3537 switch(uAccessType)
3538 {
3539 case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
3540 case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
3541 {
3542 RTGCPHYS GCPhys;
3543 PDMApicGetBase(pVM, &GCPhys);
3544 GCPhys &= PAGE_BASE_GC_MASK;
3545 GCPhys += VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(exitQualification);
3546
3547 LogFlow(("Apic access at %RGp\n", GCPhys));
3548 rc = IOMMMIOPhysHandler(pVM, (uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ) ? 0 : X86_TRAP_PF_RW, CPUMCTX2CORE(pCtx), GCPhys);
3549 if (rc == VINF_SUCCESS)
3550 {
3551 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub1, y1);
3552 goto ResumeExecution; /* rip already updated */
3553 }
3554 break;
3555 }
3556
3557 default:
3558 rc = VINF_EM_RAW_EMULATE_INSTR;
3559 break;
3560 }
3561 break;
3562 }
3563
3564 case VMX_EXIT_PREEMPTION_TIMER: /* 52 VMX-preemption timer expired. The preemption timer counted down to zero. */
3565 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub1, y1);
3566 goto ResumeExecution;
3567
3568 default:
3569 /* The rest is handled after syncing the entire CPU state. */
3570 break;
3571 }
3572
3573 /* Note: the guest state isn't entirely synced back at this stage. */
3574
3575 /* Investigate why there was a VM-exit. (part 2) */
3576 switch (exitReason)
3577 {
3578 case VMX_EXIT_EXCEPTION: /* 0 Exception or non-maskable interrupt (NMI). */
3579 case VMX_EXIT_EXTERNAL_IRQ: /* 1 External interrupt. */
3580 case VMX_EXIT_EPT_VIOLATION:
3581 case VMX_EXIT_PREEMPTION_TIMER: /* 52 VMX-preemption timer expired. The preemption timer counted down to zero. */
3582 /* Already handled above. */
3583 break;
3584
3585 case VMX_EXIT_TRIPLE_FAULT: /* 2 Triple fault. */
3586 rc = VINF_EM_RESET; /* Triple fault equals a reset. */
3587 break;
3588
3589 case VMX_EXIT_INIT_SIGNAL: /* 3 INIT signal. */
3590 case VMX_EXIT_SIPI: /* 4 Start-up IPI (SIPI). */
3591 rc = VINF_EM_RAW_INTERRUPT;
3592 AssertFailed(); /* Can't happen. Yet. */
3593 break;
3594
3595 case VMX_EXIT_IO_SMI_IRQ: /* 5 I/O system-management interrupt (SMI). */
3596 case VMX_EXIT_SMI_IRQ: /* 6 Other SMI. */
3597 rc = VINF_EM_RAW_INTERRUPT;
3598 AssertFailed(); /* Can't happen afaik. */
3599 break;
3600
3601 case VMX_EXIT_TASK_SWITCH: /* 9 Task switch. */
3602 rc = VERR_EM_INTERPRETER;
3603 break;
3604
3605 case VMX_EXIT_HLT: /* 12 Guest software attempted to execute HLT. */
3606 /** Check if external interrupts are pending; if so, don't switch back. */
3607 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitHlt);
3608 pCtx->rip++; /* skip hlt */
3609 if ( pCtx->eflags.Bits.u1IF
3610 && VMCPU_FF_ISPENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC|VMCPU_FF_INTERRUPT_PIC)))
3611 goto ResumeExecution;
3612
3613 rc = VINF_EM_HALT;
3614 break;
3615
3616 case VMX_EXIT_MWAIT: /* 36 Guest software executed MWAIT. */
3617 Log2(("VMX: mwait\n"));
3618 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitMwait);
3619 rc = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pCtx));
3620 if ( rc == VINF_EM_HALT
3621 || rc == VINF_SUCCESS)
3622 {
3623 /* Update EIP and continue execution. */
3624 pCtx->rip += cbInstr;
3625
3626 /** Check if external interrupts are pending; if so, don't switch back. */
3627 if ( rc == VINF_SUCCESS
3628 || ( rc == VINF_EM_HALT
3629 && pCtx->eflags.Bits.u1IF
3630 && VMCPU_FF_ISPENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC|VMCPU_FF_INTERRUPT_PIC)))
3631 )
3632 goto ResumeExecution;
3633 }
3634 AssertMsg(rc == VERR_EM_INTERPRETER || rc == VINF_EM_HALT, ("EMU: mwait failed with %Rrc\n", rc));
3635 break;
3636
3637 case VMX_EXIT_RSM: /* 17 Guest software attempted to execute RSM in SMM. */
3638 AssertFailed(); /* can't happen. */
3639 rc = VERR_EM_INTERPRETER;
3640 break;
3641
3642 case VMX_EXIT_VMCALL: /* 18 Guest software executed VMCALL. */
3643 case VMX_EXIT_VMCLEAR: /* 19 Guest software executed VMCLEAR. */
3644 case VMX_EXIT_VMLAUNCH: /* 20 Guest software executed VMLAUNCH. */
3645 case VMX_EXIT_VMPTRLD: /* 21 Guest software executed VMPTRLD. */
3646 case VMX_EXIT_VMPTRST: /* 22 Guest software executed VMPTRST. */
3647 case VMX_EXIT_VMREAD: /* 23 Guest software executed VMREAD. */
3648 case VMX_EXIT_VMRESUME: /* 24 Guest software executed VMRESUME. */
3649 case VMX_EXIT_VMWRITE: /* 25 Guest software executed VMWRITE. */
3650 case VMX_EXIT_VMXOFF: /* 26 Guest software executed VMXOFF. */
3651 case VMX_EXIT_VMXON: /* 27 Guest software executed VMXON. */
3652 /** @todo inject #UD immediately */
3653 rc = VERR_EM_INTERPRETER;
3654 break;
3655
3656 case VMX_EXIT_CPUID: /* 10 Guest software attempted to execute CPUID. */
3657 case VMX_EXIT_RDTSC: /* 16 Guest software attempted to execute RDTSC. */
3658 case VMX_EXIT_INVPG: /* 14 Guest software attempted to execute INVPG. */
3659 case VMX_EXIT_CRX_MOVE: /* 28 Control-register accesses. */
3660 case VMX_EXIT_DRX_MOVE: /* 29 Debug-register accesses. */
3661 case VMX_EXIT_PORT_IO: /* 30 I/O instruction. */
3662 case VMX_EXIT_RDPMC: /* 15 Guest software attempted to execute RDPMC. */
3663 /* already handled above */
3664 AssertMsg( rc == VINF_PGM_CHANGE_MODE
3665 || rc == VINF_EM_RAW_INTERRUPT
3666 || rc == VERR_EM_INTERPRETER
3667 || rc == VINF_EM_RAW_EMULATE_INSTR
3668 || rc == VINF_PGM_SYNC_CR3
3669 || rc == VINF_IOM_HC_IOPORT_READ
3670 || rc == VINF_IOM_HC_IOPORT_WRITE
3671 || rc == VINF_EM_RAW_GUEST_TRAP
3672 || rc == VINF_TRPM_XCPT_DISPATCHED
3673 || rc == VINF_EM_RESCHEDULE_REM,
3674 ("rc = %d\n", rc));
3675 break;
3676
3677 case VMX_EXIT_TPR: /* 43 TPR below threshold. Guest software executed MOV to CR8. */
3678 case VMX_EXIT_APIC_ACCESS: /* 44 APIC access. Guest software attempted to access memory at a physical address on the APIC-access page. */
3679 case VMX_EXIT_RDMSR: /* 31 RDMSR. Guest software attempted to execute RDMSR. */
3680 case VMX_EXIT_WRMSR: /* 32 WRMSR. Guest software attempted to execute WRMSR. */
3681 case VMX_EXIT_MONITOR: /* 39 Guest software attempted to execute MONITOR. */
3682 case VMX_EXIT_PAUSE: /* 40 Guest software attempted to execute PAUSE. */
3683 /* Note: If we decide to emulate them here, then we must sync the MSRs that could have been changed (sysenter, fs/gs base)!!! */
3684 rc = VERR_EM_INTERPRETER;
3685 break;
3686
3687 case VMX_EXIT_IRQ_WINDOW: /* 7 Interrupt window. */
3688 Assert(rc == VINF_EM_RAW_INTERRUPT);
3689 break;
3690
3691 case VMX_EXIT_ERR_INVALID_GUEST_STATE: /* 33 VM-entry failure due to invalid guest state. */
3692 {
3693#ifdef VBOX_STRICT
3694 RTCCUINTREG val = 0;
3695
3696 Log(("VMX_EXIT_ERR_INVALID_GUEST_STATE\n"));
3697
3698 VMXReadVMCS(VMX_VMCS64_GUEST_RIP, &val);
3699 Log(("Old eip %RGv new %RGv\n", (RTGCPTR)pCtx->rip, (RTGCPTR)val));
3700
3701 VMXReadVMCS(VMX_VMCS64_GUEST_CR0, &val);
3702 Log(("VMX_VMCS_GUEST_CR0 %RX64\n", (uint64_t)val));
3703
3704 VMXReadVMCS(VMX_VMCS64_GUEST_CR3, &val);
3705 Log(("VMX_VMCS_GUEST_CR3 %RX64\n", (uint64_t)val));
3706
3707 VMXReadVMCS(VMX_VMCS64_GUEST_CR4, &val);
3708 Log(("VMX_VMCS_GUEST_CR4 %RX64\n", (uint64_t)val));
3709
3710 VMXReadVMCS(VMX_VMCS_GUEST_RFLAGS, &val);
3711 Log(("VMX_VMCS_GUEST_RFLAGS %08x\n", val));
3712
3713 VMX_LOG_SELREG(CS, "CS");
3714 VMX_LOG_SELREG(DS, "DS");
3715 VMX_LOG_SELREG(ES, "ES");
3716 VMX_LOG_SELREG(FS, "FS");
3717 VMX_LOG_SELREG(GS, "GS");
3718 VMX_LOG_SELREG(SS, "SS");
3719 VMX_LOG_SELREG(TR, "TR");
3720 VMX_LOG_SELREG(LDTR, "LDTR");
3721
3722 VMXReadVMCS(VMX_VMCS64_GUEST_GDTR_BASE, &val);
3723 Log(("VMX_VMCS_GUEST_GDTR_BASE %RX64\n", (uint64_t)val));
3724 VMXReadVMCS(VMX_VMCS64_GUEST_IDTR_BASE, &val);
3725 Log(("VMX_VMCS_GUEST_IDTR_BASE %RX64\n", (uint64_t)val));
3726#endif /* VBOX_STRICT */
3727 rc = VERR_VMX_INVALID_GUEST_STATE;
3728 break;
3729 }
3730
3731 case VMX_EXIT_ERR_MSR_LOAD: /* 34 VM-entry failure due to MSR loading. */
3732 case VMX_EXIT_ERR_MACHINE_CHECK: /* 41 VM-entry failure due to machine-check. */
3733 default:
3734 rc = VERR_VMX_UNEXPECTED_EXIT_CODE;
3735 AssertMsgFailed(("Unexpected exit code %d\n", exitReason)); /* Can't happen. */
3736 break;
3737
3738 }
3739end:
3740
3741 /* Signal changes for the recompiler. */
3742 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR | CPUM_CHANGED_LDTR | CPUM_CHANGED_GDTR | CPUM_CHANGED_IDTR | CPUM_CHANGED_TR | CPUM_CHANGED_HIDDEN_SEL_REGS);
3743
3744 /* If we executed vmlaunch/vmresume and an external irq was pending, then we don't have to do a full sync the next time. */
3745 if ( exitReason == VMX_EXIT_EXTERNAL_IRQ
3746 && !VMX_EXIT_INTERRUPTION_INFO_VALID(intInfo))
3747 {
3748 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatPendingHostIrq);
3749 /* On the next entry we'll only sync the host context. */
3750 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_HOST_CONTEXT;
3751 }
3752 else
3753 {
3754 /* On the next entry we'll sync everything. */
3755 /** @todo we can do better than this */
3756 /* Not in the VINF_PGM_CHANGE_MODE though! */
3757 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_ALL;
3758 }
3759
3760 /* translate into a less severe return code */
3761 if (rc == VERR_EM_INTERPRETER)
3762 rc = VINF_EM_RAW_EMULATE_INSTR;
3763 else
3764 /* Try to extract more information about what might have gone wrong here. */
3765 if (rc == VERR_VMX_INVALID_VMCS_PTR)
3766 {
3767 VMXGetActivateVMCS(&pVCpu->hwaccm.s.vmx.lasterror.u64VMCSPhys);
3768 pVCpu->hwaccm.s.vmx.lasterror.ulVMCSRevision = *(uint32_t *)pVCpu->hwaccm.s.vmx.pVMCS;
3769 pVCpu->hwaccm.s.vmx.lasterror.idEnteredCpu = pVCpu->hwaccm.s.idEnteredCpu;
3770 pVCpu->hwaccm.s.vmx.lasterror.idCurrentCpu = RTMpCpuId();
3771 }
3772
3773 /* Just set the correct state here instead of trying to catch every goto above. */
3774 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED, VMCPUSTATE_STARTED_EXEC);
3775
3776#ifdef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
3777 /* Restore interrupts if we exitted after disabling them. */
3778 if (uOldEFlags != ~(RTCCUINTREG)0)
3779 ASMSetFlags(uOldEFlags);
3780#endif
3781
3782 STAM_STATS({
3783 if (fStatExit2Started) STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2, y);
3784 else if (fStatEntryStarted) STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatEntry, x);
3785 });
3786 Log2(("X"));
3787 return rc;
3788}
3789
3790
3791/**
3792 * Enters the VT-x session
3793 *
3794 * @returns VBox status code.
3795 * @param pVM The VM to operate on.
3796 * @param pVCpu The VMCPU to operate on.
3797 * @param pCpu CPU info struct
3798 */
3799VMMR0DECL(int) VMXR0Enter(PVM pVM, PVMCPU pVCpu, PHWACCM_CPUINFO pCpu)
3800{
3801 Assert(pVM->hwaccm.s.vmx.fSupported);
3802
3803 unsigned cr4 = ASMGetCR4();
3804 if (!(cr4 & X86_CR4_VMXE))
3805 {
3806 AssertMsgFailed(("X86_CR4_VMXE should be set!\n"));
3807 return VERR_VMX_X86_CR4_VMXE_CLEARED;
3808 }
3809
3810 /* Activate the VM Control Structure. */
3811 int rc = VMXActivateVMCS(pVCpu->hwaccm.s.vmx.pVMCSPhys);
3812 if (RT_FAILURE(rc))
3813 return rc;
3814
3815 pVCpu->hwaccm.s.fResumeVM = false;
3816 return VINF_SUCCESS;
3817}
3818
3819
3820/**
3821 * Leaves the VT-x session
3822 *
3823 * @returns VBox status code.
3824 * @param pVM The VM to operate on.
3825 * @param pVCpu The VMCPU to operate on.
3826 * @param pCtx CPU context
3827 */
3828VMMR0DECL(int) VMXR0Leave(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
3829{
3830 Assert(pVM->hwaccm.s.vmx.fSupported);
3831
3832#ifdef DEBUG
3833 if (CPUMIsHyperDebugStateActive(pVCpu))
3834 {
3835 CPUMR0LoadHostDebugState(pVM, pVCpu);
3836 }
3837 else
3838#endif
3839 /* Save the guest debug state if necessary. */
3840 if (CPUMIsGuestDebugStateActive(pVCpu))
3841 {
3842 CPUMR0SaveGuestDebugState(pVM, pVCpu, pCtx, true /* save DR6 */);
3843
3844 /* Enable drx move intercepts again. */
3845 pVCpu->hwaccm.s.vmx.proc_ctls |= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT;
3846 int rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
3847 AssertRC(rc);
3848
3849 /* Resync the debug registers the next time. */
3850 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_GUEST_DEBUG;
3851 }
3852 else
3853 Assert(pVCpu->hwaccm.s.vmx.proc_ctls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT);
3854
3855 /* Clear VM Control Structure. Marking it inactive, clearing implementation specific data and writing back VMCS data to memory. */
3856 int rc = VMXClearVMCS(pVCpu->hwaccm.s.vmx.pVMCSPhys);
3857 AssertRC(rc);
3858
3859 return VINF_SUCCESS;
3860}
3861
3862/**
3863 * Flush the TLB (EPT)
3864 *
3865 * @returns VBox status code.
3866 * @param pVM The VM to operate on.
3867 * @param pVCpu The VM CPU to operate on.
3868 * @param enmFlush Type of flush
3869 * @param GCPhys Physical address of the page to flush
3870 */
3871static void vmxR0FlushEPT(PVM pVM, PVMCPU pVCpu, VMX_FLUSH enmFlush, RTGCPHYS GCPhys)
3872{
3873 uint64_t descriptor[2];
3874
3875 LogFlow(("vmxR0FlushEPT %d %RGv\n", enmFlush, GCPhys));
3876 Assert(pVM->hwaccm.s.fNestedPaging);
3877 descriptor[0] = pVCpu->hwaccm.s.vmx.GCPhysEPTP;
3878 descriptor[1] = GCPhys;
3879 int rc = VMXR0InvEPT(enmFlush, &descriptor[0]);
3880 AssertRC(rc);
3881}
3882
3883#ifdef HWACCM_VTX_WITH_VPID
3884/**
3885 * Flush the TLB (EPT)
3886 *
3887 * @returns VBox status code.
3888 * @param pVM The VM to operate on.
3889 * @param pVCpu The VM CPU to operate on.
3890 * @param enmFlush Type of flush
3891 * @param GCPtr Virtual address of the page to flush
3892 */
3893static void vmxR0FlushVPID(PVM pVM, PVMCPU pVCpu, VMX_FLUSH enmFlush, RTGCPTR GCPtr)
3894{
3895#if HC_ARCH_BITS == 32
3896 /* If we get a flush in 64 bits guest mode, then force a full TLB flush. Invvpid probably takes only 32 bits addresses. (@todo) */
3897 if ( CPUMIsGuestInLongMode(pVCpu)
3898 && !VMX_IS_64BIT_HOST_MODE())
3899 {
3900 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
3901 }
3902 else
3903#endif
3904 {
3905 uint64_t descriptor[2];
3906
3907 Assert(pVM->hwaccm.s.vmx.fVPID);
3908 descriptor[0] = pVCpu->hwaccm.s.uCurrentASID;
3909 descriptor[1] = GCPtr;
3910 int rc = VMXR0InvVPID(enmFlush, &descriptor[0]);
3911 AssertRC(rc);
3912 }
3913}
3914#endif /* HWACCM_VTX_WITH_VPID */
3915
3916/**
3917 * Invalidates a guest page
3918 *
3919 * @returns VBox status code.
3920 * @param pVM The VM to operate on.
3921 * @param pVCpu The VM CPU to operate on.
3922 * @param GCVirt Page to invalidate
3923 */
3924VMMR0DECL(int) VMXR0InvalidatePage(PVM pVM, PVMCPU pVCpu, RTGCPTR GCVirt)
3925{
3926 bool fFlushPending = VMCPU_FF_ISSET(pVCpu, VMCPU_FF_TLB_FLUSH);
3927
3928 Log2(("VMXR0InvalidatePage %RGv\n", GCVirt));
3929
3930 /* Only relevant if we want to use VPID.
3931 * In the nested paging case we still see such calls, but
3932 * can safely ignore them. (e.g. after cr3 updates)
3933 */
3934#ifdef HWACCM_VTX_WITH_VPID
3935 /* Skip it if a TLB flush is already pending. */
3936 if ( !fFlushPending
3937 && pVM->hwaccm.s.vmx.fVPID)
3938 vmxR0FlushVPID(pVM, pVCpu, pVM->hwaccm.s.vmx.enmFlushPage, GCVirt);
3939#endif /* HWACCM_VTX_WITH_VPID */
3940
3941 return VINF_SUCCESS;
3942}
3943
3944/**
3945 * Invalidates a guest page by physical address
3946 *
3947 * NOTE: Assumes the current instruction references this physical page though a virtual address!!
3948 *
3949 * @returns VBox status code.
3950 * @param pVM The VM to operate on.
3951 * @param pVCpu The VM CPU to operate on.
3952 * @param GCPhys Page to invalidate
3953 */
3954VMMR0DECL(int) VMXR0InvalidatePhysPage(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys)
3955{
3956 bool fFlushPending = VMCPU_FF_ISSET(pVCpu, VMCPU_FF_TLB_FLUSH);
3957
3958 Assert(pVM->hwaccm.s.fNestedPaging);
3959
3960 LogFlow(("VMXR0InvalidatePhysPage %RGp\n", GCPhys));
3961
3962 /* Skip it if a TLB flush is already pending. */
3963 if (!fFlushPending)
3964 vmxR0FlushEPT(pVM, pVCpu, pVM->hwaccm.s.vmx.enmFlushPage, GCPhys);
3965
3966 return VINF_SUCCESS;
3967}
3968
3969/**
3970 * Report world switch error and dump some useful debug info
3971 *
3972 * @param pVM The VM to operate on.
3973 * @param pVCpu The VMCPU to operate on.
3974 * @param rc Return code
3975 * @param pCtx Current CPU context (not updated)
3976 */
3977static void VMXR0ReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, int rc, PCPUMCTX pCtx)
3978{
3979 switch (rc)
3980 {
3981 case VERR_VMX_INVALID_VMXON_PTR:
3982 AssertFailed();
3983 break;
3984
3985 case VERR_VMX_UNABLE_TO_START_VM:
3986 case VERR_VMX_UNABLE_TO_RESUME_VM:
3987 {
3988 int rc;
3989 RTCCUINTREG exitReason, instrError;
3990
3991 rc = VMXReadVMCS(VMX_VMCS32_RO_EXIT_REASON, &exitReason);
3992 rc |= VMXReadVMCS(VMX_VMCS32_RO_VM_INSTR_ERROR, &instrError);
3993 AssertRC(rc);
3994 if (rc == VINF_SUCCESS)
3995 {
3996 Log(("Unable to start/resume VM for reason: %x. Instruction error %x\n", (uint32_t)exitReason, (uint32_t)instrError));
3997 Log(("Current stack %08x\n", &rc));
3998
3999 pVCpu->hwaccm.s.vmx.lasterror.ulInstrError = instrError;
4000 pVCpu->hwaccm.s.vmx.lasterror.ulExitReason = exitReason;
4001
4002#ifdef VBOX_STRICT
4003 RTGDTR gdtr;
4004 PX86DESCHC pDesc;
4005 RTCCUINTREG val;
4006
4007 ASMGetGDTR(&gdtr);
4008
4009 VMXReadVMCS(VMX_VMCS64_GUEST_RIP, &val);
4010 Log(("Old eip %RGv new %RGv\n", (RTGCPTR)pCtx->rip, (RTGCPTR)val));
4011 VMXReadVMCS(VMX_VMCS_CTRL_PIN_EXEC_CONTROLS, &val);
4012 Log(("VMX_VMCS_CTRL_PIN_EXEC_CONTROLS %08x\n", val));
4013 VMXReadVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, &val);
4014 Log(("VMX_VMCS_CTRL_PROC_EXEC_CONTROLS %08x\n", val));
4015 VMXReadVMCS(VMX_VMCS_CTRL_ENTRY_CONTROLS, &val);
4016 Log(("VMX_VMCS_CTRL_ENTRY_CONTROLS %08x\n", val));
4017 VMXReadVMCS(VMX_VMCS_CTRL_EXIT_CONTROLS, &val);
4018 Log(("VMX_VMCS_CTRL_EXIT_CONTROLS %08x\n", val));
4019
4020 VMXReadVMCS(VMX_VMCS_HOST_CR0, &val);
4021 Log(("VMX_VMCS_HOST_CR0 %08x\n", val));
4022
4023 VMXReadVMCS(VMX_VMCS_HOST_CR3, &val);
4024 Log(("VMX_VMCS_HOST_CR3 %08x\n", val));
4025
4026 VMXReadVMCS(VMX_VMCS_HOST_CR4, &val);
4027 Log(("VMX_VMCS_HOST_CR4 %08x\n", val));
4028
4029 VMXReadVMCS(VMX_VMCS16_HOST_FIELD_CS, &val);
4030 Log(("VMX_VMCS_HOST_FIELD_CS %08x\n", val));
4031
4032 VMXReadVMCS(VMX_VMCS_GUEST_RFLAGS, &val);
4033 Log(("VMX_VMCS_GUEST_RFLAGS %08x\n", val));
4034
4035 if (val < gdtr.cbGdt)
4036 {
4037 pDesc = &((PX86DESCHC)gdtr.pGdt)[val >> X86_SEL_SHIFT_HC];
4038 HWACCMR0DumpDescriptor(pDesc, val, "CS: ");
4039 }
4040
4041 VMXReadVMCS(VMX_VMCS16_HOST_FIELD_DS, &val);
4042 Log(("VMX_VMCS_HOST_FIELD_DS %08x\n", val));
4043 if (val < gdtr.cbGdt)
4044 {
4045 pDesc = &((PX86DESCHC)gdtr.pGdt)[val >> X86_SEL_SHIFT_HC];
4046 HWACCMR0DumpDescriptor(pDesc, val, "DS: ");
4047 }
4048
4049 VMXReadVMCS(VMX_VMCS16_HOST_FIELD_ES, &val);
4050 Log(("VMX_VMCS_HOST_FIELD_ES %08x\n", val));
4051 if (val < gdtr.cbGdt)
4052 {
4053 pDesc = &((PX86DESCHC)gdtr.pGdt)[val >> X86_SEL_SHIFT_HC];
4054 HWACCMR0DumpDescriptor(pDesc, val, "ES: ");
4055 }
4056
4057 VMXReadVMCS(VMX_VMCS16_HOST_FIELD_FS, &val);
4058 Log(("VMX_VMCS16_HOST_FIELD_FS %08x\n", val));
4059 if (val < gdtr.cbGdt)
4060 {
4061 pDesc = &((PX86DESCHC)gdtr.pGdt)[val >> X86_SEL_SHIFT_HC];
4062 HWACCMR0DumpDescriptor(pDesc, val, "FS: ");
4063 }
4064
4065 VMXReadVMCS(VMX_VMCS16_HOST_FIELD_GS, &val);
4066 Log(("VMX_VMCS16_HOST_FIELD_GS %08x\n", val));
4067 if (val < gdtr.cbGdt)
4068 {
4069 pDesc = &((PX86DESCHC)gdtr.pGdt)[val >> X86_SEL_SHIFT_HC];
4070 HWACCMR0DumpDescriptor(pDesc, val, "GS: ");
4071 }
4072
4073 VMXReadVMCS(VMX_VMCS16_HOST_FIELD_SS, &val);
4074 Log(("VMX_VMCS16_HOST_FIELD_SS %08x\n", val));
4075 if (val < gdtr.cbGdt)
4076 {
4077 pDesc = &((PX86DESCHC)gdtr.pGdt)[val >> X86_SEL_SHIFT_HC];
4078 HWACCMR0DumpDescriptor(pDesc, val, "SS: ");
4079 }
4080
4081 VMXReadVMCS(VMX_VMCS16_HOST_FIELD_TR, &val);
4082 Log(("VMX_VMCS16_HOST_FIELD_TR %08x\n", val));
4083 if (val < gdtr.cbGdt)
4084 {
4085 pDesc = &((PX86DESCHC)gdtr.pGdt)[val >> X86_SEL_SHIFT_HC];
4086 HWACCMR0DumpDescriptor(pDesc, val, "TR: ");
4087 }
4088
4089 VMXReadVMCS(VMX_VMCS_HOST_TR_BASE, &val);
4090 Log(("VMX_VMCS_HOST_TR_BASE %RHv\n", val));
4091
4092 VMXReadVMCS(VMX_VMCS_HOST_GDTR_BASE, &val);
4093 Log(("VMX_VMCS_HOST_GDTR_BASE %RHv\n", val));
4094 VMXReadVMCS(VMX_VMCS_HOST_IDTR_BASE, &val);
4095 Log(("VMX_VMCS_HOST_IDTR_BASE %RHv\n", val));
4096
4097 VMXReadVMCS(VMX_VMCS32_HOST_SYSENTER_CS, &val);
4098 Log(("VMX_VMCS_HOST_SYSENTER_CS %08x\n", val));
4099
4100 VMXReadVMCS(VMX_VMCS_HOST_SYSENTER_EIP, &val);
4101 Log(("VMX_VMCS_HOST_SYSENTER_EIP %RHv\n", val));
4102
4103 VMXReadVMCS(VMX_VMCS_HOST_SYSENTER_ESP, &val);
4104 Log(("VMX_VMCS_HOST_SYSENTER_ESP %RHv\n", val));
4105
4106 VMXReadVMCS(VMX_VMCS_HOST_RSP, &val);
4107 Log(("VMX_VMCS_HOST_RSP %RHv\n", val));
4108 VMXReadVMCS(VMX_VMCS_HOST_RIP, &val);
4109 Log(("VMX_VMCS_HOST_RIP %RHv\n", val));
4110
4111# if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4112 if (VMX_IS_64BIT_HOST_MODE())
4113 {
4114 Log(("MSR_K6_EFER = %RX64\n", ASMRdMsr(MSR_K6_EFER)));
4115 Log(("MSR_K6_STAR = %RX64\n", ASMRdMsr(MSR_K6_STAR)));
4116 Log(("MSR_K8_LSTAR = %RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
4117 Log(("MSR_K8_CSTAR = %RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
4118 Log(("MSR_K8_SF_MASK = %RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
4119 }
4120# endif
4121#endif /* VBOX_STRICT */
4122 }
4123 break;
4124 }
4125
4126 default:
4127 /* impossible */
4128 AssertMsgFailed(("%Rrc (%#x)\n", rc, rc));
4129 break;
4130 }
4131}
4132
4133#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4134/**
4135 * Prepares for and executes VMLAUNCH (64 bits guest mode)
4136 *
4137 * @returns VBox status code
4138 * @param fResume vmlauch/vmresume
4139 * @param pCtx Guest context
4140 * @param pCache VMCS cache
4141 * @param pVM The VM to operate on.
4142 * @param pVCpu The VMCPU to operate on.
4143 */
4144DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
4145{
4146 uint32_t aParam[6];
4147 PHWACCM_CPUINFO pCpu;
4148 RTHCPHYS pPageCpuPhys;
4149 int rc;
4150
4151 pCpu = HWACCMR0GetCurrentCpu();
4152 pPageCpuPhys = RTR0MemObjGetPagePhysAddr(pCpu->pMemObj, 0);
4153
4154#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4155 pCache->uPos = 1;
4156 pCache->interPD = PGMGetInterPaeCR3(pVM);
4157 pCache->pSwitcher = (uint64_t)pVM->hwaccm.s.pfnHost32ToGuest64R0;
4158#endif
4159
4160#ifdef DEBUG
4161 pCache->TestIn.pPageCpuPhys = 0;
4162 pCache->TestIn.pVMCSPhys = 0;
4163 pCache->TestIn.pCache = 0;
4164 pCache->TestOut.pVMCSPhys = 0;
4165 pCache->TestOut.pCache = 0;
4166 pCache->TestOut.pCtx = 0;
4167 pCache->TestOut.eflags = 0;
4168#endif
4169
4170 aParam[0] = (uint32_t)(pPageCpuPhys); /* Param 1: VMXON physical address - Lo. */
4171 aParam[1] = (uint32_t)(pPageCpuPhys >> 32); /* Param 1: VMXON physical address - Hi. */
4172 aParam[2] = (uint32_t)(pVCpu->hwaccm.s.vmx.pVMCSPhys); /* Param 2: VMCS physical address - Lo. */
4173 aParam[3] = (uint32_t)(pVCpu->hwaccm.s.vmx.pVMCSPhys >> 32); /* Param 2: VMCS physical address - Hi. */
4174 aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hwaccm.s.vmx.VMCSCache);
4175 aParam[5] = 0;
4176
4177#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4178 pCtx->dr[4] = pVM->hwaccm.s.vmx.pScratchPhys + 16 + 8;
4179 *(uint32_t *)(pVM->hwaccm.s.vmx.pScratch + 16 + 8) = 1;
4180#endif
4181 rc = VMXR0Execute64BitsHandler(pVM, pVCpu, pCtx, pVM->hwaccm.s.pfnVMXGCStartVM64, 6, &aParam[0]);
4182
4183#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4184 Assert(*(uint32_t *)(pVM->hwaccm.s.vmx.pScratch + 16 + 8) == 5);
4185 Assert(pCtx->dr[4] == 10);
4186 *(uint32_t *)(pVM->hwaccm.s.vmx.pScratch + 16 + 8) = 0xff;
4187#endif
4188
4189#ifdef DEBUG
4190 AssertMsg(pCache->TestIn.pPageCpuPhys == pPageCpuPhys, ("%RHp vs %RHp\n", pCache->TestIn.pPageCpuPhys, pPageCpuPhys));
4191 AssertMsg(pCache->TestIn.pVMCSPhys == pVCpu->hwaccm.s.vmx.pVMCSPhys, ("%RHp vs %RHp\n", pCache->TestIn.pVMCSPhys, pVCpu->hwaccm.s.vmx.pVMCSPhys));
4192 AssertMsg(pCache->TestIn.pVMCSPhys == pCache->TestOut.pVMCSPhys, ("%RHp vs %RHp\n", pCache->TestIn.pVMCSPhys, pCache->TestOut.pVMCSPhys));
4193 AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache, pCache->TestOut.pCache));
4194 AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hwaccm.s.vmx.VMCSCache), ("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hwaccm.s.vmx.VMCSCache)));
4195 AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx, pCache->TestOut.pCtx));
4196 Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
4197#endif
4198 return rc;
4199}
4200
4201/**
4202 * Executes the specified handler in 64 mode
4203 *
4204 * @returns VBox status code.
4205 * @param pVM The VM to operate on.
4206 * @param pVCpu The VMCPU to operate on.
4207 * @param pCtx Guest context
4208 * @param pfnHandler RC handler
4209 * @param cbParam Number of parameters
4210 * @param paParam Array of 32 bits parameters
4211 */
4212VMMR0DECL(int) VMXR0Execute64BitsHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, RTRCPTR pfnHandler, uint32_t cbParam, uint32_t *paParam)
4213{
4214 int rc, rc2;
4215 PHWACCM_CPUINFO pCpu;
4216 RTHCPHYS pPageCpuPhys;
4217
4218 /* @todo This code is not guest SMP safe (hyper stack and switchers) */
4219 AssertReturn(pVM->cCPUs == 1, VERR_TOO_MANY_CPUS);
4220 AssertReturn(pVM->hwaccm.s.pfnHost32ToGuest64R0, VERR_INTERNAL_ERROR);
4221 Assert(pVCpu->hwaccm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hwaccm.s.vmx.VMCSCache.Write.aField));
4222 Assert(pVCpu->hwaccm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hwaccm.s.vmx.VMCSCache.Read.aField));
4223
4224#ifdef VBOX_STRICT
4225 for (unsigned i=0;i<pVCpu->hwaccm.s.vmx.VMCSCache.Write.cValidEntries;i++)
4226 Assert(vmxR0IsValidWriteField(pVCpu->hwaccm.s.vmx.VMCSCache.Write.aField[i]));
4227
4228 for (unsigned i=0;i<pVCpu->hwaccm.s.vmx.VMCSCache.Read.cValidEntries;i++)
4229 Assert(vmxR0IsValidReadField(pVCpu->hwaccm.s.vmx.VMCSCache.Read.aField[i]));
4230#endif
4231
4232 pCpu = HWACCMR0GetCurrentCpu();
4233 pPageCpuPhys = RTR0MemObjGetPagePhysAddr(pCpu->pMemObj, 0);
4234
4235 /* Clear VM Control Structure. Marking it inactive, clearing implementation specific data and writing back VMCS data to memory. */
4236 VMXClearVMCS(pVCpu->hwaccm.s.vmx.pVMCSPhys);
4237
4238 /* Leave VMX Root Mode. */
4239 VMXDisable();
4240
4241 ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
4242
4243 CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVM));
4244 CPUMSetHyperEIP(pVCpu, pfnHandler);
4245 for (int i=(int)cbParam-1;i>=0;i--)
4246 CPUMPushHyper(pVCpu, paParam[i]);
4247
4248 STAM_PROFILE_ADV_START(&pVCpu->hwaccm.s.StatWorldSwitch3264, z);
4249 /* Call switcher. */
4250 rc = pVM->hwaccm.s.pfnHost32ToGuest64R0(pVM);
4251 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatWorldSwitch3264, z);
4252
4253 /* Make sure the VMX instructions don't cause #UD faults. */
4254 ASMSetCR4(ASMGetCR4() | X86_CR4_VMXE);
4255
4256 /* Enter VMX Root Mode */
4257 rc2 = VMXEnable(pPageCpuPhys);
4258 if (RT_FAILURE(rc2))
4259 {
4260 if (pVM)
4261 VMXR0CheckError(pVM, pVCpu, rc2);
4262 ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
4263 return VERR_VMX_VMXON_FAILED;
4264 }
4265
4266 rc2 = VMXActivateVMCS(pVCpu->hwaccm.s.vmx.pVMCSPhys);
4267 AssertRCReturn(rc2, rc2);
4268 Assert(!(ASMGetFlags() & X86_EFL_IF));
4269 return rc;
4270}
4271
4272#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL) */
4273
4274
4275#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
4276/**
4277 * Executes VMWRITE
4278 *
4279 * @returns VBox status code
4280 * @param pVCpu The VMCPU to operate on.
4281 * @param idxField VMCS index
4282 * @param u64Val 16, 32 or 64 bits value
4283 */
4284VMMR0DECL(int) VMXWriteVMCS64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
4285{
4286 int rc;
4287
4288 switch (idxField)
4289 {
4290 case VMX_VMCS_CTRL_TSC_OFFSET_FULL:
4291 case VMX_VMCS_CTRL_IO_BITMAP_A_FULL:
4292 case VMX_VMCS_CTRL_IO_BITMAP_B_FULL:
4293 case VMX_VMCS_CTRL_MSR_BITMAP_FULL:
4294 case VMX_VMCS_CTRL_VMEXIT_MSR_STORE_FULL:
4295 case VMX_VMCS_CTRL_VMEXIT_MSR_LOAD_FULL:
4296 case VMX_VMCS_CTRL_VMENTRY_MSR_LOAD_FULL:
4297 case VMX_VMCS_CTRL_VAPIC_PAGEADDR_FULL:
4298 case VMX_VMCS_CTRL_APIC_ACCESSADDR_FULL:
4299 case VMX_VMCS_GUEST_LINK_PTR_FULL:
4300 case VMX_VMCS_GUEST_PDPTR0_FULL:
4301 case VMX_VMCS_GUEST_PDPTR1_FULL:
4302 case VMX_VMCS_GUEST_PDPTR2_FULL:
4303 case VMX_VMCS_GUEST_PDPTR3_FULL:
4304 case VMX_VMCS_GUEST_DEBUGCTL_FULL:
4305 case VMX_VMCS_GUEST_EFER_FULL:
4306 case VMX_VMCS_CTRL_EPTP_FULL:
4307 /* These fields consist of two parts, which are both writable in 32 bits mode. */
4308 rc = VMXWriteVMCS32(idxField, u64Val);
4309 rc |= VMXWriteVMCS32(idxField + 1, (uint32_t)(u64Val >> 32ULL));
4310 AssertRC(rc);
4311 return rc;
4312
4313 case VMX_VMCS64_GUEST_LDTR_BASE:
4314 case VMX_VMCS64_GUEST_TR_BASE:
4315 case VMX_VMCS64_GUEST_GDTR_BASE:
4316 case VMX_VMCS64_GUEST_IDTR_BASE:
4317 case VMX_VMCS64_GUEST_SYSENTER_EIP:
4318 case VMX_VMCS64_GUEST_SYSENTER_ESP:
4319 case VMX_VMCS64_GUEST_CR0:
4320 case VMX_VMCS64_GUEST_CR4:
4321 case VMX_VMCS64_GUEST_CR3:
4322 case VMX_VMCS64_GUEST_DR7:
4323 case VMX_VMCS64_GUEST_RIP:
4324 case VMX_VMCS64_GUEST_RSP:
4325 case VMX_VMCS64_GUEST_CS_BASE:
4326 case VMX_VMCS64_GUEST_DS_BASE:
4327 case VMX_VMCS64_GUEST_ES_BASE:
4328 case VMX_VMCS64_GUEST_FS_BASE:
4329 case VMX_VMCS64_GUEST_GS_BASE:
4330 case VMX_VMCS64_GUEST_SS_BASE:
4331 /* Queue a 64 bits value as we can't set it in 32 bits host mode. */
4332 if (u64Val >> 32ULL)
4333 rc = VMXWriteCachedVMCSEx(pVCpu, idxField, u64Val);
4334 else
4335 rc = VMXWriteVMCS32(idxField, (uint32_t)u64Val);
4336
4337 return rc;
4338
4339 default:
4340 AssertMsgFailed(("Unexpected field %x\n", idxField));
4341 return VERR_INVALID_PARAMETER;
4342 }
4343}
4344
4345/**
4346 * Cache VMCS writes for performance reasons (Darwin) and for running 64 bits guests on 32 bits hosts.
4347 *
4348 * @param pVCpu The VMCPU to operate on.
4349 * @param idxField VMCS field
4350 * @param u64Val Value
4351 */
4352VMMR0DECL(int) VMXWriteCachedVMCSEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
4353{
4354 PVMCSCACHE pCache = &pVCpu->hwaccm.s.vmx.VMCSCache;
4355
4356 AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1, ("entries=%x\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
4357
4358 /* Make sure there are no duplicates. */
4359 for (unsigned i=0;i<pCache->Write.cValidEntries;i++)
4360 {
4361 if (pCache->Write.aField[i] == idxField)
4362 {
4363 pCache->Write.aFieldVal[i] = u64Val;
4364 return VINF_SUCCESS;
4365 }
4366 }
4367
4368 pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
4369 pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
4370 pCache->Write.cValidEntries++;
4371 return VINF_SUCCESS;
4372}
4373
4374#endif /* HC_ARCH_BITS == 32 && !VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0 */
4375
4376#ifdef VBOX_STRICT
4377static bool vmxR0IsValidReadField(uint32_t idxField)
4378{
4379 switch(idxField)
4380 {
4381 case VMX_VMCS64_GUEST_RIP:
4382 case VMX_VMCS64_GUEST_RSP:
4383 case VMX_VMCS_GUEST_RFLAGS:
4384 case VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE:
4385 case VMX_VMCS_CTRL_CR0_READ_SHADOW:
4386 case VMX_VMCS64_GUEST_CR0:
4387 case VMX_VMCS_CTRL_CR4_READ_SHADOW:
4388 case VMX_VMCS64_GUEST_CR4:
4389 case VMX_VMCS64_GUEST_DR7:
4390 case VMX_VMCS32_GUEST_SYSENTER_CS:
4391 case VMX_VMCS64_GUEST_SYSENTER_EIP:
4392 case VMX_VMCS64_GUEST_SYSENTER_ESP:
4393 case VMX_VMCS32_GUEST_GDTR_LIMIT:
4394 case VMX_VMCS64_GUEST_GDTR_BASE:
4395 case VMX_VMCS32_GUEST_IDTR_LIMIT:
4396 case VMX_VMCS64_GUEST_IDTR_BASE:
4397 case VMX_VMCS16_GUEST_FIELD_CS:
4398 case VMX_VMCS32_GUEST_CS_LIMIT:
4399 case VMX_VMCS64_GUEST_CS_BASE:
4400 case VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS:
4401 case VMX_VMCS16_GUEST_FIELD_DS:
4402 case VMX_VMCS32_GUEST_DS_LIMIT:
4403 case VMX_VMCS64_GUEST_DS_BASE:
4404 case VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS:
4405 case VMX_VMCS16_GUEST_FIELD_ES:
4406 case VMX_VMCS32_GUEST_ES_LIMIT:
4407 case VMX_VMCS64_GUEST_ES_BASE:
4408 case VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS:
4409 case VMX_VMCS16_GUEST_FIELD_FS:
4410 case VMX_VMCS32_GUEST_FS_LIMIT:
4411 case VMX_VMCS64_GUEST_FS_BASE:
4412 case VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS:
4413 case VMX_VMCS16_GUEST_FIELD_GS:
4414 case VMX_VMCS32_GUEST_GS_LIMIT:
4415 case VMX_VMCS64_GUEST_GS_BASE:
4416 case VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS:
4417 case VMX_VMCS16_GUEST_FIELD_SS:
4418 case VMX_VMCS32_GUEST_SS_LIMIT:
4419 case VMX_VMCS64_GUEST_SS_BASE:
4420 case VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS:
4421 case VMX_VMCS16_GUEST_FIELD_LDTR:
4422 case VMX_VMCS32_GUEST_LDTR_LIMIT:
4423 case VMX_VMCS64_GUEST_LDTR_BASE:
4424 case VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS:
4425 case VMX_VMCS16_GUEST_FIELD_TR:
4426 case VMX_VMCS32_GUEST_TR_LIMIT:
4427 case VMX_VMCS64_GUEST_TR_BASE:
4428 case VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS:
4429 case VMX_VMCS32_RO_EXIT_REASON:
4430 case VMX_VMCS32_RO_VM_INSTR_ERROR:
4431 case VMX_VMCS32_RO_EXIT_INSTR_LENGTH:
4432 case VMX_VMCS32_RO_EXIT_INTERRUPTION_ERRCODE:
4433 case VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO:
4434 case VMX_VMCS32_RO_EXIT_INSTR_INFO:
4435 case VMX_VMCS_RO_EXIT_QUALIFICATION:
4436 case VMX_VMCS32_RO_IDT_INFO:
4437 case VMX_VMCS32_RO_IDT_ERRCODE:
4438 case VMX_VMCS64_GUEST_CR3:
4439 case VMX_VMCS_EXIT_PHYS_ADDR_FULL:
4440 return true;
4441 }
4442 return false;
4443}
4444
4445static bool vmxR0IsValidWriteField(uint32_t idxField)
4446{
4447 switch(idxField)
4448 {
4449 case VMX_VMCS64_GUEST_LDTR_BASE:
4450 case VMX_VMCS64_GUEST_TR_BASE:
4451 case VMX_VMCS64_GUEST_GDTR_BASE:
4452 case VMX_VMCS64_GUEST_IDTR_BASE:
4453 case VMX_VMCS64_GUEST_SYSENTER_EIP:
4454 case VMX_VMCS64_GUEST_SYSENTER_ESP:
4455 case VMX_VMCS64_GUEST_CR0:
4456 case VMX_VMCS64_GUEST_CR4:
4457 case VMX_VMCS64_GUEST_CR3:
4458 case VMX_VMCS64_GUEST_DR7:
4459 case VMX_VMCS64_GUEST_RIP:
4460 case VMX_VMCS64_GUEST_RSP:
4461 case VMX_VMCS64_GUEST_CS_BASE:
4462 case VMX_VMCS64_GUEST_DS_BASE:
4463 case VMX_VMCS64_GUEST_ES_BASE:
4464 case VMX_VMCS64_GUEST_FS_BASE:
4465 case VMX_VMCS64_GUEST_GS_BASE:
4466 case VMX_VMCS64_GUEST_SS_BASE:
4467 return true;
4468 }
4469 return false;
4470}
4471
4472#endif
Note: See TracBrowser for help on using the repository browser.

© 2024 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette