VirtualBox

source: vbox/trunk/src/VBox/VMM/VMMR0/HWVMXR0.cpp@ 27561

Last change on this file since 27561 was 27445, checked in by vboxsync, 15 years ago

Logging update
  • Property svn:eol-style set to native
  • Property svn:keywords set to Id
File size: 194.8 KB
Line 
1/* $Id: HWVMXR0.cpp 27445 2010-03-17 12:56:35Z vboxsync $ */
2/** @file
3 * HWACCM VMX - Host Context Ring 0.
4 */
5
6/*
7 * Copyright (C) 2006-2007 Sun Microsystems, Inc.
8 *
9 * This file is part of VirtualBox Open Source Edition (OSE), as
10 * available from http://www.virtualbox.org. This file is free software;
11 * you can redistribute it and/or modify it under the terms of the GNU
12 * General Public License (GPL) as published by the Free Software
13 * Foundation, in version 2 as it comes in the "COPYING" file of the
14 * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15 * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16 *
17 * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa
18 * Clara, CA 95054 USA or visit http://www.sun.com if you need
19 * additional information or have any questions.
20 */
21
22
23/*******************************************************************************
24* Header Files *
25*******************************************************************************/
26#define LOG_GROUP LOG_GROUP_HWACCM
27#include <VBox/hwaccm.h>
28#include <VBox/pgm.h>
29#include <VBox/dbgf.h>
30#include <VBox/selm.h>
31#include <VBox/iom.h>
32#include <VBox/rem.h>
33#include <VBox/tm.h>
34#include "HWACCMInternal.h"
35#include <VBox/vm.h>
36#include <VBox/x86.h>
37#include <VBox/pdmapi.h>
38#include <VBox/err.h>
39#include <VBox/log.h>
40#include <iprt/asm.h>
41#include <iprt/assert.h>
42#include <iprt/param.h>
43#include <iprt/string.h>
44#include <iprt/time.h>
45#ifdef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
46# include <iprt/thread.h>
47#endif
48#include "HWVMXR0.h"
49
50/*******************************************************************************
51* Defined Constants And Macros *
52*******************************************************************************/
53#if defined(RT_ARCH_AMD64)
54# define VMX_IS_64BIT_HOST_MODE() (true)
55#elif defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
56# define VMX_IS_64BIT_HOST_MODE() (g_fVMXIs64bitHost != 0)
57#else
58# define VMX_IS_64BIT_HOST_MODE() (false)
59#endif
60
61/*******************************************************************************
62* Global Variables *
63*******************************************************************************/
64/* IO operation lookup arrays. */
65static uint32_t const g_aIOSize[4] = {1, 2, 0, 4};
66static uint32_t const g_aIOOpAnd[4] = {0xff, 0xffff, 0, 0xffffffff};
67
68#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
69/** See HWACCMR0A.asm. */
70extern "C" uint32_t g_fVMXIs64bitHost;
71#endif
72
73/*******************************************************************************
74* Local Functions *
75*******************************************************************************/
76static void VMXR0ReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, int rc, PCPUMCTX pCtx);
77static void vmxR0SetupTLBEPT(PVM pVM, PVMCPU pVCpu);
78static void vmxR0SetupTLBVPID(PVM pVM, PVMCPU pVCpu);
79static void vmxR0SetupTLBDummy(PVM pVM, PVMCPU pVCpu);
80static void vmxR0FlushEPT(PVM pVM, PVMCPU pVCpu, VMX_FLUSH enmFlush, RTGCPHYS GCPhys);
81static void vmxR0FlushVPID(PVM pVM, PVMCPU pVCpu, VMX_FLUSH enmFlush, RTGCPTR GCPtr);
82static void vmxR0UpdateExceptionBitmap(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx);
83#ifdef VBOX_STRICT
84static bool vmxR0IsValidReadField(uint32_t idxField);
85static bool vmxR0IsValidWriteField(uint32_t idxField);
86#endif
87static void vmxR0SetMSRPermission(PVMCPU pVCpu, unsigned ulMSR, bool fRead, bool fWrite);
88
89static void VMXR0CheckError(PVM pVM, PVMCPU pVCpu, int rc)
90{
91 if (rc == VERR_VMX_GENERIC)
92 {
93 RTCCUINTREG instrError;
94
95 VMXReadVMCS(VMX_VMCS32_RO_VM_INSTR_ERROR, &instrError);
96 pVCpu->hwaccm.s.vmx.lasterror.ulInstrError = instrError;
97 }
98 pVM->hwaccm.s.lLastError = rc;
99}
100
101/**
102 * Sets up and activates VT-x on the current CPU
103 *
104 * @returns VBox status code.
105 * @param pCpu CPU info struct
106 * @param pVM The VM to operate on. (can be NULL after a resume!!)
107 * @param pvPageCpu Pointer to the global cpu page
108 * @param pPageCpuPhys Physical address of the global cpu page
109 */
110VMMR0DECL(int) VMXR0EnableCpu(PHWACCM_CPUINFO pCpu, PVM pVM, void *pvPageCpu, RTHCPHYS pPageCpuPhys)
111{
112 AssertReturn(pPageCpuPhys, VERR_INVALID_PARAMETER);
113 AssertReturn(pvPageCpu, VERR_INVALID_PARAMETER);
114
115 if (pVM)
116 {
117 /* Set revision dword at the beginning of the VMXON structure. */
118 *(uint32_t *)pvPageCpu = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hwaccm.s.vmx.msr.vmx_basic_info);
119 }
120
121 /** @todo we should unmap the two pages from the virtual address space in order to prevent accidental corruption.
122 * (which can have very bad consequences!!!)
123 */
124
125 if (ASMGetCR4() & X86_CR4_VMXE)
126 return VERR_VMX_IN_VMX_ROOT_MODE;
127
128 /* Make sure the VMX instructions don't cause #UD faults. */
129 ASMSetCR4(ASMGetCR4() | X86_CR4_VMXE);
130
131 /* Enter VMX Root Mode */
132 int rc = VMXEnable(pPageCpuPhys);
133 if (RT_FAILURE(rc))
134 {
135 if (pVM)
136 VMXR0CheckError(pVM, &pVM->aCpus[0], rc);
137 ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
138 return VERR_VMX_VMXON_FAILED;
139 }
140 return VINF_SUCCESS;
141}
142
143/**
144 * Deactivates VT-x on the current CPU
145 *
146 * @returns VBox status code.
147 * @param pCpu CPU info struct
148 * @param pvPageCpu Pointer to the global cpu page
149 * @param pPageCpuPhys Physical address of the global cpu page
150 */
151VMMR0DECL(int) VMXR0DisableCpu(PHWACCM_CPUINFO pCpu, void *pvPageCpu, RTHCPHYS pPageCpuPhys)
152{
153 AssertReturn(pPageCpuPhys, VERR_INVALID_PARAMETER);
154 AssertReturn(pvPageCpu, VERR_INVALID_PARAMETER);
155
156 /* Leave VMX Root Mode. */
157 VMXDisable();
158
159 /* And clear the X86_CR4_VMXE bit */
160 ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
161 return VINF_SUCCESS;
162}
163
164/**
165 * Does Ring-0 per VM VT-x init.
166 *
167 * @returns VBox status code.
168 * @param pVM The VM to operate on.
169 */
170VMMR0DECL(int) VMXR0InitVM(PVM pVM)
171{
172 int rc;
173
174#ifdef LOG_ENABLED
175 SUPR0Printf("VMXR0InitVM %x\n", pVM);
176#endif
177
178 pVM->hwaccm.s.vmx.pMemObjAPIC = NIL_RTR0MEMOBJ;
179
180 if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
181 {
182 /* Allocate one page for the APIC physical page (serves for filtering accesses). */
183 rc = RTR0MemObjAllocCont(&pVM->hwaccm.s.vmx.pMemObjAPIC, 1 << PAGE_SHIFT, true /* executable R0 mapping */);
184 AssertRC(rc);
185 if (RT_FAILURE(rc))
186 return rc;
187
188 pVM->hwaccm.s.vmx.pAPIC = (uint8_t *)RTR0MemObjAddress(pVM->hwaccm.s.vmx.pMemObjAPIC);
189 pVM->hwaccm.s.vmx.pAPICPhys = RTR0MemObjGetPagePhysAddr(pVM->hwaccm.s.vmx.pMemObjAPIC, 0);
190 ASMMemZero32(pVM->hwaccm.s.vmx.pAPIC, PAGE_SIZE);
191 }
192 else
193 {
194 pVM->hwaccm.s.vmx.pMemObjAPIC = 0;
195 pVM->hwaccm.s.vmx.pAPIC = 0;
196 pVM->hwaccm.s.vmx.pAPICPhys = 0;
197 }
198
199#ifdef VBOX_WITH_CRASHDUMP_MAGIC
200 {
201 rc = RTR0MemObjAllocCont(&pVM->hwaccm.s.vmx.pMemObjScratch, 1 << PAGE_SHIFT, true /* executable R0 mapping */);
202 AssertRC(rc);
203 if (RT_FAILURE(rc))
204 return rc;
205
206 pVM->hwaccm.s.vmx.pScratch = (uint8_t *)RTR0MemObjAddress(pVM->hwaccm.s.vmx.pMemObjScratch);
207 pVM->hwaccm.s.vmx.pScratchPhys = RTR0MemObjGetPagePhysAddr(pVM->hwaccm.s.vmx.pMemObjScratch, 0);
208
209 ASMMemZero32(pVM->hwaccm.s.vmx.pScratch, PAGE_SIZE);
210 strcpy((char *)pVM->hwaccm.s.vmx.pScratch, "SCRATCH Magic");
211 *(uint64_t *)(pVM->hwaccm.s.vmx.pScratch + 16) = UINT64_C(0xDEADBEEFDEADBEEF);
212 }
213#endif
214
215 /* Allocate VMCBs for all guest CPUs. */
216 for (VMCPUID i = 0; i < pVM->cCpus; i++)
217 {
218 PVMCPU pVCpu = &pVM->aCpus[i];
219
220 pVCpu->hwaccm.s.vmx.pMemObjVMCS = NIL_RTR0MEMOBJ;
221
222 /* Allocate one page for the VM control structure (VMCS). */
223 rc = RTR0MemObjAllocCont(&pVCpu->hwaccm.s.vmx.pMemObjVMCS, 1 << PAGE_SHIFT, true /* executable R0 mapping */);
224 AssertRC(rc);
225 if (RT_FAILURE(rc))
226 return rc;
227
228 pVCpu->hwaccm.s.vmx.pVMCS = RTR0MemObjAddress(pVCpu->hwaccm.s.vmx.pMemObjVMCS);
229 pVCpu->hwaccm.s.vmx.pVMCSPhys = RTR0MemObjGetPagePhysAddr(pVCpu->hwaccm.s.vmx.pMemObjVMCS, 0);
230 ASMMemZero32(pVCpu->hwaccm.s.vmx.pVMCS, PAGE_SIZE);
231
232 pVCpu->hwaccm.s.vmx.cr0_mask = 0;
233 pVCpu->hwaccm.s.vmx.cr4_mask = 0;
234
235 /* Allocate one page for the virtual APIC page for TPR caching. */
236 rc = RTR0MemObjAllocCont(&pVCpu->hwaccm.s.vmx.pMemObjVAPIC, 1 << PAGE_SHIFT, true /* executable R0 mapping */);
237 AssertRC(rc);
238 if (RT_FAILURE(rc))
239 return rc;
240
241 pVCpu->hwaccm.s.vmx.pVAPIC = (uint8_t *)RTR0MemObjAddress(pVCpu->hwaccm.s.vmx.pMemObjVAPIC);
242 pVCpu->hwaccm.s.vmx.pVAPICPhys = RTR0MemObjGetPagePhysAddr(pVCpu->hwaccm.s.vmx.pMemObjVAPIC, 0);
243 ASMMemZero32(pVCpu->hwaccm.s.vmx.pVAPIC, PAGE_SIZE);
244
245 /* Allocate the MSR bitmap if this feature is supported. */
246 if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
247 {
248 rc = RTR0MemObjAllocCont(&pVCpu->hwaccm.s.vmx.pMemObjMSRBitmap, 1 << PAGE_SHIFT, true /* executable R0 mapping */);
249 AssertRC(rc);
250 if (RT_FAILURE(rc))
251 return rc;
252
253 pVCpu->hwaccm.s.vmx.pMSRBitmap = (uint8_t *)RTR0MemObjAddress(pVCpu->hwaccm.s.vmx.pMemObjMSRBitmap);
254 pVCpu->hwaccm.s.vmx.pMSRBitmapPhys = RTR0MemObjGetPagePhysAddr(pVCpu->hwaccm.s.vmx.pMemObjMSRBitmap, 0);
255 memset(pVCpu->hwaccm.s.vmx.pMSRBitmap, 0xff, PAGE_SIZE);
256 }
257
258#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
259 /* Allocate one page for the guest MSR load area (for preloading guest MSRs during the world switch). */
260 rc = RTR0MemObjAllocCont(&pVCpu->hwaccm.s.vmx.pMemObjGuestMSR, 1 << PAGE_SHIFT, true /* executable R0 mapping */);
261 AssertRC(rc);
262 if (RT_FAILURE(rc))
263 return rc;
264
265 pVCpu->hwaccm.s.vmx.pGuestMSR = (uint8_t *)RTR0MemObjAddress(pVCpu->hwaccm.s.vmx.pMemObjGuestMSR);
266 pVCpu->hwaccm.s.vmx.pGuestMSRPhys = RTR0MemObjGetPagePhysAddr(pVCpu->hwaccm.s.vmx.pMemObjGuestMSR, 0);
267 memset(pVCpu->hwaccm.s.vmx.pGuestMSR, 0, PAGE_SIZE);
268
269 /* Allocate one page for the host MSR load area (for restoring host MSRs after the world switch back). */
270 rc = RTR0MemObjAllocCont(&pVCpu->hwaccm.s.vmx.pMemObjHostMSR, 1 << PAGE_SHIFT, true /* executable R0 mapping */);
271 AssertRC(rc);
272 if (RT_FAILURE(rc))
273 return rc;
274
275 pVCpu->hwaccm.s.vmx.pHostMSR = (uint8_t *)RTR0MemObjAddress(pVCpu->hwaccm.s.vmx.pMemObjHostMSR);
276 pVCpu->hwaccm.s.vmx.pHostMSRPhys = RTR0MemObjGetPagePhysAddr(pVCpu->hwaccm.s.vmx.pMemObjHostMSR, 0);
277 memset(pVCpu->hwaccm.s.vmx.pHostMSR, 0, PAGE_SIZE);
278#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
279
280 /* Current guest paging mode. */
281 pVCpu->hwaccm.s.vmx.enmLastSeenGuestMode = PGMMODE_REAL;
282
283#ifdef LOG_ENABLED
284 SUPR0Printf("VMXR0InitVM %x VMCS=%x (%x)\n", pVM, pVCpu->hwaccm.s.vmx.pVMCS, (uint32_t)pVCpu->hwaccm.s.vmx.pVMCSPhys);
285#endif
286 }
287
288 return VINF_SUCCESS;
289}
290
291/**
292 * Does Ring-0 per VM VT-x termination.
293 *
294 * @returns VBox status code.
295 * @param pVM The VM to operate on.
296 */
297VMMR0DECL(int) VMXR0TermVM(PVM pVM)
298{
299 for (VMCPUID i = 0; i < pVM->cCpus; i++)
300 {
301 PVMCPU pVCpu = &pVM->aCpus[i];
302
303 if (pVCpu->hwaccm.s.vmx.pMemObjVMCS != NIL_RTR0MEMOBJ)
304 {
305 RTR0MemObjFree(pVCpu->hwaccm.s.vmx.pMemObjVMCS, false);
306 pVCpu->hwaccm.s.vmx.pMemObjVMCS = NIL_RTR0MEMOBJ;
307 pVCpu->hwaccm.s.vmx.pVMCS = 0;
308 pVCpu->hwaccm.s.vmx.pVMCSPhys = 0;
309 }
310 if (pVCpu->hwaccm.s.vmx.pMemObjVAPIC != NIL_RTR0MEMOBJ)
311 {
312 RTR0MemObjFree(pVCpu->hwaccm.s.vmx.pMemObjVAPIC, false);
313 pVCpu->hwaccm.s.vmx.pMemObjVAPIC = NIL_RTR0MEMOBJ;
314 pVCpu->hwaccm.s.vmx.pVAPIC = 0;
315 pVCpu->hwaccm.s.vmx.pVAPICPhys = 0;
316 }
317 if (pVCpu->hwaccm.s.vmx.pMemObjMSRBitmap != NIL_RTR0MEMOBJ)
318 {
319 RTR0MemObjFree(pVCpu->hwaccm.s.vmx.pMemObjMSRBitmap, false);
320 pVCpu->hwaccm.s.vmx.pMemObjMSRBitmap = NIL_RTR0MEMOBJ;
321 pVCpu->hwaccm.s.vmx.pMSRBitmap = 0;
322 pVCpu->hwaccm.s.vmx.pMSRBitmapPhys = 0;
323 }
324#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
325 if (pVCpu->hwaccm.s.vmx.pMemObjHostMSR != NIL_RTR0MEMOBJ)
326 {
327 RTR0MemObjFree(pVCpu->hwaccm.s.vmx.pMemObjHostMSR, false);
328 pVCpu->hwaccm.s.vmx.pMemObjHostMSR = NIL_RTR0MEMOBJ;
329 pVCpu->hwaccm.s.vmx.pHostMSR = 0;
330 pVCpu->hwaccm.s.vmx.pHostMSRPhys = 0;
331 }
332 if (pVCpu->hwaccm.s.vmx.pMemObjGuestMSR != NIL_RTR0MEMOBJ)
333 {
334 RTR0MemObjFree(pVCpu->hwaccm.s.vmx.pMemObjGuestMSR, false);
335 pVCpu->hwaccm.s.vmx.pMemObjGuestMSR = NIL_RTR0MEMOBJ;
336 pVCpu->hwaccm.s.vmx.pGuestMSR = 0;
337 pVCpu->hwaccm.s.vmx.pGuestMSRPhys = 0;
338 }
339#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
340 }
341 if (pVM->hwaccm.s.vmx.pMemObjAPIC != NIL_RTR0MEMOBJ)
342 {
343 RTR0MemObjFree(pVM->hwaccm.s.vmx.pMemObjAPIC, false);
344 pVM->hwaccm.s.vmx.pMemObjAPIC = NIL_RTR0MEMOBJ;
345 pVM->hwaccm.s.vmx.pAPIC = 0;
346 pVM->hwaccm.s.vmx.pAPICPhys = 0;
347 }
348#ifdef VBOX_WITH_CRASHDUMP_MAGIC
349 if (pVM->hwaccm.s.vmx.pMemObjScratch != NIL_RTR0MEMOBJ)
350 {
351 ASMMemZero32(pVM->hwaccm.s.vmx.pScratch, PAGE_SIZE);
352 RTR0MemObjFree(pVM->hwaccm.s.vmx.pMemObjScratch, false);
353 pVM->hwaccm.s.vmx.pMemObjScratch = NIL_RTR0MEMOBJ;
354 pVM->hwaccm.s.vmx.pScratch = 0;
355 pVM->hwaccm.s.vmx.pScratchPhys = 0;
356 }
357#endif
358 return VINF_SUCCESS;
359}
360
361/**
362 * Sets up VT-x for the specified VM
363 *
364 * @returns VBox status code.
365 * @param pVM The VM to operate on.
366 */
367VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
368{
369 int rc = VINF_SUCCESS;
370 uint32_t val;
371
372 AssertReturn(pVM, VERR_INVALID_PARAMETER);
373
374 for (VMCPUID i = 0; i < pVM->cCpus; i++)
375 {
376 PVMCPU pVCpu = &pVM->aCpus[i];
377
378 Assert(pVCpu->hwaccm.s.vmx.pVMCS);
379
380 /* Set revision dword at the beginning of the VMCS structure. */
381 *(uint32_t *)pVCpu->hwaccm.s.vmx.pVMCS = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hwaccm.s.vmx.msr.vmx_basic_info);
382
383 /* Clear VM Control Structure. */
384 Log(("pVMCSPhys = %RHp\n", pVCpu->hwaccm.s.vmx.pVMCSPhys));
385 rc = VMXClearVMCS(pVCpu->hwaccm.s.vmx.pVMCSPhys);
386 if (RT_FAILURE(rc))
387 goto vmx_end;
388
389 /* Activate the VM Control Structure. */
390 rc = VMXActivateVMCS(pVCpu->hwaccm.s.vmx.pVMCSPhys);
391 if (RT_FAILURE(rc))
392 goto vmx_end;
393
394 /* VMX_VMCS_CTRL_PIN_EXEC_CONTROLS
395 * Set required bits to one and zero according to the MSR capabilities.
396 */
397 val = pVM->hwaccm.s.vmx.msr.vmx_pin_ctls.n.disallowed0;
398 /* External and non-maskable interrupts cause VM-exits. */
399 val = val | VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_EXT_INT_EXIT | VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_NMI_EXIT;
400 val &= pVM->hwaccm.s.vmx.msr.vmx_pin_ctls.n.allowed1;
401
402 rc = VMXWriteVMCS(VMX_VMCS_CTRL_PIN_EXEC_CONTROLS, val);
403 AssertRC(rc);
404
405 /* VMX_VMCS_CTRL_PROC_EXEC_CONTROLS
406 * Set required bits to one and zero according to the MSR capabilities.
407 */
408 val = pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.disallowed0;
409 /* Program which event cause VM-exits and which features we want to use. */
410 val = val | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_HLT_EXIT
411 | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_TSC_OFFSET
412 | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT
413 | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_UNCOND_IO_EXIT
414 | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDPMC_EXIT
415 | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_EXIT
416 | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MWAIT_EXIT; /* don't execute mwait or else we'll idle inside the guest (host thinks the cpu load is high) */
417
418 /* Without nested paging we should intercept invlpg and cr3 mov instructions. */
419 if (!pVM->hwaccm.s.fNestedPaging)
420 val |= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INVLPG_EXIT
421 | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
422 | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT;
423
424 /* Note: VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MWAIT_EXIT might cause a vmlaunch failure with an invalid control fields error. (combined with some other exit reasons) */
425 if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
426 {
427 /* CR8 reads from the APIC shadow page; writes cause an exit is they lower the TPR below the threshold */
428 val |= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW;
429 Assert(pVM->hwaccm.s.vmx.pAPIC);
430 }
431 else
432 /* Exit on CR8 reads & writes in case the TPR shadow feature isn't present. */
433 val |= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR8_STORE_EXIT | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR8_LOAD_EXIT;
434
435 if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
436 {
437 Assert(pVCpu->hwaccm.s.vmx.pMSRBitmapPhys);
438 val |= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS;
439 }
440
441 /* We will use the secondary control if it's present. */
442 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
443
444 /* Mask away the bits that the CPU doesn't support */
445 /** @todo make sure they don't conflict with the above requirements. */
446 val &= pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1;
447 pVCpu->hwaccm.s.vmx.proc_ctls = val;
448
449 rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, val);
450 AssertRC(rc);
451
452 if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
453 {
454 /* VMX_VMCS_CTRL_PROC_EXEC_CONTROLS2
455 * Set required bits to one and zero according to the MSR capabilities.
456 */
457 val = pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2.n.disallowed0;
458 val |= VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT;
459
460#ifdef HWACCM_VTX_WITH_EPT
461 if (pVM->hwaccm.s.fNestedPaging)
462 val |= VMX_VMCS_CTRL_PROC_EXEC2_EPT;
463#endif /* HWACCM_VTX_WITH_EPT */
464#ifdef HWACCM_VTX_WITH_VPID
465 else
466 if (pVM->hwaccm.s.vmx.fVPID)
467 val |= VMX_VMCS_CTRL_PROC_EXEC2_VPID;
468#endif /* HWACCM_VTX_WITH_VPID */
469
470 if (pVM->hwaccm.s.fHasIoApic)
471 val |= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC;
472
473 if (pVM->hwaccm.s.vmx.fUnrestrictedGuest)
474 val |= VMX_VMCS_CTRL_PROC_EXEC2_REAL_MODE;
475
476 /* Mask away the bits that the CPU doesn't support */
477 /** @todo make sure they don't conflict with the above requirements. */
478 val &= pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2.n.allowed1;
479 pVCpu->hwaccm.s.vmx.proc_ctls2 = val;
480 rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS2, val);
481 AssertRC(rc);
482 }
483
484 /* VMX_VMCS_CTRL_CR3_TARGET_COUNT
485 * Set required bits to one and zero according to the MSR capabilities.
486 */
487 rc = VMXWriteVMCS(VMX_VMCS_CTRL_CR3_TARGET_COUNT, 0);
488 AssertRC(rc);
489
490 /* Forward all exception except #NM & #PF to the guest.
491 * We always need to check pagefaults since our shadow page table can be out of sync.
492 * And we always lazily sync the FPU & XMM state.
493 */
494
495 /** @todo Possible optimization:
496 * Keep the FPU and XMM state current in the EM thread. That way there's no need to
497 * lazily sync anything, but the downside is that we can't use the FPU stack or XMM
498 * registers ourselves of course.
499 *
500 * Note: only possible if the current state is actually ours (X86_CR0_TS flag)
501 */
502
503 /* Don't filter page faults; all of them should cause a switch. */
504 rc = VMXWriteVMCS(VMX_VMCS_CTRL_PAGEFAULT_ERROR_MASK, 0);
505 rc |= VMXWriteVMCS(VMX_VMCS_CTRL_PAGEFAULT_ERROR_MATCH, 0);
506 AssertRC(rc);
507
508 /* Init TSC offset to zero. */
509 rc = VMXWriteVMCS64(VMX_VMCS_CTRL_TSC_OFFSET_FULL, 0);
510 AssertRC(rc);
511
512 rc = VMXWriteVMCS64(VMX_VMCS_CTRL_IO_BITMAP_A_FULL, 0);
513 AssertRC(rc);
514
515 rc = VMXWriteVMCS64(VMX_VMCS_CTRL_IO_BITMAP_B_FULL, 0);
516 AssertRC(rc);
517
518 /* Set the MSR bitmap address. */
519 if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
520 {
521 Assert(pVCpu->hwaccm.s.vmx.pMSRBitmapPhys);
522
523 rc = VMXWriteVMCS64(VMX_VMCS_CTRL_MSR_BITMAP_FULL, pVCpu->hwaccm.s.vmx.pMSRBitmapPhys);
524 AssertRC(rc);
525
526 /* Allow the guest to directly modify these MSRs; they are restored and saved automatically. */
527 vmxR0SetMSRPermission(pVCpu, MSR_IA32_SYSENTER_CS, true, true);
528 vmxR0SetMSRPermission(pVCpu, MSR_IA32_SYSENTER_ESP, true, true);
529 vmxR0SetMSRPermission(pVCpu, MSR_IA32_SYSENTER_EIP, true, true);
530 vmxR0SetMSRPermission(pVCpu, MSR_K8_LSTAR, true, true);
531 vmxR0SetMSRPermission(pVCpu, MSR_K6_STAR, true, true);
532 vmxR0SetMSRPermission(pVCpu, MSR_K8_SF_MASK, true, true);
533 vmxR0SetMSRPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, true, true);
534 vmxR0SetMSRPermission(pVCpu, MSR_K8_GS_BASE, true, true);
535 vmxR0SetMSRPermission(pVCpu, MSR_K8_FS_BASE, true, true);
536 }
537
538#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
539 /* Set the guest & host MSR load/store physical addresses. */
540 Assert(pVCpu->hwaccm.s.vmx.pGuestMSRPhys);
541 rc = VMXWriteVMCS64(VMX_VMCS_CTRL_VMENTRY_MSR_LOAD_FULL, pVCpu->hwaccm.s.vmx.pGuestMSRPhys);
542 AssertRC(rc);
543 rc = VMXWriteVMCS64(VMX_VMCS_CTRL_VMEXIT_MSR_STORE_FULL, pVCpu->hwaccm.s.vmx.pGuestMSRPhys);
544 AssertRC(rc);
545
546 Assert(pVCpu->hwaccm.s.vmx.pHostMSRPhys);
547 rc = VMXWriteVMCS64(VMX_VMCS_CTRL_VMEXIT_MSR_LOAD_FULL, pVCpu->hwaccm.s.vmx.pHostMSRPhys);
548 AssertRC(rc);
549#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
550
551 rc = VMXWriteVMCS(VMX_VMCS_CTRL_ENTRY_MSR_LOAD_COUNT, 0);
552 AssertRC(rc);
553
554 rc = VMXWriteVMCS(VMX_VMCS_CTRL_EXIT_MSR_STORE_COUNT, 0);
555 AssertRC(rc);
556
557 if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
558 {
559 Assert(pVM->hwaccm.s.vmx.pMemObjAPIC);
560 /* Optional */
561 rc = VMXWriteVMCS(VMX_VMCS_CTRL_TPR_THRESHOLD, 0);
562 rc |= VMXWriteVMCS64(VMX_VMCS_CTRL_VAPIC_PAGEADDR_FULL, pVCpu->hwaccm.s.vmx.pVAPICPhys);
563
564 if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
565 rc |= VMXWriteVMCS64(VMX_VMCS_CTRL_APIC_ACCESSADDR_FULL, pVM->hwaccm.s.vmx.pAPICPhys);
566
567 AssertRC(rc);
568 }
569
570 /* Set link pointer to -1. Not currently used. */
571 rc = VMXWriteVMCS64(VMX_VMCS_GUEST_LINK_PTR_FULL, 0xFFFFFFFFFFFFFFFFULL);
572 AssertRC(rc);
573
574 /* Clear VM Control Structure. Marking it inactive, clearing implementation specific data and writing back VMCS data to memory. */
575 rc = VMXClearVMCS(pVCpu->hwaccm.s.vmx.pVMCSPhys);
576 AssertRC(rc);
577
578 /* Configure the VMCS read cache. */
579 PVMCSCACHE pCache = &pVCpu->hwaccm.s.vmx.VMCSCache;
580
581 VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_RIP);
582 VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_RSP);
583 VMXSetupCachedReadVMCS(pCache, VMX_VMCS_GUEST_RFLAGS);
584 VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE);
585 VMXSetupCachedReadVMCS(pCache, VMX_VMCS_CTRL_CR0_READ_SHADOW);
586 VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_CR0);
587 VMXSetupCachedReadVMCS(pCache, VMX_VMCS_CTRL_CR4_READ_SHADOW);
588 VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_CR4);
589 VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_DR7);
590 VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_GUEST_SYSENTER_CS);
591 VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_SYSENTER_EIP);
592 VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_SYSENTER_ESP);
593 VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_GUEST_GDTR_LIMIT);
594 VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_GDTR_BASE);
595 VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_GUEST_IDTR_LIMIT);
596 VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_IDTR_BASE);
597
598 VMX_SETUP_SELREG(ES, pCache);
599 VMX_SETUP_SELREG(SS, pCache);
600 VMX_SETUP_SELREG(CS, pCache);
601 VMX_SETUP_SELREG(DS, pCache);
602 VMX_SETUP_SELREG(FS, pCache);
603 VMX_SETUP_SELREG(GS, pCache);
604 VMX_SETUP_SELREG(LDTR, pCache);
605 VMX_SETUP_SELREG(TR, pCache);
606
607 /* Status code VMCS reads. */
608 VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_EXIT_REASON);
609 VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_VM_INSTR_ERROR);
610 VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_EXIT_INSTR_LENGTH);
611 VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_EXIT_INTERRUPTION_ERRCODE);
612 VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO);
613 VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_EXIT_INSTR_INFO);
614 VMXSetupCachedReadVMCS(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
615 VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_IDT_INFO);
616 VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_IDT_ERRCODE);
617
618 if (pVM->hwaccm.s.fNestedPaging)
619 {
620 VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_CR3);
621 VMXSetupCachedReadVMCS(pCache, VMX_VMCS_EXIT_PHYS_ADDR_FULL);
622 pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
623 }
624 else
625 pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
626 } /* for each VMCPU */
627
628 /* Choose the right TLB setup function. */
629 if (pVM->hwaccm.s.fNestedPaging)
630 {
631 pVM->hwaccm.s.vmx.pfnSetupTaggedTLB = vmxR0SetupTLBEPT;
632
633 /* Default values for flushing. */
634 pVM->hwaccm.s.vmx.enmFlushPage = VMX_FLUSH_ALL_CONTEXTS;
635 pVM->hwaccm.s.vmx.enmFlushContext = VMX_FLUSH_ALL_CONTEXTS;
636
637 /* If the capabilities specify we can do more, then make use of it. */
638 if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVEPT_CAPS_INDIV)
639 pVM->hwaccm.s.vmx.enmFlushPage = VMX_FLUSH_PAGE;
640 else
641 if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVEPT_CAPS_CONTEXT)
642 pVM->hwaccm.s.vmx.enmFlushPage = VMX_FLUSH_SINGLE_CONTEXT;
643
644 if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVEPT_CAPS_CONTEXT)
645 pVM->hwaccm.s.vmx.enmFlushContext = VMX_FLUSH_SINGLE_CONTEXT;
646 }
647#ifdef HWACCM_VTX_WITH_VPID
648 else
649 if (pVM->hwaccm.s.vmx.fVPID)
650 {
651 pVM->hwaccm.s.vmx.pfnSetupTaggedTLB = vmxR0SetupTLBVPID;
652
653 /* Default values for flushing. */
654 pVM->hwaccm.s.vmx.enmFlushPage = VMX_FLUSH_ALL_CONTEXTS;
655 pVM->hwaccm.s.vmx.enmFlushContext = VMX_FLUSH_ALL_CONTEXTS;
656
657 /* If the capabilities specify we can do more, then make use of it. */
658 if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVVPID_CAPS_INDIV)
659 pVM->hwaccm.s.vmx.enmFlushPage = VMX_FLUSH_PAGE;
660 else
661 if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVVPID_CAPS_CONTEXT)
662 pVM->hwaccm.s.vmx.enmFlushPage = VMX_FLUSH_SINGLE_CONTEXT;
663
664 if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVVPID_CAPS_CONTEXT)
665 pVM->hwaccm.s.vmx.enmFlushContext = VMX_FLUSH_SINGLE_CONTEXT;
666 }
667#endif /* HWACCM_VTX_WITH_VPID */
668 else
669 pVM->hwaccm.s.vmx.pfnSetupTaggedTLB = vmxR0SetupTLBDummy;
670
671vmx_end:
672 VMXR0CheckError(pVM, &pVM->aCpus[0], rc);
673 return rc;
674}
675
676/**
677 * Sets the permission bits for the specified MSR
678 *
679 * @param pVCpu The VMCPU to operate on.
680 * @param ulMSR MSR value
681 * @param fRead Reading allowed/disallowed
682 * @param fWrite Writing allowed/disallowed
683 */
684static void vmxR0SetMSRPermission(PVMCPU pVCpu, unsigned ulMSR, bool fRead, bool fWrite)
685{
686 unsigned ulBit;
687 uint8_t *pMSRBitmap = (uint8_t *)pVCpu->hwaccm.s.vmx.pMSRBitmap;
688
689 /* Layout:
690 * 0x000 - 0x3ff - Low MSR read bits
691 * 0x400 - 0x7ff - High MSR read bits
692 * 0x800 - 0xbff - Low MSR write bits
693 * 0xc00 - 0xfff - High MSR write bits
694 */
695 if (ulMSR <= 0x00001FFF)
696 {
697 /* Pentium-compatible MSRs */
698 ulBit = ulMSR;
699 }
700 else
701 if ( ulMSR >= 0xC0000000
702 && ulMSR <= 0xC0001FFF)
703 {
704 /* AMD Sixth Generation x86 Processor MSRs */
705 ulBit = (ulMSR - 0xC0000000);
706 pMSRBitmap += 0x400;
707 }
708 else
709 {
710 AssertFailed();
711 return;
712 }
713
714 Assert(ulBit <= 0x1fff);
715 if (fRead)
716 ASMBitClear(pMSRBitmap, ulBit);
717 else
718 ASMBitSet(pMSRBitmap, ulBit);
719
720 if (fWrite)
721 ASMBitClear(pMSRBitmap + 0x800, ulBit);
722 else
723 ASMBitSet(pMSRBitmap + 0x800, ulBit);
724}
725
726
727/**
728 * Injects an event (trap or external interrupt)
729 *
730 * @returns VBox status code.
731 * @param pVM The VM to operate on.
732 * @param pVCpu The VMCPU to operate on.
733 * @param pCtx CPU Context
734 * @param intInfo VMX interrupt info
735 * @param cbInstr Opcode length of faulting instruction
736 * @param errCode Error code (optional)
737 */
738static int VMXR0InjectEvent(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, uint32_t intInfo, uint32_t cbInstr, uint32_t errCode)
739{
740 int rc;
741 uint32_t iGate = VMX_EXIT_INTERRUPTION_INFO_VECTOR(intInfo);
742
743#ifdef VBOX_WITH_STATISTICS
744 STAM_COUNTER_INC(&pVCpu->hwaccm.s.paStatInjectedIrqsR0[iGate & MASK_INJECT_IRQ_STAT]);
745#endif
746
747#ifdef VBOX_STRICT
748 if (iGate == 0xE)
749 LogFlow(("VMXR0InjectEvent: Injecting interrupt %d at %RGv error code=%08x CR2=%RGv intInfo=%08x\n", iGate, (RTGCPTR)pCtx->rip, errCode, pCtx->cr2, intInfo));
750 else
751 if (iGate < 0x20)
752 LogFlow(("VMXR0InjectEvent: Injecting interrupt %d at %RGv error code=%08x\n", iGate, (RTGCPTR)pCtx->rip, errCode));
753 else
754 {
755 LogFlow(("INJ-EI: %x at %RGv\n", iGate, (RTGCPTR)pCtx->rip));
756 Assert(VMX_EXIT_INTERRUPTION_INFO_TYPE(intInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW || !VMCPU_FF_ISSET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
757 Assert(VMX_EXIT_INTERRUPTION_INFO_TYPE(intInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW || pCtx->eflags.u32 & X86_EFL_IF);
758 }
759#endif
760
761#ifdef HWACCM_VMX_EMULATE_REALMODE
762 if ( CPUMIsGuestInRealModeEx(pCtx)
763 && pVM->hwaccm.s.vmx.pRealModeTSS)
764 {
765 RTGCPHYS GCPhysHandler;
766 uint16_t offset, ip;
767 RTSEL sel;
768
769 /* Injecting events doesn't work right with real mode emulation.
770 * (#GP if we try to inject external hardware interrupts)
771 * Inject the interrupt or trap directly instead.
772 *
773 * ASSUMES no access handlers for the bits we read or write below (should be safe).
774 */
775 Log(("Manual interrupt/trap '%x' inject (real mode)\n", iGate));
776
777 /* Check if the interrupt handler is present. */
778 if (iGate * 4 + 3 > pCtx->idtr.cbIdt)
779 {
780 Log(("IDT cbIdt violation\n"));
781 if (iGate != X86_XCPT_DF)
782 {
783 uint32_t intInfo2;
784
785 intInfo2 = (iGate == X86_XCPT_GP) ? (uint32_t)X86_XCPT_DF : iGate;
786 intInfo2 |= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
787 intInfo2 |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
788 intInfo2 |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HWEXCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
789
790 return VMXR0InjectEvent(pVM, pVCpu, pCtx, intInfo2, 0, 0 /* no error code according to the Intel docs */);
791 }
792 Log(("Triple fault -> reset the VM!\n"));
793 return VINF_EM_RESET;
794 }
795 if ( VMX_EXIT_INTERRUPTION_INFO_TYPE(intInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW
796 || iGate == 3 /* Both #BP and #OF point to the instruction after. */
797 || iGate == 4)
798 {
799 ip = pCtx->ip + cbInstr;
800 }
801 else
802 ip = pCtx->ip;
803
804 /* Read the selector:offset pair of the interrupt handler. */
805 GCPhysHandler = (RTGCPHYS)pCtx->idtr.pIdt + iGate * 4;
806 rc = PGMPhysSimpleReadGCPhys(pVM, &offset, GCPhysHandler, sizeof(offset)); AssertRC(rc);
807 rc = PGMPhysSimpleReadGCPhys(pVM, &sel, GCPhysHandler + 2, sizeof(sel)); AssertRC(rc);
808
809 LogFlow(("IDT handler %04X:%04X\n", sel, offset));
810
811 /* Construct the stack frame. */
812 /** @todo should check stack limit. */
813 pCtx->sp -= 2;
814 LogFlow(("ss:sp %04X:%04X eflags=%x\n", pCtx->ss, pCtx->sp, pCtx->eflags.u));
815 rc = PGMPhysSimpleWriteGCPhys(pVM, pCtx->ssHid.u64Base + pCtx->sp, &pCtx->eflags, sizeof(uint16_t)); AssertRC(rc);
816 pCtx->sp -= 2;
817 LogFlow(("ss:sp %04X:%04X cs=%x\n", pCtx->ss, pCtx->sp, pCtx->cs));
818 rc = PGMPhysSimpleWriteGCPhys(pVM, pCtx->ssHid.u64Base + pCtx->sp, &pCtx->cs, sizeof(uint16_t)); AssertRC(rc);
819 pCtx->sp -= 2;
820 LogFlow(("ss:sp %04X:%04X ip=%x\n", pCtx->ss, pCtx->sp, ip));
821 rc = PGMPhysSimpleWriteGCPhys(pVM, pCtx->ssHid.u64Base + pCtx->sp, &ip, sizeof(ip)); AssertRC(rc);
822
823 /* Update the CPU state for executing the handler. */
824 pCtx->rip = offset;
825 pCtx->cs = sel;
826 pCtx->csHid.u64Base = sel << 4;
827 pCtx->eflags.u &= ~(X86_EFL_IF|X86_EFL_TF|X86_EFL_RF|X86_EFL_AC);
828
829 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_GUEST_SEGMENT_REGS;
830 return VINF_SUCCESS;
831 }
832#endif /* HWACCM_VMX_EMULATE_REALMODE */
833
834 /* Set event injection state. */
835 rc = VMXWriteVMCS(VMX_VMCS_CTRL_ENTRY_IRQ_INFO, intInfo | (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT));
836
837 rc |= VMXWriteVMCS(VMX_VMCS_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
838 rc |= VMXWriteVMCS(VMX_VMCS_CTRL_ENTRY_EXCEPTION_ERRCODE, errCode);
839
840 AssertRC(rc);
841 return rc;
842}
843
844
845/**
846 * Checks for pending guest interrupts and injects them
847 *
848 * @returns VBox status code.
849 * @param pVM The VM to operate on.
850 * @param pVCpu The VMCPU to operate on.
851 * @param pCtx CPU Context
852 */
853static int VMXR0CheckPendingInterrupt(PVM pVM, PVMCPU pVCpu, CPUMCTX *pCtx)
854{
855 int rc;
856
857 /* Dispatch any pending interrupts. (injected before, but a VM exit occurred prematurely) */
858 if (pVCpu->hwaccm.s.Event.fPending)
859 {
860 Log(("CPU%d: Reinjecting event %RX64 %08x at %RGv cr2=%RX64\n", pVCpu->idCpu, pVCpu->hwaccm.s.Event.intInfo, pVCpu->hwaccm.s.Event.errCode, (RTGCPTR)pCtx->rip, pCtx->cr2));
861 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatIntReinject);
862 rc = VMXR0InjectEvent(pVM, pVCpu, pCtx, pVCpu->hwaccm.s.Event.intInfo, 0, pVCpu->hwaccm.s.Event.errCode);
863 AssertRC(rc);
864
865 pVCpu->hwaccm.s.Event.fPending = false;
866 return VINF_SUCCESS;
867 }
868
869 /* If an active trap is already pending, then we must forward it first! */
870 if (!TRPMHasTrap(pVCpu))
871 {
872 if (VMCPU_FF_TESTANDCLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI))
873 {
874 RTGCUINTPTR intInfo;
875
876 Log(("CPU%d: injecting #NMI\n", pVCpu->idCpu));
877
878 intInfo = X86_XCPT_NMI;
879 intInfo |= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
880 intInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
881
882 rc = VMXR0InjectEvent(pVM, pVCpu, pCtx, intInfo, 0, 0);
883 AssertRC(rc);
884
885 return VINF_SUCCESS;
886 }
887
888 /* @todo SMI interrupts. */
889
890 /* When external interrupts are pending, we should exit the VM when IF is set. */
891 if (VMCPU_FF_ISPENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC|VMCPU_FF_INTERRUPT_PIC)))
892 {
893 if (!(pCtx->eflags.u32 & X86_EFL_IF))
894 {
895 if (!(pVCpu->hwaccm.s.vmx.proc_ctls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_IRQ_WINDOW_EXIT))
896 {
897 LogFlow(("Enable irq window exit!\n"));
898 pVCpu->hwaccm.s.vmx.proc_ctls |= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_IRQ_WINDOW_EXIT;
899 rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
900 AssertRC(rc);
901 }
902 /* else nothing to do but wait */
903 }
904 else
905 if (!VMCPU_FF_ISSET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
906 {
907 uint8_t u8Interrupt;
908
909 rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
910 Log(("CPU%d: Dispatch interrupt: u8Interrupt=%x (%d) rc=%Rrc cs:rip=%04X:%RGv\n", pVCpu->idCpu, u8Interrupt, u8Interrupt, rc, pCtx->cs, (RTGCPTR)pCtx->rip));
911 if (RT_SUCCESS(rc))
912 {
913 rc = TRPMAssertTrap(pVCpu, u8Interrupt, TRPM_HARDWARE_INT);
914 AssertRC(rc);
915 }
916 else
917 {
918 /* Can only happen in rare cases where a pending interrupt is cleared behind our back */
919 Assert(!VMCPU_FF_ISPENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC|VMCPU_FF_INTERRUPT_PIC)));
920 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatSwitchGuestIrq);
921 /* Just continue */
922 }
923 }
924 else
925 Log(("Pending interrupt blocked at %RGv by VM_FF_INHIBIT_INTERRUPTS!!\n", (RTGCPTR)pCtx->rip));
926 }
927 }
928
929#ifdef VBOX_STRICT
930 if (TRPMHasTrap(pVCpu))
931 {
932 uint8_t u8Vector;
933 rc = TRPMQueryTrapAll(pVCpu, &u8Vector, 0, 0, 0);
934 AssertRC(rc);
935 }
936#endif
937
938 if ( (pCtx->eflags.u32 & X86_EFL_IF)
939 && (!VMCPU_FF_ISSET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
940 && TRPMHasTrap(pVCpu)
941 )
942 {
943 uint8_t u8Vector;
944 TRPMEVENT enmType;
945 RTGCUINTPTR intInfo;
946 RTGCUINT errCode;
947
948 /* If a new event is pending, then dispatch it now. */
949 rc = TRPMQueryTrapAll(pVCpu, &u8Vector, &enmType, &errCode, 0);
950 AssertRC(rc);
951 Assert(pCtx->eflags.Bits.u1IF == 1 || enmType == TRPM_TRAP);
952 Assert(enmType != TRPM_SOFTWARE_INT);
953
954 /* Clear the pending trap. */
955 rc = TRPMResetTrap(pVCpu);
956 AssertRC(rc);
957
958 intInfo = u8Vector;
959 intInfo |= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
960
961 if (enmType == TRPM_TRAP)
962 {
963 switch (u8Vector) {
964 case 8:
965 case 10:
966 case 11:
967 case 12:
968 case 13:
969 case 14:
970 case 17:
971 /* Valid error codes. */
972 intInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
973 break;
974 default:
975 break;
976 }
977 if (u8Vector == X86_XCPT_BP || u8Vector == X86_XCPT_OF)
978 intInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SWEXCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
979 else
980 intInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HWEXCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
981 }
982 else
983 intInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
984
985 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatIntInject);
986 rc = VMXR0InjectEvent(pVM, pVCpu, pCtx, intInfo, 0, errCode);
987 AssertRC(rc);
988 } /* if (interrupts can be dispatched) */
989
990 return VINF_SUCCESS;
991}
992
993/**
994 * Save the host state
995 *
996 * @returns VBox status code.
997 * @param pVM The VM to operate on.
998 * @param pVCpu The VMCPU to operate on.
999 */
1000VMMR0DECL(int) VMXR0SaveHostState(PVM pVM, PVMCPU pVCpu)
1001{
1002 int rc = VINF_SUCCESS;
1003
1004 /*
1005 * Host CPU Context
1006 */
1007 if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_HOST_CONTEXT)
1008 {
1009 RTIDTR idtr;
1010 RTGDTR gdtr;
1011 RTSEL SelTR;
1012 PCX86DESCHC pDesc;
1013 uintptr_t trBase;
1014 RTSEL cs;
1015 RTSEL ss;
1016 uint64_t cr3;
1017
1018 /* Control registers */
1019 rc = VMXWriteVMCS(VMX_VMCS_HOST_CR0, ASMGetCR0());
1020#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
1021 if (VMX_IS_64BIT_HOST_MODE())
1022 {
1023 cr3 = hwaccmR0Get64bitCR3();
1024 rc |= VMXWriteVMCS64(VMX_VMCS_HOST_CR3, cr3);
1025 }
1026 else
1027#endif
1028 {
1029 cr3 = ASMGetCR3();
1030 rc |= VMXWriteVMCS(VMX_VMCS_HOST_CR3, cr3);
1031 }
1032 rc |= VMXWriteVMCS(VMX_VMCS_HOST_CR4, ASMGetCR4());
1033 AssertRC(rc);
1034 Log2(("VMX_VMCS_HOST_CR0 %08x\n", ASMGetCR0()));
1035 Log2(("VMX_VMCS_HOST_CR3 %08RX64\n", cr3));
1036 Log2(("VMX_VMCS_HOST_CR4 %08x\n", ASMGetCR4()));
1037
1038 /* Selector registers. */
1039#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
1040 if (VMX_IS_64BIT_HOST_MODE())
1041 {
1042 cs = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelCS;
1043 ss = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelSS;
1044 }
1045 else
1046 {
1047 /* sysenter loads LDT cs & ss, VMX doesn't like this. Load the GDT ones (safe). */
1048 cs = (RTSEL)(uintptr_t)&SUPR0AbsKernelCS;
1049 ss = (RTSEL)(uintptr_t)&SUPR0AbsKernelSS;
1050 }
1051#else
1052 cs = ASMGetCS();
1053 ss = ASMGetSS();
1054#endif
1055 Assert(!(cs & X86_SEL_LDT)); Assert((cs & X86_SEL_RPL) == 0);
1056 Assert(!(ss & X86_SEL_LDT)); Assert((ss & X86_SEL_RPL) == 0);
1057 rc = VMXWriteVMCS(VMX_VMCS16_HOST_FIELD_CS, cs);
1058 /* Note: VMX is (again) very picky about the RPL of the selectors here; we'll restore them manually. */
1059 rc |= VMXWriteVMCS(VMX_VMCS16_HOST_FIELD_DS, 0);
1060 rc |= VMXWriteVMCS(VMX_VMCS16_HOST_FIELD_ES, 0);
1061#if HC_ARCH_BITS == 32
1062 if (!VMX_IS_64BIT_HOST_MODE())
1063 {
1064 rc |= VMXWriteVMCS(VMX_VMCS16_HOST_FIELD_FS, 0);
1065 rc |= VMXWriteVMCS(VMX_VMCS16_HOST_FIELD_GS, 0);
1066 }
1067#endif
1068 rc |= VMXWriteVMCS(VMX_VMCS16_HOST_FIELD_SS, ss);
1069 SelTR = ASMGetTR();
1070 rc |= VMXWriteVMCS(VMX_VMCS16_HOST_FIELD_TR, SelTR);
1071 AssertRC(rc);
1072 Log2(("VMX_VMCS_HOST_FIELD_CS %08x (%08x)\n", cs, ASMGetSS()));
1073 Log2(("VMX_VMCS_HOST_FIELD_DS 00000000 (%08x)\n", ASMGetDS()));
1074 Log2(("VMX_VMCS_HOST_FIELD_ES 00000000 (%08x)\n", ASMGetES()));
1075 Log2(("VMX_VMCS_HOST_FIELD_FS 00000000 (%08x)\n", ASMGetFS()));
1076 Log2(("VMX_VMCS_HOST_FIELD_GS 00000000 (%08x)\n", ASMGetGS()));
1077 Log2(("VMX_VMCS_HOST_FIELD_SS %08x (%08x)\n", ss, ASMGetSS()));
1078 Log2(("VMX_VMCS_HOST_FIELD_TR %08x\n", ASMGetTR()));
1079
1080 /* GDTR & IDTR */
1081#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
1082 if (VMX_IS_64BIT_HOST_MODE())
1083 {
1084 X86XDTR64 gdtr64, idtr64;
1085 hwaccmR0Get64bitGDTRandIDTR(&gdtr64, &idtr64);
1086 rc = VMXWriteVMCS64(VMX_VMCS_HOST_GDTR_BASE, gdtr64.uAddr);
1087 rc |= VMXWriteVMCS64(VMX_VMCS_HOST_IDTR_BASE, gdtr64.uAddr);
1088 AssertRC(rc);
1089 Log2(("VMX_VMCS_HOST_GDTR_BASE %RX64\n", gdtr64.uAddr));
1090 Log2(("VMX_VMCS_HOST_IDTR_BASE %RX64\n", idtr64.uAddr));
1091 gdtr.cbGdt = gdtr64.cb;
1092 gdtr.pGdt = (uintptr_t)gdtr64.uAddr;
1093 }
1094 else
1095#endif
1096 {
1097 ASMGetGDTR(&gdtr);
1098 rc = VMXWriteVMCS(VMX_VMCS_HOST_GDTR_BASE, gdtr.pGdt);
1099 ASMGetIDTR(&idtr);
1100 rc |= VMXWriteVMCS(VMX_VMCS_HOST_IDTR_BASE, idtr.pIdt);
1101 AssertRC(rc);
1102 Log2(("VMX_VMCS_HOST_GDTR_BASE %RHv\n", gdtr.pGdt));
1103 Log2(("VMX_VMCS_HOST_IDTR_BASE %RHv\n", idtr.pIdt));
1104 }
1105
1106 /* Save the base address of the TR selector. */
1107 if (SelTR > gdtr.cbGdt)
1108 {
1109 AssertMsgFailed(("Invalid TR selector %x. GDTR.cbGdt=%x\n", SelTR, gdtr.cbGdt));
1110 return VERR_VMX_INVALID_HOST_STATE;
1111 }
1112
1113 pDesc = (PCX86DESCHC)(gdtr.pGdt + (SelTR & X86_SEL_MASK));
1114#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
1115 if (VMX_IS_64BIT_HOST_MODE())
1116 {
1117 uint64_t trBase64 = X86DESC64_BASE(*(PX86DESC64)pDesc);
1118 rc = VMXWriteVMCS64(VMX_VMCS_HOST_TR_BASE, trBase64);
1119 Log2(("VMX_VMCS_HOST_TR_BASE %RX64\n", trBase64));
1120 AssertRC(rc);
1121 }
1122 else
1123#endif
1124 {
1125#if HC_ARCH_BITS == 64
1126 trBase = X86DESC64_BASE(*pDesc);
1127#else
1128 trBase = X86DESC_BASE(*pDesc);
1129#endif
1130 rc = VMXWriteVMCS(VMX_VMCS_HOST_TR_BASE, trBase);
1131 AssertRC(rc);
1132 Log2(("VMX_VMCS_HOST_TR_BASE %RHv\n", trBase));
1133 }
1134
1135 /* FS and GS base. */
1136#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
1137 if (VMX_IS_64BIT_HOST_MODE())
1138 {
1139 Log2(("MSR_K8_FS_BASE = %RX64\n", ASMRdMsr(MSR_K8_FS_BASE)));
1140 Log2(("MSR_K8_GS_BASE = %RX64\n", ASMRdMsr(MSR_K8_GS_BASE)));
1141 rc = VMXWriteVMCS64(VMX_VMCS_HOST_FS_BASE, ASMRdMsr(MSR_K8_FS_BASE));
1142 rc |= VMXWriteVMCS64(VMX_VMCS_HOST_GS_BASE, ASMRdMsr(MSR_K8_GS_BASE));
1143 }
1144#endif
1145 AssertRC(rc);
1146
1147 /* Sysenter MSRs. */
1148 /** @todo expensive!! */
1149 rc = VMXWriteVMCS(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
1150 Log2(("VMX_VMCS_HOST_SYSENTER_CS %08x\n", ASMRdMsr_Low(MSR_IA32_SYSENTER_CS)));
1151#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
1152 if (VMX_IS_64BIT_HOST_MODE())
1153 {
1154 Log2(("VMX_VMCS_HOST_SYSENTER_EIP %RX64\n", ASMRdMsr(MSR_IA32_SYSENTER_EIP)));
1155 Log2(("VMX_VMCS_HOST_SYSENTER_ESP %RX64\n", ASMRdMsr(MSR_IA32_SYSENTER_ESP)));
1156 rc |= VMXWriteVMCS64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
1157 rc |= VMXWriteVMCS64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
1158 }
1159 else
1160 {
1161 rc |= VMXWriteVMCS(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
1162 rc |= VMXWriteVMCS(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
1163 Log2(("VMX_VMCS_HOST_SYSENTER_EIP %RX32\n", ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP)));
1164 Log2(("VMX_VMCS_HOST_SYSENTER_ESP %RX32\n", ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP)));
1165 }
1166#elif HC_ARCH_BITS == 32
1167 rc |= VMXWriteVMCS(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
1168 rc |= VMXWriteVMCS(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
1169 Log2(("VMX_VMCS_HOST_SYSENTER_EIP %RX32\n", ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP)));
1170 Log2(("VMX_VMCS_HOST_SYSENTER_ESP %RX32\n", ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP)));
1171#else
1172 Log2(("VMX_VMCS_HOST_SYSENTER_EIP %RX64\n", ASMRdMsr(MSR_IA32_SYSENTER_EIP)));
1173 Log2(("VMX_VMCS_HOST_SYSENTER_ESP %RX64\n", ASMRdMsr(MSR_IA32_SYSENTER_ESP)));
1174 rc |= VMXWriteVMCS64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
1175 rc |= VMXWriteVMCS64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
1176#endif
1177 AssertRC(rc);
1178
1179#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
1180 /* Store all host MSRs in the VM-Exit load area, so they will be reloaded after the world switch back to the host. */
1181 PVMXMSR pMsr = (PVMXMSR)pVCpu->hwaccm.s.vmx.pHostMSR;
1182 unsigned idxMsr = 0;
1183
1184 /* EFER MSR present? */
1185 if (ASMCpuId_EDX(0x80000001) & (X86_CPUID_AMD_FEATURE_EDX_NX|X86_CPUID_AMD_FEATURE_EDX_LONG_MODE))
1186 {
1187 if (ASMCpuId_EDX(0x80000001) & X86_CPUID_AMD_FEATURE_EDX_SEP)
1188 {
1189 pMsr->u32IndexMSR = MSR_K6_STAR;
1190 pMsr->u32Reserved = 0;
1191 pMsr->u64Value = ASMRdMsr(MSR_K6_STAR); /* legacy syscall eip, cs & ss */
1192 pMsr++; idxMsr++;
1193 }
1194
1195 pMsr->u32IndexMSR = MSR_K6_EFER;
1196 pMsr->u32Reserved = 0;
1197# if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
1198 if (CPUMIsGuestInLongMode(pVCpu))
1199 {
1200 /* Must match the efer value in our 64 bits switcher. */
1201 pMsr->u64Value = ASMRdMsr(MSR_K6_EFER) | MSR_K6_EFER_LME | MSR_K6_EFER_SCE | MSR_K6_EFER_NXE;
1202 }
1203 else
1204# endif
1205 pMsr->u64Value = ASMRdMsr(MSR_K6_EFER);
1206 pMsr++; idxMsr++;
1207 }
1208
1209# if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
1210 if (VMX_IS_64BIT_HOST_MODE())
1211 {
1212 pMsr->u32IndexMSR = MSR_K8_LSTAR;
1213 pMsr->u32Reserved = 0;
1214 pMsr->u64Value = ASMRdMsr(MSR_K8_LSTAR); /* 64 bits mode syscall rip */
1215 pMsr++; idxMsr++;
1216 pMsr->u32IndexMSR = MSR_K8_SF_MASK;
1217 pMsr->u32Reserved = 0;
1218 pMsr->u64Value = ASMRdMsr(MSR_K8_SF_MASK); /* syscall flag mask */
1219 pMsr++; idxMsr++;
1220 pMsr->u32IndexMSR = MSR_K8_KERNEL_GS_BASE;
1221 pMsr->u32Reserved = 0;
1222 pMsr->u64Value = ASMRdMsr(MSR_K8_KERNEL_GS_BASE); /* swapgs exchange value */
1223 pMsr++; idxMsr++;
1224 }
1225# endif
1226 rc = VMXWriteVMCS(VMX_VMCS_CTRL_EXIT_MSR_LOAD_COUNT, idxMsr);
1227 AssertRC(rc);
1228#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
1229
1230 pVCpu->hwaccm.s.fContextUseFlags &= ~HWACCM_CHANGED_HOST_CONTEXT;
1231 }
1232 return rc;
1233}
1234
1235/**
1236 * Prefetch the 4 PDPT pointers (PAE and nested paging only)
1237 *
1238 * @param pVM The VM to operate on.
1239 * @param pVCpu The VMCPU to operate on.
1240 * @param pCtx Guest context
1241 */
1242static void vmxR0PrefetchPAEPdptrs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
1243{
1244 if (CPUMIsGuestInPAEModeEx(pCtx))
1245 {
1246 X86PDPE Pdpe;
1247
1248 for (unsigned i=0;i<4;i++)
1249 {
1250 Pdpe = PGMGstGetPaePDPtr(pVCpu, i);
1251 int rc = VMXWriteVMCS64(VMX_VMCS_GUEST_PDPTR0_FULL + i*2, Pdpe.u);
1252 AssertRC(rc);
1253 }
1254 }
1255}
1256
1257/**
1258 * Update the exception bitmap according to the current CPU state
1259 *
1260 * @param pVM The VM to operate on.
1261 * @param pVCpu The VMCPU to operate on.
1262 * @param pCtx Guest context
1263 */
1264static void vmxR0UpdateExceptionBitmap(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
1265{
1266 uint32_t u32TrapMask;
1267 Assert(pCtx);
1268
1269 u32TrapMask = HWACCM_VMX_TRAP_MASK;
1270#ifndef DEBUG
1271 if (pVM->hwaccm.s.fNestedPaging)
1272 u32TrapMask &= ~RT_BIT(X86_XCPT_PF); /* no longer need to intercept #PF. */
1273#endif
1274
1275 /* Also catch floating point exceptions as we need to report them to the guest in a different way. */
1276 if ( CPUMIsGuestFPUStateActive(pVCpu) == true
1277 && !(pCtx->cr0 & X86_CR0_NE)
1278 && !pVCpu->hwaccm.s.fFPUOldStyleOverride)
1279 {
1280 u32TrapMask |= RT_BIT(X86_XCPT_MF);
1281 pVCpu->hwaccm.s.fFPUOldStyleOverride = true;
1282 }
1283
1284#ifdef VBOX_STRICT
1285 Assert(u32TrapMask & RT_BIT(X86_XCPT_GP));
1286#endif
1287
1288# ifdef HWACCM_VMX_EMULATE_REALMODE
1289 /* Intercept all exceptions in real mode as none of them can be injected directly (#GP otherwise). */
1290 if ( CPUMIsGuestInRealModeEx(pCtx)
1291 && pVM->hwaccm.s.vmx.pRealModeTSS)
1292 u32TrapMask |= HWACCM_VMX_TRAP_MASK_REALMODE;
1293# endif /* HWACCM_VMX_EMULATE_REALMODE */
1294
1295 int rc = VMXWriteVMCS(VMX_VMCS_CTRL_EXCEPTION_BITMAP, u32TrapMask);
1296 AssertRC(rc);
1297}
1298
1299/**
1300 * Loads the guest state
1301 *
1302 * NOTE: Don't do anything here that can cause a jump back to ring 3!!!!!
1303 *
1304 * @returns VBox status code.
1305 * @param pVM The VM to operate on.
1306 * @param pVCpu The VMCPU to operate on.
1307 * @param pCtx Guest context
1308 */
1309VMMR0DECL(int) VMXR0LoadGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
1310{
1311 int rc = VINF_SUCCESS;
1312 RTGCUINTPTR val;
1313 X86EFLAGS eflags;
1314
1315 /* VMX_VMCS_CTRL_ENTRY_CONTROLS
1316 * Set required bits to one and zero according to the MSR capabilities.
1317 */
1318 val = pVM->hwaccm.s.vmx.msr.vmx_entry.n.disallowed0;
1319 /* Load guest debug controls (dr7 & IA32_DEBUGCTL_MSR) (forced to 1 on the 'first' VT-x capable CPUs; this actually includes the newest Nehalem CPUs) */
1320 val |= VMX_VMCS_CTRL_ENTRY_CONTROLS_LOAD_DEBUG;
1321 /* 64 bits guest mode? */
1322 if (CPUMIsGuestInLongModeEx(pCtx))
1323 val |= VMX_VMCS_CTRL_ENTRY_CONTROLS_IA64_MODE;
1324 /* else Must be zero when AMD64 is not available. */
1325
1326 /* Mask away the bits that the CPU doesn't support */
1327 val &= pVM->hwaccm.s.vmx.msr.vmx_entry.n.allowed1;
1328 rc = VMXWriteVMCS(VMX_VMCS_CTRL_ENTRY_CONTROLS, val);
1329 AssertRC(rc);
1330
1331 /* VMX_VMCS_CTRL_EXIT_CONTROLS
1332 * Set required bits to one and zero according to the MSR capabilities.
1333 */
1334 val = pVM->hwaccm.s.vmx.msr.vmx_exit.n.disallowed0;
1335
1336 /* Save debug controls (dr7 & IA32_DEBUGCTL_MSR) (forced to 1 on the 'first' VT-x capable CPUs; this actually includes the newest Nehalem CPUs) */
1337 val |= VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_DEBUG;
1338
1339#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
1340 if (VMX_IS_64BIT_HOST_MODE())
1341 val |= VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_AMD64;
1342 /* else: Must be zero when AMD64 is not available. */
1343#elif HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
1344 if (CPUMIsGuestInLongModeEx(pCtx))
1345 val |= VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_AMD64; /* our switcher goes to long mode */
1346 else
1347 Assert(!(val & VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_AMD64));
1348#endif
1349 val &= pVM->hwaccm.s.vmx.msr.vmx_exit.n.allowed1;
1350 /* Don't acknowledge external interrupts on VM-exit. */
1351 rc = VMXWriteVMCS(VMX_VMCS_CTRL_EXIT_CONTROLS, val);
1352 AssertRC(rc);
1353
1354 /* Guest CPU context: ES, CS, SS, DS, FS, GS. */
1355 if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_SEGMENT_REGS)
1356 {
1357#ifdef HWACCM_VMX_EMULATE_REALMODE
1358 if (pVM->hwaccm.s.vmx.pRealModeTSS)
1359 {
1360 PGMMODE enmGuestMode = PGMGetGuestMode(pVCpu);
1361 if (pVCpu->hwaccm.s.vmx.enmLastSeenGuestMode != enmGuestMode)
1362 {
1363 /* Correct weird requirements for switching to protected mode. */
1364 if ( pVCpu->hwaccm.s.vmx.enmLastSeenGuestMode == PGMMODE_REAL
1365 && enmGuestMode >= PGMMODE_PROTECTED)
1366 {
1367 /* Flush the recompiler code cache as it's not unlikely
1368 * the guest will rewrite code it will later execute in real
1369 * mode (OpenBSD 4.0 is one such example)
1370 */
1371 REMFlushTBs(pVM);
1372
1373 /* DPL of all hidden selector registers must match the current CPL (0). */
1374 pCtx->csHid.Attr.n.u2Dpl = 0;
1375 pCtx->csHid.Attr.n.u4Type = X86_SEL_TYPE_CODE | X86_SEL_TYPE_RW_ACC;
1376
1377 pCtx->dsHid.Attr.n.u2Dpl = 0;
1378 pCtx->esHid.Attr.n.u2Dpl = 0;
1379 pCtx->fsHid.Attr.n.u2Dpl = 0;
1380 pCtx->gsHid.Attr.n.u2Dpl = 0;
1381 pCtx->ssHid.Attr.n.u2Dpl = 0;
1382
1383 /* The limit must correspond to the 32 bits setting. */
1384 if (!pCtx->csHid.Attr.n.u1DefBig)
1385 pCtx->csHid.u32Limit &= 0xffff;
1386 if (!pCtx->dsHid.Attr.n.u1DefBig)
1387 pCtx->dsHid.u32Limit &= 0xffff;
1388 if (!pCtx->esHid.Attr.n.u1DefBig)
1389 pCtx->esHid.u32Limit &= 0xffff;
1390 if (!pCtx->fsHid.Attr.n.u1DefBig)
1391 pCtx->fsHid.u32Limit &= 0xffff;
1392 if (!pCtx->gsHid.Attr.n.u1DefBig)
1393 pCtx->gsHid.u32Limit &= 0xffff;
1394 if (!pCtx->ssHid.Attr.n.u1DefBig)
1395 pCtx->ssHid.u32Limit &= 0xffff;
1396 }
1397 else
1398 /* Switching from protected mode to real mode. */
1399 if ( pVCpu->hwaccm.s.vmx.enmLastSeenGuestMode >= PGMMODE_PROTECTED
1400 && enmGuestMode == PGMMODE_REAL)
1401 {
1402 /* The limit must also be set to 0xffff. */
1403 pCtx->csHid.u32Limit = 0xffff;
1404 pCtx->dsHid.u32Limit = 0xffff;
1405 pCtx->esHid.u32Limit = 0xffff;
1406 pCtx->fsHid.u32Limit = 0xffff;
1407 pCtx->gsHid.u32Limit = 0xffff;
1408 pCtx->ssHid.u32Limit = 0xffff;
1409
1410 Assert(pCtx->csHid.u64Base <= 0xfffff);
1411 Assert(pCtx->dsHid.u64Base <= 0xfffff);
1412 Assert(pCtx->esHid.u64Base <= 0xfffff);
1413 Assert(pCtx->fsHid.u64Base <= 0xfffff);
1414 Assert(pCtx->gsHid.u64Base <= 0xfffff);
1415 }
1416 pVCpu->hwaccm.s.vmx.enmLastSeenGuestMode = enmGuestMode;
1417 }
1418 else
1419 /* VT-x will fail with a guest invalid state otherwise... (CPU state after a reset) */
1420 if ( CPUMIsGuestInRealModeEx(pCtx)
1421 && pCtx->csHid.u64Base == 0xffff0000)
1422 {
1423 pCtx->csHid.u64Base = 0xf0000;
1424 pCtx->cs = 0xf000;
1425 }
1426 }
1427#endif /* HWACCM_VMX_EMULATE_REALMODE */
1428
1429 VMX_WRITE_SELREG(ES, es);
1430 AssertRC(rc);
1431
1432 VMX_WRITE_SELREG(CS, cs);
1433 AssertRC(rc);
1434
1435 VMX_WRITE_SELREG(SS, ss);
1436 AssertRC(rc);
1437
1438 VMX_WRITE_SELREG(DS, ds);
1439 AssertRC(rc);
1440
1441 VMX_WRITE_SELREG(FS, fs);
1442 AssertRC(rc);
1443
1444 VMX_WRITE_SELREG(GS, gs);
1445 AssertRC(rc);
1446 }
1447
1448 /* Guest CPU context: LDTR. */
1449 if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_LDTR)
1450 {
1451 if (pCtx->ldtr == 0)
1452 {
1453 rc = VMXWriteVMCS(VMX_VMCS16_GUEST_FIELD_LDTR, 0);
1454 rc |= VMXWriteVMCS(VMX_VMCS32_GUEST_LDTR_LIMIT, 0);
1455 rc |= VMXWriteVMCS64(VMX_VMCS64_GUEST_LDTR_BASE, 0);
1456 /* Note: vmlaunch will fail with 0 or just 0x02. No idea why. */
1457 rc |= VMXWriteVMCS(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, 0x82 /* present, LDT */);
1458 }
1459 else
1460 {
1461 rc = VMXWriteVMCS(VMX_VMCS16_GUEST_FIELD_LDTR, pCtx->ldtr);
1462 rc |= VMXWriteVMCS(VMX_VMCS32_GUEST_LDTR_LIMIT, pCtx->ldtrHid.u32Limit);
1463 rc |= VMXWriteVMCS64(VMX_VMCS64_GUEST_LDTR_BASE, pCtx->ldtrHid.u64Base);
1464 rc |= VMXWriteVMCS(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, pCtx->ldtrHid.Attr.u);
1465 }
1466 AssertRC(rc);
1467 }
1468 /* Guest CPU context: TR. */
1469 if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_TR)
1470 {
1471#ifdef HWACCM_VMX_EMULATE_REALMODE
1472 /* Real mode emulation using v86 mode with CR4.VME (interrupt redirection using the int bitmap in the TSS) */
1473 if ( CPUMIsGuestInRealModeEx(pCtx)
1474 && pVM->hwaccm.s.vmx.pRealModeTSS)
1475 {
1476 RTGCPHYS GCPhys;
1477
1478 /* We convert it here every time as pci regions could be reconfigured. */
1479 rc = PDMVMMDevHeapR3ToGCPhys(pVM, pVM->hwaccm.s.vmx.pRealModeTSS, &GCPhys);
1480 AssertRC(rc);
1481
1482 rc = VMXWriteVMCS(VMX_VMCS16_GUEST_FIELD_TR, 0);
1483 rc |= VMXWriteVMCS(VMX_VMCS32_GUEST_TR_LIMIT, HWACCM_VTX_TSS_SIZE);
1484 rc |= VMXWriteVMCS64(VMX_VMCS64_GUEST_TR_BASE, GCPhys /* phys = virt in this mode */);
1485
1486 X86DESCATTR attr;
1487
1488 attr.u = 0;
1489 attr.n.u1Present = 1;
1490 attr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
1491 val = attr.u;
1492 }
1493 else
1494#endif /* HWACCM_VMX_EMULATE_REALMODE */
1495 {
1496 rc = VMXWriteVMCS(VMX_VMCS16_GUEST_FIELD_TR, pCtx->tr);
1497 rc |= VMXWriteVMCS(VMX_VMCS32_GUEST_TR_LIMIT, pCtx->trHid.u32Limit);
1498 rc |= VMXWriteVMCS64(VMX_VMCS64_GUEST_TR_BASE, pCtx->trHid.u64Base);
1499
1500 val = pCtx->trHid.Attr.u;
1501
1502 /* The TSS selector must be busy. */
1503 if ((val & 0xF) == X86_SEL_TYPE_SYS_286_TSS_AVAIL)
1504 val = (val & ~0xF) | X86_SEL_TYPE_SYS_286_TSS_BUSY;
1505 else
1506 /* Default even if no TR selector has been set (otherwise vmlaunch will fail!) */
1507 val = (val & ~0xF) | X86_SEL_TYPE_SYS_386_TSS_BUSY;
1508
1509 }
1510 rc |= VMXWriteVMCS(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, val);
1511 AssertRC(rc);
1512 }
1513 /* Guest CPU context: GDTR. */
1514 if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_GDTR)
1515 {
1516 rc = VMXWriteVMCS(VMX_VMCS32_GUEST_GDTR_LIMIT, pCtx->gdtr.cbGdt);
1517 rc |= VMXWriteVMCS64(VMX_VMCS64_GUEST_GDTR_BASE, pCtx->gdtr.pGdt);
1518 AssertRC(rc);
1519 }
1520 /* Guest CPU context: IDTR. */
1521 if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_IDTR)
1522 {
1523 rc = VMXWriteVMCS(VMX_VMCS32_GUEST_IDTR_LIMIT, pCtx->idtr.cbIdt);
1524 rc |= VMXWriteVMCS64(VMX_VMCS64_GUEST_IDTR_BASE, pCtx->idtr.pIdt);
1525 AssertRC(rc);
1526 }
1527
1528 /*
1529 * Sysenter MSRs (unconditional)
1530 */
1531 rc = VMXWriteVMCS(VMX_VMCS32_GUEST_SYSENTER_CS, pCtx->SysEnter.cs);
1532 rc |= VMXWriteVMCS64(VMX_VMCS64_GUEST_SYSENTER_EIP, pCtx->SysEnter.eip);
1533 rc |= VMXWriteVMCS64(VMX_VMCS64_GUEST_SYSENTER_ESP, pCtx->SysEnter.esp);
1534 AssertRC(rc);
1535
1536 /* Control registers */
1537 if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_CR0)
1538 {
1539 val = pCtx->cr0;
1540 rc = VMXWriteVMCS(VMX_VMCS_CTRL_CR0_READ_SHADOW, val);
1541 Log2(("Guest CR0-shadow %08x\n", val));
1542 if (CPUMIsGuestFPUStateActive(pVCpu) == false)
1543 {
1544 /* Always use #NM exceptions to load the FPU/XMM state on demand. */
1545 val |= X86_CR0_TS | X86_CR0_ET | X86_CR0_NE | X86_CR0_MP;
1546 }
1547 else
1548 {
1549 /** @todo check if we support the old style mess correctly. */
1550 if (!(val & X86_CR0_NE))
1551 Log(("Forcing X86_CR0_NE!!!\n"));
1552
1553 val |= X86_CR0_NE; /* always turn on the native mechanism to report FPU errors (old style uses interrupts) */
1554 }
1555 /* Note: protected mode & paging are always enabled; we use them for emulating real and protected mode without paging too. */
1556 if (!pVM->hwaccm.s.vmx.fUnrestrictedGuest)
1557 val |= X86_CR0_PE | X86_CR0_PG;
1558
1559 if (pVM->hwaccm.s.fNestedPaging)
1560 {
1561 if (CPUMIsGuestInPagedProtectedModeEx(pCtx))
1562 {
1563 /* Disable cr3 read/write monitoring as we don't need it for EPT. */
1564 pVCpu->hwaccm.s.vmx.proc_ctls &= ~( VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
1565 | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT);
1566 }
1567 else
1568 {
1569 /* Reenable cr3 read/write monitoring as our identity mapped page table is active. */
1570 pVCpu->hwaccm.s.vmx.proc_ctls |= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
1571 | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT;
1572 }
1573 rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
1574 AssertRC(rc);
1575 }
1576 else
1577 {
1578 /* Note: We must also set this as we rely on protecting various pages for which supervisor writes must be caught. */
1579 val |= X86_CR0_WP;
1580 }
1581
1582 /* Always enable caching. */
1583 val &= ~(X86_CR0_CD|X86_CR0_NW);
1584
1585 rc |= VMXWriteVMCS64(VMX_VMCS64_GUEST_CR0, val);
1586 Log2(("Guest CR0 %08x\n", val));
1587 /* CR0 flags owned by the host; if the guests attempts to change them, then
1588 * the VM will exit.
1589 */
1590 val = X86_CR0_PE /* Must monitor this bit (assumptions are made for real mode emulation) */
1591 | X86_CR0_WP /* Must monitor this bit (it must always be enabled). */
1592 | X86_CR0_PG /* Must monitor this bit (assumptions are made for real mode & protected mode without paging emulation) */
1593 | X86_CR0_TS
1594 | X86_CR0_ET /* Bit not restored during VM-exit! */
1595 | X86_CR0_CD /* Bit not restored during VM-exit! */
1596 | X86_CR0_NW /* Bit not restored during VM-exit! */
1597 | X86_CR0_NE
1598 | X86_CR0_MP;
1599 pVCpu->hwaccm.s.vmx.cr0_mask = val;
1600
1601 rc |= VMXWriteVMCS(VMX_VMCS_CTRL_CR0_MASK, val);
1602 Log2(("Guest CR0-mask %08x\n", val));
1603 AssertRC(rc);
1604 }
1605 if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_CR4)
1606 {
1607 /* CR4 */
1608 rc = VMXWriteVMCS(VMX_VMCS_CTRL_CR4_READ_SHADOW, pCtx->cr4);
1609 Log2(("Guest CR4-shadow %08x\n", pCtx->cr4));
1610 /* Set the required bits in cr4 too (currently X86_CR4_VMXE). */
1611 val = pCtx->cr4 | (uint32_t)pVM->hwaccm.s.vmx.msr.vmx_cr4_fixed0;
1612
1613 if (!pVM->hwaccm.s.fNestedPaging)
1614 {
1615 switch(pVCpu->hwaccm.s.enmShadowMode)
1616 {
1617 case PGMMODE_REAL: /* Real mode -> emulated using v86 mode */
1618 case PGMMODE_PROTECTED: /* Protected mode, no paging -> emulated using identity mapping. */
1619 case PGMMODE_32_BIT: /* 32-bit paging. */
1620 val &= ~X86_CR4_PAE;
1621 break;
1622
1623 case PGMMODE_PAE: /* PAE paging. */
1624 case PGMMODE_PAE_NX: /* PAE paging with NX enabled. */
1625 /** @todo use normal 32 bits paging */
1626 val |= X86_CR4_PAE;
1627 break;
1628
1629 case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
1630 case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
1631#ifdef VBOX_ENABLE_64_BITS_GUESTS
1632 break;
1633#else
1634 AssertFailed();
1635 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
1636#endif
1637 default: /* shut up gcc */
1638 AssertFailed();
1639 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
1640 }
1641 }
1642 else
1643 if ( !CPUMIsGuestInPagedProtectedModeEx(pCtx)
1644 && !pVM->hwaccm.s.vmx.fUnrestrictedGuest)
1645 {
1646 /* We use 4 MB pages in our identity mapping page table for real and protected mode without paging. */
1647 val |= X86_CR4_PSE;
1648 /* Our identity mapping is a 32 bits page directory. */
1649 val &= ~X86_CR4_PAE;
1650 }
1651
1652#ifdef HWACCM_VMX_EMULATE_REALMODE
1653 /* Turn off VME if we're in emulated real mode. */
1654 if ( CPUMIsGuestInRealModeEx(pCtx)
1655 && pVM->hwaccm.s.vmx.pRealModeTSS)
1656 val &= ~X86_CR4_VME;
1657#endif /* HWACCM_VMX_EMULATE_REALMODE */
1658
1659 rc |= VMXWriteVMCS64(VMX_VMCS64_GUEST_CR4, val);
1660 Log2(("Guest CR4 %08x\n", val));
1661 /* CR4 flags owned by the host; if the guests attempts to change them, then
1662 * the VM will exit.
1663 */
1664 val = 0
1665 | X86_CR4_VME
1666 | X86_CR4_PAE
1667 | X86_CR4_PGE
1668 | X86_CR4_PSE
1669 | X86_CR4_VMXE;
1670 pVCpu->hwaccm.s.vmx.cr4_mask = val;
1671
1672 rc |= VMXWriteVMCS(VMX_VMCS_CTRL_CR4_MASK, val);
1673 Log2(("Guest CR4-mask %08x\n", val));
1674 AssertRC(rc);
1675 }
1676
1677 if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_CR3)
1678 {
1679 if (pVM->hwaccm.s.fNestedPaging)
1680 {
1681 Assert(PGMGetHyperCR3(pVCpu));
1682 pVCpu->hwaccm.s.vmx.GCPhysEPTP = PGMGetHyperCR3(pVCpu);
1683
1684 Assert(!(pVCpu->hwaccm.s.vmx.GCPhysEPTP & 0xfff));
1685 /** @todo Check the IA32_VMX_EPT_VPID_CAP MSR for other supported memory types. */
1686 pVCpu->hwaccm.s.vmx.GCPhysEPTP |= VMX_EPT_MEMTYPE_WB
1687 | (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
1688
1689 rc = VMXWriteVMCS64(VMX_VMCS_CTRL_EPTP_FULL, pVCpu->hwaccm.s.vmx.GCPhysEPTP);
1690 AssertRC(rc);
1691
1692 if ( !CPUMIsGuestInPagedProtectedModeEx(pCtx)
1693 && !pVM->hwaccm.s.vmx.fUnrestrictedGuest)
1694 {
1695 RTGCPHYS GCPhys;
1696
1697 /* We convert it here every time as pci regions could be reconfigured. */
1698 rc = PDMVMMDevHeapR3ToGCPhys(pVM, pVM->hwaccm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
1699 AssertMsgRC(rc, ("pNonPagingModeEPTPageTable = %RGv\n", pVM->hwaccm.s.vmx.pNonPagingModeEPTPageTable));
1700
1701 /* We use our identity mapping page table here as we need to map guest virtual to guest physical addresses; EPT will
1702 * take care of the translation to host physical addresses.
1703 */
1704 val = GCPhys;
1705 }
1706 else
1707 {
1708 /* Save the real guest CR3 in VMX_VMCS_GUEST_CR3 */
1709 val = pCtx->cr3;
1710 /* Prefetch the four PDPT entries in PAE mode. */
1711 vmxR0PrefetchPAEPdptrs(pVM, pVCpu, pCtx);
1712 }
1713 }
1714 else
1715 {
1716 val = PGMGetHyperCR3(pVCpu);
1717 Assert(val || VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL));
1718 }
1719
1720 /* Save our shadow CR3 register. */
1721 rc = VMXWriteVMCS64(VMX_VMCS64_GUEST_CR3, val);
1722 AssertRC(rc);
1723 }
1724
1725 /* Debug registers. */
1726 if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_DEBUG)
1727 {
1728 pCtx->dr[6] |= X86_DR6_INIT_VAL; /* set all reserved bits to 1. */
1729 pCtx->dr[6] &= ~RT_BIT(12); /* must be zero. */
1730
1731 pCtx->dr[7] &= 0xffffffff; /* upper 32 bits reserved */
1732 pCtx->dr[7] &= ~(RT_BIT(11) | RT_BIT(12) | RT_BIT(14) | RT_BIT(15)); /* must be zero */
1733 pCtx->dr[7] |= 0x400; /* must be one */
1734
1735 /* Resync DR7 */
1736 rc = VMXWriteVMCS64(VMX_VMCS64_GUEST_DR7, pCtx->dr[7]);
1737 AssertRC(rc);
1738
1739#ifdef DEBUG
1740 /* Sync the hypervisor debug state now if any breakpoint is armed. */
1741 if ( CPUMGetHyperDR7(pVCpu) & (X86_DR7_ENABLED_MASK|X86_DR7_GD)
1742 && !CPUMIsHyperDebugStateActive(pVCpu)
1743 && !DBGFIsStepping(pVCpu))
1744 {
1745 /* Save the host and load the hypervisor debug state. */
1746 rc = CPUMR0LoadHyperDebugState(pVM, pVCpu, pCtx, true /* include DR6 */);
1747 AssertRC(rc);
1748
1749 /* DRx intercepts remain enabled. */
1750
1751 /* Override dr7 with the hypervisor value. */
1752 rc = VMXWriteVMCS64(VMX_VMCS64_GUEST_DR7, CPUMGetHyperDR7(pVCpu));
1753 AssertRC(rc);
1754 }
1755 else
1756#endif
1757 /* Sync the debug state now if any breakpoint is armed. */
1758 if ( (pCtx->dr[7] & (X86_DR7_ENABLED_MASK|X86_DR7_GD))
1759 && !CPUMIsGuestDebugStateActive(pVCpu)
1760 && !DBGFIsStepping(pVCpu))
1761 {
1762 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatDRxArmed);
1763
1764 /* Disable drx move intercepts. */
1765 pVCpu->hwaccm.s.vmx.proc_ctls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT;
1766 rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
1767 AssertRC(rc);
1768
1769 /* Save the host and load the guest debug state. */
1770 rc = CPUMR0LoadGuestDebugState(pVM, pVCpu, pCtx, true /* include DR6 */);
1771 AssertRC(rc);
1772 }
1773
1774 /* IA32_DEBUGCTL MSR. */
1775 rc = VMXWriteVMCS64(VMX_VMCS_GUEST_DEBUGCTL_FULL, 0);
1776 AssertRC(rc);
1777
1778 /** @todo do we really ever need this? */
1779 rc |= VMXWriteVMCS(VMX_VMCS_GUEST_DEBUG_EXCEPTIONS, 0);
1780 AssertRC(rc);
1781 }
1782
1783 /* EIP, ESP and EFLAGS */
1784 rc = VMXWriteVMCS64(VMX_VMCS64_GUEST_RIP, pCtx->rip);
1785 rc |= VMXWriteVMCS64(VMX_VMCS64_GUEST_RSP, pCtx->rsp);
1786 AssertRC(rc);
1787
1788 /* Bits 22-31, 15, 5 & 3 must be zero. Bit 1 must be 1. */
1789 eflags = pCtx->eflags;
1790 eflags.u32 &= VMX_EFLAGS_RESERVED_0;
1791 eflags.u32 |= VMX_EFLAGS_RESERVED_1;
1792
1793#ifdef HWACCM_VMX_EMULATE_REALMODE
1794 /* Real mode emulation using v86 mode. */
1795 if ( CPUMIsGuestInRealModeEx(pCtx)
1796 && pVM->hwaccm.s.vmx.pRealModeTSS)
1797 {
1798 pVCpu->hwaccm.s.vmx.RealMode.eflags = eflags;
1799
1800 eflags.Bits.u1VM = 1;
1801 eflags.Bits.u2IOPL = 0; /* must always be 0 or else certain instructions won't cause faults. */
1802 }
1803#endif /* HWACCM_VMX_EMULATE_REALMODE */
1804 rc = VMXWriteVMCS(VMX_VMCS_GUEST_RFLAGS, eflags.u32);
1805 AssertRC(rc);
1806
1807 if (TMCpuTickCanUseRealTSC(pVCpu, &pVCpu->hwaccm.s.vmx.u64TSCOffset))
1808 {
1809 uint64_t u64CurTSC = ASMReadTSC();
1810 if (u64CurTSC + pVCpu->hwaccm.s.vmx.u64TSCOffset >= TMCpuTickGetLastSeen(pVCpu))
1811 {
1812 /* Note: VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT takes precedence over TSC_OFFSET */
1813 rc = VMXWriteVMCS64(VMX_VMCS_CTRL_TSC_OFFSET_FULL, pVCpu->hwaccm.s.vmx.u64TSCOffset);
1814 AssertRC(rc);
1815
1816 pVCpu->hwaccm.s.vmx.proc_ctls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT;
1817 rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
1818 AssertRC(rc);
1819 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatTSCOffset);
1820 }
1821 else
1822 {
1823 /* Fall back to rdtsc emulation as we would otherwise pass decreasing tsc values to the guest. */
1824 LogFlow(("TSC %RX64 offset %RX64 time=%RX64 last=%RX64 (diff=%RX64, virt_tsc=%RX64)\n", u64CurTSC, pVCpu->hwaccm.s.vmx.u64TSCOffset, u64CurTSC + pVCpu->hwaccm.s.vmx.u64TSCOffset, TMCpuTickGetLastSeen(pVCpu), TMCpuTickGetLastSeen(pVCpu) - u64CurTSC - pVCpu->hwaccm.s.vmx.u64TSCOffset, TMCpuTickGet(pVCpu)));
1825 pVCpu->hwaccm.s.vmx.proc_ctls |= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT;
1826 rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
1827 AssertRC(rc);
1828 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatTSCInterceptOverFlow);
1829 }
1830 }
1831 else
1832 {
1833 pVCpu->hwaccm.s.vmx.proc_ctls |= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT;
1834 rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
1835 AssertRC(rc);
1836 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatTSCIntercept);
1837 }
1838
1839 /* 64 bits guest mode? */
1840 if (CPUMIsGuestInLongModeEx(pCtx))
1841 {
1842#if !defined(VBOX_ENABLE_64_BITS_GUESTS)
1843 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
1844#elif HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
1845 pVCpu->hwaccm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
1846#else
1847# ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
1848 if (!pVM->hwaccm.s.fAllow64BitGuests)
1849 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
1850# endif
1851 pVCpu->hwaccm.s.vmx.pfnStartVM = VMXR0StartVM64;
1852#endif
1853 /* Unconditionally update these as wrmsr might have changed them. */
1854 rc = VMXWriteVMCS64(VMX_VMCS64_GUEST_FS_BASE, pCtx->fsHid.u64Base);
1855 AssertRC(rc);
1856 rc = VMXWriteVMCS64(VMX_VMCS64_GUEST_GS_BASE, pCtx->gsHid.u64Base);
1857 AssertRC(rc);
1858 }
1859 else
1860 {
1861 pVCpu->hwaccm.s.vmx.pfnStartVM = VMXR0StartVM32;
1862 }
1863
1864 vmxR0UpdateExceptionBitmap(pVM, pVCpu, pCtx);
1865
1866#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
1867 /* Store all guest MSRs in the VM-Entry load area, so they will be loaded during the world switch. */
1868 PVMXMSR pMsr = (PVMXMSR)pVCpu->hwaccm.s.vmx.pGuestMSR;
1869 unsigned idxMsr = 0;
1870
1871 uint32_t ulEdx;
1872 uint32_t ulTemp;
1873 CPUMGetGuestCpuId(pVCpu, 0x80000001, &ulTemp, &ulTemp, &ulTemp, &ulEdx);
1874 /* EFER MSR present? */
1875 if (ulEdx & (X86_CPUID_AMD_FEATURE_EDX_NX|X86_CPUID_AMD_FEATURE_EDX_LONG_MODE))
1876 {
1877 pMsr->u32IndexMSR = MSR_K6_EFER;
1878 pMsr->u32Reserved = 0;
1879 pMsr->u64Value = pCtx->msrEFER;
1880 /* VT-x will complain if only MSR_K6_EFER_LME is set. */
1881 if (!CPUMIsGuestInLongModeEx(pCtx))
1882 pMsr->u64Value &= ~(MSR_K6_EFER_LMA|MSR_K6_EFER_LME);
1883 pMsr++; idxMsr++;
1884
1885 if (ulEdx & X86_CPUID_AMD_FEATURE_EDX_LONG_MODE)
1886 {
1887 pMsr->u32IndexMSR = MSR_K8_LSTAR;
1888 pMsr->u32Reserved = 0;
1889 pMsr->u64Value = pCtx->msrLSTAR; /* 64 bits mode syscall rip */
1890 pMsr++; idxMsr++;
1891 pMsr->u32IndexMSR = MSR_K6_STAR;
1892 pMsr->u32Reserved = 0;
1893 pMsr->u64Value = pCtx->msrSTAR; /* legacy syscall eip, cs & ss */
1894 pMsr++; idxMsr++;
1895 pMsr->u32IndexMSR = MSR_K8_SF_MASK;
1896 pMsr->u32Reserved = 0;
1897 pMsr->u64Value = pCtx->msrSFMASK; /* syscall flag mask */
1898 pMsr++; idxMsr++;
1899 pMsr->u32IndexMSR = MSR_K8_KERNEL_GS_BASE;
1900 pMsr->u32Reserved = 0;
1901 pMsr->u64Value = pCtx->msrKERNELGSBASE; /* swapgs exchange value */
1902 pMsr++; idxMsr++;
1903 }
1904 }
1905 pVCpu->hwaccm.s.vmx.cCachedMSRs = idxMsr;
1906
1907 rc = VMXWriteVMCS(VMX_VMCS_CTRL_ENTRY_MSR_LOAD_COUNT, idxMsr);
1908 AssertRC(rc);
1909
1910 rc = VMXWriteVMCS(VMX_VMCS_CTRL_EXIT_MSR_STORE_COUNT, idxMsr);
1911 AssertRC(rc);
1912#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
1913
1914 /* Done. */
1915 pVCpu->hwaccm.s.fContextUseFlags &= ~HWACCM_CHANGED_ALL_GUEST;
1916
1917 return rc;
1918}
1919
1920/**
1921 * Syncs back the guest state
1922 *
1923 * @returns VBox status code.
1924 * @param pVM The VM to operate on.
1925 * @param pVCpu The VMCPU to operate on.
1926 * @param pCtx Guest context
1927 */
1928DECLINLINE(int) VMXR0SaveGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
1929{
1930 RTGCUINTREG val, valShadow;
1931 RTGCUINTPTR uInterruptState;
1932 int rc;
1933
1934 /* Let's first sync back eip, esp, and eflags. */
1935 rc = VMXReadCachedVMCS(VMX_VMCS64_GUEST_RIP, &val);
1936 AssertRC(rc);
1937 pCtx->rip = val;
1938 rc = VMXReadCachedVMCS(VMX_VMCS64_GUEST_RSP, &val);
1939 AssertRC(rc);
1940 pCtx->rsp = val;
1941 rc = VMXReadCachedVMCS(VMX_VMCS_GUEST_RFLAGS, &val);
1942 AssertRC(rc);
1943 pCtx->eflags.u32 = val;
1944
1945 /* Take care of instruction fusing (sti, mov ss) */
1946 rc |= VMXReadCachedVMCS(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &val);
1947 uInterruptState = val;
1948 if (uInterruptState != 0)
1949 {
1950 Assert(uInterruptState <= 2); /* only sti & mov ss */
1951 Log(("uInterruptState %x eip=%RGv\n", (uint32_t)uInterruptState, pCtx->rip));
1952 EMSetInhibitInterruptsPC(pVCpu, pCtx->rip);
1953 }
1954 else
1955 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
1956
1957 /* Control registers. */
1958 VMXReadCachedVMCS(VMX_VMCS_CTRL_CR0_READ_SHADOW, &valShadow);
1959 VMXReadCachedVMCS(VMX_VMCS64_GUEST_CR0, &val);
1960 val = (valShadow & pVCpu->hwaccm.s.vmx.cr0_mask) | (val & ~pVCpu->hwaccm.s.vmx.cr0_mask);
1961 CPUMSetGuestCR0(pVCpu, val);
1962
1963 VMXReadCachedVMCS(VMX_VMCS_CTRL_CR4_READ_SHADOW, &valShadow);
1964 VMXReadCachedVMCS(VMX_VMCS64_GUEST_CR4, &val);
1965 val = (valShadow & pVCpu->hwaccm.s.vmx.cr4_mask) | (val & ~pVCpu->hwaccm.s.vmx.cr4_mask);
1966 CPUMSetGuestCR4(pVCpu, val);
1967
1968 /* Note: no reason to sync back the CRx registers. They can't be changed by the guest. */
1969 /* Note: only in the nested paging case can CR3 & CR4 be changed by the guest. */
1970 if ( pVM->hwaccm.s.fNestedPaging
1971 && CPUMIsGuestInPagedProtectedModeEx(pCtx))
1972 {
1973 PVMCSCACHE pCache = &pVCpu->hwaccm.s.vmx.VMCSCache;
1974
1975 /* Can be updated behind our back in the nested paging case. */
1976 CPUMSetGuestCR2(pVCpu, pCache->cr2);
1977
1978 VMXReadCachedVMCS(VMX_VMCS64_GUEST_CR3, &val);
1979
1980 if (val != pCtx->cr3)
1981 {
1982 CPUMSetGuestCR3(pVCpu, val);
1983 PGMUpdateCR3(pVCpu, val);
1984 }
1985 /* Prefetch the four PDPT entries in PAE mode. */
1986 vmxR0PrefetchPAEPdptrs(pVM, pVCpu, pCtx);
1987 }
1988
1989 /* Sync back DR7 here. */
1990 VMXReadCachedVMCS(VMX_VMCS64_GUEST_DR7, &val);
1991 pCtx->dr[7] = val;
1992
1993 /* Guest CPU context: ES, CS, SS, DS, FS, GS. */
1994 VMX_READ_SELREG(ES, es);
1995 VMX_READ_SELREG(SS, ss);
1996 VMX_READ_SELREG(CS, cs);
1997 VMX_READ_SELREG(DS, ds);
1998 VMX_READ_SELREG(FS, fs);
1999 VMX_READ_SELREG(GS, gs);
2000
2001 /*
2002 * System MSRs
2003 */
2004 VMXReadCachedVMCS(VMX_VMCS32_GUEST_SYSENTER_CS, &val);
2005 pCtx->SysEnter.cs = val;
2006 VMXReadCachedVMCS(VMX_VMCS64_GUEST_SYSENTER_EIP, &val);
2007 pCtx->SysEnter.eip = val;
2008 VMXReadCachedVMCS(VMX_VMCS64_GUEST_SYSENTER_ESP, &val);
2009 pCtx->SysEnter.esp = val;
2010
2011 /* Misc. registers; must sync everything otherwise we can get out of sync when jumping to ring 3. */
2012 VMX_READ_SELREG(LDTR, ldtr);
2013
2014 VMXReadCachedVMCS(VMX_VMCS32_GUEST_GDTR_LIMIT, &val);
2015 pCtx->gdtr.cbGdt = val;
2016 VMXReadCachedVMCS(VMX_VMCS64_GUEST_GDTR_BASE, &val);
2017 pCtx->gdtr.pGdt = val;
2018
2019 VMXReadCachedVMCS(VMX_VMCS32_GUEST_IDTR_LIMIT, &val);
2020 pCtx->idtr.cbIdt = val;
2021 VMXReadCachedVMCS(VMX_VMCS64_GUEST_IDTR_BASE, &val);
2022 pCtx->idtr.pIdt = val;
2023
2024#ifdef HWACCM_VMX_EMULATE_REALMODE
2025 /* Real mode emulation using v86 mode. */
2026 if ( CPUMIsGuestInRealModeEx(pCtx)
2027 && pVM->hwaccm.s.vmx.pRealModeTSS)
2028 {
2029 /* Hide our emulation flags */
2030 pCtx->eflags.Bits.u1VM = 0;
2031
2032 /* Restore original IOPL setting as we always use 0. */
2033 pCtx->eflags.Bits.u2IOPL = pVCpu->hwaccm.s.vmx.RealMode.eflags.Bits.u2IOPL;
2034
2035 /* Force a TR resync every time in case we switch modes. */
2036 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_GUEST_TR;
2037 }
2038 else
2039#endif /* HWACCM_VMX_EMULATE_REALMODE */
2040 {
2041 /* In real mode we have a fake TSS, so only sync it back when it's supposed to be valid. */
2042 VMX_READ_SELREG(TR, tr);
2043 }
2044
2045#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
2046 /* Save the possibly changed MSRs that we automatically restore and save during a world switch. */
2047 for (unsigned i = 0; i < pVCpu->hwaccm.s.vmx.cCachedMSRs; i++)
2048 {
2049 PVMXMSR pMsr = (PVMXMSR)pVCpu->hwaccm.s.vmx.pGuestMSR;
2050 pMsr += i;
2051
2052 switch (pMsr->u32IndexMSR)
2053 {
2054 case MSR_K8_LSTAR:
2055 pCtx->msrLSTAR = pMsr->u64Value;
2056 break;
2057 case MSR_K6_STAR:
2058 pCtx->msrSTAR = pMsr->u64Value;
2059 break;
2060 case MSR_K8_SF_MASK:
2061 pCtx->msrSFMASK = pMsr->u64Value;
2062 break;
2063 case MSR_K8_KERNEL_GS_BASE:
2064 pCtx->msrKERNELGSBASE = pMsr->u64Value;
2065 break;
2066 case MSR_K6_EFER:
2067 /* EFER can't be changed without causing a VM-exit. */
2068// Assert(pCtx->msrEFER == pMsr->u64Value);
2069 break;
2070 default:
2071 AssertFailed();
2072 return VERR_INTERNAL_ERROR;
2073 }
2074 }
2075#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
2076 return VINF_SUCCESS;
2077}
2078
2079/**
2080 * Dummy placeholder
2081 *
2082 * @param pVM The VM to operate on.
2083 * @param pVCpu The VMCPU to operate on.
2084 */
2085static void vmxR0SetupTLBDummy(PVM pVM, PVMCPU pVCpu)
2086{
2087 NOREF(pVM);
2088 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
2089 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
2090 pVCpu->hwaccm.s.TlbShootdown.cPages = 0;
2091 return;
2092}
2093
2094/**
2095 * Setup the tagged TLB for EPT
2096 *
2097 * @returns VBox status code.
2098 * @param pVM The VM to operate on.
2099 * @param pVCpu The VMCPU to operate on.
2100 */
2101static void vmxR0SetupTLBEPT(PVM pVM, PVMCPU pVCpu)
2102{
2103 PHWACCM_CPUINFO pCpu;
2104
2105 Assert(pVM->hwaccm.s.fNestedPaging);
2106 Assert(!pVM->hwaccm.s.vmx.fVPID);
2107
2108 /* Deal with tagged TLBs if VPID or EPT is supported. */
2109 pCpu = HWACCMR0GetCurrentCpu();
2110 /* Force a TLB flush for the first world switch if the current cpu differs from the one we ran on last. */
2111 /* Note that this can happen both for start and resume due to long jumps back to ring 3. */
2112 if ( pVCpu->hwaccm.s.idLastCpu != pCpu->idCpu
2113 /* if the tlb flush count has changed, another VM has flushed the TLB of this cpu, so we can't use our current ASID anymore. */
2114 || pVCpu->hwaccm.s.cTLBFlushes != pCpu->cTLBFlushes)
2115 {
2116 /* Force a TLB flush on VM entry. */
2117 pVCpu->hwaccm.s.fForceTLBFlush = true;
2118 }
2119 else
2120 Assert(!pCpu->fFlushTLB);
2121
2122 /* Check for tlb shootdown flushes. */
2123 if (VMCPU_FF_TESTANDCLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2124 pVCpu->hwaccm.s.fForceTLBFlush = true;
2125
2126 pVCpu->hwaccm.s.idLastCpu = pCpu->idCpu;
2127 pCpu->fFlushTLB = false;
2128
2129 if (pVCpu->hwaccm.s.fForceTLBFlush)
2130 {
2131 vmxR0FlushEPT(pVM, pVCpu, pVM->hwaccm.s.vmx.enmFlushContext, 0);
2132 }
2133 else
2134 if (VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
2135 {
2136 /* Deal with pending TLB shootdown actions which were queued when we were not executing code. */
2137 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatTlbShootdown);
2138
2139 for (unsigned i=0;i<pVCpu->hwaccm.s.TlbShootdown.cPages;i++)
2140 {
2141 /* aTlbShootdownPages contains physical addresses in this case. */
2142 vmxR0FlushEPT(pVM, pVCpu, pVM->hwaccm.s.vmx.enmFlushPage, pVCpu->hwaccm.s.TlbShootdown.aPages[i]);
2143 }
2144 }
2145 pVCpu->hwaccm.s.TlbShootdown.cPages= 0;
2146 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
2147
2148#ifdef VBOX_WITH_STATISTICS
2149 if (pVCpu->hwaccm.s.fForceTLBFlush)
2150 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatFlushTLBWorldSwitch);
2151 else
2152 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatNoFlushTLBWorldSwitch);
2153#endif
2154}
2155
2156#ifdef HWACCM_VTX_WITH_VPID
2157/**
2158 * Setup the tagged TLB for VPID
2159 *
2160 * @returns VBox status code.
2161 * @param pVM The VM to operate on.
2162 * @param pVCpu The VMCPU to operate on.
2163 */
2164static void vmxR0SetupTLBVPID(PVM pVM, PVMCPU pVCpu)
2165{
2166 PHWACCM_CPUINFO pCpu;
2167
2168 Assert(pVM->hwaccm.s.vmx.fVPID);
2169 Assert(!pVM->hwaccm.s.fNestedPaging);
2170
2171 /* Deal with tagged TLBs if VPID or EPT is supported. */
2172 pCpu = HWACCMR0GetCurrentCpu();
2173 /* Force a TLB flush for the first world switch if the current cpu differs from the one we ran on last. */
2174 /* Note that this can happen both for start and resume due to long jumps back to ring 3. */
2175 if ( pVCpu->hwaccm.s.idLastCpu != pCpu->idCpu
2176 /* if the tlb flush count has changed, another VM has flushed the TLB of this cpu, so we can't use our current ASID anymore. */
2177 || pVCpu->hwaccm.s.cTLBFlushes != pCpu->cTLBFlushes)
2178 {
2179 /* Force a TLB flush on VM entry. */
2180 pVCpu->hwaccm.s.fForceTLBFlush = true;
2181 }
2182 else
2183 Assert(!pCpu->fFlushTLB);
2184
2185 pVCpu->hwaccm.s.idLastCpu = pCpu->idCpu;
2186
2187 /* Check for tlb shootdown flushes. */
2188 if (VMCPU_FF_TESTANDCLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2189 pVCpu->hwaccm.s.fForceTLBFlush = true;
2190
2191 /* Make sure we flush the TLB when required. Switch ASID to achieve the same thing, but without actually flushing the whole TLB (which is expensive). */
2192 if (pVCpu->hwaccm.s.fForceTLBFlush)
2193 {
2194 if ( ++pCpu->uCurrentASID >= pVM->hwaccm.s.uMaxASID
2195 || pCpu->fFlushTLB)
2196 {
2197 pCpu->fFlushTLB = false;
2198 pCpu->uCurrentASID = 1; /* start at 1; host uses 0 */
2199 pCpu->cTLBFlushes++;
2200 vmxR0FlushVPID(pVM, pVCpu, VMX_FLUSH_ALL_CONTEXTS, 0);
2201 }
2202 else
2203 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatFlushASID);
2204
2205 pVCpu->hwaccm.s.fForceTLBFlush = false;
2206 pVCpu->hwaccm.s.cTLBFlushes = pCpu->cTLBFlushes;
2207 pVCpu->hwaccm.s.uCurrentASID = pCpu->uCurrentASID;
2208 }
2209 else
2210 {
2211 Assert(!pCpu->fFlushTLB);
2212 Assert(pVCpu->hwaccm.s.uCurrentASID && pCpu->uCurrentASID);
2213
2214 if (VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
2215 {
2216 /* Deal with pending TLB shootdown actions which were queued when we were not executing code. */
2217 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatTlbShootdown);
2218 for (unsigned i=0;i<pVCpu->hwaccm.s.TlbShootdown.cPages;i++)
2219 vmxR0FlushVPID(pVM, pVCpu, pVM->hwaccm.s.vmx.enmFlushPage, pVCpu->hwaccm.s.TlbShootdown.aPages[i]);
2220 }
2221 }
2222 pVCpu->hwaccm.s.TlbShootdown.cPages = 0;
2223 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
2224
2225 AssertMsg(pVCpu->hwaccm.s.cTLBFlushes == pCpu->cTLBFlushes, ("Flush count mismatch for cpu %d (%x vs %x)\n", pCpu->idCpu, pVCpu->hwaccm.s.cTLBFlushes, pCpu->cTLBFlushes));
2226 AssertMsg(pCpu->uCurrentASID >= 1 && pCpu->uCurrentASID < pVM->hwaccm.s.uMaxASID, ("cpu%d uCurrentASID = %x\n", pCpu->idCpu, pCpu->uCurrentASID));
2227 AssertMsg(pVCpu->hwaccm.s.uCurrentASID >= 1 && pVCpu->hwaccm.s.uCurrentASID < pVM->hwaccm.s.uMaxASID, ("cpu%d VM uCurrentASID = %x\n", pCpu->idCpu, pVCpu->hwaccm.s.uCurrentASID));
2228
2229 int rc = VMXWriteVMCS(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hwaccm.s.uCurrentASID);
2230 AssertRC(rc);
2231
2232 if (pVCpu->hwaccm.s.fForceTLBFlush)
2233 vmxR0FlushVPID(pVM, pVCpu, pVM->hwaccm.s.vmx.enmFlushContext, 0);
2234
2235#ifdef VBOX_WITH_STATISTICS
2236 if (pVCpu->hwaccm.s.fForceTLBFlush)
2237 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatFlushTLBWorldSwitch);
2238 else
2239 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatNoFlushTLBWorldSwitch);
2240#endif
2241}
2242#endif /* HWACCM_VTX_WITH_VPID */
2243
2244/**
2245 * Runs guest code in a VT-x VM.
2246 *
2247 * @returns VBox status code.
2248 * @param pVM The VM to operate on.
2249 * @param pVCpu The VMCPU to operate on.
2250 * @param pCtx Guest context
2251 */
2252VMMR0DECL(int) VMXR0RunGuestCode(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
2253{
2254 int rc = VINF_SUCCESS;
2255 RTGCUINTREG val;
2256 RTGCUINTREG exitReason = (RTGCUINTREG)VMX_EXIT_INVALID;
2257 RTGCUINTREG instrError, cbInstr;
2258 RTGCUINTPTR exitQualification = 0;
2259 RTGCUINTPTR intInfo = 0; /* shut up buggy gcc 4 */
2260 RTGCUINTPTR errCode, instrInfo;
2261 bool fSetupTPRCaching = false;
2262 uint64_t u64OldLSTAR = 0;
2263 uint8_t u8LastTPR = 0;
2264 RTCCUINTREG uOldEFlags = ~(RTCCUINTREG)0;
2265 unsigned cResume = 0;
2266#ifdef VBOX_STRICT
2267 RTCPUID idCpuCheck;
2268 bool fWasInLongMode = false;
2269#endif
2270#ifdef VBOX_HIGH_RES_TIMERS_HACK_IN_RING0
2271 uint64_t u64LastTime = RTTimeMilliTS();
2272#endif
2273#ifdef VBOX_WITH_STATISTICS
2274 bool fStatEntryStarted = true;
2275 bool fStatExit2Started = false;
2276#endif
2277
2278 Assert(!(pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC) || (pVCpu->hwaccm.s.vmx.pVAPIC && pVM->hwaccm.s.vmx.pAPIC));
2279
2280 /* Check if we need to use TPR shadowing. */
2281 if ( CPUMIsGuestInLongModeEx(pCtx)
2282 || ( ((pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC) || pVM->hwaccm.s.fTRPPatchingAllowed)
2283 && pVM->hwaccm.s.fHasIoApic)
2284 )
2285 {
2286 fSetupTPRCaching = true;
2287 }
2288
2289 Log2(("\nE"));
2290
2291 STAM_PROFILE_ADV_START(&pVCpu->hwaccm.s.StatEntry, x);
2292
2293#ifdef VBOX_STRICT
2294 {
2295 RTCCUINTREG val2;
2296
2297 rc = VMXReadVMCS(VMX_VMCS_CTRL_PIN_EXEC_CONTROLS, &val2);
2298 AssertRC(rc);
2299 Log2(("VMX_VMCS_CTRL_PIN_EXEC_CONTROLS = %08x\n", val2));
2300
2301 /* allowed zero */
2302 if ((val2 & pVM->hwaccm.s.vmx.msr.vmx_pin_ctls.n.disallowed0) != pVM->hwaccm.s.vmx.msr.vmx_pin_ctls.n.disallowed0)
2303 Log(("Invalid VMX_VMCS_CTRL_PIN_EXEC_CONTROLS: zero\n"));
2304
2305 /* allowed one */
2306 if ((val2 & ~pVM->hwaccm.s.vmx.msr.vmx_pin_ctls.n.allowed1) != 0)
2307 Log(("Invalid VMX_VMCS_CTRL_PIN_EXEC_CONTROLS: one\n"));
2308
2309 rc = VMXReadVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, &val2);
2310 AssertRC(rc);
2311 Log2(("VMX_VMCS_CTRL_PROC_EXEC_CONTROLS = %08x\n", val2));
2312
2313 /* Must be set according to the MSR, but can be cleared in case of EPT. */
2314 if (pVM->hwaccm.s.fNestedPaging)
2315 val2 |= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INVLPG_EXIT
2316 | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
2317 | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT;
2318
2319 /* allowed zero */
2320 if ((val2 & pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.disallowed0) != pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.disallowed0)
2321 Log(("Invalid VMX_VMCS_CTRL_PROC_EXEC_CONTROLS: zero\n"));
2322
2323 /* allowed one */
2324 if ((val2 & ~pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1) != 0)
2325 Log(("Invalid VMX_VMCS_CTRL_PROC_EXEC_CONTROLS: one\n"));
2326
2327 rc = VMXReadVMCS(VMX_VMCS_CTRL_ENTRY_CONTROLS, &val2);
2328 AssertRC(rc);
2329 Log2(("VMX_VMCS_CTRL_ENTRY_CONTROLS = %08x\n", val2));
2330
2331 /* allowed zero */
2332 if ((val2 & pVM->hwaccm.s.vmx.msr.vmx_entry.n.disallowed0) != pVM->hwaccm.s.vmx.msr.vmx_entry.n.disallowed0)
2333 Log(("Invalid VMX_VMCS_CTRL_ENTRY_CONTROLS: zero\n"));
2334
2335 /* allowed one */
2336 if ((val2 & ~pVM->hwaccm.s.vmx.msr.vmx_entry.n.allowed1) != 0)
2337 Log(("Invalid VMX_VMCS_CTRL_ENTRY_CONTROLS: one\n"));
2338
2339 rc = VMXReadVMCS(VMX_VMCS_CTRL_EXIT_CONTROLS, &val2);
2340 AssertRC(rc);
2341 Log2(("VMX_VMCS_CTRL_EXIT_CONTROLS = %08x\n", val2));
2342
2343 /* allowed zero */
2344 if ((val2 & pVM->hwaccm.s.vmx.msr.vmx_exit.n.disallowed0) != pVM->hwaccm.s.vmx.msr.vmx_exit.n.disallowed0)
2345 Log(("Invalid VMX_VMCS_CTRL_EXIT_CONTROLS: zero\n"));
2346
2347 /* allowed one */
2348 if ((val2 & ~pVM->hwaccm.s.vmx.msr.vmx_exit.n.allowed1) != 0)
2349 Log(("Invalid VMX_VMCS_CTRL_EXIT_CONTROLS: one\n"));
2350 }
2351 fWasInLongMode = CPUMIsGuestInLongModeEx(pCtx);
2352#endif /* VBOX_STRICT */
2353
2354#ifdef VBOX_WITH_CRASHDUMP_MAGIC
2355 pVCpu->hwaccm.s.vmx.VMCSCache.u64TimeEntry = RTTimeNanoTS();
2356#endif
2357
2358 /* We can jump to this point to resume execution after determining that a VM-exit is innocent.
2359 */
2360ResumeExecution:
2361 STAM_STATS({
2362 if (fStatExit2Started) { STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2, y); fStatExit2Started = false; }
2363 if (!fStatEntryStarted) { STAM_PROFILE_ADV_START(&pVCpu->hwaccm.s.StatEntry, x); fStatEntryStarted = true; }
2364 });
2365 AssertMsg(pVCpu->hwaccm.s.idEnteredCpu == RTMpCpuId(),
2366 ("Expected %d, I'm %d; cResume=%d exitReason=%RGv exitQualification=%RGv\n",
2367 (int)pVCpu->hwaccm.s.idEnteredCpu, (int)RTMpCpuId(), cResume, exitReason, exitQualification));
2368 Assert(!HWACCMR0SuspendPending());
2369 /* Not allowed to switch modes without reloading the host state (32->64 switcher)!! */
2370 Assert(fWasInLongMode == CPUMIsGuestInLongModeEx(pCtx));
2371
2372 /* Safety precaution; looping for too long here can have a very bad effect on the host */
2373 if (RT_UNLIKELY(++cResume > pVM->hwaccm.s.cMaxResumeLoops))
2374 {
2375 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitMaxResume);
2376 rc = VINF_EM_RAW_INTERRUPT;
2377 goto end;
2378 }
2379
2380 /* Check for irq inhibition due to instruction fusing (sti, mov ss). */
2381 if (VMCPU_FF_ISSET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
2382 {
2383 Log(("VM_FF_INHIBIT_INTERRUPTS at %RGv successor %RGv\n", (RTGCPTR)pCtx->rip, EMGetInhibitInterruptsPC(pVCpu)));
2384 if (pCtx->rip != EMGetInhibitInterruptsPC(pVCpu))
2385 {
2386 /* Note: we intentionally don't clear VM_FF_INHIBIT_INTERRUPTS here.
2387 * Before we are able to execute this instruction in raw mode (iret to guest code) an external interrupt might
2388 * force a world switch again. Possibly allowing a guest interrupt to be dispatched in the process. This could
2389 * break the guest. Sounds very unlikely, but such timing sensitive problems are not as rare as you might think.
2390 */
2391 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
2392 /* Irq inhibition is no longer active; clear the corresponding VMX state. */
2393 rc = VMXWriteVMCS(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, 0);
2394 AssertRC(rc);
2395 }
2396 }
2397 else
2398 {
2399 /* Irq inhibition is no longer active; clear the corresponding VMX state. */
2400 rc = VMXWriteVMCS(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, 0);
2401 AssertRC(rc);
2402 }
2403
2404#ifdef VBOX_HIGH_RES_TIMERS_HACK_IN_RING0
2405 if (RT_UNLIKELY(cResume & 0xf) == 0)
2406 {
2407 uint64_t u64CurTime = RTTimeMilliTS();
2408
2409 if (RT_UNLIKELY(u64CurTime > u64LastTime))
2410 {
2411 u64LastTime = u64CurTime;
2412 TMTimerPollVoid(pVM, pVCpu);
2413 }
2414 }
2415#endif
2416
2417 /* Check for pending actions that force us to go back to ring 3. */
2418 if ( VM_FF_ISPENDING(pVM, VM_FF_HWACCM_TO_R3_MASK | VM_FF_REQUEST | VM_FF_PGM_POOL_FLUSH_PENDING)
2419 || VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_HWACCM_TO_R3_MASK | VMCPU_FF_PGM_SYNC_CR3 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL | VMCPU_FF_REQUEST))
2420 {
2421 /* Check if a sync operation is pending. */
2422 if (VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
2423 {
2424 rc = PGMSyncCR3(pVCpu, pCtx->cr0, pCtx->cr3, pCtx->cr4, VMCPU_FF_ISSET(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
2425 AssertRC(rc);
2426 if (rc != VINF_SUCCESS)
2427 {
2428 Log(("Pending pool sync is forcing us back to ring 3; rc=%d\n", rc));
2429 goto end;
2430 }
2431 }
2432
2433#ifdef DEBUG
2434 /* Intercept X86_XCPT_DB if stepping is enabled */
2435 if (!DBGFIsStepping(pVCpu))
2436#endif
2437 {
2438 if ( VM_FF_ISPENDING(pVM, VM_FF_HWACCM_TO_R3_MASK)
2439 || VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_HWACCM_TO_R3_MASK))
2440 {
2441 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
2442 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatSwitchToR3);
2443 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatEntry, x);
2444 rc = RT_UNLIKELY(VM_FF_ISPENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
2445 goto end;
2446 }
2447 }
2448
2449 /* Pending request packets might contain actions that need immediate attention, such as pending hardware interrupts. */
2450 if ( VM_FF_ISPENDING(pVM, VM_FF_REQUEST)
2451 || VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_REQUEST))
2452 {
2453 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatEntry, x);
2454 rc = VINF_EM_PENDING_REQUEST;
2455 goto end;
2456 }
2457
2458 /* Check if a pgm pool flush is in progress. */
2459 if (VM_FF_ISPENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
2460 {
2461 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatEntry, x);
2462 rc = VINF_PGM_POOL_FLUSH_PENDING;
2463 goto end;
2464 }
2465 }
2466
2467#ifdef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
2468 /*
2469 * Exit to ring-3 preemption/work is pending.
2470 *
2471 * Interrupts are disabled before the call to make sure we don't miss any interrupt
2472 * that would flag preemption (IPI, timer tick, ++). (Would've been nice to do this
2473 * further down, but VMXR0CheckPendingInterrupt makes that impossible.)
2474 *
2475 * Note! Interrupts must be disabled done *before* we check for TLB flushes; TLB
2476 * shootdowns rely on this.
2477 */
2478 uOldEFlags = ASMIntDisableFlags();
2479 if (RTThreadPreemptIsPending(NIL_RTTHREAD))
2480 {
2481 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitPreemptPending);
2482 rc = VINF_EM_RAW_INTERRUPT;
2483 goto end;
2484 }
2485 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
2486#endif
2487
2488 /* When external interrupts are pending, we should exit the VM when IF is set. */
2489 /* Note! *After* VM_FF_INHIBIT_INTERRUPTS check!!! */
2490 rc = VMXR0CheckPendingInterrupt(pVM, pVCpu, pCtx);
2491 if (RT_FAILURE(rc))
2492 goto end;
2493
2494 /** @todo check timers?? */
2495
2496 /* TPR caching using CR8 is only available in 64 bits mode */
2497 /* Note the 32 bits exception for AMD (X86_CPUID_AMD_FEATURE_ECX_CR8L), but that appears missing in Intel CPUs */
2498 /* Note: we can't do this in LoadGuestState as PDMApicGetTPR can jump back to ring 3 (lock)!!!!! (no longer true) */
2499 /**
2500 * @todo query and update the TPR only when it could have been changed (mmio access & wrmsr (x2apic))
2501 */
2502 if (fSetupTPRCaching)
2503 {
2504 /* TPR caching in CR8 */
2505 bool fPending;
2506
2507 int rc2 = PDMApicGetTPR(pVCpu, &u8LastTPR, &fPending);
2508 AssertRC(rc2);
2509 /* The TPR can be found at offset 0x80 in the APIC mmio page. */
2510 pVCpu->hwaccm.s.vmx.pVAPIC[0x80] = u8LastTPR;
2511
2512 /* Two options here:
2513 * - external interrupt pending, but masked by the TPR value.
2514 * -> a CR8 update that lower the current TPR value should cause an exit
2515 * - no pending interrupts
2516 * -> We don't need to be explicitely notified. There are enough world switches for detecting pending interrupts.
2517 */
2518 rc = VMXWriteVMCS(VMX_VMCS_CTRL_TPR_THRESHOLD, (fPending) ? (u8LastTPR >> 4) : 0); /* cr8 bits 3-0 correspond to bits 7-4 of the task priority mmio register. */
2519 AssertRC(rc);
2520
2521 if (pVM->hwaccm.s.fTPRPatchingActive)
2522 {
2523 Assert(!CPUMIsGuestInLongModeEx(pCtx));
2524 /* Our patch code uses LSTAR for TPR caching. */
2525 pCtx->msrLSTAR = u8LastTPR;
2526
2527 if (fPending)
2528 {
2529 /* A TPR change could activate a pending interrupt, so catch lstar writes. */
2530 vmxR0SetMSRPermission(pVCpu, MSR_K8_LSTAR, true, false);
2531 }
2532 else
2533 {
2534 /* No interrupts are pending, so we don't need to be explicitely notified.
2535 * There are enough world switches for detecting pending interrupts.
2536 */
2537 vmxR0SetMSRPermission(pVCpu, MSR_K8_LSTAR, true, true);
2538 }
2539 }
2540 }
2541
2542#if defined(HWACCM_VTX_WITH_EPT) && defined(LOG_ENABLED)
2543 if ( pVM->hwaccm.s.fNestedPaging
2544# ifdef HWACCM_VTX_WITH_VPID
2545 || pVM->hwaccm.s.vmx.fVPID
2546# endif /* HWACCM_VTX_WITH_VPID */
2547 )
2548 {
2549 PHWACCM_CPUINFO pCpu;
2550
2551 pCpu = HWACCMR0GetCurrentCpu();
2552 if ( pVCpu->hwaccm.s.idLastCpu != pCpu->idCpu
2553 || pVCpu->hwaccm.s.cTLBFlushes != pCpu->cTLBFlushes)
2554 {
2555 if (pVCpu->hwaccm.s.idLastCpu != pCpu->idCpu)
2556 LogFlow(("Force TLB flush due to rescheduling to a different cpu (%d vs %d)\n", pVCpu->hwaccm.s.idLastCpu, pCpu->idCpu));
2557 else
2558 LogFlow(("Force TLB flush due to changed TLB flush count (%x vs %x)\n", pVCpu->hwaccm.s.cTLBFlushes, pCpu->cTLBFlushes));
2559 }
2560 if (pCpu->fFlushTLB)
2561 LogFlow(("Force TLB flush: first time cpu %d is used -> flush\n", pCpu->idCpu));
2562 else
2563 if (pVCpu->hwaccm.s.fForceTLBFlush)
2564 LogFlow(("Manual TLB flush\n"));
2565 }
2566#endif
2567#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
2568 PGMDynMapFlushAutoSet(pVCpu);
2569#endif
2570
2571 /*
2572 * NOTE: DO NOT DO ANYTHING AFTER THIS POINT THAT MIGHT JUMP BACK TO RING 3!
2573 * (until the actual world switch)
2574 */
2575#ifdef VBOX_STRICT
2576 idCpuCheck = RTMpCpuId();
2577#endif
2578#ifdef LOG_ENABLED
2579 VMMR0LogFlushDisable(pVCpu);
2580#endif
2581 /* Save the host state first. */
2582 rc = VMXR0SaveHostState(pVM, pVCpu);
2583 if (RT_UNLIKELY(rc != VINF_SUCCESS))
2584 {
2585 VMMR0LogFlushEnable(pVCpu);
2586 goto end;
2587 }
2588 /* Load the guest state */
2589 rc = VMXR0LoadGuestState(pVM, pVCpu, pCtx);
2590 if (RT_UNLIKELY(rc != VINF_SUCCESS))
2591 {
2592 VMMR0LogFlushEnable(pVCpu);
2593 goto end;
2594 }
2595
2596#ifndef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
2597 /* Disable interrupts to make sure a poke will interrupt execution.
2598 * This must be done *before* we check for TLB flushes; TLB shootdowns rely on this.
2599 */
2600 uOldEFlags = ASMIntDisableFlags();
2601 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
2602#endif
2603
2604 /* Non-register state Guest Context */
2605 /** @todo change me according to cpu state */
2606 rc = VMXWriteVMCS(VMX_VMCS32_GUEST_ACTIVITY_STATE, VMX_CMS_GUEST_ACTIVITY_ACTIVE);
2607 AssertRC(rc);
2608
2609 /** Set TLB flush state as checked until we return from the world switch. */
2610 ASMAtomicWriteU8(&pVCpu->hwaccm.s.fCheckedTLBFlush, true);
2611 /* Deal with tagged TLB setup and invalidation. */
2612 pVM->hwaccm.s.vmx.pfnSetupTaggedTLB(pVM, pVCpu);
2613
2614 STAM_STATS({ STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatEntry, x); fStatEntryStarted = false; });
2615
2616 /* Manual save and restore:
2617 * - General purpose registers except RIP, RSP
2618 *
2619 * Trashed:
2620 * - CR2 (we don't care)
2621 * - LDTR (reset to 0)
2622 * - DRx (presumably not changed at all)
2623 * - DR7 (reset to 0x400)
2624 * - EFLAGS (reset to RT_BIT(1); not relevant)
2625 *
2626 */
2627
2628 /* All done! Let's start VM execution. */
2629 STAM_PROFILE_ADV_START(&pVCpu->hwaccm.s.StatInGC, z);
2630 Assert(idCpuCheck == RTMpCpuId());
2631
2632#ifdef VBOX_WITH_CRASHDUMP_MAGIC
2633 pVCpu->hwaccm.s.vmx.VMCSCache.cResume = cResume;
2634 pVCpu->hwaccm.s.vmx.VMCSCache.u64TimeSwitch = RTTimeNanoTS();
2635#endif
2636
2637 /* Save the current TPR value in the LSTAR msr so our patches can access it. */
2638 if (pVM->hwaccm.s.fTPRPatchingActive)
2639 {
2640 Assert(pVM->hwaccm.s.fTPRPatchingActive);
2641 u64OldLSTAR = ASMRdMsr(MSR_K8_LSTAR);
2642 ASMWrMsr(MSR_K8_LSTAR, u8LastTPR);
2643 }
2644
2645 TMNotifyStartOfExecution(pVCpu);
2646#ifdef VBOX_WITH_KERNEL_USING_XMM
2647 rc = hwaccmR0VMXStartVMWrapXMM(pVCpu->hwaccm.s.fResumeVM, pCtx, &pVCpu->hwaccm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hwaccm.s.vmx.pfnStartVM);
2648#else
2649 rc = pVCpu->hwaccm.s.vmx.pfnStartVM(pVCpu->hwaccm.s.fResumeVM, pCtx, &pVCpu->hwaccm.s.vmx.VMCSCache, pVM, pVCpu);
2650#endif
2651 ASMAtomicWriteU8(&pVCpu->hwaccm.s.fCheckedTLBFlush, false);
2652 ASMAtomicIncU32(&pVCpu->hwaccm.s.cWorldSwitchExit);
2653 /* Possibly the last TSC value seen by the guest (too high) (only when we're in tsc offset mode). */
2654 if (!(pVCpu->hwaccm.s.vmx.proc_ctls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT))
2655 TMCpuTickSetLastSeen(pVCpu, ASMReadTSC() + pVCpu->hwaccm.s.vmx.u64TSCOffset - 0x400 /* guestimate of world switch overhead in clock ticks */);
2656
2657 TMNotifyEndOfExecution(pVCpu);
2658 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED);
2659 Assert(!(ASMGetFlags() & X86_EFL_IF));
2660
2661 /* Restore the host LSTAR msr if the guest could have changed it. */
2662 if (pVM->hwaccm.s.fTPRPatchingActive)
2663 {
2664 Assert(pVM->hwaccm.s.fTPRPatchingActive);
2665 pVCpu->hwaccm.s.vmx.pVAPIC[0x80] = pCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR);
2666 ASMWrMsr(MSR_K8_LSTAR, u64OldLSTAR);
2667 }
2668
2669 ASMSetFlags(uOldEFlags);
2670#ifdef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
2671 uOldEFlags = ~(RTCCUINTREG)0;
2672#endif
2673
2674 AssertMsg(!pVCpu->hwaccm.s.vmx.VMCSCache.Write.cValidEntries, ("pVCpu->hwaccm.s.vmx.VMCSCache.Write.cValidEntries=%d\n", pVCpu->hwaccm.s.vmx.VMCSCache.Write.cValidEntries));
2675
2676 /* In case we execute a goto ResumeExecution later on. */
2677 pVCpu->hwaccm.s.fResumeVM = true;
2678 pVCpu->hwaccm.s.fForceTLBFlush = false;
2679
2680 /*
2681 * !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
2682 * IMPORTANT: WE CAN'T DO ANY LOGGING OR OPERATIONS THAT CAN DO A LONGJMP BACK TO RING 3 *BEFORE* WE'VE SYNCED BACK (MOST OF) THE GUEST STATE
2683 * !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
2684 */
2685 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatInGC, z);
2686 STAM_PROFILE_ADV_START(&pVCpu->hwaccm.s.StatExit1, v);
2687
2688 if (RT_UNLIKELY(rc != VINF_SUCCESS))
2689 {
2690 VMXR0ReportWorldSwitchError(pVM, pVCpu, rc, pCtx);
2691 VMMR0LogFlushEnable(pVCpu);
2692 goto end;
2693 }
2694
2695 /* Success. Query the guest state and figure out what has happened. */
2696
2697 /* Investigate why there was a VM-exit. */
2698 rc = VMXReadCachedVMCS(VMX_VMCS32_RO_EXIT_REASON, &exitReason);
2699 STAM_COUNTER_INC(&pVCpu->hwaccm.s.paStatExitReasonR0[exitReason & MASK_EXITREASON_STAT]);
2700
2701 exitReason &= 0xffff; /* bit 0-15 contain the exit code. */
2702 rc |= VMXReadCachedVMCS(VMX_VMCS32_RO_VM_INSTR_ERROR, &instrError);
2703 rc |= VMXReadCachedVMCS(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &cbInstr);
2704 rc |= VMXReadCachedVMCS(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &intInfo);
2705 /* might not be valid; depends on VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID. */
2706 rc |= VMXReadCachedVMCS(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERRCODE, &errCode);
2707 rc |= VMXReadCachedVMCS(VMX_VMCS32_RO_EXIT_INSTR_INFO, &instrInfo);
2708 rc |= VMXReadCachedVMCS(VMX_VMCS_RO_EXIT_QUALIFICATION, &exitQualification);
2709 AssertRC(rc);
2710
2711 /* Sync back the guest state */
2712 rc = VMXR0SaveGuestState(pVM, pVCpu, pCtx);
2713 AssertRC(rc);
2714
2715 /* Note! NOW IT'S SAFE FOR LOGGING! */
2716 VMMR0LogFlushEnable(pVCpu);
2717 Log2(("Raw exit reason %08x\n", exitReason));
2718
2719 /* Check if an injected event was interrupted prematurely. */
2720 rc = VMXReadCachedVMCS(VMX_VMCS32_RO_IDT_INFO, &val);
2721 AssertRC(rc);
2722 pVCpu->hwaccm.s.Event.intInfo = VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(val);
2723 if ( VMX_EXIT_INTERRUPTION_INFO_VALID(pVCpu->hwaccm.s.Event.intInfo)
2724 /* Ignore 'int xx' as they'll be restarted anyway. */
2725 && VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hwaccm.s.Event.intInfo) != VMX_EXIT_INTERRUPTION_INFO_TYPE_SW
2726 /* Ignore software exceptions (such as int3) as they'll reoccur when we restart the instruction anyway. */
2727 && VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hwaccm.s.Event.intInfo) != VMX_EXIT_INTERRUPTION_INFO_TYPE_SWEXCPT)
2728 {
2729 Assert(!pVCpu->hwaccm.s.Event.fPending);
2730 pVCpu->hwaccm.s.Event.fPending = true;
2731 /* Error code present? */
2732 if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(pVCpu->hwaccm.s.Event.intInfo))
2733 {
2734 rc = VMXReadCachedVMCS(VMX_VMCS32_RO_IDT_ERRCODE, &val);
2735 AssertRC(rc);
2736 pVCpu->hwaccm.s.Event.errCode = val;
2737 Log(("Pending inject %RX64 at %RGv exit=%08x intInfo=%08x exitQualification=%RGv pending error=%RX64\n", pVCpu->hwaccm.s.Event.intInfo, (RTGCPTR)pCtx->rip, exitReason, intInfo, exitQualification, val));
2738 }
2739 else
2740 {
2741 Log(("Pending inject %RX64 at %RGv exit=%08x intInfo=%08x exitQualification=%RGv\n", pVCpu->hwaccm.s.Event.intInfo, (RTGCPTR)pCtx->rip, exitReason, intInfo, exitQualification));
2742 pVCpu->hwaccm.s.Event.errCode = 0;
2743 }
2744 }
2745#ifdef VBOX_STRICT
2746 else
2747 if ( VMX_EXIT_INTERRUPTION_INFO_VALID(pVCpu->hwaccm.s.Event.intInfo)
2748 /* Ignore software exceptions (such as int3) as they're reoccur when we restart the instruction anyway. */
2749 && VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hwaccm.s.Event.intInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_SWEXCPT)
2750 {
2751 Log(("Ignore pending inject %RX64 at %RGv exit=%08x intInfo=%08x exitQualification=%RGv\n", pVCpu->hwaccm.s.Event.intInfo, (RTGCPTR)pCtx->rip, exitReason, intInfo, exitQualification));
2752 }
2753
2754 if (exitReason == VMX_EXIT_ERR_INVALID_GUEST_STATE)
2755 HWACCMDumpRegs(pVM, pVCpu, pCtx);
2756#endif
2757
2758 Log2(("E%d: New EIP=%x:%RGv\n", (uint32_t)exitReason, pCtx->cs, (RTGCPTR)pCtx->rip));
2759 Log2(("Exit reason %d, exitQualification %RGv\n", (uint32_t)exitReason, exitQualification));
2760 Log2(("instrInfo=%d instrError=%d instr length=%d\n", (uint32_t)instrInfo, (uint32_t)instrError, (uint32_t)cbInstr));
2761 Log2(("Interruption error code %d\n", (uint32_t)errCode));
2762 Log2(("IntInfo = %08x\n", (uint32_t)intInfo));
2763
2764 /* Sync back the TPR if it was changed. */
2765 if ( fSetupTPRCaching
2766 && u8LastTPR != pVCpu->hwaccm.s.vmx.pVAPIC[0x80])
2767 {
2768 rc = PDMApicSetTPR(pVCpu, pVCpu->hwaccm.s.vmx.pVAPIC[0x80]);
2769 AssertRC(rc);
2770 }
2771
2772 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit1, v);
2773 STAM_STATS({ STAM_PROFILE_ADV_START(&pVCpu->hwaccm.s.StatExit2, y); fStatExit2Started = true; });
2774
2775 /* Some cases don't need a complete resync of the guest CPU state; handle them here. */
2776 switch (exitReason)
2777 {
2778 case VMX_EXIT_EXCEPTION: /* 0 Exception or non-maskable interrupt (NMI). */
2779 case VMX_EXIT_EXTERNAL_IRQ: /* 1 External interrupt. */
2780 {
2781 uint32_t vector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(intInfo);
2782
2783 if (!VMX_EXIT_INTERRUPTION_INFO_VALID(intInfo))
2784 {
2785 Assert(exitReason == VMX_EXIT_EXTERNAL_IRQ);
2786 /* External interrupt; leave to allow it to be dispatched again. */
2787 rc = VINF_EM_RAW_INTERRUPT;
2788 break;
2789 }
2790 STAM_PROFILE_ADV_START(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
2791 switch (VMX_EXIT_INTERRUPTION_INFO_TYPE(intInfo))
2792 {
2793 case VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI: /* Non-maskable interrupt. */
2794 /* External interrupt; leave to allow it to be dispatched again. */
2795 rc = VINF_EM_RAW_INTERRUPT;
2796 break;
2797
2798 case VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT: /* External hardware interrupt. */
2799 AssertFailed(); /* can't come here; fails the first check. */
2800 break;
2801
2802 case VMX_EXIT_INTERRUPTION_INFO_TYPE_DBEXCPT: /* Unknown why we get this type for #DB */
2803 case VMX_EXIT_INTERRUPTION_INFO_TYPE_SWEXCPT: /* Software exception. (#BP or #OF) */
2804 Assert(vector == 1 || vector == 3 || vector == 4);
2805 /* no break */
2806 case VMX_EXIT_INTERRUPTION_INFO_TYPE_HWEXCPT: /* Hardware exception. */
2807 Log2(("Hardware/software interrupt %d\n", vector));
2808 switch (vector)
2809 {
2810 case X86_XCPT_NM:
2811 {
2812 Log(("#NM fault at %RGv error code %x\n", (RTGCPTR)pCtx->rip, errCode));
2813
2814 /** @todo don't intercept #NM exceptions anymore when we've activated the guest FPU state. */
2815 /* If we sync the FPU/XMM state on-demand, then we can continue execution as if nothing has happened. */
2816 rc = CPUMR0LoadGuestFPU(pVM, pVCpu, pCtx);
2817 if (rc == VINF_SUCCESS)
2818 {
2819 Assert(CPUMIsGuestFPUStateActive(pVCpu));
2820
2821 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitShadowNM);
2822
2823 /* Continue execution. */
2824 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_GUEST_CR0;
2825
2826 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
2827 goto ResumeExecution;
2828 }
2829
2830 Log(("Forward #NM fault to the guest\n"));
2831 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestNM);
2832 rc = VMXR0InjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), cbInstr, 0);
2833 AssertRC(rc);
2834 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
2835 goto ResumeExecution;
2836 }
2837
2838 case X86_XCPT_PF: /* Page fault */
2839 {
2840#ifdef DEBUG
2841 if (pVM->hwaccm.s.fNestedPaging)
2842 { /* A genuine pagefault.
2843 * Forward the trap to the guest by injecting the exception and resuming execution.
2844 */
2845 Log(("Guest page fault at %RGv cr2=%RGv error code %x rsp=%RGv\n", (RTGCPTR)pCtx->rip, exitQualification, errCode, (RTGCPTR)pCtx->rsp));
2846
2847 Assert(CPUMIsGuestInPagedProtectedModeEx(pCtx));
2848
2849 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestPF);
2850
2851 /* Now we must update CR2. */
2852 pCtx->cr2 = exitQualification;
2853 rc = VMXR0InjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), cbInstr, errCode);
2854 AssertRC(rc);
2855
2856 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
2857 goto ResumeExecution;
2858 }
2859#endif
2860 Assert(!pVM->hwaccm.s.fNestedPaging);
2861
2862#ifdef VBOX_HWACCM_WITH_GUEST_PATCHING
2863 /* Shortcut for APIC TPR reads and writes; 32 bits guests only */
2864 if ( pVM->hwaccm.s.fTRPPatchingAllowed
2865 && pVM->hwaccm.s.pGuestPatchMem
2866 && (exitQualification & 0xfff) == 0x080
2867 && !(errCode & X86_TRAP_PF_P) /* not present */
2868 && CPUMGetGuestCPL(pVCpu, CPUMCTX2CORE(pCtx)) == 0
2869 && !CPUMIsGuestInLongModeEx(pCtx)
2870 && pVM->hwaccm.s.cPatches < RT_ELEMENTS(pVM->hwaccm.s.aPatches))
2871 {
2872 RTGCPHYS GCPhysApicBase, GCPhys;
2873 PDMApicGetBase(pVM, &GCPhysApicBase); /* @todo cache this */
2874 GCPhysApicBase &= PAGE_BASE_GC_MASK;
2875
2876 rc = PGMGstGetPage(pVCpu, (RTGCPTR)exitQualification, NULL, &GCPhys);
2877 if ( rc == VINF_SUCCESS
2878 && GCPhys == GCPhysApicBase)
2879 {
2880 /* Only attempt to patch the instruction once. */
2881 PHWACCMTPRPATCH pPatch = (PHWACCMTPRPATCH)RTAvloU32Get(&pVM->hwaccm.s.PatchTree, (AVLOU32KEY)pCtx->eip);
2882 if (!pPatch)
2883 {
2884 rc = VINF_EM_HWACCM_PATCH_TPR_INSTR;
2885 break;
2886 }
2887 }
2888 }
2889#endif
2890
2891 Log2(("Page fault at %RGv error code %x\n", exitQualification, errCode));
2892 /* Exit qualification contains the linear address of the page fault. */
2893 TRPMAssertTrap(pVCpu, X86_XCPT_PF, TRPM_TRAP);
2894 TRPMSetErrorCode(pVCpu, errCode);
2895 TRPMSetFaultAddress(pVCpu, exitQualification);
2896
2897 /* Shortcut for APIC TPR reads and writes. */
2898 if ( (exitQualification & 0xfff) == 0x080
2899 && !(errCode & X86_TRAP_PF_P) /* not present */
2900 && fSetupTPRCaching
2901 && (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC))
2902 {
2903 RTGCPHYS GCPhysApicBase, GCPhys;
2904 PDMApicGetBase(pVM, &GCPhysApicBase); /* @todo cache this */
2905 GCPhysApicBase &= PAGE_BASE_GC_MASK;
2906
2907 rc = PGMGstGetPage(pVCpu, (RTGCPTR)exitQualification, NULL, &GCPhys);
2908 if ( rc == VINF_SUCCESS
2909 && GCPhys == GCPhysApicBase)
2910 {
2911 Log(("Enable VT-x virtual APIC access filtering\n"));
2912 rc = IOMMMIOMapMMIOHCPage(pVM, GCPhysApicBase, pVM->hwaccm.s.vmx.pAPICPhys, X86_PTE_RW | X86_PTE_P);
2913 AssertRC(rc);
2914 }
2915 }
2916
2917 /* Forward it to our trap handler first, in case our shadow pages are out of sync. */
2918 rc = PGMTrap0eHandler(pVCpu, errCode, CPUMCTX2CORE(pCtx), (RTGCPTR)exitQualification);
2919 Log2(("PGMTrap0eHandler %RGv returned %Rrc\n", (RTGCPTR)pCtx->rip, rc));
2920
2921 if (rc == VINF_SUCCESS)
2922 { /* We've successfully synced our shadow pages, so let's just continue execution. */
2923 Log2(("Shadow page fault at %RGv cr2=%RGv error code %x\n", (RTGCPTR)pCtx->rip, exitQualification ,errCode));
2924 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitShadowPF);
2925
2926 TRPMResetTrap(pVCpu);
2927 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
2928 goto ResumeExecution;
2929 }
2930 else
2931 if (rc == VINF_EM_RAW_GUEST_TRAP)
2932 { /* A genuine pagefault.
2933 * Forward the trap to the guest by injecting the exception and resuming execution.
2934 */
2935 Log2(("Forward page fault to the guest\n"));
2936
2937 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestPF);
2938 /* The error code might have been changed. */
2939 errCode = TRPMGetErrorCode(pVCpu);
2940
2941 TRPMResetTrap(pVCpu);
2942
2943 /* Now we must update CR2. */
2944 pCtx->cr2 = exitQualification;
2945 rc = VMXR0InjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), cbInstr, errCode);
2946 AssertRC(rc);
2947
2948 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
2949 goto ResumeExecution;
2950 }
2951#ifdef VBOX_STRICT
2952 if (rc != VINF_EM_RAW_EMULATE_INSTR && rc != VINF_EM_RAW_EMULATE_IO_BLOCK)
2953 Log2(("PGMTrap0eHandler failed with %d\n", rc));
2954#endif
2955 /* Need to go back to the recompiler to emulate the instruction. */
2956 TRPMResetTrap(pVCpu);
2957 break;
2958 }
2959
2960 case X86_XCPT_MF: /* Floating point exception. */
2961 {
2962 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestMF);
2963 if (!(pCtx->cr0 & X86_CR0_NE))
2964 {
2965 /* old style FPU error reporting needs some extra work. */
2966 /** @todo don't fall back to the recompiler, but do it manually. */
2967 rc = VINF_EM_RAW_EMULATE_INSTR;
2968 break;
2969 }
2970 Log(("Trap %x at %04X:%RGv\n", vector, pCtx->cs, (RTGCPTR)pCtx->rip));
2971 rc = VMXR0InjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), cbInstr, errCode);
2972 AssertRC(rc);
2973
2974 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
2975 goto ResumeExecution;
2976 }
2977
2978 case X86_XCPT_DB: /* Debug exception. */
2979 {
2980 uint64_t uDR6;
2981
2982 /* DR6, DR7.GD and IA32_DEBUGCTL.LBR are not updated yet.
2983 *
2984 * Exit qualification bits:
2985 * 3:0 B0-B3 which breakpoint condition was met
2986 * 12:4 Reserved (0)
2987 * 13 BD - debug register access detected
2988 * 14 BS - single step execution or branch taken
2989 * 63:15 Reserved (0)
2990 */
2991 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestDB);
2992
2993 /* Note that we don't support guest and host-initiated debugging at the same time. */
2994
2995 uDR6 = X86_DR6_INIT_VAL;
2996 uDR6 |= (exitQualification & (X86_DR6_B0|X86_DR6_B1|X86_DR6_B2|X86_DR6_B3|X86_DR6_BD|X86_DR6_BS));
2997 rc = DBGFRZTrap01Handler(pVM, pVCpu, CPUMCTX2CORE(pCtx), uDR6);
2998 if (rc == VINF_EM_RAW_GUEST_TRAP)
2999 {
3000 /* Update DR6 here. */
3001 pCtx->dr[6] = uDR6;
3002
3003 /* Resync DR6 if the debug state is active. */
3004 if (CPUMIsGuestDebugStateActive(pVCpu))
3005 ASMSetDR6(pCtx->dr[6]);
3006
3007 /* X86_DR7_GD will be cleared if drx accesses should be trapped inside the guest. */
3008 pCtx->dr[7] &= ~X86_DR7_GD;
3009
3010 /* Paranoia. */
3011 pCtx->dr[7] &= 0xffffffff; /* upper 32 bits reserved */
3012 pCtx->dr[7] &= ~(RT_BIT(11) | RT_BIT(12) | RT_BIT(14) | RT_BIT(15)); /* must be zero */
3013 pCtx->dr[7] |= 0x400; /* must be one */
3014
3015 /* Resync DR7 */
3016 rc = VMXWriteVMCS64(VMX_VMCS64_GUEST_DR7, pCtx->dr[7]);
3017 AssertRC(rc);
3018
3019 Log(("Trap %x (debug) at %RGv exit qualification %RX64 dr6=%x dr7=%x\n", vector, (RTGCPTR)pCtx->rip, exitQualification, (uint32_t)pCtx->dr[6], (uint32_t)pCtx->dr[7]));
3020 rc = VMXR0InjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), cbInstr, errCode);
3021 AssertRC(rc);
3022
3023 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3024 goto ResumeExecution;
3025 }
3026 /* Return to ring 3 to deal with the debug exit code. */
3027 Log(("Debugger hardware BP at %04x:%RGv (rc=%Rrc)\n", pCtx->cs, pCtx->rip, rc));
3028 break;
3029 }
3030
3031 case X86_XCPT_BP: /* Breakpoint. */
3032 {
3033 rc = DBGFRZTrap03Handler(pVM, pVCpu, CPUMCTX2CORE(pCtx));
3034 if (rc == VINF_EM_RAW_GUEST_TRAP)
3035 {
3036 Log(("Guest #BP at %04x:%RGv\n", pCtx->cs, pCtx->rip));
3037 rc = VMXR0InjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), cbInstr, errCode);
3038 AssertRC(rc);
3039 goto ResumeExecution;
3040 }
3041 if (rc == VINF_SUCCESS)
3042 goto ResumeExecution;
3043 Log(("Debugger BP at %04x:%RGv (rc=%Rrc)\n", pCtx->cs, pCtx->rip, rc));
3044 break;
3045 }
3046
3047 case X86_XCPT_GP: /* General protection failure exception.*/
3048 {
3049 uint32_t cbOp;
3050 uint32_t cbSize;
3051 PDISCPUSTATE pDis = &pVCpu->hwaccm.s.DisState;
3052
3053 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestGP);
3054#ifdef VBOX_STRICT
3055 if ( !CPUMIsGuestInRealModeEx(pCtx)
3056 || !pVM->hwaccm.s.vmx.pRealModeTSS)
3057 {
3058 Log(("Trap %x at %04X:%RGv errorCode=%x\n", vector, pCtx->cs, (RTGCPTR)pCtx->rip, errCode));
3059 rc = VMXR0InjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), cbInstr, errCode);
3060 AssertRC(rc);
3061 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3062 goto ResumeExecution;
3063 }
3064#endif
3065 Assert(CPUMIsGuestInRealModeEx(pCtx));
3066
3067 LogFlow(("Real mode X86_XCPT_GP instruction emulation at %x:%RGv\n", pCtx->cs, (RTGCPTR)pCtx->rip));
3068
3069 rc = EMInterpretDisasOne(pVM, pVCpu, CPUMCTX2CORE(pCtx), pDis, &cbOp);
3070 if (RT_SUCCESS(rc))
3071 {
3072 bool fUpdateRIP = true;
3073
3074 Assert(cbOp == pDis->opsize);
3075 switch (pDis->pCurInstr->opcode)
3076 {
3077 case OP_CLI:
3078 pCtx->eflags.Bits.u1IF = 0;
3079 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitCli);
3080 break;
3081
3082 case OP_STI:
3083 pCtx->eflags.Bits.u1IF = 1;
3084 EMSetInhibitInterruptsPC(pVCpu, pCtx->rip + pDis->opsize);
3085 Assert(VMCPU_FF_ISSET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
3086 rc = VMXWriteVMCS(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
3087 AssertRC(rc);
3088 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitSti);
3089 break;
3090
3091 case OP_HLT:
3092 fUpdateRIP = false;
3093 rc = VINF_EM_HALT;
3094 pCtx->rip += pDis->opsize;
3095 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitHlt);
3096 break;
3097
3098 case OP_POPF:
3099 {
3100 RTGCPTR GCPtrStack;
3101 uint32_t cbParm;
3102 uint32_t uMask;
3103 X86EFLAGS eflags;
3104
3105 if (pDis->prefix & PREFIX_OPSIZE)
3106 {
3107 cbParm = 4;
3108 uMask = 0xffffffff;
3109 }
3110 else
3111 {
3112 cbParm = 2;
3113 uMask = 0xffff;
3114 }
3115
3116 rc = SELMToFlatEx(pVM, DIS_SELREG_SS, CPUMCTX2CORE(pCtx), pCtx->esp & uMask, 0, &GCPtrStack);
3117 if (RT_FAILURE(rc))
3118 {
3119 rc = VERR_EM_INTERPRETER;
3120 break;
3121 }
3122 eflags.u = 0;
3123 rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &eflags.u, cbParm);
3124 if (RT_FAILURE(rc))
3125 {
3126 rc = VERR_EM_INTERPRETER;
3127 break;
3128 }
3129 LogFlow(("POPF %x -> %RGv mask=%x\n", eflags.u, pCtx->rsp, uMask));
3130 pCtx->eflags.u = (pCtx->eflags.u & ~(X86_EFL_POPF_BITS & uMask)) | (eflags.u & X86_EFL_POPF_BITS & uMask);
3131 /* RF cleared when popped in real mode; see pushf description in AMD manual. */
3132 pCtx->eflags.Bits.u1RF = 0;
3133 pCtx->esp += cbParm;
3134 pCtx->esp &= uMask;
3135
3136 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitPopf);
3137 break;
3138 }
3139
3140 case OP_PUSHF:
3141 {
3142 RTGCPTR GCPtrStack;
3143 uint32_t cbParm;
3144 uint32_t uMask;
3145 X86EFLAGS eflags;
3146
3147 if (pDis->prefix & PREFIX_OPSIZE)
3148 {
3149 cbParm = 4;
3150 uMask = 0xffffffff;
3151 }
3152 else
3153 {
3154 cbParm = 2;
3155 uMask = 0xffff;
3156 }
3157
3158 rc = SELMToFlatEx(pVM, DIS_SELREG_SS, CPUMCTX2CORE(pCtx), (pCtx->esp - cbParm) & uMask, 0, &GCPtrStack);
3159 if (RT_FAILURE(rc))
3160 {
3161 rc = VERR_EM_INTERPRETER;
3162 break;
3163 }
3164 eflags = pCtx->eflags;
3165 /* RF & VM cleared when pushed in real mode; see pushf description in AMD manual. */
3166 eflags.Bits.u1RF = 0;
3167 eflags.Bits.u1VM = 0;
3168
3169 rc = PGMPhysWrite(pVM, (RTGCPHYS)GCPtrStack, &eflags.u, cbParm);
3170 if (RT_FAILURE(rc))
3171 {
3172 rc = VERR_EM_INTERPRETER;
3173 break;
3174 }
3175 LogFlow(("PUSHF %x -> %RGv\n", eflags.u, GCPtrStack));
3176 pCtx->esp -= cbParm;
3177 pCtx->esp &= uMask;
3178 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitPushf);
3179 break;
3180 }
3181
3182 case OP_IRET:
3183 {
3184 RTGCPTR GCPtrStack;
3185 uint32_t uMask = 0xffff;
3186 uint16_t aIretFrame[3];
3187
3188 if (pDis->prefix & (PREFIX_OPSIZE | PREFIX_ADDRSIZE))
3189 {
3190 rc = VERR_EM_INTERPRETER;
3191 break;
3192 }
3193
3194 rc = SELMToFlatEx(pVM, DIS_SELREG_SS, CPUMCTX2CORE(pCtx), pCtx->esp & uMask, 0, &GCPtrStack);
3195 if (RT_FAILURE(rc))
3196 {
3197 rc = VERR_EM_INTERPRETER;
3198 break;
3199 }
3200 rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &aIretFrame[0], sizeof(aIretFrame));
3201 if (RT_FAILURE(rc))
3202 {
3203 rc = VERR_EM_INTERPRETER;
3204 break;
3205 }
3206 pCtx->ip = aIretFrame[0];
3207 pCtx->cs = aIretFrame[1];
3208 pCtx->csHid.u64Base = pCtx->cs << 4;
3209 pCtx->eflags.u = (pCtx->eflags.u & ~(X86_EFL_POPF_BITS & uMask)) | (aIretFrame[2] & X86_EFL_POPF_BITS & uMask);
3210 pCtx->sp += sizeof(aIretFrame);
3211
3212 LogFlow(("iret to %04x:%x\n", pCtx->cs, pCtx->ip));
3213 fUpdateRIP = false;
3214 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitIret);
3215 break;
3216 }
3217
3218 case OP_INT:
3219 {
3220 uint32_t intInfo2;
3221
3222 LogFlow(("Realmode: INT %x\n", pDis->param1.parval & 0xff));
3223 intInfo2 = pDis->param1.parval & 0xff;
3224 intInfo2 |= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
3225 intInfo2 |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
3226
3227 rc = VMXR0InjectEvent(pVM, pVCpu, pCtx, intInfo2, cbOp, 0);
3228 AssertRC(rc);
3229 fUpdateRIP = false;
3230 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitInt);
3231 break;
3232 }
3233
3234 case OP_INTO:
3235 {
3236 if (pCtx->eflags.Bits.u1OF)
3237 {
3238 uint32_t intInfo2;
3239
3240 LogFlow(("Realmode: INTO\n"));
3241 intInfo2 = X86_XCPT_OF;
3242 intInfo2 |= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
3243 intInfo2 |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
3244
3245 rc = VMXR0InjectEvent(pVM, pVCpu, pCtx, intInfo2, cbOp, 0);
3246 AssertRC(rc);
3247 fUpdateRIP = false;
3248 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitInt);
3249 }
3250 break;
3251 }
3252
3253 case OP_INT3:
3254 {
3255 uint32_t intInfo2;
3256
3257 LogFlow(("Realmode: INT 3\n"));
3258 intInfo2 = 3;
3259 intInfo2 |= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
3260 intInfo2 |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
3261
3262 rc = VMXR0InjectEvent(pVM, pVCpu, pCtx, intInfo2, cbOp, 0);
3263 AssertRC(rc);
3264 fUpdateRIP = false;
3265 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitInt);
3266 break;
3267 }
3268
3269 default:
3270 rc = EMInterpretInstructionCPU(pVM, pVCpu, pDis, CPUMCTX2CORE(pCtx), 0, &cbSize);
3271 break;
3272 }
3273
3274 if (rc == VINF_SUCCESS)
3275 {
3276 if (fUpdateRIP)
3277 pCtx->rip += cbOp; /* Move on to the next instruction. */
3278
3279 /* lidt, lgdt can end up here. In the future crx changes as well. Just reload the whole context to be done with it. */
3280 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_ALL;
3281
3282 /* Only resume if successful. */
3283 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3284 goto ResumeExecution;
3285 }
3286 }
3287 else
3288 rc = VERR_EM_INTERPRETER;
3289
3290 AssertMsg(rc == VERR_EM_INTERPRETER || rc == VINF_PGM_CHANGE_MODE || rc == VINF_EM_HALT, ("Unexpected rc=%Rrc\n", rc));
3291 break;
3292 }
3293
3294#ifdef VBOX_STRICT
3295 case X86_XCPT_XF: /* SIMD exception. */
3296 case X86_XCPT_DE: /* Divide error. */
3297 case X86_XCPT_UD: /* Unknown opcode exception. */
3298 case X86_XCPT_SS: /* Stack segment exception. */
3299 case X86_XCPT_NP: /* Segment not present exception. */
3300 {
3301 switch(vector)
3302 {
3303 case X86_XCPT_DE:
3304 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestDE);
3305 break;
3306 case X86_XCPT_UD:
3307 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestUD);
3308 break;
3309 case X86_XCPT_SS:
3310 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestSS);
3311 break;
3312 case X86_XCPT_NP:
3313 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestNP);
3314 break;
3315 }
3316
3317 Log(("Trap %x at %04X:%RGv\n", vector, pCtx->cs, (RTGCPTR)pCtx->rip));
3318 rc = VMXR0InjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), cbInstr, errCode);
3319 AssertRC(rc);
3320
3321 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3322 goto ResumeExecution;
3323 }
3324#endif
3325 default:
3326#ifdef HWACCM_VMX_EMULATE_REALMODE
3327 if ( CPUMIsGuestInRealModeEx(pCtx)
3328 && pVM->hwaccm.s.vmx.pRealModeTSS)
3329 {
3330 Log(("Real Mode Trap %x at %04x:%04X error code %x\n", vector, pCtx->cs, pCtx->eip, errCode));
3331 rc = VMXR0InjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), cbInstr, errCode);
3332 AssertRC(rc);
3333
3334 /* Go back to ring 3 in case of a triple fault. */
3335 if ( vector == X86_XCPT_DF
3336 && rc == VINF_EM_RESET)
3337 break;
3338
3339 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3340 goto ResumeExecution;
3341 }
3342#endif
3343 AssertMsgFailed(("Unexpected vm-exit caused by exception %x\n", vector));
3344 rc = VERR_VMX_UNEXPECTED_EXCEPTION;
3345 break;
3346 } /* switch (vector) */
3347
3348 break;
3349
3350 default:
3351 rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_CODE;
3352 AssertMsgFailed(("Unexpected interuption code %x\n", intInfo));
3353 break;
3354 }
3355
3356 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3357 break;
3358 }
3359
3360 case VMX_EXIT_EPT_VIOLATION: /* 48 EPT violation. An attempt to access memory with a guest-physical address was disallowed by the configuration of the EPT paging structures. */
3361 {
3362 RTGCPHYS GCPhys;
3363
3364 Assert(pVM->hwaccm.s.fNestedPaging);
3365
3366 rc = VMXReadVMCS64(VMX_VMCS_EXIT_PHYS_ADDR_FULL, &GCPhys);
3367 AssertRC(rc);
3368 Assert(((exitQualification >> 7) & 3) != 2);
3369
3370 /* Determine the kind of violation. */
3371 errCode = 0;
3372 if (exitQualification & VMX_EXIT_QUALIFICATION_EPT_INSTR_FETCH)
3373 errCode |= X86_TRAP_PF_ID;
3374
3375 if (exitQualification & VMX_EXIT_QUALIFICATION_EPT_DATA_WRITE)
3376 errCode |= X86_TRAP_PF_RW;
3377
3378 /* If the page is present, then it's a page level protection fault. */
3379 if (exitQualification & VMX_EXIT_QUALIFICATION_EPT_ENTRY_PRESENT)
3380 {
3381 errCode |= X86_TRAP_PF_P;
3382 }
3383 else
3384 {
3385 /* Shortcut for APIC TPR reads and writes. */
3386 if ( (GCPhys & 0xfff) == 0x080
3387 && GCPhys > 0x1000000 /* to skip VGA frame buffer accesses */
3388 && fSetupTPRCaching
3389 && (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC))
3390 {
3391 RTGCPHYS GCPhysApicBase;
3392 PDMApicGetBase(pVM, &GCPhysApicBase); /* @todo cache this */
3393 GCPhysApicBase &= PAGE_BASE_GC_MASK;
3394 if (GCPhys == GCPhysApicBase + 0x80)
3395 {
3396 Log(("Enable VT-x virtual APIC access filtering\n"));
3397 rc = IOMMMIOMapMMIOHCPage(pVM, GCPhysApicBase, pVM->hwaccm.s.vmx.pAPICPhys, X86_PTE_RW | X86_PTE_P);
3398 AssertRC(rc);
3399 }
3400 }
3401 }
3402 Log(("EPT Page fault %x at %RGp error code %x\n", (uint32_t)exitQualification, GCPhys, errCode));
3403
3404 /* GCPhys contains the guest physical address of the page fault. */
3405 TRPMAssertTrap(pVCpu, X86_XCPT_PF, TRPM_TRAP);
3406 TRPMSetErrorCode(pVCpu, errCode);
3407 TRPMSetFaultAddress(pVCpu, GCPhys);
3408
3409 /* Handle the pagefault trap for the nested shadow table. */
3410 rc = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, errCode, CPUMCTX2CORE(pCtx), GCPhys);
3411 Log2(("PGMR0Trap0eHandlerNestedPaging %RGv returned %Rrc\n", (RTGCPTR)pCtx->rip, rc));
3412 if (rc == VINF_SUCCESS)
3413 { /* We've successfully synced our shadow pages, so let's just continue execution. */
3414 Log2(("Shadow page fault at %RGv cr2=%RGp error code %x\n", (RTGCPTR)pCtx->rip, exitQualification , errCode));
3415 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitReasonNPF);
3416
3417 TRPMResetTrap(pVCpu);
3418 goto ResumeExecution;
3419 }
3420
3421#ifdef VBOX_STRICT
3422 if (rc != VINF_EM_RAW_EMULATE_INSTR)
3423 LogFlow(("PGMTrap0eHandlerNestedPaging failed with %d\n", rc));
3424#endif
3425 /* Need to go back to the recompiler to emulate the instruction. */
3426 TRPMResetTrap(pVCpu);
3427 break;
3428 }
3429
3430 case VMX_EXIT_EPT_MISCONFIG:
3431 {
3432 RTGCPHYS GCPhys;
3433
3434 Assert(pVM->hwaccm.s.fNestedPaging);
3435
3436 rc = VMXReadVMCS64(VMX_VMCS_EXIT_PHYS_ADDR_FULL, &GCPhys);
3437 AssertRC(rc);
3438
3439 Log(("VMX_EXIT_EPT_MISCONFIG for %RGp\n", GCPhys));
3440 break;
3441 }
3442
3443 case VMX_EXIT_IRQ_WINDOW: /* 7 Interrupt window. */
3444 /* Clear VM-exit on IF=1 change. */
3445 LogFlow(("VMX_EXIT_IRQ_WINDOW %RGv pending=%d IF=%d\n", (RTGCPTR)pCtx->rip, VMCPU_FF_ISPENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC|VMCPU_FF_INTERRUPT_PIC)), pCtx->eflags.Bits.u1IF));
3446 pVCpu->hwaccm.s.vmx.proc_ctls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_IRQ_WINDOW_EXIT;
3447 rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
3448 AssertRC(rc);
3449 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitIrqWindow);
3450 goto ResumeExecution; /* we check for pending guest interrupts there */
3451
3452 case VMX_EXIT_WBINVD: /* 54 Guest software attempted to execute WBINVD. (conditional) */
3453 case VMX_EXIT_INVD: /* 13 Guest software attempted to execute INVD. (unconditional) */
3454 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitInvd);
3455 /* Skip instruction and continue directly. */
3456 pCtx->rip += cbInstr;
3457 /* Continue execution.*/
3458 goto ResumeExecution;
3459
3460 case VMX_EXIT_CPUID: /* 10 Guest software attempted to execute CPUID. */
3461 {
3462 Log2(("VMX: Cpuid %x\n", pCtx->eax));
3463 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitCpuid);
3464 rc = EMInterpretCpuId(pVM, pVCpu, CPUMCTX2CORE(pCtx));
3465 if (rc == VINF_SUCCESS)
3466 {
3467 /* Update EIP and continue execution. */
3468 Assert(cbInstr == 2);
3469 pCtx->rip += cbInstr;
3470 goto ResumeExecution;
3471 }
3472 AssertMsgFailed(("EMU: cpuid failed with %Rrc\n", rc));
3473 rc = VINF_EM_RAW_EMULATE_INSTR;
3474 break;
3475 }
3476
3477 case VMX_EXIT_RDPMC: /* 15 Guest software attempted to execute RDPMC. */
3478 {
3479 Log2(("VMX: Rdpmc %x\n", pCtx->ecx));
3480 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitRdpmc);
3481 rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pCtx));
3482 if (rc == VINF_SUCCESS)
3483 {
3484 /* Update EIP and continue execution. */
3485 Assert(cbInstr == 2);
3486 pCtx->rip += cbInstr;
3487 goto ResumeExecution;
3488 }
3489 rc = VINF_EM_RAW_EMULATE_INSTR;
3490 break;
3491 }
3492
3493 case VMX_EXIT_RDTSC: /* 16 Guest software attempted to execute RDTSC. */
3494 {
3495 Log2(("VMX: Rdtsc\n"));
3496 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitRdtsc);
3497 rc = EMInterpretRdtsc(pVM, pVCpu, CPUMCTX2CORE(pCtx));
3498 if (rc == VINF_SUCCESS)
3499 {
3500 /* Update EIP and continue execution. */
3501 Assert(cbInstr == 2);
3502 pCtx->rip += cbInstr;
3503 goto ResumeExecution;
3504 }
3505 rc = VINF_EM_RAW_EMULATE_INSTR;
3506 break;
3507 }
3508
3509 case VMX_EXIT_INVPG: /* 14 Guest software attempted to execute INVPG. */
3510 {
3511 Log2(("VMX: invlpg\n"));
3512 Assert(!pVM->hwaccm.s.fNestedPaging);
3513
3514 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitInvpg);
3515 rc = EMInterpretInvlpg(pVM, pVCpu, CPUMCTX2CORE(pCtx), exitQualification);
3516 if (rc == VINF_SUCCESS)
3517 {
3518 /* Update EIP and continue execution. */
3519 pCtx->rip += cbInstr;
3520 goto ResumeExecution;
3521 }
3522 AssertMsg(rc == VERR_EM_INTERPRETER, ("EMU: invlpg %RGv failed with %Rrc\n", exitQualification, rc));
3523 break;
3524 }
3525
3526 case VMX_EXIT_MONITOR: /* 39 Guest software attempted to execute MONITOR. */
3527 {
3528 Log2(("VMX: monitor\n"));
3529
3530 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitMonitor);
3531 rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pCtx));
3532 if (rc == VINF_SUCCESS)
3533 {
3534 /* Update EIP and continue execution. */
3535 pCtx->rip += cbInstr;
3536 goto ResumeExecution;
3537 }
3538 AssertMsg(rc == VERR_EM_INTERPRETER, ("EMU: monitor failed with %Rrc\n", rc));
3539 break;
3540 }
3541
3542 case VMX_EXIT_WRMSR: /* 32 WRMSR. Guest software attempted to execute WRMSR. */
3543 /* When an interrupt is pending, we'll let MSR_K8_LSTAR writes fault in our TPR patch code. */
3544 if ( pVM->hwaccm.s.fTPRPatchingActive
3545 && pCtx->ecx == MSR_K8_LSTAR)
3546 {
3547 Assert(!CPUMIsGuestInLongModeEx(pCtx));
3548 if ((pCtx->eax & 0xff) != u8LastTPR)
3549 {
3550 Log(("VMX: Faulting MSR_K8_LSTAR write with new TPR value %x\n", pCtx->eax & 0xff));
3551
3552 /* Our patch code uses LSTAR for TPR caching. */
3553 rc = PDMApicSetTPR(pVCpu, pCtx->eax & 0xff);
3554 AssertRC(rc);
3555 }
3556
3557 /* Skip the instruction and continue. */
3558 pCtx->rip += cbInstr; /* wrmsr = [0F 30] */
3559
3560 /* Only resume if successful. */
3561 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit1, x);
3562 goto ResumeExecution;
3563 }
3564 /* no break */
3565 case VMX_EXIT_RDMSR: /* 31 RDMSR. Guest software attempted to execute RDMSR. */
3566 {
3567 uint32_t cbSize;
3568
3569 STAM_COUNTER_INC((exitReason == VMX_EXIT_RDMSR) ? &pVCpu->hwaccm.s.StatExitRdmsr : &pVCpu->hwaccm.s.StatExitWrmsr);
3570
3571 /* Note: the intel manual claims there's a REX version of RDMSR that's slightly different, so we play safe by completely disassembling the instruction. */
3572 Log2(("VMX: %s\n", (exitReason == VMX_EXIT_RDMSR) ? "rdmsr" : "wrmsr"));
3573 rc = EMInterpretInstruction(pVM, pVCpu, CPUMCTX2CORE(pCtx), 0, &cbSize);
3574 if (rc == VINF_SUCCESS)
3575 {
3576 /* EIP has been updated already. */
3577
3578 /* Only resume if successful. */
3579 goto ResumeExecution;
3580 }
3581 AssertMsg(rc == VERR_EM_INTERPRETER, ("EMU: %s failed with %Rrc\n", (exitReason == VMX_EXIT_RDMSR) ? "rdmsr" : "wrmsr", rc));
3582 break;
3583 }
3584
3585 case VMX_EXIT_CRX_MOVE: /* 28 Control-register accesses. */
3586 {
3587 STAM_PROFILE_ADV_START(&pVCpu->hwaccm.s.StatExit2Sub2, y2);
3588
3589 switch (VMX_EXIT_QUALIFICATION_CRX_ACCESS(exitQualification))
3590 {
3591 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_WRITE:
3592 Log2(("VMX: %RGv mov cr%d, x\n", (RTGCPTR)pCtx->rip, VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification)));
3593 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitCRxWrite[VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification)]);
3594 rc = EMInterpretCRxWrite(pVM, pVCpu, CPUMCTX2CORE(pCtx),
3595 VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification),
3596 VMX_EXIT_QUALIFICATION_CRX_GENREG(exitQualification));
3597
3598 switch (VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification))
3599 {
3600 case 0:
3601 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_GUEST_CR0 | HWACCM_CHANGED_GUEST_CR3;
3602 break;
3603 case 2:
3604 break;
3605 case 3:
3606 Assert(!pVM->hwaccm.s.fNestedPaging || !CPUMIsGuestInPagedProtectedModeEx(pCtx));
3607 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_GUEST_CR3;
3608 break;
3609 case 4:
3610 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_GUEST_CR4;
3611 break;
3612 case 8:
3613 /* CR8 contains the APIC TPR */
3614 Assert(!(pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW));
3615 break;
3616
3617 default:
3618 AssertFailed();
3619 break;
3620 }
3621 break;
3622
3623 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ:
3624 Log2(("VMX: mov x, crx\n"));
3625 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitCRxRead[VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification)]);
3626
3627 Assert(!pVM->hwaccm.s.fNestedPaging || !CPUMIsGuestInPagedProtectedModeEx(pCtx) || VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification) != USE_REG_CR3);
3628
3629 /* CR8 reads only cause an exit when the TPR shadow feature isn't present. */
3630 Assert(VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification) != 8 || !(pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW));
3631
3632 rc = EMInterpretCRxRead(pVM, pVCpu, CPUMCTX2CORE(pCtx),
3633 VMX_EXIT_QUALIFICATION_CRX_GENREG(exitQualification),
3634 VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification));
3635 break;
3636
3637 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_CLTS:
3638 Log2(("VMX: clts\n"));
3639 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitCLTS);
3640 rc = EMInterpretCLTS(pVM, pVCpu);
3641 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_GUEST_CR0;
3642 break;
3643
3644 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_LMSW:
3645 Log2(("VMX: lmsw %x\n", VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(exitQualification)));
3646 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitLMSW);
3647 rc = EMInterpretLMSW(pVM, pVCpu, CPUMCTX2CORE(pCtx), VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(exitQualification));
3648 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_GUEST_CR0;
3649 break;
3650 }
3651
3652 /* Update EIP if no error occurred. */
3653 if (RT_SUCCESS(rc))
3654 pCtx->rip += cbInstr;
3655
3656 if (rc == VINF_SUCCESS)
3657 {
3658 /* Only resume if successful. */
3659 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub2, y2);
3660 goto ResumeExecution;
3661 }
3662 Assert(rc == VERR_EM_INTERPRETER || rc == VINF_PGM_CHANGE_MODE || rc == VINF_PGM_SYNC_CR3);
3663 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub2, y2);
3664 break;
3665 }
3666
3667 case VMX_EXIT_DRX_MOVE: /* 29 Debug-register accesses. */
3668 {
3669 if ( !DBGFIsStepping(pVCpu)
3670 && !CPUMIsHyperDebugStateActive(pVCpu))
3671 {
3672 /* Disable drx move intercepts. */
3673 pVCpu->hwaccm.s.vmx.proc_ctls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT;
3674 rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
3675 AssertRC(rc);
3676
3677 /* Save the host and load the guest debug state. */
3678 rc = CPUMR0LoadGuestDebugState(pVM, pVCpu, pCtx, true /* include DR6 */);
3679 AssertRC(rc);
3680
3681#ifdef LOG_ENABLED
3682 if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(exitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
3683 Log(("VMX_EXIT_DRX_MOVE: write DR%d genreg %d\n", VMX_EXIT_QUALIFICATION_DRX_REGISTER(exitQualification), VMX_EXIT_QUALIFICATION_DRX_GENREG(exitQualification)));
3684 else
3685 Log(("VMX_EXIT_DRX_MOVE: read DR%d\n", VMX_EXIT_QUALIFICATION_DRX_REGISTER(exitQualification)));
3686#endif
3687
3688#ifdef VBOX_WITH_STATISTICS
3689 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatDRxContextSwitch);
3690 if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(exitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
3691 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitDRxWrite);
3692 else
3693 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitDRxRead);
3694#endif
3695
3696 goto ResumeExecution;
3697 }
3698
3699 /** @todo clear VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT after the first time and restore drx registers afterwards */
3700 if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(exitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
3701 {
3702 Log2(("VMX: mov drx%d, genreg%d\n", VMX_EXIT_QUALIFICATION_DRX_REGISTER(exitQualification), VMX_EXIT_QUALIFICATION_DRX_GENREG(exitQualification)));
3703 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitDRxWrite);
3704 rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pCtx),
3705 VMX_EXIT_QUALIFICATION_DRX_REGISTER(exitQualification),
3706 VMX_EXIT_QUALIFICATION_DRX_GENREG(exitQualification));
3707 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_GUEST_DEBUG;
3708 Log2(("DR7=%08x\n", pCtx->dr[7]));
3709 }
3710 else
3711 {
3712 Log2(("VMX: mov x, drx\n"));
3713 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitDRxRead);
3714 rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pCtx),
3715 VMX_EXIT_QUALIFICATION_DRX_GENREG(exitQualification),
3716 VMX_EXIT_QUALIFICATION_DRX_REGISTER(exitQualification));
3717 }
3718 /* Update EIP if no error occurred. */
3719 if (RT_SUCCESS(rc))
3720 pCtx->rip += cbInstr;
3721
3722 if (rc == VINF_SUCCESS)
3723 {
3724 /* Only resume if successful. */
3725 goto ResumeExecution;
3726 }
3727 Assert(rc == VERR_EM_INTERPRETER);
3728 break;
3729 }
3730
3731 /* Note: We'll get a #GP if the IO instruction isn't allowed (IOPL or TSS bitmap); no need to double check. */
3732 case VMX_EXIT_PORT_IO: /* 30 I/O instruction. */
3733 {
3734 STAM_PROFILE_ADV_START(&pVCpu->hwaccm.s.StatExit2Sub1, y1);
3735 uint32_t uIOWidth = VMX_EXIT_QUALIFICATION_IO_WIDTH(exitQualification);
3736 uint32_t uPort;
3737 bool fIOWrite = (VMX_EXIT_QUALIFICATION_IO_DIRECTION(exitQualification) == VMX_EXIT_QUALIFICATION_IO_DIRECTION_OUT);
3738
3739 /** @todo necessary to make the distinction? */
3740 if (VMX_EXIT_QUALIFICATION_IO_ENCODING(exitQualification) == VMX_EXIT_QUALIFICATION_IO_ENCODING_DX)
3741 {
3742 uPort = pCtx->edx & 0xffff;
3743 }
3744 else
3745 uPort = VMX_EXIT_QUALIFICATION_IO_PORT(exitQualification); /* Immediate encoding. */
3746
3747 /* paranoia */
3748 if (RT_UNLIKELY(uIOWidth == 2 || uIOWidth >= 4))
3749 {
3750 rc = fIOWrite ? VINF_IOM_HC_IOPORT_WRITE : VINF_IOM_HC_IOPORT_READ;
3751 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub1, y1);
3752 break;
3753 }
3754
3755 uint32_t cbSize = g_aIOSize[uIOWidth];
3756
3757 if (VMX_EXIT_QUALIFICATION_IO_STRING(exitQualification))
3758 {
3759 /* ins/outs */
3760 PDISCPUSTATE pDis = &pVCpu->hwaccm.s.DisState;
3761
3762 /* Disassemble manually to deal with segment prefixes. */
3763 /** @todo VMX_VMCS_EXIT_GUEST_LINEAR_ADDR contains the flat pointer operand of the instruction. */
3764 /** @todo VMX_VMCS32_RO_EXIT_INSTR_INFO also contains segment prefix info. */
3765 rc = EMInterpretDisasOne(pVM, pVCpu, CPUMCTX2CORE(pCtx), pDis, NULL);
3766 if (rc == VINF_SUCCESS)
3767 {
3768 if (fIOWrite)
3769 {
3770 Log2(("IOMInterpretOUTSEx %RGv %x size=%d\n", (RTGCPTR)pCtx->rip, uPort, cbSize));
3771 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitIOStringWrite);
3772 rc = VBOXSTRICTRC_TODO(IOMInterpretOUTSEx(pVM, CPUMCTX2CORE(pCtx), uPort, pDis->prefix, cbSize));
3773 }
3774 else
3775 {
3776 Log2(("IOMInterpretINSEx %RGv %x size=%d\n", (RTGCPTR)pCtx->rip, uPort, cbSize));
3777 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitIOStringRead);
3778 rc = VBOXSTRICTRC_TODO(IOMInterpretINSEx(pVM, CPUMCTX2CORE(pCtx), uPort, pDis->prefix, cbSize));
3779 }
3780 }
3781 else
3782 rc = VINF_EM_RAW_EMULATE_INSTR;
3783 }
3784 else
3785 {
3786 /* normal in/out */
3787 uint32_t uAndVal = g_aIOOpAnd[uIOWidth];
3788
3789 Assert(!VMX_EXIT_QUALIFICATION_IO_REP(exitQualification));
3790
3791 if (fIOWrite)
3792 {
3793 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitIOWrite);
3794 rc = VBOXSTRICTRC_TODO(IOMIOPortWrite(pVM, uPort, pCtx->eax & uAndVal, cbSize));
3795 if (rc == VINF_IOM_HC_IOPORT_WRITE)
3796 HWACCMR0SavePendingIOPortWrite(pVCpu, pCtx->rip, pCtx->rip + cbInstr, uPort, uAndVal, cbSize);
3797 }
3798 else
3799 {
3800 uint32_t u32Val = 0;
3801
3802 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitIORead);
3803 rc = VBOXSTRICTRC_TODO(IOMIOPortRead(pVM, uPort, &u32Val, cbSize));
3804 if (IOM_SUCCESS(rc))
3805 {
3806 /* Write back to the EAX register. */
3807 pCtx->eax = (pCtx->eax & ~uAndVal) | (u32Val & uAndVal);
3808 }
3809 else
3810 if (rc == VINF_IOM_HC_IOPORT_READ)
3811 HWACCMR0SavePendingIOPortRead(pVCpu, pCtx->rip, pCtx->rip + cbInstr, uPort, uAndVal, cbSize);
3812 }
3813 }
3814 /*
3815 * Handled the I/O return codes.
3816 * (The unhandled cases end up with rc == VINF_EM_RAW_EMULATE_INSTR.)
3817 */
3818 if (IOM_SUCCESS(rc))
3819 {
3820 /* Update EIP and continue execution. */
3821 pCtx->rip += cbInstr;
3822 if (RT_LIKELY(rc == VINF_SUCCESS))
3823 {
3824 /* If any IO breakpoints are armed, then we should check if a debug trap needs to be generated. */
3825 if (pCtx->dr[7] & X86_DR7_ENABLED_MASK)
3826 {
3827 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatDRxIOCheck);
3828 for (unsigned i=0;i<4;i++)
3829 {
3830 unsigned uBPLen = g_aIOSize[X86_DR7_GET_LEN(pCtx->dr[7], i)];
3831
3832 if ( (uPort >= pCtx->dr[i] && uPort < pCtx->dr[i] + uBPLen)
3833 && (pCtx->dr[7] & (X86_DR7_L(i) | X86_DR7_G(i)))
3834 && (pCtx->dr[7] & X86_DR7_RW(i, X86_DR7_RW_IO)) == X86_DR7_RW(i, X86_DR7_RW_IO))
3835 {
3836 uint64_t uDR6;
3837
3838 Assert(CPUMIsGuestDebugStateActive(pVCpu));
3839
3840 uDR6 = ASMGetDR6();
3841
3842 /* Clear all breakpoint status flags and set the one we just hit. */
3843 uDR6 &= ~(X86_DR6_B0|X86_DR6_B1|X86_DR6_B2|X86_DR6_B3);
3844 uDR6 |= (uint64_t)RT_BIT(i);
3845
3846 /* Note: AMD64 Architecture Programmer's Manual 13.1:
3847 * Bits 15:13 of the DR6 register is never cleared by the processor and must be cleared by software after
3848 * the contents have been read.
3849 */
3850 ASMSetDR6(uDR6);
3851
3852 /* X86_DR7_GD will be cleared if drx accesses should be trapped inside the guest. */
3853 pCtx->dr[7] &= ~X86_DR7_GD;
3854
3855 /* Paranoia. */
3856 pCtx->dr[7] &= 0xffffffff; /* upper 32 bits reserved */
3857 pCtx->dr[7] &= ~(RT_BIT(11) | RT_BIT(12) | RT_BIT(14) | RT_BIT(15)); /* must be zero */
3858 pCtx->dr[7] |= 0x400; /* must be one */
3859
3860 /* Resync DR7 */
3861 rc = VMXWriteVMCS64(VMX_VMCS64_GUEST_DR7, pCtx->dr[7]);
3862 AssertRC(rc);
3863
3864 /* Construct inject info. */
3865 intInfo = X86_XCPT_DB;
3866 intInfo |= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
3867 intInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HWEXCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
3868
3869 Log(("Inject IO debug trap at %RGv\n", (RTGCPTR)pCtx->rip));
3870 rc = VMXR0InjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), 0, 0);
3871 AssertRC(rc);
3872
3873 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub1, y1);
3874 goto ResumeExecution;
3875 }
3876 }
3877 }
3878
3879 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub1, y1);
3880 goto ResumeExecution;
3881 }
3882 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub1, y1);
3883 break;
3884 }
3885
3886#ifdef VBOX_STRICT
3887 if (rc == VINF_IOM_HC_IOPORT_READ)
3888 Assert(!fIOWrite);
3889 else if (rc == VINF_IOM_HC_IOPORT_WRITE)
3890 Assert(fIOWrite);
3891 else
3892 AssertMsg(RT_FAILURE(rc) || rc == VINF_EM_RAW_EMULATE_INSTR || rc == VINF_EM_RAW_GUEST_TRAP || rc == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", rc));
3893#endif
3894 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub1, y1);
3895 break;
3896 }
3897
3898 case VMX_EXIT_TPR: /* 43 TPR below threshold. Guest software executed MOV to CR8. */
3899 LogFlow(("VMX_EXIT_TPR\n"));
3900 /* RIP is already set to the next instruction and the TPR has been synced back. Just resume. */
3901 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub1, y1);
3902 goto ResumeExecution;
3903
3904 case VMX_EXIT_APIC_ACCESS: /* 44 APIC access. Guest software attempted to access memory at a physical address on the APIC-access page. */
3905 {
3906 LogFlow(("VMX_EXIT_APIC_ACCESS\n"));
3907 unsigned uAccessType = VMX_EXIT_QUALIFICATION_APIC_ACCESS_TYPE(exitQualification);
3908
3909 switch(uAccessType)
3910 {
3911 case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
3912 case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
3913 {
3914 RTGCPHYS GCPhys;
3915 PDMApicGetBase(pVM, &GCPhys);
3916 GCPhys &= PAGE_BASE_GC_MASK;
3917 GCPhys += VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(exitQualification);
3918
3919 LogFlow(("Apic access at %RGp\n", GCPhys));
3920 rc = VBOXSTRICTRC_TODO(IOMMMIOPhysHandler(pVM, (uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ) ? 0 : X86_TRAP_PF_RW, CPUMCTX2CORE(pCtx), GCPhys));
3921 if (rc == VINF_SUCCESS)
3922 {
3923 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub1, y1);
3924 goto ResumeExecution; /* rip already updated */
3925 }
3926 break;
3927 }
3928
3929 default:
3930 rc = VINF_EM_RAW_EMULATE_INSTR;
3931 break;
3932 }
3933 break;
3934 }
3935
3936 case VMX_EXIT_PREEMPTION_TIMER: /* 52 VMX-preemption timer expired. The preemption timer counted down to zero. */
3937 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub1, y1);
3938 goto ResumeExecution;
3939
3940 default:
3941 /* The rest is handled after syncing the entire CPU state. */
3942 break;
3943 }
3944
3945 /* Note: the guest state isn't entirely synced back at this stage. */
3946
3947 /* Investigate why there was a VM-exit. (part 2) */
3948 switch (exitReason)
3949 {
3950 case VMX_EXIT_EXCEPTION: /* 0 Exception or non-maskable interrupt (NMI). */
3951 case VMX_EXIT_EXTERNAL_IRQ: /* 1 External interrupt. */
3952 case VMX_EXIT_EPT_VIOLATION:
3953 case VMX_EXIT_PREEMPTION_TIMER: /* 52 VMX-preemption timer expired. The preemption timer counted down to zero. */
3954 /* Already handled above. */
3955 break;
3956
3957 case VMX_EXIT_TRIPLE_FAULT: /* 2 Triple fault. */
3958 rc = VINF_EM_RESET; /* Triple fault equals a reset. */
3959 break;
3960
3961 case VMX_EXIT_INIT_SIGNAL: /* 3 INIT signal. */
3962 case VMX_EXIT_SIPI: /* 4 Start-up IPI (SIPI). */
3963 rc = VINF_EM_RAW_INTERRUPT;
3964 AssertFailed(); /* Can't happen. Yet. */
3965 break;
3966
3967 case VMX_EXIT_IO_SMI_IRQ: /* 5 I/O system-management interrupt (SMI). */
3968 case VMX_EXIT_SMI_IRQ: /* 6 Other SMI. */
3969 rc = VINF_EM_RAW_INTERRUPT;
3970 AssertFailed(); /* Can't happen afaik. */
3971 break;
3972
3973 case VMX_EXIT_TASK_SWITCH: /* 9 Task switch: too complicated to emulate, so fall back to the recompiler */
3974 Log(("VMX_EXIT_TASK_SWITCH: exit=%RX64\n", exitQualification));
3975 if ( (VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE(exitQualification) == VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE_IDT)
3976 && pVCpu->hwaccm.s.Event.fPending)
3977 {
3978 /* Caused by an injected interrupt. */
3979 pVCpu->hwaccm.s.Event.fPending = false;
3980
3981 Log(("VMX_EXIT_TASK_SWITCH: reassert trap %d\n", VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVCpu->hwaccm.s.Event.intInfo)));
3982 Assert(!VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(pVCpu->hwaccm.s.Event.intInfo));
3983 rc = TRPMAssertTrap(pVCpu, VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVCpu->hwaccm.s.Event.intInfo), TRPM_HARDWARE_INT);
3984 AssertRC(rc);
3985 }
3986 /* else Exceptions and software interrupts can just be restarted. */
3987 rc = VERR_EM_INTERPRETER;
3988 break;
3989
3990 case VMX_EXIT_HLT: /* 12 Guest software attempted to execute HLT. */
3991 /** Check if external interrupts are pending; if so, don't switch back. */
3992 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitHlt);
3993 pCtx->rip++; /* skip hlt */
3994 if (EMShouldContinueAfterHalt(pVCpu, pCtx))
3995 goto ResumeExecution;
3996
3997 rc = VINF_EM_HALT;
3998 break;
3999
4000 case VMX_EXIT_MWAIT: /* 36 Guest software executed MWAIT. */
4001 Log2(("VMX: mwait\n"));
4002 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitMwait);
4003 rc = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pCtx));
4004 if ( rc == VINF_EM_HALT
4005 || rc == VINF_SUCCESS)
4006 {
4007 /* Update EIP and continue execution. */
4008 pCtx->rip += cbInstr;
4009
4010 /** Check if external interrupts are pending; if so, don't switch back. */
4011 if ( rc == VINF_SUCCESS
4012 || ( rc == VINF_EM_HALT
4013 && EMShouldContinueAfterHalt(pVCpu, pCtx))
4014 )
4015 goto ResumeExecution;
4016 }
4017 AssertMsg(rc == VERR_EM_INTERPRETER || rc == VINF_EM_HALT, ("EMU: mwait failed with %Rrc\n", rc));
4018 break;
4019
4020 case VMX_EXIT_RSM: /* 17 Guest software attempted to execute RSM in SMM. */
4021 AssertFailed(); /* can't happen. */
4022 rc = VERR_EM_INTERPRETER;
4023 break;
4024
4025 case VMX_EXIT_VMCALL: /* 18 Guest software executed VMCALL. */
4026 case VMX_EXIT_VMCLEAR: /* 19 Guest software executed VMCLEAR. */
4027 case VMX_EXIT_VMLAUNCH: /* 20 Guest software executed VMLAUNCH. */
4028 case VMX_EXIT_VMPTRLD: /* 21 Guest software executed VMPTRLD. */
4029 case VMX_EXIT_VMPTRST: /* 22 Guest software executed VMPTRST. */
4030 case VMX_EXIT_VMREAD: /* 23 Guest software executed VMREAD. */
4031 case VMX_EXIT_VMRESUME: /* 24 Guest software executed VMRESUME. */
4032 case VMX_EXIT_VMWRITE: /* 25 Guest software executed VMWRITE. */
4033 case VMX_EXIT_VMXOFF: /* 26 Guest software executed VMXOFF. */
4034 case VMX_EXIT_VMXON: /* 27 Guest software executed VMXON. */
4035 /** @todo inject #UD immediately */
4036 rc = VERR_EM_INTERPRETER;
4037 break;
4038
4039 case VMX_EXIT_CPUID: /* 10 Guest software attempted to execute CPUID. */
4040 case VMX_EXIT_RDTSC: /* 16 Guest software attempted to execute RDTSC. */
4041 case VMX_EXIT_INVPG: /* 14 Guest software attempted to execute INVPG. */
4042 case VMX_EXIT_CRX_MOVE: /* 28 Control-register accesses. */
4043 case VMX_EXIT_DRX_MOVE: /* 29 Debug-register accesses. */
4044 case VMX_EXIT_PORT_IO: /* 30 I/O instruction. */
4045 case VMX_EXIT_RDPMC: /* 15 Guest software attempted to execute RDPMC. */
4046 /* already handled above */
4047 AssertMsg( rc == VINF_PGM_CHANGE_MODE
4048 || rc == VINF_EM_RAW_INTERRUPT
4049 || rc == VERR_EM_INTERPRETER
4050 || rc == VINF_EM_RAW_EMULATE_INSTR
4051 || rc == VINF_PGM_SYNC_CR3
4052 || rc == VINF_IOM_HC_IOPORT_READ
4053 || rc == VINF_IOM_HC_IOPORT_WRITE
4054 || rc == VINF_EM_RAW_GUEST_TRAP
4055 || rc == VINF_TRPM_XCPT_DISPATCHED
4056 || rc == VINF_EM_RESCHEDULE_REM,
4057 ("rc = %d\n", rc));
4058 break;
4059
4060 case VMX_EXIT_TPR: /* 43 TPR below threshold. Guest software executed MOV to CR8. */
4061 case VMX_EXIT_APIC_ACCESS: /* 44 APIC access. Guest software attempted to access memory at a physical address on the APIC-access page. */
4062 case VMX_EXIT_RDMSR: /* 31 RDMSR. Guest software attempted to execute RDMSR. */
4063 case VMX_EXIT_WRMSR: /* 32 WRMSR. Guest software attempted to execute WRMSR. */
4064 case VMX_EXIT_PAUSE: /* 40 Guest software attempted to execute PAUSE. */
4065 case VMX_EXIT_MONITOR: /* 39 Guest software attempted to execute MONITOR. */
4066 /* Note: If we decide to emulate them here, then we must sync the MSRs that could have been changed (sysenter, fs/gs base)!!! */
4067 rc = VERR_EM_INTERPRETER;
4068 break;
4069
4070 case VMX_EXIT_IRQ_WINDOW: /* 7 Interrupt window. */
4071 Assert(rc == VINF_EM_RAW_INTERRUPT);
4072 break;
4073
4074 case VMX_EXIT_ERR_INVALID_GUEST_STATE: /* 33 VM-entry failure due to invalid guest state. */
4075 {
4076#ifdef VBOX_STRICT
4077 RTCCUINTREG val2 = 0;
4078
4079 Log(("VMX_EXIT_ERR_INVALID_GUEST_STATE\n"));
4080
4081 VMXReadVMCS(VMX_VMCS64_GUEST_RIP, &val2);
4082 Log(("Old eip %RGv new %RGv\n", (RTGCPTR)pCtx->rip, (RTGCPTR)val2));
4083
4084 VMXReadVMCS(VMX_VMCS64_GUEST_CR0, &val2);
4085 Log(("VMX_VMCS_GUEST_CR0 %RX64\n", (uint64_t)val2));
4086
4087 VMXReadVMCS(VMX_VMCS64_GUEST_CR3, &val2);
4088 Log(("VMX_VMCS_GUEST_CR3 %RX64\n", (uint64_t)val2));
4089
4090 VMXReadVMCS(VMX_VMCS64_GUEST_CR4, &val2);
4091 Log(("VMX_VMCS_GUEST_CR4 %RX64\n", (uint64_t)val2));
4092
4093 VMXReadVMCS(VMX_VMCS_GUEST_RFLAGS, &val2);
4094 Log(("VMX_VMCS_GUEST_RFLAGS %08x\n", val2));
4095
4096 VMX_LOG_SELREG(CS, "CS", val2);
4097 VMX_LOG_SELREG(DS, "DS", val2);
4098 VMX_LOG_SELREG(ES, "ES", val2);
4099 VMX_LOG_SELREG(FS, "FS", val2);
4100 VMX_LOG_SELREG(GS, "GS", val2);
4101 VMX_LOG_SELREG(SS, "SS", val2);
4102 VMX_LOG_SELREG(TR, "TR", val2);
4103 VMX_LOG_SELREG(LDTR, "LDTR", val2);
4104
4105 VMXReadVMCS(VMX_VMCS64_GUEST_GDTR_BASE, &val2);
4106 Log(("VMX_VMCS_GUEST_GDTR_BASE %RX64\n", (uint64_t)val2));
4107 VMXReadVMCS(VMX_VMCS64_GUEST_IDTR_BASE, &val2);
4108 Log(("VMX_VMCS_GUEST_IDTR_BASE %RX64\n", (uint64_t)val2));
4109#endif /* VBOX_STRICT */
4110 rc = VERR_VMX_INVALID_GUEST_STATE;
4111 break;
4112 }
4113
4114 case VMX_EXIT_ERR_MSR_LOAD: /* 34 VM-entry failure due to MSR loading. */
4115 case VMX_EXIT_ERR_MACHINE_CHECK: /* 41 VM-entry failure due to machine-check. */
4116 default:
4117 rc = VERR_VMX_UNEXPECTED_EXIT_CODE;
4118 AssertMsgFailed(("Unexpected exit code %d\n", exitReason)); /* Can't happen. */
4119 break;
4120
4121 }
4122end:
4123
4124 /* Signal changes for the recompiler. */
4125 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR | CPUM_CHANGED_LDTR | CPUM_CHANGED_GDTR | CPUM_CHANGED_IDTR | CPUM_CHANGED_TR | CPUM_CHANGED_HIDDEN_SEL_REGS);
4126
4127 /* If we executed vmlaunch/vmresume and an external irq was pending, then we don't have to do a full sync the next time. */
4128 if ( exitReason == VMX_EXIT_EXTERNAL_IRQ
4129 && !VMX_EXIT_INTERRUPTION_INFO_VALID(intInfo))
4130 {
4131 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatPendingHostIrq);
4132 /* On the next entry we'll only sync the host context. */
4133 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_HOST_CONTEXT;
4134 }
4135 else
4136 {
4137 /* On the next entry we'll sync everything. */
4138 /** @todo we can do better than this */
4139 /* Not in the VINF_PGM_CHANGE_MODE though! */
4140 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_ALL;
4141 }
4142
4143 /* translate into a less severe return code */
4144 if (rc == VERR_EM_INTERPRETER)
4145 rc = VINF_EM_RAW_EMULATE_INSTR;
4146 else
4147 /* Try to extract more information about what might have gone wrong here. */
4148 if (rc == VERR_VMX_INVALID_VMCS_PTR)
4149 {
4150 VMXGetActivateVMCS(&pVCpu->hwaccm.s.vmx.lasterror.u64VMCSPhys);
4151 pVCpu->hwaccm.s.vmx.lasterror.ulVMCSRevision = *(uint32_t *)pVCpu->hwaccm.s.vmx.pVMCS;
4152 pVCpu->hwaccm.s.vmx.lasterror.idEnteredCpu = pVCpu->hwaccm.s.idEnteredCpu;
4153 pVCpu->hwaccm.s.vmx.lasterror.idCurrentCpu = RTMpCpuId();
4154 }
4155
4156 /* Just set the correct state here instead of trying to catch every goto above. */
4157 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED, VMCPUSTATE_STARTED_EXEC);
4158
4159#ifdef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
4160 /* Restore interrupts if we exitted after disabling them. */
4161 if (uOldEFlags != ~(RTCCUINTREG)0)
4162 ASMSetFlags(uOldEFlags);
4163#endif
4164
4165 STAM_STATS({
4166 if (fStatExit2Started) STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2, y);
4167 else if (fStatEntryStarted) STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatEntry, x);
4168 });
4169 Log2(("X"));
4170 return rc;
4171}
4172
4173
4174/**
4175 * Enters the VT-x session
4176 *
4177 * @returns VBox status code.
4178 * @param pVM The VM to operate on.
4179 * @param pVCpu The VMCPU to operate on.
4180 * @param pCpu CPU info struct
4181 */
4182VMMR0DECL(int) VMXR0Enter(PVM pVM, PVMCPU pVCpu, PHWACCM_CPUINFO pCpu)
4183{
4184 Assert(pVM->hwaccm.s.vmx.fSupported);
4185
4186 unsigned cr4 = ASMGetCR4();
4187 if (!(cr4 & X86_CR4_VMXE))
4188 {
4189 AssertMsgFailed(("X86_CR4_VMXE should be set!\n"));
4190 return VERR_VMX_X86_CR4_VMXE_CLEARED;
4191 }
4192
4193 /* Activate the VM Control Structure. */
4194 int rc = VMXActivateVMCS(pVCpu->hwaccm.s.vmx.pVMCSPhys);
4195 if (RT_FAILURE(rc))
4196 return rc;
4197
4198 pVCpu->hwaccm.s.fResumeVM = false;
4199 return VINF_SUCCESS;
4200}
4201
4202
4203/**
4204 * Leaves the VT-x session
4205 *
4206 * @returns VBox status code.
4207 * @param pVM The VM to operate on.
4208 * @param pVCpu The VMCPU to operate on.
4209 * @param pCtx CPU context
4210 */
4211VMMR0DECL(int) VMXR0Leave(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
4212{
4213 Assert(pVM->hwaccm.s.vmx.fSupported);
4214
4215#ifdef DEBUG
4216 if (CPUMIsHyperDebugStateActive(pVCpu))
4217 {
4218 CPUMR0LoadHostDebugState(pVM, pVCpu);
4219 Assert(pVCpu->hwaccm.s.vmx.proc_ctls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT);
4220 }
4221 else
4222#endif
4223 /* Save the guest debug state if necessary. */
4224 if (CPUMIsGuestDebugStateActive(pVCpu))
4225 {
4226 CPUMR0SaveGuestDebugState(pVM, pVCpu, pCtx, true /* save DR6 */);
4227
4228 /* Enable drx move intercepts again. */
4229 pVCpu->hwaccm.s.vmx.proc_ctls |= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT;
4230 int rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
4231 AssertRC(rc);
4232
4233 /* Resync the debug registers the next time. */
4234 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_GUEST_DEBUG;
4235 }
4236 else
4237 Assert(pVCpu->hwaccm.s.vmx.proc_ctls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT);
4238
4239 /* Clear VM Control Structure. Marking it inactive, clearing implementation specific data and writing back VMCS data to memory. */
4240 int rc = VMXClearVMCS(pVCpu->hwaccm.s.vmx.pVMCSPhys);
4241 AssertRC(rc);
4242
4243 return VINF_SUCCESS;
4244}
4245
4246/**
4247 * Flush the TLB (EPT)
4248 *
4249 * @returns VBox status code.
4250 * @param pVM The VM to operate on.
4251 * @param pVCpu The VM CPU to operate on.
4252 * @param enmFlush Type of flush
4253 * @param GCPhys Physical address of the page to flush
4254 */
4255static void vmxR0FlushEPT(PVM pVM, PVMCPU pVCpu, VMX_FLUSH enmFlush, RTGCPHYS GCPhys)
4256{
4257 uint64_t descriptor[2];
4258
4259 LogFlow(("vmxR0FlushEPT %d %RGv\n", enmFlush, GCPhys));
4260 Assert(pVM->hwaccm.s.fNestedPaging);
4261 descriptor[0] = pVCpu->hwaccm.s.vmx.GCPhysEPTP;
4262 descriptor[1] = GCPhys;
4263 int rc = VMXR0InvEPT(enmFlush, &descriptor[0]);
4264 AssertRC(rc);
4265}
4266
4267#ifdef HWACCM_VTX_WITH_VPID
4268/**
4269 * Flush the TLB (EPT)
4270 *
4271 * @returns VBox status code.
4272 * @param pVM The VM to operate on.
4273 * @param pVCpu The VM CPU to operate on.
4274 * @param enmFlush Type of flush
4275 * @param GCPtr Virtual address of the page to flush
4276 */
4277static void vmxR0FlushVPID(PVM pVM, PVMCPU pVCpu, VMX_FLUSH enmFlush, RTGCPTR GCPtr)
4278{
4279#if HC_ARCH_BITS == 32
4280 /* If we get a flush in 64 bits guest mode, then force a full TLB flush. Invvpid probably takes only 32 bits addresses. (@todo) */
4281 if ( CPUMIsGuestInLongMode(pVCpu)
4282 && !VMX_IS_64BIT_HOST_MODE())
4283 {
4284 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
4285 }
4286 else
4287#endif
4288 {
4289 uint64_t descriptor[2];
4290
4291 Assert(pVM->hwaccm.s.vmx.fVPID);
4292 descriptor[0] = pVCpu->hwaccm.s.uCurrentASID;
4293 descriptor[1] = GCPtr;
4294 int rc = VMXR0InvVPID(enmFlush, &descriptor[0]);
4295 AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvVPID %x %x %RGv failed with %d\n", enmFlush, pVCpu->hwaccm.s.uCurrentASID, GCPtr, rc));
4296 }
4297}
4298#endif /* HWACCM_VTX_WITH_VPID */
4299
4300/**
4301 * Invalidates a guest page
4302 *
4303 * @returns VBox status code.
4304 * @param pVM The VM to operate on.
4305 * @param pVCpu The VM CPU to operate on.
4306 * @param GCVirt Page to invalidate
4307 */
4308VMMR0DECL(int) VMXR0InvalidatePage(PVM pVM, PVMCPU pVCpu, RTGCPTR GCVirt)
4309{
4310 bool fFlushPending = VMCPU_FF_ISSET(pVCpu, VMCPU_FF_TLB_FLUSH);
4311
4312 Log2(("VMXR0InvalidatePage %RGv\n", GCVirt));
4313
4314 /* Only relevant if we want to use VPID.
4315 * In the nested paging case we still see such calls, but
4316 * can safely ignore them. (e.g. after cr3 updates)
4317 */
4318#ifdef HWACCM_VTX_WITH_VPID
4319 /* Skip it if a TLB flush is already pending. */
4320 if ( !fFlushPending
4321 && pVM->hwaccm.s.vmx.fVPID)
4322 vmxR0FlushVPID(pVM, pVCpu, pVM->hwaccm.s.vmx.enmFlushPage, GCVirt);
4323#endif /* HWACCM_VTX_WITH_VPID */
4324
4325 return VINF_SUCCESS;
4326}
4327
4328/**
4329 * Invalidates a guest page by physical address
4330 *
4331 * NOTE: Assumes the current instruction references this physical page though a virtual address!!
4332 *
4333 * @returns VBox status code.
4334 * @param pVM The VM to operate on.
4335 * @param pVCpu The VM CPU to operate on.
4336 * @param GCPhys Page to invalidate
4337 */
4338VMMR0DECL(int) VMXR0InvalidatePhysPage(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys)
4339{
4340 bool fFlushPending = VMCPU_FF_ISSET(pVCpu, VMCPU_FF_TLB_FLUSH);
4341
4342 Assert(pVM->hwaccm.s.fNestedPaging);
4343
4344 LogFlow(("VMXR0InvalidatePhysPage %RGp\n", GCPhys));
4345
4346 /* Skip it if a TLB flush is already pending. */
4347 if (!fFlushPending)
4348 vmxR0FlushEPT(pVM, pVCpu, pVM->hwaccm.s.vmx.enmFlushPage, GCPhys);
4349
4350 return VINF_SUCCESS;
4351}
4352
4353/**
4354 * Report world switch error and dump some useful debug info
4355 *
4356 * @param pVM The VM to operate on.
4357 * @param pVCpu The VMCPU to operate on.
4358 * @param rc Return code
4359 * @param pCtx Current CPU context (not updated)
4360 */
4361static void VMXR0ReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, int rc, PCPUMCTX pCtx)
4362{
4363 switch (rc)
4364 {
4365 case VERR_VMX_INVALID_VMXON_PTR:
4366 AssertFailed();
4367 break;
4368
4369 case VERR_VMX_UNABLE_TO_START_VM:
4370 case VERR_VMX_UNABLE_TO_RESUME_VM:
4371 {
4372 int rc2;
4373 RTCCUINTREG exitReason, instrError;
4374
4375 rc2 = VMXReadVMCS(VMX_VMCS32_RO_EXIT_REASON, &exitReason);
4376 rc2 |= VMXReadVMCS(VMX_VMCS32_RO_VM_INSTR_ERROR, &instrError);
4377 AssertRC(rc2);
4378 if (rc2 == VINF_SUCCESS)
4379 {
4380 Log(("Unable to start/resume VM for reason: %x. Instruction error %x\n", (uint32_t)exitReason, (uint32_t)instrError));
4381 Log(("Current stack %08x\n", &rc2));
4382
4383 pVCpu->hwaccm.s.vmx.lasterror.ulInstrError = instrError;
4384 pVCpu->hwaccm.s.vmx.lasterror.ulExitReason = exitReason;
4385
4386#ifdef VBOX_STRICT
4387 RTGDTR gdtr;
4388 PCX86DESCHC pDesc;
4389 RTCCUINTREG val;
4390
4391 ASMGetGDTR(&gdtr);
4392
4393 VMXReadVMCS(VMX_VMCS64_GUEST_RIP, &val);
4394 Log(("Old eip %RGv new %RGv\n", (RTGCPTR)pCtx->rip, (RTGCPTR)val));
4395 VMXReadVMCS(VMX_VMCS_CTRL_PIN_EXEC_CONTROLS, &val);
4396 Log(("VMX_VMCS_CTRL_PIN_EXEC_CONTROLS %08x\n", val));
4397 VMXReadVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, &val);
4398 Log(("VMX_VMCS_CTRL_PROC_EXEC_CONTROLS %08x\n", val));
4399 VMXReadVMCS(VMX_VMCS_CTRL_ENTRY_CONTROLS, &val);
4400 Log(("VMX_VMCS_CTRL_ENTRY_CONTROLS %08x\n", val));
4401 VMXReadVMCS(VMX_VMCS_CTRL_EXIT_CONTROLS, &val);
4402 Log(("VMX_VMCS_CTRL_EXIT_CONTROLS %08x\n", val));
4403
4404 VMXReadVMCS(VMX_VMCS_HOST_CR0, &val);
4405 Log(("VMX_VMCS_HOST_CR0 %08x\n", val));
4406
4407 VMXReadVMCS(VMX_VMCS_HOST_CR3, &val);
4408 Log(("VMX_VMCS_HOST_CR3 %08x\n", val));
4409
4410 VMXReadVMCS(VMX_VMCS_HOST_CR4, &val);
4411 Log(("VMX_VMCS_HOST_CR4 %08x\n", val));
4412
4413 VMXReadVMCS(VMX_VMCS16_HOST_FIELD_CS, &val);
4414 Log(("VMX_VMCS_HOST_FIELD_CS %08x\n", val));
4415
4416 VMXReadVMCS(VMX_VMCS_GUEST_RFLAGS, &val);
4417 Log(("VMX_VMCS_GUEST_RFLAGS %08x\n", val));
4418
4419 if (val < gdtr.cbGdt)
4420 {
4421 pDesc = (PCX86DESCHC)(gdtr.pGdt + (val & X86_SEL_MASK));
4422 HWACCMR0DumpDescriptor(pDesc, val, "CS: ");
4423 }
4424
4425 VMXReadVMCS(VMX_VMCS16_HOST_FIELD_DS, &val);
4426 Log(("VMX_VMCS_HOST_FIELD_DS %08x\n", val));
4427 if (val < gdtr.cbGdt)
4428 {
4429 pDesc = (PCX86DESCHC)(gdtr.pGdt + (val & X86_SEL_MASK));
4430 HWACCMR0DumpDescriptor(pDesc, val, "DS: ");
4431 }
4432
4433 VMXReadVMCS(VMX_VMCS16_HOST_FIELD_ES, &val);
4434 Log(("VMX_VMCS_HOST_FIELD_ES %08x\n", val));
4435 if (val < gdtr.cbGdt)
4436 {
4437 pDesc = (PCX86DESCHC)(gdtr.pGdt + (val & X86_SEL_MASK));
4438 HWACCMR0DumpDescriptor(pDesc, val, "ES: ");
4439 }
4440
4441 VMXReadVMCS(VMX_VMCS16_HOST_FIELD_FS, &val);
4442 Log(("VMX_VMCS16_HOST_FIELD_FS %08x\n", val));
4443 if (val < gdtr.cbGdt)
4444 {
4445 pDesc = (PCX86DESCHC)(gdtr.pGdt + (val & X86_SEL_MASK));
4446 HWACCMR0DumpDescriptor(pDesc, val, "FS: ");
4447 }
4448
4449 VMXReadVMCS(VMX_VMCS16_HOST_FIELD_GS, &val);
4450 Log(("VMX_VMCS16_HOST_FIELD_GS %08x\n", val));
4451 if (val < gdtr.cbGdt)
4452 {
4453 pDesc = (PCX86DESCHC)(gdtr.pGdt + (val & X86_SEL_MASK));
4454 HWACCMR0DumpDescriptor(pDesc, val, "GS: ");
4455 }
4456
4457 VMXReadVMCS(VMX_VMCS16_HOST_FIELD_SS, &val);
4458 Log(("VMX_VMCS16_HOST_FIELD_SS %08x\n", val));
4459 if (val < gdtr.cbGdt)
4460 {
4461 pDesc = (PCX86DESCHC)(gdtr.pGdt + (val & X86_SEL_MASK));
4462 HWACCMR0DumpDescriptor(pDesc, val, "SS: ");
4463 }
4464
4465 VMXReadVMCS(VMX_VMCS16_HOST_FIELD_TR, &val);
4466 Log(("VMX_VMCS16_HOST_FIELD_TR %08x\n", val));
4467 if (val < gdtr.cbGdt)
4468 {
4469 pDesc = (PCX86DESCHC)(gdtr.pGdt + (val & X86_SEL_MASK));
4470 HWACCMR0DumpDescriptor(pDesc, val, "TR: ");
4471 }
4472
4473 VMXReadVMCS(VMX_VMCS_HOST_TR_BASE, &val);
4474 Log(("VMX_VMCS_HOST_TR_BASE %RHv\n", val));
4475
4476 VMXReadVMCS(VMX_VMCS_HOST_GDTR_BASE, &val);
4477 Log(("VMX_VMCS_HOST_GDTR_BASE %RHv\n", val));
4478 VMXReadVMCS(VMX_VMCS_HOST_IDTR_BASE, &val);
4479 Log(("VMX_VMCS_HOST_IDTR_BASE %RHv\n", val));
4480
4481 VMXReadVMCS(VMX_VMCS32_HOST_SYSENTER_CS, &val);
4482 Log(("VMX_VMCS_HOST_SYSENTER_CS %08x\n", val));
4483
4484 VMXReadVMCS(VMX_VMCS_HOST_SYSENTER_EIP, &val);
4485 Log(("VMX_VMCS_HOST_SYSENTER_EIP %RHv\n", val));
4486
4487 VMXReadVMCS(VMX_VMCS_HOST_SYSENTER_ESP, &val);
4488 Log(("VMX_VMCS_HOST_SYSENTER_ESP %RHv\n", val));
4489
4490 VMXReadVMCS(VMX_VMCS_HOST_RSP, &val);
4491 Log(("VMX_VMCS_HOST_RSP %RHv\n", val));
4492 VMXReadVMCS(VMX_VMCS_HOST_RIP, &val);
4493 Log(("VMX_VMCS_HOST_RIP %RHv\n", val));
4494
4495# if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4496 if (VMX_IS_64BIT_HOST_MODE())
4497 {
4498 Log(("MSR_K6_EFER = %RX64\n", ASMRdMsr(MSR_K6_EFER)));
4499 Log(("MSR_K6_STAR = %RX64\n", ASMRdMsr(MSR_K6_STAR)));
4500 Log(("MSR_K8_LSTAR = %RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
4501 Log(("MSR_K8_CSTAR = %RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
4502 Log(("MSR_K8_SF_MASK = %RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
4503 }
4504# endif
4505#endif /* VBOX_STRICT */
4506 }
4507 break;
4508 }
4509
4510 default:
4511 /* impossible */
4512 AssertMsgFailed(("%Rrc (%#x)\n", rc, rc));
4513 break;
4514 }
4515}
4516
4517#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4518/**
4519 * Prepares for and executes VMLAUNCH (64 bits guest mode)
4520 *
4521 * @returns VBox status code
4522 * @param fResume vmlauch/vmresume
4523 * @param pCtx Guest context
4524 * @param pCache VMCS cache
4525 * @param pVM The VM to operate on.
4526 * @param pVCpu The VMCPU to operate on.
4527 */
4528DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
4529{
4530 uint32_t aParam[6];
4531 PHWACCM_CPUINFO pCpu;
4532 RTHCPHYS pPageCpuPhys;
4533 int rc;
4534
4535 pCpu = HWACCMR0GetCurrentCpu();
4536 pPageCpuPhys = RTR0MemObjGetPagePhysAddr(pCpu->pMemObj, 0);
4537
4538#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4539 pCache->uPos = 1;
4540 pCache->interPD = PGMGetInterPaeCR3(pVM);
4541 pCache->pSwitcher = (uint64_t)pVM->hwaccm.s.pfnHost32ToGuest64R0;
4542#endif
4543
4544#ifdef DEBUG
4545 pCache->TestIn.pPageCpuPhys = 0;
4546 pCache->TestIn.pVMCSPhys = 0;
4547 pCache->TestIn.pCache = 0;
4548 pCache->TestOut.pVMCSPhys = 0;
4549 pCache->TestOut.pCache = 0;
4550 pCache->TestOut.pCtx = 0;
4551 pCache->TestOut.eflags = 0;
4552#endif
4553
4554 aParam[0] = (uint32_t)(pPageCpuPhys); /* Param 1: VMXON physical address - Lo. */
4555 aParam[1] = (uint32_t)(pPageCpuPhys >> 32); /* Param 1: VMXON physical address - Hi. */
4556 aParam[2] = (uint32_t)(pVCpu->hwaccm.s.vmx.pVMCSPhys); /* Param 2: VMCS physical address - Lo. */
4557 aParam[3] = (uint32_t)(pVCpu->hwaccm.s.vmx.pVMCSPhys >> 32); /* Param 2: VMCS physical address - Hi. */
4558 aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hwaccm.s.vmx.VMCSCache);
4559 aParam[5] = 0;
4560
4561#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4562 pCtx->dr[4] = pVM->hwaccm.s.vmx.pScratchPhys + 16 + 8;
4563 *(uint32_t *)(pVM->hwaccm.s.vmx.pScratch + 16 + 8) = 1;
4564#endif
4565 rc = VMXR0Execute64BitsHandler(pVM, pVCpu, pCtx, pVM->hwaccm.s.pfnVMXGCStartVM64, 6, &aParam[0]);
4566
4567#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4568 Assert(*(uint32_t *)(pVM->hwaccm.s.vmx.pScratch + 16 + 8) == 5);
4569 Assert(pCtx->dr[4] == 10);
4570 *(uint32_t *)(pVM->hwaccm.s.vmx.pScratch + 16 + 8) = 0xff;
4571#endif
4572
4573#ifdef DEBUG
4574 AssertMsg(pCache->TestIn.pPageCpuPhys == pPageCpuPhys, ("%RHp vs %RHp\n", pCache->TestIn.pPageCpuPhys, pPageCpuPhys));
4575 AssertMsg(pCache->TestIn.pVMCSPhys == pVCpu->hwaccm.s.vmx.pVMCSPhys, ("%RHp vs %RHp\n", pCache->TestIn.pVMCSPhys, pVCpu->hwaccm.s.vmx.pVMCSPhys));
4576 AssertMsg(pCache->TestIn.pVMCSPhys == pCache->TestOut.pVMCSPhys, ("%RHp vs %RHp\n", pCache->TestIn.pVMCSPhys, pCache->TestOut.pVMCSPhys));
4577 AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache, pCache->TestOut.pCache));
4578 AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hwaccm.s.vmx.VMCSCache), ("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hwaccm.s.vmx.VMCSCache)));
4579 AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx, pCache->TestOut.pCtx));
4580 Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
4581#endif
4582 return rc;
4583}
4584
4585/**
4586 * Executes the specified handler in 64 mode
4587 *
4588 * @returns VBox status code.
4589 * @param pVM The VM to operate on.
4590 * @param pVCpu The VMCPU to operate on.
4591 * @param pCtx Guest context
4592 * @param pfnHandler RC handler
4593 * @param cbParam Number of parameters
4594 * @param paParam Array of 32 bits parameters
4595 */
4596VMMR0DECL(int) VMXR0Execute64BitsHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, RTRCPTR pfnHandler, uint32_t cbParam, uint32_t *paParam)
4597{
4598 int rc, rc2;
4599 PHWACCM_CPUINFO pCpu;
4600 RTHCPHYS pPageCpuPhys;
4601 RTHCUINTREG uOldEFlags;
4602
4603 /* @todo This code is not guest SMP safe (hyper stack and switchers) */
4604 AssertReturn(pVM->cCpus == 1, VERR_TOO_MANY_CPUS);
4605 AssertReturn(pVM->hwaccm.s.pfnHost32ToGuest64R0, VERR_INTERNAL_ERROR);
4606 Assert(pVCpu->hwaccm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hwaccm.s.vmx.VMCSCache.Write.aField));
4607 Assert(pVCpu->hwaccm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hwaccm.s.vmx.VMCSCache.Read.aField));
4608
4609#ifdef VBOX_STRICT
4610 for (unsigned i=0;i<pVCpu->hwaccm.s.vmx.VMCSCache.Write.cValidEntries;i++)
4611 Assert(vmxR0IsValidWriteField(pVCpu->hwaccm.s.vmx.VMCSCache.Write.aField[i]));
4612
4613 for (unsigned i=0;i<pVCpu->hwaccm.s.vmx.VMCSCache.Read.cValidEntries;i++)
4614 Assert(vmxR0IsValidReadField(pVCpu->hwaccm.s.vmx.VMCSCache.Read.aField[i]));
4615#endif
4616
4617 /* Disable interrupts. */
4618 uOldEFlags = ASMIntDisableFlags();
4619
4620 pCpu = HWACCMR0GetCurrentCpu();
4621 pPageCpuPhys = RTR0MemObjGetPagePhysAddr(pCpu->pMemObj, 0);
4622
4623 /* Clear VM Control Structure. Marking it inactive, clearing implementation specific data and writing back VMCS data to memory. */
4624 VMXClearVMCS(pVCpu->hwaccm.s.vmx.pVMCSPhys);
4625
4626 /* Leave VMX Root Mode. */
4627 VMXDisable();
4628
4629 ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
4630
4631 CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVM));
4632 CPUMSetHyperEIP(pVCpu, pfnHandler);
4633 for (int i=(int)cbParam-1;i>=0;i--)
4634 CPUMPushHyper(pVCpu, paParam[i]);
4635
4636 STAM_PROFILE_ADV_START(&pVCpu->hwaccm.s.StatWorldSwitch3264, z);
4637 /* Call switcher. */
4638 rc = pVM->hwaccm.s.pfnHost32ToGuest64R0(pVM);
4639 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatWorldSwitch3264, z);
4640
4641 /* Make sure the VMX instructions don't cause #UD faults. */
4642 ASMSetCR4(ASMGetCR4() | X86_CR4_VMXE);
4643
4644 /* Enter VMX Root Mode */
4645 rc2 = VMXEnable(pPageCpuPhys);
4646 if (RT_FAILURE(rc2))
4647 {
4648 if (pVM)
4649 VMXR0CheckError(pVM, pVCpu, rc2);
4650 ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
4651 ASMSetFlags(uOldEFlags);
4652 return VERR_VMX_VMXON_FAILED;
4653 }
4654
4655 rc2 = VMXActivateVMCS(pVCpu->hwaccm.s.vmx.pVMCSPhys);
4656 AssertRC(rc2);
4657 Assert(!(ASMGetFlags() & X86_EFL_IF));
4658 ASMSetFlags(uOldEFlags);
4659 return rc;
4660}
4661
4662#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL) */
4663
4664
4665#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
4666/**
4667 * Executes VMWRITE
4668 *
4669 * @returns VBox status code
4670 * @param pVCpu The VMCPU to operate on.
4671 * @param idxField VMCS index
4672 * @param u64Val 16, 32 or 64 bits value
4673 */
4674VMMR0DECL(int) VMXWriteVMCS64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
4675{
4676 int rc;
4677
4678 switch (idxField)
4679 {
4680 case VMX_VMCS_CTRL_TSC_OFFSET_FULL:
4681 case VMX_VMCS_CTRL_IO_BITMAP_A_FULL:
4682 case VMX_VMCS_CTRL_IO_BITMAP_B_FULL:
4683 case VMX_VMCS_CTRL_MSR_BITMAP_FULL:
4684 case VMX_VMCS_CTRL_VMEXIT_MSR_STORE_FULL:
4685 case VMX_VMCS_CTRL_VMEXIT_MSR_LOAD_FULL:
4686 case VMX_VMCS_CTRL_VMENTRY_MSR_LOAD_FULL:
4687 case VMX_VMCS_CTRL_VAPIC_PAGEADDR_FULL:
4688 case VMX_VMCS_CTRL_APIC_ACCESSADDR_FULL:
4689 case VMX_VMCS_GUEST_LINK_PTR_FULL:
4690 case VMX_VMCS_GUEST_PDPTR0_FULL:
4691 case VMX_VMCS_GUEST_PDPTR1_FULL:
4692 case VMX_VMCS_GUEST_PDPTR2_FULL:
4693 case VMX_VMCS_GUEST_PDPTR3_FULL:
4694 case VMX_VMCS_GUEST_DEBUGCTL_FULL:
4695 case VMX_VMCS_GUEST_EFER_FULL:
4696 case VMX_VMCS_CTRL_EPTP_FULL:
4697 /* These fields consist of two parts, which are both writable in 32 bits mode. */
4698 rc = VMXWriteVMCS32(idxField, u64Val);
4699 rc |= VMXWriteVMCS32(idxField + 1, (uint32_t)(u64Val >> 32ULL));
4700 AssertRC(rc);
4701 return rc;
4702
4703 case VMX_VMCS64_GUEST_LDTR_BASE:
4704 case VMX_VMCS64_GUEST_TR_BASE:
4705 case VMX_VMCS64_GUEST_GDTR_BASE:
4706 case VMX_VMCS64_GUEST_IDTR_BASE:
4707 case VMX_VMCS64_GUEST_SYSENTER_EIP:
4708 case VMX_VMCS64_GUEST_SYSENTER_ESP:
4709 case VMX_VMCS64_GUEST_CR0:
4710 case VMX_VMCS64_GUEST_CR4:
4711 case VMX_VMCS64_GUEST_CR3:
4712 case VMX_VMCS64_GUEST_DR7:
4713 case VMX_VMCS64_GUEST_RIP:
4714 case VMX_VMCS64_GUEST_RSP:
4715 case VMX_VMCS64_GUEST_CS_BASE:
4716 case VMX_VMCS64_GUEST_DS_BASE:
4717 case VMX_VMCS64_GUEST_ES_BASE:
4718 case VMX_VMCS64_GUEST_FS_BASE:
4719 case VMX_VMCS64_GUEST_GS_BASE:
4720 case VMX_VMCS64_GUEST_SS_BASE:
4721 /* Queue a 64 bits value as we can't set it in 32 bits host mode. */
4722 if (u64Val >> 32ULL)
4723 rc = VMXWriteCachedVMCSEx(pVCpu, idxField, u64Val);
4724 else
4725 rc = VMXWriteVMCS32(idxField, (uint32_t)u64Val);
4726
4727 return rc;
4728
4729 default:
4730 AssertMsgFailed(("Unexpected field %x\n", idxField));
4731 return VERR_INVALID_PARAMETER;
4732 }
4733}
4734
4735/**
4736 * Cache VMCS writes for performance reasons (Darwin) and for running 64 bits guests on 32 bits hosts.
4737 *
4738 * @param pVCpu The VMCPU to operate on.
4739 * @param idxField VMCS field
4740 * @param u64Val Value
4741 */
4742VMMR0DECL(int) VMXWriteCachedVMCSEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
4743{
4744 PVMCSCACHE pCache = &pVCpu->hwaccm.s.vmx.VMCSCache;
4745
4746 AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1, ("entries=%x\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
4747
4748 /* Make sure there are no duplicates. */
4749 for (unsigned i=0;i<pCache->Write.cValidEntries;i++)
4750 {
4751 if (pCache->Write.aField[i] == idxField)
4752 {
4753 pCache->Write.aFieldVal[i] = u64Val;
4754 return VINF_SUCCESS;
4755 }
4756 }
4757
4758 pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
4759 pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
4760 pCache->Write.cValidEntries++;
4761 return VINF_SUCCESS;
4762}
4763
4764#endif /* HC_ARCH_BITS == 32 && !VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0 */
4765
4766#ifdef VBOX_STRICT
4767static bool vmxR0IsValidReadField(uint32_t idxField)
4768{
4769 switch(idxField)
4770 {
4771 case VMX_VMCS64_GUEST_RIP:
4772 case VMX_VMCS64_GUEST_RSP:
4773 case VMX_VMCS_GUEST_RFLAGS:
4774 case VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE:
4775 case VMX_VMCS_CTRL_CR0_READ_SHADOW:
4776 case VMX_VMCS64_GUEST_CR0:
4777 case VMX_VMCS_CTRL_CR4_READ_SHADOW:
4778 case VMX_VMCS64_GUEST_CR4:
4779 case VMX_VMCS64_GUEST_DR7:
4780 case VMX_VMCS32_GUEST_SYSENTER_CS:
4781 case VMX_VMCS64_GUEST_SYSENTER_EIP:
4782 case VMX_VMCS64_GUEST_SYSENTER_ESP:
4783 case VMX_VMCS32_GUEST_GDTR_LIMIT:
4784 case VMX_VMCS64_GUEST_GDTR_BASE:
4785 case VMX_VMCS32_GUEST_IDTR_LIMIT:
4786 case VMX_VMCS64_GUEST_IDTR_BASE:
4787 case VMX_VMCS16_GUEST_FIELD_CS:
4788 case VMX_VMCS32_GUEST_CS_LIMIT:
4789 case VMX_VMCS64_GUEST_CS_BASE:
4790 case VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS:
4791 case VMX_VMCS16_GUEST_FIELD_DS:
4792 case VMX_VMCS32_GUEST_DS_LIMIT:
4793 case VMX_VMCS64_GUEST_DS_BASE:
4794 case VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS:
4795 case VMX_VMCS16_GUEST_FIELD_ES:
4796 case VMX_VMCS32_GUEST_ES_LIMIT:
4797 case VMX_VMCS64_GUEST_ES_BASE:
4798 case VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS:
4799 case VMX_VMCS16_GUEST_FIELD_FS:
4800 case VMX_VMCS32_GUEST_FS_LIMIT:
4801 case VMX_VMCS64_GUEST_FS_BASE:
4802 case VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS:
4803 case VMX_VMCS16_GUEST_FIELD_GS:
4804 case VMX_VMCS32_GUEST_GS_LIMIT:
4805 case VMX_VMCS64_GUEST_GS_BASE:
4806 case VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS:
4807 case VMX_VMCS16_GUEST_FIELD_SS:
4808 case VMX_VMCS32_GUEST_SS_LIMIT:
4809 case VMX_VMCS64_GUEST_SS_BASE:
4810 case VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS:
4811 case VMX_VMCS16_GUEST_FIELD_LDTR:
4812 case VMX_VMCS32_GUEST_LDTR_LIMIT:
4813 case VMX_VMCS64_GUEST_LDTR_BASE:
4814 case VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS:
4815 case VMX_VMCS16_GUEST_FIELD_TR:
4816 case VMX_VMCS32_GUEST_TR_LIMIT:
4817 case VMX_VMCS64_GUEST_TR_BASE:
4818 case VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS:
4819 case VMX_VMCS32_RO_EXIT_REASON:
4820 case VMX_VMCS32_RO_VM_INSTR_ERROR:
4821 case VMX_VMCS32_RO_EXIT_INSTR_LENGTH:
4822 case VMX_VMCS32_RO_EXIT_INTERRUPTION_ERRCODE:
4823 case VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO:
4824 case VMX_VMCS32_RO_EXIT_INSTR_INFO:
4825 case VMX_VMCS_RO_EXIT_QUALIFICATION:
4826 case VMX_VMCS32_RO_IDT_INFO:
4827 case VMX_VMCS32_RO_IDT_ERRCODE:
4828 case VMX_VMCS64_GUEST_CR3:
4829 case VMX_VMCS_EXIT_PHYS_ADDR_FULL:
4830 return true;
4831 }
4832 return false;
4833}
4834
4835static bool vmxR0IsValidWriteField(uint32_t idxField)
4836{
4837 switch(idxField)
4838 {
4839 case VMX_VMCS64_GUEST_LDTR_BASE:
4840 case VMX_VMCS64_GUEST_TR_BASE:
4841 case VMX_VMCS64_GUEST_GDTR_BASE:
4842 case VMX_VMCS64_GUEST_IDTR_BASE:
4843 case VMX_VMCS64_GUEST_SYSENTER_EIP:
4844 case VMX_VMCS64_GUEST_SYSENTER_ESP:
4845 case VMX_VMCS64_GUEST_CR0:
4846 case VMX_VMCS64_GUEST_CR4:
4847 case VMX_VMCS64_GUEST_CR3:
4848 case VMX_VMCS64_GUEST_DR7:
4849 case VMX_VMCS64_GUEST_RIP:
4850 case VMX_VMCS64_GUEST_RSP:
4851 case VMX_VMCS64_GUEST_CS_BASE:
4852 case VMX_VMCS64_GUEST_DS_BASE:
4853 case VMX_VMCS64_GUEST_ES_BASE:
4854 case VMX_VMCS64_GUEST_FS_BASE:
4855 case VMX_VMCS64_GUEST_GS_BASE:
4856 case VMX_VMCS64_GUEST_SS_BASE:
4857 return true;
4858 }
4859 return false;
4860}
4861
4862#endif
Note: See TracBrowser for help on using the repository browser.

© 2025 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette