VirtualBox

source: vbox/trunk/src/VBox/VMM/VMMR0/HWVMXR0.cpp@ 26296

Last change on this file since 26296 was 26296, checked in by vboxsync, 15 years ago

more warnings.
  • Property svn:eol-style set to native
  • Property svn:keywords set to Id
File size: 194.2 KB
Line 
1/* $Id: HWVMXR0.cpp 26296 2010-02-05 14:53:35Z vboxsync $ */
2/** @file
3 * HWACCM VMX - Host Context Ring 0.
4 */
5
6/*
7 * Copyright (C) 2006-2007 Sun Microsystems, Inc.
8 *
9 * This file is part of VirtualBox Open Source Edition (OSE), as
10 * available from http://www.virtualbox.org. This file is free software;
11 * you can redistribute it and/or modify it under the terms of the GNU
12 * General Public License (GPL) as published by the Free Software
13 * Foundation, in version 2 as it comes in the "COPYING" file of the
14 * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15 * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16 *
17 * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa
18 * Clara, CA 95054 USA or visit http://www.sun.com if you need
19 * additional information or have any questions.
20 */
21
22
23/*******************************************************************************
24* Header Files *
25*******************************************************************************/
26#define LOG_GROUP LOG_GROUP_HWACCM
27#include <VBox/hwaccm.h>
28#include <VBox/pgm.h>
29#include <VBox/dbgf.h>
30#include <VBox/selm.h>
31#include <VBox/iom.h>
32#include <VBox/rem.h>
33#include <VBox/tm.h>
34#include "HWACCMInternal.h"
35#include <VBox/vm.h>
36#include <VBox/x86.h>
37#include <VBox/pdmapi.h>
38#include <VBox/err.h>
39#include <VBox/log.h>
40#include <iprt/asm.h>
41#include <iprt/assert.h>
42#include <iprt/param.h>
43#include <iprt/string.h>
44#include <iprt/time.h>
45#ifdef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
46# include <iprt/thread.h>
47#endif
48#include "HWVMXR0.h"
49
50/*******************************************************************************
51* Defined Constants And Macros *
52*******************************************************************************/
53#if defined(RT_ARCH_AMD64)
54# define VMX_IS_64BIT_HOST_MODE() (true)
55#elif defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
56# define VMX_IS_64BIT_HOST_MODE() (g_fVMXIs64bitHost != 0)
57#else
58# define VMX_IS_64BIT_HOST_MODE() (false)
59#endif
60
61/*******************************************************************************
62* Global Variables *
63*******************************************************************************/
64/* IO operation lookup arrays. */
65static uint32_t const g_aIOSize[4] = {1, 2, 0, 4};
66static uint32_t const g_aIOOpAnd[4] = {0xff, 0xffff, 0, 0xffffffff};
67
68#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
69/** See HWACCMR0A.asm. */
70extern "C" uint32_t g_fVMXIs64bitHost;
71#endif
72
73/*******************************************************************************
74* Local Functions *
75*******************************************************************************/
76static void VMXR0ReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, int rc, PCPUMCTX pCtx);
77static void vmxR0SetupTLBEPT(PVM pVM, PVMCPU pVCpu);
78static void vmxR0SetupTLBVPID(PVM pVM, PVMCPU pVCpu);
79static void vmxR0SetupTLBDummy(PVM pVM, PVMCPU pVCpu);
80static void vmxR0FlushEPT(PVM pVM, PVMCPU pVCpu, VMX_FLUSH enmFlush, RTGCPHYS GCPhys);
81static void vmxR0FlushVPID(PVM pVM, PVMCPU pVCpu, VMX_FLUSH enmFlush, RTGCPTR GCPtr);
82static void vmxR0UpdateExceptionBitmap(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx);
83#ifdef VBOX_STRICT
84static bool vmxR0IsValidReadField(uint32_t idxField);
85static bool vmxR0IsValidWriteField(uint32_t idxField);
86#endif
87static void vmxR0SetMSRPermission(PVMCPU pVCpu, unsigned ulMSR, bool fRead, bool fWrite);
88
89static void VMXR0CheckError(PVM pVM, PVMCPU pVCpu, int rc)
90{
91 if (rc == VERR_VMX_GENERIC)
92 {
93 RTCCUINTREG instrError;
94
95 VMXReadVMCS(VMX_VMCS32_RO_VM_INSTR_ERROR, &instrError);
96 pVCpu->hwaccm.s.vmx.lasterror.ulInstrError = instrError;
97 }
98 pVM->hwaccm.s.lLastError = rc;
99}
100
101/**
102 * Sets up and activates VT-x on the current CPU
103 *
104 * @returns VBox status code.
105 * @param pCpu CPU info struct
106 * @param pVM The VM to operate on. (can be NULL after a resume!!)
107 * @param pvPageCpu Pointer to the global cpu page
108 * @param pPageCpuPhys Physical address of the global cpu page
109 */
110VMMR0DECL(int) VMXR0EnableCpu(PHWACCM_CPUINFO pCpu, PVM pVM, void *pvPageCpu, RTHCPHYS pPageCpuPhys)
111{
112 AssertReturn(pPageCpuPhys, VERR_INVALID_PARAMETER);
113 AssertReturn(pvPageCpu, VERR_INVALID_PARAMETER);
114
115 if (pVM)
116 {
117 /* Set revision dword at the beginning of the VMXON structure. */
118 *(uint32_t *)pvPageCpu = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hwaccm.s.vmx.msr.vmx_basic_info);
119 }
120
121 /** @todo we should unmap the two pages from the virtual address space in order to prevent accidental corruption.
122 * (which can have very bad consequences!!!)
123 */
124
125 if (ASMGetCR4() & X86_CR4_VMXE)
126 return VERR_VMX_IN_VMX_ROOT_MODE;
127
128 /* Make sure the VMX instructions don't cause #UD faults. */
129 ASMSetCR4(ASMGetCR4() | X86_CR4_VMXE);
130
131 /* Enter VMX Root Mode */
132 int rc = VMXEnable(pPageCpuPhys);
133 if (RT_FAILURE(rc))
134 {
135 if (pVM)
136 VMXR0CheckError(pVM, &pVM->aCpus[0], rc);
137 ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
138 return VERR_VMX_VMXON_FAILED;
139 }
140 return VINF_SUCCESS;
141}
142
143/**
144 * Deactivates VT-x on the current CPU
145 *
146 * @returns VBox status code.
147 * @param pCpu CPU info struct
148 * @param pvPageCpu Pointer to the global cpu page
149 * @param pPageCpuPhys Physical address of the global cpu page
150 */
151VMMR0DECL(int) VMXR0DisableCpu(PHWACCM_CPUINFO pCpu, void *pvPageCpu, RTHCPHYS pPageCpuPhys)
152{
153 AssertReturn(pPageCpuPhys, VERR_INVALID_PARAMETER);
154 AssertReturn(pvPageCpu, VERR_INVALID_PARAMETER);
155
156 /* Leave VMX Root Mode. */
157 VMXDisable();
158
159 /* And clear the X86_CR4_VMXE bit */
160 ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
161 return VINF_SUCCESS;
162}
163
164/**
165 * Does Ring-0 per VM VT-x init.
166 *
167 * @returns VBox status code.
168 * @param pVM The VM to operate on.
169 */
170VMMR0DECL(int) VMXR0InitVM(PVM pVM)
171{
172 int rc;
173
174#ifdef LOG_ENABLED
175 SUPR0Printf("VMXR0InitVM %x\n", pVM);
176#endif
177
178 pVM->hwaccm.s.vmx.pMemObjAPIC = NIL_RTR0MEMOBJ;
179
180 if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
181 {
182 /* Allocate one page for the APIC physical page (serves for filtering accesses). */
183 rc = RTR0MemObjAllocCont(&pVM->hwaccm.s.vmx.pMemObjAPIC, 1 << PAGE_SHIFT, true /* executable R0 mapping */);
184 AssertRC(rc);
185 if (RT_FAILURE(rc))
186 return rc;
187
188 pVM->hwaccm.s.vmx.pAPIC = (uint8_t *)RTR0MemObjAddress(pVM->hwaccm.s.vmx.pMemObjAPIC);
189 pVM->hwaccm.s.vmx.pAPICPhys = RTR0MemObjGetPagePhysAddr(pVM->hwaccm.s.vmx.pMemObjAPIC, 0);
190 ASMMemZero32(pVM->hwaccm.s.vmx.pAPIC, PAGE_SIZE);
191 }
192 else
193 {
194 pVM->hwaccm.s.vmx.pMemObjAPIC = 0;
195 pVM->hwaccm.s.vmx.pAPIC = 0;
196 pVM->hwaccm.s.vmx.pAPICPhys = 0;
197 }
198
199#ifdef VBOX_WITH_CRASHDUMP_MAGIC
200 {
201 rc = RTR0MemObjAllocCont(&pVM->hwaccm.s.vmx.pMemObjScratch, 1 << PAGE_SHIFT, true /* executable R0 mapping */);
202 AssertRC(rc);
203 if (RT_FAILURE(rc))
204 return rc;
205
206 pVM->hwaccm.s.vmx.pScratch = (uint8_t *)RTR0MemObjAddress(pVM->hwaccm.s.vmx.pMemObjScratch);
207 pVM->hwaccm.s.vmx.pScratchPhys = RTR0MemObjGetPagePhysAddr(pVM->hwaccm.s.vmx.pMemObjScratch, 0);
208
209 ASMMemZero32(pVM->hwaccm.s.vmx.pScratch, PAGE_SIZE);
210 strcpy((char *)pVM->hwaccm.s.vmx.pScratch, "SCRATCH Magic");
211 *(uint64_t *)(pVM->hwaccm.s.vmx.pScratch + 16) = UINT64_C(0xDEADBEEFDEADBEEF);
212 }
213#endif
214
215 /* Allocate VMCBs for all guest CPUs. */
216 for (VMCPUID i = 0; i < pVM->cCpus; i++)
217 {
218 PVMCPU pVCpu = &pVM->aCpus[i];
219
220 pVCpu->hwaccm.s.vmx.pMemObjVMCS = NIL_RTR0MEMOBJ;
221
222 /* Allocate one page for the VM control structure (VMCS). */
223 rc = RTR0MemObjAllocCont(&pVCpu->hwaccm.s.vmx.pMemObjVMCS, 1 << PAGE_SHIFT, true /* executable R0 mapping */);
224 AssertRC(rc);
225 if (RT_FAILURE(rc))
226 return rc;
227
228 pVCpu->hwaccm.s.vmx.pVMCS = RTR0MemObjAddress(pVCpu->hwaccm.s.vmx.pMemObjVMCS);
229 pVCpu->hwaccm.s.vmx.pVMCSPhys = RTR0MemObjGetPagePhysAddr(pVCpu->hwaccm.s.vmx.pMemObjVMCS, 0);
230 ASMMemZero32(pVCpu->hwaccm.s.vmx.pVMCS, PAGE_SIZE);
231
232 pVCpu->hwaccm.s.vmx.cr0_mask = 0;
233 pVCpu->hwaccm.s.vmx.cr4_mask = 0;
234
235 /* Allocate one page for the virtual APIC page for TPR caching. */
236 rc = RTR0MemObjAllocCont(&pVCpu->hwaccm.s.vmx.pMemObjVAPIC, 1 << PAGE_SHIFT, true /* executable R0 mapping */);
237 AssertRC(rc);
238 if (RT_FAILURE(rc))
239 return rc;
240
241 pVCpu->hwaccm.s.vmx.pVAPIC = (uint8_t *)RTR0MemObjAddress(pVCpu->hwaccm.s.vmx.pMemObjVAPIC);
242 pVCpu->hwaccm.s.vmx.pVAPICPhys = RTR0MemObjGetPagePhysAddr(pVCpu->hwaccm.s.vmx.pMemObjVAPIC, 0);
243 ASMMemZero32(pVCpu->hwaccm.s.vmx.pVAPIC, PAGE_SIZE);
244
245 /* Allocate the MSR bitmap if this feature is supported. */
246 if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
247 {
248 rc = RTR0MemObjAllocCont(&pVCpu->hwaccm.s.vmx.pMemObjMSRBitmap, 1 << PAGE_SHIFT, true /* executable R0 mapping */);
249 AssertRC(rc);
250 if (RT_FAILURE(rc))
251 return rc;
252
253 pVCpu->hwaccm.s.vmx.pMSRBitmap = (uint8_t *)RTR0MemObjAddress(pVCpu->hwaccm.s.vmx.pMemObjMSRBitmap);
254 pVCpu->hwaccm.s.vmx.pMSRBitmapPhys = RTR0MemObjGetPagePhysAddr(pVCpu->hwaccm.s.vmx.pMemObjMSRBitmap, 0);
255 memset(pVCpu->hwaccm.s.vmx.pMSRBitmap, 0xff, PAGE_SIZE);
256 }
257
258#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
259 /* Allocate one page for the guest MSR load area (for preloading guest MSRs during the world switch). */
260 rc = RTR0MemObjAllocCont(&pVCpu->hwaccm.s.vmx.pMemObjGuestMSR, 1 << PAGE_SHIFT, true /* executable R0 mapping */);
261 AssertRC(rc);
262 if (RT_FAILURE(rc))
263 return rc;
264
265 pVCpu->hwaccm.s.vmx.pGuestMSR = (uint8_t *)RTR0MemObjAddress(pVCpu->hwaccm.s.vmx.pMemObjGuestMSR);
266 pVCpu->hwaccm.s.vmx.pGuestMSRPhys = RTR0MemObjGetPagePhysAddr(pVCpu->hwaccm.s.vmx.pMemObjGuestMSR, 0);
267 memset(pVCpu->hwaccm.s.vmx.pGuestMSR, 0, PAGE_SIZE);
268
269 /* Allocate one page for the host MSR load area (for restoring host MSRs after the world switch back). */
270 rc = RTR0MemObjAllocCont(&pVCpu->hwaccm.s.vmx.pMemObjHostMSR, 1 << PAGE_SHIFT, true /* executable R0 mapping */);
271 AssertRC(rc);
272 if (RT_FAILURE(rc))
273 return rc;
274
275 pVCpu->hwaccm.s.vmx.pHostMSR = (uint8_t *)RTR0MemObjAddress(pVCpu->hwaccm.s.vmx.pMemObjHostMSR);
276 pVCpu->hwaccm.s.vmx.pHostMSRPhys = RTR0MemObjGetPagePhysAddr(pVCpu->hwaccm.s.vmx.pMemObjHostMSR, 0);
277 memset(pVCpu->hwaccm.s.vmx.pHostMSR, 0, PAGE_SIZE);
278#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
279
280 /* Current guest paging mode. */
281 pVCpu->hwaccm.s.vmx.enmLastSeenGuestMode = PGMMODE_REAL;
282
283#ifdef LOG_ENABLED
284 SUPR0Printf("VMXR0InitVM %x VMCS=%x (%x)\n", pVM, pVCpu->hwaccm.s.vmx.pVMCS, (uint32_t)pVCpu->hwaccm.s.vmx.pVMCSPhys);
285#endif
286 }
287
288 return VINF_SUCCESS;
289}
290
291/**
292 * Does Ring-0 per VM VT-x termination.
293 *
294 * @returns VBox status code.
295 * @param pVM The VM to operate on.
296 */
297VMMR0DECL(int) VMXR0TermVM(PVM pVM)
298{
299 for (VMCPUID i = 0; i < pVM->cCpus; i++)
300 {
301 PVMCPU pVCpu = &pVM->aCpus[i];
302
303 if (pVCpu->hwaccm.s.vmx.pMemObjVMCS != NIL_RTR0MEMOBJ)
304 {
305 RTR0MemObjFree(pVCpu->hwaccm.s.vmx.pMemObjVMCS, false);
306 pVCpu->hwaccm.s.vmx.pMemObjVMCS = NIL_RTR0MEMOBJ;
307 pVCpu->hwaccm.s.vmx.pVMCS = 0;
308 pVCpu->hwaccm.s.vmx.pVMCSPhys = 0;
309 }
310 if (pVCpu->hwaccm.s.vmx.pMemObjVAPIC != NIL_RTR0MEMOBJ)
311 {
312 RTR0MemObjFree(pVCpu->hwaccm.s.vmx.pMemObjVAPIC, false);
313 pVCpu->hwaccm.s.vmx.pMemObjVAPIC = NIL_RTR0MEMOBJ;
314 pVCpu->hwaccm.s.vmx.pVAPIC = 0;
315 pVCpu->hwaccm.s.vmx.pVAPICPhys = 0;
316 }
317 if (pVCpu->hwaccm.s.vmx.pMemObjMSRBitmap != NIL_RTR0MEMOBJ)
318 {
319 RTR0MemObjFree(pVCpu->hwaccm.s.vmx.pMemObjMSRBitmap, false);
320 pVCpu->hwaccm.s.vmx.pMemObjMSRBitmap = NIL_RTR0MEMOBJ;
321 pVCpu->hwaccm.s.vmx.pMSRBitmap = 0;
322 pVCpu->hwaccm.s.vmx.pMSRBitmapPhys = 0;
323 }
324#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
325 if (pVCpu->hwaccm.s.vmx.pMemObjHostMSR != NIL_RTR0MEMOBJ)
326 {
327 RTR0MemObjFree(pVCpu->hwaccm.s.vmx.pMemObjHostMSR, false);
328 pVCpu->hwaccm.s.vmx.pMemObjHostMSR = NIL_RTR0MEMOBJ;
329 pVCpu->hwaccm.s.vmx.pHostMSR = 0;
330 pVCpu->hwaccm.s.vmx.pHostMSRPhys = 0;
331 }
332 if (pVCpu->hwaccm.s.vmx.pMemObjGuestMSR != NIL_RTR0MEMOBJ)
333 {
334 RTR0MemObjFree(pVCpu->hwaccm.s.vmx.pMemObjGuestMSR, false);
335 pVCpu->hwaccm.s.vmx.pMemObjGuestMSR = NIL_RTR0MEMOBJ;
336 pVCpu->hwaccm.s.vmx.pGuestMSR = 0;
337 pVCpu->hwaccm.s.vmx.pGuestMSRPhys = 0;
338 }
339#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
340 }
341 if (pVM->hwaccm.s.vmx.pMemObjAPIC != NIL_RTR0MEMOBJ)
342 {
343 RTR0MemObjFree(pVM->hwaccm.s.vmx.pMemObjAPIC, false);
344 pVM->hwaccm.s.vmx.pMemObjAPIC = NIL_RTR0MEMOBJ;
345 pVM->hwaccm.s.vmx.pAPIC = 0;
346 pVM->hwaccm.s.vmx.pAPICPhys = 0;
347 }
348#ifdef VBOX_WITH_CRASHDUMP_MAGIC
349 if (pVM->hwaccm.s.vmx.pMemObjScratch != NIL_RTR0MEMOBJ)
350 {
351 ASMMemZero32(pVM->hwaccm.s.vmx.pScratch, PAGE_SIZE);
352 RTR0MemObjFree(pVM->hwaccm.s.vmx.pMemObjScratch, false);
353 pVM->hwaccm.s.vmx.pMemObjScratch = NIL_RTR0MEMOBJ;
354 pVM->hwaccm.s.vmx.pScratch = 0;
355 pVM->hwaccm.s.vmx.pScratchPhys = 0;
356 }
357#endif
358 return VINF_SUCCESS;
359}
360
361/**
362 * Sets up VT-x for the specified VM
363 *
364 * @returns VBox status code.
365 * @param pVM The VM to operate on.
366 */
367VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
368{
369 int rc = VINF_SUCCESS;
370 uint32_t val;
371
372 AssertReturn(pVM, VERR_INVALID_PARAMETER);
373
374 for (VMCPUID i = 0; i < pVM->cCpus; i++)
375 {
376 PVMCPU pVCpu = &pVM->aCpus[i];
377
378 Assert(pVCpu->hwaccm.s.vmx.pVMCS);
379
380 /* Set revision dword at the beginning of the VMCS structure. */
381 *(uint32_t *)pVCpu->hwaccm.s.vmx.pVMCS = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hwaccm.s.vmx.msr.vmx_basic_info);
382
383 /* Clear VM Control Structure. */
384 Log(("pVMCSPhys = %RHp\n", pVCpu->hwaccm.s.vmx.pVMCSPhys));
385 rc = VMXClearVMCS(pVCpu->hwaccm.s.vmx.pVMCSPhys);
386 if (RT_FAILURE(rc))
387 goto vmx_end;
388
389 /* Activate the VM Control Structure. */
390 rc = VMXActivateVMCS(pVCpu->hwaccm.s.vmx.pVMCSPhys);
391 if (RT_FAILURE(rc))
392 goto vmx_end;
393
394 /* VMX_VMCS_CTRL_PIN_EXEC_CONTROLS
395 * Set required bits to one and zero according to the MSR capabilities.
396 */
397 val = pVM->hwaccm.s.vmx.msr.vmx_pin_ctls.n.disallowed0;
398 /* External and non-maskable interrupts cause VM-exits. */
399 val = val | VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_EXT_INT_EXIT | VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_NMI_EXIT;
400 val &= pVM->hwaccm.s.vmx.msr.vmx_pin_ctls.n.allowed1;
401
402 rc = VMXWriteVMCS(VMX_VMCS_CTRL_PIN_EXEC_CONTROLS, val);
403 AssertRC(rc);
404
405 /* VMX_VMCS_CTRL_PROC_EXEC_CONTROLS
406 * Set required bits to one and zero according to the MSR capabilities.
407 */
408 val = pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.disallowed0;
409 /* Program which event cause VM-exits and which features we want to use. */
410 val = val | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_HLT_EXIT
411 | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_TSC_OFFSET
412 | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT
413 | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_UNCOND_IO_EXIT
414 | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDPMC_EXIT
415 | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MWAIT_EXIT; /* don't execute mwait or else we'll idle inside the guest (host thinks the cpu load is high) */
416
417 /* Without nested paging we should intercept invlpg and cr3 mov instructions. */
418 if (!pVM->hwaccm.s.fNestedPaging)
419 val |= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INVLPG_EXIT
420 | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
421 | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT;
422
423 /* Note: VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MWAIT_EXIT might cause a vmlaunch failure with an invalid control fields error. (combined with some other exit reasons) */
424 if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
425 {
426 /* CR8 reads from the APIC shadow page; writes cause an exit is they lower the TPR below the threshold */
427 val |= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW;
428 Assert(pVM->hwaccm.s.vmx.pAPIC);
429 }
430 else
431 /* Exit on CR8 reads & writes in case the TPR shadow feature isn't present. */
432 val |= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR8_STORE_EXIT | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR8_LOAD_EXIT;
433
434 if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
435 {
436 Assert(pVCpu->hwaccm.s.vmx.pMSRBitmapPhys);
437 val |= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS;
438 }
439
440 /* We will use the secondary control if it's present. */
441 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
442
443 /* Mask away the bits that the CPU doesn't support */
444 /** @todo make sure they don't conflict with the above requirements. */
445 val &= pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1;
446 pVCpu->hwaccm.s.vmx.proc_ctls = val;
447
448 rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, val);
449 AssertRC(rc);
450
451 if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
452 {
453 /* VMX_VMCS_CTRL_PROC_EXEC_CONTROLS2
454 * Set required bits to one and zero according to the MSR capabilities.
455 */
456 val = pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2.n.disallowed0;
457 val |= VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT;
458
459#ifdef HWACCM_VTX_WITH_EPT
460 if (pVM->hwaccm.s.fNestedPaging)
461 val |= VMX_VMCS_CTRL_PROC_EXEC2_EPT;
462#endif /* HWACCM_VTX_WITH_EPT */
463#ifdef HWACCM_VTX_WITH_VPID
464 else
465 if (pVM->hwaccm.s.vmx.fVPID)
466 val |= VMX_VMCS_CTRL_PROC_EXEC2_VPID;
467#endif /* HWACCM_VTX_WITH_VPID */
468
469 if (pVM->hwaccm.s.fHasIoApic)
470 val |= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC;
471
472 if (pVM->hwaccm.s.vmx.fUnrestrictedGuest)
473 val |= VMX_VMCS_CTRL_PROC_EXEC2_REAL_MODE;
474
475 /* Mask away the bits that the CPU doesn't support */
476 /** @todo make sure they don't conflict with the above requirements. */
477 val &= pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2.n.allowed1;
478 pVCpu->hwaccm.s.vmx.proc_ctls2 = val;
479 rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS2, val);
480 AssertRC(rc);
481 }
482
483 /* VMX_VMCS_CTRL_CR3_TARGET_COUNT
484 * Set required bits to one and zero according to the MSR capabilities.
485 */
486 rc = VMXWriteVMCS(VMX_VMCS_CTRL_CR3_TARGET_COUNT, 0);
487 AssertRC(rc);
488
489 /* Forward all exception except #NM & #PF to the guest.
490 * We always need to check pagefaults since our shadow page table can be out of sync.
491 * And we always lazily sync the FPU & XMM state.
492 */
493
494 /** @todo Possible optimization:
495 * Keep the FPU and XMM state current in the EM thread. That way there's no need to
496 * lazily sync anything, but the downside is that we can't use the FPU stack or XMM
497 * registers ourselves of course.
498 *
499 * Note: only possible if the current state is actually ours (X86_CR0_TS flag)
500 */
501
502 /* Don't filter page faults; all of them should cause a switch. */
503 rc = VMXWriteVMCS(VMX_VMCS_CTRL_PAGEFAULT_ERROR_MASK, 0);
504 rc |= VMXWriteVMCS(VMX_VMCS_CTRL_PAGEFAULT_ERROR_MATCH, 0);
505 AssertRC(rc);
506
507 /* Init TSC offset to zero. */
508 rc = VMXWriteVMCS64(VMX_VMCS_CTRL_TSC_OFFSET_FULL, 0);
509 AssertRC(rc);
510
511 rc = VMXWriteVMCS64(VMX_VMCS_CTRL_IO_BITMAP_A_FULL, 0);
512 AssertRC(rc);
513
514 rc = VMXWriteVMCS64(VMX_VMCS_CTRL_IO_BITMAP_B_FULL, 0);
515 AssertRC(rc);
516
517 /* Set the MSR bitmap address. */
518 if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
519 {
520 Assert(pVCpu->hwaccm.s.vmx.pMSRBitmapPhys);
521
522 rc = VMXWriteVMCS64(VMX_VMCS_CTRL_MSR_BITMAP_FULL, pVCpu->hwaccm.s.vmx.pMSRBitmapPhys);
523 AssertRC(rc);
524
525 /* Allow the guest to directly modify these MSRs; they are restored and saved automatically. */
526 vmxR0SetMSRPermission(pVCpu, MSR_IA32_SYSENTER_CS, true, true);
527 vmxR0SetMSRPermission(pVCpu, MSR_IA32_SYSENTER_ESP, true, true);
528 vmxR0SetMSRPermission(pVCpu, MSR_IA32_SYSENTER_EIP, true, true);
529 vmxR0SetMSRPermission(pVCpu, MSR_K8_LSTAR, true, true);
530 vmxR0SetMSRPermission(pVCpu, MSR_K6_STAR, true, true);
531 vmxR0SetMSRPermission(pVCpu, MSR_K8_SF_MASK, true, true);
532 vmxR0SetMSRPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, true, true);
533 vmxR0SetMSRPermission(pVCpu, MSR_K8_GS_BASE, true, true);
534 vmxR0SetMSRPermission(pVCpu, MSR_K8_FS_BASE, true, true);
535 }
536
537#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
538 /* Set the guest & host MSR load/store physical addresses. */
539 Assert(pVCpu->hwaccm.s.vmx.pGuestMSRPhys);
540 rc = VMXWriteVMCS64(VMX_VMCS_CTRL_VMENTRY_MSR_LOAD_FULL, pVCpu->hwaccm.s.vmx.pGuestMSRPhys);
541 AssertRC(rc);
542 rc = VMXWriteVMCS64(VMX_VMCS_CTRL_VMEXIT_MSR_STORE_FULL, pVCpu->hwaccm.s.vmx.pGuestMSRPhys);
543 AssertRC(rc);
544
545 Assert(pVCpu->hwaccm.s.vmx.pHostMSRPhys);
546 rc = VMXWriteVMCS64(VMX_VMCS_CTRL_VMEXIT_MSR_LOAD_FULL, pVCpu->hwaccm.s.vmx.pHostMSRPhys);
547 AssertRC(rc);
548#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
549
550 rc = VMXWriteVMCS(VMX_VMCS_CTRL_ENTRY_MSR_LOAD_COUNT, 0);
551 AssertRC(rc);
552
553 rc = VMXWriteVMCS(VMX_VMCS_CTRL_EXIT_MSR_STORE_COUNT, 0);
554 AssertRC(rc);
555
556 if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
557 {
558 Assert(pVM->hwaccm.s.vmx.pMemObjAPIC);
559 /* Optional */
560 rc = VMXWriteVMCS(VMX_VMCS_CTRL_TPR_THRESHOLD, 0);
561 rc |= VMXWriteVMCS64(VMX_VMCS_CTRL_VAPIC_PAGEADDR_FULL, pVCpu->hwaccm.s.vmx.pVAPICPhys);
562
563 if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
564 rc |= VMXWriteVMCS64(VMX_VMCS_CTRL_APIC_ACCESSADDR_FULL, pVM->hwaccm.s.vmx.pAPICPhys);
565
566 AssertRC(rc);
567 }
568
569 /* Set link pointer to -1. Not currently used. */
570 rc = VMXWriteVMCS64(VMX_VMCS_GUEST_LINK_PTR_FULL, 0xFFFFFFFFFFFFFFFFULL);
571 AssertRC(rc);
572
573 /* Clear VM Control Structure. Marking it inactive, clearing implementation specific data and writing back VMCS data to memory. */
574 rc = VMXClearVMCS(pVCpu->hwaccm.s.vmx.pVMCSPhys);
575 AssertRC(rc);
576
577 /* Configure the VMCS read cache. */
578 PVMCSCACHE pCache = &pVCpu->hwaccm.s.vmx.VMCSCache;
579
580 VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_RIP);
581 VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_RSP);
582 VMXSetupCachedReadVMCS(pCache, VMX_VMCS_GUEST_RFLAGS);
583 VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE);
584 VMXSetupCachedReadVMCS(pCache, VMX_VMCS_CTRL_CR0_READ_SHADOW);
585 VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_CR0);
586 VMXSetupCachedReadVMCS(pCache, VMX_VMCS_CTRL_CR4_READ_SHADOW);
587 VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_CR4);
588 VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_DR7);
589 VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_GUEST_SYSENTER_CS);
590 VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_SYSENTER_EIP);
591 VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_SYSENTER_ESP);
592 VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_GUEST_GDTR_LIMIT);
593 VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_GDTR_BASE);
594 VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_GUEST_IDTR_LIMIT);
595 VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_IDTR_BASE);
596
597 VMX_SETUP_SELREG(ES, pCache);
598 VMX_SETUP_SELREG(SS, pCache);
599 VMX_SETUP_SELREG(CS, pCache);
600 VMX_SETUP_SELREG(DS, pCache);
601 VMX_SETUP_SELREG(FS, pCache);
602 VMX_SETUP_SELREG(GS, pCache);
603 VMX_SETUP_SELREG(LDTR, pCache);
604 VMX_SETUP_SELREG(TR, pCache);
605
606 /* Status code VMCS reads. */
607 VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_EXIT_REASON);
608 VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_VM_INSTR_ERROR);
609 VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_EXIT_INSTR_LENGTH);
610 VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_EXIT_INTERRUPTION_ERRCODE);
611 VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO);
612 VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_EXIT_INSTR_INFO);
613 VMXSetupCachedReadVMCS(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
614 VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_IDT_INFO);
615 VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_IDT_ERRCODE);
616
617 if (pVM->hwaccm.s.fNestedPaging)
618 {
619 VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_CR3);
620 VMXSetupCachedReadVMCS(pCache, VMX_VMCS_EXIT_PHYS_ADDR_FULL);
621 pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
622 }
623 else
624 pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
625 } /* for each VMCPU */
626
627 /* Choose the right TLB setup function. */
628 if (pVM->hwaccm.s.fNestedPaging)
629 {
630 pVM->hwaccm.s.vmx.pfnSetupTaggedTLB = vmxR0SetupTLBEPT;
631
632 /* Default values for flushing. */
633 pVM->hwaccm.s.vmx.enmFlushPage = VMX_FLUSH_ALL_CONTEXTS;
634 pVM->hwaccm.s.vmx.enmFlushContext = VMX_FLUSH_ALL_CONTEXTS;
635
636 /* If the capabilities specify we can do more, then make use of it. */
637 if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVEPT_CAPS_INDIV)
638 pVM->hwaccm.s.vmx.enmFlushPage = VMX_FLUSH_PAGE;
639 else
640 if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVEPT_CAPS_CONTEXT)
641 pVM->hwaccm.s.vmx.enmFlushPage = VMX_FLUSH_SINGLE_CONTEXT;
642
643 if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVEPT_CAPS_CONTEXT)
644 pVM->hwaccm.s.vmx.enmFlushContext = VMX_FLUSH_SINGLE_CONTEXT;
645 }
646#ifdef HWACCM_VTX_WITH_VPID
647 else
648 if (pVM->hwaccm.s.vmx.fVPID)
649 {
650 pVM->hwaccm.s.vmx.pfnSetupTaggedTLB = vmxR0SetupTLBVPID;
651
652 /* Default values for flushing. */
653 pVM->hwaccm.s.vmx.enmFlushPage = VMX_FLUSH_ALL_CONTEXTS;
654 pVM->hwaccm.s.vmx.enmFlushContext = VMX_FLUSH_ALL_CONTEXTS;
655
656 /* If the capabilities specify we can do more, then make use of it. */
657 if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVVPID_CAPS_INDIV)
658 pVM->hwaccm.s.vmx.enmFlushPage = VMX_FLUSH_PAGE;
659 else
660 if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVVPID_CAPS_CONTEXT)
661 pVM->hwaccm.s.vmx.enmFlushPage = VMX_FLUSH_SINGLE_CONTEXT;
662
663 if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVVPID_CAPS_CONTEXT)
664 pVM->hwaccm.s.vmx.enmFlushContext = VMX_FLUSH_SINGLE_CONTEXT;
665 }
666#endif /* HWACCM_VTX_WITH_VPID */
667 else
668 pVM->hwaccm.s.vmx.pfnSetupTaggedTLB = vmxR0SetupTLBDummy;
669
670vmx_end:
671 VMXR0CheckError(pVM, &pVM->aCpus[0], rc);
672 return rc;
673}
674
675/**
676 * Sets the permission bits for the specified MSR
677 *
678 * @param pVCpu The VMCPU to operate on.
679 * @param ulMSR MSR value
680 * @param fRead Reading allowed/disallowed
681 * @param fWrite Writing allowed/disallowed
682 */
683static void vmxR0SetMSRPermission(PVMCPU pVCpu, unsigned ulMSR, bool fRead, bool fWrite)
684{
685 unsigned ulBit;
686 uint8_t *pMSRBitmap = (uint8_t *)pVCpu->hwaccm.s.vmx.pMSRBitmap;
687
688 /* Layout:
689 * 0x000 - 0x3ff - Low MSR read bits
690 * 0x400 - 0x7ff - High MSR read bits
691 * 0x800 - 0xbff - Low MSR write bits
692 * 0xc00 - 0xfff - High MSR write bits
693 */
694 if (ulMSR <= 0x00001FFF)
695 {
696 /* Pentium-compatible MSRs */
697 ulBit = ulMSR;
698 }
699 else
700 if ( ulMSR >= 0xC0000000
701 && ulMSR <= 0xC0001FFF)
702 {
703 /* AMD Sixth Generation x86 Processor MSRs */
704 ulBit = (ulMSR - 0xC0000000);
705 pMSRBitmap += 0x400;
706 }
707 else
708 {
709 AssertFailed();
710 return;
711 }
712
713 Assert(ulBit <= 0x1fff);
714 if (fRead)
715 ASMBitClear(pMSRBitmap, ulBit);
716 else
717 ASMBitSet(pMSRBitmap, ulBit);
718
719 if (fWrite)
720 ASMBitClear(pMSRBitmap + 0x800, ulBit);
721 else
722 ASMBitSet(pMSRBitmap + 0x800, ulBit);
723}
724
725
726/**
727 * Injects an event (trap or external interrupt)
728 *
729 * @returns VBox status code.
730 * @param pVM The VM to operate on.
731 * @param pVCpu The VMCPU to operate on.
732 * @param pCtx CPU Context
733 * @param intInfo VMX interrupt info
734 * @param cbInstr Opcode length of faulting instruction
735 * @param errCode Error code (optional)
736 */
737static int VMXR0InjectEvent(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, uint32_t intInfo, uint32_t cbInstr, uint32_t errCode)
738{
739 int rc;
740 uint32_t iGate = VMX_EXIT_INTERRUPTION_INFO_VECTOR(intInfo);
741
742#ifdef VBOX_WITH_STATISTICS
743 STAM_COUNTER_INC(&pVCpu->hwaccm.s.paStatInjectedIrqsR0[iGate & MASK_INJECT_IRQ_STAT]);
744#endif
745
746#ifdef VBOX_STRICT
747 if (iGate == 0xE)
748 LogFlow(("VMXR0InjectEvent: Injecting interrupt %d at %RGv error code=%08x CR2=%RGv intInfo=%08x\n", iGate, (RTGCPTR)pCtx->rip, errCode, pCtx->cr2, intInfo));
749 else
750 if (iGate < 0x20)
751 LogFlow(("VMXR0InjectEvent: Injecting interrupt %d at %RGv error code=%08x\n", iGate, (RTGCPTR)pCtx->rip, errCode));
752 else
753 {
754 LogFlow(("INJ-EI: %x at %RGv\n", iGate, (RTGCPTR)pCtx->rip));
755 Assert(VMX_EXIT_INTERRUPTION_INFO_TYPE(intInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW || !VMCPU_FF_ISSET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
756 Assert(VMX_EXIT_INTERRUPTION_INFO_TYPE(intInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW || pCtx->eflags.u32 & X86_EFL_IF);
757 }
758#endif
759
760#ifdef HWACCM_VMX_EMULATE_REALMODE
761 if ( CPUMIsGuestInRealModeEx(pCtx)
762 && pVM->hwaccm.s.vmx.pRealModeTSS)
763 {
764 RTGCPHYS GCPhysHandler;
765 uint16_t offset, ip;
766 RTSEL sel;
767
768 /* Injecting events doesn't work right with real mode emulation.
769 * (#GP if we try to inject external hardware interrupts)
770 * Inject the interrupt or trap directly instead.
771 *
772 * ASSUMES no access handlers for the bits we read or write below (should be safe).
773 */
774 Log(("Manual interrupt/trap '%x' inject (real mode)\n", iGate));
775
776 /* Check if the interrupt handler is present. */
777 if (iGate * 4 + 3 > pCtx->idtr.cbIdt)
778 {
779 Log(("IDT cbIdt violation\n"));
780 if (iGate != X86_XCPT_DF)
781 {
782 uint32_t intInfo2;
783
784 intInfo2 = (iGate == X86_XCPT_GP) ? (uint32_t)X86_XCPT_DF : iGate;
785 intInfo2 |= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
786 intInfo2 |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
787 intInfo2 |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HWEXCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
788
789 return VMXR0InjectEvent(pVM, pVCpu, pCtx, intInfo2, 0, 0 /* no error code according to the Intel docs */);
790 }
791 Log(("Triple fault -> reset the VM!\n"));
792 return VINF_EM_RESET;
793 }
794 if ( VMX_EXIT_INTERRUPTION_INFO_TYPE(intInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW
795 || iGate == 3 /* Both #BP and #OF point to the instruction after. */
796 || iGate == 4)
797 {
798 ip = pCtx->ip + cbInstr;
799 }
800 else
801 ip = pCtx->ip;
802
803 /* Read the selector:offset pair of the interrupt handler. */
804 GCPhysHandler = (RTGCPHYS)pCtx->idtr.pIdt + iGate * 4;
805 rc = PGMPhysSimpleReadGCPhys(pVM, &offset, GCPhysHandler, sizeof(offset)); AssertRC(rc);
806 rc = PGMPhysSimpleReadGCPhys(pVM, &sel, GCPhysHandler + 2, sizeof(sel)); AssertRC(rc);
807
808 LogFlow(("IDT handler %04X:%04X\n", sel, offset));
809
810 /* Construct the stack frame. */
811 /** @todo should check stack limit. */
812 pCtx->sp -= 2;
813 LogFlow(("ss:sp %04X:%04X eflags=%x\n", pCtx->ss, pCtx->sp, pCtx->eflags.u));
814 rc = PGMPhysSimpleWriteGCPhys(pVM, pCtx->ssHid.u64Base + pCtx->sp, &pCtx->eflags, sizeof(uint16_t)); AssertRC(rc);
815 pCtx->sp -= 2;
816 LogFlow(("ss:sp %04X:%04X cs=%x\n", pCtx->ss, pCtx->sp, pCtx->cs));
817 rc = PGMPhysSimpleWriteGCPhys(pVM, pCtx->ssHid.u64Base + pCtx->sp, &pCtx->cs, sizeof(uint16_t)); AssertRC(rc);
818 pCtx->sp -= 2;
819 LogFlow(("ss:sp %04X:%04X ip=%x\n", pCtx->ss, pCtx->sp, ip));
820 rc = PGMPhysSimpleWriteGCPhys(pVM, pCtx->ssHid.u64Base + pCtx->sp, &ip, sizeof(ip)); AssertRC(rc);
821
822 /* Update the CPU state for executing the handler. */
823 pCtx->rip = offset;
824 pCtx->cs = sel;
825 pCtx->csHid.u64Base = sel << 4;
826 pCtx->eflags.u &= ~(X86_EFL_IF|X86_EFL_TF|X86_EFL_RF|X86_EFL_AC);
827
828 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_GUEST_SEGMENT_REGS;
829 return VINF_SUCCESS;
830 }
831#endif /* HWACCM_VMX_EMULATE_REALMODE */
832
833 /* Set event injection state. */
834 rc = VMXWriteVMCS(VMX_VMCS_CTRL_ENTRY_IRQ_INFO, intInfo | (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT));
835
836 rc |= VMXWriteVMCS(VMX_VMCS_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
837 rc |= VMXWriteVMCS(VMX_VMCS_CTRL_ENTRY_EXCEPTION_ERRCODE, errCode);
838
839 AssertRC(rc);
840 return rc;
841}
842
843
844/**
845 * Checks for pending guest interrupts and injects them
846 *
847 * @returns VBox status code.
848 * @param pVM The VM to operate on.
849 * @param pVCpu The VMCPU to operate on.
850 * @param pCtx CPU Context
851 */
852static int VMXR0CheckPendingInterrupt(PVM pVM, PVMCPU pVCpu, CPUMCTX *pCtx)
853{
854 int rc;
855
856 /* Dispatch any pending interrupts. (injected before, but a VM exit occurred prematurely) */
857 if (pVCpu->hwaccm.s.Event.fPending)
858 {
859 Log(("CPU%d: Reinjecting event %RX64 %08x at %RGv cr2=%RX64\n", pVCpu->idCpu, pVCpu->hwaccm.s.Event.intInfo, pVCpu->hwaccm.s.Event.errCode, (RTGCPTR)pCtx->rip, pCtx->cr2));
860 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatIntReinject);
861 rc = VMXR0InjectEvent(pVM, pVCpu, pCtx, pVCpu->hwaccm.s.Event.intInfo, 0, pVCpu->hwaccm.s.Event.errCode);
862 AssertRC(rc);
863
864 pVCpu->hwaccm.s.Event.fPending = false;
865 return VINF_SUCCESS;
866 }
867
868 /* If an active trap is already pending, then we must forward it first! */
869 if (!TRPMHasTrap(pVCpu))
870 {
871 if (VMCPU_FF_TESTANDCLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI))
872 {
873 RTGCUINTPTR intInfo;
874
875 Log(("CPU%d: injecting #NMI\n", pVCpu->idCpu));
876
877 intInfo = X86_XCPT_NMI;
878 intInfo |= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
879 intInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
880
881 rc = VMXR0InjectEvent(pVM, pVCpu, pCtx, intInfo, 0, 0);
882 AssertRC(rc);
883
884 return VINF_SUCCESS;
885 }
886
887 /* @todo SMI interrupts. */
888
889 /* When external interrupts are pending, we should exit the VM when IF is set. */
890 if (VMCPU_FF_ISPENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC|VMCPU_FF_INTERRUPT_PIC)))
891 {
892 if (!(pCtx->eflags.u32 & X86_EFL_IF))
893 {
894 if (!(pVCpu->hwaccm.s.vmx.proc_ctls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_IRQ_WINDOW_EXIT))
895 {
896 LogFlow(("Enable irq window exit!\n"));
897 pVCpu->hwaccm.s.vmx.proc_ctls |= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_IRQ_WINDOW_EXIT;
898 rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
899 AssertRC(rc);
900 }
901 /* else nothing to do but wait */
902 }
903 else
904 if (!VMCPU_FF_ISSET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
905 {
906 uint8_t u8Interrupt;
907
908 rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
909 Log(("CPU%d: Dispatch interrupt: u8Interrupt=%x (%d) rc=%Rrc cs:rip=%04X:%RGv\n", pVCpu->idCpu, u8Interrupt, u8Interrupt, rc, pCtx->cs, (RTGCPTR)pCtx->rip));
910 if (RT_SUCCESS(rc))
911 {
912 rc = TRPMAssertTrap(pVCpu, u8Interrupt, TRPM_HARDWARE_INT);
913 AssertRC(rc);
914 }
915 else
916 {
917 /* Can only happen in rare cases where a pending interrupt is cleared behind our back */
918 Assert(!VMCPU_FF_ISPENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC|VMCPU_FF_INTERRUPT_PIC)));
919 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatSwitchGuestIrq);
920 /* Just continue */
921 }
922 }
923 else
924 Log(("Pending interrupt blocked at %RGv by VM_FF_INHIBIT_INTERRUPTS!!\n", (RTGCPTR)pCtx->rip));
925 }
926 }
927
928#ifdef VBOX_STRICT
929 if (TRPMHasTrap(pVCpu))
930 {
931 uint8_t u8Vector;
932 rc = TRPMQueryTrapAll(pVCpu, &u8Vector, 0, 0, 0);
933 AssertRC(rc);
934 }
935#endif
936
937 if ( (pCtx->eflags.u32 & X86_EFL_IF)
938 && (!VMCPU_FF_ISSET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
939 && TRPMHasTrap(pVCpu)
940 )
941 {
942 uint8_t u8Vector;
943 TRPMEVENT enmType;
944 RTGCUINTPTR intInfo;
945 RTGCUINT errCode;
946
947 /* If a new event is pending, then dispatch it now. */
948 rc = TRPMQueryTrapAll(pVCpu, &u8Vector, &enmType, &errCode, 0);
949 AssertRC(rc);
950 Assert(pCtx->eflags.Bits.u1IF == 1 || enmType == TRPM_TRAP);
951 Assert(enmType != TRPM_SOFTWARE_INT);
952
953 /* Clear the pending trap. */
954 rc = TRPMResetTrap(pVCpu);
955 AssertRC(rc);
956
957 intInfo = u8Vector;
958 intInfo |= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
959
960 if (enmType == TRPM_TRAP)
961 {
962 switch (u8Vector) {
963 case 8:
964 case 10:
965 case 11:
966 case 12:
967 case 13:
968 case 14:
969 case 17:
970 /* Valid error codes. */
971 intInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
972 break;
973 default:
974 break;
975 }
976 if (u8Vector == X86_XCPT_BP || u8Vector == X86_XCPT_OF)
977 intInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SWEXCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
978 else
979 intInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HWEXCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
980 }
981 else
982 intInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
983
984 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatIntInject);
985 rc = VMXR0InjectEvent(pVM, pVCpu, pCtx, intInfo, 0, errCode);
986 AssertRC(rc);
987 } /* if (interrupts can be dispatched) */
988
989 return VINF_SUCCESS;
990}
991
992/**
993 * Save the host state
994 *
995 * @returns VBox status code.
996 * @param pVM The VM to operate on.
997 * @param pVCpu The VMCPU to operate on.
998 */
999VMMR0DECL(int) VMXR0SaveHostState(PVM pVM, PVMCPU pVCpu)
1000{
1001 int rc = VINF_SUCCESS;
1002
1003 /*
1004 * Host CPU Context
1005 */
1006 if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_HOST_CONTEXT)
1007 {
1008 RTIDTR idtr;
1009 RTGDTR gdtr;
1010 RTSEL SelTR;
1011 PCX86DESCHC pDesc;
1012 uintptr_t trBase;
1013 RTSEL cs;
1014 RTSEL ss;
1015 uint64_t cr3;
1016
1017 /* Control registers */
1018 rc = VMXWriteVMCS(VMX_VMCS_HOST_CR0, ASMGetCR0());
1019#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
1020 if (VMX_IS_64BIT_HOST_MODE())
1021 {
1022 cr3 = hwaccmR0Get64bitCR3();
1023 rc |= VMXWriteVMCS64(VMX_VMCS_HOST_CR3, cr3);
1024 }
1025 else
1026#endif
1027 {
1028 cr3 = ASMGetCR3();
1029 rc |= VMXWriteVMCS(VMX_VMCS_HOST_CR3, cr3);
1030 }
1031 rc |= VMXWriteVMCS(VMX_VMCS_HOST_CR4, ASMGetCR4());
1032 AssertRC(rc);
1033 Log2(("VMX_VMCS_HOST_CR0 %08x\n", ASMGetCR0()));
1034 Log2(("VMX_VMCS_HOST_CR3 %08RX64\n", cr3));
1035 Log2(("VMX_VMCS_HOST_CR4 %08x\n", ASMGetCR4()));
1036
1037 /* Selector registers. */
1038#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
1039 if (VMX_IS_64BIT_HOST_MODE())
1040 {
1041 cs = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelCS;
1042 ss = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelSS;
1043 }
1044 else
1045 {
1046 /* sysenter loads LDT cs & ss, VMX doesn't like this. Load the GDT ones (safe). */
1047 cs = (RTSEL)(uintptr_t)&SUPR0AbsKernelCS;
1048 ss = (RTSEL)(uintptr_t)&SUPR0AbsKernelSS;
1049 }
1050#else
1051 cs = ASMGetCS();
1052 ss = ASMGetSS();
1053#endif
1054 Assert(!(cs & X86_SEL_LDT)); Assert((cs & X86_SEL_RPL) == 0);
1055 Assert(!(ss & X86_SEL_LDT)); Assert((ss & X86_SEL_RPL) == 0);
1056 rc = VMXWriteVMCS(VMX_VMCS16_HOST_FIELD_CS, cs);
1057 /* Note: VMX is (again) very picky about the RPL of the selectors here; we'll restore them manually. */
1058 rc |= VMXWriteVMCS(VMX_VMCS16_HOST_FIELD_DS, 0);
1059 rc |= VMXWriteVMCS(VMX_VMCS16_HOST_FIELD_ES, 0);
1060#if HC_ARCH_BITS == 32
1061 if (!VMX_IS_64BIT_HOST_MODE())
1062 {
1063 rc |= VMXWriteVMCS(VMX_VMCS16_HOST_FIELD_FS, 0);
1064 rc |= VMXWriteVMCS(VMX_VMCS16_HOST_FIELD_GS, 0);
1065 }
1066#endif
1067 rc |= VMXWriteVMCS(VMX_VMCS16_HOST_FIELD_SS, ss);
1068 SelTR = ASMGetTR();
1069 rc |= VMXWriteVMCS(VMX_VMCS16_HOST_FIELD_TR, SelTR);
1070 AssertRC(rc);
1071 Log2(("VMX_VMCS_HOST_FIELD_CS %08x (%08x)\n", cs, ASMGetSS()));
1072 Log2(("VMX_VMCS_HOST_FIELD_DS 00000000 (%08x)\n", ASMGetDS()));
1073 Log2(("VMX_VMCS_HOST_FIELD_ES 00000000 (%08x)\n", ASMGetES()));
1074 Log2(("VMX_VMCS_HOST_FIELD_FS 00000000 (%08x)\n", ASMGetFS()));
1075 Log2(("VMX_VMCS_HOST_FIELD_GS 00000000 (%08x)\n", ASMGetGS()));
1076 Log2(("VMX_VMCS_HOST_FIELD_SS %08x (%08x)\n", ss, ASMGetSS()));
1077 Log2(("VMX_VMCS_HOST_FIELD_TR %08x\n", ASMGetTR()));
1078
1079 /* GDTR & IDTR */
1080#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
1081 if (VMX_IS_64BIT_HOST_MODE())
1082 {
1083 X86XDTR64 gdtr64, idtr64;
1084 hwaccmR0Get64bitGDTRandIDTR(&gdtr64, &idtr64);
1085 rc = VMXWriteVMCS64(VMX_VMCS_HOST_GDTR_BASE, gdtr64.uAddr);
1086 rc |= VMXWriteVMCS64(VMX_VMCS_HOST_IDTR_BASE, gdtr64.uAddr);
1087 AssertRC(rc);
1088 Log2(("VMX_VMCS_HOST_GDTR_BASE %RX64\n", gdtr64.uAddr));
1089 Log2(("VMX_VMCS_HOST_IDTR_BASE %RX64\n", idtr64.uAddr));
1090 gdtr.cbGdt = gdtr64.cb;
1091 gdtr.pGdt = (uintptr_t)gdtr64.uAddr;
1092 }
1093 else
1094#endif
1095 {
1096 ASMGetGDTR(&gdtr);
1097 rc = VMXWriteVMCS(VMX_VMCS_HOST_GDTR_BASE, gdtr.pGdt);
1098 ASMGetIDTR(&idtr);
1099 rc |= VMXWriteVMCS(VMX_VMCS_HOST_IDTR_BASE, idtr.pIdt);
1100 AssertRC(rc);
1101 Log2(("VMX_VMCS_HOST_GDTR_BASE %RHv\n", gdtr.pGdt));
1102 Log2(("VMX_VMCS_HOST_IDTR_BASE %RHv\n", idtr.pIdt));
1103 }
1104
1105 /* Save the base address of the TR selector. */
1106 if (SelTR > gdtr.cbGdt)
1107 {
1108 AssertMsgFailed(("Invalid TR selector %x. GDTR.cbGdt=%x\n", SelTR, gdtr.cbGdt));
1109 return VERR_VMX_INVALID_HOST_STATE;
1110 }
1111
1112 pDesc = (PCX86DESCHC)(gdtr.pGdt + (SelTR & X86_SEL_MASK));
1113#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
1114 if (VMX_IS_64BIT_HOST_MODE())
1115 {
1116 uint64_t trBase64 = X86DESC64_BASE(*(PX86DESC64)pDesc);
1117 rc = VMXWriteVMCS64(VMX_VMCS_HOST_TR_BASE, trBase64);
1118 Log2(("VMX_VMCS_HOST_TR_BASE %RX64\n", trBase64));
1119 AssertRC(rc);
1120 }
1121 else
1122#endif
1123 {
1124#if HC_ARCH_BITS == 64
1125 trBase = X86DESC64_BASE(*pDesc);
1126#else
1127 trBase = X86DESC_BASE(*pDesc);
1128#endif
1129 rc = VMXWriteVMCS(VMX_VMCS_HOST_TR_BASE, trBase);
1130 AssertRC(rc);
1131 Log2(("VMX_VMCS_HOST_TR_BASE %RHv\n", trBase));
1132 }
1133
1134 /* FS and GS base. */
1135#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
1136 if (VMX_IS_64BIT_HOST_MODE())
1137 {
1138 Log2(("MSR_K8_FS_BASE = %RX64\n", ASMRdMsr(MSR_K8_FS_BASE)));
1139 Log2(("MSR_K8_GS_BASE = %RX64\n", ASMRdMsr(MSR_K8_GS_BASE)));
1140 rc = VMXWriteVMCS64(VMX_VMCS_HOST_FS_BASE, ASMRdMsr(MSR_K8_FS_BASE));
1141 rc |= VMXWriteVMCS64(VMX_VMCS_HOST_GS_BASE, ASMRdMsr(MSR_K8_GS_BASE));
1142 }
1143#endif
1144 AssertRC(rc);
1145
1146 /* Sysenter MSRs. */
1147 /** @todo expensive!! */
1148 rc = VMXWriteVMCS(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
1149 Log2(("VMX_VMCS_HOST_SYSENTER_CS %08x\n", ASMRdMsr_Low(MSR_IA32_SYSENTER_CS)));
1150#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
1151 if (VMX_IS_64BIT_HOST_MODE())
1152 {
1153 Log2(("VMX_VMCS_HOST_SYSENTER_EIP %RX64\n", ASMRdMsr(MSR_IA32_SYSENTER_EIP)));
1154 Log2(("VMX_VMCS_HOST_SYSENTER_ESP %RX64\n", ASMRdMsr(MSR_IA32_SYSENTER_ESP)));
1155 rc |= VMXWriteVMCS64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
1156 rc |= VMXWriteVMCS64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
1157 }
1158 else
1159 {
1160 rc |= VMXWriteVMCS(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
1161 rc |= VMXWriteVMCS(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
1162 Log2(("VMX_VMCS_HOST_SYSENTER_EIP %RX32\n", ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP)));
1163 Log2(("VMX_VMCS_HOST_SYSENTER_ESP %RX32\n", ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP)));
1164 }
1165#elif HC_ARCH_BITS == 32
1166 rc |= VMXWriteVMCS(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
1167 rc |= VMXWriteVMCS(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
1168 Log2(("VMX_VMCS_HOST_SYSENTER_EIP %RX32\n", ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP)));
1169 Log2(("VMX_VMCS_HOST_SYSENTER_ESP %RX32\n", ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP)));
1170#else
1171 Log2(("VMX_VMCS_HOST_SYSENTER_EIP %RX64\n", ASMRdMsr(MSR_IA32_SYSENTER_EIP)));
1172 Log2(("VMX_VMCS_HOST_SYSENTER_ESP %RX64\n", ASMRdMsr(MSR_IA32_SYSENTER_ESP)));
1173 rc |= VMXWriteVMCS64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
1174 rc |= VMXWriteVMCS64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
1175#endif
1176 AssertRC(rc);
1177
1178#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
1179 /* Store all host MSRs in the VM-Exit load area, so they will be reloaded after the world switch back to the host. */
1180 PVMXMSR pMsr = (PVMXMSR)pVCpu->hwaccm.s.vmx.pHostMSR;
1181 unsigned idxMsr = 0;
1182
1183 /* EFER MSR present? */
1184 if (ASMCpuId_EDX(0x80000001) & (X86_CPUID_AMD_FEATURE_EDX_NX|X86_CPUID_AMD_FEATURE_EDX_LONG_MODE))
1185 {
1186 if (ASMCpuId_EDX(0x80000001) & X86_CPUID_AMD_FEATURE_EDX_SEP)
1187 {
1188 pMsr->u32IndexMSR = MSR_K6_STAR;
1189 pMsr->u32Reserved = 0;
1190 pMsr->u64Value = ASMRdMsr(MSR_K6_STAR); /* legacy syscall eip, cs & ss */
1191 pMsr++; idxMsr++;
1192 }
1193
1194 pMsr->u32IndexMSR = MSR_K6_EFER;
1195 pMsr->u32Reserved = 0;
1196# if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
1197 if (CPUMIsGuestInLongMode(pVCpu))
1198 {
1199 /* Must match the efer value in our 64 bits switcher. */
1200 pMsr->u64Value = ASMRdMsr(MSR_K6_EFER) | MSR_K6_EFER_LME | MSR_K6_EFER_SCE | MSR_K6_EFER_NXE;
1201 }
1202 else
1203# endif
1204 pMsr->u64Value = ASMRdMsr(MSR_K6_EFER);
1205 pMsr++; idxMsr++;
1206 }
1207
1208# if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
1209 if (VMX_IS_64BIT_HOST_MODE())
1210 {
1211 pMsr->u32IndexMSR = MSR_K8_LSTAR;
1212 pMsr->u32Reserved = 0;
1213 pMsr->u64Value = ASMRdMsr(MSR_K8_LSTAR); /* 64 bits mode syscall rip */
1214 pMsr++; idxMsr++;
1215 pMsr->u32IndexMSR = MSR_K8_SF_MASK;
1216 pMsr->u32Reserved = 0;
1217 pMsr->u64Value = ASMRdMsr(MSR_K8_SF_MASK); /* syscall flag mask */
1218 pMsr++; idxMsr++;
1219 pMsr->u32IndexMSR = MSR_K8_KERNEL_GS_BASE;
1220 pMsr->u32Reserved = 0;
1221 pMsr->u64Value = ASMRdMsr(MSR_K8_KERNEL_GS_BASE); /* swapgs exchange value */
1222 pMsr++; idxMsr++;
1223 }
1224# endif
1225 rc = VMXWriteVMCS(VMX_VMCS_CTRL_EXIT_MSR_LOAD_COUNT, idxMsr);
1226 AssertRC(rc);
1227#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
1228
1229 pVCpu->hwaccm.s.fContextUseFlags &= ~HWACCM_CHANGED_HOST_CONTEXT;
1230 }
1231 return rc;
1232}
1233
1234/**
1235 * Prefetch the 4 PDPT pointers (PAE and nested paging only)
1236 *
1237 * @param pVM The VM to operate on.
1238 * @param pVCpu The VMCPU to operate on.
1239 * @param pCtx Guest context
1240 */
1241static void vmxR0PrefetchPAEPdptrs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
1242{
1243 if (CPUMIsGuestInPAEModeEx(pCtx))
1244 {
1245 X86PDPE Pdpe;
1246
1247 for (unsigned i=0;i<4;i++)
1248 {
1249 Pdpe = PGMGstGetPaePDPtr(pVCpu, i);
1250 int rc = VMXWriteVMCS64(VMX_VMCS_GUEST_PDPTR0_FULL + i*2, Pdpe.u);
1251 AssertRC(rc);
1252 }
1253 }
1254}
1255
1256/**
1257 * Update the exception bitmap according to the current CPU state
1258 *
1259 * @param pVM The VM to operate on.
1260 * @param pVCpu The VMCPU to operate on.
1261 * @param pCtx Guest context
1262 */
1263static void vmxR0UpdateExceptionBitmap(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
1264{
1265 uint32_t u32TrapMask;
1266 Assert(pCtx);
1267
1268 u32TrapMask = HWACCM_VMX_TRAP_MASK;
1269#ifndef DEBUG
1270 if (pVM->hwaccm.s.fNestedPaging)
1271 u32TrapMask &= ~RT_BIT(X86_XCPT_PF); /* no longer need to intercept #PF. */
1272#endif
1273
1274 /* Also catch floating point exceptions as we need to report them to the guest in a different way. */
1275 if ( CPUMIsGuestFPUStateActive(pVCpu) == true
1276 && !(pCtx->cr0 & X86_CR0_NE)
1277 && !pVCpu->hwaccm.s.fFPUOldStyleOverride)
1278 {
1279 u32TrapMask |= RT_BIT(X86_XCPT_MF);
1280 pVCpu->hwaccm.s.fFPUOldStyleOverride = true;
1281 }
1282
1283#ifdef DEBUG /* till after branching, enable it by default then. */
1284 /* Intercept X86_XCPT_DB if stepping is enabled */
1285 if ( DBGFIsStepping(pVCpu)
1286 || CPUMIsHyperDebugStateActive(pVCpu))
1287 u32TrapMask |= RT_BIT(X86_XCPT_DB);
1288 /** @todo Don't trap it unless the debugger has armed breakpoints. */
1289 u32TrapMask |= RT_BIT(X86_XCPT_BP);
1290#endif
1291
1292#ifdef VBOX_STRICT
1293 Assert(u32TrapMask & RT_BIT(X86_XCPT_GP));
1294#endif
1295
1296# ifdef HWACCM_VMX_EMULATE_REALMODE
1297 /* Intercept all exceptions in real mode as none of them can be injected directly (#GP otherwise). */
1298 if ( CPUMIsGuestInRealModeEx(pCtx)
1299 && pVM->hwaccm.s.vmx.pRealModeTSS)
1300 u32TrapMask |= HWACCM_VMX_TRAP_MASK_REALMODE;
1301# endif /* HWACCM_VMX_EMULATE_REALMODE */
1302
1303 int rc = VMXWriteVMCS(VMX_VMCS_CTRL_EXCEPTION_BITMAP, u32TrapMask);
1304 AssertRC(rc);
1305}
1306
1307/**
1308 * Loads the guest state
1309 *
1310 * NOTE: Don't do anything here that can cause a jump back to ring 3!!!!!
1311 *
1312 * @returns VBox status code.
1313 * @param pVM The VM to operate on.
1314 * @param pVCpu The VMCPU to operate on.
1315 * @param pCtx Guest context
1316 */
1317VMMR0DECL(int) VMXR0LoadGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
1318{
1319 int rc = VINF_SUCCESS;
1320 RTGCUINTPTR val;
1321 X86EFLAGS eflags;
1322
1323 /* VMX_VMCS_CTRL_ENTRY_CONTROLS
1324 * Set required bits to one and zero according to the MSR capabilities.
1325 */
1326 val = pVM->hwaccm.s.vmx.msr.vmx_entry.n.disallowed0;
1327 /* Load guest debug controls (dr7 & IA32_DEBUGCTL_MSR) (forced to 1 on the 'first' VT-x capable CPUs; this actually includes the newest Nehalem CPUs) */
1328 val |= VMX_VMCS_CTRL_ENTRY_CONTROLS_LOAD_DEBUG;
1329 /* 64 bits guest mode? */
1330 if (CPUMIsGuestInLongModeEx(pCtx))
1331 val |= VMX_VMCS_CTRL_ENTRY_CONTROLS_IA64_MODE;
1332 /* else Must be zero when AMD64 is not available. */
1333
1334 /* Mask away the bits that the CPU doesn't support */
1335 val &= pVM->hwaccm.s.vmx.msr.vmx_entry.n.allowed1;
1336 rc = VMXWriteVMCS(VMX_VMCS_CTRL_ENTRY_CONTROLS, val);
1337 AssertRC(rc);
1338
1339 /* VMX_VMCS_CTRL_EXIT_CONTROLS
1340 * Set required bits to one and zero according to the MSR capabilities.
1341 */
1342 val = pVM->hwaccm.s.vmx.msr.vmx_exit.n.disallowed0;
1343
1344 /* Save debug controls (dr7 & IA32_DEBUGCTL_MSR) (forced to 1 on the 'first' VT-x capable CPUs; this actually includes the newest Nehalem CPUs) */
1345 val |= VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_DEBUG;
1346
1347#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
1348 if (VMX_IS_64BIT_HOST_MODE())
1349 val |= VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_AMD64;
1350 /* else: Must be zero when AMD64 is not available. */
1351#elif HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
1352 if (CPUMIsGuestInLongModeEx(pCtx))
1353 val |= VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_AMD64; /* our switcher goes to long mode */
1354 else
1355 Assert(!(val & VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_AMD64));
1356#endif
1357 val &= pVM->hwaccm.s.vmx.msr.vmx_exit.n.allowed1;
1358 /* Don't acknowledge external interrupts on VM-exit. */
1359 rc = VMXWriteVMCS(VMX_VMCS_CTRL_EXIT_CONTROLS, val);
1360 AssertRC(rc);
1361
1362 /* Guest CPU context: ES, CS, SS, DS, FS, GS. */
1363 if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_SEGMENT_REGS)
1364 {
1365#ifdef HWACCM_VMX_EMULATE_REALMODE
1366 if (pVM->hwaccm.s.vmx.pRealModeTSS)
1367 {
1368 PGMMODE enmGuestMode = PGMGetGuestMode(pVCpu);
1369 if (pVCpu->hwaccm.s.vmx.enmLastSeenGuestMode != enmGuestMode)
1370 {
1371 /* Correct weird requirements for switching to protected mode. */
1372 if ( pVCpu->hwaccm.s.vmx.enmLastSeenGuestMode == PGMMODE_REAL
1373 && enmGuestMode >= PGMMODE_PROTECTED)
1374 {
1375 /* Flush the recompiler code cache as it's not unlikely
1376 * the guest will rewrite code it will later execute in real
1377 * mode (OpenBSD 4.0 is one such example)
1378 */
1379 REMFlushTBs(pVM);
1380
1381 /* DPL of all hidden selector registers must match the current CPL (0). */
1382 pCtx->csHid.Attr.n.u2Dpl = 0;
1383 pCtx->csHid.Attr.n.u4Type = X86_SEL_TYPE_CODE | X86_SEL_TYPE_RW_ACC;
1384
1385 pCtx->dsHid.Attr.n.u2Dpl = 0;
1386 pCtx->esHid.Attr.n.u2Dpl = 0;
1387 pCtx->fsHid.Attr.n.u2Dpl = 0;
1388 pCtx->gsHid.Attr.n.u2Dpl = 0;
1389 pCtx->ssHid.Attr.n.u2Dpl = 0;
1390
1391 /* The limit must correspond to the 32 bits setting. */
1392 if (!pCtx->csHid.Attr.n.u1DefBig)
1393 pCtx->csHid.u32Limit &= 0xffff;
1394 if (!pCtx->dsHid.Attr.n.u1DefBig)
1395 pCtx->dsHid.u32Limit &= 0xffff;
1396 if (!pCtx->esHid.Attr.n.u1DefBig)
1397 pCtx->esHid.u32Limit &= 0xffff;
1398 if (!pCtx->fsHid.Attr.n.u1DefBig)
1399 pCtx->fsHid.u32Limit &= 0xffff;
1400 if (!pCtx->gsHid.Attr.n.u1DefBig)
1401 pCtx->gsHid.u32Limit &= 0xffff;
1402 if (!pCtx->ssHid.Attr.n.u1DefBig)
1403 pCtx->ssHid.u32Limit &= 0xffff;
1404 }
1405 else
1406 /* Switching from protected mode to real mode. */
1407 if ( pVCpu->hwaccm.s.vmx.enmLastSeenGuestMode >= PGMMODE_PROTECTED
1408 && enmGuestMode == PGMMODE_REAL)
1409 {
1410 /* The limit must also be set to 0xffff. */
1411 pCtx->csHid.u32Limit = 0xffff;
1412 pCtx->dsHid.u32Limit = 0xffff;
1413 pCtx->esHid.u32Limit = 0xffff;
1414 pCtx->fsHid.u32Limit = 0xffff;
1415 pCtx->gsHid.u32Limit = 0xffff;
1416 pCtx->ssHid.u32Limit = 0xffff;
1417
1418 Assert(pCtx->csHid.u64Base <= 0xfffff);
1419 Assert(pCtx->dsHid.u64Base <= 0xfffff);
1420 Assert(pCtx->esHid.u64Base <= 0xfffff);
1421 Assert(pCtx->fsHid.u64Base <= 0xfffff);
1422 Assert(pCtx->gsHid.u64Base <= 0xfffff);
1423 }
1424 pVCpu->hwaccm.s.vmx.enmLastSeenGuestMode = enmGuestMode;
1425 }
1426 else
1427 /* VT-x will fail with a guest invalid state otherwise... (CPU state after a reset) */
1428 if ( CPUMIsGuestInRealModeEx(pCtx)
1429 && pCtx->csHid.u64Base == 0xffff0000)
1430 {
1431 pCtx->csHid.u64Base = 0xf0000;
1432 pCtx->cs = 0xf000;
1433 }
1434 }
1435#endif /* HWACCM_VMX_EMULATE_REALMODE */
1436
1437 VMX_WRITE_SELREG(ES, es);
1438 AssertRC(rc);
1439
1440 VMX_WRITE_SELREG(CS, cs);
1441 AssertRC(rc);
1442
1443 VMX_WRITE_SELREG(SS, ss);
1444 AssertRC(rc);
1445
1446 VMX_WRITE_SELREG(DS, ds);
1447 AssertRC(rc);
1448
1449 VMX_WRITE_SELREG(FS, fs);
1450 AssertRC(rc);
1451
1452 VMX_WRITE_SELREG(GS, gs);
1453 AssertRC(rc);
1454 }
1455
1456 /* Guest CPU context: LDTR. */
1457 if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_LDTR)
1458 {
1459 if (pCtx->ldtr == 0)
1460 {
1461 rc = VMXWriteVMCS(VMX_VMCS16_GUEST_FIELD_LDTR, 0);
1462 rc |= VMXWriteVMCS(VMX_VMCS32_GUEST_LDTR_LIMIT, 0);
1463 rc |= VMXWriteVMCS64(VMX_VMCS64_GUEST_LDTR_BASE, 0);
1464 /* Note: vmlaunch will fail with 0 or just 0x02. No idea why. */
1465 rc |= VMXWriteVMCS(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, 0x82 /* present, LDT */);
1466 }
1467 else
1468 {
1469 rc = VMXWriteVMCS(VMX_VMCS16_GUEST_FIELD_LDTR, pCtx->ldtr);
1470 rc |= VMXWriteVMCS(VMX_VMCS32_GUEST_LDTR_LIMIT, pCtx->ldtrHid.u32Limit);
1471 rc |= VMXWriteVMCS64(VMX_VMCS64_GUEST_LDTR_BASE, pCtx->ldtrHid.u64Base);
1472 rc |= VMXWriteVMCS(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, pCtx->ldtrHid.Attr.u);
1473 }
1474 AssertRC(rc);
1475 }
1476 /* Guest CPU context: TR. */
1477 if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_TR)
1478 {
1479#ifdef HWACCM_VMX_EMULATE_REALMODE
1480 /* Real mode emulation using v86 mode with CR4.VME (interrupt redirection using the int bitmap in the TSS) */
1481 if ( CPUMIsGuestInRealModeEx(pCtx)
1482 && pVM->hwaccm.s.vmx.pRealModeTSS)
1483 {
1484 RTGCPHYS GCPhys;
1485
1486 /* We convert it here every time as pci regions could be reconfigured. */
1487 rc = PDMVMMDevHeapR3ToGCPhys(pVM, pVM->hwaccm.s.vmx.pRealModeTSS, &GCPhys);
1488 AssertRC(rc);
1489
1490 rc = VMXWriteVMCS(VMX_VMCS16_GUEST_FIELD_TR, 0);
1491 rc |= VMXWriteVMCS(VMX_VMCS32_GUEST_TR_LIMIT, HWACCM_VTX_TSS_SIZE);
1492 rc |= VMXWriteVMCS64(VMX_VMCS64_GUEST_TR_BASE, GCPhys /* phys = virt in this mode */);
1493
1494 X86DESCATTR attr;
1495
1496 attr.u = 0;
1497 attr.n.u1Present = 1;
1498 attr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
1499 val = attr.u;
1500 }
1501 else
1502#endif /* HWACCM_VMX_EMULATE_REALMODE */
1503 {
1504 rc = VMXWriteVMCS(VMX_VMCS16_GUEST_FIELD_TR, pCtx->tr);
1505 rc |= VMXWriteVMCS(VMX_VMCS32_GUEST_TR_LIMIT, pCtx->trHid.u32Limit);
1506 rc |= VMXWriteVMCS64(VMX_VMCS64_GUEST_TR_BASE, pCtx->trHid.u64Base);
1507
1508 val = pCtx->trHid.Attr.u;
1509
1510 /* The TSS selector must be busy. */
1511 if ((val & 0xF) == X86_SEL_TYPE_SYS_286_TSS_AVAIL)
1512 val = (val & ~0xF) | X86_SEL_TYPE_SYS_286_TSS_BUSY;
1513 else
1514 /* Default even if no TR selector has been set (otherwise vmlaunch will fail!) */
1515 val = (val & ~0xF) | X86_SEL_TYPE_SYS_386_TSS_BUSY;
1516
1517 }
1518 rc |= VMXWriteVMCS(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, val);
1519 AssertRC(rc);
1520 }
1521 /* Guest CPU context: GDTR. */
1522 if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_GDTR)
1523 {
1524 rc = VMXWriteVMCS(VMX_VMCS32_GUEST_GDTR_LIMIT, pCtx->gdtr.cbGdt);
1525 rc |= VMXWriteVMCS64(VMX_VMCS64_GUEST_GDTR_BASE, pCtx->gdtr.pGdt);
1526 AssertRC(rc);
1527 }
1528 /* Guest CPU context: IDTR. */
1529 if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_IDTR)
1530 {
1531 rc = VMXWriteVMCS(VMX_VMCS32_GUEST_IDTR_LIMIT, pCtx->idtr.cbIdt);
1532 rc |= VMXWriteVMCS64(VMX_VMCS64_GUEST_IDTR_BASE, pCtx->idtr.pIdt);
1533 AssertRC(rc);
1534 }
1535
1536 /*
1537 * Sysenter MSRs (unconditional)
1538 */
1539 rc = VMXWriteVMCS(VMX_VMCS32_GUEST_SYSENTER_CS, pCtx->SysEnter.cs);
1540 rc |= VMXWriteVMCS64(VMX_VMCS64_GUEST_SYSENTER_EIP, pCtx->SysEnter.eip);
1541 rc |= VMXWriteVMCS64(VMX_VMCS64_GUEST_SYSENTER_ESP, pCtx->SysEnter.esp);
1542 AssertRC(rc);
1543
1544 /* Control registers */
1545 if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_CR0)
1546 {
1547 val = pCtx->cr0;
1548 rc = VMXWriteVMCS(VMX_VMCS_CTRL_CR0_READ_SHADOW, val);
1549 Log2(("Guest CR0-shadow %08x\n", val));
1550 if (CPUMIsGuestFPUStateActive(pVCpu) == false)
1551 {
1552 /* Always use #NM exceptions to load the FPU/XMM state on demand. */
1553 val |= X86_CR0_TS | X86_CR0_ET | X86_CR0_NE | X86_CR0_MP;
1554 }
1555 else
1556 {
1557 /** @todo check if we support the old style mess correctly. */
1558 if (!(val & X86_CR0_NE))
1559 Log(("Forcing X86_CR0_NE!!!\n"));
1560
1561 val |= X86_CR0_NE; /* always turn on the native mechanism to report FPU errors (old style uses interrupts) */
1562 }
1563 /* Note: protected mode & paging are always enabled; we use them for emulating real and protected mode without paging too. */
1564 if (!pVM->hwaccm.s.vmx.fUnrestrictedGuest)
1565 val |= X86_CR0_PE | X86_CR0_PG;
1566
1567 if (pVM->hwaccm.s.fNestedPaging)
1568 {
1569 if (CPUMIsGuestInPagedProtectedModeEx(pCtx))
1570 {
1571 /* Disable cr3 read/write monitoring as we don't need it for EPT. */
1572 pVCpu->hwaccm.s.vmx.proc_ctls &= ~( VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
1573 | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT);
1574 }
1575 else
1576 {
1577 /* Reenable cr3 read/write monitoring as our identity mapped page table is active. */
1578 pVCpu->hwaccm.s.vmx.proc_ctls |= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
1579 | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT;
1580 }
1581 rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
1582 AssertRC(rc);
1583 }
1584 else
1585 {
1586 /* Note: We must also set this as we rely on protecting various pages for which supervisor writes must be caught. */
1587 val |= X86_CR0_WP;
1588 }
1589
1590 /* Always enable caching. */
1591 val &= ~(X86_CR0_CD|X86_CR0_NW);
1592
1593 rc |= VMXWriteVMCS64(VMX_VMCS64_GUEST_CR0, val);
1594 Log2(("Guest CR0 %08x\n", val));
1595 /* CR0 flags owned by the host; if the guests attempts to change them, then
1596 * the VM will exit.
1597 */
1598 val = X86_CR0_PE /* Must monitor this bit (assumptions are made for real mode emulation) */
1599 | X86_CR0_WP /* Must monitor this bit (it must always be enabled). */
1600 | X86_CR0_PG /* Must monitor this bit (assumptions are made for real mode & protected mode without paging emulation) */
1601 | X86_CR0_TS
1602 | X86_CR0_ET /* Bit not restored during VM-exit! */
1603 | X86_CR0_CD /* Bit not restored during VM-exit! */
1604 | X86_CR0_NW /* Bit not restored during VM-exit! */
1605 | X86_CR0_NE
1606 | X86_CR0_MP;
1607 pVCpu->hwaccm.s.vmx.cr0_mask = val;
1608
1609 rc |= VMXWriteVMCS(VMX_VMCS_CTRL_CR0_MASK, val);
1610 Log2(("Guest CR0-mask %08x\n", val));
1611 AssertRC(rc);
1612 }
1613 if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_CR4)
1614 {
1615 /* CR4 */
1616 rc = VMXWriteVMCS(VMX_VMCS_CTRL_CR4_READ_SHADOW, pCtx->cr4);
1617 Log2(("Guest CR4-shadow %08x\n", pCtx->cr4));
1618 /* Set the required bits in cr4 too (currently X86_CR4_VMXE). */
1619 val = pCtx->cr4 | (uint32_t)pVM->hwaccm.s.vmx.msr.vmx_cr4_fixed0;
1620
1621 if (!pVM->hwaccm.s.fNestedPaging)
1622 {
1623 switch(pVCpu->hwaccm.s.enmShadowMode)
1624 {
1625 case PGMMODE_REAL: /* Real mode -> emulated using v86 mode */
1626 case PGMMODE_PROTECTED: /* Protected mode, no paging -> emulated using identity mapping. */
1627 case PGMMODE_32_BIT: /* 32-bit paging. */
1628 val &= ~X86_CR4_PAE;
1629 break;
1630
1631 case PGMMODE_PAE: /* PAE paging. */
1632 case PGMMODE_PAE_NX: /* PAE paging with NX enabled. */
1633 /** @todo use normal 32 bits paging */
1634 val |= X86_CR4_PAE;
1635 break;
1636
1637 case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
1638 case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
1639#ifdef VBOX_ENABLE_64_BITS_GUESTS
1640 break;
1641#else
1642 AssertFailed();
1643 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
1644#endif
1645 default: /* shut up gcc */
1646 AssertFailed();
1647 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
1648 }
1649 }
1650 else
1651 if ( !CPUMIsGuestInPagedProtectedModeEx(pCtx)
1652 && !pVM->hwaccm.s.vmx.fUnrestrictedGuest)
1653 {
1654 /* We use 4 MB pages in our identity mapping page table for real and protected mode without paging. */
1655 val |= X86_CR4_PSE;
1656 /* Our identity mapping is a 32 bits page directory. */
1657 val &= ~X86_CR4_PAE;
1658 }
1659
1660#ifdef HWACCM_VMX_EMULATE_REALMODE
1661 /* Turn off VME if we're in emulated real mode. */
1662 if ( CPUMIsGuestInRealModeEx(pCtx)
1663 && pVM->hwaccm.s.vmx.pRealModeTSS)
1664 val &= ~X86_CR4_VME;
1665#endif /* HWACCM_VMX_EMULATE_REALMODE */
1666
1667 rc |= VMXWriteVMCS64(VMX_VMCS64_GUEST_CR4, val);
1668 Log2(("Guest CR4 %08x\n", val));
1669 /* CR4 flags owned by the host; if the guests attempts to change them, then
1670 * the VM will exit.
1671 */
1672 val = 0
1673 | X86_CR4_VME
1674 | X86_CR4_PAE
1675 | X86_CR4_PGE
1676 | X86_CR4_PSE
1677 | X86_CR4_VMXE;
1678 pVCpu->hwaccm.s.vmx.cr4_mask = val;
1679
1680 rc |= VMXWriteVMCS(VMX_VMCS_CTRL_CR4_MASK, val);
1681 Log2(("Guest CR4-mask %08x\n", val));
1682 AssertRC(rc);
1683 }
1684
1685 if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_CR3)
1686 {
1687 if (pVM->hwaccm.s.fNestedPaging)
1688 {
1689 Assert(PGMGetHyperCR3(pVCpu));
1690 pVCpu->hwaccm.s.vmx.GCPhysEPTP = PGMGetHyperCR3(pVCpu);
1691
1692 Assert(!(pVCpu->hwaccm.s.vmx.GCPhysEPTP & 0xfff));
1693 /** @todo Check the IA32_VMX_EPT_VPID_CAP MSR for other supported memory types. */
1694 pVCpu->hwaccm.s.vmx.GCPhysEPTP |= VMX_EPT_MEMTYPE_WB
1695 | (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
1696
1697 rc = VMXWriteVMCS64(VMX_VMCS_CTRL_EPTP_FULL, pVCpu->hwaccm.s.vmx.GCPhysEPTP);
1698 AssertRC(rc);
1699
1700 if ( !CPUMIsGuestInPagedProtectedModeEx(pCtx)
1701 && !pVM->hwaccm.s.vmx.fUnrestrictedGuest)
1702 {
1703 RTGCPHYS GCPhys;
1704
1705 /* We convert it here every time as pci regions could be reconfigured. */
1706 rc = PDMVMMDevHeapR3ToGCPhys(pVM, pVM->hwaccm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
1707 AssertMsgRC(rc, ("pNonPagingModeEPTPageTable = %RGv\n", pVM->hwaccm.s.vmx.pNonPagingModeEPTPageTable));
1708
1709 /* We use our identity mapping page table here as we need to map guest virtual to guest physical addresses; EPT will
1710 * take care of the translation to host physical addresses.
1711 */
1712 val = GCPhys;
1713 }
1714 else
1715 {
1716 /* Save the real guest CR3 in VMX_VMCS_GUEST_CR3 */
1717 val = pCtx->cr3;
1718 /* Prefetch the four PDPT entries in PAE mode. */
1719 vmxR0PrefetchPAEPdptrs(pVM, pVCpu, pCtx);
1720 }
1721 }
1722 else
1723 {
1724 val = PGMGetHyperCR3(pVCpu);
1725 Assert(val || VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL));
1726 }
1727
1728 /* Save our shadow CR3 register. */
1729 rc = VMXWriteVMCS64(VMX_VMCS64_GUEST_CR3, val);
1730 AssertRC(rc);
1731 }
1732
1733 /* Debug registers. */
1734 if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_DEBUG)
1735 {
1736 pCtx->dr[6] |= X86_DR6_INIT_VAL; /* set all reserved bits to 1. */
1737 pCtx->dr[6] &= ~RT_BIT(12); /* must be zero. */
1738
1739 pCtx->dr[7] &= 0xffffffff; /* upper 32 bits reserved */
1740 pCtx->dr[7] &= ~(RT_BIT(11) | RT_BIT(12) | RT_BIT(14) | RT_BIT(15)); /* must be zero */
1741 pCtx->dr[7] |= 0x400; /* must be one */
1742
1743 /* Resync DR7 */
1744 rc = VMXWriteVMCS64(VMX_VMCS64_GUEST_DR7, pCtx->dr[7]);
1745 AssertRC(rc);
1746
1747#ifdef DEBUG
1748 /* Sync the hypervisor debug state now if any breakpoint is armed. */
1749 if ( CPUMGetHyperDR7(pVCpu) & (X86_DR7_ENABLED_MASK|X86_DR7_GD)
1750 && !CPUMIsHyperDebugStateActive(pVCpu)
1751 && !DBGFIsStepping(pVCpu))
1752 {
1753 /* Save the host and load the hypervisor debug state. */
1754 rc = CPUMR0LoadHyperDebugState(pVM, pVCpu, pCtx, true /* include DR6 */);
1755 AssertRC(rc);
1756
1757 /* DRx intercepts remain enabled. */
1758
1759 /* Override dr7 with the hypervisor value. */
1760 rc = VMXWriteVMCS64(VMX_VMCS64_GUEST_DR7, CPUMGetHyperDR7(pVCpu));
1761 AssertRC(rc);
1762 }
1763 else
1764#endif
1765 /* Sync the debug state now if any breakpoint is armed. */
1766 if ( (pCtx->dr[7] & (X86_DR7_ENABLED_MASK|X86_DR7_GD))
1767 && !CPUMIsGuestDebugStateActive(pVCpu)
1768 && !DBGFIsStepping(pVCpu))
1769 {
1770 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatDRxArmed);
1771
1772 /* Disable drx move intercepts. */
1773 pVCpu->hwaccm.s.vmx.proc_ctls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT;
1774 rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
1775 AssertRC(rc);
1776
1777 /* Save the host and load the guest debug state. */
1778 rc = CPUMR0LoadGuestDebugState(pVM, pVCpu, pCtx, true /* include DR6 */);
1779 AssertRC(rc);
1780 }
1781
1782 /* IA32_DEBUGCTL MSR. */
1783 rc = VMXWriteVMCS64(VMX_VMCS_GUEST_DEBUGCTL_FULL, 0);
1784 AssertRC(rc);
1785
1786 /** @todo do we really ever need this? */
1787 rc |= VMXWriteVMCS(VMX_VMCS_GUEST_DEBUG_EXCEPTIONS, 0);
1788 AssertRC(rc);
1789 }
1790
1791 /* EIP, ESP and EFLAGS */
1792 rc = VMXWriteVMCS64(VMX_VMCS64_GUEST_RIP, pCtx->rip);
1793 rc |= VMXWriteVMCS64(VMX_VMCS64_GUEST_RSP, pCtx->rsp);
1794 AssertRC(rc);
1795
1796 /* Bits 22-31, 15, 5 & 3 must be zero. Bit 1 must be 1. */
1797 eflags = pCtx->eflags;
1798 eflags.u32 &= VMX_EFLAGS_RESERVED_0;
1799 eflags.u32 |= VMX_EFLAGS_RESERVED_1;
1800
1801#ifdef HWACCM_VMX_EMULATE_REALMODE
1802 /* Real mode emulation using v86 mode. */
1803 if ( CPUMIsGuestInRealModeEx(pCtx)
1804 && pVM->hwaccm.s.vmx.pRealModeTSS)
1805 {
1806 pVCpu->hwaccm.s.vmx.RealMode.eflags = eflags;
1807
1808 eflags.Bits.u1VM = 1;
1809 eflags.Bits.u2IOPL = 0; /* must always be 0 or else certain instructions won't cause faults. */
1810 }
1811#endif /* HWACCM_VMX_EMULATE_REALMODE */
1812 rc = VMXWriteVMCS(VMX_VMCS_GUEST_RFLAGS, eflags.u32);
1813 AssertRC(rc);
1814
1815 if (TMCpuTickCanUseRealTSC(pVCpu, &pVCpu->hwaccm.s.vmx.u64TSCOffset))
1816 {
1817 uint64_t u64CurTSC = ASMReadTSC();
1818 if (u64CurTSC + pVCpu->hwaccm.s.vmx.u64TSCOffset >= TMCpuTickGetLastSeen(pVCpu))
1819 {
1820 /* Note: VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT takes precedence over TSC_OFFSET */
1821 rc = VMXWriteVMCS64(VMX_VMCS_CTRL_TSC_OFFSET_FULL, pVCpu->hwaccm.s.vmx.u64TSCOffset);
1822 AssertRC(rc);
1823
1824 pVCpu->hwaccm.s.vmx.proc_ctls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT;
1825 rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
1826 AssertRC(rc);
1827 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatTSCOffset);
1828 }
1829 else
1830 {
1831 /* Fall back to rdtsc emulation as we would otherwise pass decreasing tsc values to the guest. */
1832 LogFlow(("TSC %RX64 offset %RX64 time=%RX64 last=%RX64 (diff=%RX64, virt_tsc=%RX64)\n", u64CurTSC, pVCpu->hwaccm.s.vmx.u64TSCOffset, u64CurTSC + pVCpu->hwaccm.s.vmx.u64TSCOffset, TMCpuTickGetLastSeen(pVCpu), TMCpuTickGetLastSeen(pVCpu) - u64CurTSC - pVCpu->hwaccm.s.vmx.u64TSCOffset, TMCpuTickGet(pVCpu)));
1833 pVCpu->hwaccm.s.vmx.proc_ctls |= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT;
1834 rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
1835 AssertRC(rc);
1836 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatTSCInterceptOverFlow);
1837 }
1838 }
1839 else
1840 {
1841 pVCpu->hwaccm.s.vmx.proc_ctls |= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT;
1842 rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
1843 AssertRC(rc);
1844 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatTSCIntercept);
1845 }
1846
1847 /* 64 bits guest mode? */
1848 if (CPUMIsGuestInLongModeEx(pCtx))
1849 {
1850#if !defined(VBOX_ENABLE_64_BITS_GUESTS)
1851 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
1852#elif HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
1853 pVCpu->hwaccm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
1854#else
1855# ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
1856 if (!pVM->hwaccm.s.fAllow64BitGuests)
1857 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
1858# endif
1859 pVCpu->hwaccm.s.vmx.pfnStartVM = VMXR0StartVM64;
1860#endif
1861 /* Unconditionally update these as wrmsr might have changed them. */
1862 rc = VMXWriteVMCS64(VMX_VMCS64_GUEST_FS_BASE, pCtx->fsHid.u64Base);
1863 AssertRC(rc);
1864 rc = VMXWriteVMCS64(VMX_VMCS64_GUEST_GS_BASE, pCtx->gsHid.u64Base);
1865 AssertRC(rc);
1866 }
1867 else
1868 {
1869 pVCpu->hwaccm.s.vmx.pfnStartVM = VMXR0StartVM32;
1870 }
1871
1872 vmxR0UpdateExceptionBitmap(pVM, pVCpu, pCtx);
1873
1874#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
1875 /* Store all guest MSRs in the VM-Entry load area, so they will be loaded during the world switch. */
1876 PVMXMSR pMsr = (PVMXMSR)pVCpu->hwaccm.s.vmx.pGuestMSR;
1877 unsigned idxMsr = 0;
1878
1879 uint32_t ulEdx;
1880 uint32_t ulTemp;
1881 CPUMGetGuestCpuId(pVCpu, 0x80000001, &ulTemp, &ulTemp, &ulTemp, &ulEdx);
1882 /* EFER MSR present? */
1883 if (ulEdx & (X86_CPUID_AMD_FEATURE_EDX_NX|X86_CPUID_AMD_FEATURE_EDX_LONG_MODE))
1884 {
1885 pMsr->u32IndexMSR = MSR_K6_EFER;
1886 pMsr->u32Reserved = 0;
1887 pMsr->u64Value = pCtx->msrEFER;
1888 /* VT-x will complain if only MSR_K6_EFER_LME is set. */
1889 if (!CPUMIsGuestInLongModeEx(pCtx))
1890 pMsr->u64Value &= ~(MSR_K6_EFER_LMA|MSR_K6_EFER_LME);
1891 pMsr++; idxMsr++;
1892
1893 if (ulEdx & X86_CPUID_AMD_FEATURE_EDX_LONG_MODE)
1894 {
1895 pMsr->u32IndexMSR = MSR_K8_LSTAR;
1896 pMsr->u32Reserved = 0;
1897 pMsr->u64Value = pCtx->msrLSTAR; /* 64 bits mode syscall rip */
1898 pMsr++; idxMsr++;
1899 pMsr->u32IndexMSR = MSR_K6_STAR;
1900 pMsr->u32Reserved = 0;
1901 pMsr->u64Value = pCtx->msrSTAR; /* legacy syscall eip, cs & ss */
1902 pMsr++; idxMsr++;
1903 pMsr->u32IndexMSR = MSR_K8_SF_MASK;
1904 pMsr->u32Reserved = 0;
1905 pMsr->u64Value = pCtx->msrSFMASK; /* syscall flag mask */
1906 pMsr++; idxMsr++;
1907 pMsr->u32IndexMSR = MSR_K8_KERNEL_GS_BASE;
1908 pMsr->u32Reserved = 0;
1909 pMsr->u64Value = pCtx->msrKERNELGSBASE; /* swapgs exchange value */
1910 pMsr++; idxMsr++;
1911 }
1912 }
1913 pVCpu->hwaccm.s.vmx.cCachedMSRs = idxMsr;
1914
1915 rc = VMXWriteVMCS(VMX_VMCS_CTRL_ENTRY_MSR_LOAD_COUNT, idxMsr);
1916 AssertRC(rc);
1917
1918 rc = VMXWriteVMCS(VMX_VMCS_CTRL_EXIT_MSR_STORE_COUNT, idxMsr);
1919 AssertRC(rc);
1920#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
1921
1922 /* Done. */
1923 pVCpu->hwaccm.s.fContextUseFlags &= ~HWACCM_CHANGED_ALL_GUEST;
1924
1925 return rc;
1926}
1927
1928/**
1929 * Syncs back the guest state
1930 *
1931 * @returns VBox status code.
1932 * @param pVM The VM to operate on.
1933 * @param pVCpu The VMCPU to operate on.
1934 * @param pCtx Guest context
1935 */
1936DECLINLINE(int) VMXR0SaveGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
1937{
1938 RTGCUINTREG val, valShadow;
1939 RTGCUINTPTR uInterruptState;
1940 int rc;
1941
1942 /* Let's first sync back eip, esp, and eflags. */
1943 rc = VMXReadCachedVMCS(VMX_VMCS64_GUEST_RIP, &val);
1944 AssertRC(rc);
1945 pCtx->rip = val;
1946 rc = VMXReadCachedVMCS(VMX_VMCS64_GUEST_RSP, &val);
1947 AssertRC(rc);
1948 pCtx->rsp = val;
1949 rc = VMXReadCachedVMCS(VMX_VMCS_GUEST_RFLAGS, &val);
1950 AssertRC(rc);
1951 pCtx->eflags.u32 = val;
1952
1953 /* Take care of instruction fusing (sti, mov ss) */
1954 rc |= VMXReadCachedVMCS(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &val);
1955 uInterruptState = val;
1956 if (uInterruptState != 0)
1957 {
1958 Assert(uInterruptState <= 2); /* only sti & mov ss */
1959 Log(("uInterruptState %x eip=%RGv\n", (uint32_t)uInterruptState, pCtx->rip));
1960 EMSetInhibitInterruptsPC(pVCpu, pCtx->rip);
1961 }
1962 else
1963 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
1964
1965 /* Control registers. */
1966 VMXReadCachedVMCS(VMX_VMCS_CTRL_CR0_READ_SHADOW, &valShadow);
1967 VMXReadCachedVMCS(VMX_VMCS64_GUEST_CR0, &val);
1968 val = (valShadow & pVCpu->hwaccm.s.vmx.cr0_mask) | (val & ~pVCpu->hwaccm.s.vmx.cr0_mask);
1969 CPUMSetGuestCR0(pVCpu, val);
1970
1971 VMXReadCachedVMCS(VMX_VMCS_CTRL_CR4_READ_SHADOW, &valShadow);
1972 VMXReadCachedVMCS(VMX_VMCS64_GUEST_CR4, &val);
1973 val = (valShadow & pVCpu->hwaccm.s.vmx.cr4_mask) | (val & ~pVCpu->hwaccm.s.vmx.cr4_mask);
1974 CPUMSetGuestCR4(pVCpu, val);
1975
1976 /* Note: no reason to sync back the CRx registers. They can't be changed by the guest. */
1977 /* Note: only in the nested paging case can CR3 & CR4 be changed by the guest. */
1978 if ( pVM->hwaccm.s.fNestedPaging
1979 && CPUMIsGuestInPagedProtectedModeEx(pCtx))
1980 {
1981 PVMCSCACHE pCache = &pVCpu->hwaccm.s.vmx.VMCSCache;
1982
1983 /* Can be updated behind our back in the nested paging case. */
1984 CPUMSetGuestCR2(pVCpu, pCache->cr2);
1985
1986 VMXReadCachedVMCS(VMX_VMCS64_GUEST_CR3, &val);
1987
1988 if (val != pCtx->cr3)
1989 {
1990 CPUMSetGuestCR3(pVCpu, val);
1991 PGMUpdateCR3(pVCpu, val);
1992 }
1993 /* Prefetch the four PDPT entries in PAE mode. */
1994 vmxR0PrefetchPAEPdptrs(pVM, pVCpu, pCtx);
1995 }
1996
1997 /* Sync back DR7 here. */
1998 VMXReadCachedVMCS(VMX_VMCS64_GUEST_DR7, &val);
1999 pCtx->dr[7] = val;
2000
2001 /* Guest CPU context: ES, CS, SS, DS, FS, GS. */
2002 VMX_READ_SELREG(ES, es);
2003 VMX_READ_SELREG(SS, ss);
2004 VMX_READ_SELREG(CS, cs);
2005 VMX_READ_SELREG(DS, ds);
2006 VMX_READ_SELREG(FS, fs);
2007 VMX_READ_SELREG(GS, gs);
2008
2009 /*
2010 * System MSRs
2011 */
2012 VMXReadCachedVMCS(VMX_VMCS32_GUEST_SYSENTER_CS, &val);
2013 pCtx->SysEnter.cs = val;
2014 VMXReadCachedVMCS(VMX_VMCS64_GUEST_SYSENTER_EIP, &val);
2015 pCtx->SysEnter.eip = val;
2016 VMXReadCachedVMCS(VMX_VMCS64_GUEST_SYSENTER_ESP, &val);
2017 pCtx->SysEnter.esp = val;
2018
2019 /* Misc. registers; must sync everything otherwise we can get out of sync when jumping to ring 3. */
2020 VMX_READ_SELREG(LDTR, ldtr);
2021
2022 VMXReadCachedVMCS(VMX_VMCS32_GUEST_GDTR_LIMIT, &val);
2023 pCtx->gdtr.cbGdt = val;
2024 VMXReadCachedVMCS(VMX_VMCS64_GUEST_GDTR_BASE, &val);
2025 pCtx->gdtr.pGdt = val;
2026
2027 VMXReadCachedVMCS(VMX_VMCS32_GUEST_IDTR_LIMIT, &val);
2028 pCtx->idtr.cbIdt = val;
2029 VMXReadCachedVMCS(VMX_VMCS64_GUEST_IDTR_BASE, &val);
2030 pCtx->idtr.pIdt = val;
2031
2032#ifdef HWACCM_VMX_EMULATE_REALMODE
2033 /* Real mode emulation using v86 mode. */
2034 if ( CPUMIsGuestInRealModeEx(pCtx)
2035 && pVM->hwaccm.s.vmx.pRealModeTSS)
2036 {
2037 /* Hide our emulation flags */
2038 pCtx->eflags.Bits.u1VM = 0;
2039
2040 /* Restore original IOPL setting as we always use 0. */
2041 pCtx->eflags.Bits.u2IOPL = pVCpu->hwaccm.s.vmx.RealMode.eflags.Bits.u2IOPL;
2042
2043 /* Force a TR resync every time in case we switch modes. */
2044 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_GUEST_TR;
2045 }
2046 else
2047#endif /* HWACCM_VMX_EMULATE_REALMODE */
2048 {
2049 /* In real mode we have a fake TSS, so only sync it back when it's supposed to be valid. */
2050 VMX_READ_SELREG(TR, tr);
2051 }
2052
2053#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
2054 /* Save the possibly changed MSRs that we automatically restore and save during a world switch. */
2055 for (unsigned i = 0; i < pVCpu->hwaccm.s.vmx.cCachedMSRs; i++)
2056 {
2057 PVMXMSR pMsr = (PVMXMSR)pVCpu->hwaccm.s.vmx.pGuestMSR;
2058 pMsr += i;
2059
2060 switch (pMsr->u32IndexMSR)
2061 {
2062 case MSR_K8_LSTAR:
2063 pCtx->msrLSTAR = pMsr->u64Value;
2064 break;
2065 case MSR_K6_STAR:
2066 pCtx->msrSTAR = pMsr->u64Value;
2067 break;
2068 case MSR_K8_SF_MASK:
2069 pCtx->msrSFMASK = pMsr->u64Value;
2070 break;
2071 case MSR_K8_KERNEL_GS_BASE:
2072 pCtx->msrKERNELGSBASE = pMsr->u64Value;
2073 break;
2074 case MSR_K6_EFER:
2075 /* EFER can't be changed without causing a VM-exit. */
2076// Assert(pCtx->msrEFER == pMsr->u64Value);
2077 break;
2078 default:
2079 AssertFailed();
2080 return VERR_INTERNAL_ERROR;
2081 }
2082 }
2083#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
2084 return VINF_SUCCESS;
2085}
2086
2087/**
2088 * Dummy placeholder
2089 *
2090 * @param pVM The VM to operate on.
2091 * @param pVCpu The VMCPU to operate on.
2092 */
2093static void vmxR0SetupTLBDummy(PVM pVM, PVMCPU pVCpu)
2094{
2095 NOREF(pVM);
2096 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
2097 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
2098 pVCpu->hwaccm.s.TlbShootdown.cPages = 0;
2099 return;
2100}
2101
2102/**
2103 * Setup the tagged TLB for EPT
2104 *
2105 * @returns VBox status code.
2106 * @param pVM The VM to operate on.
2107 * @param pVCpu The VMCPU to operate on.
2108 */
2109static void vmxR0SetupTLBEPT(PVM pVM, PVMCPU pVCpu)
2110{
2111 PHWACCM_CPUINFO pCpu;
2112
2113 Assert(pVM->hwaccm.s.fNestedPaging);
2114 Assert(!pVM->hwaccm.s.vmx.fVPID);
2115
2116 /* Deal with tagged TLBs if VPID or EPT is supported. */
2117 pCpu = HWACCMR0GetCurrentCpu();
2118 /* Force a TLB flush for the first world switch if the current cpu differs from the one we ran on last. */
2119 /* Note that this can happen both for start and resume due to long jumps back to ring 3. */
2120 if ( pVCpu->hwaccm.s.idLastCpu != pCpu->idCpu
2121 /* if the tlb flush count has changed, another VM has flushed the TLB of this cpu, so we can't use our current ASID anymore. */
2122 || pVCpu->hwaccm.s.cTLBFlushes != pCpu->cTLBFlushes)
2123 {
2124 /* Force a TLB flush on VM entry. */
2125 pVCpu->hwaccm.s.fForceTLBFlush = true;
2126 }
2127 else
2128 Assert(!pCpu->fFlushTLB);
2129
2130 /* Check for tlb shootdown flushes. */
2131 if (VMCPU_FF_TESTANDCLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2132 pVCpu->hwaccm.s.fForceTLBFlush = true;
2133
2134 pVCpu->hwaccm.s.idLastCpu = pCpu->idCpu;
2135 pCpu->fFlushTLB = false;
2136
2137 if (pVCpu->hwaccm.s.fForceTLBFlush)
2138 {
2139 vmxR0FlushEPT(pVM, pVCpu, pVM->hwaccm.s.vmx.enmFlushContext, 0);
2140 }
2141 else
2142 if (VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
2143 {
2144 /* Deal with pending TLB shootdown actions which were queued when we were not executing code. */
2145 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatTlbShootdown);
2146
2147 for (unsigned i=0;i<pVCpu->hwaccm.s.TlbShootdown.cPages;i++)
2148 {
2149 /* aTlbShootdownPages contains physical addresses in this case. */
2150 vmxR0FlushEPT(pVM, pVCpu, pVM->hwaccm.s.vmx.enmFlushPage, pVCpu->hwaccm.s.TlbShootdown.aPages[i]);
2151 }
2152 }
2153 pVCpu->hwaccm.s.TlbShootdown.cPages= 0;
2154 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
2155
2156#ifdef VBOX_WITH_STATISTICS
2157 if (pVCpu->hwaccm.s.fForceTLBFlush)
2158 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatFlushTLBWorldSwitch);
2159 else
2160 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatNoFlushTLBWorldSwitch);
2161#endif
2162}
2163
2164#ifdef HWACCM_VTX_WITH_VPID
2165/**
2166 * Setup the tagged TLB for VPID
2167 *
2168 * @returns VBox status code.
2169 * @param pVM The VM to operate on.
2170 * @param pVCpu The VMCPU to operate on.
2171 */
2172static void vmxR0SetupTLBVPID(PVM pVM, PVMCPU pVCpu)
2173{
2174 PHWACCM_CPUINFO pCpu;
2175
2176 Assert(pVM->hwaccm.s.vmx.fVPID);
2177 Assert(!pVM->hwaccm.s.fNestedPaging);
2178
2179 /* Deal with tagged TLBs if VPID or EPT is supported. */
2180 pCpu = HWACCMR0GetCurrentCpu();
2181 /* Force a TLB flush for the first world switch if the current cpu differs from the one we ran on last. */
2182 /* Note that this can happen both for start and resume due to long jumps back to ring 3. */
2183 if ( pVCpu->hwaccm.s.idLastCpu != pCpu->idCpu
2184 /* if the tlb flush count has changed, another VM has flushed the TLB of this cpu, so we can't use our current ASID anymore. */
2185 || pVCpu->hwaccm.s.cTLBFlushes != pCpu->cTLBFlushes)
2186 {
2187 /* Force a TLB flush on VM entry. */
2188 pVCpu->hwaccm.s.fForceTLBFlush = true;
2189 }
2190 else
2191 Assert(!pCpu->fFlushTLB);
2192
2193 pVCpu->hwaccm.s.idLastCpu = pCpu->idCpu;
2194
2195 /* Check for tlb shootdown flushes. */
2196 if (VMCPU_FF_TESTANDCLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2197 pVCpu->hwaccm.s.fForceTLBFlush = true;
2198
2199 /* Make sure we flush the TLB when required. Switch ASID to achieve the same thing, but without actually flushing the whole TLB (which is expensive). */
2200 if (pVCpu->hwaccm.s.fForceTLBFlush)
2201 {
2202 if ( ++pCpu->uCurrentASID >= pVM->hwaccm.s.uMaxASID
2203 || pCpu->fFlushTLB)
2204 {
2205 pCpu->fFlushTLB = false;
2206 pCpu->uCurrentASID = 1; /* start at 1; host uses 0 */
2207 pCpu->cTLBFlushes++;
2208 vmxR0FlushVPID(pVM, pVCpu, VMX_FLUSH_ALL_CONTEXTS, 0);
2209 }
2210 else
2211 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatFlushASID);
2212
2213 pVCpu->hwaccm.s.fForceTLBFlush = false;
2214 pVCpu->hwaccm.s.cTLBFlushes = pCpu->cTLBFlushes;
2215 pVCpu->hwaccm.s.uCurrentASID = pCpu->uCurrentASID;
2216 }
2217 else
2218 {
2219 Assert(!pCpu->fFlushTLB);
2220 Assert(pVCpu->hwaccm.s.uCurrentASID && pCpu->uCurrentASID);
2221
2222 if (VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
2223 {
2224 /* Deal with pending TLB shootdown actions which were queued when we were not executing code. */
2225 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatTlbShootdown);
2226 for (unsigned i=0;i<pVCpu->hwaccm.s.TlbShootdown.cPages;i++)
2227 vmxR0FlushVPID(pVM, pVCpu, pVM->hwaccm.s.vmx.enmFlushPage, pVCpu->hwaccm.s.TlbShootdown.aPages[i]);
2228 }
2229 }
2230 pVCpu->hwaccm.s.TlbShootdown.cPages = 0;
2231 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
2232
2233 AssertMsg(pVCpu->hwaccm.s.cTLBFlushes == pCpu->cTLBFlushes, ("Flush count mismatch for cpu %d (%x vs %x)\n", pCpu->idCpu, pVCpu->hwaccm.s.cTLBFlushes, pCpu->cTLBFlushes));
2234 AssertMsg(pCpu->uCurrentASID >= 1 && pCpu->uCurrentASID < pVM->hwaccm.s.uMaxASID, ("cpu%d uCurrentASID = %x\n", pCpu->idCpu, pCpu->uCurrentASID));
2235 AssertMsg(pVCpu->hwaccm.s.uCurrentASID >= 1 && pVCpu->hwaccm.s.uCurrentASID < pVM->hwaccm.s.uMaxASID, ("cpu%d VM uCurrentASID = %x\n", pCpu->idCpu, pVCpu->hwaccm.s.uCurrentASID));
2236
2237 int rc = VMXWriteVMCS(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hwaccm.s.uCurrentASID);
2238 AssertRC(rc);
2239
2240 if (pVCpu->hwaccm.s.fForceTLBFlush)
2241 vmxR0FlushVPID(pVM, pVCpu, pVM->hwaccm.s.vmx.enmFlushContext, 0);
2242
2243#ifdef VBOX_WITH_STATISTICS
2244 if (pVCpu->hwaccm.s.fForceTLBFlush)
2245 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatFlushTLBWorldSwitch);
2246 else
2247 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatNoFlushTLBWorldSwitch);
2248#endif
2249}
2250#endif /* HWACCM_VTX_WITH_VPID */
2251
2252/**
2253 * Runs guest code in a VT-x VM.
2254 *
2255 * @returns VBox status code.
2256 * @param pVM The VM to operate on.
2257 * @param pVCpu The VMCPU to operate on.
2258 * @param pCtx Guest context
2259 */
2260VMMR0DECL(int) VMXR0RunGuestCode(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
2261{
2262 int rc = VINF_SUCCESS;
2263 RTGCUINTREG val;
2264 RTGCUINTREG exitReason = (RTGCUINTREG)VMX_EXIT_INVALID;
2265 RTGCUINTREG instrError, cbInstr;
2266 RTGCUINTPTR exitQualification = 0;
2267 RTGCUINTPTR intInfo = 0; /* shut up buggy gcc 4 */
2268 RTGCUINTPTR errCode, instrInfo;
2269 bool fSetupTPRCaching = false;
2270 uint64_t u64OldLSTAR = 0;
2271 uint8_t u8LastTPR = 0;
2272 RTCCUINTREG uOldEFlags = ~(RTCCUINTREG)0;
2273 unsigned cResume = 0;
2274#ifdef VBOX_STRICT
2275 RTCPUID idCpuCheck;
2276 bool fWasInLongMode = false;
2277#endif
2278#ifdef VBOX_HIGH_RES_TIMERS_HACK_IN_RING0
2279 uint64_t u64LastTime = RTTimeMilliTS();
2280#endif
2281#ifdef VBOX_WITH_STATISTICS
2282 bool fStatEntryStarted = true;
2283 bool fStatExit2Started = false;
2284#endif
2285
2286 Assert(!(pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC) || (pVCpu->hwaccm.s.vmx.pVAPIC && pVM->hwaccm.s.vmx.pAPIC));
2287
2288 /* Check if we need to use TPR shadowing. */
2289 if ( CPUMIsGuestInLongModeEx(pCtx)
2290 || ( ((pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC) || pVM->hwaccm.s.fTRPPatchingAllowed)
2291 && pVM->hwaccm.s.fHasIoApic)
2292 )
2293 {
2294 fSetupTPRCaching = true;
2295 }
2296
2297 Log2(("\nE"));
2298
2299 STAM_PROFILE_ADV_START(&pVCpu->hwaccm.s.StatEntry, x);
2300
2301#ifdef VBOX_STRICT
2302 {
2303 RTCCUINTREG val2;
2304
2305 rc = VMXReadVMCS(VMX_VMCS_CTRL_PIN_EXEC_CONTROLS, &val2);
2306 AssertRC(rc);
2307 Log2(("VMX_VMCS_CTRL_PIN_EXEC_CONTROLS = %08x\n", val2));
2308
2309 /* allowed zero */
2310 if ((val2 & pVM->hwaccm.s.vmx.msr.vmx_pin_ctls.n.disallowed0) != pVM->hwaccm.s.vmx.msr.vmx_pin_ctls.n.disallowed0)
2311 Log(("Invalid VMX_VMCS_CTRL_PIN_EXEC_CONTROLS: zero\n"));
2312
2313 /* allowed one */
2314 if ((val2 & ~pVM->hwaccm.s.vmx.msr.vmx_pin_ctls.n.allowed1) != 0)
2315 Log(("Invalid VMX_VMCS_CTRL_PIN_EXEC_CONTROLS: one\n"));
2316
2317 rc = VMXReadVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, &val2);
2318 AssertRC(rc);
2319 Log2(("VMX_VMCS_CTRL_PROC_EXEC_CONTROLS = %08x\n", val2));
2320
2321 /* Must be set according to the MSR, but can be cleared in case of EPT. */
2322 if (pVM->hwaccm.s.fNestedPaging)
2323 val2 |= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INVLPG_EXIT
2324 | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
2325 | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT;
2326
2327 /* allowed zero */
2328 if ((val2 & pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.disallowed0) != pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.disallowed0)
2329 Log(("Invalid VMX_VMCS_CTRL_PROC_EXEC_CONTROLS: zero\n"));
2330
2331 /* allowed one */
2332 if ((val2 & ~pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1) != 0)
2333 Log(("Invalid VMX_VMCS_CTRL_PROC_EXEC_CONTROLS: one\n"));
2334
2335 rc = VMXReadVMCS(VMX_VMCS_CTRL_ENTRY_CONTROLS, &val2);
2336 AssertRC(rc);
2337 Log2(("VMX_VMCS_CTRL_ENTRY_CONTROLS = %08x\n", val2));
2338
2339 /* allowed zero */
2340 if ((val2 & pVM->hwaccm.s.vmx.msr.vmx_entry.n.disallowed0) != pVM->hwaccm.s.vmx.msr.vmx_entry.n.disallowed0)
2341 Log(("Invalid VMX_VMCS_CTRL_ENTRY_CONTROLS: zero\n"));
2342
2343 /* allowed one */
2344 if ((val2 & ~pVM->hwaccm.s.vmx.msr.vmx_entry.n.allowed1) != 0)
2345 Log(("Invalid VMX_VMCS_CTRL_ENTRY_CONTROLS: one\n"));
2346
2347 rc = VMXReadVMCS(VMX_VMCS_CTRL_EXIT_CONTROLS, &val2);
2348 AssertRC(rc);
2349 Log2(("VMX_VMCS_CTRL_EXIT_CONTROLS = %08x\n", val2));
2350
2351 /* allowed zero */
2352 if ((val2 & pVM->hwaccm.s.vmx.msr.vmx_exit.n.disallowed0) != pVM->hwaccm.s.vmx.msr.vmx_exit.n.disallowed0)
2353 Log(("Invalid VMX_VMCS_CTRL_EXIT_CONTROLS: zero\n"));
2354
2355 /* allowed one */
2356 if ((val2 & ~pVM->hwaccm.s.vmx.msr.vmx_exit.n.allowed1) != 0)
2357 Log(("Invalid VMX_VMCS_CTRL_EXIT_CONTROLS: one\n"));
2358 }
2359 fWasInLongMode = CPUMIsGuestInLongModeEx(pCtx);
2360#endif /* VBOX_STRICT */
2361
2362#ifdef VBOX_WITH_CRASHDUMP_MAGIC
2363 pVCpu->hwaccm.s.vmx.VMCSCache.u64TimeEntry = RTTimeNanoTS();
2364#endif
2365
2366 /* We can jump to this point to resume execution after determining that a VM-exit is innocent.
2367 */
2368ResumeExecution:
2369 STAM_STATS({
2370 if (fStatExit2Started) { STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2, y); fStatExit2Started = false; }
2371 if (!fStatEntryStarted) { STAM_PROFILE_ADV_START(&pVCpu->hwaccm.s.StatEntry, x); fStatEntryStarted = true; }
2372 });
2373 AssertMsg(pVCpu->hwaccm.s.idEnteredCpu == RTMpCpuId(),
2374 ("Expected %d, I'm %d; cResume=%d exitReason=%RGv exitQualification=%RGv\n",
2375 (int)pVCpu->hwaccm.s.idEnteredCpu, (int)RTMpCpuId(), cResume, exitReason, exitQualification));
2376 Assert(!HWACCMR0SuspendPending());
2377 /* Not allowed to switch modes without reloading the host state (32->64 switcher)!! */
2378 Assert(fWasInLongMode == CPUMIsGuestInLongModeEx(pCtx));
2379
2380 /* Safety precaution; looping for too long here can have a very bad effect on the host */
2381 if (RT_UNLIKELY(++cResume > pVM->hwaccm.s.cMaxResumeLoops))
2382 {
2383 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitMaxResume);
2384 rc = VINF_EM_RAW_INTERRUPT;
2385 goto end;
2386 }
2387
2388 /* Check for irq inhibition due to instruction fusing (sti, mov ss). */
2389 if (VMCPU_FF_ISSET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
2390 {
2391 Log(("VM_FF_INHIBIT_INTERRUPTS at %RGv successor %RGv\n", (RTGCPTR)pCtx->rip, EMGetInhibitInterruptsPC(pVCpu)));
2392 if (pCtx->rip != EMGetInhibitInterruptsPC(pVCpu))
2393 {
2394 /* Note: we intentionally don't clear VM_FF_INHIBIT_INTERRUPTS here.
2395 * Before we are able to execute this instruction in raw mode (iret to guest code) an external interrupt might
2396 * force a world switch again. Possibly allowing a guest interrupt to be dispatched in the process. This could
2397 * break the guest. Sounds very unlikely, but such timing sensitive problems are not as rare as you might think.
2398 */
2399 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
2400 /* Irq inhibition is no longer active; clear the corresponding VMX state. */
2401 rc = VMXWriteVMCS(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, 0);
2402 AssertRC(rc);
2403 }
2404 }
2405 else
2406 {
2407 /* Irq inhibition is no longer active; clear the corresponding VMX state. */
2408 rc = VMXWriteVMCS(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, 0);
2409 AssertRC(rc);
2410 }
2411
2412#ifdef VBOX_HIGH_RES_TIMERS_HACK_IN_RING0
2413 if (RT_UNLIKELY(cResume & 0xf) == 0)
2414 {
2415 uint64_t u64CurTime = RTTimeMilliTS();
2416
2417 if (RT_UNLIKELY(u64CurTime > u64LastTime))
2418 {
2419 u64LastTime = u64CurTime;
2420 TMTimerPollVoid(pVM, pVCpu);
2421 }
2422 }
2423#endif
2424
2425 /* Check for pending actions that force us to go back to ring 3. */
2426 if ( VM_FF_ISPENDING(pVM, VM_FF_HWACCM_TO_R3_MASK | VM_FF_REQUEST | VM_FF_PGM_POOL_FLUSH_PENDING)
2427 || VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_HWACCM_TO_R3_MASK | VMCPU_FF_PGM_SYNC_CR3 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL | VMCPU_FF_REQUEST))
2428 {
2429 /* Check if a sync operation is pending. */
2430 if (VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
2431 {
2432 rc = PGMSyncCR3(pVCpu, pCtx->cr0, pCtx->cr3, pCtx->cr4, VMCPU_FF_ISSET(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
2433 AssertRC(rc);
2434 if (rc != VINF_SUCCESS)
2435 {
2436 Log(("Pending pool sync is forcing us back to ring 3; rc=%d\n", rc));
2437 goto end;
2438 }
2439 }
2440
2441#ifdef DEBUG
2442 /* Intercept X86_XCPT_DB if stepping is enabled */
2443 if (!DBGFIsStepping(pVCpu))
2444#endif
2445 {
2446 if ( VM_FF_ISPENDING(pVM, VM_FF_HWACCM_TO_R3_MASK)
2447 || VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_HWACCM_TO_R3_MASK))
2448 {
2449 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
2450 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatSwitchToR3);
2451 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatEntry, x);
2452 rc = RT_UNLIKELY(VM_FF_ISPENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
2453 goto end;
2454 }
2455 }
2456
2457 /* Pending request packets might contain actions that need immediate attention, such as pending hardware interrupts. */
2458 if ( VM_FF_ISPENDING(pVM, VM_FF_REQUEST)
2459 || VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_REQUEST))
2460 {
2461 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatEntry, x);
2462 rc = VINF_EM_PENDING_REQUEST;
2463 goto end;
2464 }
2465
2466 /* Check if a pgm pool flush is in progress. */
2467 if (VM_FF_ISPENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
2468 {
2469 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatEntry, x);
2470 rc = VINF_PGM_POOL_FLUSH_PENDING;
2471 goto end;
2472 }
2473 }
2474
2475#ifdef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
2476 /*
2477 * Exit to ring-3 preemption/work is pending.
2478 *
2479 * Interrupts are disabled before the call to make sure we don't miss any interrupt
2480 * that would flag preemption (IPI, timer tick, ++). (Would've been nice to do this
2481 * further down, but VMXR0CheckPendingInterrupt makes that impossible.)
2482 *
2483 * Note! Interrupts must be disabled done *before* we check for TLB flushes; TLB
2484 * shootdowns rely on this.
2485 */
2486 uOldEFlags = ASMIntDisableFlags();
2487 if (RTThreadPreemptIsPending(NIL_RTTHREAD))
2488 {
2489 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitPreemptPending);
2490 rc = VINF_EM_RAW_INTERRUPT;
2491 goto end;
2492 }
2493 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
2494#endif
2495
2496 /* When external interrupts are pending, we should exit the VM when IF is set. */
2497 /* Note! *After* VM_FF_INHIBIT_INTERRUPTS check!!! */
2498 rc = VMXR0CheckPendingInterrupt(pVM, pVCpu, pCtx);
2499 if (RT_FAILURE(rc))
2500 goto end;
2501
2502 /** @todo check timers?? */
2503
2504 /* TPR caching using CR8 is only available in 64 bits mode */
2505 /* Note the 32 bits exception for AMD (X86_CPUID_AMD_FEATURE_ECX_CR8L), but that appears missing in Intel CPUs */
2506 /* Note: we can't do this in LoadGuestState as PDMApicGetTPR can jump back to ring 3 (lock)!!!!! (no longer true) */
2507 /**
2508 * @todo query and update the TPR only when it could have been changed (mmio access & wrmsr (x2apic))
2509 */
2510 if (fSetupTPRCaching)
2511 {
2512 /* TPR caching in CR8 */
2513 bool fPending;
2514
2515 int rc2 = PDMApicGetTPR(pVCpu, &u8LastTPR, &fPending);
2516 AssertRC(rc2);
2517 /* The TPR can be found at offset 0x80 in the APIC mmio page. */
2518 pVCpu->hwaccm.s.vmx.pVAPIC[0x80] = u8LastTPR;
2519
2520 /* Two options here:
2521 * - external interrupt pending, but masked by the TPR value.
2522 * -> a CR8 update that lower the current TPR value should cause an exit
2523 * - no pending interrupts
2524 * -> We don't need to be explicitely notified. There are enough world switches for detecting pending interrupts.
2525 */
2526 rc = VMXWriteVMCS(VMX_VMCS_CTRL_TPR_THRESHOLD, (fPending) ? (u8LastTPR >> 4) : 0); /* cr8 bits 3-0 correspond to bits 7-4 of the task priority mmio register. */
2527 AssertRC(rc);
2528
2529 if (pVM->hwaccm.s.fTPRPatchingActive)
2530 {
2531 Assert(!CPUMIsGuestInLongModeEx(pCtx));
2532 /* Our patch code uses LSTAR for TPR caching. */
2533 pCtx->msrLSTAR = u8LastTPR;
2534
2535 if (fPending)
2536 {
2537 /* A TPR change could activate a pending interrupt, so catch lstar writes. */
2538 vmxR0SetMSRPermission(pVCpu, MSR_K8_LSTAR, true, false);
2539 }
2540 else
2541 {
2542 /* No interrupts are pending, so we don't need to be explicitely notified.
2543 * There are enough world switches for detecting pending interrupts.
2544 */
2545 vmxR0SetMSRPermission(pVCpu, MSR_K8_LSTAR, true, true);
2546 }
2547 }
2548 }
2549
2550#if defined(HWACCM_VTX_WITH_EPT) && defined(LOG_ENABLED)
2551 if ( pVM->hwaccm.s.fNestedPaging
2552# ifdef HWACCM_VTX_WITH_VPID
2553 || pVM->hwaccm.s.vmx.fVPID
2554# endif /* HWACCM_VTX_WITH_VPID */
2555 )
2556 {
2557 PHWACCM_CPUINFO pCpu;
2558
2559 pCpu = HWACCMR0GetCurrentCpu();
2560 if ( pVCpu->hwaccm.s.idLastCpu != pCpu->idCpu
2561 || pVCpu->hwaccm.s.cTLBFlushes != pCpu->cTLBFlushes)
2562 {
2563 if (pVCpu->hwaccm.s.idLastCpu != pCpu->idCpu)
2564 LogFlow(("Force TLB flush due to rescheduling to a different cpu (%d vs %d)\n", pVCpu->hwaccm.s.idLastCpu, pCpu->idCpu));
2565 else
2566 LogFlow(("Force TLB flush due to changed TLB flush count (%x vs %x)\n", pVCpu->hwaccm.s.cTLBFlushes, pCpu->cTLBFlushes));
2567 }
2568 if (pCpu->fFlushTLB)
2569 LogFlow(("Force TLB flush: first time cpu %d is used -> flush\n", pCpu->idCpu));
2570 else
2571 if (pVCpu->hwaccm.s.fForceTLBFlush)
2572 LogFlow(("Manual TLB flush\n"));
2573 }
2574#endif
2575#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
2576 PGMDynMapFlushAutoSet(pVCpu);
2577#endif
2578
2579 /*
2580 * NOTE: DO NOT DO ANYTHING AFTER THIS POINT THAT MIGHT JUMP BACK TO RING 3!
2581 * (until the actual world switch)
2582 */
2583#ifdef VBOX_STRICT
2584 idCpuCheck = RTMpCpuId();
2585#endif
2586#ifdef LOG_ENABLED
2587 VMMR0LogFlushDisable(pVCpu);
2588#endif
2589 /* Save the host state first. */
2590 rc = VMXR0SaveHostState(pVM, pVCpu);
2591 if (RT_UNLIKELY(rc != VINF_SUCCESS))
2592 {
2593 VMMR0LogFlushEnable(pVCpu);
2594 goto end;
2595 }
2596 /* Load the guest state */
2597 rc = VMXR0LoadGuestState(pVM, pVCpu, pCtx);
2598 if (RT_UNLIKELY(rc != VINF_SUCCESS))
2599 {
2600 VMMR0LogFlushEnable(pVCpu);
2601 goto end;
2602 }
2603
2604#ifndef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
2605 /* Disable interrupts to make sure a poke will interrupt execution.
2606 * This must be done *before* we check for TLB flushes; TLB shootdowns rely on this.
2607 */
2608 uOldEFlags = ASMIntDisableFlags();
2609 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
2610#endif
2611
2612 /* Non-register state Guest Context */
2613 /** @todo change me according to cpu state */
2614 rc = VMXWriteVMCS(VMX_VMCS32_GUEST_ACTIVITY_STATE, VMX_CMS_GUEST_ACTIVITY_ACTIVE);
2615 AssertRC(rc);
2616
2617 /** Set TLB flush state as checked until we return from the world switch. */
2618 ASMAtomicWriteU8(&pVCpu->hwaccm.s.fCheckedTLBFlush, true);
2619 /* Deal with tagged TLB setup and invalidation. */
2620 pVM->hwaccm.s.vmx.pfnSetupTaggedTLB(pVM, pVCpu);
2621
2622 STAM_STATS({ STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatEntry, x); fStatEntryStarted = false; });
2623
2624 /* Manual save and restore:
2625 * - General purpose registers except RIP, RSP
2626 *
2627 * Trashed:
2628 * - CR2 (we don't care)
2629 * - LDTR (reset to 0)
2630 * - DRx (presumably not changed at all)
2631 * - DR7 (reset to 0x400)
2632 * - EFLAGS (reset to RT_BIT(1); not relevant)
2633 *
2634 */
2635
2636 /* All done! Let's start VM execution. */
2637 STAM_PROFILE_ADV_START(&pVCpu->hwaccm.s.StatInGC, z);
2638 Assert(idCpuCheck == RTMpCpuId());
2639
2640#ifdef VBOX_WITH_CRASHDUMP_MAGIC
2641 pVCpu->hwaccm.s.vmx.VMCSCache.cResume = cResume;
2642 pVCpu->hwaccm.s.vmx.VMCSCache.u64TimeSwitch = RTTimeNanoTS();
2643#endif
2644
2645 /* Save the current TPR value in the LSTAR msr so our patches can access it. */
2646 if (pVM->hwaccm.s.fTPRPatchingActive)
2647 {
2648 Assert(pVM->hwaccm.s.fTPRPatchingActive);
2649 u64OldLSTAR = ASMRdMsr(MSR_K8_LSTAR);
2650 ASMWrMsr(MSR_K8_LSTAR, u8LastTPR);
2651 }
2652
2653 TMNotifyStartOfExecution(pVCpu);
2654#ifdef VBOX_WITH_KERNEL_USING_XMM
2655 rc = hwaccmR0VMXStartVMWrapXMM(pVCpu->hwaccm.s.fResumeVM, pCtx, &pVCpu->hwaccm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hwaccm.s.vmx.pfnStartVM);
2656#else
2657 rc = pVCpu->hwaccm.s.vmx.pfnStartVM(pVCpu->hwaccm.s.fResumeVM, pCtx, &pVCpu->hwaccm.s.vmx.VMCSCache, pVM, pVCpu);
2658#endif
2659 ASMAtomicWriteU8(&pVCpu->hwaccm.s.fCheckedTLBFlush, false);
2660 ASMAtomicIncU32(&pVCpu->hwaccm.s.cWorldSwitchExit);
2661 /* Possibly the last TSC value seen by the guest (too high) (only when we're in tsc offset mode). */
2662 if (!(pVCpu->hwaccm.s.vmx.proc_ctls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT))
2663 TMCpuTickSetLastSeen(pVCpu, ASMReadTSC() + pVCpu->hwaccm.s.vmx.u64TSCOffset - 0x400 /* guestimate of world switch overhead in clock ticks */);
2664
2665 TMNotifyEndOfExecution(pVCpu);
2666 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED);
2667 Assert(!(ASMGetFlags() & X86_EFL_IF));
2668
2669 /* Restore the host LSTAR msr if the guest could have changed it. */
2670 if (pVM->hwaccm.s.fTPRPatchingActive)
2671 {
2672 Assert(pVM->hwaccm.s.fTPRPatchingActive);
2673 pVCpu->hwaccm.s.vmx.pVAPIC[0x80] = pCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR);
2674 ASMWrMsr(MSR_K8_LSTAR, u64OldLSTAR);
2675 }
2676
2677 ASMSetFlags(uOldEFlags);
2678#ifdef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
2679 uOldEFlags = ~(RTCCUINTREG)0;
2680#endif
2681
2682 AssertMsg(!pVCpu->hwaccm.s.vmx.VMCSCache.Write.cValidEntries, ("pVCpu->hwaccm.s.vmx.VMCSCache.Write.cValidEntries=%d\n", pVCpu->hwaccm.s.vmx.VMCSCache.Write.cValidEntries));
2683
2684 /* In case we execute a goto ResumeExecution later on. */
2685 pVCpu->hwaccm.s.fResumeVM = true;
2686 pVCpu->hwaccm.s.fForceTLBFlush = false;
2687
2688 /*
2689 * !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
2690 * IMPORTANT: WE CAN'T DO ANY LOGGING OR OPERATIONS THAT CAN DO A LONGJMP BACK TO RING 3 *BEFORE* WE'VE SYNCED BACK (MOST OF) THE GUEST STATE
2691 * !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
2692 */
2693 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatInGC, z);
2694 STAM_PROFILE_ADV_START(&pVCpu->hwaccm.s.StatExit1, v);
2695
2696 if (RT_UNLIKELY(rc != VINF_SUCCESS))
2697 {
2698 VMXR0ReportWorldSwitchError(pVM, pVCpu, rc, pCtx);
2699 VMMR0LogFlushEnable(pVCpu);
2700 goto end;
2701 }
2702
2703 /* Success. Query the guest state and figure out what has happened. */
2704
2705 /* Investigate why there was a VM-exit. */
2706 rc = VMXReadCachedVMCS(VMX_VMCS32_RO_EXIT_REASON, &exitReason);
2707 STAM_COUNTER_INC(&pVCpu->hwaccm.s.paStatExitReasonR0[exitReason & MASK_EXITREASON_STAT]);
2708
2709 exitReason &= 0xffff; /* bit 0-15 contain the exit code. */
2710 rc |= VMXReadCachedVMCS(VMX_VMCS32_RO_VM_INSTR_ERROR, &instrError);
2711 rc |= VMXReadCachedVMCS(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &cbInstr);
2712 rc |= VMXReadCachedVMCS(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &intInfo);
2713 /* might not be valid; depends on VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID. */
2714 rc |= VMXReadCachedVMCS(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERRCODE, &errCode);
2715 rc |= VMXReadCachedVMCS(VMX_VMCS32_RO_EXIT_INSTR_INFO, &instrInfo);
2716 rc |= VMXReadCachedVMCS(VMX_VMCS_RO_EXIT_QUALIFICATION, &exitQualification);
2717 AssertRC(rc);
2718
2719 /* Sync back the guest state */
2720 rc = VMXR0SaveGuestState(pVM, pVCpu, pCtx);
2721 AssertRC(rc);
2722
2723 /* Note! NOW IT'S SAFE FOR LOGGING! */
2724 VMMR0LogFlushEnable(pVCpu);
2725 Log2(("Raw exit reason %08x\n", exitReason));
2726
2727 /* Check if an injected event was interrupted prematurely. */
2728 rc = VMXReadCachedVMCS(VMX_VMCS32_RO_IDT_INFO, &val);
2729 AssertRC(rc);
2730 pVCpu->hwaccm.s.Event.intInfo = VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(val);
2731 if ( VMX_EXIT_INTERRUPTION_INFO_VALID(pVCpu->hwaccm.s.Event.intInfo)
2732 /* Ignore 'int xx' as they'll be restarted anyway. */
2733 && VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hwaccm.s.Event.intInfo) != VMX_EXIT_INTERRUPTION_INFO_TYPE_SW
2734 /* Ignore software exceptions (such as int3) as they'll reoccur when we restart the instruction anyway. */
2735 && VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hwaccm.s.Event.intInfo) != VMX_EXIT_INTERRUPTION_INFO_TYPE_SWEXCPT)
2736 {
2737 Assert(!pVCpu->hwaccm.s.Event.fPending);
2738 pVCpu->hwaccm.s.Event.fPending = true;
2739 /* Error code present? */
2740 if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(pVCpu->hwaccm.s.Event.intInfo))
2741 {
2742 rc = VMXReadCachedVMCS(VMX_VMCS32_RO_IDT_ERRCODE, &val);
2743 AssertRC(rc);
2744 pVCpu->hwaccm.s.Event.errCode = val;
2745 Log(("Pending inject %RX64 at %RGv exit=%08x intInfo=%08x exitQualification=%RGv pending error=%RX64\n", pVCpu->hwaccm.s.Event.intInfo, (RTGCPTR)pCtx->rip, exitReason, intInfo, exitQualification, val));
2746 }
2747 else
2748 {
2749 Log(("Pending inject %RX64 at %RGv exit=%08x intInfo=%08x exitQualification=%RGv\n", pVCpu->hwaccm.s.Event.intInfo, (RTGCPTR)pCtx->rip, exitReason, intInfo, exitQualification));
2750 pVCpu->hwaccm.s.Event.errCode = 0;
2751 }
2752 }
2753#ifdef VBOX_STRICT
2754 else
2755 if ( VMX_EXIT_INTERRUPTION_INFO_VALID(pVCpu->hwaccm.s.Event.intInfo)
2756 /* Ignore software exceptions (such as int3) as they're reoccur when we restart the instruction anyway. */
2757 && VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hwaccm.s.Event.intInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_SWEXCPT)
2758 {
2759 Log(("Ignore pending inject %RX64 at %RGv exit=%08x intInfo=%08x exitQualification=%RGv\n", pVCpu->hwaccm.s.Event.intInfo, (RTGCPTR)pCtx->rip, exitReason, intInfo, exitQualification));
2760 }
2761
2762 if (exitReason == VMX_EXIT_ERR_INVALID_GUEST_STATE)
2763 HWACCMDumpRegs(pVM, pVCpu, pCtx);
2764#endif
2765
2766 Log2(("E%d: New EIP=%x:%RGv\n", (uint32_t)exitReason, pCtx->cs, (RTGCPTR)pCtx->rip));
2767 Log2(("Exit reason %d, exitQualification %RGv\n", (uint32_t)exitReason, exitQualification));
2768 Log2(("instrInfo=%d instrError=%d instr length=%d\n", (uint32_t)instrInfo, (uint32_t)instrError, (uint32_t)cbInstr));
2769 Log2(("Interruption error code %d\n", (uint32_t)errCode));
2770 Log2(("IntInfo = %08x\n", (uint32_t)intInfo));
2771
2772 /* Sync back the TPR if it was changed. */
2773 if ( fSetupTPRCaching
2774 && u8LastTPR != pVCpu->hwaccm.s.vmx.pVAPIC[0x80])
2775 {
2776 rc = PDMApicSetTPR(pVCpu, pVCpu->hwaccm.s.vmx.pVAPIC[0x80]);
2777 AssertRC(rc);
2778 }
2779
2780 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit1, v);
2781 STAM_STATS({ STAM_PROFILE_ADV_START(&pVCpu->hwaccm.s.StatExit2, y); fStatExit2Started = true; });
2782
2783 /* Some cases don't need a complete resync of the guest CPU state; handle them here. */
2784 switch (exitReason)
2785 {
2786 case VMX_EXIT_EXCEPTION: /* 0 Exception or non-maskable interrupt (NMI). */
2787 case VMX_EXIT_EXTERNAL_IRQ: /* 1 External interrupt. */
2788 {
2789 uint32_t vector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(intInfo);
2790
2791 if (!VMX_EXIT_INTERRUPTION_INFO_VALID(intInfo))
2792 {
2793 Assert(exitReason == VMX_EXIT_EXTERNAL_IRQ);
2794 /* External interrupt; leave to allow it to be dispatched again. */
2795 rc = VINF_EM_RAW_INTERRUPT;
2796 break;
2797 }
2798 STAM_PROFILE_ADV_START(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
2799 switch (VMX_EXIT_INTERRUPTION_INFO_TYPE(intInfo))
2800 {
2801 case VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI: /* Non-maskable interrupt. */
2802 /* External interrupt; leave to allow it to be dispatched again. */
2803 rc = VINF_EM_RAW_INTERRUPT;
2804 break;
2805
2806 case VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT: /* External hardware interrupt. */
2807 AssertFailed(); /* can't come here; fails the first check. */
2808 break;
2809
2810 case VMX_EXIT_INTERRUPTION_INFO_TYPE_DBEXCPT: /* Unknown why we get this type for #DB */
2811 case VMX_EXIT_INTERRUPTION_INFO_TYPE_SWEXCPT: /* Software exception. (#BP or #OF) */
2812 Assert(vector == 1 || vector == 3 || vector == 4);
2813 /* no break */
2814 case VMX_EXIT_INTERRUPTION_INFO_TYPE_HWEXCPT: /* Hardware exception. */
2815 Log2(("Hardware/software interrupt %d\n", vector));
2816 switch (vector)
2817 {
2818 case X86_XCPT_NM:
2819 {
2820 Log(("#NM fault at %RGv error code %x\n", (RTGCPTR)pCtx->rip, errCode));
2821
2822 /** @todo don't intercept #NM exceptions anymore when we've activated the guest FPU state. */
2823 /* If we sync the FPU/XMM state on-demand, then we can continue execution as if nothing has happened. */
2824 rc = CPUMR0LoadGuestFPU(pVM, pVCpu, pCtx);
2825 if (rc == VINF_SUCCESS)
2826 {
2827 Assert(CPUMIsGuestFPUStateActive(pVCpu));
2828
2829 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitShadowNM);
2830
2831 /* Continue execution. */
2832 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_GUEST_CR0;
2833
2834 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
2835 goto ResumeExecution;
2836 }
2837
2838 Log(("Forward #NM fault to the guest\n"));
2839 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestNM);
2840 rc = VMXR0InjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), cbInstr, 0);
2841 AssertRC(rc);
2842 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
2843 goto ResumeExecution;
2844 }
2845
2846 case X86_XCPT_PF: /* Page fault */
2847 {
2848#ifdef DEBUG
2849 if (pVM->hwaccm.s.fNestedPaging)
2850 { /* A genuine pagefault.
2851 * Forward the trap to the guest by injecting the exception and resuming execution.
2852 */
2853 Log(("Guest page fault at %RGv cr2=%RGv error code %x rsp=%RGv\n", (RTGCPTR)pCtx->rip, exitQualification, errCode, (RTGCPTR)pCtx->rsp));
2854
2855 Assert(CPUMIsGuestInPagedProtectedModeEx(pCtx));
2856
2857 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestPF);
2858
2859 /* Now we must update CR2. */
2860 pCtx->cr2 = exitQualification;
2861 rc = VMXR0InjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), cbInstr, errCode);
2862 AssertRC(rc);
2863
2864 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
2865 goto ResumeExecution;
2866 }
2867#endif
2868 Assert(!pVM->hwaccm.s.fNestedPaging);
2869
2870#ifdef VBOX_HWACCM_WITH_GUEST_PATCHING
2871 /* Shortcut for APIC TPR reads and writes; 32 bits guests only */
2872 if ( pVM->hwaccm.s.fTRPPatchingAllowed
2873 && pVM->hwaccm.s.pGuestPatchMem
2874 && (exitQualification & 0xfff) == 0x080
2875 && !(errCode & X86_TRAP_PF_P) /* not present */
2876 && CPUMGetGuestCPL(pVCpu, CPUMCTX2CORE(pCtx)) == 0
2877 && !CPUMIsGuestInLongModeEx(pCtx)
2878 && pVM->hwaccm.s.cPatches < RT_ELEMENTS(pVM->hwaccm.s.aPatches))
2879 {
2880 RTGCPHYS GCPhysApicBase, GCPhys;
2881 PDMApicGetBase(pVM, &GCPhysApicBase); /* @todo cache this */
2882 GCPhysApicBase &= PAGE_BASE_GC_MASK;
2883
2884 rc = PGMGstGetPage(pVCpu, (RTGCPTR)exitQualification, NULL, &GCPhys);
2885 if ( rc == VINF_SUCCESS
2886 && GCPhys == GCPhysApicBase)
2887 {
2888 /* Only attempt to patch the instruction once. */
2889 PHWACCMTPRPATCH pPatch = (PHWACCMTPRPATCH)RTAvloU32Get(&pVM->hwaccm.s.PatchTree, (AVLOU32KEY)pCtx->eip);
2890 if (!pPatch)
2891 {
2892 rc = VINF_EM_HWACCM_PATCH_TPR_INSTR;
2893 break;
2894 }
2895 }
2896 }
2897#endif
2898
2899 Log2(("Page fault at %RGv error code %x\n", exitQualification, errCode));
2900 /* Exit qualification contains the linear address of the page fault. */
2901 TRPMAssertTrap(pVCpu, X86_XCPT_PF, TRPM_TRAP);
2902 TRPMSetErrorCode(pVCpu, errCode);
2903 TRPMSetFaultAddress(pVCpu, exitQualification);
2904
2905 /* Shortcut for APIC TPR reads and writes. */
2906 if ( (exitQualification & 0xfff) == 0x080
2907 && !(errCode & X86_TRAP_PF_P) /* not present */
2908 && fSetupTPRCaching
2909 && (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC))
2910 {
2911 RTGCPHYS GCPhysApicBase, GCPhys;
2912 PDMApicGetBase(pVM, &GCPhysApicBase); /* @todo cache this */
2913 GCPhysApicBase &= PAGE_BASE_GC_MASK;
2914
2915 rc = PGMGstGetPage(pVCpu, (RTGCPTR)exitQualification, NULL, &GCPhys);
2916 if ( rc == VINF_SUCCESS
2917 && GCPhys == GCPhysApicBase)
2918 {
2919 Log(("Enable VT-x virtual APIC access filtering\n"));
2920 rc = IOMMMIOMapMMIOHCPage(pVM, GCPhysApicBase, pVM->hwaccm.s.vmx.pAPICPhys, X86_PTE_RW | X86_PTE_P);
2921 AssertRC(rc);
2922 }
2923 }
2924
2925 /* Forward it to our trap handler first, in case our shadow pages are out of sync. */
2926 rc = PGMTrap0eHandler(pVCpu, errCode, CPUMCTX2CORE(pCtx), (RTGCPTR)exitQualification);
2927 Log2(("PGMTrap0eHandler %RGv returned %Rrc\n", (RTGCPTR)pCtx->rip, rc));
2928
2929 if (rc == VINF_SUCCESS)
2930 { /* We've successfully synced our shadow pages, so let's just continue execution. */
2931 Log2(("Shadow page fault at %RGv cr2=%RGv error code %x\n", (RTGCPTR)pCtx->rip, exitQualification ,errCode));
2932 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitShadowPF);
2933
2934 TRPMResetTrap(pVCpu);
2935 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
2936 goto ResumeExecution;
2937 }
2938 else
2939 if (rc == VINF_EM_RAW_GUEST_TRAP)
2940 { /* A genuine pagefault.
2941 * Forward the trap to the guest by injecting the exception and resuming execution.
2942 */
2943 Log2(("Forward page fault to the guest\n"));
2944
2945 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestPF);
2946 /* The error code might have been changed. */
2947 errCode = TRPMGetErrorCode(pVCpu);
2948
2949 TRPMResetTrap(pVCpu);
2950
2951 /* Now we must update CR2. */
2952 pCtx->cr2 = exitQualification;
2953 rc = VMXR0InjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), cbInstr, errCode);
2954 AssertRC(rc);
2955
2956 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
2957 goto ResumeExecution;
2958 }
2959#ifdef VBOX_STRICT
2960 if (rc != VINF_EM_RAW_EMULATE_INSTR && rc != VINF_EM_RAW_EMULATE_IO_BLOCK)
2961 Log2(("PGMTrap0eHandler failed with %d\n", rc));
2962#endif
2963 /* Need to go back to the recompiler to emulate the instruction. */
2964 TRPMResetTrap(pVCpu);
2965 break;
2966 }
2967
2968 case X86_XCPT_MF: /* Floating point exception. */
2969 {
2970 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestMF);
2971 if (!(pCtx->cr0 & X86_CR0_NE))
2972 {
2973 /* old style FPU error reporting needs some extra work. */
2974 /** @todo don't fall back to the recompiler, but do it manually. */
2975 rc = VINF_EM_RAW_EMULATE_INSTR;
2976 break;
2977 }
2978 Log(("Trap %x at %04X:%RGv\n", vector, pCtx->cs, (RTGCPTR)pCtx->rip));
2979 rc = VMXR0InjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), cbInstr, errCode);
2980 AssertRC(rc);
2981
2982 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
2983 goto ResumeExecution;
2984 }
2985
2986 case X86_XCPT_DB: /* Debug exception. */
2987 {
2988 uint64_t uDR6;
2989
2990 /* DR6, DR7.GD and IA32_DEBUGCTL.LBR are not updated yet.
2991 *
2992 * Exit qualification bits:
2993 * 3:0 B0-B3 which breakpoint condition was met
2994 * 12:4 Reserved (0)
2995 * 13 BD - debug register access detected
2996 * 14 BS - single step execution or branch taken
2997 * 63:15 Reserved (0)
2998 */
2999 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestDB);
3000
3001 /* Note that we don't support guest and host-initiated debugging at the same time. */
3002 Assert(DBGFIsStepping(pVCpu) || CPUMIsGuestInRealModeEx(pCtx) || CPUMIsHyperDebugStateActive(pVCpu));
3003
3004 uDR6 = X86_DR6_INIT_VAL;
3005 uDR6 |= (exitQualification & (X86_DR6_B0|X86_DR6_B1|X86_DR6_B2|X86_DR6_B3|X86_DR6_BD|X86_DR6_BS));
3006 rc = DBGFRZTrap01Handler(pVM, pVCpu, CPUMCTX2CORE(pCtx), uDR6);
3007 if (rc == VINF_EM_RAW_GUEST_TRAP)
3008 {
3009 /** @todo this isn't working, but we'll never get here normally. */
3010
3011 /* Update DR6 here. */
3012 pCtx->dr[6] = uDR6;
3013
3014 /* X86_DR7_GD will be cleared if drx accesses should be trapped inside the guest. */
3015 pCtx->dr[7] &= ~X86_DR7_GD;
3016
3017 /* Paranoia. */
3018 pCtx->dr[7] &= 0xffffffff; /* upper 32 bits reserved */
3019 pCtx->dr[7] &= ~(RT_BIT(11) | RT_BIT(12) | RT_BIT(14) | RT_BIT(15)); /* must be zero */
3020 pCtx->dr[7] |= 0x400; /* must be one */
3021
3022 /* Resync DR7 */
3023 rc = VMXWriteVMCS64(VMX_VMCS64_GUEST_DR7, pCtx->dr[7]);
3024 AssertRC(rc);
3025
3026 Log(("Trap %x (debug) at %RGv exit qualification %RX64\n", vector, (RTGCPTR)pCtx->rip, exitQualification));
3027 rc = VMXR0InjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), cbInstr, errCode);
3028 AssertRC(rc);
3029
3030 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3031 goto ResumeExecution;
3032 }
3033 /* Return to ring 3 to deal with the debug exit code. */
3034 Log(("Debugger hardware BP at %04x:%RGv (rc=%Rrc)\n", pCtx->cs, pCtx->rip, rc));
3035 break;
3036 }
3037
3038 case X86_XCPT_BP: /* Breakpoint. */
3039 {
3040 rc = DBGFRZTrap03Handler(pVM, pVCpu, CPUMCTX2CORE(pCtx));
3041 if (rc == VINF_EM_RAW_GUEST_TRAP)
3042 {
3043 Log(("Guest #BP at %04x:%RGv\n", pCtx->cs, pCtx->rip));
3044 rc = VMXR0InjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), cbInstr, errCode);
3045 AssertRC(rc);
3046 goto ResumeExecution;
3047 }
3048 if (rc == VINF_SUCCESS)
3049 goto ResumeExecution;
3050 Log(("Debugger BP at %04x:%RGv (rc=%Rrc)\n", pCtx->cs, pCtx->rip, rc));
3051 break;
3052 }
3053
3054 case X86_XCPT_GP: /* General protection failure exception.*/
3055 {
3056 uint32_t cbOp;
3057 uint32_t cbSize;
3058 PDISCPUSTATE pDis = &pVCpu->hwaccm.s.DisState;
3059
3060 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestGP);
3061#ifdef VBOX_STRICT
3062 if ( !CPUMIsGuestInRealModeEx(pCtx)
3063 || !pVM->hwaccm.s.vmx.pRealModeTSS)
3064 {
3065 Log(("Trap %x at %04X:%RGv errorCode=%x\n", vector, pCtx->cs, (RTGCPTR)pCtx->rip, errCode));
3066 rc = VMXR0InjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), cbInstr, errCode);
3067 AssertRC(rc);
3068 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3069 goto ResumeExecution;
3070 }
3071#endif
3072 Assert(CPUMIsGuestInRealModeEx(pCtx));
3073
3074 LogFlow(("Real mode X86_XCPT_GP instruction emulation at %x:%RGv\n", pCtx->cs, (RTGCPTR)pCtx->rip));
3075
3076 rc = EMInterpretDisasOne(pVM, pVCpu, CPUMCTX2CORE(pCtx), pDis, &cbOp);
3077 if (RT_SUCCESS(rc))
3078 {
3079 bool fUpdateRIP = true;
3080
3081 Assert(cbOp == pDis->opsize);
3082 switch (pDis->pCurInstr->opcode)
3083 {
3084 case OP_CLI:
3085 pCtx->eflags.Bits.u1IF = 0;
3086 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitCli);
3087 break;
3088
3089 case OP_STI:
3090 pCtx->eflags.Bits.u1IF = 1;
3091 EMSetInhibitInterruptsPC(pVCpu, pCtx->rip + pDis->opsize);
3092 Assert(VMCPU_FF_ISSET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
3093 rc = VMXWriteVMCS(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
3094 AssertRC(rc);
3095 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitSti);
3096 break;
3097
3098 case OP_HLT:
3099 fUpdateRIP = false;
3100 rc = VINF_EM_HALT;
3101 pCtx->rip += pDis->opsize;
3102 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitHlt);
3103 break;
3104
3105 case OP_POPF:
3106 {
3107 RTGCPTR GCPtrStack;
3108 uint32_t cbParm;
3109 uint32_t uMask;
3110 X86EFLAGS eflags;
3111
3112 if (pDis->prefix & PREFIX_OPSIZE)
3113 {
3114 cbParm = 4;
3115 uMask = 0xffffffff;
3116 }
3117 else
3118 {
3119 cbParm = 2;
3120 uMask = 0xffff;
3121 }
3122
3123 rc = SELMToFlatEx(pVM, DIS_SELREG_SS, CPUMCTX2CORE(pCtx), pCtx->esp & uMask, 0, &GCPtrStack);
3124 if (RT_FAILURE(rc))
3125 {
3126 rc = VERR_EM_INTERPRETER;
3127 break;
3128 }
3129 eflags.u = 0;
3130 rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &eflags.u, cbParm);
3131 if (RT_FAILURE(rc))
3132 {
3133 rc = VERR_EM_INTERPRETER;
3134 break;
3135 }
3136 LogFlow(("POPF %x -> %RGv mask=%x\n", eflags.u, pCtx->rsp, uMask));
3137 pCtx->eflags.u = (pCtx->eflags.u & ~(X86_EFL_POPF_BITS & uMask)) | (eflags.u & X86_EFL_POPF_BITS & uMask);
3138 /* RF cleared when popped in real mode; see pushf description in AMD manual. */
3139 pCtx->eflags.Bits.u1RF = 0;
3140 pCtx->esp += cbParm;
3141 pCtx->esp &= uMask;
3142
3143 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitPopf);
3144 break;
3145 }
3146
3147 case OP_PUSHF:
3148 {
3149 RTGCPTR GCPtrStack;
3150 uint32_t cbParm;
3151 uint32_t uMask;
3152 X86EFLAGS eflags;
3153
3154 if (pDis->prefix & PREFIX_OPSIZE)
3155 {
3156 cbParm = 4;
3157 uMask = 0xffffffff;
3158 }
3159 else
3160 {
3161 cbParm = 2;
3162 uMask = 0xffff;
3163 }
3164
3165 rc = SELMToFlatEx(pVM, DIS_SELREG_SS, CPUMCTX2CORE(pCtx), (pCtx->esp - cbParm) & uMask, 0, &GCPtrStack);
3166 if (RT_FAILURE(rc))
3167 {
3168 rc = VERR_EM_INTERPRETER;
3169 break;
3170 }
3171 eflags = pCtx->eflags;
3172 /* RF & VM cleared when pushed in real mode; see pushf description in AMD manual. */
3173 eflags.Bits.u1RF = 0;
3174 eflags.Bits.u1VM = 0;
3175
3176 rc = PGMPhysWrite(pVM, (RTGCPHYS)GCPtrStack, &eflags.u, cbParm);
3177 if (RT_FAILURE(rc))
3178 {
3179 rc = VERR_EM_INTERPRETER;
3180 break;
3181 }
3182 LogFlow(("PUSHF %x -> %RGv\n", eflags.u, GCPtrStack));
3183 pCtx->esp -= cbParm;
3184 pCtx->esp &= uMask;
3185 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitPushf);
3186 break;
3187 }
3188
3189 case OP_IRET:
3190 {
3191 RTGCPTR GCPtrStack;
3192 uint32_t uMask = 0xffff;
3193 uint16_t aIretFrame[3];
3194
3195 if (pDis->prefix & (PREFIX_OPSIZE | PREFIX_ADDRSIZE))
3196 {
3197 rc = VERR_EM_INTERPRETER;
3198 break;
3199 }
3200
3201 rc = SELMToFlatEx(pVM, DIS_SELREG_SS, CPUMCTX2CORE(pCtx), pCtx->esp & uMask, 0, &GCPtrStack);
3202 if (RT_FAILURE(rc))
3203 {
3204 rc = VERR_EM_INTERPRETER;
3205 break;
3206 }
3207 rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &aIretFrame[0], sizeof(aIretFrame));
3208 if (RT_FAILURE(rc))
3209 {
3210 rc = VERR_EM_INTERPRETER;
3211 break;
3212 }
3213 pCtx->ip = aIretFrame[0];
3214 pCtx->cs = aIretFrame[1];
3215 pCtx->csHid.u64Base = pCtx->cs << 4;
3216 pCtx->eflags.u = (pCtx->eflags.u & ~(X86_EFL_POPF_BITS & uMask)) | (aIretFrame[2] & X86_EFL_POPF_BITS & uMask);
3217 pCtx->sp += sizeof(aIretFrame);
3218
3219 LogFlow(("iret to %04x:%x\n", pCtx->cs, pCtx->ip));
3220 fUpdateRIP = false;
3221 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitIret);
3222 break;
3223 }
3224
3225 case OP_INT:
3226 {
3227 uint32_t intInfo2;
3228
3229 LogFlow(("Realmode: INT %x\n", pDis->param1.parval & 0xff));
3230 intInfo2 = pDis->param1.parval & 0xff;
3231 intInfo2 |= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
3232 intInfo2 |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
3233
3234 rc = VMXR0InjectEvent(pVM, pVCpu, pCtx, intInfo2, cbOp, 0);
3235 AssertRC(rc);
3236 fUpdateRIP = false;
3237 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitInt);
3238 break;
3239 }
3240
3241 case OP_INTO:
3242 {
3243 if (pCtx->eflags.Bits.u1OF)
3244 {
3245 uint32_t intInfo2;
3246
3247 LogFlow(("Realmode: INTO\n"));
3248 intInfo2 = X86_XCPT_OF;
3249 intInfo2 |= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
3250 intInfo2 |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
3251
3252 rc = VMXR0InjectEvent(pVM, pVCpu, pCtx, intInfo2, cbOp, 0);
3253 AssertRC(rc);
3254 fUpdateRIP = false;
3255 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitInt);
3256 }
3257 break;
3258 }
3259
3260 case OP_INT3:
3261 {
3262 uint32_t intInfo2;
3263
3264 LogFlow(("Realmode: INT 3\n"));
3265 intInfo2 = 3;
3266 intInfo2 |= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
3267 intInfo2 |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
3268
3269 rc = VMXR0InjectEvent(pVM, pVCpu, pCtx, intInfo2, cbOp, 0);
3270 AssertRC(rc);
3271 fUpdateRIP = false;
3272 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitInt);
3273 break;
3274 }
3275
3276 default:
3277 rc = EMInterpretInstructionCPU(pVM, pVCpu, pDis, CPUMCTX2CORE(pCtx), 0, &cbSize);
3278 break;
3279 }
3280
3281 if (rc == VINF_SUCCESS)
3282 {
3283 if (fUpdateRIP)
3284 pCtx->rip += cbOp; /* Move on to the next instruction. */
3285
3286 /* lidt, lgdt can end up here. In the future crx changes as well. Just reload the whole context to be done with it. */
3287 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_ALL;
3288
3289 /* Only resume if successful. */
3290 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3291 goto ResumeExecution;
3292 }
3293 }
3294 else
3295 rc = VERR_EM_INTERPRETER;
3296
3297 AssertMsg(rc == VERR_EM_INTERPRETER || rc == VINF_PGM_CHANGE_MODE || rc == VINF_EM_HALT, ("Unexpected rc=%Rrc\n", rc));
3298 break;
3299 }
3300
3301#ifdef VBOX_STRICT
3302 case X86_XCPT_XF: /* SIMD exception. */
3303 case X86_XCPT_DE: /* Divide error. */
3304 case X86_XCPT_UD: /* Unknown opcode exception. */
3305 case X86_XCPT_SS: /* Stack segment exception. */
3306 case X86_XCPT_NP: /* Segment not present exception. */
3307 {
3308 switch(vector)
3309 {
3310 case X86_XCPT_DE:
3311 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestDE);
3312 break;
3313 case X86_XCPT_UD:
3314 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestUD);
3315 break;
3316 case X86_XCPT_SS:
3317 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestSS);
3318 break;
3319 case X86_XCPT_NP:
3320 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestNP);
3321 break;
3322 }
3323
3324 Log(("Trap %x at %04X:%RGv\n", vector, pCtx->cs, (RTGCPTR)pCtx->rip));
3325 rc = VMXR0InjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), cbInstr, errCode);
3326 AssertRC(rc);
3327
3328 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3329 goto ResumeExecution;
3330 }
3331#endif
3332 default:
3333#ifdef HWACCM_VMX_EMULATE_REALMODE
3334 if ( CPUMIsGuestInRealModeEx(pCtx)
3335 && pVM->hwaccm.s.vmx.pRealModeTSS)
3336 {
3337 Log(("Real Mode Trap %x at %04x:%04X error code %x\n", vector, pCtx->cs, pCtx->eip, errCode));
3338 rc = VMXR0InjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), cbInstr, errCode);
3339 AssertRC(rc);
3340
3341 /* Go back to ring 3 in case of a triple fault. */
3342 if ( vector == X86_XCPT_DF
3343 && rc == VINF_EM_RESET)
3344 break;
3345
3346 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3347 goto ResumeExecution;
3348 }
3349#endif
3350 AssertMsgFailed(("Unexpected vm-exit caused by exception %x\n", vector));
3351 rc = VERR_VMX_UNEXPECTED_EXCEPTION;
3352 break;
3353 } /* switch (vector) */
3354
3355 break;
3356
3357 default:
3358 rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_CODE;
3359 AssertMsgFailed(("Unexpected interuption code %x\n", intInfo));
3360 break;
3361 }
3362
3363 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3364 break;
3365 }
3366
3367 case VMX_EXIT_EPT_VIOLATION: /* 48 EPT violation. An attempt to access memory with a guest-physical address was disallowed by the configuration of the EPT paging structures. */
3368 {
3369 RTGCPHYS GCPhys;
3370
3371 Assert(pVM->hwaccm.s.fNestedPaging);
3372
3373 rc = VMXReadVMCS64(VMX_VMCS_EXIT_PHYS_ADDR_FULL, &GCPhys);
3374 AssertRC(rc);
3375 Assert(((exitQualification >> 7) & 3) != 2);
3376
3377 /* Determine the kind of violation. */
3378 errCode = 0;
3379 if (exitQualification & VMX_EXIT_QUALIFICATION_EPT_INSTR_FETCH)
3380 errCode |= X86_TRAP_PF_ID;
3381
3382 if (exitQualification & VMX_EXIT_QUALIFICATION_EPT_DATA_WRITE)
3383 errCode |= X86_TRAP_PF_RW;
3384
3385 /* If the page is present, then it's a page level protection fault. */
3386 if (exitQualification & VMX_EXIT_QUALIFICATION_EPT_ENTRY_PRESENT)
3387 {
3388 errCode |= X86_TRAP_PF_P;
3389 }
3390 else {
3391 /* Shortcut for APIC TPR reads and writes. */
3392 if ( (GCPhys & 0xfff) == 0x080
3393 && GCPhys > 0x1000000 /* to skip VGA frame buffer accesses */
3394 && fSetupTPRCaching
3395 && (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC))
3396 {
3397 RTGCPHYS GCPhysApicBase;
3398 PDMApicGetBase(pVM, &GCPhysApicBase); /* @todo cache this */
3399 GCPhysApicBase &= PAGE_BASE_GC_MASK;
3400 if (GCPhys == GCPhysApicBase + 0x80)
3401 {
3402 Log(("Enable VT-x virtual APIC access filtering\n"));
3403 rc = IOMMMIOMapMMIOHCPage(pVM, GCPhysApicBase, pVM->hwaccm.s.vmx.pAPICPhys, X86_PTE_RW | X86_PTE_P);
3404 AssertRC(rc);
3405 }
3406 }
3407 }
3408 Log(("EPT Page fault %x at %RGp error code %x\n", (uint32_t)exitQualification, GCPhys, errCode));
3409
3410 /* GCPhys contains the guest physical address of the page fault. */
3411 TRPMAssertTrap(pVCpu, X86_XCPT_PF, TRPM_TRAP);
3412 TRPMSetErrorCode(pVCpu, errCode);
3413 TRPMSetFaultAddress(pVCpu, GCPhys);
3414
3415 /* Handle the pagefault trap for the nested shadow table. */
3416 rc = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, errCode, CPUMCTX2CORE(pCtx), GCPhys);
3417 Log2(("PGMR0Trap0eHandlerNestedPaging %RGv returned %Rrc\n", (RTGCPTR)pCtx->rip, rc));
3418 if (rc == VINF_SUCCESS)
3419 { /* We've successfully synced our shadow pages, so let's just continue execution. */
3420 Log2(("Shadow page fault at %RGv cr2=%RGp error code %x\n", (RTGCPTR)pCtx->rip, exitQualification , errCode));
3421 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitReasonNPF);
3422
3423 TRPMResetTrap(pVCpu);
3424 goto ResumeExecution;
3425 }
3426
3427#ifdef VBOX_STRICT
3428 if (rc != VINF_EM_RAW_EMULATE_INSTR)
3429 LogFlow(("PGMTrap0eHandlerNestedPaging failed with %d\n", rc));
3430#endif
3431 /* Need to go back to the recompiler to emulate the instruction. */
3432 TRPMResetTrap(pVCpu);
3433 break;
3434 }
3435
3436 case VMX_EXIT_EPT_MISCONFIG:
3437 {
3438 RTGCPHYS GCPhys;
3439
3440 Assert(pVM->hwaccm.s.fNestedPaging);
3441
3442 rc = VMXReadVMCS64(VMX_VMCS_EXIT_PHYS_ADDR_FULL, &GCPhys);
3443 AssertRC(rc);
3444
3445 Log(("VMX_EXIT_EPT_MISCONFIG for %RGp\n", GCPhys));
3446 break;
3447 }
3448
3449 case VMX_EXIT_IRQ_WINDOW: /* 7 Interrupt window. */
3450 /* Clear VM-exit on IF=1 change. */
3451 LogFlow(("VMX_EXIT_IRQ_WINDOW %RGv pending=%d IF=%d\n", (RTGCPTR)pCtx->rip, VMCPU_FF_ISPENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC|VMCPU_FF_INTERRUPT_PIC)), pCtx->eflags.Bits.u1IF));
3452 pVCpu->hwaccm.s.vmx.proc_ctls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_IRQ_WINDOW_EXIT;
3453 rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
3454 AssertRC(rc);
3455 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitIrqWindow);
3456 goto ResumeExecution; /* we check for pending guest interrupts there */
3457
3458 case VMX_EXIT_WBINVD: /* 54 Guest software attempted to execute WBINVD. (conditional) */
3459 case VMX_EXIT_INVD: /* 13 Guest software attempted to execute INVD. (unconditional) */
3460 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitInvd);
3461 /* Skip instruction and continue directly. */
3462 pCtx->rip += cbInstr;
3463 /* Continue execution.*/
3464 goto ResumeExecution;
3465
3466 case VMX_EXIT_CPUID: /* 10 Guest software attempted to execute CPUID. */
3467 {
3468 Log2(("VMX: Cpuid %x\n", pCtx->eax));
3469 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitCpuid);
3470 rc = EMInterpretCpuId(pVM, pVCpu, CPUMCTX2CORE(pCtx));
3471 if (rc == VINF_SUCCESS)
3472 {
3473 /* Update EIP and continue execution. */
3474 Assert(cbInstr == 2);
3475 pCtx->rip += cbInstr;
3476 goto ResumeExecution;
3477 }
3478 AssertMsgFailed(("EMU: cpuid failed with %Rrc\n", rc));
3479 rc = VINF_EM_RAW_EMULATE_INSTR;
3480 break;
3481 }
3482
3483 case VMX_EXIT_RDPMC: /* 15 Guest software attempted to execute RDPMC. */
3484 {
3485 Log2(("VMX: Rdpmc %x\n", pCtx->ecx));
3486 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitRdpmc);
3487 rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pCtx));
3488 if (rc == VINF_SUCCESS)
3489 {
3490 /* Update EIP and continue execution. */
3491 Assert(cbInstr == 2);
3492 pCtx->rip += cbInstr;
3493 goto ResumeExecution;
3494 }
3495 rc = VINF_EM_RAW_EMULATE_INSTR;
3496 break;
3497 }
3498
3499 case VMX_EXIT_RDTSC: /* 16 Guest software attempted to execute RDTSC. */
3500 {
3501 Log2(("VMX: Rdtsc\n"));
3502 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitRdtsc);
3503 rc = EMInterpretRdtsc(pVM, pVCpu, CPUMCTX2CORE(pCtx));
3504 if (rc == VINF_SUCCESS)
3505 {
3506 /* Update EIP and continue execution. */
3507 Assert(cbInstr == 2);
3508 pCtx->rip += cbInstr;
3509 goto ResumeExecution;
3510 }
3511 rc = VINF_EM_RAW_EMULATE_INSTR;
3512 break;
3513 }
3514
3515 case VMX_EXIT_INVPG: /* 14 Guest software attempted to execute INVPG. */
3516 {
3517 Log2(("VMX: invlpg\n"));
3518 Assert(!pVM->hwaccm.s.fNestedPaging);
3519
3520 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitInvpg);
3521 rc = EMInterpretInvlpg(pVM, pVCpu, CPUMCTX2CORE(pCtx), exitQualification);
3522 if (rc == VINF_SUCCESS)
3523 {
3524 /* Update EIP and continue execution. */
3525 pCtx->rip += cbInstr;
3526 goto ResumeExecution;
3527 }
3528 AssertMsg(rc == VERR_EM_INTERPRETER, ("EMU: invlpg %RGv failed with %Rrc\n", exitQualification, rc));
3529 break;
3530 }
3531
3532 case VMX_EXIT_WRMSR: /* 32 WRMSR. Guest software attempted to execute WRMSR. */
3533 /* When an interrupt is pending, we'll let MSR_K8_LSTAR writes fault in our TPR patch code. */
3534 if ( pVM->hwaccm.s.fTPRPatchingActive
3535 && pCtx->ecx == MSR_K8_LSTAR)
3536 {
3537 Assert(!CPUMIsGuestInLongModeEx(pCtx));
3538 if ((pCtx->eax & 0xff) != u8LastTPR)
3539 {
3540 Log(("VMX: Faulting MSR_K8_LSTAR write with new TPR value %x\n", pCtx->eax & 0xff));
3541
3542 /* Our patch code uses LSTAR for TPR caching. */
3543 rc = PDMApicSetTPR(pVCpu, pCtx->eax & 0xff);
3544 AssertRC(rc);
3545 }
3546
3547 /* Skip the instruction and continue. */
3548 pCtx->rip += cbInstr; /* wrmsr = [0F 30] */
3549
3550 /* Only resume if successful. */
3551 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit1, x);
3552 goto ResumeExecution;
3553 }
3554 /* no break */
3555 case VMX_EXIT_RDMSR: /* 31 RDMSR. Guest software attempted to execute RDMSR. */
3556 {
3557 uint32_t cbSize;
3558
3559 STAM_COUNTER_INC((exitReason == VMX_EXIT_RDMSR) ? &pVCpu->hwaccm.s.StatExitRdmsr : &pVCpu->hwaccm.s.StatExitWrmsr);
3560
3561 /* Note: the intel manual claims there's a REX version of RDMSR that's slightly different, so we play safe by completely disassembling the instruction. */
3562 Log2(("VMX: %s\n", (exitReason == VMX_EXIT_RDMSR) ? "rdmsr" : "wrmsr"));
3563 rc = EMInterpretInstruction(pVM, pVCpu, CPUMCTX2CORE(pCtx), 0, &cbSize);
3564 if (rc == VINF_SUCCESS)
3565 {
3566 /* EIP has been updated already. */
3567
3568 /* Only resume if successful. */
3569 goto ResumeExecution;
3570 }
3571 AssertMsg(rc == VERR_EM_INTERPRETER, ("EMU: %s failed with %Rrc\n", (exitReason == VMX_EXIT_RDMSR) ? "rdmsr" : "wrmsr", rc));
3572 break;
3573 }
3574
3575 case VMX_EXIT_CRX_MOVE: /* 28 Control-register accesses. */
3576 {
3577 STAM_PROFILE_ADV_START(&pVCpu->hwaccm.s.StatExit2Sub2, y2);
3578
3579 switch (VMX_EXIT_QUALIFICATION_CRX_ACCESS(exitQualification))
3580 {
3581 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_WRITE:
3582 Log2(("VMX: %RGv mov cr%d, x\n", (RTGCPTR)pCtx->rip, VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification)));
3583 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitCRxWrite[VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification)]);
3584 rc = EMInterpretCRxWrite(pVM, pVCpu, CPUMCTX2CORE(pCtx),
3585 VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification),
3586 VMX_EXIT_QUALIFICATION_CRX_GENREG(exitQualification));
3587
3588 switch (VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification))
3589 {
3590 case 0:
3591 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_GUEST_CR0 | HWACCM_CHANGED_GUEST_CR3;
3592 break;
3593 case 2:
3594 break;
3595 case 3:
3596 Assert(!pVM->hwaccm.s.fNestedPaging || !CPUMIsGuestInPagedProtectedModeEx(pCtx));
3597 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_GUEST_CR3;
3598 break;
3599 case 4:
3600 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_GUEST_CR4;
3601 break;
3602 case 8:
3603 /* CR8 contains the APIC TPR */
3604 Assert(!(pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW));
3605 break;
3606
3607 default:
3608 AssertFailed();
3609 break;
3610 }
3611 break;
3612
3613 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ:
3614 Log2(("VMX: mov x, crx\n"));
3615 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitCRxRead[VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification)]);
3616
3617 Assert(!pVM->hwaccm.s.fNestedPaging || !CPUMIsGuestInPagedProtectedModeEx(pCtx) || VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification) != USE_REG_CR3);
3618
3619 /* CR8 reads only cause an exit when the TPR shadow feature isn't present. */
3620 Assert(VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification) != 8 || !(pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW));
3621
3622 rc = EMInterpretCRxRead(pVM, pVCpu, CPUMCTX2CORE(pCtx),
3623 VMX_EXIT_QUALIFICATION_CRX_GENREG(exitQualification),
3624 VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification));
3625 break;
3626
3627 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_CLTS:
3628 Log2(("VMX: clts\n"));
3629 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitCLTS);
3630 rc = EMInterpretCLTS(pVM, pVCpu);
3631 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_GUEST_CR0;
3632 break;
3633
3634 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_LMSW:
3635 Log2(("VMX: lmsw %x\n", VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(exitQualification)));
3636 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitLMSW);
3637 rc = EMInterpretLMSW(pVM, pVCpu, CPUMCTX2CORE(pCtx), VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(exitQualification));
3638 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_GUEST_CR0;
3639 break;
3640 }
3641
3642 /* Update EIP if no error occurred. */
3643 if (RT_SUCCESS(rc))
3644 pCtx->rip += cbInstr;
3645
3646 if (rc == VINF_SUCCESS)
3647 {
3648 /* Only resume if successful. */
3649 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub2, y2);
3650 goto ResumeExecution;
3651 }
3652 Assert(rc == VERR_EM_INTERPRETER || rc == VINF_PGM_CHANGE_MODE || rc == VINF_PGM_SYNC_CR3);
3653 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub2, y2);
3654 break;
3655 }
3656
3657 case VMX_EXIT_DRX_MOVE: /* 29 Debug-register accesses. */
3658 {
3659 if ( !DBGFIsStepping(pVCpu)
3660 && !CPUMIsHyperDebugStateActive(pVCpu))
3661 {
3662 /* Disable drx move intercepts. */
3663 pVCpu->hwaccm.s.vmx.proc_ctls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT;
3664 rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
3665 AssertRC(rc);
3666
3667 /* Save the host and load the guest debug state. */
3668 rc = CPUMR0LoadGuestDebugState(pVM, pVCpu, pCtx, true /* include DR6 */);
3669 AssertRC(rc);
3670
3671#ifdef VBOX_WITH_STATISTICS
3672 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatDRxContextSwitch);
3673 if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(exitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
3674 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitDRxWrite);
3675 else
3676 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitDRxRead);
3677#endif
3678
3679 goto ResumeExecution;
3680 }
3681
3682 /** @todo clear VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT after the first time and restore drx registers afterwards */
3683 if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(exitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
3684 {
3685 Log2(("VMX: mov drx%d, genreg%d\n", VMX_EXIT_QUALIFICATION_DRX_REGISTER(exitQualification), VMX_EXIT_QUALIFICATION_DRX_GENREG(exitQualification)));
3686 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitDRxWrite);
3687 rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pCtx),
3688 VMX_EXIT_QUALIFICATION_DRX_REGISTER(exitQualification),
3689 VMX_EXIT_QUALIFICATION_DRX_GENREG(exitQualification));
3690 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_GUEST_DEBUG;
3691 Log2(("DR7=%08x\n", pCtx->dr[7]));
3692 }
3693 else
3694 {
3695 Log2(("VMX: mov x, drx\n"));
3696 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitDRxRead);
3697 rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pCtx),
3698 VMX_EXIT_QUALIFICATION_DRX_GENREG(exitQualification),
3699 VMX_EXIT_QUALIFICATION_DRX_REGISTER(exitQualification));
3700 }
3701 /* Update EIP if no error occurred. */
3702 if (RT_SUCCESS(rc))
3703 pCtx->rip += cbInstr;
3704
3705 if (rc == VINF_SUCCESS)
3706 {
3707 /* Only resume if successful. */
3708 goto ResumeExecution;
3709 }
3710 Assert(rc == VERR_EM_INTERPRETER);
3711 break;
3712 }
3713
3714 /* Note: We'll get a #GP if the IO instruction isn't allowed (IOPL or TSS bitmap); no need to double check. */
3715 case VMX_EXIT_PORT_IO: /* 30 I/O instruction. */
3716 {
3717 STAM_PROFILE_ADV_START(&pVCpu->hwaccm.s.StatExit2Sub1, y1);
3718 uint32_t uIOWidth = VMX_EXIT_QUALIFICATION_IO_WIDTH(exitQualification);
3719 uint32_t uPort;
3720 bool fIOWrite = (VMX_EXIT_QUALIFICATION_IO_DIRECTION(exitQualification) == VMX_EXIT_QUALIFICATION_IO_DIRECTION_OUT);
3721
3722 /** @todo necessary to make the distinction? */
3723 if (VMX_EXIT_QUALIFICATION_IO_ENCODING(exitQualification) == VMX_EXIT_QUALIFICATION_IO_ENCODING_DX)
3724 {
3725 uPort = pCtx->edx & 0xffff;
3726 }
3727 else
3728 uPort = VMX_EXIT_QUALIFICATION_IO_PORT(exitQualification); /* Immediate encoding. */
3729
3730 /* paranoia */
3731 if (RT_UNLIKELY(uIOWidth == 2 || uIOWidth >= 4))
3732 {
3733 rc = fIOWrite ? VINF_IOM_HC_IOPORT_WRITE : VINF_IOM_HC_IOPORT_READ;
3734 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub1, y1);
3735 break;
3736 }
3737
3738 uint32_t cbSize = g_aIOSize[uIOWidth];
3739
3740 if (VMX_EXIT_QUALIFICATION_IO_STRING(exitQualification))
3741 {
3742 /* ins/outs */
3743 PDISCPUSTATE pDis = &pVCpu->hwaccm.s.DisState;
3744
3745 /* Disassemble manually to deal with segment prefixes. */
3746 /** @todo VMX_VMCS_EXIT_GUEST_LINEAR_ADDR contains the flat pointer operand of the instruction. */
3747 /** @todo VMX_VMCS32_RO_EXIT_INSTR_INFO also contains segment prefix info. */
3748 rc = EMInterpretDisasOne(pVM, pVCpu, CPUMCTX2CORE(pCtx), pDis, NULL);
3749 if (rc == VINF_SUCCESS)
3750 {
3751 if (fIOWrite)
3752 {
3753 Log2(("IOMInterpretOUTSEx %RGv %x size=%d\n", (RTGCPTR)pCtx->rip, uPort, cbSize));
3754 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitIOStringWrite);
3755 rc = VBOXSTRICTRC_TODO(IOMInterpretOUTSEx(pVM, CPUMCTX2CORE(pCtx), uPort, pDis->prefix, cbSize));
3756 }
3757 else
3758 {
3759 Log2(("IOMInterpretINSEx %RGv %x size=%d\n", (RTGCPTR)pCtx->rip, uPort, cbSize));
3760 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitIOStringRead);
3761 rc = VBOXSTRICTRC_TODO(IOMInterpretINSEx(pVM, CPUMCTX2CORE(pCtx), uPort, pDis->prefix, cbSize));
3762 }
3763 }
3764 else
3765 rc = VINF_EM_RAW_EMULATE_INSTR;
3766 }
3767 else
3768 {
3769 /* normal in/out */
3770 uint32_t uAndVal = g_aIOOpAnd[uIOWidth];
3771
3772 Assert(!VMX_EXIT_QUALIFICATION_IO_REP(exitQualification));
3773
3774 if (fIOWrite)
3775 {
3776 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitIOWrite);
3777 rc = VBOXSTRICTRC_TODO(IOMIOPortWrite(pVM, uPort, pCtx->eax & uAndVal, cbSize));
3778 if (rc == VINF_IOM_HC_IOPORT_WRITE)
3779 HWACCMR0SavePendingIOPortWrite(pVCpu, pCtx->rip, pCtx->rip + cbInstr, uPort, uAndVal, cbSize);
3780 }
3781 else
3782 {
3783 uint32_t u32Val = 0;
3784
3785 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitIORead);
3786 rc = VBOXSTRICTRC_TODO(IOMIOPortRead(pVM, uPort, &u32Val, cbSize));
3787 if (IOM_SUCCESS(rc))
3788 {
3789 /* Write back to the EAX register. */
3790 pCtx->eax = (pCtx->eax & ~uAndVal) | (u32Val & uAndVal);
3791 }
3792 else
3793 if (rc == VINF_IOM_HC_IOPORT_READ)
3794 HWACCMR0SavePendingIOPortRead(pVCpu, pCtx->rip, pCtx->rip + cbInstr, uPort, uAndVal, cbSize);
3795 }
3796 }
3797 /*
3798 * Handled the I/O return codes.
3799 * (The unhandled cases end up with rc == VINF_EM_RAW_EMULATE_INSTR.)
3800 */
3801 if (IOM_SUCCESS(rc))
3802 {
3803 /* Update EIP and continue execution. */
3804 pCtx->rip += cbInstr;
3805 if (RT_LIKELY(rc == VINF_SUCCESS))
3806 {
3807 /* If any IO breakpoints are armed, then we should check if a debug trap needs to be generated. */
3808 if (pCtx->dr[7] & X86_DR7_ENABLED_MASK)
3809 {
3810 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatDRxIOCheck);
3811 for (unsigned i=0;i<4;i++)
3812 {
3813 unsigned uBPLen = g_aIOSize[X86_DR7_GET_LEN(pCtx->dr[7], i)];
3814
3815 if ( (uPort >= pCtx->dr[i] && uPort < pCtx->dr[i] + uBPLen)
3816 && (pCtx->dr[7] & (X86_DR7_L(i) | X86_DR7_G(i)))
3817 && (pCtx->dr[7] & X86_DR7_RW(i, X86_DR7_RW_IO)) == X86_DR7_RW(i, X86_DR7_RW_IO))
3818 {
3819 uint64_t uDR6;
3820
3821 Assert(CPUMIsGuestDebugStateActive(pVCpu));
3822
3823 uDR6 = ASMGetDR6();
3824
3825 /* Clear all breakpoint status flags and set the one we just hit. */
3826 uDR6 &= ~(X86_DR6_B0|X86_DR6_B1|X86_DR6_B2|X86_DR6_B3);
3827 uDR6 |= (uint64_t)RT_BIT(i);
3828
3829 /* Note: AMD64 Architecture Programmer's Manual 13.1:
3830 * Bits 15:13 of the DR6 register is never cleared by the processor and must be cleared by software after
3831 * the contents have been read.
3832 */
3833 ASMSetDR6(uDR6);
3834
3835 /* X86_DR7_GD will be cleared if drx accesses should be trapped inside the guest. */
3836 pCtx->dr[7] &= ~X86_DR7_GD;
3837
3838 /* Paranoia. */
3839 pCtx->dr[7] &= 0xffffffff; /* upper 32 bits reserved */
3840 pCtx->dr[7] &= ~(RT_BIT(11) | RT_BIT(12) | RT_BIT(14) | RT_BIT(15)); /* must be zero */
3841 pCtx->dr[7] |= 0x400; /* must be one */
3842
3843 /* Resync DR7 */
3844 rc = VMXWriteVMCS64(VMX_VMCS64_GUEST_DR7, pCtx->dr[7]);
3845 AssertRC(rc);
3846
3847 /* Construct inject info. */
3848 intInfo = X86_XCPT_DB;
3849 intInfo |= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
3850 intInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HWEXCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
3851
3852 Log(("Inject IO debug trap at %RGv\n", (RTGCPTR)pCtx->rip));
3853 rc = VMXR0InjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), 0, 0);
3854 AssertRC(rc);
3855
3856 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub1, y1);
3857 goto ResumeExecution;
3858 }
3859 }
3860 }
3861
3862 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub1, y1);
3863 goto ResumeExecution;
3864 }
3865 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub1, y1);
3866 break;
3867 }
3868
3869#ifdef VBOX_STRICT
3870 if (rc == VINF_IOM_HC_IOPORT_READ)
3871 Assert(!fIOWrite);
3872 else if (rc == VINF_IOM_HC_IOPORT_WRITE)
3873 Assert(fIOWrite);
3874 else
3875 AssertMsg(RT_FAILURE(rc) || rc == VINF_EM_RAW_EMULATE_INSTR || rc == VINF_EM_RAW_GUEST_TRAP || rc == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", rc));
3876#endif
3877 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub1, y1);
3878 break;
3879 }
3880
3881 case VMX_EXIT_TPR: /* 43 TPR below threshold. Guest software executed MOV to CR8. */
3882 LogFlow(("VMX_EXIT_TPR\n"));
3883 /* RIP is already set to the next instruction and the TPR has been synced back. Just resume. */
3884 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub1, y1);
3885 goto ResumeExecution;
3886
3887 case VMX_EXIT_APIC_ACCESS: /* 44 APIC access. Guest software attempted to access memory at a physical address on the APIC-access page. */
3888 {
3889 LogFlow(("VMX_EXIT_APIC_ACCESS\n"));
3890 unsigned uAccessType = VMX_EXIT_QUALIFICATION_APIC_ACCESS_TYPE(exitQualification);
3891
3892 switch(uAccessType)
3893 {
3894 case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
3895 case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
3896 {
3897 RTGCPHYS GCPhys;
3898 PDMApicGetBase(pVM, &GCPhys);
3899 GCPhys &= PAGE_BASE_GC_MASK;
3900 GCPhys += VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(exitQualification);
3901
3902 LogFlow(("Apic access at %RGp\n", GCPhys));
3903 rc = VBOXSTRICTRC_TODO(IOMMMIOPhysHandler(pVM, (uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ) ? 0 : X86_TRAP_PF_RW, CPUMCTX2CORE(pCtx), GCPhys));
3904 if (rc == VINF_SUCCESS)
3905 {
3906 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub1, y1);
3907 goto ResumeExecution; /* rip already updated */
3908 }
3909 break;
3910 }
3911
3912 default:
3913 rc = VINF_EM_RAW_EMULATE_INSTR;
3914 break;
3915 }
3916 break;
3917 }
3918
3919 case VMX_EXIT_PREEMPTION_TIMER: /* 52 VMX-preemption timer expired. The preemption timer counted down to zero. */
3920 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub1, y1);
3921 goto ResumeExecution;
3922
3923 default:
3924 /* The rest is handled after syncing the entire CPU state. */
3925 break;
3926 }
3927
3928 /* Note: the guest state isn't entirely synced back at this stage. */
3929
3930 /* Investigate why there was a VM-exit. (part 2) */
3931 switch (exitReason)
3932 {
3933 case VMX_EXIT_EXCEPTION: /* 0 Exception or non-maskable interrupt (NMI). */
3934 case VMX_EXIT_EXTERNAL_IRQ: /* 1 External interrupt. */
3935 case VMX_EXIT_EPT_VIOLATION:
3936 case VMX_EXIT_PREEMPTION_TIMER: /* 52 VMX-preemption timer expired. The preemption timer counted down to zero. */
3937 /* Already handled above. */
3938 break;
3939
3940 case VMX_EXIT_TRIPLE_FAULT: /* 2 Triple fault. */
3941 rc = VINF_EM_RESET; /* Triple fault equals a reset. */
3942 break;
3943
3944 case VMX_EXIT_INIT_SIGNAL: /* 3 INIT signal. */
3945 case VMX_EXIT_SIPI: /* 4 Start-up IPI (SIPI). */
3946 rc = VINF_EM_RAW_INTERRUPT;
3947 AssertFailed(); /* Can't happen. Yet. */
3948 break;
3949
3950 case VMX_EXIT_IO_SMI_IRQ: /* 5 I/O system-management interrupt (SMI). */
3951 case VMX_EXIT_SMI_IRQ: /* 6 Other SMI. */
3952 rc = VINF_EM_RAW_INTERRUPT;
3953 AssertFailed(); /* Can't happen afaik. */
3954 break;
3955
3956 case VMX_EXIT_TASK_SWITCH: /* 9 Task switch: too complicated to emulate, so fall back to the recompiler */
3957 Log(("VMX_EXIT_TASK_SWITCH: exit=%RX64\n", exitQualification));
3958 if ( (VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE(exitQualification) == VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE_IDT)
3959 && pVCpu->hwaccm.s.Event.fPending)
3960 {
3961 /* Caused by an injected interrupt. */
3962 pVCpu->hwaccm.s.Event.fPending = false;
3963
3964 Log(("VMX_EXIT_TASK_SWITCH: reassert trap %d\n", VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVCpu->hwaccm.s.Event.intInfo)));
3965 Assert(!VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(pVCpu->hwaccm.s.Event.intInfo));
3966 rc = TRPMAssertTrap(pVCpu, VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVCpu->hwaccm.s.Event.intInfo), TRPM_HARDWARE_INT);
3967 AssertRC(rc);
3968 }
3969 /* else Exceptions and software interrupts can just be restarted. */
3970 rc = VERR_EM_INTERPRETER;
3971 break;
3972
3973 case VMX_EXIT_HLT: /* 12 Guest software attempted to execute HLT. */
3974 /** Check if external interrupts are pending; if so, don't switch back. */
3975 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitHlt);
3976 pCtx->rip++; /* skip hlt */
3977 if ( pCtx->eflags.Bits.u1IF
3978 && VMCPU_FF_ISPENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC|VMCPU_FF_INTERRUPT_PIC)))
3979 goto ResumeExecution;
3980
3981 rc = VINF_EM_HALT;
3982 break;
3983
3984 case VMX_EXIT_MWAIT: /* 36 Guest software executed MWAIT. */
3985 Log2(("VMX: mwait\n"));
3986 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitMwait);
3987 rc = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pCtx));
3988 if ( rc == VINF_EM_HALT
3989 || rc == VINF_SUCCESS)
3990 {
3991 /* Update EIP and continue execution. */
3992 pCtx->rip += cbInstr;
3993
3994 /** Check if external interrupts are pending; if so, don't switch back. */
3995 if ( rc == VINF_SUCCESS
3996 || ( rc == VINF_EM_HALT
3997 && pCtx->eflags.Bits.u1IF
3998 && VMCPU_FF_ISPENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC|VMCPU_FF_INTERRUPT_PIC)))
3999 )
4000 goto ResumeExecution;
4001 }
4002 AssertMsg(rc == VERR_EM_INTERPRETER || rc == VINF_EM_HALT, ("EMU: mwait failed with %Rrc\n", rc));
4003 break;
4004
4005 case VMX_EXIT_RSM: /* 17 Guest software attempted to execute RSM in SMM. */
4006 AssertFailed(); /* can't happen. */
4007 rc = VERR_EM_INTERPRETER;
4008 break;
4009
4010 case VMX_EXIT_VMCALL: /* 18 Guest software executed VMCALL. */
4011 case VMX_EXIT_VMCLEAR: /* 19 Guest software executed VMCLEAR. */
4012 case VMX_EXIT_VMLAUNCH: /* 20 Guest software executed VMLAUNCH. */
4013 case VMX_EXIT_VMPTRLD: /* 21 Guest software executed VMPTRLD. */
4014 case VMX_EXIT_VMPTRST: /* 22 Guest software executed VMPTRST. */
4015 case VMX_EXIT_VMREAD: /* 23 Guest software executed VMREAD. */
4016 case VMX_EXIT_VMRESUME: /* 24 Guest software executed VMRESUME. */
4017 case VMX_EXIT_VMWRITE: /* 25 Guest software executed VMWRITE. */
4018 case VMX_EXIT_VMXOFF: /* 26 Guest software executed VMXOFF. */
4019 case VMX_EXIT_VMXON: /* 27 Guest software executed VMXON. */
4020 /** @todo inject #UD immediately */
4021 rc = VERR_EM_INTERPRETER;
4022 break;
4023
4024 case VMX_EXIT_CPUID: /* 10 Guest software attempted to execute CPUID. */
4025 case VMX_EXIT_RDTSC: /* 16 Guest software attempted to execute RDTSC. */
4026 case VMX_EXIT_INVPG: /* 14 Guest software attempted to execute INVPG. */
4027 case VMX_EXIT_CRX_MOVE: /* 28 Control-register accesses. */
4028 case VMX_EXIT_DRX_MOVE: /* 29 Debug-register accesses. */
4029 case VMX_EXIT_PORT_IO: /* 30 I/O instruction. */
4030 case VMX_EXIT_RDPMC: /* 15 Guest software attempted to execute RDPMC. */
4031 /* already handled above */
4032 AssertMsg( rc == VINF_PGM_CHANGE_MODE
4033 || rc == VINF_EM_RAW_INTERRUPT
4034 || rc == VERR_EM_INTERPRETER
4035 || rc == VINF_EM_RAW_EMULATE_INSTR
4036 || rc == VINF_PGM_SYNC_CR3
4037 || rc == VINF_IOM_HC_IOPORT_READ
4038 || rc == VINF_IOM_HC_IOPORT_WRITE
4039 || rc == VINF_EM_RAW_GUEST_TRAP
4040 || rc == VINF_TRPM_XCPT_DISPATCHED
4041 || rc == VINF_EM_RESCHEDULE_REM,
4042 ("rc = %d\n", rc));
4043 break;
4044
4045 case VMX_EXIT_TPR: /* 43 TPR below threshold. Guest software executed MOV to CR8. */
4046 case VMX_EXIT_APIC_ACCESS: /* 44 APIC access. Guest software attempted to access memory at a physical address on the APIC-access page. */
4047 case VMX_EXIT_RDMSR: /* 31 RDMSR. Guest software attempted to execute RDMSR. */
4048 case VMX_EXIT_WRMSR: /* 32 WRMSR. Guest software attempted to execute WRMSR. */
4049 case VMX_EXIT_MONITOR: /* 39 Guest software attempted to execute MONITOR. */
4050 case VMX_EXIT_PAUSE: /* 40 Guest software attempted to execute PAUSE. */
4051 /* Note: If we decide to emulate them here, then we must sync the MSRs that could have been changed (sysenter, fs/gs base)!!! */
4052 rc = VERR_EM_INTERPRETER;
4053 break;
4054
4055 case VMX_EXIT_IRQ_WINDOW: /* 7 Interrupt window. */
4056 Assert(rc == VINF_EM_RAW_INTERRUPT);
4057 break;
4058
4059 case VMX_EXIT_ERR_INVALID_GUEST_STATE: /* 33 VM-entry failure due to invalid guest state. */
4060 {
4061#ifdef VBOX_STRICT
4062 RTCCUINTREG val2 = 0;
4063
4064 Log(("VMX_EXIT_ERR_INVALID_GUEST_STATE\n"));
4065
4066 VMXReadVMCS(VMX_VMCS64_GUEST_RIP, &val2);
4067 Log(("Old eip %RGv new %RGv\n", (RTGCPTR)pCtx->rip, (RTGCPTR)val2));
4068
4069 VMXReadVMCS(VMX_VMCS64_GUEST_CR0, &val2);
4070 Log(("VMX_VMCS_GUEST_CR0 %RX64\n", (uint64_t)val2));
4071
4072 VMXReadVMCS(VMX_VMCS64_GUEST_CR3, &val2);
4073 Log(("VMX_VMCS_GUEST_CR3 %RX64\n", (uint64_t)val2));
4074
4075 VMXReadVMCS(VMX_VMCS64_GUEST_CR4, &val2);
4076 Log(("VMX_VMCS_GUEST_CR4 %RX64\n", (uint64_t)val2));
4077
4078 VMXReadVMCS(VMX_VMCS_GUEST_RFLAGS, &val2);
4079 Log(("VMX_VMCS_GUEST_RFLAGS %08x\n", val2));
4080
4081 VMX_LOG_SELREG(CS, "CS", val2);
4082 VMX_LOG_SELREG(DS, "DS", val2);
4083 VMX_LOG_SELREG(ES, "ES", val2);
4084 VMX_LOG_SELREG(FS, "FS", val2);
4085 VMX_LOG_SELREG(GS, "GS", val2);
4086 VMX_LOG_SELREG(SS, "SS", val2);
4087 VMX_LOG_SELREG(TR, "TR", val2);
4088 VMX_LOG_SELREG(LDTR, "LDTR", val2);
4089
4090 VMXReadVMCS(VMX_VMCS64_GUEST_GDTR_BASE, &val2);
4091 Log(("VMX_VMCS_GUEST_GDTR_BASE %RX64\n", (uint64_t)val2));
4092 VMXReadVMCS(VMX_VMCS64_GUEST_IDTR_BASE, &val2);
4093 Log(("VMX_VMCS_GUEST_IDTR_BASE %RX64\n", (uint64_t)val2));
4094#endif /* VBOX_STRICT */
4095 rc = VERR_VMX_INVALID_GUEST_STATE;
4096 break;
4097 }
4098
4099 case VMX_EXIT_ERR_MSR_LOAD: /* 34 VM-entry failure due to MSR loading. */
4100 case VMX_EXIT_ERR_MACHINE_CHECK: /* 41 VM-entry failure due to machine-check. */
4101 default:
4102 rc = VERR_VMX_UNEXPECTED_EXIT_CODE;
4103 AssertMsgFailed(("Unexpected exit code %d\n", exitReason)); /* Can't happen. */
4104 break;
4105
4106 }
4107end:
4108
4109 /* Signal changes for the recompiler. */
4110 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR | CPUM_CHANGED_LDTR | CPUM_CHANGED_GDTR | CPUM_CHANGED_IDTR | CPUM_CHANGED_TR | CPUM_CHANGED_HIDDEN_SEL_REGS);
4111
4112 /* If we executed vmlaunch/vmresume and an external irq was pending, then we don't have to do a full sync the next time. */
4113 if ( exitReason == VMX_EXIT_EXTERNAL_IRQ
4114 && !VMX_EXIT_INTERRUPTION_INFO_VALID(intInfo))
4115 {
4116 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatPendingHostIrq);
4117 /* On the next entry we'll only sync the host context. */
4118 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_HOST_CONTEXT;
4119 }
4120 else
4121 {
4122 /* On the next entry we'll sync everything. */
4123 /** @todo we can do better than this */
4124 /* Not in the VINF_PGM_CHANGE_MODE though! */
4125 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_ALL;
4126 }
4127
4128 /* translate into a less severe return code */
4129 if (rc == VERR_EM_INTERPRETER)
4130 rc = VINF_EM_RAW_EMULATE_INSTR;
4131 else
4132 /* Try to extract more information about what might have gone wrong here. */
4133 if (rc == VERR_VMX_INVALID_VMCS_PTR)
4134 {
4135 VMXGetActivateVMCS(&pVCpu->hwaccm.s.vmx.lasterror.u64VMCSPhys);
4136 pVCpu->hwaccm.s.vmx.lasterror.ulVMCSRevision = *(uint32_t *)pVCpu->hwaccm.s.vmx.pVMCS;
4137 pVCpu->hwaccm.s.vmx.lasterror.idEnteredCpu = pVCpu->hwaccm.s.idEnteredCpu;
4138 pVCpu->hwaccm.s.vmx.lasterror.idCurrentCpu = RTMpCpuId();
4139 }
4140
4141 /* Just set the correct state here instead of trying to catch every goto above. */
4142 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED, VMCPUSTATE_STARTED_EXEC);
4143
4144#ifdef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
4145 /* Restore interrupts if we exitted after disabling them. */
4146 if (uOldEFlags != ~(RTCCUINTREG)0)
4147 ASMSetFlags(uOldEFlags);
4148#endif
4149
4150 STAM_STATS({
4151 if (fStatExit2Started) STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2, y);
4152 else if (fStatEntryStarted) STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatEntry, x);
4153 });
4154 Log2(("X"));
4155 return rc;
4156}
4157
4158
4159/**
4160 * Enters the VT-x session
4161 *
4162 * @returns VBox status code.
4163 * @param pVM The VM to operate on.
4164 * @param pVCpu The VMCPU to operate on.
4165 * @param pCpu CPU info struct
4166 */
4167VMMR0DECL(int) VMXR0Enter(PVM pVM, PVMCPU pVCpu, PHWACCM_CPUINFO pCpu)
4168{
4169 Assert(pVM->hwaccm.s.vmx.fSupported);
4170
4171 unsigned cr4 = ASMGetCR4();
4172 if (!(cr4 & X86_CR4_VMXE))
4173 {
4174 AssertMsgFailed(("X86_CR4_VMXE should be set!\n"));
4175 return VERR_VMX_X86_CR4_VMXE_CLEARED;
4176 }
4177
4178 /* Activate the VM Control Structure. */
4179 int rc = VMXActivateVMCS(pVCpu->hwaccm.s.vmx.pVMCSPhys);
4180 if (RT_FAILURE(rc))
4181 return rc;
4182
4183 pVCpu->hwaccm.s.fResumeVM = false;
4184 return VINF_SUCCESS;
4185}
4186
4187
4188/**
4189 * Leaves the VT-x session
4190 *
4191 * @returns VBox status code.
4192 * @param pVM The VM to operate on.
4193 * @param pVCpu The VMCPU to operate on.
4194 * @param pCtx CPU context
4195 */
4196VMMR0DECL(int) VMXR0Leave(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
4197{
4198 Assert(pVM->hwaccm.s.vmx.fSupported);
4199
4200#ifdef DEBUG
4201 if (CPUMIsHyperDebugStateActive(pVCpu))
4202 {
4203 CPUMR0LoadHostDebugState(pVM, pVCpu);
4204 }
4205 else
4206#endif
4207 /* Save the guest debug state if necessary. */
4208 if (CPUMIsGuestDebugStateActive(pVCpu))
4209 {
4210 CPUMR0SaveGuestDebugState(pVM, pVCpu, pCtx, true /* save DR6 */);
4211
4212 /* Enable drx move intercepts again. */
4213 pVCpu->hwaccm.s.vmx.proc_ctls |= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT;
4214 int rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
4215 AssertRC(rc);
4216
4217 /* Resync the debug registers the next time. */
4218 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_GUEST_DEBUG;
4219 }
4220 else
4221 Assert(pVCpu->hwaccm.s.vmx.proc_ctls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT);
4222
4223 /* Clear VM Control Structure. Marking it inactive, clearing implementation specific data and writing back VMCS data to memory. */
4224 int rc = VMXClearVMCS(pVCpu->hwaccm.s.vmx.pVMCSPhys);
4225 AssertRC(rc);
4226
4227 return VINF_SUCCESS;
4228}
4229
4230/**
4231 * Flush the TLB (EPT)
4232 *
4233 * @returns VBox status code.
4234 * @param pVM The VM to operate on.
4235 * @param pVCpu The VM CPU to operate on.
4236 * @param enmFlush Type of flush
4237 * @param GCPhys Physical address of the page to flush
4238 */
4239static void vmxR0FlushEPT(PVM pVM, PVMCPU pVCpu, VMX_FLUSH enmFlush, RTGCPHYS GCPhys)
4240{
4241 uint64_t descriptor[2];
4242
4243 LogFlow(("vmxR0FlushEPT %d %RGv\n", enmFlush, GCPhys));
4244 Assert(pVM->hwaccm.s.fNestedPaging);
4245 descriptor[0] = pVCpu->hwaccm.s.vmx.GCPhysEPTP;
4246 descriptor[1] = GCPhys;
4247 int rc = VMXR0InvEPT(enmFlush, &descriptor[0]);
4248 AssertRC(rc);
4249}
4250
4251#ifdef HWACCM_VTX_WITH_VPID
4252/**
4253 * Flush the TLB (EPT)
4254 *
4255 * @returns VBox status code.
4256 * @param pVM The VM to operate on.
4257 * @param pVCpu The VM CPU to operate on.
4258 * @param enmFlush Type of flush
4259 * @param GCPtr Virtual address of the page to flush
4260 */
4261static void vmxR0FlushVPID(PVM pVM, PVMCPU pVCpu, VMX_FLUSH enmFlush, RTGCPTR GCPtr)
4262{
4263#if HC_ARCH_BITS == 32
4264 /* If we get a flush in 64 bits guest mode, then force a full TLB flush. Invvpid probably takes only 32 bits addresses. (@todo) */
4265 if ( CPUMIsGuestInLongMode(pVCpu)
4266 && !VMX_IS_64BIT_HOST_MODE())
4267 {
4268 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
4269 }
4270 else
4271#endif
4272 {
4273 uint64_t descriptor[2];
4274
4275 Assert(pVM->hwaccm.s.vmx.fVPID);
4276 descriptor[0] = pVCpu->hwaccm.s.uCurrentASID;
4277 descriptor[1] = GCPtr;
4278 int rc = VMXR0InvVPID(enmFlush, &descriptor[0]);
4279 AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvVPID %x %x %RGv failed with %d\n", enmFlush, pVCpu->hwaccm.s.uCurrentASID, GCPtr, rc));
4280 }
4281}
4282#endif /* HWACCM_VTX_WITH_VPID */
4283
4284/**
4285 * Invalidates a guest page
4286 *
4287 * @returns VBox status code.
4288 * @param pVM The VM to operate on.
4289 * @param pVCpu The VM CPU to operate on.
4290 * @param GCVirt Page to invalidate
4291 */
4292VMMR0DECL(int) VMXR0InvalidatePage(PVM pVM, PVMCPU pVCpu, RTGCPTR GCVirt)
4293{
4294 bool fFlushPending = VMCPU_FF_ISSET(pVCpu, VMCPU_FF_TLB_FLUSH);
4295
4296 Log2(("VMXR0InvalidatePage %RGv\n", GCVirt));
4297
4298 /* Only relevant if we want to use VPID.
4299 * In the nested paging case we still see such calls, but
4300 * can safely ignore them. (e.g. after cr3 updates)
4301 */
4302#ifdef HWACCM_VTX_WITH_VPID
4303 /* Skip it if a TLB flush is already pending. */
4304 if ( !fFlushPending
4305 && pVM->hwaccm.s.vmx.fVPID)
4306 vmxR0FlushVPID(pVM, pVCpu, pVM->hwaccm.s.vmx.enmFlushPage, GCVirt);
4307#endif /* HWACCM_VTX_WITH_VPID */
4308
4309 return VINF_SUCCESS;
4310}
4311
4312/**
4313 * Invalidates a guest page by physical address
4314 *
4315 * NOTE: Assumes the current instruction references this physical page though a virtual address!!
4316 *
4317 * @returns VBox status code.
4318 * @param pVM The VM to operate on.
4319 * @param pVCpu The VM CPU to operate on.
4320 * @param GCPhys Page to invalidate
4321 */
4322VMMR0DECL(int) VMXR0InvalidatePhysPage(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys)
4323{
4324 bool fFlushPending = VMCPU_FF_ISSET(pVCpu, VMCPU_FF_TLB_FLUSH);
4325
4326 Assert(pVM->hwaccm.s.fNestedPaging);
4327
4328 LogFlow(("VMXR0InvalidatePhysPage %RGp\n", GCPhys));
4329
4330 /* Skip it if a TLB flush is already pending. */
4331 if (!fFlushPending)
4332 vmxR0FlushEPT(pVM, pVCpu, pVM->hwaccm.s.vmx.enmFlushPage, GCPhys);
4333
4334 return VINF_SUCCESS;
4335}
4336
4337/**
4338 * Report world switch error and dump some useful debug info
4339 *
4340 * @param pVM The VM to operate on.
4341 * @param pVCpu The VMCPU to operate on.
4342 * @param rc Return code
4343 * @param pCtx Current CPU context (not updated)
4344 */
4345static void VMXR0ReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, int rc, PCPUMCTX pCtx)
4346{
4347 switch (rc)
4348 {
4349 case VERR_VMX_INVALID_VMXON_PTR:
4350 AssertFailed();
4351 break;
4352
4353 case VERR_VMX_UNABLE_TO_START_VM:
4354 case VERR_VMX_UNABLE_TO_RESUME_VM:
4355 {
4356 int rc2;
4357 RTCCUINTREG exitReason, instrError;
4358
4359 rc2 = VMXReadVMCS(VMX_VMCS32_RO_EXIT_REASON, &exitReason);
4360 rc2 |= VMXReadVMCS(VMX_VMCS32_RO_VM_INSTR_ERROR, &instrError);
4361 AssertRC(rc2);
4362 if (rc2 == VINF_SUCCESS)
4363 {
4364 Log(("Unable to start/resume VM for reason: %x. Instruction error %x\n", (uint32_t)exitReason, (uint32_t)instrError));
4365 Log(("Current stack %08x\n", &rc2));
4366
4367 pVCpu->hwaccm.s.vmx.lasterror.ulInstrError = instrError;
4368 pVCpu->hwaccm.s.vmx.lasterror.ulExitReason = exitReason;
4369
4370#ifdef VBOX_STRICT
4371 RTGDTR gdtr;
4372 PCX86DESCHC pDesc;
4373 RTCCUINTREG val;
4374
4375 ASMGetGDTR(&gdtr);
4376
4377 VMXReadVMCS(VMX_VMCS64_GUEST_RIP, &val);
4378 Log(("Old eip %RGv new %RGv\n", (RTGCPTR)pCtx->rip, (RTGCPTR)val));
4379 VMXReadVMCS(VMX_VMCS_CTRL_PIN_EXEC_CONTROLS, &val);
4380 Log(("VMX_VMCS_CTRL_PIN_EXEC_CONTROLS %08x\n", val));
4381 VMXReadVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, &val);
4382 Log(("VMX_VMCS_CTRL_PROC_EXEC_CONTROLS %08x\n", val));
4383 VMXReadVMCS(VMX_VMCS_CTRL_ENTRY_CONTROLS, &val);
4384 Log(("VMX_VMCS_CTRL_ENTRY_CONTROLS %08x\n", val));
4385 VMXReadVMCS(VMX_VMCS_CTRL_EXIT_CONTROLS, &val);
4386 Log(("VMX_VMCS_CTRL_EXIT_CONTROLS %08x\n", val));
4387
4388 VMXReadVMCS(VMX_VMCS_HOST_CR0, &val);
4389 Log(("VMX_VMCS_HOST_CR0 %08x\n", val));
4390
4391 VMXReadVMCS(VMX_VMCS_HOST_CR3, &val);
4392 Log(("VMX_VMCS_HOST_CR3 %08x\n", val));
4393
4394 VMXReadVMCS(VMX_VMCS_HOST_CR4, &val);
4395 Log(("VMX_VMCS_HOST_CR4 %08x\n", val));
4396
4397 VMXReadVMCS(VMX_VMCS16_HOST_FIELD_CS, &val);
4398 Log(("VMX_VMCS_HOST_FIELD_CS %08x\n", val));
4399
4400 VMXReadVMCS(VMX_VMCS_GUEST_RFLAGS, &val);
4401 Log(("VMX_VMCS_GUEST_RFLAGS %08x\n", val));
4402
4403 if (val < gdtr.cbGdt)
4404 {
4405 pDesc = (PCX86DESCHC)(gdtr.pGdt + (val & X86_SEL_MASK));
4406 HWACCMR0DumpDescriptor(pDesc, val, "CS: ");
4407 }
4408
4409 VMXReadVMCS(VMX_VMCS16_HOST_FIELD_DS, &val);
4410 Log(("VMX_VMCS_HOST_FIELD_DS %08x\n", val));
4411 if (val < gdtr.cbGdt)
4412 {
4413 pDesc = (PCX86DESCHC)(gdtr.pGdt + (val & X86_SEL_MASK));
4414 HWACCMR0DumpDescriptor(pDesc, val, "DS: ");
4415 }
4416
4417 VMXReadVMCS(VMX_VMCS16_HOST_FIELD_ES, &val);
4418 Log(("VMX_VMCS_HOST_FIELD_ES %08x\n", val));
4419 if (val < gdtr.cbGdt)
4420 {
4421 pDesc = (PCX86DESCHC)(gdtr.pGdt + (val & X86_SEL_MASK));
4422 HWACCMR0DumpDescriptor(pDesc, val, "ES: ");
4423 }
4424
4425 VMXReadVMCS(VMX_VMCS16_HOST_FIELD_FS, &val);
4426 Log(("VMX_VMCS16_HOST_FIELD_FS %08x\n", val));
4427 if (val < gdtr.cbGdt)
4428 {
4429 pDesc = (PCX86DESCHC)(gdtr.pGdt + (val & X86_SEL_MASK));
4430 HWACCMR0DumpDescriptor(pDesc, val, "FS: ");
4431 }
4432
4433 VMXReadVMCS(VMX_VMCS16_HOST_FIELD_GS, &val);
4434 Log(("VMX_VMCS16_HOST_FIELD_GS %08x\n", val));
4435 if (val < gdtr.cbGdt)
4436 {
4437 pDesc = (PCX86DESCHC)(gdtr.pGdt + (val & X86_SEL_MASK));
4438 HWACCMR0DumpDescriptor(pDesc, val, "GS: ");
4439 }
4440
4441 VMXReadVMCS(VMX_VMCS16_HOST_FIELD_SS, &val);
4442 Log(("VMX_VMCS16_HOST_FIELD_SS %08x\n", val));
4443 if (val < gdtr.cbGdt)
4444 {
4445 pDesc = (PCX86DESCHC)(gdtr.pGdt + (val & X86_SEL_MASK));
4446 HWACCMR0DumpDescriptor(pDesc, val, "SS: ");
4447 }
4448
4449 VMXReadVMCS(VMX_VMCS16_HOST_FIELD_TR, &val);
4450 Log(("VMX_VMCS16_HOST_FIELD_TR %08x\n", val));
4451 if (val < gdtr.cbGdt)
4452 {
4453 pDesc = (PCX86DESCHC)(gdtr.pGdt + (val & X86_SEL_MASK));
4454 HWACCMR0DumpDescriptor(pDesc, val, "TR: ");
4455 }
4456
4457 VMXReadVMCS(VMX_VMCS_HOST_TR_BASE, &val);
4458 Log(("VMX_VMCS_HOST_TR_BASE %RHv\n", val));
4459
4460 VMXReadVMCS(VMX_VMCS_HOST_GDTR_BASE, &val);
4461 Log(("VMX_VMCS_HOST_GDTR_BASE %RHv\n", val));
4462 VMXReadVMCS(VMX_VMCS_HOST_IDTR_BASE, &val);
4463 Log(("VMX_VMCS_HOST_IDTR_BASE %RHv\n", val));
4464
4465 VMXReadVMCS(VMX_VMCS32_HOST_SYSENTER_CS, &val);
4466 Log(("VMX_VMCS_HOST_SYSENTER_CS %08x\n", val));
4467
4468 VMXReadVMCS(VMX_VMCS_HOST_SYSENTER_EIP, &val);
4469 Log(("VMX_VMCS_HOST_SYSENTER_EIP %RHv\n", val));
4470
4471 VMXReadVMCS(VMX_VMCS_HOST_SYSENTER_ESP, &val);
4472 Log(("VMX_VMCS_HOST_SYSENTER_ESP %RHv\n", val));
4473
4474 VMXReadVMCS(VMX_VMCS_HOST_RSP, &val);
4475 Log(("VMX_VMCS_HOST_RSP %RHv\n", val));
4476 VMXReadVMCS(VMX_VMCS_HOST_RIP, &val);
4477 Log(("VMX_VMCS_HOST_RIP %RHv\n", val));
4478
4479# if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4480 if (VMX_IS_64BIT_HOST_MODE())
4481 {
4482 Log(("MSR_K6_EFER = %RX64\n", ASMRdMsr(MSR_K6_EFER)));
4483 Log(("MSR_K6_STAR = %RX64\n", ASMRdMsr(MSR_K6_STAR)));
4484 Log(("MSR_K8_LSTAR = %RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
4485 Log(("MSR_K8_CSTAR = %RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
4486 Log(("MSR_K8_SF_MASK = %RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
4487 }
4488# endif
4489#endif /* VBOX_STRICT */
4490 }
4491 break;
4492 }
4493
4494 default:
4495 /* impossible */
4496 AssertMsgFailed(("%Rrc (%#x)\n", rc, rc));
4497 break;
4498 }
4499}
4500
4501#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4502/**
4503 * Prepares for and executes VMLAUNCH (64 bits guest mode)
4504 *
4505 * @returns VBox status code
4506 * @param fResume vmlauch/vmresume
4507 * @param pCtx Guest context
4508 * @param pCache VMCS cache
4509 * @param pVM The VM to operate on.
4510 * @param pVCpu The VMCPU to operate on.
4511 */
4512DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
4513{
4514 uint32_t aParam[6];
4515 PHWACCM_CPUINFO pCpu;
4516 RTHCPHYS pPageCpuPhys;
4517 int rc;
4518
4519 pCpu = HWACCMR0GetCurrentCpu();
4520 pPageCpuPhys = RTR0MemObjGetPagePhysAddr(pCpu->pMemObj, 0);
4521
4522#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4523 pCache->uPos = 1;
4524 pCache->interPD = PGMGetInterPaeCR3(pVM);
4525 pCache->pSwitcher = (uint64_t)pVM->hwaccm.s.pfnHost32ToGuest64R0;
4526#endif
4527
4528#ifdef DEBUG
4529 pCache->TestIn.pPageCpuPhys = 0;
4530 pCache->TestIn.pVMCSPhys = 0;
4531 pCache->TestIn.pCache = 0;
4532 pCache->TestOut.pVMCSPhys = 0;
4533 pCache->TestOut.pCache = 0;
4534 pCache->TestOut.pCtx = 0;
4535 pCache->TestOut.eflags = 0;
4536#endif
4537
4538 aParam[0] = (uint32_t)(pPageCpuPhys); /* Param 1: VMXON physical address - Lo. */
4539 aParam[1] = (uint32_t)(pPageCpuPhys >> 32); /* Param 1: VMXON physical address - Hi. */
4540 aParam[2] = (uint32_t)(pVCpu->hwaccm.s.vmx.pVMCSPhys); /* Param 2: VMCS physical address - Lo. */
4541 aParam[3] = (uint32_t)(pVCpu->hwaccm.s.vmx.pVMCSPhys >> 32); /* Param 2: VMCS physical address - Hi. */
4542 aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hwaccm.s.vmx.VMCSCache);
4543 aParam[5] = 0;
4544
4545#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4546 pCtx->dr[4] = pVM->hwaccm.s.vmx.pScratchPhys + 16 + 8;
4547 *(uint32_t *)(pVM->hwaccm.s.vmx.pScratch + 16 + 8) = 1;
4548#endif
4549 rc = VMXR0Execute64BitsHandler(pVM, pVCpu, pCtx, pVM->hwaccm.s.pfnVMXGCStartVM64, 6, &aParam[0]);
4550
4551#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4552 Assert(*(uint32_t *)(pVM->hwaccm.s.vmx.pScratch + 16 + 8) == 5);
4553 Assert(pCtx->dr[4] == 10);
4554 *(uint32_t *)(pVM->hwaccm.s.vmx.pScratch + 16 + 8) = 0xff;
4555#endif
4556
4557#ifdef DEBUG
4558 AssertMsg(pCache->TestIn.pPageCpuPhys == pPageCpuPhys, ("%RHp vs %RHp\n", pCache->TestIn.pPageCpuPhys, pPageCpuPhys));
4559 AssertMsg(pCache->TestIn.pVMCSPhys == pVCpu->hwaccm.s.vmx.pVMCSPhys, ("%RHp vs %RHp\n", pCache->TestIn.pVMCSPhys, pVCpu->hwaccm.s.vmx.pVMCSPhys));
4560 AssertMsg(pCache->TestIn.pVMCSPhys == pCache->TestOut.pVMCSPhys, ("%RHp vs %RHp\n", pCache->TestIn.pVMCSPhys, pCache->TestOut.pVMCSPhys));
4561 AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache, pCache->TestOut.pCache));
4562 AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hwaccm.s.vmx.VMCSCache), ("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hwaccm.s.vmx.VMCSCache)));
4563 AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx, pCache->TestOut.pCtx));
4564 Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
4565#endif
4566 return rc;
4567}
4568
4569/**
4570 * Executes the specified handler in 64 mode
4571 *
4572 * @returns VBox status code.
4573 * @param pVM The VM to operate on.
4574 * @param pVCpu The VMCPU to operate on.
4575 * @param pCtx Guest context
4576 * @param pfnHandler RC handler
4577 * @param cbParam Number of parameters
4578 * @param paParam Array of 32 bits parameters
4579 */
4580VMMR0DECL(int) VMXR0Execute64BitsHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, RTRCPTR pfnHandler, uint32_t cbParam, uint32_t *paParam)
4581{
4582 int rc, rc2;
4583 PHWACCM_CPUINFO pCpu;
4584 RTHCPHYS pPageCpuPhys;
4585 RTHCUINTREG uOldEFlags;
4586
4587 /* @todo This code is not guest SMP safe (hyper stack and switchers) */
4588 AssertReturn(pVM->cCpus == 1, VERR_TOO_MANY_CPUS);
4589 AssertReturn(pVM->hwaccm.s.pfnHost32ToGuest64R0, VERR_INTERNAL_ERROR);
4590 Assert(pVCpu->hwaccm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hwaccm.s.vmx.VMCSCache.Write.aField));
4591 Assert(pVCpu->hwaccm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hwaccm.s.vmx.VMCSCache.Read.aField));
4592
4593#ifdef VBOX_STRICT
4594 for (unsigned i=0;i<pVCpu->hwaccm.s.vmx.VMCSCache.Write.cValidEntries;i++)
4595 Assert(vmxR0IsValidWriteField(pVCpu->hwaccm.s.vmx.VMCSCache.Write.aField[i]));
4596
4597 for (unsigned i=0;i<pVCpu->hwaccm.s.vmx.VMCSCache.Read.cValidEntries;i++)
4598 Assert(vmxR0IsValidReadField(pVCpu->hwaccm.s.vmx.VMCSCache.Read.aField[i]));
4599#endif
4600
4601 /* Disable interrupts. */
4602 uOldEFlags = ASMIntDisableFlags();
4603
4604 pCpu = HWACCMR0GetCurrentCpu();
4605 pPageCpuPhys = RTR0MemObjGetPagePhysAddr(pCpu->pMemObj, 0);
4606
4607 /* Clear VM Control Structure. Marking it inactive, clearing implementation specific data and writing back VMCS data to memory. */
4608 VMXClearVMCS(pVCpu->hwaccm.s.vmx.pVMCSPhys);
4609
4610 /* Leave VMX Root Mode. */
4611 VMXDisable();
4612
4613 ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
4614
4615 CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVM));
4616 CPUMSetHyperEIP(pVCpu, pfnHandler);
4617 for (int i=(int)cbParam-1;i>=0;i--)
4618 CPUMPushHyper(pVCpu, paParam[i]);
4619
4620 STAM_PROFILE_ADV_START(&pVCpu->hwaccm.s.StatWorldSwitch3264, z);
4621 /* Call switcher. */
4622 rc = pVM->hwaccm.s.pfnHost32ToGuest64R0(pVM);
4623 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatWorldSwitch3264, z);
4624
4625 /* Make sure the VMX instructions don't cause #UD faults. */
4626 ASMSetCR4(ASMGetCR4() | X86_CR4_VMXE);
4627
4628 /* Enter VMX Root Mode */
4629 rc2 = VMXEnable(pPageCpuPhys);
4630 if (RT_FAILURE(rc2))
4631 {
4632 if (pVM)
4633 VMXR0CheckError(pVM, pVCpu, rc2);
4634 ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
4635 ASMSetFlags(uOldEFlags);
4636 return VERR_VMX_VMXON_FAILED;
4637 }
4638
4639 rc2 = VMXActivateVMCS(pVCpu->hwaccm.s.vmx.pVMCSPhys);
4640 AssertRC(rc2);
4641 Assert(!(ASMGetFlags() & X86_EFL_IF));
4642 ASMSetFlags(uOldEFlags);
4643 return rc;
4644}
4645
4646#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL) */
4647
4648
4649#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
4650/**
4651 * Executes VMWRITE
4652 *
4653 * @returns VBox status code
4654 * @param pVCpu The VMCPU to operate on.
4655 * @param idxField VMCS index
4656 * @param u64Val 16, 32 or 64 bits value
4657 */
4658VMMR0DECL(int) VMXWriteVMCS64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
4659{
4660 int rc;
4661
4662 switch (idxField)
4663 {
4664 case VMX_VMCS_CTRL_TSC_OFFSET_FULL:
4665 case VMX_VMCS_CTRL_IO_BITMAP_A_FULL:
4666 case VMX_VMCS_CTRL_IO_BITMAP_B_FULL:
4667 case VMX_VMCS_CTRL_MSR_BITMAP_FULL:
4668 case VMX_VMCS_CTRL_VMEXIT_MSR_STORE_FULL:
4669 case VMX_VMCS_CTRL_VMEXIT_MSR_LOAD_FULL:
4670 case VMX_VMCS_CTRL_VMENTRY_MSR_LOAD_FULL:
4671 case VMX_VMCS_CTRL_VAPIC_PAGEADDR_FULL:
4672 case VMX_VMCS_CTRL_APIC_ACCESSADDR_FULL:
4673 case VMX_VMCS_GUEST_LINK_PTR_FULL:
4674 case VMX_VMCS_GUEST_PDPTR0_FULL:
4675 case VMX_VMCS_GUEST_PDPTR1_FULL:
4676 case VMX_VMCS_GUEST_PDPTR2_FULL:
4677 case VMX_VMCS_GUEST_PDPTR3_FULL:
4678 case VMX_VMCS_GUEST_DEBUGCTL_FULL:
4679 case VMX_VMCS_GUEST_EFER_FULL:
4680 case VMX_VMCS_CTRL_EPTP_FULL:
4681 /* These fields consist of two parts, which are both writable in 32 bits mode. */
4682 rc = VMXWriteVMCS32(idxField, u64Val);
4683 rc |= VMXWriteVMCS32(idxField + 1, (uint32_t)(u64Val >> 32ULL));
4684 AssertRC(rc);
4685 return rc;
4686
4687 case VMX_VMCS64_GUEST_LDTR_BASE:
4688 case VMX_VMCS64_GUEST_TR_BASE:
4689 case VMX_VMCS64_GUEST_GDTR_BASE:
4690 case VMX_VMCS64_GUEST_IDTR_BASE:
4691 case VMX_VMCS64_GUEST_SYSENTER_EIP:
4692 case VMX_VMCS64_GUEST_SYSENTER_ESP:
4693 case VMX_VMCS64_GUEST_CR0:
4694 case VMX_VMCS64_GUEST_CR4:
4695 case VMX_VMCS64_GUEST_CR3:
4696 case VMX_VMCS64_GUEST_DR7:
4697 case VMX_VMCS64_GUEST_RIP:
4698 case VMX_VMCS64_GUEST_RSP:
4699 case VMX_VMCS64_GUEST_CS_BASE:
4700 case VMX_VMCS64_GUEST_DS_BASE:
4701 case VMX_VMCS64_GUEST_ES_BASE:
4702 case VMX_VMCS64_GUEST_FS_BASE:
4703 case VMX_VMCS64_GUEST_GS_BASE:
4704 case VMX_VMCS64_GUEST_SS_BASE:
4705 /* Queue a 64 bits value as we can't set it in 32 bits host mode. */
4706 if (u64Val >> 32ULL)
4707 rc = VMXWriteCachedVMCSEx(pVCpu, idxField, u64Val);
4708 else
4709 rc = VMXWriteVMCS32(idxField, (uint32_t)u64Val);
4710
4711 return rc;
4712
4713 default:
4714 AssertMsgFailed(("Unexpected field %x\n", idxField));
4715 return VERR_INVALID_PARAMETER;
4716 }
4717}
4718
4719/**
4720 * Cache VMCS writes for performance reasons (Darwin) and for running 64 bits guests on 32 bits hosts.
4721 *
4722 * @param pVCpu The VMCPU to operate on.
4723 * @param idxField VMCS field
4724 * @param u64Val Value
4725 */
4726VMMR0DECL(int) VMXWriteCachedVMCSEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
4727{
4728 PVMCSCACHE pCache = &pVCpu->hwaccm.s.vmx.VMCSCache;
4729
4730 AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1, ("entries=%x\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
4731
4732 /* Make sure there are no duplicates. */
4733 for (unsigned i=0;i<pCache->Write.cValidEntries;i++)
4734 {
4735 if (pCache->Write.aField[i] == idxField)
4736 {
4737 pCache->Write.aFieldVal[i] = u64Val;
4738 return VINF_SUCCESS;
4739 }
4740 }
4741
4742 pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
4743 pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
4744 pCache->Write.cValidEntries++;
4745 return VINF_SUCCESS;
4746}
4747
4748#endif /* HC_ARCH_BITS == 32 && !VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0 */
4749
4750#ifdef VBOX_STRICT
4751static bool vmxR0IsValidReadField(uint32_t idxField)
4752{
4753 switch(idxField)
4754 {
4755 case VMX_VMCS64_GUEST_RIP:
4756 case VMX_VMCS64_GUEST_RSP:
4757 case VMX_VMCS_GUEST_RFLAGS:
4758 case VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE:
4759 case VMX_VMCS_CTRL_CR0_READ_SHADOW:
4760 case VMX_VMCS64_GUEST_CR0:
4761 case VMX_VMCS_CTRL_CR4_READ_SHADOW:
4762 case VMX_VMCS64_GUEST_CR4:
4763 case VMX_VMCS64_GUEST_DR7:
4764 case VMX_VMCS32_GUEST_SYSENTER_CS:
4765 case VMX_VMCS64_GUEST_SYSENTER_EIP:
4766 case VMX_VMCS64_GUEST_SYSENTER_ESP:
4767 case VMX_VMCS32_GUEST_GDTR_LIMIT:
4768 case VMX_VMCS64_GUEST_GDTR_BASE:
4769 case VMX_VMCS32_GUEST_IDTR_LIMIT:
4770 case VMX_VMCS64_GUEST_IDTR_BASE:
4771 case VMX_VMCS16_GUEST_FIELD_CS:
4772 case VMX_VMCS32_GUEST_CS_LIMIT:
4773 case VMX_VMCS64_GUEST_CS_BASE:
4774 case VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS:
4775 case VMX_VMCS16_GUEST_FIELD_DS:
4776 case VMX_VMCS32_GUEST_DS_LIMIT:
4777 case VMX_VMCS64_GUEST_DS_BASE:
4778 case VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS:
4779 case VMX_VMCS16_GUEST_FIELD_ES:
4780 case VMX_VMCS32_GUEST_ES_LIMIT:
4781 case VMX_VMCS64_GUEST_ES_BASE:
4782 case VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS:
4783 case VMX_VMCS16_GUEST_FIELD_FS:
4784 case VMX_VMCS32_GUEST_FS_LIMIT:
4785 case VMX_VMCS64_GUEST_FS_BASE:
4786 case VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS:
4787 case VMX_VMCS16_GUEST_FIELD_GS:
4788 case VMX_VMCS32_GUEST_GS_LIMIT:
4789 case VMX_VMCS64_GUEST_GS_BASE:
4790 case VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS:
4791 case VMX_VMCS16_GUEST_FIELD_SS:
4792 case VMX_VMCS32_GUEST_SS_LIMIT:
4793 case VMX_VMCS64_GUEST_SS_BASE:
4794 case VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS:
4795 case VMX_VMCS16_GUEST_FIELD_LDTR:
4796 case VMX_VMCS32_GUEST_LDTR_LIMIT:
4797 case VMX_VMCS64_GUEST_LDTR_BASE:
4798 case VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS:
4799 case VMX_VMCS16_GUEST_FIELD_TR:
4800 case VMX_VMCS32_GUEST_TR_LIMIT:
4801 case VMX_VMCS64_GUEST_TR_BASE:
4802 case VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS:
4803 case VMX_VMCS32_RO_EXIT_REASON:
4804 case VMX_VMCS32_RO_VM_INSTR_ERROR:
4805 case VMX_VMCS32_RO_EXIT_INSTR_LENGTH:
4806 case VMX_VMCS32_RO_EXIT_INTERRUPTION_ERRCODE:
4807 case VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO:
4808 case VMX_VMCS32_RO_EXIT_INSTR_INFO:
4809 case VMX_VMCS_RO_EXIT_QUALIFICATION:
4810 case VMX_VMCS32_RO_IDT_INFO:
4811 case VMX_VMCS32_RO_IDT_ERRCODE:
4812 case VMX_VMCS64_GUEST_CR3:
4813 case VMX_VMCS_EXIT_PHYS_ADDR_FULL:
4814 return true;
4815 }
4816 return false;
4817}
4818
4819static bool vmxR0IsValidWriteField(uint32_t idxField)
4820{
4821 switch(idxField)
4822 {
4823 case VMX_VMCS64_GUEST_LDTR_BASE:
4824 case VMX_VMCS64_GUEST_TR_BASE:
4825 case VMX_VMCS64_GUEST_GDTR_BASE:
4826 case VMX_VMCS64_GUEST_IDTR_BASE:
4827 case VMX_VMCS64_GUEST_SYSENTER_EIP:
4828 case VMX_VMCS64_GUEST_SYSENTER_ESP:
4829 case VMX_VMCS64_GUEST_CR0:
4830 case VMX_VMCS64_GUEST_CR4:
4831 case VMX_VMCS64_GUEST_CR3:
4832 case VMX_VMCS64_GUEST_DR7:
4833 case VMX_VMCS64_GUEST_RIP:
4834 case VMX_VMCS64_GUEST_RSP:
4835 case VMX_VMCS64_GUEST_CS_BASE:
4836 case VMX_VMCS64_GUEST_DS_BASE:
4837 case VMX_VMCS64_GUEST_ES_BASE:
4838 case VMX_VMCS64_GUEST_FS_BASE:
4839 case VMX_VMCS64_GUEST_GS_BASE:
4840 case VMX_VMCS64_GUEST_SS_BASE:
4841 return true;
4842 }
4843 return false;
4844}
4845
4846#endif
Note: See TracBrowser for help on using the repository browser.

© 2024 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette