VirtualBox

source: vbox/trunk/src/VBox/VMM/VMMR0/HWVMXR0.cpp@ 31646

Last change on this file since 31646 was 31646, checked in by vboxsync, 14 years ago

HWVMXR0.cpp: Made the PGM/MMIO hack work with EPT.
  • Property svn:eol-style set to native
  • Property svn:keywords set to Id
File size: 195.9 KB
Line 
1/* $Id: HWVMXR0.cpp 31646 2010-08-13 12:49:15Z vboxsync $ */
2/** @file
3 * HWACCM VMX - Host Context Ring 0.
4 */
5
6/*
7 * Copyright (C) 2006-2007 Oracle Corporation
8 *
9 * This file is part of VirtualBox Open Source Edition (OSE), as
10 * available from http://www.virtualbox.org. This file is free software;
11 * you can redistribute it and/or modify it under the terms of the GNU
12 * General Public License (GPL) as published by the Free Software
13 * Foundation, in version 2 as it comes in the "COPYING" file of the
14 * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15 * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16 */
17
18
19/*******************************************************************************
20* Header Files *
21*******************************************************************************/
22#define LOG_GROUP LOG_GROUP_HWACCM
23#include <VBox/hwaccm.h>
24#include <VBox/pgm.h>
25#include <VBox/dbgf.h>
26#include <VBox/selm.h>
27#include <VBox/iom.h>
28#include <VBox/rem.h>
29#include <VBox/tm.h>
30#include "HWACCMInternal.h"
31#include <VBox/vm.h>
32#include <VBox/x86.h>
33#include <VBox/pdmapi.h>
34#include <VBox/err.h>
35#include <VBox/log.h>
36#include <iprt/asm-amd64-x86.h>
37#include <iprt/assert.h>
38#include <iprt/param.h>
39#include <iprt/string.h>
40#include <iprt/time.h>
41#ifdef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
42# include <iprt/thread.h>
43#endif
44#include "HWVMXR0.h"
45
46/*******************************************************************************
47* Defined Constants And Macros *
48*******************************************************************************/
49#if defined(RT_ARCH_AMD64)
50# define VMX_IS_64BIT_HOST_MODE() (true)
51#elif defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
52# define VMX_IS_64BIT_HOST_MODE() (g_fVMXIs64bitHost != 0)
53#else
54# define VMX_IS_64BIT_HOST_MODE() (false)
55#endif
56
57/*******************************************************************************
58* Global Variables *
59*******************************************************************************/
60/* IO operation lookup arrays. */
61static uint32_t const g_aIOSize[4] = {1, 2, 0, 4};
62static uint32_t const g_aIOOpAnd[4] = {0xff, 0xffff, 0, 0xffffffff};
63
64#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
65/** See HWACCMR0A.asm. */
66extern "C" uint32_t g_fVMXIs64bitHost;
67#endif
68
69/*******************************************************************************
70* Local Functions *
71*******************************************************************************/
72static void VMXR0ReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, VBOXSTRICTRC rc, PCPUMCTX pCtx);
73static void vmxR0SetupTLBEPT(PVM pVM, PVMCPU pVCpu);
74static void vmxR0SetupTLBVPID(PVM pVM, PVMCPU pVCpu);
75static void vmxR0SetupTLBDummy(PVM pVM, PVMCPU pVCpu);
76static void vmxR0FlushEPT(PVM pVM, PVMCPU pVCpu, VMX_FLUSH enmFlush, RTGCPHYS GCPhys);
77static void vmxR0FlushVPID(PVM pVM, PVMCPU pVCpu, VMX_FLUSH enmFlush, RTGCPTR GCPtr);
78static void vmxR0UpdateExceptionBitmap(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx);
79#ifdef VBOX_STRICT
80static bool vmxR0IsValidReadField(uint32_t idxField);
81static bool vmxR0IsValidWriteField(uint32_t idxField);
82#endif
83static void vmxR0SetMSRPermission(PVMCPU pVCpu, unsigned ulMSR, bool fRead, bool fWrite);
84
85static void VMXR0CheckError(PVM pVM, PVMCPU pVCpu, int rc)
86{
87 if (rc == VERR_VMX_GENERIC)
88 {
89 RTCCUINTREG instrError;
90
91 VMXReadVMCS(VMX_VMCS32_RO_VM_INSTR_ERROR, &instrError);
92 pVCpu->hwaccm.s.vmx.lasterror.ulInstrError = instrError;
93 }
94 pVM->hwaccm.s.lLastError = rc;
95}
96
97/**
98 * Sets up and activates VT-x on the current CPU
99 *
100 * @returns VBox status code.
101 * @param pCpu CPU info struct
102 * @param pVM The VM to operate on. (can be NULL after a resume!!)
103 * @param pvPageCpu Pointer to the global cpu page
104 * @param pPageCpuPhys Physical address of the global cpu page
105 */
106VMMR0DECL(int) VMXR0EnableCpu(PHWACCM_CPUINFO pCpu, PVM pVM, void *pvPageCpu, RTHCPHYS pPageCpuPhys)
107{
108 AssertReturn(pPageCpuPhys, VERR_INVALID_PARAMETER);
109 AssertReturn(pvPageCpu, VERR_INVALID_PARAMETER);
110
111 if (pVM)
112 {
113 /* Set revision dword at the beginning of the VMXON structure. */
114 *(uint32_t *)pvPageCpu = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hwaccm.s.vmx.msr.vmx_basic_info);
115 }
116
117 /** @todo we should unmap the two pages from the virtual address space in order to prevent accidental corruption.
118 * (which can have very bad consequences!!!)
119 */
120
121 if (ASMGetCR4() & X86_CR4_VMXE)
122 return VERR_VMX_IN_VMX_ROOT_MODE;
123
124 /* Make sure the VMX instructions don't cause #UD faults. */
125 ASMSetCR4(ASMGetCR4() | X86_CR4_VMXE);
126
127 /* Enter VMX Root Mode */
128 int rc = VMXEnable(pPageCpuPhys);
129 if (RT_FAILURE(rc))
130 {
131 ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
132 return VERR_VMX_VMXON_FAILED;
133 }
134 return VINF_SUCCESS;
135}
136
137/**
138 * Deactivates VT-x on the current CPU
139 *
140 * @returns VBox status code.
141 * @param pCpu CPU info struct
142 * @param pvPageCpu Pointer to the global cpu page
143 * @param pPageCpuPhys Physical address of the global cpu page
144 */
145VMMR0DECL(int) VMXR0DisableCpu(PHWACCM_CPUINFO pCpu, void *pvPageCpu, RTHCPHYS pPageCpuPhys)
146{
147 AssertReturn(pPageCpuPhys, VERR_INVALID_PARAMETER);
148 AssertReturn(pvPageCpu, VERR_INVALID_PARAMETER);
149
150 /* If we're somehow not in VMX root mode, then we shouldn't dare leaving it. */
151 if (!(ASMGetCR4() & X86_CR4_VMXE))
152 return VERR_VMX_NOT_IN_VMX_ROOT_MODE;
153
154 /* Leave VMX Root Mode. */
155 VMXDisable();
156
157 /* And clear the X86_CR4_VMXE bit */
158 ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
159 return VINF_SUCCESS;
160}
161
162/**
163 * Does Ring-0 per VM VT-x init.
164 *
165 * @returns VBox status code.
166 * @param pVM The VM to operate on.
167 */
168VMMR0DECL(int) VMXR0InitVM(PVM pVM)
169{
170 int rc;
171
172#ifdef LOG_ENABLED
173 SUPR0Printf("VMXR0InitVM %x\n", pVM);
174#endif
175
176 pVM->hwaccm.s.vmx.pMemObjAPIC = NIL_RTR0MEMOBJ;
177
178 if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
179 {
180 /* Allocate one page for the APIC physical page (serves for filtering accesses). */
181 rc = RTR0MemObjAllocCont(&pVM->hwaccm.s.vmx.pMemObjAPIC, 1 << PAGE_SHIFT, true /* executable R0 mapping */);
182 AssertRC(rc);
183 if (RT_FAILURE(rc))
184 return rc;
185
186 pVM->hwaccm.s.vmx.pAPIC = (uint8_t *)RTR0MemObjAddress(pVM->hwaccm.s.vmx.pMemObjAPIC);
187 pVM->hwaccm.s.vmx.pAPICPhys = RTR0MemObjGetPagePhysAddr(pVM->hwaccm.s.vmx.pMemObjAPIC, 0);
188 ASMMemZero32(pVM->hwaccm.s.vmx.pAPIC, PAGE_SIZE);
189 }
190 else
191 {
192 pVM->hwaccm.s.vmx.pMemObjAPIC = 0;
193 pVM->hwaccm.s.vmx.pAPIC = 0;
194 pVM->hwaccm.s.vmx.pAPICPhys = 0;
195 }
196
197#ifdef VBOX_WITH_CRASHDUMP_MAGIC
198 {
199 rc = RTR0MemObjAllocCont(&pVM->hwaccm.s.vmx.pMemObjScratch, 1 << PAGE_SHIFT, true /* executable R0 mapping */);
200 AssertRC(rc);
201 if (RT_FAILURE(rc))
202 return rc;
203
204 pVM->hwaccm.s.vmx.pScratch = (uint8_t *)RTR0MemObjAddress(pVM->hwaccm.s.vmx.pMemObjScratch);
205 pVM->hwaccm.s.vmx.pScratchPhys = RTR0MemObjGetPagePhysAddr(pVM->hwaccm.s.vmx.pMemObjScratch, 0);
206
207 ASMMemZero32(pVM->hwaccm.s.vmx.pScratch, PAGE_SIZE);
208 strcpy((char *)pVM->hwaccm.s.vmx.pScratch, "SCRATCH Magic");
209 *(uint64_t *)(pVM->hwaccm.s.vmx.pScratch + 16) = UINT64_C(0xDEADBEEFDEADBEEF);
210 }
211#endif
212
213 /* Allocate VMCBs for all guest CPUs. */
214 for (VMCPUID i = 0; i < pVM->cCpus; i++)
215 {
216 PVMCPU pVCpu = &pVM->aCpus[i];
217
218 pVCpu->hwaccm.s.vmx.pMemObjVMCS = NIL_RTR0MEMOBJ;
219
220 /* Allocate one page for the VM control structure (VMCS). */
221 rc = RTR0MemObjAllocCont(&pVCpu->hwaccm.s.vmx.pMemObjVMCS, 1 << PAGE_SHIFT, true /* executable R0 mapping */);
222 AssertRC(rc);
223 if (RT_FAILURE(rc))
224 return rc;
225
226 pVCpu->hwaccm.s.vmx.pVMCS = RTR0MemObjAddress(pVCpu->hwaccm.s.vmx.pMemObjVMCS);
227 pVCpu->hwaccm.s.vmx.pVMCSPhys = RTR0MemObjGetPagePhysAddr(pVCpu->hwaccm.s.vmx.pMemObjVMCS, 0);
228 ASMMemZero32(pVCpu->hwaccm.s.vmx.pVMCS, PAGE_SIZE);
229
230 pVCpu->hwaccm.s.vmx.cr0_mask = 0;
231 pVCpu->hwaccm.s.vmx.cr4_mask = 0;
232
233 /* Allocate one page for the virtual APIC page for TPR caching. */
234 rc = RTR0MemObjAllocCont(&pVCpu->hwaccm.s.vmx.pMemObjVAPIC, 1 << PAGE_SHIFT, true /* executable R0 mapping */);
235 AssertRC(rc);
236 if (RT_FAILURE(rc))
237 return rc;
238
239 pVCpu->hwaccm.s.vmx.pVAPIC = (uint8_t *)RTR0MemObjAddress(pVCpu->hwaccm.s.vmx.pMemObjVAPIC);
240 pVCpu->hwaccm.s.vmx.pVAPICPhys = RTR0MemObjGetPagePhysAddr(pVCpu->hwaccm.s.vmx.pMemObjVAPIC, 0);
241 ASMMemZero32(pVCpu->hwaccm.s.vmx.pVAPIC, PAGE_SIZE);
242
243 /* Allocate the MSR bitmap if this feature is supported. */
244 if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
245 {
246 rc = RTR0MemObjAllocCont(&pVCpu->hwaccm.s.vmx.pMemObjMSRBitmap, 1 << PAGE_SHIFT, true /* executable R0 mapping */);
247 AssertRC(rc);
248 if (RT_FAILURE(rc))
249 return rc;
250
251 pVCpu->hwaccm.s.vmx.pMSRBitmap = (uint8_t *)RTR0MemObjAddress(pVCpu->hwaccm.s.vmx.pMemObjMSRBitmap);
252 pVCpu->hwaccm.s.vmx.pMSRBitmapPhys = RTR0MemObjGetPagePhysAddr(pVCpu->hwaccm.s.vmx.pMemObjMSRBitmap, 0);
253 memset(pVCpu->hwaccm.s.vmx.pMSRBitmap, 0xff, PAGE_SIZE);
254 }
255
256#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
257 /* Allocate one page for the guest MSR load area (for preloading guest MSRs during the world switch). */
258 rc = RTR0MemObjAllocCont(&pVCpu->hwaccm.s.vmx.pMemObjGuestMSR, 1 << PAGE_SHIFT, true /* executable R0 mapping */);
259 AssertRC(rc);
260 if (RT_FAILURE(rc))
261 return rc;
262
263 pVCpu->hwaccm.s.vmx.pGuestMSR = (uint8_t *)RTR0MemObjAddress(pVCpu->hwaccm.s.vmx.pMemObjGuestMSR);
264 pVCpu->hwaccm.s.vmx.pGuestMSRPhys = RTR0MemObjGetPagePhysAddr(pVCpu->hwaccm.s.vmx.pMemObjGuestMSR, 0);
265 memset(pVCpu->hwaccm.s.vmx.pGuestMSR, 0, PAGE_SIZE);
266
267 /* Allocate one page for the host MSR load area (for restoring host MSRs after the world switch back). */
268 rc = RTR0MemObjAllocCont(&pVCpu->hwaccm.s.vmx.pMemObjHostMSR, 1 << PAGE_SHIFT, true /* executable R0 mapping */);
269 AssertRC(rc);
270 if (RT_FAILURE(rc))
271 return rc;
272
273 pVCpu->hwaccm.s.vmx.pHostMSR = (uint8_t *)RTR0MemObjAddress(pVCpu->hwaccm.s.vmx.pMemObjHostMSR);
274 pVCpu->hwaccm.s.vmx.pHostMSRPhys = RTR0MemObjGetPagePhysAddr(pVCpu->hwaccm.s.vmx.pMemObjHostMSR, 0);
275 memset(pVCpu->hwaccm.s.vmx.pHostMSR, 0, PAGE_SIZE);
276#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
277
278 /* Current guest paging mode. */
279 pVCpu->hwaccm.s.vmx.enmLastSeenGuestMode = PGMMODE_REAL;
280
281#ifdef LOG_ENABLED
282 SUPR0Printf("VMXR0InitVM %x VMCS=%x (%x)\n", pVM, pVCpu->hwaccm.s.vmx.pVMCS, (uint32_t)pVCpu->hwaccm.s.vmx.pVMCSPhys);
283#endif
284 }
285
286 return VINF_SUCCESS;
287}
288
289/**
290 * Does Ring-0 per VM VT-x termination.
291 *
292 * @returns VBox status code.
293 * @param pVM The VM to operate on.
294 */
295VMMR0DECL(int) VMXR0TermVM(PVM pVM)
296{
297 for (VMCPUID i = 0; i < pVM->cCpus; i++)
298 {
299 PVMCPU pVCpu = &pVM->aCpus[i];
300
301 if (pVCpu->hwaccm.s.vmx.pMemObjVMCS != NIL_RTR0MEMOBJ)
302 {
303 RTR0MemObjFree(pVCpu->hwaccm.s.vmx.pMemObjVMCS, false);
304 pVCpu->hwaccm.s.vmx.pMemObjVMCS = NIL_RTR0MEMOBJ;
305 pVCpu->hwaccm.s.vmx.pVMCS = 0;
306 pVCpu->hwaccm.s.vmx.pVMCSPhys = 0;
307 }
308 if (pVCpu->hwaccm.s.vmx.pMemObjVAPIC != NIL_RTR0MEMOBJ)
309 {
310 RTR0MemObjFree(pVCpu->hwaccm.s.vmx.pMemObjVAPIC, false);
311 pVCpu->hwaccm.s.vmx.pMemObjVAPIC = NIL_RTR0MEMOBJ;
312 pVCpu->hwaccm.s.vmx.pVAPIC = 0;
313 pVCpu->hwaccm.s.vmx.pVAPICPhys = 0;
314 }
315 if (pVCpu->hwaccm.s.vmx.pMemObjMSRBitmap != NIL_RTR0MEMOBJ)
316 {
317 RTR0MemObjFree(pVCpu->hwaccm.s.vmx.pMemObjMSRBitmap, false);
318 pVCpu->hwaccm.s.vmx.pMemObjMSRBitmap = NIL_RTR0MEMOBJ;
319 pVCpu->hwaccm.s.vmx.pMSRBitmap = 0;
320 pVCpu->hwaccm.s.vmx.pMSRBitmapPhys = 0;
321 }
322#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
323 if (pVCpu->hwaccm.s.vmx.pMemObjHostMSR != NIL_RTR0MEMOBJ)
324 {
325 RTR0MemObjFree(pVCpu->hwaccm.s.vmx.pMemObjHostMSR, false);
326 pVCpu->hwaccm.s.vmx.pMemObjHostMSR = NIL_RTR0MEMOBJ;
327 pVCpu->hwaccm.s.vmx.pHostMSR = 0;
328 pVCpu->hwaccm.s.vmx.pHostMSRPhys = 0;
329 }
330 if (pVCpu->hwaccm.s.vmx.pMemObjGuestMSR != NIL_RTR0MEMOBJ)
331 {
332 RTR0MemObjFree(pVCpu->hwaccm.s.vmx.pMemObjGuestMSR, false);
333 pVCpu->hwaccm.s.vmx.pMemObjGuestMSR = NIL_RTR0MEMOBJ;
334 pVCpu->hwaccm.s.vmx.pGuestMSR = 0;
335 pVCpu->hwaccm.s.vmx.pGuestMSRPhys = 0;
336 }
337#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
338 }
339 if (pVM->hwaccm.s.vmx.pMemObjAPIC != NIL_RTR0MEMOBJ)
340 {
341 RTR0MemObjFree(pVM->hwaccm.s.vmx.pMemObjAPIC, false);
342 pVM->hwaccm.s.vmx.pMemObjAPIC = NIL_RTR0MEMOBJ;
343 pVM->hwaccm.s.vmx.pAPIC = 0;
344 pVM->hwaccm.s.vmx.pAPICPhys = 0;
345 }
346#ifdef VBOX_WITH_CRASHDUMP_MAGIC
347 if (pVM->hwaccm.s.vmx.pMemObjScratch != NIL_RTR0MEMOBJ)
348 {
349 ASMMemZero32(pVM->hwaccm.s.vmx.pScratch, PAGE_SIZE);
350 RTR0MemObjFree(pVM->hwaccm.s.vmx.pMemObjScratch, false);
351 pVM->hwaccm.s.vmx.pMemObjScratch = NIL_RTR0MEMOBJ;
352 pVM->hwaccm.s.vmx.pScratch = 0;
353 pVM->hwaccm.s.vmx.pScratchPhys = 0;
354 }
355#endif
356 return VINF_SUCCESS;
357}
358
359/**
360 * Sets up VT-x for the specified VM
361 *
362 * @returns VBox status code.
363 * @param pVM The VM to operate on.
364 */
365VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
366{
367 int rc = VINF_SUCCESS;
368 uint32_t val;
369
370 AssertReturn(pVM, VERR_INVALID_PARAMETER);
371
372 for (VMCPUID i = 0; i < pVM->cCpus; i++)
373 {
374 PVMCPU pVCpu = &pVM->aCpus[i];
375
376 Assert(pVCpu->hwaccm.s.vmx.pVMCS);
377
378 /* Set revision dword at the beginning of the VMCS structure. */
379 *(uint32_t *)pVCpu->hwaccm.s.vmx.pVMCS = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hwaccm.s.vmx.msr.vmx_basic_info);
380
381 /* Clear VM Control Structure. */
382 Log(("pVMCSPhys = %RHp\n", pVCpu->hwaccm.s.vmx.pVMCSPhys));
383 rc = VMXClearVMCS(pVCpu->hwaccm.s.vmx.pVMCSPhys);
384 if (RT_FAILURE(rc))
385 goto vmx_end;
386
387 /* Activate the VM Control Structure. */
388 rc = VMXActivateVMCS(pVCpu->hwaccm.s.vmx.pVMCSPhys);
389 if (RT_FAILURE(rc))
390 goto vmx_end;
391
392 /* VMX_VMCS_CTRL_PIN_EXEC_CONTROLS
393 * Set required bits to one and zero according to the MSR capabilities.
394 */
395 val = pVM->hwaccm.s.vmx.msr.vmx_pin_ctls.n.disallowed0;
396 /* External and non-maskable interrupts cause VM-exits. */
397 val = val | VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_EXT_INT_EXIT | VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_NMI_EXIT;
398 val &= pVM->hwaccm.s.vmx.msr.vmx_pin_ctls.n.allowed1;
399
400 rc = VMXWriteVMCS(VMX_VMCS_CTRL_PIN_EXEC_CONTROLS, val);
401 AssertRC(rc);
402
403 /* VMX_VMCS_CTRL_PROC_EXEC_CONTROLS
404 * Set required bits to one and zero according to the MSR capabilities.
405 */
406 val = pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.disallowed0;
407 /* Program which event cause VM-exits and which features we want to use. */
408 val = val | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_HLT_EXIT
409 | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_TSC_OFFSET
410 | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT
411 | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_UNCOND_IO_EXIT
412 | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDPMC_EXIT
413 | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_EXIT
414 | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MWAIT_EXIT; /* don't execute mwait or else we'll idle inside the guest (host thinks the cpu load is high) */
415
416 /* Without nested paging we should intercept invlpg and cr3 mov instructions. */
417 if (!pVM->hwaccm.s.fNestedPaging)
418 val |= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INVLPG_EXIT
419 | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
420 | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT;
421
422 /* Note: VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MWAIT_EXIT might cause a vmlaunch failure with an invalid control fields error. (combined with some other exit reasons) */
423 if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
424 {
425 /* CR8 reads from the APIC shadow page; writes cause an exit is they lower the TPR below the threshold */
426 val |= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW;
427 Assert(pVM->hwaccm.s.vmx.pAPIC);
428 }
429 else
430 /* Exit on CR8 reads & writes in case the TPR shadow feature isn't present. */
431 val |= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR8_STORE_EXIT | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR8_LOAD_EXIT;
432
433 if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
434 {
435 Assert(pVCpu->hwaccm.s.vmx.pMSRBitmapPhys);
436 val |= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS;
437 }
438
439 /* We will use the secondary control if it's present. */
440 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
441
442 /* Mask away the bits that the CPU doesn't support */
443 /** @todo make sure they don't conflict with the above requirements. */
444 val &= pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1;
445 pVCpu->hwaccm.s.vmx.proc_ctls = val;
446
447 rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, val);
448 AssertRC(rc);
449
450 if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
451 {
452 /* VMX_VMCS_CTRL_PROC_EXEC_CONTROLS2
453 * Set required bits to one and zero according to the MSR capabilities.
454 */
455 val = pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2.n.disallowed0;
456 val |= VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT;
457
458#ifdef HWACCM_VTX_WITH_EPT
459 if (pVM->hwaccm.s.fNestedPaging)
460 val |= VMX_VMCS_CTRL_PROC_EXEC2_EPT;
461#endif /* HWACCM_VTX_WITH_EPT */
462#ifdef HWACCM_VTX_WITH_VPID
463 else
464 if (pVM->hwaccm.s.vmx.fVPID)
465 val |= VMX_VMCS_CTRL_PROC_EXEC2_VPID;
466#endif /* HWACCM_VTX_WITH_VPID */
467
468 if (pVM->hwaccm.s.fHasIoApic)
469 val |= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC;
470
471 if (pVM->hwaccm.s.vmx.fUnrestrictedGuest)
472 val |= VMX_VMCS_CTRL_PROC_EXEC2_REAL_MODE;
473
474 /* Mask away the bits that the CPU doesn't support */
475 /** @todo make sure they don't conflict with the above requirements. */
476 val &= pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2.n.allowed1;
477 pVCpu->hwaccm.s.vmx.proc_ctls2 = val;
478 rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS2, val);
479 AssertRC(rc);
480 }
481
482 /* VMX_VMCS_CTRL_CR3_TARGET_COUNT
483 * Set required bits to one and zero according to the MSR capabilities.
484 */
485 rc = VMXWriteVMCS(VMX_VMCS_CTRL_CR3_TARGET_COUNT, 0);
486 AssertRC(rc);
487
488 /* Forward all exception except #NM & #PF to the guest.
489 * We always need to check pagefaults since our shadow page table can be out of sync.
490 * And we always lazily sync the FPU & XMM state.
491 */
492
493 /** @todo Possible optimization:
494 * Keep the FPU and XMM state current in the EM thread. That way there's no need to
495 * lazily sync anything, but the downside is that we can't use the FPU stack or XMM
496 * registers ourselves of course.
497 *
498 * Note: only possible if the current state is actually ours (X86_CR0_TS flag)
499 */
500
501 /* Don't filter page faults; all of them should cause a switch. */
502 rc = VMXWriteVMCS(VMX_VMCS_CTRL_PAGEFAULT_ERROR_MASK, 0);
503 rc |= VMXWriteVMCS(VMX_VMCS_CTRL_PAGEFAULT_ERROR_MATCH, 0);
504 AssertRC(rc);
505
506 /* Init TSC offset to zero. */
507 rc = VMXWriteVMCS64(VMX_VMCS_CTRL_TSC_OFFSET_FULL, 0);
508 AssertRC(rc);
509
510 rc = VMXWriteVMCS64(VMX_VMCS_CTRL_IO_BITMAP_A_FULL, 0);
511 AssertRC(rc);
512
513 rc = VMXWriteVMCS64(VMX_VMCS_CTRL_IO_BITMAP_B_FULL, 0);
514 AssertRC(rc);
515
516 /* Set the MSR bitmap address. */
517 if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
518 {
519 Assert(pVCpu->hwaccm.s.vmx.pMSRBitmapPhys);
520
521 rc = VMXWriteVMCS64(VMX_VMCS_CTRL_MSR_BITMAP_FULL, pVCpu->hwaccm.s.vmx.pMSRBitmapPhys);
522 AssertRC(rc);
523
524 /* Allow the guest to directly modify these MSRs; they are restored and saved automatically. */
525 vmxR0SetMSRPermission(pVCpu, MSR_IA32_SYSENTER_CS, true, true);
526 vmxR0SetMSRPermission(pVCpu, MSR_IA32_SYSENTER_ESP, true, true);
527 vmxR0SetMSRPermission(pVCpu, MSR_IA32_SYSENTER_EIP, true, true);
528 vmxR0SetMSRPermission(pVCpu, MSR_K8_LSTAR, true, true);
529 vmxR0SetMSRPermission(pVCpu, MSR_K6_STAR, true, true);
530 vmxR0SetMSRPermission(pVCpu, MSR_K8_SF_MASK, true, true);
531 vmxR0SetMSRPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, true, true);
532 vmxR0SetMSRPermission(pVCpu, MSR_K8_GS_BASE, true, true);
533 vmxR0SetMSRPermission(pVCpu, MSR_K8_FS_BASE, true, true);
534 }
535
536#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
537 /* Set the guest & host MSR load/store physical addresses. */
538 Assert(pVCpu->hwaccm.s.vmx.pGuestMSRPhys);
539 rc = VMXWriteVMCS64(VMX_VMCS_CTRL_VMENTRY_MSR_LOAD_FULL, pVCpu->hwaccm.s.vmx.pGuestMSRPhys);
540 AssertRC(rc);
541 rc = VMXWriteVMCS64(VMX_VMCS_CTRL_VMEXIT_MSR_STORE_FULL, pVCpu->hwaccm.s.vmx.pGuestMSRPhys);
542 AssertRC(rc);
543
544 Assert(pVCpu->hwaccm.s.vmx.pHostMSRPhys);
545 rc = VMXWriteVMCS64(VMX_VMCS_CTRL_VMEXIT_MSR_LOAD_FULL, pVCpu->hwaccm.s.vmx.pHostMSRPhys);
546 AssertRC(rc);
547#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
548
549 rc = VMXWriteVMCS(VMX_VMCS_CTRL_ENTRY_MSR_LOAD_COUNT, 0);
550 AssertRC(rc);
551
552 rc = VMXWriteVMCS(VMX_VMCS_CTRL_EXIT_MSR_STORE_COUNT, 0);
553 AssertRC(rc);
554
555 if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
556 {
557 Assert(pVM->hwaccm.s.vmx.pMemObjAPIC);
558 /* Optional */
559 rc = VMXWriteVMCS(VMX_VMCS_CTRL_TPR_THRESHOLD, 0);
560 rc |= VMXWriteVMCS64(VMX_VMCS_CTRL_VAPIC_PAGEADDR_FULL, pVCpu->hwaccm.s.vmx.pVAPICPhys);
561
562 if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
563 rc |= VMXWriteVMCS64(VMX_VMCS_CTRL_APIC_ACCESSADDR_FULL, pVM->hwaccm.s.vmx.pAPICPhys);
564
565 AssertRC(rc);
566 }
567
568 /* Set link pointer to -1. Not currently used. */
569 rc = VMXWriteVMCS64(VMX_VMCS_GUEST_LINK_PTR_FULL, 0xFFFFFFFFFFFFFFFFULL);
570 AssertRC(rc);
571
572 /* Clear VM Control Structure. Marking it inactive, clearing implementation specific data and writing back VMCS data to memory. */
573 rc = VMXClearVMCS(pVCpu->hwaccm.s.vmx.pVMCSPhys);
574 AssertRC(rc);
575
576 /* Configure the VMCS read cache. */
577 PVMCSCACHE pCache = &pVCpu->hwaccm.s.vmx.VMCSCache;
578
579 VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_RIP);
580 VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_RSP);
581 VMXSetupCachedReadVMCS(pCache, VMX_VMCS_GUEST_RFLAGS);
582 VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE);
583 VMXSetupCachedReadVMCS(pCache, VMX_VMCS_CTRL_CR0_READ_SHADOW);
584 VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_CR0);
585 VMXSetupCachedReadVMCS(pCache, VMX_VMCS_CTRL_CR4_READ_SHADOW);
586 VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_CR4);
587 VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_DR7);
588 VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_GUEST_SYSENTER_CS);
589 VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_SYSENTER_EIP);
590 VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_SYSENTER_ESP);
591 VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_GUEST_GDTR_LIMIT);
592 VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_GDTR_BASE);
593 VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_GUEST_IDTR_LIMIT);
594 VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_IDTR_BASE);
595
596 VMX_SETUP_SELREG(ES, pCache);
597 VMX_SETUP_SELREG(SS, pCache);
598 VMX_SETUP_SELREG(CS, pCache);
599 VMX_SETUP_SELREG(DS, pCache);
600 VMX_SETUP_SELREG(FS, pCache);
601 VMX_SETUP_SELREG(GS, pCache);
602 VMX_SETUP_SELREG(LDTR, pCache);
603 VMX_SETUP_SELREG(TR, pCache);
604
605 /* Status code VMCS reads. */
606 VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_EXIT_REASON);
607 VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_VM_INSTR_ERROR);
608 VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_EXIT_INSTR_LENGTH);
609 VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_EXIT_INTERRUPTION_ERRCODE);
610 VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO);
611 VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_EXIT_INSTR_INFO);
612 VMXSetupCachedReadVMCS(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
613 VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_IDT_INFO);
614 VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_IDT_ERRCODE);
615
616 if (pVM->hwaccm.s.fNestedPaging)
617 {
618 VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_CR3);
619 VMXSetupCachedReadVMCS(pCache, VMX_VMCS_EXIT_PHYS_ADDR_FULL);
620 pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
621 }
622 else
623 pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
624 } /* for each VMCPU */
625
626 /* Choose the right TLB setup function. */
627 if (pVM->hwaccm.s.fNestedPaging)
628 {
629 pVM->hwaccm.s.vmx.pfnSetupTaggedTLB = vmxR0SetupTLBEPT;
630
631 /* Default values for flushing. */
632 pVM->hwaccm.s.vmx.enmFlushPage = VMX_FLUSH_ALL_CONTEXTS;
633 pVM->hwaccm.s.vmx.enmFlushContext = VMX_FLUSH_ALL_CONTEXTS;
634
635 /* If the capabilities specify we can do more, then make use of it. */
636 if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVEPT_CAPS_INDIV)
637 pVM->hwaccm.s.vmx.enmFlushPage = VMX_FLUSH_PAGE;
638 else
639 if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVEPT_CAPS_CONTEXT)
640 pVM->hwaccm.s.vmx.enmFlushPage = VMX_FLUSH_SINGLE_CONTEXT;
641
642 if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVEPT_CAPS_CONTEXT)
643 pVM->hwaccm.s.vmx.enmFlushContext = VMX_FLUSH_SINGLE_CONTEXT;
644 }
645#ifdef HWACCM_VTX_WITH_VPID
646 else
647 if (pVM->hwaccm.s.vmx.fVPID)
648 {
649 pVM->hwaccm.s.vmx.pfnSetupTaggedTLB = vmxR0SetupTLBVPID;
650
651 /* Default values for flushing. */
652 pVM->hwaccm.s.vmx.enmFlushPage = VMX_FLUSH_ALL_CONTEXTS;
653 pVM->hwaccm.s.vmx.enmFlushContext = VMX_FLUSH_ALL_CONTEXTS;
654
655 /* If the capabilities specify we can do more, then make use of it. */
656 if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVVPID_CAPS_INDIV)
657 pVM->hwaccm.s.vmx.enmFlushPage = VMX_FLUSH_PAGE;
658 else
659 if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVVPID_CAPS_CONTEXT)
660 pVM->hwaccm.s.vmx.enmFlushPage = VMX_FLUSH_SINGLE_CONTEXT;
661
662 if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVVPID_CAPS_CONTEXT)
663 pVM->hwaccm.s.vmx.enmFlushContext = VMX_FLUSH_SINGLE_CONTEXT;
664 }
665#endif /* HWACCM_VTX_WITH_VPID */
666 else
667 pVM->hwaccm.s.vmx.pfnSetupTaggedTLB = vmxR0SetupTLBDummy;
668
669vmx_end:
670 VMXR0CheckError(pVM, &pVM->aCpus[0], rc);
671 return rc;
672}
673
674/**
675 * Sets the permission bits for the specified MSR
676 *
677 * @param pVCpu The VMCPU to operate on.
678 * @param ulMSR MSR value
679 * @param fRead Reading allowed/disallowed
680 * @param fWrite Writing allowed/disallowed
681 */
682static void vmxR0SetMSRPermission(PVMCPU pVCpu, unsigned ulMSR, bool fRead, bool fWrite)
683{
684 unsigned ulBit;
685 uint8_t *pMSRBitmap = (uint8_t *)pVCpu->hwaccm.s.vmx.pMSRBitmap;
686
687 /* Layout:
688 * 0x000 - 0x3ff - Low MSR read bits
689 * 0x400 - 0x7ff - High MSR read bits
690 * 0x800 - 0xbff - Low MSR write bits
691 * 0xc00 - 0xfff - High MSR write bits
692 */
693 if (ulMSR <= 0x00001FFF)
694 {
695 /* Pentium-compatible MSRs */
696 ulBit = ulMSR;
697 }
698 else
699 if ( ulMSR >= 0xC0000000
700 && ulMSR <= 0xC0001FFF)
701 {
702 /* AMD Sixth Generation x86 Processor MSRs */
703 ulBit = (ulMSR - 0xC0000000);
704 pMSRBitmap += 0x400;
705 }
706 else
707 {
708 AssertFailed();
709 return;
710 }
711
712 Assert(ulBit <= 0x1fff);
713 if (fRead)
714 ASMBitClear(pMSRBitmap, ulBit);
715 else
716 ASMBitSet(pMSRBitmap, ulBit);
717
718 if (fWrite)
719 ASMBitClear(pMSRBitmap + 0x800, ulBit);
720 else
721 ASMBitSet(pMSRBitmap + 0x800, ulBit);
722}
723
724
725/**
726 * Injects an event (trap or external interrupt)
727 *
728 * @returns VBox status code. Note that it may return VINF_EM_RESET to
729 * indicate a triple fault when injecting X86_XCPT_DF.
730 *
731 * @param pVM The VM to operate on.
732 * @param pVCpu The VMCPU to operate on.
733 * @param pCtx CPU Context
734 * @param intInfo VMX interrupt info
735 * @param cbInstr Opcode length of faulting instruction
736 * @param errCode Error code (optional)
737 */
738static int VMXR0InjectEvent(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, uint32_t intInfo, uint32_t cbInstr, uint32_t errCode)
739{
740 int rc;
741 uint32_t iGate = VMX_EXIT_INTERRUPTION_INFO_VECTOR(intInfo);
742
743#ifdef VBOX_WITH_STATISTICS
744 STAM_COUNTER_INC(&pVCpu->hwaccm.s.paStatInjectedIrqsR0[iGate & MASK_INJECT_IRQ_STAT]);
745#endif
746
747#ifdef VBOX_STRICT
748 if (iGate == 0xE)
749 LogFlow(("VMXR0InjectEvent: Injecting interrupt %d at %RGv error code=%08x CR2=%RGv intInfo=%08x\n", iGate, (RTGCPTR)pCtx->rip, errCode, pCtx->cr2, intInfo));
750 else
751 if (iGate < 0x20)
752 LogFlow(("VMXR0InjectEvent: Injecting interrupt %d at %RGv error code=%08x\n", iGate, (RTGCPTR)pCtx->rip, errCode));
753 else
754 {
755 LogFlow(("INJ-EI: %x at %RGv\n", iGate, (RTGCPTR)pCtx->rip));
756 Assert(VMX_EXIT_INTERRUPTION_INFO_TYPE(intInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW || !VMCPU_FF_ISSET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
757 Assert(VMX_EXIT_INTERRUPTION_INFO_TYPE(intInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW || pCtx->eflags.u32 & X86_EFL_IF);
758 }
759#endif
760
761 if ( CPUMIsGuestInRealModeEx(pCtx)
762 && pVM->hwaccm.s.vmx.pRealModeTSS)
763 {
764 RTGCPHYS GCPhysHandler;
765 uint16_t offset, ip;
766 RTSEL sel;
767
768 /* Injecting events doesn't work right with real mode emulation.
769 * (#GP if we try to inject external hardware interrupts)
770 * Inject the interrupt or trap directly instead.
771 *
772 * ASSUMES no access handlers for the bits we read or write below (should be safe).
773 */
774 Log(("Manual interrupt/trap '%x' inject (real mode)\n", iGate));
775
776 /* Check if the interrupt handler is present. */
777 if (iGate * 4 + 3 > pCtx->idtr.cbIdt)
778 {
779 Log(("IDT cbIdt violation\n"));
780 if (iGate != X86_XCPT_DF)
781 {
782 uint32_t intInfo2;
783
784 intInfo2 = (iGate == X86_XCPT_GP) ? (uint32_t)X86_XCPT_DF : iGate;
785 intInfo2 |= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
786 intInfo2 |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
787 intInfo2 |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HWEXCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
788
789 return VMXR0InjectEvent(pVM, pVCpu, pCtx, intInfo2, 0, 0 /* no error code according to the Intel docs */);
790 }
791 Log(("Triple fault -> reset the VM!\n"));
792 return VINF_EM_RESET;
793 }
794 if ( VMX_EXIT_INTERRUPTION_INFO_TYPE(intInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW
795 || iGate == 3 /* Both #BP and #OF point to the instruction after. */
796 || iGate == 4)
797 {
798 ip = pCtx->ip + cbInstr;
799 }
800 else
801 ip = pCtx->ip;
802
803 /* Read the selector:offset pair of the interrupt handler. */
804 GCPhysHandler = (RTGCPHYS)pCtx->idtr.pIdt + iGate * 4;
805 rc = PGMPhysSimpleReadGCPhys(pVM, &offset, GCPhysHandler, sizeof(offset)); AssertRC(rc);
806 rc = PGMPhysSimpleReadGCPhys(pVM, &sel, GCPhysHandler + 2, sizeof(sel)); AssertRC(rc);
807
808 LogFlow(("IDT handler %04X:%04X\n", sel, offset));
809
810 /* Construct the stack frame. */
811 /** @todo should check stack limit. */
812 pCtx->sp -= 2;
813 LogFlow(("ss:sp %04X:%04X eflags=%x\n", pCtx->ss, pCtx->sp, pCtx->eflags.u));
814 rc = PGMPhysSimpleWriteGCPhys(pVM, pCtx->ssHid.u64Base + pCtx->sp, &pCtx->eflags, sizeof(uint16_t)); AssertRC(rc);
815 pCtx->sp -= 2;
816 LogFlow(("ss:sp %04X:%04X cs=%x\n", pCtx->ss, pCtx->sp, pCtx->cs));
817 rc = PGMPhysSimpleWriteGCPhys(pVM, pCtx->ssHid.u64Base + pCtx->sp, &pCtx->cs, sizeof(uint16_t)); AssertRC(rc);
818 pCtx->sp -= 2;
819 LogFlow(("ss:sp %04X:%04X ip=%x\n", pCtx->ss, pCtx->sp, ip));
820 rc = PGMPhysSimpleWriteGCPhys(pVM, pCtx->ssHid.u64Base + pCtx->sp, &ip, sizeof(ip)); AssertRC(rc);
821
822 /* Update the CPU state for executing the handler. */
823 pCtx->rip = offset;
824 pCtx->cs = sel;
825 pCtx->csHid.u64Base = sel << 4;
826 pCtx->eflags.u &= ~(X86_EFL_IF|X86_EFL_TF|X86_EFL_RF|X86_EFL_AC);
827
828 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_GUEST_SEGMENT_REGS;
829 return VINF_SUCCESS;
830 }
831
832 /* Set event injection state. */
833 rc = VMXWriteVMCS(VMX_VMCS_CTRL_ENTRY_IRQ_INFO, intInfo | (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT));
834
835 rc |= VMXWriteVMCS(VMX_VMCS_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
836 rc |= VMXWriteVMCS(VMX_VMCS_CTRL_ENTRY_EXCEPTION_ERRCODE, errCode);
837
838 AssertRC(rc);
839 return rc;
840}
841
842
843/**
844 * Checks for pending guest interrupts and injects them
845 *
846 * @returns VBox status code.
847 * @param pVM The VM to operate on.
848 * @param pVCpu The VMCPU to operate on.
849 * @param pCtx CPU Context
850 */
851static int VMXR0CheckPendingInterrupt(PVM pVM, PVMCPU pVCpu, CPUMCTX *pCtx)
852{
853 int rc;
854
855 /* Dispatch any pending interrupts. (injected before, but a VM exit occurred prematurely) */
856 if (pVCpu->hwaccm.s.Event.fPending)
857 {
858 Log(("CPU%d: Reinjecting event %RX64 %08x at %RGv cr2=%RX64\n", pVCpu->idCpu, pVCpu->hwaccm.s.Event.intInfo, pVCpu->hwaccm.s.Event.errCode, (RTGCPTR)pCtx->rip, pCtx->cr2));
859 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatIntReinject);
860 rc = VMXR0InjectEvent(pVM, pVCpu, pCtx, pVCpu->hwaccm.s.Event.intInfo, 0, pVCpu->hwaccm.s.Event.errCode);
861 AssertRC(rc);
862
863 pVCpu->hwaccm.s.Event.fPending = false;
864 return VINF_SUCCESS;
865 }
866
867 /* If an active trap is already pending, then we must forward it first! */
868 if (!TRPMHasTrap(pVCpu))
869 {
870 if (VMCPU_FF_TESTANDCLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI))
871 {
872 RTGCUINTPTR intInfo;
873
874 Log(("CPU%d: injecting #NMI\n", pVCpu->idCpu));
875
876 intInfo = X86_XCPT_NMI;
877 intInfo |= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
878 intInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
879
880 rc = VMXR0InjectEvent(pVM, pVCpu, pCtx, intInfo, 0, 0);
881 AssertRC(rc);
882
883 return VINF_SUCCESS;
884 }
885
886 /* @todo SMI interrupts. */
887
888 /* When external interrupts are pending, we should exit the VM when IF is set. */
889 if (VMCPU_FF_ISPENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC|VMCPU_FF_INTERRUPT_PIC)))
890 {
891 if (!(pCtx->eflags.u32 & X86_EFL_IF))
892 {
893 if (!(pVCpu->hwaccm.s.vmx.proc_ctls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_IRQ_WINDOW_EXIT))
894 {
895 LogFlow(("Enable irq window exit!\n"));
896 pVCpu->hwaccm.s.vmx.proc_ctls |= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_IRQ_WINDOW_EXIT;
897 rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
898 AssertRC(rc);
899 }
900 /* else nothing to do but wait */
901 }
902 else
903 if (!VMCPU_FF_ISSET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
904 {
905 uint8_t u8Interrupt;
906
907 rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
908 Log(("CPU%d: Dispatch interrupt: u8Interrupt=%x (%d) rc=%Rrc cs:rip=%04X:%RGv\n", pVCpu->idCpu, u8Interrupt, u8Interrupt, rc, pCtx->cs, (RTGCPTR)pCtx->rip));
909 if (RT_SUCCESS(rc))
910 {
911 rc = TRPMAssertTrap(pVCpu, u8Interrupt, TRPM_HARDWARE_INT);
912 AssertRC(rc);
913 }
914 else
915 {
916 /* Can only happen in rare cases where a pending interrupt is cleared behind our back */
917 Assert(!VMCPU_FF_ISPENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC|VMCPU_FF_INTERRUPT_PIC)));
918 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatSwitchGuestIrq);
919 /* Just continue */
920 }
921 }
922 else
923 Log(("Pending interrupt blocked at %RGv by VM_FF_INHIBIT_INTERRUPTS!!\n", (RTGCPTR)pCtx->rip));
924 }
925 }
926
927#ifdef VBOX_STRICT
928 if (TRPMHasTrap(pVCpu))
929 {
930 uint8_t u8Vector;
931 rc = TRPMQueryTrapAll(pVCpu, &u8Vector, 0, 0, 0);
932 AssertRC(rc);
933 }
934#endif
935
936 if ( (pCtx->eflags.u32 & X86_EFL_IF)
937 && (!VMCPU_FF_ISSET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
938 && TRPMHasTrap(pVCpu)
939 )
940 {
941 uint8_t u8Vector;
942 TRPMEVENT enmType;
943 RTGCUINTPTR intInfo;
944 RTGCUINT errCode;
945
946 /* If a new event is pending, then dispatch it now. */
947 rc = TRPMQueryTrapAll(pVCpu, &u8Vector, &enmType, &errCode, 0);
948 AssertRC(rc);
949 Assert(pCtx->eflags.Bits.u1IF == 1 || enmType == TRPM_TRAP);
950 Assert(enmType != TRPM_SOFTWARE_INT);
951
952 /* Clear the pending trap. */
953 rc = TRPMResetTrap(pVCpu);
954 AssertRC(rc);
955
956 intInfo = u8Vector;
957 intInfo |= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
958
959 if (enmType == TRPM_TRAP)
960 {
961 switch (u8Vector) {
962 case 8:
963 case 10:
964 case 11:
965 case 12:
966 case 13:
967 case 14:
968 case 17:
969 /* Valid error codes. */
970 intInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
971 break;
972 default:
973 break;
974 }
975 if (u8Vector == X86_XCPT_BP || u8Vector == X86_XCPT_OF)
976 intInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SWEXCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
977 else
978 intInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HWEXCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
979 }
980 else
981 intInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
982
983 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatIntInject);
984 rc = VMXR0InjectEvent(pVM, pVCpu, pCtx, intInfo, 0, errCode);
985 AssertRC(rc);
986 } /* if (interrupts can be dispatched) */
987
988 return VINF_SUCCESS;
989}
990
991/**
992 * Save the host state
993 *
994 * @returns VBox status code.
995 * @param pVM The VM to operate on.
996 * @param pVCpu The VMCPU to operate on.
997 */
998VMMR0DECL(int) VMXR0SaveHostState(PVM pVM, PVMCPU pVCpu)
999{
1000 int rc = VINF_SUCCESS;
1001
1002 /*
1003 * Host CPU Context
1004 */
1005 if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_HOST_CONTEXT)
1006 {
1007 RTIDTR idtr;
1008 RTGDTR gdtr;
1009 RTSEL SelTR;
1010 PCX86DESCHC pDesc;
1011 uintptr_t trBase;
1012 RTSEL cs;
1013 RTSEL ss;
1014 uint64_t cr3;
1015
1016 /* Control registers */
1017 rc = VMXWriteVMCS(VMX_VMCS_HOST_CR0, ASMGetCR0());
1018#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
1019 if (VMX_IS_64BIT_HOST_MODE())
1020 {
1021 cr3 = hwaccmR0Get64bitCR3();
1022 rc |= VMXWriteVMCS64(VMX_VMCS_HOST_CR3, cr3);
1023 }
1024 else
1025#endif
1026 {
1027 cr3 = ASMGetCR3();
1028 rc |= VMXWriteVMCS(VMX_VMCS_HOST_CR3, cr3);
1029 }
1030 rc |= VMXWriteVMCS(VMX_VMCS_HOST_CR4, ASMGetCR4());
1031 AssertRC(rc);
1032 Log2(("VMX_VMCS_HOST_CR0 %08x\n", ASMGetCR0()));
1033 Log2(("VMX_VMCS_HOST_CR3 %08RX64\n", cr3));
1034 Log2(("VMX_VMCS_HOST_CR4 %08x\n", ASMGetCR4()));
1035
1036 /* Selector registers. */
1037#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
1038 if (VMX_IS_64BIT_HOST_MODE())
1039 {
1040 cs = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelCS;
1041 ss = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelSS;
1042 }
1043 else
1044 {
1045 /* sysenter loads LDT cs & ss, VMX doesn't like this. Load the GDT ones (safe). */
1046 cs = (RTSEL)(uintptr_t)&SUPR0AbsKernelCS;
1047 ss = (RTSEL)(uintptr_t)&SUPR0AbsKernelSS;
1048 }
1049#else
1050 cs = ASMGetCS();
1051 ss = ASMGetSS();
1052#endif
1053 Assert(!(cs & X86_SEL_LDT)); Assert((cs & X86_SEL_RPL) == 0);
1054 Assert(!(ss & X86_SEL_LDT)); Assert((ss & X86_SEL_RPL) == 0);
1055 rc = VMXWriteVMCS(VMX_VMCS16_HOST_FIELD_CS, cs);
1056 /* Note: VMX is (again) very picky about the RPL of the selectors here; we'll restore them manually. */
1057 rc |= VMXWriteVMCS(VMX_VMCS16_HOST_FIELD_DS, 0);
1058 rc |= VMXWriteVMCS(VMX_VMCS16_HOST_FIELD_ES, 0);
1059#if HC_ARCH_BITS == 32
1060 if (!VMX_IS_64BIT_HOST_MODE())
1061 {
1062 rc |= VMXWriteVMCS(VMX_VMCS16_HOST_FIELD_FS, 0);
1063 rc |= VMXWriteVMCS(VMX_VMCS16_HOST_FIELD_GS, 0);
1064 }
1065#endif
1066 rc |= VMXWriteVMCS(VMX_VMCS16_HOST_FIELD_SS, ss);
1067 SelTR = ASMGetTR();
1068 rc |= VMXWriteVMCS(VMX_VMCS16_HOST_FIELD_TR, SelTR);
1069 AssertRC(rc);
1070 Log2(("VMX_VMCS_HOST_FIELD_CS %08x (%08x)\n", cs, ASMGetSS()));
1071 Log2(("VMX_VMCS_HOST_FIELD_DS 00000000 (%08x)\n", ASMGetDS()));
1072 Log2(("VMX_VMCS_HOST_FIELD_ES 00000000 (%08x)\n", ASMGetES()));
1073 Log2(("VMX_VMCS_HOST_FIELD_FS 00000000 (%08x)\n", ASMGetFS()));
1074 Log2(("VMX_VMCS_HOST_FIELD_GS 00000000 (%08x)\n", ASMGetGS()));
1075 Log2(("VMX_VMCS_HOST_FIELD_SS %08x (%08x)\n", ss, ASMGetSS()));
1076 Log2(("VMX_VMCS_HOST_FIELD_TR %08x\n", ASMGetTR()));
1077
1078 /* GDTR & IDTR */
1079#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
1080 if (VMX_IS_64BIT_HOST_MODE())
1081 {
1082 X86XDTR64 gdtr64, idtr64;
1083 hwaccmR0Get64bitGDTRandIDTR(&gdtr64, &idtr64);
1084 rc = VMXWriteVMCS64(VMX_VMCS_HOST_GDTR_BASE, gdtr64.uAddr);
1085 rc |= VMXWriteVMCS64(VMX_VMCS_HOST_IDTR_BASE, gdtr64.uAddr);
1086 AssertRC(rc);
1087 Log2(("VMX_VMCS_HOST_GDTR_BASE %RX64\n", gdtr64.uAddr));
1088 Log2(("VMX_VMCS_HOST_IDTR_BASE %RX64\n", idtr64.uAddr));
1089 gdtr.cbGdt = gdtr64.cb;
1090 gdtr.pGdt = (uintptr_t)gdtr64.uAddr;
1091 }
1092 else
1093#endif
1094 {
1095 ASMGetGDTR(&gdtr);
1096 rc = VMXWriteVMCS(VMX_VMCS_HOST_GDTR_BASE, gdtr.pGdt);
1097 ASMGetIDTR(&idtr);
1098 rc |= VMXWriteVMCS(VMX_VMCS_HOST_IDTR_BASE, idtr.pIdt);
1099 AssertRC(rc);
1100 Log2(("VMX_VMCS_HOST_GDTR_BASE %RHv\n", gdtr.pGdt));
1101 Log2(("VMX_VMCS_HOST_IDTR_BASE %RHv\n", idtr.pIdt));
1102 }
1103
1104 /* Save the base address of the TR selector. */
1105 if (SelTR > gdtr.cbGdt)
1106 {
1107 AssertMsgFailed(("Invalid TR selector %x. GDTR.cbGdt=%x\n", SelTR, gdtr.cbGdt));
1108 return VERR_VMX_INVALID_HOST_STATE;
1109 }
1110
1111 pDesc = (PCX86DESCHC)(gdtr.pGdt + (SelTR & X86_SEL_MASK));
1112#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
1113 if (VMX_IS_64BIT_HOST_MODE())
1114 {
1115 uint64_t trBase64 = X86DESC64_BASE(*(PX86DESC64)pDesc);
1116 rc = VMXWriteVMCS64(VMX_VMCS_HOST_TR_BASE, trBase64);
1117 Log2(("VMX_VMCS_HOST_TR_BASE %RX64\n", trBase64));
1118 AssertRC(rc);
1119 }
1120 else
1121#endif
1122 {
1123#if HC_ARCH_BITS == 64
1124 trBase = X86DESC64_BASE(*pDesc);
1125#else
1126 trBase = X86DESC_BASE(*pDesc);
1127#endif
1128 rc = VMXWriteVMCS(VMX_VMCS_HOST_TR_BASE, trBase);
1129 AssertRC(rc);
1130 Log2(("VMX_VMCS_HOST_TR_BASE %RHv\n", trBase));
1131 }
1132
1133 /* FS and GS base. */
1134#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
1135 if (VMX_IS_64BIT_HOST_MODE())
1136 {
1137 Log2(("MSR_K8_FS_BASE = %RX64\n", ASMRdMsr(MSR_K8_FS_BASE)));
1138 Log2(("MSR_K8_GS_BASE = %RX64\n", ASMRdMsr(MSR_K8_GS_BASE)));
1139 rc = VMXWriteVMCS64(VMX_VMCS_HOST_FS_BASE, ASMRdMsr(MSR_K8_FS_BASE));
1140 rc |= VMXWriteVMCS64(VMX_VMCS_HOST_GS_BASE, ASMRdMsr(MSR_K8_GS_BASE));
1141 }
1142#endif
1143 AssertRC(rc);
1144
1145 /* Sysenter MSRs. */
1146 /** @todo expensive!! */
1147 rc = VMXWriteVMCS(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
1148 Log2(("VMX_VMCS_HOST_SYSENTER_CS %08x\n", ASMRdMsr_Low(MSR_IA32_SYSENTER_CS)));
1149#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
1150 if (VMX_IS_64BIT_HOST_MODE())
1151 {
1152 Log2(("VMX_VMCS_HOST_SYSENTER_EIP %RX64\n", ASMRdMsr(MSR_IA32_SYSENTER_EIP)));
1153 Log2(("VMX_VMCS_HOST_SYSENTER_ESP %RX64\n", ASMRdMsr(MSR_IA32_SYSENTER_ESP)));
1154 rc |= VMXWriteVMCS64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
1155 rc |= VMXWriteVMCS64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
1156 }
1157 else
1158 {
1159 rc |= VMXWriteVMCS(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
1160 rc |= VMXWriteVMCS(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
1161 Log2(("VMX_VMCS_HOST_SYSENTER_EIP %RX32\n", ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP)));
1162 Log2(("VMX_VMCS_HOST_SYSENTER_ESP %RX32\n", ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP)));
1163 }
1164#elif HC_ARCH_BITS == 32
1165 rc |= VMXWriteVMCS(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
1166 rc |= VMXWriteVMCS(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
1167 Log2(("VMX_VMCS_HOST_SYSENTER_EIP %RX32\n", ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP)));
1168 Log2(("VMX_VMCS_HOST_SYSENTER_ESP %RX32\n", ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP)));
1169#else
1170 Log2(("VMX_VMCS_HOST_SYSENTER_EIP %RX64\n", ASMRdMsr(MSR_IA32_SYSENTER_EIP)));
1171 Log2(("VMX_VMCS_HOST_SYSENTER_ESP %RX64\n", ASMRdMsr(MSR_IA32_SYSENTER_ESP)));
1172 rc |= VMXWriteVMCS64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
1173 rc |= VMXWriteVMCS64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
1174#endif
1175 AssertRC(rc);
1176
1177#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
1178 /* Store all host MSRs in the VM-Exit load area, so they will be reloaded after the world switch back to the host. */
1179 PVMXMSR pMsr = (PVMXMSR)pVCpu->hwaccm.s.vmx.pHostMSR;
1180 unsigned idxMsr = 0;
1181
1182 /* EFER MSR present? */
1183 if (ASMCpuId_EDX(0x80000001) & (X86_CPUID_AMD_FEATURE_EDX_NX|X86_CPUID_AMD_FEATURE_EDX_LONG_MODE))
1184 {
1185 if (ASMCpuId_EDX(0x80000001) & X86_CPUID_AMD_FEATURE_EDX_SEP)
1186 {
1187 pMsr->u32IndexMSR = MSR_K6_STAR;
1188 pMsr->u32Reserved = 0;
1189 pMsr->u64Value = ASMRdMsr(MSR_K6_STAR); /* legacy syscall eip, cs & ss */
1190 pMsr++; idxMsr++;
1191 }
1192
1193 pMsr->u32IndexMSR = MSR_K6_EFER;
1194 pMsr->u32Reserved = 0;
1195# if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
1196 if (CPUMIsGuestInLongMode(pVCpu))
1197 {
1198 /* Must match the efer value in our 64 bits switcher. */
1199 pMsr->u64Value = ASMRdMsr(MSR_K6_EFER) | MSR_K6_EFER_LME | MSR_K6_EFER_SCE | MSR_K6_EFER_NXE;
1200 }
1201 else
1202# endif
1203 pMsr->u64Value = ASMRdMsr(MSR_K6_EFER);
1204 pMsr++; idxMsr++;
1205 }
1206
1207# if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
1208 if (VMX_IS_64BIT_HOST_MODE())
1209 {
1210 pMsr->u32IndexMSR = MSR_K8_LSTAR;
1211 pMsr->u32Reserved = 0;
1212 pMsr->u64Value = ASMRdMsr(MSR_K8_LSTAR); /* 64 bits mode syscall rip */
1213 pMsr++; idxMsr++;
1214 pMsr->u32IndexMSR = MSR_K8_SF_MASK;
1215 pMsr->u32Reserved = 0;
1216 pMsr->u64Value = ASMRdMsr(MSR_K8_SF_MASK); /* syscall flag mask */
1217 pMsr++; idxMsr++;
1218 pMsr->u32IndexMSR = MSR_K8_KERNEL_GS_BASE;
1219 pMsr->u32Reserved = 0;
1220 pMsr->u64Value = ASMRdMsr(MSR_K8_KERNEL_GS_BASE); /* swapgs exchange value */
1221 pMsr++; idxMsr++;
1222 }
1223# endif
1224 rc = VMXWriteVMCS(VMX_VMCS_CTRL_EXIT_MSR_LOAD_COUNT, idxMsr);
1225 AssertRC(rc);
1226#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
1227
1228 pVCpu->hwaccm.s.fContextUseFlags &= ~HWACCM_CHANGED_HOST_CONTEXT;
1229 }
1230 return rc;
1231}
1232
1233/**
1234 * Prefetch the 4 PDPT pointers (PAE and nested paging only)
1235 *
1236 * @param pVM The VM to operate on.
1237 * @param pVCpu The VMCPU to operate on.
1238 * @param pCtx Guest context
1239 */
1240static void vmxR0PrefetchPAEPdptrs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
1241{
1242 if (CPUMIsGuestInPAEModeEx(pCtx))
1243 {
1244 X86PDPE Pdpe;
1245
1246 for (unsigned i=0;i<4;i++)
1247 {
1248 Pdpe = PGMGstGetPaePDPtr(pVCpu, i);
1249 int rc = VMXWriteVMCS64(VMX_VMCS_GUEST_PDPTR0_FULL + i*2, Pdpe.u);
1250 AssertRC(rc);
1251 }
1252 }
1253}
1254
1255/**
1256 * Update the exception bitmap according to the current CPU state
1257 *
1258 * @param pVM The VM to operate on.
1259 * @param pVCpu The VMCPU to operate on.
1260 * @param pCtx Guest context
1261 */
1262static void vmxR0UpdateExceptionBitmap(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
1263{
1264 uint32_t u32TrapMask;
1265 Assert(pCtx);
1266
1267 u32TrapMask = HWACCM_VMX_TRAP_MASK;
1268#ifndef DEBUG
1269 if (pVM->hwaccm.s.fNestedPaging)
1270 u32TrapMask &= ~RT_BIT(X86_XCPT_PF); /* no longer need to intercept #PF. */
1271#endif
1272
1273 /* Also catch floating point exceptions as we need to report them to the guest in a different way. */
1274 if ( CPUMIsGuestFPUStateActive(pVCpu) == true
1275 && !(pCtx->cr0 & X86_CR0_NE)
1276 && !pVCpu->hwaccm.s.fFPUOldStyleOverride)
1277 {
1278 u32TrapMask |= RT_BIT(X86_XCPT_MF);
1279 pVCpu->hwaccm.s.fFPUOldStyleOverride = true;
1280 }
1281
1282#ifdef VBOX_STRICT
1283 Assert(u32TrapMask & RT_BIT(X86_XCPT_GP));
1284#endif
1285
1286 /* Intercept all exceptions in real mode as none of them can be injected directly (#GP otherwise). */
1287 if ( CPUMIsGuestInRealModeEx(pCtx)
1288 && pVM->hwaccm.s.vmx.pRealModeTSS)
1289 u32TrapMask |= HWACCM_VMX_TRAP_MASK_REALMODE;
1290
1291 int rc = VMXWriteVMCS(VMX_VMCS_CTRL_EXCEPTION_BITMAP, u32TrapMask);
1292 AssertRC(rc);
1293}
1294
1295/**
1296 * Loads a minimal guest state
1297 *
1298 * NOTE: Don't do anything here that can cause a jump back to ring 3!!!!!
1299 *
1300 * @param pVM The VM to operate on.
1301 * @param pVCpu The VMCPU to operate on.
1302 * @param pCtx Guest context
1303 */
1304VMMR0DECL(void) VMXR0LoadMinimalGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
1305{
1306 int rc;
1307 X86EFLAGS eflags;
1308
1309 Assert(!(pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_ALL_GUEST));
1310
1311 /* EIP, ESP and EFLAGS */
1312 rc = VMXWriteVMCS64(VMX_VMCS64_GUEST_RIP, pCtx->rip);
1313 rc |= VMXWriteVMCS64(VMX_VMCS64_GUEST_RSP, pCtx->rsp);
1314 AssertRC(rc);
1315
1316 /* Bits 22-31, 15, 5 & 3 must be zero. Bit 1 must be 1. */
1317 eflags = pCtx->eflags;
1318 eflags.u32 &= VMX_EFLAGS_RESERVED_0;
1319 eflags.u32 |= VMX_EFLAGS_RESERVED_1;
1320
1321 /* Real mode emulation using v86 mode. */
1322 if ( CPUMIsGuestInRealModeEx(pCtx)
1323 && pVM->hwaccm.s.vmx.pRealModeTSS)
1324 {
1325 pVCpu->hwaccm.s.vmx.RealMode.eflags = eflags;
1326
1327 eflags.Bits.u1VM = 1;
1328 eflags.Bits.u2IOPL = 0; /* must always be 0 or else certain instructions won't cause faults. */
1329 }
1330 rc = VMXWriteVMCS(VMX_VMCS_GUEST_RFLAGS, eflags.u32);
1331 AssertRC(rc);
1332}
1333
1334/**
1335 * Loads the guest state
1336 *
1337 * NOTE: Don't do anything here that can cause a jump back to ring 3!!!!!
1338 *
1339 * @returns VBox status code.
1340 * @param pVM The VM to operate on.
1341 * @param pVCpu The VMCPU to operate on.
1342 * @param pCtx Guest context
1343 */
1344VMMR0DECL(int) VMXR0LoadGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
1345{
1346 int rc = VINF_SUCCESS;
1347 RTGCUINTPTR val;
1348
1349 /* VMX_VMCS_CTRL_ENTRY_CONTROLS
1350 * Set required bits to one and zero according to the MSR capabilities.
1351 */
1352 val = pVM->hwaccm.s.vmx.msr.vmx_entry.n.disallowed0;
1353 /* Load guest debug controls (dr7 & IA32_DEBUGCTL_MSR) (forced to 1 on the 'first' VT-x capable CPUs; this actually includes the newest Nehalem CPUs) */
1354 val |= VMX_VMCS_CTRL_ENTRY_CONTROLS_LOAD_DEBUG;
1355 /* 64 bits guest mode? */
1356 if (CPUMIsGuestInLongModeEx(pCtx))
1357 val |= VMX_VMCS_CTRL_ENTRY_CONTROLS_IA64_MODE;
1358 /* else Must be zero when AMD64 is not available. */
1359
1360 /* Mask away the bits that the CPU doesn't support */
1361 val &= pVM->hwaccm.s.vmx.msr.vmx_entry.n.allowed1;
1362 rc = VMXWriteVMCS(VMX_VMCS_CTRL_ENTRY_CONTROLS, val);
1363 AssertRC(rc);
1364
1365 /* VMX_VMCS_CTRL_EXIT_CONTROLS
1366 * Set required bits to one and zero according to the MSR capabilities.
1367 */
1368 val = pVM->hwaccm.s.vmx.msr.vmx_exit.n.disallowed0;
1369
1370 /* Save debug controls (dr7 & IA32_DEBUGCTL_MSR) (forced to 1 on the 'first' VT-x capable CPUs; this actually includes the newest Nehalem CPUs) */
1371 val |= VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_DEBUG;
1372
1373#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
1374 if (VMX_IS_64BIT_HOST_MODE())
1375 val |= VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_AMD64;
1376 /* else: Must be zero when AMD64 is not available. */
1377#elif HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
1378 if (CPUMIsGuestInLongModeEx(pCtx))
1379 val |= VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_AMD64; /* our switcher goes to long mode */
1380 else
1381 Assert(!(val & VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_AMD64));
1382#endif
1383 val &= pVM->hwaccm.s.vmx.msr.vmx_exit.n.allowed1;
1384 /* Don't acknowledge external interrupts on VM-exit. */
1385 rc = VMXWriteVMCS(VMX_VMCS_CTRL_EXIT_CONTROLS, val);
1386 AssertRC(rc);
1387
1388 /* Guest CPU context: ES, CS, SS, DS, FS, GS. */
1389 if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_SEGMENT_REGS)
1390 {
1391 if (pVM->hwaccm.s.vmx.pRealModeTSS)
1392 {
1393 PGMMODE enmGuestMode = PGMGetGuestMode(pVCpu);
1394 if (pVCpu->hwaccm.s.vmx.enmLastSeenGuestMode != enmGuestMode)
1395 {
1396 /* Correct weird requirements for switching to protected mode. */
1397 if ( pVCpu->hwaccm.s.vmx.enmLastSeenGuestMode == PGMMODE_REAL
1398 && enmGuestMode >= PGMMODE_PROTECTED)
1399 {
1400 /* Flush the recompiler code cache as it's not unlikely
1401 * the guest will rewrite code it will later execute in real
1402 * mode (OpenBSD 4.0 is one such example)
1403 */
1404 REMFlushTBs(pVM);
1405
1406 /* DPL of all hidden selector registers must match the current CPL (0). */
1407 pCtx->csHid.Attr.n.u2Dpl = 0;
1408 pCtx->csHid.Attr.n.u4Type = X86_SEL_TYPE_CODE | X86_SEL_TYPE_RW_ACC;
1409
1410 pCtx->dsHid.Attr.n.u2Dpl = 0;
1411 pCtx->esHid.Attr.n.u2Dpl = 0;
1412 pCtx->fsHid.Attr.n.u2Dpl = 0;
1413 pCtx->gsHid.Attr.n.u2Dpl = 0;
1414 pCtx->ssHid.Attr.n.u2Dpl = 0;
1415
1416 /* The limit must correspond to the 32 bits setting. */
1417 if (!pCtx->csHid.Attr.n.u1DefBig)
1418 pCtx->csHid.u32Limit &= 0xffff;
1419 if (!pCtx->dsHid.Attr.n.u1DefBig)
1420 pCtx->dsHid.u32Limit &= 0xffff;
1421 if (!pCtx->esHid.Attr.n.u1DefBig)
1422 pCtx->esHid.u32Limit &= 0xffff;
1423 if (!pCtx->fsHid.Attr.n.u1DefBig)
1424 pCtx->fsHid.u32Limit &= 0xffff;
1425 if (!pCtx->gsHid.Attr.n.u1DefBig)
1426 pCtx->gsHid.u32Limit &= 0xffff;
1427 if (!pCtx->ssHid.Attr.n.u1DefBig)
1428 pCtx->ssHid.u32Limit &= 0xffff;
1429 }
1430 else
1431 /* Switching from protected mode to real mode. */
1432 if ( pVCpu->hwaccm.s.vmx.enmLastSeenGuestMode >= PGMMODE_PROTECTED
1433 && enmGuestMode == PGMMODE_REAL)
1434 {
1435 /* The limit must also be set to 0xffff. */
1436 pCtx->csHid.u32Limit = 0xffff;
1437 pCtx->dsHid.u32Limit = 0xffff;
1438 pCtx->esHid.u32Limit = 0xffff;
1439 pCtx->fsHid.u32Limit = 0xffff;
1440 pCtx->gsHid.u32Limit = 0xffff;
1441 pCtx->ssHid.u32Limit = 0xffff;
1442
1443 Assert(pCtx->csHid.u64Base <= 0xfffff);
1444 Assert(pCtx->dsHid.u64Base <= 0xfffff);
1445 Assert(pCtx->esHid.u64Base <= 0xfffff);
1446 Assert(pCtx->fsHid.u64Base <= 0xfffff);
1447 Assert(pCtx->gsHid.u64Base <= 0xfffff);
1448 }
1449 pVCpu->hwaccm.s.vmx.enmLastSeenGuestMode = enmGuestMode;
1450 }
1451 else
1452 /* VT-x will fail with a guest invalid state otherwise... (CPU state after a reset) */
1453 if ( CPUMIsGuestInRealModeEx(pCtx)
1454 && pCtx->csHid.u64Base == 0xffff0000)
1455 {
1456 pCtx->csHid.u64Base = 0xf0000;
1457 pCtx->cs = 0xf000;
1458 }
1459 }
1460
1461 VMX_WRITE_SELREG(ES, es);
1462 AssertRC(rc);
1463
1464 VMX_WRITE_SELREG(CS, cs);
1465 AssertRC(rc);
1466
1467 VMX_WRITE_SELREG(SS, ss);
1468 AssertRC(rc);
1469
1470 VMX_WRITE_SELREG(DS, ds);
1471 AssertRC(rc);
1472
1473 VMX_WRITE_SELREG(FS, fs);
1474 AssertRC(rc);
1475
1476 VMX_WRITE_SELREG(GS, gs);
1477 AssertRC(rc);
1478 }
1479
1480 /* Guest CPU context: LDTR. */
1481 if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_LDTR)
1482 {
1483 if (pCtx->ldtr == 0)
1484 {
1485 rc = VMXWriteVMCS(VMX_VMCS16_GUEST_FIELD_LDTR, 0);
1486 rc |= VMXWriteVMCS(VMX_VMCS32_GUEST_LDTR_LIMIT, 0);
1487 rc |= VMXWriteVMCS64(VMX_VMCS64_GUEST_LDTR_BASE, 0);
1488 /* Note: vmlaunch will fail with 0 or just 0x02. No idea why. */
1489 rc |= VMXWriteVMCS(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, 0x82 /* present, LDT */);
1490 }
1491 else
1492 {
1493 rc = VMXWriteVMCS(VMX_VMCS16_GUEST_FIELD_LDTR, pCtx->ldtr);
1494 rc |= VMXWriteVMCS(VMX_VMCS32_GUEST_LDTR_LIMIT, pCtx->ldtrHid.u32Limit);
1495 rc |= VMXWriteVMCS64(VMX_VMCS64_GUEST_LDTR_BASE, pCtx->ldtrHid.u64Base);
1496 rc |= VMXWriteVMCS(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, pCtx->ldtrHid.Attr.u);
1497 }
1498 AssertRC(rc);
1499 }
1500 /* Guest CPU context: TR. */
1501 if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_TR)
1502 {
1503 /* Real mode emulation using v86 mode with CR4.VME (interrupt redirection using the int bitmap in the TSS) */
1504 if ( CPUMIsGuestInRealModeEx(pCtx)
1505 && pVM->hwaccm.s.vmx.pRealModeTSS)
1506 {
1507 RTGCPHYS GCPhys;
1508
1509 /* We convert it here every time as pci regions could be reconfigured. */
1510 rc = PDMVMMDevHeapR3ToGCPhys(pVM, pVM->hwaccm.s.vmx.pRealModeTSS, &GCPhys);
1511 AssertRC(rc);
1512
1513 rc = VMXWriteVMCS(VMX_VMCS16_GUEST_FIELD_TR, 0);
1514 rc |= VMXWriteVMCS(VMX_VMCS32_GUEST_TR_LIMIT, HWACCM_VTX_TSS_SIZE);
1515 rc |= VMXWriteVMCS64(VMX_VMCS64_GUEST_TR_BASE, GCPhys /* phys = virt in this mode */);
1516
1517 X86DESCATTR attr;
1518
1519 attr.u = 0;
1520 attr.n.u1Present = 1;
1521 attr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
1522 val = attr.u;
1523 }
1524 else
1525 {
1526 rc = VMXWriteVMCS(VMX_VMCS16_GUEST_FIELD_TR, pCtx->tr);
1527 rc |= VMXWriteVMCS(VMX_VMCS32_GUEST_TR_LIMIT, pCtx->trHid.u32Limit);
1528 rc |= VMXWriteVMCS64(VMX_VMCS64_GUEST_TR_BASE, pCtx->trHid.u64Base);
1529
1530 val = pCtx->trHid.Attr.u;
1531
1532 /* The TSS selector must be busy. */
1533 if ((val & 0xF) == X86_SEL_TYPE_SYS_286_TSS_AVAIL)
1534 val = (val & ~0xF) | X86_SEL_TYPE_SYS_286_TSS_BUSY;
1535 else
1536 /* Default even if no TR selector has been set (otherwise vmlaunch will fail!) */
1537 val = (val & ~0xF) | X86_SEL_TYPE_SYS_386_TSS_BUSY;
1538
1539 }
1540 rc |= VMXWriteVMCS(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, val);
1541 AssertRC(rc);
1542 }
1543 /* Guest CPU context: GDTR. */
1544 if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_GDTR)
1545 {
1546 rc = VMXWriteVMCS(VMX_VMCS32_GUEST_GDTR_LIMIT, pCtx->gdtr.cbGdt);
1547 rc |= VMXWriteVMCS64(VMX_VMCS64_GUEST_GDTR_BASE, pCtx->gdtr.pGdt);
1548 AssertRC(rc);
1549 }
1550 /* Guest CPU context: IDTR. */
1551 if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_IDTR)
1552 {
1553 rc = VMXWriteVMCS(VMX_VMCS32_GUEST_IDTR_LIMIT, pCtx->idtr.cbIdt);
1554 rc |= VMXWriteVMCS64(VMX_VMCS64_GUEST_IDTR_BASE, pCtx->idtr.pIdt);
1555 AssertRC(rc);
1556 }
1557
1558 /*
1559 * Sysenter MSRs (unconditional)
1560 */
1561 rc = VMXWriteVMCS(VMX_VMCS32_GUEST_SYSENTER_CS, pCtx->SysEnter.cs);
1562 rc |= VMXWriteVMCS64(VMX_VMCS64_GUEST_SYSENTER_EIP, pCtx->SysEnter.eip);
1563 rc |= VMXWriteVMCS64(VMX_VMCS64_GUEST_SYSENTER_ESP, pCtx->SysEnter.esp);
1564 AssertRC(rc);
1565
1566 /* Control registers */
1567 if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_CR0)
1568 {
1569 val = pCtx->cr0;
1570 rc = VMXWriteVMCS(VMX_VMCS_CTRL_CR0_READ_SHADOW, val);
1571 Log2(("Guest CR0-shadow %08x\n", val));
1572 if (CPUMIsGuestFPUStateActive(pVCpu) == false)
1573 {
1574 /* Always use #NM exceptions to load the FPU/XMM state on demand. */
1575 val |= X86_CR0_TS | X86_CR0_ET | X86_CR0_NE | X86_CR0_MP;
1576 }
1577 else
1578 {
1579 /** @todo check if we support the old style mess correctly. */
1580 if (!(val & X86_CR0_NE))
1581 Log(("Forcing X86_CR0_NE!!!\n"));
1582
1583 val |= X86_CR0_NE; /* always turn on the native mechanism to report FPU errors (old style uses interrupts) */
1584 }
1585 /* Note: protected mode & paging are always enabled; we use them for emulating real and protected mode without paging too. */
1586 if (!pVM->hwaccm.s.vmx.fUnrestrictedGuest)
1587 val |= X86_CR0_PE | X86_CR0_PG;
1588
1589 if (pVM->hwaccm.s.fNestedPaging)
1590 {
1591 if (CPUMIsGuestInPagedProtectedModeEx(pCtx))
1592 {
1593 /* Disable cr3 read/write monitoring as we don't need it for EPT. */
1594 pVCpu->hwaccm.s.vmx.proc_ctls &= ~( VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
1595 | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT);
1596 }
1597 else
1598 {
1599 /* Reenable cr3 read/write monitoring as our identity mapped page table is active. */
1600 pVCpu->hwaccm.s.vmx.proc_ctls |= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
1601 | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT;
1602 }
1603 rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
1604 AssertRC(rc);
1605 }
1606 else
1607 {
1608 /* Note: We must also set this as we rely on protecting various pages for which supervisor writes must be caught. */
1609 val |= X86_CR0_WP;
1610 }
1611
1612 /* Always enable caching. */
1613 val &= ~(X86_CR0_CD|X86_CR0_NW);
1614
1615 rc |= VMXWriteVMCS64(VMX_VMCS64_GUEST_CR0, val);
1616 Log2(("Guest CR0 %08x\n", val));
1617 /* CR0 flags owned by the host; if the guests attempts to change them, then
1618 * the VM will exit.
1619 */
1620 val = X86_CR0_PE /* Must monitor this bit (assumptions are made for real mode emulation) */
1621 | X86_CR0_WP /* Must monitor this bit (it must always be enabled). */
1622 | X86_CR0_PG /* Must monitor this bit (assumptions are made for real mode & protected mode without paging emulation) */
1623 | X86_CR0_CD /* Bit not restored during VM-exit! */
1624 | X86_CR0_NW /* Bit not restored during VM-exit! */
1625 | X86_CR0_NE;
1626
1627 /* When the guest's FPU state is active, then we no longer care about
1628 * the FPU related bits.
1629 */
1630 if (CPUMIsGuestFPUStateActive(pVCpu) == false)
1631 val |= X86_CR0_TS | X86_CR0_ET | X86_CR0_MP;
1632
1633 pVCpu->hwaccm.s.vmx.cr0_mask = val;
1634
1635 rc |= VMXWriteVMCS(VMX_VMCS_CTRL_CR0_MASK, val);
1636 Log2(("Guest CR0-mask %08x\n", val));
1637 AssertRC(rc);
1638 }
1639 if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_CR4)
1640 {
1641 /* CR4 */
1642 rc = VMXWriteVMCS(VMX_VMCS_CTRL_CR4_READ_SHADOW, pCtx->cr4);
1643 Log2(("Guest CR4-shadow %08x\n", pCtx->cr4));
1644 /* Set the required bits in cr4 too (currently X86_CR4_VMXE). */
1645 val = pCtx->cr4 | (uint32_t)pVM->hwaccm.s.vmx.msr.vmx_cr4_fixed0;
1646
1647 if (!pVM->hwaccm.s.fNestedPaging)
1648 {
1649 switch(pVCpu->hwaccm.s.enmShadowMode)
1650 {
1651 case PGMMODE_REAL: /* Real mode -> emulated using v86 mode */
1652 case PGMMODE_PROTECTED: /* Protected mode, no paging -> emulated using identity mapping. */
1653 case PGMMODE_32_BIT: /* 32-bit paging. */
1654 val &= ~X86_CR4_PAE;
1655 break;
1656
1657 case PGMMODE_PAE: /* PAE paging. */
1658 case PGMMODE_PAE_NX: /* PAE paging with NX enabled. */
1659 /** @todo use normal 32 bits paging */
1660 val |= X86_CR4_PAE;
1661 break;
1662
1663 case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
1664 case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
1665#ifdef VBOX_ENABLE_64_BITS_GUESTS
1666 break;
1667#else
1668 AssertFailed();
1669 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
1670#endif
1671 default: /* shut up gcc */
1672 AssertFailed();
1673 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
1674 }
1675 }
1676 else
1677 if ( !CPUMIsGuestInPagedProtectedModeEx(pCtx)
1678 && !pVM->hwaccm.s.vmx.fUnrestrictedGuest)
1679 {
1680 /* We use 4 MB pages in our identity mapping page table for real and protected mode without paging. */
1681 val |= X86_CR4_PSE;
1682 /* Our identity mapping is a 32 bits page directory. */
1683 val &= ~X86_CR4_PAE;
1684 }
1685
1686 /* Turn off VME if we're in emulated real mode. */
1687 if ( CPUMIsGuestInRealModeEx(pCtx)
1688 && pVM->hwaccm.s.vmx.pRealModeTSS)
1689 val &= ~X86_CR4_VME;
1690
1691 rc |= VMXWriteVMCS64(VMX_VMCS64_GUEST_CR4, val);
1692 Log2(("Guest CR4 %08x\n", val));
1693 /* CR4 flags owned by the host; if the guests attempts to change them, then
1694 * the VM will exit.
1695 */
1696 val = 0
1697 | X86_CR4_VME
1698 | X86_CR4_PAE
1699 | X86_CR4_PGE
1700 | X86_CR4_PSE
1701 | X86_CR4_VMXE;
1702 pVCpu->hwaccm.s.vmx.cr4_mask = val;
1703
1704 rc |= VMXWriteVMCS(VMX_VMCS_CTRL_CR4_MASK, val);
1705 Log2(("Guest CR4-mask %08x\n", val));
1706 AssertRC(rc);
1707 }
1708
1709 if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_CR3)
1710 {
1711 if (pVM->hwaccm.s.fNestedPaging)
1712 {
1713 Assert(PGMGetHyperCR3(pVCpu));
1714 pVCpu->hwaccm.s.vmx.GCPhysEPTP = PGMGetHyperCR3(pVCpu);
1715
1716 Assert(!(pVCpu->hwaccm.s.vmx.GCPhysEPTP & 0xfff));
1717 /** @todo Check the IA32_VMX_EPT_VPID_CAP MSR for other supported memory types. */
1718 pVCpu->hwaccm.s.vmx.GCPhysEPTP |= VMX_EPT_MEMTYPE_WB
1719 | (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
1720
1721 rc = VMXWriteVMCS64(VMX_VMCS_CTRL_EPTP_FULL, pVCpu->hwaccm.s.vmx.GCPhysEPTP);
1722 AssertRC(rc);
1723
1724 if ( !CPUMIsGuestInPagedProtectedModeEx(pCtx)
1725 && !pVM->hwaccm.s.vmx.fUnrestrictedGuest)
1726 {
1727 RTGCPHYS GCPhys;
1728
1729 /* We convert it here every time as pci regions could be reconfigured. */
1730 rc = PDMVMMDevHeapR3ToGCPhys(pVM, pVM->hwaccm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
1731 AssertMsgRC(rc, ("pNonPagingModeEPTPageTable = %RGv\n", pVM->hwaccm.s.vmx.pNonPagingModeEPTPageTable));
1732
1733 /* We use our identity mapping page table here as we need to map guest virtual to guest physical addresses; EPT will
1734 * take care of the translation to host physical addresses.
1735 */
1736 val = GCPhys;
1737 }
1738 else
1739 {
1740 /* Save the real guest CR3 in VMX_VMCS_GUEST_CR3 */
1741 val = pCtx->cr3;
1742 /* Prefetch the four PDPT entries in PAE mode. */
1743 vmxR0PrefetchPAEPdptrs(pVM, pVCpu, pCtx);
1744 }
1745 }
1746 else
1747 {
1748 val = PGMGetHyperCR3(pVCpu);
1749 Assert(val || VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL));
1750 }
1751
1752 /* Save our shadow CR3 register. */
1753 rc = VMXWriteVMCS64(VMX_VMCS64_GUEST_CR3, val);
1754 AssertRC(rc);
1755 }
1756
1757 /* Debug registers. */
1758 if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_DEBUG)
1759 {
1760 pCtx->dr[6] |= X86_DR6_INIT_VAL; /* set all reserved bits to 1. */
1761 pCtx->dr[6] &= ~RT_BIT(12); /* must be zero. */
1762
1763 pCtx->dr[7] &= 0xffffffff; /* upper 32 bits reserved */
1764 pCtx->dr[7] &= ~(RT_BIT(11) | RT_BIT(12) | RT_BIT(14) | RT_BIT(15)); /* must be zero */
1765 pCtx->dr[7] |= 0x400; /* must be one */
1766
1767 /* Resync DR7 */
1768 rc = VMXWriteVMCS64(VMX_VMCS64_GUEST_DR7, pCtx->dr[7]);
1769 AssertRC(rc);
1770
1771#ifdef DEBUG
1772 /* Sync the hypervisor debug state now if any breakpoint is armed. */
1773 if ( CPUMGetHyperDR7(pVCpu) & (X86_DR7_ENABLED_MASK|X86_DR7_GD)
1774 && !CPUMIsHyperDebugStateActive(pVCpu)
1775 && !DBGFIsStepping(pVCpu))
1776 {
1777 /* Save the host and load the hypervisor debug state. */
1778 rc = CPUMR0LoadHyperDebugState(pVM, pVCpu, pCtx, true /* include DR6 */);
1779 AssertRC(rc);
1780
1781 /* DRx intercepts remain enabled. */
1782
1783 /* Override dr7 with the hypervisor value. */
1784 rc = VMXWriteVMCS64(VMX_VMCS64_GUEST_DR7, CPUMGetHyperDR7(pVCpu));
1785 AssertRC(rc);
1786 }
1787 else
1788#endif
1789 /* Sync the debug state now if any breakpoint is armed. */
1790 if ( (pCtx->dr[7] & (X86_DR7_ENABLED_MASK|X86_DR7_GD))
1791 && !CPUMIsGuestDebugStateActive(pVCpu)
1792 && !DBGFIsStepping(pVCpu))
1793 {
1794 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatDRxArmed);
1795
1796 /* Disable drx move intercepts. */
1797 pVCpu->hwaccm.s.vmx.proc_ctls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT;
1798 rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
1799 AssertRC(rc);
1800
1801 /* Save the host and load the guest debug state. */
1802 rc = CPUMR0LoadGuestDebugState(pVM, pVCpu, pCtx, true /* include DR6 */);
1803 AssertRC(rc);
1804 }
1805
1806 /* IA32_DEBUGCTL MSR. */
1807 rc = VMXWriteVMCS64(VMX_VMCS_GUEST_DEBUGCTL_FULL, 0);
1808 AssertRC(rc);
1809
1810 /** @todo do we really ever need this? */
1811 rc |= VMXWriteVMCS(VMX_VMCS_GUEST_DEBUG_EXCEPTIONS, 0);
1812 AssertRC(rc);
1813 }
1814
1815 /* 64 bits guest mode? */
1816 if (CPUMIsGuestInLongModeEx(pCtx))
1817 {
1818#if !defined(VBOX_ENABLE_64_BITS_GUESTS)
1819 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
1820#elif HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
1821 pVCpu->hwaccm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
1822#else
1823# ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
1824 if (!pVM->hwaccm.s.fAllow64BitGuests)
1825 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
1826# endif
1827 pVCpu->hwaccm.s.vmx.pfnStartVM = VMXR0StartVM64;
1828#endif
1829 /* Unconditionally update these as wrmsr might have changed them. */
1830 rc = VMXWriteVMCS64(VMX_VMCS64_GUEST_FS_BASE, pCtx->fsHid.u64Base);
1831 AssertRC(rc);
1832 rc = VMXWriteVMCS64(VMX_VMCS64_GUEST_GS_BASE, pCtx->gsHid.u64Base);
1833 AssertRC(rc);
1834 }
1835 else
1836 {
1837 pVCpu->hwaccm.s.vmx.pfnStartVM = VMXR0StartVM32;
1838 }
1839
1840 vmxR0UpdateExceptionBitmap(pVM, pVCpu, pCtx);
1841
1842#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
1843 /* Store all guest MSRs in the VM-Entry load area, so they will be loaded during the world switch. */
1844 PVMXMSR pMsr = (PVMXMSR)pVCpu->hwaccm.s.vmx.pGuestMSR;
1845 unsigned idxMsr = 0;
1846
1847 uint32_t ulEdx;
1848 uint32_t ulTemp;
1849 CPUMGetGuestCpuId(pVCpu, 0x80000001, &ulTemp, &ulTemp, &ulTemp, &ulEdx);
1850 /* EFER MSR present? */
1851 if (ulEdx & (X86_CPUID_AMD_FEATURE_EDX_NX|X86_CPUID_AMD_FEATURE_EDX_LONG_MODE))
1852 {
1853 pMsr->u32IndexMSR = MSR_K6_EFER;
1854 pMsr->u32Reserved = 0;
1855 pMsr->u64Value = pCtx->msrEFER;
1856 /* VT-x will complain if only MSR_K6_EFER_LME is set. */
1857 if (!CPUMIsGuestInLongModeEx(pCtx))
1858 pMsr->u64Value &= ~(MSR_K6_EFER_LMA|MSR_K6_EFER_LME);
1859 pMsr++; idxMsr++;
1860
1861 if (ulEdx & X86_CPUID_AMD_FEATURE_EDX_LONG_MODE)
1862 {
1863 pMsr->u32IndexMSR = MSR_K8_LSTAR;
1864 pMsr->u32Reserved = 0;
1865 pMsr->u64Value = pCtx->msrLSTAR; /* 64 bits mode syscall rip */
1866 pMsr++; idxMsr++;
1867 pMsr->u32IndexMSR = MSR_K6_STAR;
1868 pMsr->u32Reserved = 0;
1869 pMsr->u64Value = pCtx->msrSTAR; /* legacy syscall eip, cs & ss */
1870 pMsr++; idxMsr++;
1871 pMsr->u32IndexMSR = MSR_K8_SF_MASK;
1872 pMsr->u32Reserved = 0;
1873 pMsr->u64Value = pCtx->msrSFMASK; /* syscall flag mask */
1874 pMsr++; idxMsr++;
1875 pMsr->u32IndexMSR = MSR_K8_KERNEL_GS_BASE;
1876 pMsr->u32Reserved = 0;
1877 pMsr->u64Value = pCtx->msrKERNELGSBASE; /* swapgs exchange value */
1878 pMsr++; idxMsr++;
1879 }
1880 }
1881 pVCpu->hwaccm.s.vmx.cCachedMSRs = idxMsr;
1882
1883 rc = VMXWriteVMCS(VMX_VMCS_CTRL_ENTRY_MSR_LOAD_COUNT, idxMsr);
1884 AssertRC(rc);
1885
1886 rc = VMXWriteVMCS(VMX_VMCS_CTRL_EXIT_MSR_STORE_COUNT, idxMsr);
1887 AssertRC(rc);
1888#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
1889
1890 if (TMCpuTickCanUseRealTSC(pVCpu, &pVCpu->hwaccm.s.vmx.u64TSCOffset))
1891 {
1892 uint64_t u64CurTSC = ASMReadTSC();
1893 if (u64CurTSC + pVCpu->hwaccm.s.vmx.u64TSCOffset >= TMCpuTickGetLastSeen(pVCpu))
1894 {
1895 /* Note: VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT takes precedence over TSC_OFFSET */
1896 rc = VMXWriteVMCS64(VMX_VMCS_CTRL_TSC_OFFSET_FULL, pVCpu->hwaccm.s.vmx.u64TSCOffset);
1897 AssertRC(rc);
1898
1899 pVCpu->hwaccm.s.vmx.proc_ctls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT;
1900 rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
1901 AssertRC(rc);
1902 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatTSCOffset);
1903 }
1904 else
1905 {
1906 /* Fall back to rdtsc emulation as we would otherwise pass decreasing tsc values to the guest. */
1907 LogFlow(("TSC %RX64 offset %RX64 time=%RX64 last=%RX64 (diff=%RX64, virt_tsc=%RX64)\n", u64CurTSC, pVCpu->hwaccm.s.vmx.u64TSCOffset, u64CurTSC + pVCpu->hwaccm.s.vmx.u64TSCOffset, TMCpuTickGetLastSeen(pVCpu), TMCpuTickGetLastSeen(pVCpu) - u64CurTSC - pVCpu->hwaccm.s.vmx.u64TSCOffset, TMCpuTickGet(pVCpu)));
1908 pVCpu->hwaccm.s.vmx.proc_ctls |= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT;
1909 rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
1910 AssertRC(rc);
1911 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatTSCInterceptOverFlow);
1912 }
1913 }
1914 else
1915 {
1916 pVCpu->hwaccm.s.vmx.proc_ctls |= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT;
1917 rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
1918 AssertRC(rc);
1919 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatTSCIntercept);
1920 }
1921
1922 /* Done with the major changes */
1923 pVCpu->hwaccm.s.fContextUseFlags &= ~HWACCM_CHANGED_ALL_GUEST;
1924
1925 /* Minimal guest state update (esp, eip, eflags mostly) */
1926 VMXR0LoadMinimalGuestState(pVM, pVCpu, pCtx);
1927 return rc;
1928}
1929
1930/**
1931 * Syncs back the guest state
1932 *
1933 * @returns VBox status code.
1934 * @param pVM The VM to operate on.
1935 * @param pVCpu The VMCPU to operate on.
1936 * @param pCtx Guest context
1937 */
1938DECLINLINE(int) VMXR0SaveGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
1939{
1940 RTGCUINTREG val, valShadow;
1941 RTGCUINTPTR uInterruptState;
1942 int rc;
1943
1944 /* Let's first sync back eip, esp, and eflags. */
1945 rc = VMXReadCachedVMCS(VMX_VMCS64_GUEST_RIP, &val);
1946 AssertRC(rc);
1947 pCtx->rip = val;
1948 rc = VMXReadCachedVMCS(VMX_VMCS64_GUEST_RSP, &val);
1949 AssertRC(rc);
1950 pCtx->rsp = val;
1951 rc = VMXReadCachedVMCS(VMX_VMCS_GUEST_RFLAGS, &val);
1952 AssertRC(rc);
1953 pCtx->eflags.u32 = val;
1954
1955 /* Take care of instruction fusing (sti, mov ss) */
1956 rc |= VMXReadCachedVMCS(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &val);
1957 uInterruptState = val;
1958 if (uInterruptState != 0)
1959 {
1960 Assert(uInterruptState <= 2); /* only sti & mov ss */
1961 Log(("uInterruptState %x eip=%RGv\n", (uint32_t)uInterruptState, pCtx->rip));
1962 EMSetInhibitInterruptsPC(pVCpu, pCtx->rip);
1963 }
1964 else
1965 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
1966
1967 /* Control registers. */
1968 VMXReadCachedVMCS(VMX_VMCS_CTRL_CR0_READ_SHADOW, &valShadow);
1969 VMXReadCachedVMCS(VMX_VMCS64_GUEST_CR0, &val);
1970 val = (valShadow & pVCpu->hwaccm.s.vmx.cr0_mask) | (val & ~pVCpu->hwaccm.s.vmx.cr0_mask);
1971 CPUMSetGuestCR0(pVCpu, val);
1972
1973 VMXReadCachedVMCS(VMX_VMCS_CTRL_CR4_READ_SHADOW, &valShadow);
1974 VMXReadCachedVMCS(VMX_VMCS64_GUEST_CR4, &val);
1975 val = (valShadow & pVCpu->hwaccm.s.vmx.cr4_mask) | (val & ~pVCpu->hwaccm.s.vmx.cr4_mask);
1976 CPUMSetGuestCR4(pVCpu, val);
1977
1978 /* Note: no reason to sync back the CRx registers. They can't be changed by the guest. */
1979 /* Note: only in the nested paging case can CR3 & CR4 be changed by the guest. */
1980 if ( pVM->hwaccm.s.fNestedPaging
1981 && CPUMIsGuestInPagedProtectedModeEx(pCtx))
1982 {
1983 PVMCSCACHE pCache = &pVCpu->hwaccm.s.vmx.VMCSCache;
1984
1985 /* Can be updated behind our back in the nested paging case. */
1986 CPUMSetGuestCR2(pVCpu, pCache->cr2);
1987
1988 VMXReadCachedVMCS(VMX_VMCS64_GUEST_CR3, &val);
1989
1990 if (val != pCtx->cr3)
1991 {
1992 CPUMSetGuestCR3(pVCpu, val);
1993 PGMUpdateCR3(pVCpu, val);
1994 }
1995 /* Prefetch the four PDPT entries in PAE mode. */
1996 vmxR0PrefetchPAEPdptrs(pVM, pVCpu, pCtx);
1997 }
1998
1999 /* Sync back DR7 here. */
2000 VMXReadCachedVMCS(VMX_VMCS64_GUEST_DR7, &val);
2001 pCtx->dr[7] = val;
2002
2003 /* Guest CPU context: ES, CS, SS, DS, FS, GS. */
2004 VMX_READ_SELREG(ES, es);
2005 VMX_READ_SELREG(SS, ss);
2006 VMX_READ_SELREG(CS, cs);
2007 VMX_READ_SELREG(DS, ds);
2008 VMX_READ_SELREG(FS, fs);
2009 VMX_READ_SELREG(GS, gs);
2010
2011 /*
2012 * System MSRs
2013 */
2014 VMXReadCachedVMCS(VMX_VMCS32_GUEST_SYSENTER_CS, &val);
2015 pCtx->SysEnter.cs = val;
2016 VMXReadCachedVMCS(VMX_VMCS64_GUEST_SYSENTER_EIP, &val);
2017 pCtx->SysEnter.eip = val;
2018 VMXReadCachedVMCS(VMX_VMCS64_GUEST_SYSENTER_ESP, &val);
2019 pCtx->SysEnter.esp = val;
2020
2021 /* Misc. registers; must sync everything otherwise we can get out of sync when jumping to ring 3. */
2022 VMX_READ_SELREG(LDTR, ldtr);
2023
2024 VMXReadCachedVMCS(VMX_VMCS32_GUEST_GDTR_LIMIT, &val);
2025 pCtx->gdtr.cbGdt = val;
2026 VMXReadCachedVMCS(VMX_VMCS64_GUEST_GDTR_BASE, &val);
2027 pCtx->gdtr.pGdt = val;
2028
2029 VMXReadCachedVMCS(VMX_VMCS32_GUEST_IDTR_LIMIT, &val);
2030 pCtx->idtr.cbIdt = val;
2031 VMXReadCachedVMCS(VMX_VMCS64_GUEST_IDTR_BASE, &val);
2032 pCtx->idtr.pIdt = val;
2033
2034 /* Real mode emulation using v86 mode. */
2035 if ( CPUMIsGuestInRealModeEx(pCtx)
2036 && pVM->hwaccm.s.vmx.pRealModeTSS)
2037 {
2038 /* Hide our emulation flags */
2039 pCtx->eflags.Bits.u1VM = 0;
2040
2041 /* Restore original IOPL setting as we always use 0. */
2042 pCtx->eflags.Bits.u2IOPL = pVCpu->hwaccm.s.vmx.RealMode.eflags.Bits.u2IOPL;
2043
2044 /* Force a TR resync every time in case we switch modes. */
2045 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_GUEST_TR;
2046 }
2047 else
2048 {
2049 /* In real mode we have a fake TSS, so only sync it back when it's supposed to be valid. */
2050 VMX_READ_SELREG(TR, tr);
2051 }
2052
2053#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
2054 /* Save the possibly changed MSRs that we automatically restore and save during a world switch. */
2055 for (unsigned i = 0; i < pVCpu->hwaccm.s.vmx.cCachedMSRs; i++)
2056 {
2057 PVMXMSR pMsr = (PVMXMSR)pVCpu->hwaccm.s.vmx.pGuestMSR;
2058 pMsr += i;
2059
2060 switch (pMsr->u32IndexMSR)
2061 {
2062 case MSR_K8_LSTAR:
2063 pCtx->msrLSTAR = pMsr->u64Value;
2064 break;
2065 case MSR_K6_STAR:
2066 pCtx->msrSTAR = pMsr->u64Value;
2067 break;
2068 case MSR_K8_SF_MASK:
2069 pCtx->msrSFMASK = pMsr->u64Value;
2070 break;
2071 case MSR_K8_KERNEL_GS_BASE:
2072 pCtx->msrKERNELGSBASE = pMsr->u64Value;
2073 break;
2074 case MSR_K6_EFER:
2075 /* EFER can't be changed without causing a VM-exit. */
2076// Assert(pCtx->msrEFER == pMsr->u64Value);
2077 break;
2078 default:
2079 AssertFailed();
2080 return VERR_INTERNAL_ERROR;
2081 }
2082 }
2083#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
2084 return VINF_SUCCESS;
2085}
2086
2087/**
2088 * Dummy placeholder
2089 *
2090 * @param pVM The VM to operate on.
2091 * @param pVCpu The VMCPU to operate on.
2092 */
2093static void vmxR0SetupTLBDummy(PVM pVM, PVMCPU pVCpu)
2094{
2095 NOREF(pVM);
2096 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
2097 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
2098 pVCpu->hwaccm.s.TlbShootdown.cPages = 0;
2099 return;
2100}
2101
2102/**
2103 * Setup the tagged TLB for EPT
2104 *
2105 * @returns VBox status code.
2106 * @param pVM The VM to operate on.
2107 * @param pVCpu The VMCPU to operate on.
2108 */
2109static void vmxR0SetupTLBEPT(PVM pVM, PVMCPU pVCpu)
2110{
2111 PHWACCM_CPUINFO pCpu;
2112
2113 Assert(pVM->hwaccm.s.fNestedPaging);
2114 Assert(!pVM->hwaccm.s.vmx.fVPID);
2115
2116 /* Deal with tagged TLBs if VPID or EPT is supported. */
2117 pCpu = HWACCMR0GetCurrentCpu();
2118 /* Force a TLB flush for the first world switch if the current cpu differs from the one we ran on last. */
2119 /* Note that this can happen both for start and resume due to long jumps back to ring 3. */
2120 if ( pVCpu->hwaccm.s.idLastCpu != pCpu->idCpu
2121 /* if the tlb flush count has changed, another VM has flushed the TLB of this cpu, so we can't use our current ASID anymore. */
2122 || pVCpu->hwaccm.s.cTLBFlushes != pCpu->cTLBFlushes)
2123 {
2124 /* Force a TLB flush on VM entry. */
2125 pVCpu->hwaccm.s.fForceTLBFlush = true;
2126 }
2127 else
2128 Assert(!pCpu->fFlushTLB);
2129
2130 /* Check for tlb shootdown flushes. */
2131 if (VMCPU_FF_TESTANDCLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2132 pVCpu->hwaccm.s.fForceTLBFlush = true;
2133
2134 pVCpu->hwaccm.s.idLastCpu = pCpu->idCpu;
2135 pCpu->fFlushTLB = false;
2136
2137 if (pVCpu->hwaccm.s.fForceTLBFlush)
2138 {
2139 vmxR0FlushEPT(pVM, pVCpu, pVM->hwaccm.s.vmx.enmFlushContext, 0);
2140 }
2141 else
2142 if (VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
2143 {
2144 /* Deal with pending TLB shootdown actions which were queued when we were not executing code. */
2145 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatTlbShootdown);
2146
2147 for (unsigned i=0;i<pVCpu->hwaccm.s.TlbShootdown.cPages;i++)
2148 {
2149 /* aTlbShootdownPages contains physical addresses in this case. */
2150 vmxR0FlushEPT(pVM, pVCpu, pVM->hwaccm.s.vmx.enmFlushPage, pVCpu->hwaccm.s.TlbShootdown.aPages[i]);
2151 }
2152 }
2153 pVCpu->hwaccm.s.TlbShootdown.cPages= 0;
2154 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
2155
2156#ifdef VBOX_WITH_STATISTICS
2157 if (pVCpu->hwaccm.s.fForceTLBFlush)
2158 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatFlushTLBWorldSwitch);
2159 else
2160 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatNoFlushTLBWorldSwitch);
2161#endif
2162}
2163
2164#ifdef HWACCM_VTX_WITH_VPID
2165/**
2166 * Setup the tagged TLB for VPID
2167 *
2168 * @returns VBox status code.
2169 * @param pVM The VM to operate on.
2170 * @param pVCpu The VMCPU to operate on.
2171 */
2172static void vmxR0SetupTLBVPID(PVM pVM, PVMCPU pVCpu)
2173{
2174 PHWACCM_CPUINFO pCpu;
2175
2176 Assert(pVM->hwaccm.s.vmx.fVPID);
2177 Assert(!pVM->hwaccm.s.fNestedPaging);
2178
2179 /* Deal with tagged TLBs if VPID or EPT is supported. */
2180 pCpu = HWACCMR0GetCurrentCpu();
2181 /* Force a TLB flush for the first world switch if the current cpu differs from the one we ran on last. */
2182 /* Note that this can happen both for start and resume due to long jumps back to ring 3. */
2183 if ( pVCpu->hwaccm.s.idLastCpu != pCpu->idCpu
2184 /* if the tlb flush count has changed, another VM has flushed the TLB of this cpu, so we can't use our current ASID anymore. */
2185 || pVCpu->hwaccm.s.cTLBFlushes != pCpu->cTLBFlushes)
2186 {
2187 /* Force a TLB flush on VM entry. */
2188 pVCpu->hwaccm.s.fForceTLBFlush = true;
2189 }
2190 else
2191 Assert(!pCpu->fFlushTLB);
2192
2193 pVCpu->hwaccm.s.idLastCpu = pCpu->idCpu;
2194
2195 /* Check for tlb shootdown flushes. */
2196 if (VMCPU_FF_TESTANDCLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2197 pVCpu->hwaccm.s.fForceTLBFlush = true;
2198
2199 /* Make sure we flush the TLB when required. Switch ASID to achieve the same thing, but without actually flushing the whole TLB (which is expensive). */
2200 if (pVCpu->hwaccm.s.fForceTLBFlush)
2201 {
2202 if ( ++pCpu->uCurrentASID >= pVM->hwaccm.s.uMaxASID
2203 || pCpu->fFlushTLB)
2204 {
2205 pCpu->fFlushTLB = false;
2206 pCpu->uCurrentASID = 1; /* start at 1; host uses 0 */
2207 pCpu->cTLBFlushes++;
2208 vmxR0FlushVPID(pVM, pVCpu, VMX_FLUSH_ALL_CONTEXTS, 0);
2209 }
2210 else
2211 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatFlushASID);
2212
2213 pVCpu->hwaccm.s.fForceTLBFlush = false;
2214 pVCpu->hwaccm.s.cTLBFlushes = pCpu->cTLBFlushes;
2215 pVCpu->hwaccm.s.uCurrentASID = pCpu->uCurrentASID;
2216 }
2217 else
2218 {
2219 Assert(!pCpu->fFlushTLB);
2220 Assert(pVCpu->hwaccm.s.uCurrentASID && pCpu->uCurrentASID);
2221
2222 if (VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
2223 {
2224 /* Deal with pending TLB shootdown actions which were queued when we were not executing code. */
2225 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatTlbShootdown);
2226 for (unsigned i=0;i<pVCpu->hwaccm.s.TlbShootdown.cPages;i++)
2227 vmxR0FlushVPID(pVM, pVCpu, pVM->hwaccm.s.vmx.enmFlushPage, pVCpu->hwaccm.s.TlbShootdown.aPages[i]);
2228 }
2229 }
2230 pVCpu->hwaccm.s.TlbShootdown.cPages = 0;
2231 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
2232
2233 AssertMsg(pVCpu->hwaccm.s.cTLBFlushes == pCpu->cTLBFlushes, ("Flush count mismatch for cpu %d (%x vs %x)\n", pCpu->idCpu, pVCpu->hwaccm.s.cTLBFlushes, pCpu->cTLBFlushes));
2234 AssertMsg(pCpu->uCurrentASID >= 1 && pCpu->uCurrentASID < pVM->hwaccm.s.uMaxASID, ("cpu%d uCurrentASID = %x\n", pCpu->idCpu, pCpu->uCurrentASID));
2235 AssertMsg(pVCpu->hwaccm.s.uCurrentASID >= 1 && pVCpu->hwaccm.s.uCurrentASID < pVM->hwaccm.s.uMaxASID, ("cpu%d VM uCurrentASID = %x\n", pCpu->idCpu, pVCpu->hwaccm.s.uCurrentASID));
2236
2237 int rc = VMXWriteVMCS(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hwaccm.s.uCurrentASID);
2238 AssertRC(rc);
2239
2240 if (pVCpu->hwaccm.s.fForceTLBFlush)
2241 vmxR0FlushVPID(pVM, pVCpu, pVM->hwaccm.s.vmx.enmFlushContext, 0);
2242
2243#ifdef VBOX_WITH_STATISTICS
2244 if (pVCpu->hwaccm.s.fForceTLBFlush)
2245 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatFlushTLBWorldSwitch);
2246 else
2247 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatNoFlushTLBWorldSwitch);
2248#endif
2249}
2250#endif /* HWACCM_VTX_WITH_VPID */
2251
2252/**
2253 * Runs guest code in a VT-x VM.
2254 *
2255 * @returns VBox status code.
2256 * @param pVM The VM to operate on.
2257 * @param pVCpu The VMCPU to operate on.
2258 * @param pCtx Guest context
2259 */
2260VMMR0DECL(int) VMXR0RunGuestCode(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
2261{
2262 STAM_PROFILE_ADV_START(&pVCpu->hwaccm.s.StatEntry, x);
2263 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hwaccm.s.StatExit1);
2264 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hwaccm.s.StatExit2);
2265
2266 VBOXSTRICTRC rc = VINF_SUCCESS;
2267 int rc2;
2268 RTGCUINTREG val;
2269 RTGCUINTREG exitReason = (RTGCUINTREG)VMX_EXIT_INVALID;
2270 RTGCUINTREG instrError, cbInstr;
2271 RTGCUINTPTR exitQualification = 0;
2272 RTGCUINTPTR intInfo = 0; /* shut up buggy gcc 4 */
2273 RTGCUINTPTR errCode, instrInfo;
2274 bool fSetupTPRCaching = false;
2275 uint64_t u64OldLSTAR = 0;
2276 uint8_t u8LastTPR = 0;
2277 RTCCUINTREG uOldEFlags = ~(RTCCUINTREG)0;
2278 unsigned cResume = 0;
2279#ifdef VBOX_STRICT
2280 RTCPUID idCpuCheck;
2281 bool fWasInLongMode = false;
2282#endif
2283#ifdef VBOX_HIGH_RES_TIMERS_HACK_IN_RING0
2284 uint64_t u64LastTime = RTTimeMilliTS();
2285#endif
2286
2287 Assert(!(pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC) || (pVCpu->hwaccm.s.vmx.pVAPIC && pVM->hwaccm.s.vmx.pAPIC));
2288
2289 /* Check if we need to use TPR shadowing. */
2290 if ( CPUMIsGuestInLongModeEx(pCtx)
2291 || ( ((pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC) || pVM->hwaccm.s.fTRPPatchingAllowed)
2292 && pVM->hwaccm.s.fHasIoApic)
2293 )
2294 {
2295 fSetupTPRCaching = true;
2296 }
2297
2298 Log2(("\nE"));
2299
2300#ifdef VBOX_STRICT
2301 {
2302 RTCCUINTREG val2;
2303
2304 rc2 = VMXReadVMCS(VMX_VMCS_CTRL_PIN_EXEC_CONTROLS, &val2);
2305 AssertRC(rc2);
2306 Log2(("VMX_VMCS_CTRL_PIN_EXEC_CONTROLS = %08x\n", val2));
2307
2308 /* allowed zero */
2309 if ((val2 & pVM->hwaccm.s.vmx.msr.vmx_pin_ctls.n.disallowed0) != pVM->hwaccm.s.vmx.msr.vmx_pin_ctls.n.disallowed0)
2310 Log(("Invalid VMX_VMCS_CTRL_PIN_EXEC_CONTROLS: zero\n"));
2311
2312 /* allowed one */
2313 if ((val2 & ~pVM->hwaccm.s.vmx.msr.vmx_pin_ctls.n.allowed1) != 0)
2314 Log(("Invalid VMX_VMCS_CTRL_PIN_EXEC_CONTROLS: one\n"));
2315
2316 rc2 = VMXReadVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, &val2);
2317 AssertRC(rc2);
2318 Log2(("VMX_VMCS_CTRL_PROC_EXEC_CONTROLS = %08x\n", val2));
2319
2320 /* Must be set according to the MSR, but can be cleared in case of EPT. */
2321 if (pVM->hwaccm.s.fNestedPaging)
2322 val2 |= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INVLPG_EXIT
2323 | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
2324 | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT;
2325
2326 /* allowed zero */
2327 if ((val2 & pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.disallowed0) != pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.disallowed0)
2328 Log(("Invalid VMX_VMCS_CTRL_PROC_EXEC_CONTROLS: zero\n"));
2329
2330 /* allowed one */
2331 if ((val2 & ~pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1) != 0)
2332 Log(("Invalid VMX_VMCS_CTRL_PROC_EXEC_CONTROLS: one\n"));
2333
2334 rc2 = VMXReadVMCS(VMX_VMCS_CTRL_ENTRY_CONTROLS, &val2);
2335 AssertRC(rc2);
2336 Log2(("VMX_VMCS_CTRL_ENTRY_CONTROLS = %08x\n", val2));
2337
2338 /* allowed zero */
2339 if ((val2 & pVM->hwaccm.s.vmx.msr.vmx_entry.n.disallowed0) != pVM->hwaccm.s.vmx.msr.vmx_entry.n.disallowed0)
2340 Log(("Invalid VMX_VMCS_CTRL_ENTRY_CONTROLS: zero\n"));
2341
2342 /* allowed one */
2343 if ((val2 & ~pVM->hwaccm.s.vmx.msr.vmx_entry.n.allowed1) != 0)
2344 Log(("Invalid VMX_VMCS_CTRL_ENTRY_CONTROLS: one\n"));
2345
2346 rc2 = VMXReadVMCS(VMX_VMCS_CTRL_EXIT_CONTROLS, &val2);
2347 AssertRC(rc2);
2348 Log2(("VMX_VMCS_CTRL_EXIT_CONTROLS = %08x\n", val2));
2349
2350 /* allowed zero */
2351 if ((val2 & pVM->hwaccm.s.vmx.msr.vmx_exit.n.disallowed0) != pVM->hwaccm.s.vmx.msr.vmx_exit.n.disallowed0)
2352 Log(("Invalid VMX_VMCS_CTRL_EXIT_CONTROLS: zero\n"));
2353
2354 /* allowed one */
2355 if ((val2 & ~pVM->hwaccm.s.vmx.msr.vmx_exit.n.allowed1) != 0)
2356 Log(("Invalid VMX_VMCS_CTRL_EXIT_CONTROLS: one\n"));
2357 }
2358 fWasInLongMode = CPUMIsGuestInLongModeEx(pCtx);
2359#endif /* VBOX_STRICT */
2360
2361#ifdef VBOX_WITH_CRASHDUMP_MAGIC
2362 pVCpu->hwaccm.s.vmx.VMCSCache.u64TimeEntry = RTTimeNanoTS();
2363#endif
2364
2365 /* We can jump to this point to resume execution after determining that a VM-exit is innocent.
2366 */
2367ResumeExecution:
2368 if (!STAM_REL_PROFILE_ADV_IS_RUNNING(&pVCpu->hwaccm.s.StatEntry))
2369 STAM_REL_PROFILE_ADV_STOP_START(&pVCpu->hwaccm.s.StatExit2, &pVCpu->hwaccm.s.StatEntry, x);
2370 AssertMsg(pVCpu->hwaccm.s.idEnteredCpu == RTMpCpuId(),
2371 ("Expected %d, I'm %d; cResume=%d exitReason=%RGv exitQualification=%RGv\n",
2372 (int)pVCpu->hwaccm.s.idEnteredCpu, (int)RTMpCpuId(), cResume, exitReason, exitQualification));
2373 Assert(!HWACCMR0SuspendPending());
2374 /* Not allowed to switch modes without reloading the host state (32->64 switcher)!! */
2375 Assert(fWasInLongMode == CPUMIsGuestInLongModeEx(pCtx));
2376
2377 /* Safety precaution; looping for too long here can have a very bad effect on the host */
2378 if (RT_UNLIKELY(++cResume > pVM->hwaccm.s.cMaxResumeLoops))
2379 {
2380 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitMaxResume);
2381 rc = VINF_EM_RAW_INTERRUPT;
2382 goto end;
2383 }
2384
2385 /* Check for irq inhibition due to instruction fusing (sti, mov ss). */
2386 if (VMCPU_FF_ISSET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
2387 {
2388 Log(("VM_FF_INHIBIT_INTERRUPTS at %RGv successor %RGv\n", (RTGCPTR)pCtx->rip, EMGetInhibitInterruptsPC(pVCpu)));
2389 if (pCtx->rip != EMGetInhibitInterruptsPC(pVCpu))
2390 {
2391 /* Note: we intentionally don't clear VM_FF_INHIBIT_INTERRUPTS here.
2392 * Before we are able to execute this instruction in raw mode (iret to guest code) an external interrupt might
2393 * force a world switch again. Possibly allowing a guest interrupt to be dispatched in the process. This could
2394 * break the guest. Sounds very unlikely, but such timing sensitive problems are not as rare as you might think.
2395 */
2396 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
2397 /* Irq inhibition is no longer active; clear the corresponding VMX state. */
2398 rc2 = VMXWriteVMCS(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, 0);
2399 AssertRC(rc2);
2400 }
2401 }
2402 else
2403 {
2404 /* Irq inhibition is no longer active; clear the corresponding VMX state. */
2405 rc2 = VMXWriteVMCS(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, 0);
2406 AssertRC(rc2);
2407 }
2408
2409#ifdef VBOX_HIGH_RES_TIMERS_HACK_IN_RING0
2410 if (RT_UNLIKELY((cResume & 0xf) == 0))
2411 {
2412 uint64_t u64CurTime = RTTimeMilliTS();
2413
2414 if (RT_UNLIKELY(u64CurTime > u64LastTime))
2415 {
2416 u64LastTime = u64CurTime;
2417 TMTimerPollVoid(pVM, pVCpu);
2418 }
2419 }
2420#endif
2421
2422 /* Check for pending actions that force us to go back to ring 3. */
2423 if ( VM_FF_ISPENDING(pVM, VM_FF_HWACCM_TO_R3_MASK | VM_FF_REQUEST | VM_FF_PGM_POOL_FLUSH_PENDING)
2424 || VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_HWACCM_TO_R3_MASK | VMCPU_FF_PGM_SYNC_CR3 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL | VMCPU_FF_REQUEST))
2425 {
2426 /* Check if a sync operation is pending. */
2427 if (VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
2428 {
2429 rc = PGMSyncCR3(pVCpu, pCtx->cr0, pCtx->cr3, pCtx->cr4, VMCPU_FF_ISSET(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
2430 if (rc != VINF_SUCCESS)
2431 {
2432 AssertRC(VBOXSTRICTRC_VAL(rc));
2433 Log(("Pending pool sync is forcing us back to ring 3; rc=%d\n", VBOXSTRICTRC_VAL(rc)));
2434 goto end;
2435 }
2436 }
2437
2438#ifdef DEBUG
2439 /* Intercept X86_XCPT_DB if stepping is enabled */
2440 if (!DBGFIsStepping(pVCpu))
2441#endif
2442 {
2443 if ( VM_FF_ISPENDING(pVM, VM_FF_HWACCM_TO_R3_MASK)
2444 || VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_HWACCM_TO_R3_MASK))
2445 {
2446 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
2447 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatSwitchToR3);
2448 rc = RT_UNLIKELY(VM_FF_ISPENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
2449 goto end;
2450 }
2451 }
2452
2453 /* Pending request packets might contain actions that need immediate attention, such as pending hardware interrupts. */
2454 if ( VM_FF_ISPENDING(pVM, VM_FF_REQUEST)
2455 || VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_REQUEST))
2456 {
2457 rc = VINF_EM_PENDING_REQUEST;
2458 goto end;
2459 }
2460
2461 /* Check if a pgm pool flush is in progress. */
2462 if (VM_FF_ISPENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
2463 {
2464 rc = VINF_PGM_POOL_FLUSH_PENDING;
2465 goto end;
2466 }
2467 }
2468
2469#ifdef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
2470 /*
2471 * Exit to ring-3 preemption/work is pending.
2472 *
2473 * Interrupts are disabled before the call to make sure we don't miss any interrupt
2474 * that would flag preemption (IPI, timer tick, ++). (Would've been nice to do this
2475 * further down, but VMXR0CheckPendingInterrupt makes that impossible.)
2476 *
2477 * Note! Interrupts must be disabled done *before* we check for TLB flushes; TLB
2478 * shootdowns rely on this.
2479 */
2480 uOldEFlags = ASMIntDisableFlags();
2481 if (RTThreadPreemptIsPending(NIL_RTTHREAD))
2482 {
2483 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitPreemptPending);
2484 rc = VINF_EM_RAW_INTERRUPT;
2485 goto end;
2486 }
2487 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
2488#endif
2489
2490 /* When external interrupts are pending, we should exit the VM when IF is set. */
2491 /* Note! *After* VM_FF_INHIBIT_INTERRUPTS check!!! */
2492 rc = VMXR0CheckPendingInterrupt(pVM, pVCpu, pCtx);
2493 if (RT_FAILURE(rc))
2494 goto end;
2495
2496 /** @todo check timers?? */
2497
2498 /* TPR caching using CR8 is only available in 64 bits mode */
2499 /* Note the 32 bits exception for AMD (X86_CPUID_AMD_FEATURE_ECX_CR8L), but that appears missing in Intel CPUs */
2500 /* Note: we can't do this in LoadGuestState as PDMApicGetTPR can jump back to ring 3 (lock)!!!!! (no longer true) */
2501 /**
2502 * @todo query and update the TPR only when it could have been changed (mmio access & wrmsr (x2apic))
2503 */
2504 if (fSetupTPRCaching)
2505 {
2506 /* TPR caching in CR8 */
2507 bool fPending;
2508
2509 rc2 = PDMApicGetTPR(pVCpu, &u8LastTPR, &fPending);
2510 AssertRC(rc2);
2511 /* The TPR can be found at offset 0x80 in the APIC mmio page. */
2512 pVCpu->hwaccm.s.vmx.pVAPIC[0x80] = u8LastTPR;
2513
2514 /* Two options here:
2515 * - external interrupt pending, but masked by the TPR value.
2516 * -> a CR8 update that lower the current TPR value should cause an exit
2517 * - no pending interrupts
2518 * -> We don't need to be explicitely notified. There are enough world switches for detecting pending interrupts.
2519 */
2520 rc = VMXWriteVMCS(VMX_VMCS_CTRL_TPR_THRESHOLD, (fPending) ? (u8LastTPR >> 4) : 0); /* cr8 bits 3-0 correspond to bits 7-4 of the task priority mmio register. */
2521 AssertRC(VBOXSTRICTRC_VAL(rc));
2522
2523 if (pVM->hwaccm.s.fTPRPatchingActive)
2524 {
2525 Assert(!CPUMIsGuestInLongModeEx(pCtx));
2526 /* Our patch code uses LSTAR for TPR caching. */
2527 pCtx->msrLSTAR = u8LastTPR;
2528
2529 if (fPending)
2530 {
2531 /* A TPR change could activate a pending interrupt, so catch lstar writes. */
2532 vmxR0SetMSRPermission(pVCpu, MSR_K8_LSTAR, true, false);
2533 }
2534 else
2535 {
2536 /* No interrupts are pending, so we don't need to be explicitely notified.
2537 * There are enough world switches for detecting pending interrupts.
2538 */
2539 vmxR0SetMSRPermission(pVCpu, MSR_K8_LSTAR, true, true);
2540 }
2541 }
2542 }
2543
2544#if defined(HWACCM_VTX_WITH_EPT) && defined(LOG_ENABLED)
2545 if ( pVM->hwaccm.s.fNestedPaging
2546# ifdef HWACCM_VTX_WITH_VPID
2547 || pVM->hwaccm.s.vmx.fVPID
2548# endif /* HWACCM_VTX_WITH_VPID */
2549 )
2550 {
2551 PHWACCM_CPUINFO pCpu;
2552
2553 pCpu = HWACCMR0GetCurrentCpu();
2554 if ( pVCpu->hwaccm.s.idLastCpu != pCpu->idCpu
2555 || pVCpu->hwaccm.s.cTLBFlushes != pCpu->cTLBFlushes)
2556 {
2557 if (pVCpu->hwaccm.s.idLastCpu != pCpu->idCpu)
2558 LogFlow(("Force TLB flush due to rescheduling to a different cpu (%d vs %d)\n", pVCpu->hwaccm.s.idLastCpu, pCpu->idCpu));
2559 else
2560 LogFlow(("Force TLB flush due to changed TLB flush count (%x vs %x)\n", pVCpu->hwaccm.s.cTLBFlushes, pCpu->cTLBFlushes));
2561 }
2562 if (pCpu->fFlushTLB)
2563 LogFlow(("Force TLB flush: first time cpu %d is used -> flush\n", pCpu->idCpu));
2564 else
2565 if (pVCpu->hwaccm.s.fForceTLBFlush)
2566 LogFlow(("Manual TLB flush\n"));
2567 }
2568#endif
2569#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
2570 PGMRZDynMapFlushAutoSet(pVCpu);
2571#endif
2572
2573 /*
2574 * NOTE: DO NOT DO ANYTHING AFTER THIS POINT THAT MIGHT JUMP BACK TO RING 3!
2575 * (until the actual world switch)
2576 */
2577#ifdef VBOX_STRICT
2578 idCpuCheck = RTMpCpuId();
2579#endif
2580#ifdef LOG_ENABLED
2581 VMMR0LogFlushDisable(pVCpu);
2582#endif
2583 /* Save the host state first. */
2584 if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_HOST_CONTEXT)
2585 {
2586 rc = VMXR0SaveHostState(pVM, pVCpu);
2587 if (RT_UNLIKELY(rc != VINF_SUCCESS))
2588 {
2589 VMMR0LogFlushEnable(pVCpu);
2590 goto end;
2591 }
2592 }
2593
2594 /* Load the guest state */
2595#if 0
2596 if (!pVCpu->hwaccm.s.fContextUseFlags)
2597 {
2598 VMXR0LoadMinimalGuestState(pVM, pVCpu, pCtx);
2599 }
2600 else
2601#endif
2602 {
2603 rc = VMXR0LoadGuestState(pVM, pVCpu, pCtx);
2604 if (RT_UNLIKELY(rc != VINF_SUCCESS))
2605 {
2606 VMMR0LogFlushEnable(pVCpu);
2607 goto end;
2608 }
2609 }
2610
2611#ifndef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
2612 /* Disable interrupts to make sure a poke will interrupt execution.
2613 * This must be done *before* we check for TLB flushes; TLB shootdowns rely on this.
2614 */
2615 uOldEFlags = ASMIntDisableFlags();
2616 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
2617#endif
2618
2619 /* Non-register state Guest Context */
2620 /** @todo change me according to cpu state */
2621 rc2 = VMXWriteVMCS(VMX_VMCS32_GUEST_ACTIVITY_STATE, VMX_CMS_GUEST_ACTIVITY_ACTIVE);
2622 AssertRC(rc2);
2623
2624 /** Set TLB flush state as checked until we return from the world switch. */
2625 ASMAtomicWriteU8(&pVCpu->hwaccm.s.fCheckedTLBFlush, true);
2626 /* Deal with tagged TLB setup and invalidation. */
2627 pVM->hwaccm.s.vmx.pfnSetupTaggedTLB(pVM, pVCpu);
2628
2629 /* Manual save and restore:
2630 * - General purpose registers except RIP, RSP
2631 *
2632 * Trashed:
2633 * - CR2 (we don't care)
2634 * - LDTR (reset to 0)
2635 * - DRx (presumably not changed at all)
2636 * - DR7 (reset to 0x400)
2637 * - EFLAGS (reset to RT_BIT(1); not relevant)
2638 *
2639 */
2640
2641 /* All done! Let's start VM execution. */
2642 STAM_PROFILE_ADV_STOP_START(&pVCpu->hwaccm.s.StatEntry, &pVCpu->hwaccm.s.StatInGC, x);
2643 Assert(idCpuCheck == RTMpCpuId());
2644
2645#ifdef VBOX_WITH_CRASHDUMP_MAGIC
2646 pVCpu->hwaccm.s.vmx.VMCSCache.cResume = cResume;
2647 pVCpu->hwaccm.s.vmx.VMCSCache.u64TimeSwitch = RTTimeNanoTS();
2648#endif
2649
2650 /* Save the current TPR value in the LSTAR msr so our patches can access it. */
2651 if (pVM->hwaccm.s.fTPRPatchingActive)
2652 {
2653 Assert(pVM->hwaccm.s.fTPRPatchingActive);
2654 u64OldLSTAR = ASMRdMsr(MSR_K8_LSTAR);
2655 ASMWrMsr(MSR_K8_LSTAR, u8LastTPR);
2656 }
2657
2658 TMNotifyStartOfExecution(pVCpu);
2659#ifdef VBOX_WITH_KERNEL_USING_XMM
2660 rc = hwaccmR0VMXStartVMWrapXMM(pVCpu->hwaccm.s.fResumeVM, pCtx, &pVCpu->hwaccm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hwaccm.s.vmx.pfnStartVM);
2661#else
2662 rc = pVCpu->hwaccm.s.vmx.pfnStartVM(pVCpu->hwaccm.s.fResumeVM, pCtx, &pVCpu->hwaccm.s.vmx.VMCSCache, pVM, pVCpu);
2663#endif
2664 ASMAtomicWriteU8(&pVCpu->hwaccm.s.fCheckedTLBFlush, false);
2665 ASMAtomicIncU32(&pVCpu->hwaccm.s.cWorldSwitchExit);
2666 /* Possibly the last TSC value seen by the guest (too high) (only when we're in tsc offset mode). */
2667 if (!(pVCpu->hwaccm.s.vmx.proc_ctls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT))
2668 TMCpuTickSetLastSeen(pVCpu, ASMReadTSC() + pVCpu->hwaccm.s.vmx.u64TSCOffset - 0x400 /* guestimate of world switch overhead in clock ticks */);
2669
2670 TMNotifyEndOfExecution(pVCpu);
2671 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED);
2672 Assert(!(ASMGetFlags() & X86_EFL_IF));
2673
2674 /* Restore the host LSTAR msr if the guest could have changed it. */
2675 if (pVM->hwaccm.s.fTPRPatchingActive)
2676 {
2677 Assert(pVM->hwaccm.s.fTPRPatchingActive);
2678 pVCpu->hwaccm.s.vmx.pVAPIC[0x80] = pCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR);
2679 ASMWrMsr(MSR_K8_LSTAR, u64OldLSTAR);
2680 }
2681
2682 STAM_PROFILE_ADV_STOP_START(&pVCpu->hwaccm.s.StatInGC, &pVCpu->hwaccm.s.StatExit1, x);
2683 ASMSetFlags(uOldEFlags);
2684#ifdef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
2685 uOldEFlags = ~(RTCCUINTREG)0;
2686#endif
2687
2688 AssertMsg(!pVCpu->hwaccm.s.vmx.VMCSCache.Write.cValidEntries, ("pVCpu->hwaccm.s.vmx.VMCSCache.Write.cValidEntries=%d\n", pVCpu->hwaccm.s.vmx.VMCSCache.Write.cValidEntries));
2689
2690 /* In case we execute a goto ResumeExecution later on. */
2691 pVCpu->hwaccm.s.fResumeVM = true;
2692 pVCpu->hwaccm.s.fForceTLBFlush = false;
2693
2694 /*
2695 * !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
2696 * IMPORTANT: WE CAN'T DO ANY LOGGING OR OPERATIONS THAT CAN DO A LONGJMP BACK TO RING 3 *BEFORE* WE'VE SYNCED BACK (MOST OF) THE GUEST STATE
2697 * !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
2698 */
2699
2700 if (RT_UNLIKELY(rc != VINF_SUCCESS))
2701 {
2702 VMXR0ReportWorldSwitchError(pVM, pVCpu, rc, pCtx);
2703 VMMR0LogFlushEnable(pVCpu);
2704 goto end;
2705 }
2706
2707 /* Success. Query the guest state and figure out what has happened. */
2708
2709 /* Investigate why there was a VM-exit. */
2710 rc2 = VMXReadCachedVMCS(VMX_VMCS32_RO_EXIT_REASON, &exitReason);
2711 STAM_COUNTER_INC(&pVCpu->hwaccm.s.paStatExitReasonR0[exitReason & MASK_EXITREASON_STAT]);
2712
2713 exitReason &= 0xffff; /* bit 0-15 contain the exit code. */
2714 rc2 |= VMXReadCachedVMCS(VMX_VMCS32_RO_VM_INSTR_ERROR, &instrError);
2715 rc2 |= VMXReadCachedVMCS(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &cbInstr);
2716 rc2 |= VMXReadCachedVMCS(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &intInfo);
2717 /* might not be valid; depends on VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID. */
2718 rc2 |= VMXReadCachedVMCS(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERRCODE, &errCode);
2719 rc2 |= VMXReadCachedVMCS(VMX_VMCS32_RO_EXIT_INSTR_INFO, &instrInfo);
2720 rc2 |= VMXReadCachedVMCS(VMX_VMCS_RO_EXIT_QUALIFICATION, &exitQualification);
2721 AssertRC(rc2);
2722
2723 /* Sync back the guest state */
2724 rc2 = VMXR0SaveGuestState(pVM, pVCpu, pCtx);
2725 AssertRC(rc2);
2726
2727 /* Note! NOW IT'S SAFE FOR LOGGING! */
2728 VMMR0LogFlushEnable(pVCpu);
2729 Log2(("Raw exit reason %08x\n", exitReason));
2730
2731 /* Check if an injected event was interrupted prematurely. */
2732 rc2 = VMXReadCachedVMCS(VMX_VMCS32_RO_IDT_INFO, &val);
2733 AssertRC(rc2);
2734 pVCpu->hwaccm.s.Event.intInfo = VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(val);
2735 if ( VMX_EXIT_INTERRUPTION_INFO_VALID(pVCpu->hwaccm.s.Event.intInfo)
2736 /* Ignore 'int xx' as they'll be restarted anyway. */
2737 && VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hwaccm.s.Event.intInfo) != VMX_EXIT_INTERRUPTION_INFO_TYPE_SW
2738 /* Ignore software exceptions (such as int3) as they'll reoccur when we restart the instruction anyway. */
2739 && VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hwaccm.s.Event.intInfo) != VMX_EXIT_INTERRUPTION_INFO_TYPE_SWEXCPT)
2740 {
2741 Assert(!pVCpu->hwaccm.s.Event.fPending);
2742 pVCpu->hwaccm.s.Event.fPending = true;
2743 /* Error code present? */
2744 if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(pVCpu->hwaccm.s.Event.intInfo))
2745 {
2746 rc2 = VMXReadCachedVMCS(VMX_VMCS32_RO_IDT_ERRCODE, &val);
2747 AssertRC(rc2);
2748 pVCpu->hwaccm.s.Event.errCode = val;
2749 Log(("Pending inject %RX64 at %RGv exit=%08x intInfo=%08x exitQualification=%RGv pending error=%RX64\n", pVCpu->hwaccm.s.Event.intInfo, (RTGCPTR)pCtx->rip, exitReason, intInfo, exitQualification, val));
2750 }
2751 else
2752 {
2753 Log(("Pending inject %RX64 at %RGv exit=%08x intInfo=%08x exitQualification=%RGv\n", pVCpu->hwaccm.s.Event.intInfo, (RTGCPTR)pCtx->rip, exitReason, intInfo, exitQualification));
2754 pVCpu->hwaccm.s.Event.errCode = 0;
2755 }
2756 }
2757#ifdef VBOX_STRICT
2758 else
2759 if ( VMX_EXIT_INTERRUPTION_INFO_VALID(pVCpu->hwaccm.s.Event.intInfo)
2760 /* Ignore software exceptions (such as int3) as they're reoccur when we restart the instruction anyway. */
2761 && VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hwaccm.s.Event.intInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_SWEXCPT)
2762 {
2763 Log(("Ignore pending inject %RX64 at %RGv exit=%08x intInfo=%08x exitQualification=%RGv\n", pVCpu->hwaccm.s.Event.intInfo, (RTGCPTR)pCtx->rip, exitReason, intInfo, exitQualification));
2764 }
2765
2766 if (exitReason == VMX_EXIT_ERR_INVALID_GUEST_STATE)
2767 HWACCMDumpRegs(pVM, pVCpu, pCtx);
2768#endif
2769
2770 Log2(("E%d: New EIP=%x:%RGv\n", (uint32_t)exitReason, pCtx->cs, (RTGCPTR)pCtx->rip));
2771 Log2(("Exit reason %d, exitQualification %RGv\n", (uint32_t)exitReason, exitQualification));
2772 Log2(("instrInfo=%d instrError=%d instr length=%d\n", (uint32_t)instrInfo, (uint32_t)instrError, (uint32_t)cbInstr));
2773 Log2(("Interruption error code %d\n", (uint32_t)errCode));
2774 Log2(("IntInfo = %08x\n", (uint32_t)intInfo));
2775
2776 /* Sync back the TPR if it was changed. */
2777 if ( fSetupTPRCaching
2778 && u8LastTPR != pVCpu->hwaccm.s.vmx.pVAPIC[0x80])
2779 {
2780 rc2 = PDMApicSetTPR(pVCpu, pVCpu->hwaccm.s.vmx.pVAPIC[0x80]);
2781 AssertRC(rc2);
2782 }
2783
2784 STAM_PROFILE_ADV_STOP_START(&pVCpu->hwaccm.s.StatExit1, &pVCpu->hwaccm.s.StatExit2, x);
2785
2786 /* Some cases don't need a complete resync of the guest CPU state; handle them here. */
2787 Assert(rc == VINF_SUCCESS); /* might consider VERR_IPE_UNINITIALIZED_STATUS here later... */
2788 switch (exitReason)
2789 {
2790 case VMX_EXIT_EXCEPTION: /* 0 Exception or non-maskable interrupt (NMI). */
2791 case VMX_EXIT_EXTERNAL_IRQ: /* 1 External interrupt. */
2792 {
2793 uint32_t vector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(intInfo);
2794
2795 if (!VMX_EXIT_INTERRUPTION_INFO_VALID(intInfo))
2796 {
2797 Assert(exitReason == VMX_EXIT_EXTERNAL_IRQ);
2798#if 0 //def VBOX_WITH_VMMR0_DISABLE_PREEMPTION
2799 if ( RTThreadPreemptIsPendingTrusty()
2800 && !RTThreadPreemptIsPending(NIL_RTTHREAD))
2801 goto ResumeExecution;
2802#endif
2803 /* External interrupt; leave to allow it to be dispatched again. */
2804 rc = VINF_EM_RAW_INTERRUPT;
2805 break;
2806 }
2807 STAM_PROFILE_ADV_START(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
2808 switch (VMX_EXIT_INTERRUPTION_INFO_TYPE(intInfo))
2809 {
2810 case VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI: /* Non-maskable interrupt. */
2811 /* External interrupt; leave to allow it to be dispatched again. */
2812 rc = VINF_EM_RAW_INTERRUPT;
2813 break;
2814
2815 case VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT: /* External hardware interrupt. */
2816 AssertFailed(); /* can't come here; fails the first check. */
2817 break;
2818
2819 case VMX_EXIT_INTERRUPTION_INFO_TYPE_DBEXCPT: /* Unknown why we get this type for #DB */
2820 case VMX_EXIT_INTERRUPTION_INFO_TYPE_SWEXCPT: /* Software exception. (#BP or #OF) */
2821 Assert(vector == 1 || vector == 3 || vector == 4);
2822 /* no break */
2823 case VMX_EXIT_INTERRUPTION_INFO_TYPE_HWEXCPT: /* Hardware exception. */
2824 Log2(("Hardware/software interrupt %d\n", vector));
2825 switch (vector)
2826 {
2827 case X86_XCPT_NM:
2828 {
2829 Log(("#NM fault at %RGv error code %x\n", (RTGCPTR)pCtx->rip, errCode));
2830
2831 /** @todo don't intercept #NM exceptions anymore when we've activated the guest FPU state. */
2832 /* If we sync the FPU/XMM state on-demand, then we can continue execution as if nothing has happened. */
2833 rc = CPUMR0LoadGuestFPU(pVM, pVCpu, pCtx);
2834 if (rc == VINF_SUCCESS)
2835 {
2836 Assert(CPUMIsGuestFPUStateActive(pVCpu));
2837
2838 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitShadowNM);
2839
2840 /* Continue execution. */
2841 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_GUEST_CR0;
2842
2843 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
2844 goto ResumeExecution;
2845 }
2846
2847 Log(("Forward #NM fault to the guest\n"));
2848 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestNM);
2849 rc2 = VMXR0InjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), cbInstr, 0);
2850 AssertRC(rc2);
2851 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
2852 goto ResumeExecution;
2853 }
2854
2855 case X86_XCPT_PF: /* Page fault */
2856 {
2857#ifdef DEBUG
2858 if (pVM->hwaccm.s.fNestedPaging)
2859 { /* A genuine pagefault.
2860 * Forward the trap to the guest by injecting the exception and resuming execution.
2861 */
2862 Log(("Guest page fault at %RGv cr2=%RGv error code %RGv rsp=%RGv\n", (RTGCPTR)pCtx->rip, exitQualification, errCode, (RTGCPTR)pCtx->rsp));
2863
2864 Assert(CPUMIsGuestInPagedProtectedModeEx(pCtx));
2865
2866 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestPF);
2867
2868 /* Now we must update CR2. */
2869 pCtx->cr2 = exitQualification;
2870 rc2 = VMXR0InjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), cbInstr, errCode);
2871 AssertRC(rc2);
2872
2873 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
2874 goto ResumeExecution;
2875 }
2876#endif
2877 Assert(!pVM->hwaccm.s.fNestedPaging);
2878
2879#ifdef VBOX_HWACCM_WITH_GUEST_PATCHING
2880 /* Shortcut for APIC TPR reads and writes; 32 bits guests only */
2881 if ( pVM->hwaccm.s.fTRPPatchingAllowed
2882 && pVM->hwaccm.s.pGuestPatchMem
2883 && (exitQualification & 0xfff) == 0x080
2884 && !(errCode & X86_TRAP_PF_P) /* not present */
2885 && CPUMGetGuestCPL(pVCpu, CPUMCTX2CORE(pCtx)) == 0
2886 && !CPUMIsGuestInLongModeEx(pCtx)
2887 && pVM->hwaccm.s.cPatches < RT_ELEMENTS(pVM->hwaccm.s.aPatches))
2888 {
2889 RTGCPHYS GCPhysApicBase, GCPhys;
2890 PDMApicGetBase(pVM, &GCPhysApicBase); /* @todo cache this */
2891 GCPhysApicBase &= PAGE_BASE_GC_MASK;
2892
2893 rc = PGMGstGetPage(pVCpu, (RTGCPTR)exitQualification, NULL, &GCPhys);
2894 if ( rc == VINF_SUCCESS
2895 && GCPhys == GCPhysApicBase)
2896 {
2897 /* Only attempt to patch the instruction once. */
2898 PHWACCMTPRPATCH pPatch = (PHWACCMTPRPATCH)RTAvloU32Get(&pVM->hwaccm.s.PatchTree, (AVLOU32KEY)pCtx->eip);
2899 if (!pPatch)
2900 {
2901 rc = VINF_EM_HWACCM_PATCH_TPR_INSTR;
2902 break;
2903 }
2904 }
2905 }
2906#endif
2907
2908 Log2(("Page fault at %RGv error code %x\n", exitQualification, errCode));
2909 /* Exit qualification contains the linear address of the page fault. */
2910 TRPMAssertTrap(pVCpu, X86_XCPT_PF, TRPM_TRAP);
2911 TRPMSetErrorCode(pVCpu, errCode);
2912 TRPMSetFaultAddress(pVCpu, exitQualification);
2913
2914 /* Shortcut for APIC TPR reads and writes. */
2915 if ( (exitQualification & 0xfff) == 0x080
2916 && !(errCode & X86_TRAP_PF_P) /* not present */
2917 && fSetupTPRCaching
2918 && (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC))
2919 {
2920 RTGCPHYS GCPhysApicBase, GCPhys;
2921 PDMApicGetBase(pVM, &GCPhysApicBase); /* @todo cache this */
2922 GCPhysApicBase &= PAGE_BASE_GC_MASK;
2923
2924 rc = PGMGstGetPage(pVCpu, (RTGCPTR)exitQualification, NULL, &GCPhys);
2925 if ( rc == VINF_SUCCESS
2926 && GCPhys == GCPhysApicBase)
2927 {
2928 Log(("Enable VT-x virtual APIC access filtering\n"));
2929 rc2 = IOMMMIOMapMMIOHCPage(pVM, GCPhysApicBase, pVM->hwaccm.s.vmx.pAPICPhys, X86_PTE_RW | X86_PTE_P);
2930 AssertRC(rc2);
2931 }
2932 }
2933
2934 /* Forward it to our trap handler first, in case our shadow pages are out of sync. */
2935 rc = PGMTrap0eHandler(pVCpu, errCode, CPUMCTX2CORE(pCtx), (RTGCPTR)exitQualification);
2936 Log2(("PGMTrap0eHandler %RGv returned %Rrc\n", (RTGCPTR)pCtx->rip, VBOXSTRICTRC_VAL(rc)));
2937
2938 if (rc == VINF_SUCCESS)
2939 { /* We've successfully synced our shadow pages, so let's just continue execution. */
2940 Log2(("Shadow page fault at %RGv cr2=%RGv error code %x\n", (RTGCPTR)pCtx->rip, exitQualification ,errCode));
2941 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitShadowPF);
2942
2943 TRPMResetTrap(pVCpu);
2944 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
2945 goto ResumeExecution;
2946 }
2947 else
2948 if (rc == VINF_EM_RAW_GUEST_TRAP)
2949 { /* A genuine pagefault.
2950 * Forward the trap to the guest by injecting the exception and resuming execution.
2951 */
2952 Log2(("Forward page fault to the guest\n"));
2953
2954 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestPF);
2955 /* The error code might have been changed. */
2956 errCode = TRPMGetErrorCode(pVCpu);
2957
2958 TRPMResetTrap(pVCpu);
2959
2960 /* Now we must update CR2. */
2961 pCtx->cr2 = exitQualification;
2962 rc2 = VMXR0InjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), cbInstr, errCode);
2963 AssertRC(rc2);
2964
2965 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
2966 goto ResumeExecution;
2967 }
2968#ifdef VBOX_STRICT
2969 if (rc != VINF_EM_RAW_EMULATE_INSTR && rc != VINF_EM_RAW_EMULATE_IO_BLOCK)
2970 Log2(("PGMTrap0eHandler failed with %d\n", VBOXSTRICTRC_VAL(rc)));
2971#endif
2972 /* Need to go back to the recompiler to emulate the instruction. */
2973 TRPMResetTrap(pVCpu);
2974 break;
2975 }
2976
2977 case X86_XCPT_MF: /* Floating point exception. */
2978 {
2979 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestMF);
2980 if (!(pCtx->cr0 & X86_CR0_NE))
2981 {
2982 /* old style FPU error reporting needs some extra work. */
2983 /** @todo don't fall back to the recompiler, but do it manually. */
2984 rc = VINF_EM_RAW_EMULATE_INSTR;
2985 break;
2986 }
2987 Log(("Trap %x at %04X:%RGv\n", vector, pCtx->cs, (RTGCPTR)pCtx->rip));
2988 rc2 = VMXR0InjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), cbInstr, errCode);
2989 AssertRC(rc2);
2990
2991 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
2992 goto ResumeExecution;
2993 }
2994
2995 case X86_XCPT_DB: /* Debug exception. */
2996 {
2997 uint64_t uDR6;
2998
2999 /* DR6, DR7.GD and IA32_DEBUGCTL.LBR are not updated yet.
3000 *
3001 * Exit qualification bits:
3002 * 3:0 B0-B3 which breakpoint condition was met
3003 * 12:4 Reserved (0)
3004 * 13 BD - debug register access detected
3005 * 14 BS - single step execution or branch taken
3006 * 63:15 Reserved (0)
3007 */
3008 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestDB);
3009
3010 /* Note that we don't support guest and host-initiated debugging at the same time. */
3011
3012 uDR6 = X86_DR6_INIT_VAL;
3013 uDR6 |= (exitQualification & (X86_DR6_B0|X86_DR6_B1|X86_DR6_B2|X86_DR6_B3|X86_DR6_BD|X86_DR6_BS));
3014 rc = DBGFRZTrap01Handler(pVM, pVCpu, CPUMCTX2CORE(pCtx), uDR6);
3015 if (rc == VINF_EM_RAW_GUEST_TRAP)
3016 {
3017 /* Update DR6 here. */
3018 pCtx->dr[6] = uDR6;
3019
3020 /* Resync DR6 if the debug state is active. */
3021 if (CPUMIsGuestDebugStateActive(pVCpu))
3022 ASMSetDR6(pCtx->dr[6]);
3023
3024 /* X86_DR7_GD will be cleared if drx accesses should be trapped inside the guest. */
3025 pCtx->dr[7] &= ~X86_DR7_GD;
3026
3027 /* Paranoia. */
3028 pCtx->dr[7] &= 0xffffffff; /* upper 32 bits reserved */
3029 pCtx->dr[7] &= ~(RT_BIT(11) | RT_BIT(12) | RT_BIT(14) | RT_BIT(15)); /* must be zero */
3030 pCtx->dr[7] |= 0x400; /* must be one */
3031
3032 /* Resync DR7 */
3033 rc2 = VMXWriteVMCS64(VMX_VMCS64_GUEST_DR7, pCtx->dr[7]);
3034 AssertRC(rc2);
3035
3036 Log(("Trap %x (debug) at %RGv exit qualification %RX64 dr6=%x dr7=%x\n", vector, (RTGCPTR)pCtx->rip, exitQualification, (uint32_t)pCtx->dr[6], (uint32_t)pCtx->dr[7]));
3037 rc2 = VMXR0InjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), cbInstr, errCode);
3038 AssertRC(rc2);
3039
3040 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3041 goto ResumeExecution;
3042 }
3043 /* Return to ring 3 to deal with the debug exit code. */
3044 Log(("Debugger hardware BP at %04x:%RGv (rc=%Rrc)\n", pCtx->cs, pCtx->rip, VBOXSTRICTRC_VAL(rc)));
3045 break;
3046 }
3047
3048 case X86_XCPT_BP: /* Breakpoint. */
3049 {
3050 rc = DBGFRZTrap03Handler(pVM, pVCpu, CPUMCTX2CORE(pCtx));
3051 if (rc == VINF_EM_RAW_GUEST_TRAP)
3052 {
3053 Log(("Guest #BP at %04x:%RGv\n", pCtx->cs, pCtx->rip));
3054 rc2 = VMXR0InjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), cbInstr, errCode);
3055 AssertRC(rc2);
3056 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3057 goto ResumeExecution;
3058 }
3059 if (rc == VINF_SUCCESS)
3060 {
3061 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3062 goto ResumeExecution;
3063 }
3064 Log(("Debugger BP at %04x:%RGv (rc=%Rrc)\n", pCtx->cs, pCtx->rip, VBOXSTRICTRC_VAL(rc)));
3065 break;
3066 }
3067
3068 case X86_XCPT_GP: /* General protection failure exception.*/
3069 {
3070 uint32_t cbOp;
3071 uint32_t cbSize;
3072 PDISCPUSTATE pDis = &pVCpu->hwaccm.s.DisState;
3073
3074 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestGP);
3075#ifdef VBOX_STRICT
3076 if ( !CPUMIsGuestInRealModeEx(pCtx)
3077 || !pVM->hwaccm.s.vmx.pRealModeTSS)
3078 {
3079 Log(("Trap %x at %04X:%RGv errorCode=%RGv\n", vector, pCtx->cs, (RTGCPTR)pCtx->rip, errCode));
3080 rc2 = VMXR0InjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), cbInstr, errCode);
3081 AssertRC(rc2);
3082 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3083 goto ResumeExecution;
3084 }
3085#endif
3086 Assert(CPUMIsGuestInRealModeEx(pCtx));
3087
3088 LogFlow(("Real mode X86_XCPT_GP instruction emulation at %x:%RGv\n", pCtx->cs, (RTGCPTR)pCtx->rip));
3089
3090 rc2 = EMInterpretDisasOne(pVM, pVCpu, CPUMCTX2CORE(pCtx), pDis, &cbOp);
3091 if (RT_SUCCESS(rc2))
3092 {
3093 bool fUpdateRIP = true;
3094
3095 rc = VINF_SUCCESS;
3096 Assert(cbOp == pDis->opsize);
3097 switch (pDis->pCurInstr->opcode)
3098 {
3099 case OP_CLI:
3100 pCtx->eflags.Bits.u1IF = 0;
3101 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitCli);
3102 break;
3103
3104 case OP_STI:
3105 pCtx->eflags.Bits.u1IF = 1;
3106 EMSetInhibitInterruptsPC(pVCpu, pCtx->rip + pDis->opsize);
3107 Assert(VMCPU_FF_ISSET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
3108 rc2 = VMXWriteVMCS(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
3109 AssertRC(rc2);
3110 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitSti);
3111 break;
3112
3113 case OP_HLT:
3114 fUpdateRIP = false;
3115 rc = VINF_EM_HALT;
3116 pCtx->rip += pDis->opsize;
3117 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitHlt);
3118 break;
3119
3120 case OP_POPF:
3121 {
3122 RTGCPTR GCPtrStack;
3123 uint32_t cbParm;
3124 uint32_t uMask;
3125 X86EFLAGS eflags;
3126
3127 if (pDis->prefix & PREFIX_OPSIZE)
3128 {
3129 cbParm = 4;
3130 uMask = 0xffffffff;
3131 }
3132 else
3133 {
3134 cbParm = 2;
3135 uMask = 0xffff;
3136 }
3137
3138 rc2 = SELMToFlatEx(pVM, DIS_SELREG_SS, CPUMCTX2CORE(pCtx), pCtx->esp & uMask, 0, &GCPtrStack);
3139 if (RT_FAILURE(rc2))
3140 {
3141 rc = VERR_EM_INTERPRETER;
3142 break;
3143 }
3144 eflags.u = 0;
3145 rc2 = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &eflags.u, cbParm);
3146 if (RT_FAILURE(rc2))
3147 {
3148 rc = VERR_EM_INTERPRETER;
3149 break;
3150 }
3151 LogFlow(("POPF %x -> %RGv mask=%x\n", eflags.u, pCtx->rsp, uMask));
3152 pCtx->eflags.u = (pCtx->eflags.u & ~(X86_EFL_POPF_BITS & uMask)) | (eflags.u & X86_EFL_POPF_BITS & uMask);
3153 /* RF cleared when popped in real mode; see pushf description in AMD manual. */
3154 pCtx->eflags.Bits.u1RF = 0;
3155 pCtx->esp += cbParm;
3156 pCtx->esp &= uMask;
3157
3158 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitPopf);
3159 break;
3160 }
3161
3162 case OP_PUSHF:
3163 {
3164 RTGCPTR GCPtrStack;
3165 uint32_t cbParm;
3166 uint32_t uMask;
3167 X86EFLAGS eflags;
3168
3169 if (pDis->prefix & PREFIX_OPSIZE)
3170 {
3171 cbParm = 4;
3172 uMask = 0xffffffff;
3173 }
3174 else
3175 {
3176 cbParm = 2;
3177 uMask = 0xffff;
3178 }
3179
3180 rc2 = SELMToFlatEx(pVM, DIS_SELREG_SS, CPUMCTX2CORE(pCtx), (pCtx->esp - cbParm) & uMask, 0, &GCPtrStack);
3181 if (RT_FAILURE(rc2))
3182 {
3183 rc = VERR_EM_INTERPRETER;
3184 break;
3185 }
3186 eflags = pCtx->eflags;
3187 /* RF & VM cleared when pushed in real mode; see pushf description in AMD manual. */
3188 eflags.Bits.u1RF = 0;
3189 eflags.Bits.u1VM = 0;
3190
3191 rc2 = PGMPhysWrite(pVM, (RTGCPHYS)GCPtrStack, &eflags.u, cbParm);
3192 if (RT_FAILURE(rc2))
3193 {
3194 rc = VERR_EM_INTERPRETER;
3195 break;
3196 }
3197 LogFlow(("PUSHF %x -> %RGv\n", eflags.u, GCPtrStack));
3198 pCtx->esp -= cbParm;
3199 pCtx->esp &= uMask;
3200 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitPushf);
3201 break;
3202 }
3203
3204 case OP_IRET:
3205 {
3206 RTGCPTR GCPtrStack;
3207 uint32_t uMask = 0xffff;
3208 uint16_t aIretFrame[3];
3209
3210 if (pDis->prefix & (PREFIX_OPSIZE | PREFIX_ADDRSIZE))
3211 {
3212 rc = VERR_EM_INTERPRETER;
3213 break;
3214 }
3215
3216 rc2 = SELMToFlatEx(pVM, DIS_SELREG_SS, CPUMCTX2CORE(pCtx), pCtx->esp & uMask, 0, &GCPtrStack);
3217 if (RT_FAILURE(rc2))
3218 {
3219 rc = VERR_EM_INTERPRETER;
3220 break;
3221 }
3222 rc2 = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &aIretFrame[0], sizeof(aIretFrame));
3223 if (RT_FAILURE(rc2))
3224 {
3225 rc = VERR_EM_INTERPRETER;
3226 break;
3227 }
3228 pCtx->ip = aIretFrame[0];
3229 pCtx->cs = aIretFrame[1];
3230 pCtx->csHid.u64Base = pCtx->cs << 4;
3231 pCtx->eflags.u = (pCtx->eflags.u & ~(X86_EFL_POPF_BITS & uMask)) | (aIretFrame[2] & X86_EFL_POPF_BITS & uMask);
3232 pCtx->sp += sizeof(aIretFrame);
3233
3234 LogFlow(("iret to %04x:%x\n", pCtx->cs, pCtx->ip));
3235 fUpdateRIP = false;
3236 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitIret);
3237 break;
3238 }
3239
3240 case OP_INT:
3241 {
3242 uint32_t intInfo2;
3243
3244 LogFlow(("Realmode: INT %x\n", pDis->param1.parval & 0xff));
3245 intInfo2 = pDis->param1.parval & 0xff;
3246 intInfo2 |= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
3247 intInfo2 |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
3248
3249 rc = VMXR0InjectEvent(pVM, pVCpu, pCtx, intInfo2, cbOp, 0);
3250 AssertRC(VBOXSTRICTRC_VAL(rc));
3251 fUpdateRIP = false;
3252 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitInt);
3253 break;
3254 }
3255
3256 case OP_INTO:
3257 {
3258 if (pCtx->eflags.Bits.u1OF)
3259 {
3260 uint32_t intInfo2;
3261
3262 LogFlow(("Realmode: INTO\n"));
3263 intInfo2 = X86_XCPT_OF;
3264 intInfo2 |= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
3265 intInfo2 |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
3266
3267 rc = VMXR0InjectEvent(pVM, pVCpu, pCtx, intInfo2, cbOp, 0);
3268 AssertRC(VBOXSTRICTRC_VAL(rc));
3269 fUpdateRIP = false;
3270 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitInt);
3271 }
3272 break;
3273 }
3274
3275 case OP_INT3:
3276 {
3277 uint32_t intInfo2;
3278
3279 LogFlow(("Realmode: INT 3\n"));
3280 intInfo2 = 3;
3281 intInfo2 |= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
3282 intInfo2 |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
3283
3284 rc = VMXR0InjectEvent(pVM, pVCpu, pCtx, intInfo2, cbOp, 0);
3285 AssertRC(VBOXSTRICTRC_VAL(rc));
3286 fUpdateRIP = false;
3287 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitInt);
3288 break;
3289 }
3290
3291 default:
3292 rc = EMInterpretInstructionCPU(pVM, pVCpu, pDis, CPUMCTX2CORE(pCtx), 0, EMCODETYPE_SUPERVISOR, &cbSize);
3293 break;
3294 }
3295
3296 if (rc == VINF_SUCCESS)
3297 {
3298 if (fUpdateRIP)
3299 pCtx->rip += cbOp; /* Move on to the next instruction. */
3300
3301 /* lidt, lgdt can end up here. In the future crx changes as well. Just reload the whole context to be done with it. */
3302 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_ALL;
3303
3304 /* Only resume if successful. */
3305 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3306 goto ResumeExecution;
3307 }
3308 }
3309 else
3310 rc = VERR_EM_INTERPRETER;
3311
3312 AssertMsg(rc == VERR_EM_INTERPRETER || rc == VINF_PGM_CHANGE_MODE || rc == VINF_EM_HALT, ("Unexpected rc=%Rrc\n", VBOXSTRICTRC_VAL(rc)));
3313 break;
3314 }
3315
3316#ifdef VBOX_STRICT
3317 case X86_XCPT_XF: /* SIMD exception. */
3318 case X86_XCPT_DE: /* Divide error. */
3319 case X86_XCPT_UD: /* Unknown opcode exception. */
3320 case X86_XCPT_SS: /* Stack segment exception. */
3321 case X86_XCPT_NP: /* Segment not present exception. */
3322 {
3323 switch(vector)
3324 {
3325 case X86_XCPT_DE:
3326 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestDE);
3327 break;
3328 case X86_XCPT_UD:
3329 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestUD);
3330 break;
3331 case X86_XCPT_SS:
3332 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestSS);
3333 break;
3334 case X86_XCPT_NP:
3335 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestNP);
3336 break;
3337 }
3338
3339 Log(("Trap %x at %04X:%RGv\n", vector, pCtx->cs, (RTGCPTR)pCtx->rip));
3340 rc2 = VMXR0InjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), cbInstr, errCode);
3341 AssertRC(rc2);
3342
3343 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3344 goto ResumeExecution;
3345 }
3346#endif
3347 default:
3348 if ( CPUMIsGuestInRealModeEx(pCtx)
3349 && pVM->hwaccm.s.vmx.pRealModeTSS)
3350 {
3351 Log(("Real Mode Trap %x at %04x:%04X error code %x\n", vector, pCtx->cs, pCtx->eip, errCode));
3352 rc = VMXR0InjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), cbInstr, errCode);
3353 AssertRC(VBOXSTRICTRC_VAL(rc)); /* Strict RC check below. */
3354
3355 /* Go back to ring 3 in case of a triple fault. */
3356 if ( vector == X86_XCPT_DF
3357 && rc == VINF_EM_RESET)
3358 break;
3359
3360 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3361 goto ResumeExecution;
3362 }
3363 AssertMsgFailed(("Unexpected vm-exit caused by exception %x\n", vector));
3364 rc = VERR_VMX_UNEXPECTED_EXCEPTION;
3365 break;
3366 } /* switch (vector) */
3367
3368 break;
3369
3370 default:
3371 rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_CODE;
3372 AssertMsgFailed(("Unexpected interuption code %x\n", intInfo));
3373 break;
3374 }
3375
3376 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3377 break;
3378 }
3379
3380 case VMX_EXIT_EPT_VIOLATION: /* 48 EPT violation. An attempt to access memory with a guest-physical address was disallowed by the configuration of the EPT paging structures. */
3381 {
3382 RTGCPHYS GCPhys;
3383
3384 Assert(pVM->hwaccm.s.fNestedPaging);
3385
3386 rc2 = VMXReadVMCS64(VMX_VMCS_EXIT_PHYS_ADDR_FULL, &GCPhys);
3387 AssertRC(rc2);
3388 Assert(((exitQualification >> 7) & 3) != 2);
3389
3390 /* Determine the kind of violation. */
3391 errCode = 0;
3392 if (exitQualification & VMX_EXIT_QUALIFICATION_EPT_INSTR_FETCH)
3393 errCode |= X86_TRAP_PF_ID;
3394
3395 if (exitQualification & VMX_EXIT_QUALIFICATION_EPT_DATA_WRITE)
3396 errCode |= X86_TRAP_PF_RW;
3397
3398 /* If the page is present, then it's a page level protection fault. */
3399 if (exitQualification & VMX_EXIT_QUALIFICATION_EPT_ENTRY_PRESENT)
3400 {
3401 errCode |= X86_TRAP_PF_P;
3402 }
3403 else
3404 {
3405 /* Shortcut for APIC TPR reads and writes. */
3406 if ( (GCPhys & 0xfff) == 0x080
3407 && GCPhys > 0x1000000 /* to skip VGA frame buffer accesses */
3408 && fSetupTPRCaching
3409 && (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC))
3410 {
3411 RTGCPHYS GCPhysApicBase;
3412 PDMApicGetBase(pVM, &GCPhysApicBase); /* @todo cache this */
3413 GCPhysApicBase &= PAGE_BASE_GC_MASK;
3414 if (GCPhys == GCPhysApicBase + 0x80)
3415 {
3416 Log(("Enable VT-x virtual APIC access filtering\n"));
3417 rc2 = IOMMMIOMapMMIOHCPage(pVM, GCPhysApicBase, pVM->hwaccm.s.vmx.pAPICPhys, X86_PTE_RW | X86_PTE_P);
3418 AssertRC(rc2);
3419 }
3420 }
3421 }
3422 Log(("EPT Page fault %x at %RGp error code %x\n", (uint32_t)exitQualification, GCPhys, errCode));
3423
3424 /* GCPhys contains the guest physical address of the page fault. */
3425 TRPMAssertTrap(pVCpu, X86_XCPT_PF, TRPM_TRAP);
3426 TRPMSetErrorCode(pVCpu, errCode);
3427 TRPMSetFaultAddress(pVCpu, GCPhys);
3428
3429 /* Handle the pagefault trap for the nested shadow table. */
3430 rc = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, errCode, CPUMCTX2CORE(pCtx), GCPhys);
3431 Log2(("PGMR0Trap0eHandlerNestedPaging %RGv returned %Rrc\n", (RTGCPTR)pCtx->rip, VBOXSTRICTRC_VAL(rc)));
3432 if (rc == VINF_SUCCESS)
3433 { /* We've successfully synced our shadow pages, so let's just continue execution. */
3434 Log2(("Shadow page fault at %RGv cr2=%RGp error code %x\n", (RTGCPTR)pCtx->rip, exitQualification , errCode));
3435 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitReasonNPF);
3436
3437 TRPMResetTrap(pVCpu);
3438 goto ResumeExecution;
3439 }
3440
3441#ifdef VBOX_STRICT
3442 if (rc != VINF_EM_RAW_EMULATE_INSTR)
3443 LogFlow(("PGMTrap0eHandlerNestedPaging failed with %d\n", VBOXSTRICTRC_VAL(rc)));
3444#endif
3445 /* Need to go back to the recompiler to emulate the instruction. */
3446 TRPMResetTrap(pVCpu);
3447 break;
3448 }
3449
3450 case VMX_EXIT_EPT_MISCONFIG:
3451 {
3452 RTGCPHYS GCPhys;
3453
3454 Assert(pVM->hwaccm.s.fNestedPaging);
3455
3456 rc2 = VMXReadVMCS64(VMX_VMCS_EXIT_PHYS_ADDR_FULL, &GCPhys);
3457 AssertRC(rc2);
3458 Log(("VMX_EXIT_EPT_MISCONFIG for %RGp\n", GCPhys));
3459
3460 rc = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pCtx), GCPhys);
3461 if (rc == VINF_SUCCESS)
3462 {
3463 Log2(("PGMR0Trap0eHandlerNPMisconfig(,,,%RGp) at %RGv -> resume\n", GCPhys, (RTGCPTR)pCtx->rip));
3464 goto ResumeExecution;
3465 }
3466
3467 Log2(("PGMR0Trap0eHandlerNPMisconfig(,,,%RGp) at %RGv -> %Rrc\n", GCPhys, (RTGCPTR)pCtx->rip, VBOXSTRICTRC_VAL(rc)));
3468 break;
3469 }
3470
3471 case VMX_EXIT_IRQ_WINDOW: /* 7 Interrupt window. */
3472 /* Clear VM-exit on IF=1 change. */
3473 LogFlow(("VMX_EXIT_IRQ_WINDOW %RGv pending=%d IF=%d\n", (RTGCPTR)pCtx->rip, VMCPU_FF_ISPENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC|VMCPU_FF_INTERRUPT_PIC)), pCtx->eflags.Bits.u1IF));
3474 pVCpu->hwaccm.s.vmx.proc_ctls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_IRQ_WINDOW_EXIT;
3475 rc2 = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
3476 AssertRC(rc2);
3477 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitIrqWindow);
3478 goto ResumeExecution; /* we check for pending guest interrupts there */
3479
3480 case VMX_EXIT_WBINVD: /* 54 Guest software attempted to execute WBINVD. (conditional) */
3481 case VMX_EXIT_INVD: /* 13 Guest software attempted to execute INVD. (unconditional) */
3482 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitInvd);
3483 /* Skip instruction and continue directly. */
3484 pCtx->rip += cbInstr;
3485 /* Continue execution.*/
3486 goto ResumeExecution;
3487
3488 case VMX_EXIT_CPUID: /* 10 Guest software attempted to execute CPUID. */
3489 {
3490 Log2(("VMX: Cpuid %x\n", pCtx->eax));
3491 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitCpuid);
3492 rc = EMInterpretCpuId(pVM, pVCpu, CPUMCTX2CORE(pCtx));
3493 if (rc == VINF_SUCCESS)
3494 {
3495 /* Update EIP and continue execution. */
3496 Assert(cbInstr == 2);
3497 pCtx->rip += cbInstr;
3498 goto ResumeExecution;
3499 }
3500 AssertMsgFailed(("EMU: cpuid failed with %Rrc\n", VBOXSTRICTRC_VAL(rc)));
3501 rc = VINF_EM_RAW_EMULATE_INSTR;
3502 break;
3503 }
3504
3505 case VMX_EXIT_RDPMC: /* 15 Guest software attempted to execute RDPMC. */
3506 {
3507 Log2(("VMX: Rdpmc %x\n", pCtx->ecx));
3508 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitRdpmc);
3509 rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pCtx));
3510 if (rc == VINF_SUCCESS)
3511 {
3512 /* Update EIP and continue execution. */
3513 Assert(cbInstr == 2);
3514 pCtx->rip += cbInstr;
3515 goto ResumeExecution;
3516 }
3517 rc = VINF_EM_RAW_EMULATE_INSTR;
3518 break;
3519 }
3520
3521 case VMX_EXIT_RDTSC: /* 16 Guest software attempted to execute RDTSC. */
3522 {
3523 Log2(("VMX: Rdtsc\n"));
3524 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitRdtsc);
3525 rc = EMInterpretRdtsc(pVM, pVCpu, CPUMCTX2CORE(pCtx));
3526 if (rc == VINF_SUCCESS)
3527 {
3528 /* Update EIP and continue execution. */
3529 Assert(cbInstr == 2);
3530 pCtx->rip += cbInstr;
3531 goto ResumeExecution;
3532 }
3533 rc = VINF_EM_RAW_EMULATE_INSTR;
3534 break;
3535 }
3536
3537 case VMX_EXIT_INVPG: /* 14 Guest software attempted to execute INVPG. */
3538 {
3539 Log2(("VMX: invlpg\n"));
3540 Assert(!pVM->hwaccm.s.fNestedPaging);
3541
3542 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitInvpg);
3543 rc = EMInterpretInvlpg(pVM, pVCpu, CPUMCTX2CORE(pCtx), exitQualification);
3544 if (rc == VINF_SUCCESS)
3545 {
3546 /* Update EIP and continue execution. */
3547 pCtx->rip += cbInstr;
3548 goto ResumeExecution;
3549 }
3550 AssertMsg(rc == VERR_EM_INTERPRETER, ("EMU: invlpg %RGv failed with %Rrc\n", exitQualification, VBOXSTRICTRC_VAL(rc)));
3551 break;
3552 }
3553
3554 case VMX_EXIT_MONITOR: /* 39 Guest software attempted to execute MONITOR. */
3555 {
3556 Log2(("VMX: monitor\n"));
3557
3558 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitMonitor);
3559 rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pCtx));
3560 if (rc == VINF_SUCCESS)
3561 {
3562 /* Update EIP and continue execution. */
3563 pCtx->rip += cbInstr;
3564 goto ResumeExecution;
3565 }
3566 AssertMsg(rc == VERR_EM_INTERPRETER, ("EMU: monitor failed with %Rrc\n", VBOXSTRICTRC_VAL(rc)));
3567 break;
3568 }
3569
3570 case VMX_EXIT_WRMSR: /* 32 WRMSR. Guest software attempted to execute WRMSR. */
3571 /* When an interrupt is pending, we'll let MSR_K8_LSTAR writes fault in our TPR patch code. */
3572 if ( pVM->hwaccm.s.fTPRPatchingActive
3573 && pCtx->ecx == MSR_K8_LSTAR)
3574 {
3575 Assert(!CPUMIsGuestInLongModeEx(pCtx));
3576 if ((pCtx->eax & 0xff) != u8LastTPR)
3577 {
3578 Log(("VMX: Faulting MSR_K8_LSTAR write with new TPR value %x\n", pCtx->eax & 0xff));
3579
3580 /* Our patch code uses LSTAR for TPR caching. */
3581 rc2 = PDMApicSetTPR(pVCpu, pCtx->eax & 0xff);
3582 AssertRC(rc2);
3583 }
3584
3585 /* Skip the instruction and continue. */
3586 pCtx->rip += cbInstr; /* wrmsr = [0F 30] */
3587
3588 /* Only resume if successful. */
3589 goto ResumeExecution;
3590 }
3591 /* no break */
3592 case VMX_EXIT_RDMSR: /* 31 RDMSR. Guest software attempted to execute RDMSR. */
3593 {
3594 uint32_t cbSize;
3595
3596 STAM_COUNTER_INC((exitReason == VMX_EXIT_RDMSR) ? &pVCpu->hwaccm.s.StatExitRdmsr : &pVCpu->hwaccm.s.StatExitWrmsr);
3597
3598 /* Note: the intel manual claims there's a REX version of RDMSR that's slightly different, so we play safe by completely disassembling the instruction. */
3599 Log2(("VMX: %s\n", (exitReason == VMX_EXIT_RDMSR) ? "rdmsr" : "wrmsr"));
3600 rc = EMInterpretInstruction(pVM, pVCpu, CPUMCTX2CORE(pCtx), 0, &cbSize);
3601 if (rc == VINF_SUCCESS)
3602 {
3603 /* EIP has been updated already. */
3604
3605 /* Only resume if successful. */
3606 goto ResumeExecution;
3607 }
3608 AssertMsg(rc == VERR_EM_INTERPRETER, ("EMU: %s failed with %Rrc\n", (exitReason == VMX_EXIT_RDMSR) ? "rdmsr" : "wrmsr", VBOXSTRICTRC_VAL(rc)));
3609 break;
3610 }
3611
3612 case VMX_EXIT_CRX_MOVE: /* 28 Control-register accesses. */
3613 {
3614 STAM_PROFILE_ADV_START(&pVCpu->hwaccm.s.StatExit2Sub2, y2);
3615
3616 switch (VMX_EXIT_QUALIFICATION_CRX_ACCESS(exitQualification))
3617 {
3618 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_WRITE:
3619 Log2(("VMX: %RGv mov cr%d, x\n", (RTGCPTR)pCtx->rip, VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification)));
3620 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitCRxWrite[VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification)]);
3621 rc = EMInterpretCRxWrite(pVM, pVCpu, CPUMCTX2CORE(pCtx),
3622 VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification),
3623 VMX_EXIT_QUALIFICATION_CRX_GENREG(exitQualification));
3624
3625 switch (VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification))
3626 {
3627 case 0:
3628 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_GUEST_CR0 | HWACCM_CHANGED_GUEST_CR3;
3629 break;
3630 case 2:
3631 break;
3632 case 3:
3633 Assert(!pVM->hwaccm.s.fNestedPaging || !CPUMIsGuestInPagedProtectedModeEx(pCtx));
3634 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_GUEST_CR3;
3635 break;
3636 case 4:
3637 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_GUEST_CR4;
3638 break;
3639 case 8:
3640 /* CR8 contains the APIC TPR */
3641 Assert(!(pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW));
3642 break;
3643
3644 default:
3645 AssertFailed();
3646 break;
3647 }
3648 break;
3649
3650 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ:
3651 Log2(("VMX: mov x, crx\n"));
3652 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitCRxRead[VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification)]);
3653
3654 Assert(!pVM->hwaccm.s.fNestedPaging || !CPUMIsGuestInPagedProtectedModeEx(pCtx) || VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification) != USE_REG_CR3);
3655
3656 /* CR8 reads only cause an exit when the TPR shadow feature isn't present. */
3657 Assert(VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification) != 8 || !(pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW));
3658
3659 rc = EMInterpretCRxRead(pVM, pVCpu, CPUMCTX2CORE(pCtx),
3660 VMX_EXIT_QUALIFICATION_CRX_GENREG(exitQualification),
3661 VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification));
3662 break;
3663
3664 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_CLTS:
3665 Log2(("VMX: clts\n"));
3666 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitCLTS);
3667 rc = EMInterpretCLTS(pVM, pVCpu);
3668 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_GUEST_CR0;
3669 break;
3670
3671 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_LMSW:
3672 Log2(("VMX: lmsw %x\n", VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(exitQualification)));
3673 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitLMSW);
3674 rc = EMInterpretLMSW(pVM, pVCpu, CPUMCTX2CORE(pCtx), VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(exitQualification));
3675 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_GUEST_CR0;
3676 break;
3677 }
3678
3679 /* Update EIP if no error occurred. */
3680 if (RT_SUCCESS(rc))
3681 pCtx->rip += cbInstr;
3682
3683 if (rc == VINF_SUCCESS)
3684 {
3685 /* Only resume if successful. */
3686 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub2, y2);
3687 goto ResumeExecution;
3688 }
3689 Assert(rc == VERR_EM_INTERPRETER || rc == VINF_PGM_CHANGE_MODE || rc == VINF_PGM_SYNC_CR3);
3690 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub2, y2);
3691 break;
3692 }
3693
3694 case VMX_EXIT_DRX_MOVE: /* 29 Debug-register accesses. */
3695 {
3696 if ( !DBGFIsStepping(pVCpu)
3697 && !CPUMIsHyperDebugStateActive(pVCpu))
3698 {
3699 /* Disable drx move intercepts. */
3700 pVCpu->hwaccm.s.vmx.proc_ctls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT;
3701 rc2 = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
3702 AssertRC(rc2);
3703
3704 /* Save the host and load the guest debug state. */
3705 rc2 = CPUMR0LoadGuestDebugState(pVM, pVCpu, pCtx, true /* include DR6 */);
3706 AssertRC(rc2);
3707
3708#ifdef LOG_ENABLED
3709 if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(exitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
3710 Log(("VMX_EXIT_DRX_MOVE: write DR%d genreg %d\n", VMX_EXIT_QUALIFICATION_DRX_REGISTER(exitQualification), VMX_EXIT_QUALIFICATION_DRX_GENREG(exitQualification)));
3711 else
3712 Log(("VMX_EXIT_DRX_MOVE: read DR%d\n", VMX_EXIT_QUALIFICATION_DRX_REGISTER(exitQualification)));
3713#endif
3714
3715#ifdef VBOX_WITH_STATISTICS
3716 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatDRxContextSwitch);
3717 if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(exitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
3718 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitDRxWrite);
3719 else
3720 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitDRxRead);
3721#endif
3722
3723 goto ResumeExecution;
3724 }
3725
3726 /** @todo clear VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT after the first time and restore drx registers afterwards */
3727 if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(exitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
3728 {
3729 Log2(("VMX: mov drx%d, genreg%d\n", VMX_EXIT_QUALIFICATION_DRX_REGISTER(exitQualification), VMX_EXIT_QUALIFICATION_DRX_GENREG(exitQualification)));
3730 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitDRxWrite);
3731 rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pCtx),
3732 VMX_EXIT_QUALIFICATION_DRX_REGISTER(exitQualification),
3733 VMX_EXIT_QUALIFICATION_DRX_GENREG(exitQualification));
3734 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_GUEST_DEBUG;
3735 Log2(("DR7=%08x\n", pCtx->dr[7]));
3736 }
3737 else
3738 {
3739 Log2(("VMX: mov x, drx\n"));
3740 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitDRxRead);
3741 rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pCtx),
3742 VMX_EXIT_QUALIFICATION_DRX_GENREG(exitQualification),
3743 VMX_EXIT_QUALIFICATION_DRX_REGISTER(exitQualification));
3744 }
3745 /* Update EIP if no error occurred. */
3746 if (RT_SUCCESS(rc))
3747 pCtx->rip += cbInstr;
3748
3749 if (rc == VINF_SUCCESS)
3750 {
3751 /* Only resume if successful. */
3752 goto ResumeExecution;
3753 }
3754 Assert(rc == VERR_EM_INTERPRETER);
3755 break;
3756 }
3757
3758 /* Note: We'll get a #GP if the IO instruction isn't allowed (IOPL or TSS bitmap); no need to double check. */
3759 case VMX_EXIT_PORT_IO: /* 30 I/O instruction. */
3760 {
3761 STAM_PROFILE_ADV_START(&pVCpu->hwaccm.s.StatExit2Sub1, y1);
3762 uint32_t uIOWidth = VMX_EXIT_QUALIFICATION_IO_WIDTH(exitQualification);
3763 uint32_t uPort;
3764 bool fIOWrite = (VMX_EXIT_QUALIFICATION_IO_DIRECTION(exitQualification) == VMX_EXIT_QUALIFICATION_IO_DIRECTION_OUT);
3765
3766 /** @todo necessary to make the distinction? */
3767 if (VMX_EXIT_QUALIFICATION_IO_ENCODING(exitQualification) == VMX_EXIT_QUALIFICATION_IO_ENCODING_DX)
3768 {
3769 uPort = pCtx->edx & 0xffff;
3770 }
3771 else
3772 uPort = VMX_EXIT_QUALIFICATION_IO_PORT(exitQualification); /* Immediate encoding. */
3773
3774 /* paranoia */
3775 if (RT_UNLIKELY(uIOWidth == 2 || uIOWidth >= 4))
3776 {
3777 rc = fIOWrite ? VINF_IOM_HC_IOPORT_WRITE : VINF_IOM_HC_IOPORT_READ;
3778 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub1, y1);
3779 break;
3780 }
3781
3782 uint32_t cbSize = g_aIOSize[uIOWidth];
3783
3784 if (VMX_EXIT_QUALIFICATION_IO_STRING(exitQualification))
3785 {
3786 /* ins/outs */
3787 PDISCPUSTATE pDis = &pVCpu->hwaccm.s.DisState;
3788
3789 /* Disassemble manually to deal with segment prefixes. */
3790 /** @todo VMX_VMCS_EXIT_GUEST_LINEAR_ADDR contains the flat pointer operand of the instruction. */
3791 /** @todo VMX_VMCS32_RO_EXIT_INSTR_INFO also contains segment prefix info. */
3792 rc2 = EMInterpretDisasOne(pVM, pVCpu, CPUMCTX2CORE(pCtx), pDis, NULL);
3793 if (RT_SUCCESS(rc))
3794 {
3795 if (fIOWrite)
3796 {
3797 Log2(("IOMInterpretOUTSEx %RGv %x size=%d\n", (RTGCPTR)pCtx->rip, uPort, cbSize));
3798 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitIOStringWrite);
3799 rc = IOMInterpretOUTSEx(pVM, CPUMCTX2CORE(pCtx), uPort, pDis->prefix, cbSize);
3800 }
3801 else
3802 {
3803 Log2(("IOMInterpretINSEx %RGv %x size=%d\n", (RTGCPTR)pCtx->rip, uPort, cbSize));
3804 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitIOStringRead);
3805 rc = IOMInterpretINSEx(pVM, CPUMCTX2CORE(pCtx), uPort, pDis->prefix, cbSize);
3806 }
3807 }
3808 else
3809 rc = VINF_EM_RAW_EMULATE_INSTR;
3810 }
3811 else
3812 {
3813 /* normal in/out */
3814 uint32_t uAndVal = g_aIOOpAnd[uIOWidth];
3815
3816 Assert(!VMX_EXIT_QUALIFICATION_IO_REP(exitQualification));
3817
3818 if (fIOWrite)
3819 {
3820 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitIOWrite);
3821 rc = IOMIOPortWrite(pVM, uPort, pCtx->eax & uAndVal, cbSize);
3822 if (rc == VINF_IOM_HC_IOPORT_WRITE)
3823 HWACCMR0SavePendingIOPortWrite(pVCpu, pCtx->rip, pCtx->rip + cbInstr, uPort, uAndVal, cbSize);
3824 }
3825 else
3826 {
3827 uint32_t u32Val = 0;
3828
3829 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitIORead);
3830 rc = IOMIOPortRead(pVM, uPort, &u32Val, cbSize);
3831 if (IOM_SUCCESS(rc))
3832 {
3833 /* Write back to the EAX register. */
3834 pCtx->eax = (pCtx->eax & ~uAndVal) | (u32Val & uAndVal);
3835 }
3836 else
3837 if (rc == VINF_IOM_HC_IOPORT_READ)
3838 HWACCMR0SavePendingIOPortRead(pVCpu, pCtx->rip, pCtx->rip + cbInstr, uPort, uAndVal, cbSize);
3839 }
3840 }
3841 /*
3842 * Handled the I/O return codes.
3843 * (The unhandled cases end up with rc == VINF_EM_RAW_EMULATE_INSTR.)
3844 */
3845 if (IOM_SUCCESS(rc))
3846 {
3847 /* Update EIP and continue execution. */
3848 pCtx->rip += cbInstr;
3849 if (RT_LIKELY(rc == VINF_SUCCESS))
3850 {
3851 /* If any IO breakpoints are armed, then we should check if a debug trap needs to be generated. */
3852 if (pCtx->dr[7] & X86_DR7_ENABLED_MASK)
3853 {
3854 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatDRxIOCheck);
3855 for (unsigned i=0;i<4;i++)
3856 {
3857 unsigned uBPLen = g_aIOSize[X86_DR7_GET_LEN(pCtx->dr[7], i)];
3858
3859 if ( (uPort >= pCtx->dr[i] && uPort < pCtx->dr[i] + uBPLen)
3860 && (pCtx->dr[7] & (X86_DR7_L(i) | X86_DR7_G(i)))
3861 && (pCtx->dr[7] & X86_DR7_RW(i, X86_DR7_RW_IO)) == X86_DR7_RW(i, X86_DR7_RW_IO))
3862 {
3863 uint64_t uDR6;
3864
3865 Assert(CPUMIsGuestDebugStateActive(pVCpu));
3866
3867 uDR6 = ASMGetDR6();
3868
3869 /* Clear all breakpoint status flags and set the one we just hit. */
3870 uDR6 &= ~(X86_DR6_B0|X86_DR6_B1|X86_DR6_B2|X86_DR6_B3);
3871 uDR6 |= (uint64_t)RT_BIT(i);
3872
3873 /* Note: AMD64 Architecture Programmer's Manual 13.1:
3874 * Bits 15:13 of the DR6 register is never cleared by the processor and must be cleared by software after
3875 * the contents have been read.
3876 */
3877 ASMSetDR6(uDR6);
3878
3879 /* X86_DR7_GD will be cleared if drx accesses should be trapped inside the guest. */
3880 pCtx->dr[7] &= ~X86_DR7_GD;
3881
3882 /* Paranoia. */
3883 pCtx->dr[7] &= 0xffffffff; /* upper 32 bits reserved */
3884 pCtx->dr[7] &= ~(RT_BIT(11) | RT_BIT(12) | RT_BIT(14) | RT_BIT(15)); /* must be zero */
3885 pCtx->dr[7] |= 0x400; /* must be one */
3886
3887 /* Resync DR7 */
3888 rc2 = VMXWriteVMCS64(VMX_VMCS64_GUEST_DR7, pCtx->dr[7]);
3889 AssertRC(rc2);
3890
3891 /* Construct inject info. */
3892 intInfo = X86_XCPT_DB;
3893 intInfo |= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
3894 intInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HWEXCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
3895
3896 Log(("Inject IO debug trap at %RGv\n", (RTGCPTR)pCtx->rip));
3897 rc2 = VMXR0InjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), 0, 0);
3898 AssertRC(rc2);
3899
3900 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub1, y1);
3901 goto ResumeExecution;
3902 }
3903 }
3904 }
3905 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub1, y1);
3906 goto ResumeExecution;
3907 }
3908 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub1, y1);
3909 break;
3910 }
3911
3912#ifdef VBOX_STRICT
3913 if (rc == VINF_IOM_HC_IOPORT_READ)
3914 Assert(!fIOWrite);
3915 else if (rc == VINF_IOM_HC_IOPORT_WRITE)
3916 Assert(fIOWrite);
3917 else
3918 AssertMsg(RT_FAILURE(rc) || rc == VINF_EM_RAW_EMULATE_INSTR || rc == VINF_EM_RAW_GUEST_TRAP || rc == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", VBOXSTRICTRC_VAL(rc)));
3919#endif
3920 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub1, y1);
3921 break;
3922 }
3923
3924 case VMX_EXIT_TPR: /* 43 TPR below threshold. Guest software executed MOV to CR8. */
3925 LogFlow(("VMX_EXIT_TPR\n"));
3926 /* RIP is already set to the next instruction and the TPR has been synced back. Just resume. */
3927 goto ResumeExecution;
3928
3929 case VMX_EXIT_APIC_ACCESS: /* 44 APIC access. Guest software attempted to access memory at a physical address on the APIC-access page. */
3930 {
3931 LogFlow(("VMX_EXIT_APIC_ACCESS\n"));
3932 unsigned uAccessType = VMX_EXIT_QUALIFICATION_APIC_ACCESS_TYPE(exitQualification);
3933
3934 switch(uAccessType)
3935 {
3936 case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
3937 case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
3938 {
3939 RTGCPHYS GCPhys;
3940 PDMApicGetBase(pVM, &GCPhys);
3941 GCPhys &= PAGE_BASE_GC_MASK;
3942 GCPhys += VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(exitQualification);
3943
3944 LogFlow(("Apic access at %RGp\n", GCPhys));
3945 rc = IOMMMIOPhysHandler(pVM, (uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ) ? 0 : X86_TRAP_PF_RW, CPUMCTX2CORE(pCtx), GCPhys);
3946 if (rc == VINF_SUCCESS)
3947 goto ResumeExecution; /* rip already updated */
3948 break;
3949 }
3950
3951 default:
3952 rc = VINF_EM_RAW_EMULATE_INSTR;
3953 break;
3954 }
3955 break;
3956 }
3957
3958 case VMX_EXIT_PREEMPTION_TIMER: /* 52 VMX-preemption timer expired. The preemption timer counted down to zero. */
3959 goto ResumeExecution;
3960
3961 default:
3962 /* The rest is handled after syncing the entire CPU state. */
3963 break;
3964 }
3965
3966 /* Note: the guest state isn't entirely synced back at this stage. */
3967
3968 /* Investigate why there was a VM-exit. (part 2) */
3969 switch (exitReason)
3970 {
3971 case VMX_EXIT_EXCEPTION: /* 0 Exception or non-maskable interrupt (NMI). */
3972 case VMX_EXIT_EXTERNAL_IRQ: /* 1 External interrupt. */
3973 case VMX_EXIT_EPT_VIOLATION:
3974 case VMX_EXIT_EPT_MISCONFIG: /* 49 EPT misconfig is used by the PGM/MMIO optimizations. */
3975 case VMX_EXIT_PREEMPTION_TIMER: /* 52 VMX-preemption timer expired. The preemption timer counted down to zero. */
3976 /* Already handled above. */
3977 break;
3978
3979 case VMX_EXIT_TRIPLE_FAULT: /* 2 Triple fault. */
3980 rc = VINF_EM_RESET; /* Triple fault equals a reset. */
3981 break;
3982
3983 case VMX_EXIT_INIT_SIGNAL: /* 3 INIT signal. */
3984 case VMX_EXIT_SIPI: /* 4 Start-up IPI (SIPI). */
3985 rc = VINF_EM_RAW_INTERRUPT;
3986 AssertFailed(); /* Can't happen. Yet. */
3987 break;
3988
3989 case VMX_EXIT_IO_SMI_IRQ: /* 5 I/O system-management interrupt (SMI). */
3990 case VMX_EXIT_SMI_IRQ: /* 6 Other SMI. */
3991 rc = VINF_EM_RAW_INTERRUPT;
3992 AssertFailed(); /* Can't happen afaik. */
3993 break;
3994
3995 case VMX_EXIT_TASK_SWITCH: /* 9 Task switch: too complicated to emulate, so fall back to the recompiler */
3996 Log(("VMX_EXIT_TASK_SWITCH: exit=%RX64\n", exitQualification));
3997 if ( (VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE(exitQualification) == VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE_IDT)
3998 && pVCpu->hwaccm.s.Event.fPending)
3999 {
4000 /* Caused by an injected interrupt. */
4001 pVCpu->hwaccm.s.Event.fPending = false;
4002
4003 Log(("VMX_EXIT_TASK_SWITCH: reassert trap %d\n", VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVCpu->hwaccm.s.Event.intInfo)));
4004 Assert(!VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(pVCpu->hwaccm.s.Event.intInfo));
4005 rc2 = TRPMAssertTrap(pVCpu, VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVCpu->hwaccm.s.Event.intInfo), TRPM_HARDWARE_INT);
4006 AssertRC(rc2);
4007 }
4008 /* else Exceptions and software interrupts can just be restarted. */
4009 rc = VERR_EM_INTERPRETER;
4010 break;
4011
4012 case VMX_EXIT_HLT: /* 12 Guest software attempted to execute HLT. */
4013 /** Check if external interrupts are pending; if so, don't switch back. */
4014 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitHlt);
4015 pCtx->rip++; /* skip hlt */
4016 if (EMShouldContinueAfterHalt(pVCpu, pCtx))
4017 goto ResumeExecution;
4018
4019 rc = VINF_EM_HALT;
4020 break;
4021
4022 case VMX_EXIT_MWAIT: /* 36 Guest software executed MWAIT. */
4023 Log2(("VMX: mwait\n"));
4024 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitMwait);
4025 rc = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pCtx));
4026 if ( rc == VINF_EM_HALT
4027 || rc == VINF_SUCCESS)
4028 {
4029 /* Update EIP and continue execution. */
4030 pCtx->rip += cbInstr;
4031
4032 /** Check if external interrupts are pending; if so, don't switch back. */
4033 if ( rc == VINF_SUCCESS
4034 || ( rc == VINF_EM_HALT
4035 && EMShouldContinueAfterHalt(pVCpu, pCtx))
4036 )
4037 goto ResumeExecution;
4038 }
4039 AssertMsg(rc == VERR_EM_INTERPRETER || rc == VINF_EM_HALT, ("EMU: mwait failed with %Rrc\n", VBOXSTRICTRC_VAL(rc)));
4040 break;
4041
4042 case VMX_EXIT_RSM: /* 17 Guest software attempted to execute RSM in SMM. */
4043 AssertFailed(); /* can't happen. */
4044 rc = VERR_EM_INTERPRETER;
4045 break;
4046
4047 case VMX_EXIT_VMCALL: /* 18 Guest software executed VMCALL. */
4048 case VMX_EXIT_VMCLEAR: /* 19 Guest software executed VMCLEAR. */
4049 case VMX_EXIT_VMLAUNCH: /* 20 Guest software executed VMLAUNCH. */
4050 case VMX_EXIT_VMPTRLD: /* 21 Guest software executed VMPTRLD. */
4051 case VMX_EXIT_VMPTRST: /* 22 Guest software executed VMPTRST. */
4052 case VMX_EXIT_VMREAD: /* 23 Guest software executed VMREAD. */
4053 case VMX_EXIT_VMRESUME: /* 24 Guest software executed VMRESUME. */
4054 case VMX_EXIT_VMWRITE: /* 25 Guest software executed VMWRITE. */
4055 case VMX_EXIT_VMXOFF: /* 26 Guest software executed VMXOFF. */
4056 case VMX_EXIT_VMXON: /* 27 Guest software executed VMXON. */
4057 /** @todo inject #UD immediately */
4058 rc = VERR_EM_INTERPRETER;
4059 break;
4060
4061 case VMX_EXIT_CPUID: /* 10 Guest software attempted to execute CPUID. */
4062 case VMX_EXIT_RDTSC: /* 16 Guest software attempted to execute RDTSC. */
4063 case VMX_EXIT_INVPG: /* 14 Guest software attempted to execute INVPG. */
4064 case VMX_EXIT_CRX_MOVE: /* 28 Control-register accesses. */
4065 case VMX_EXIT_DRX_MOVE: /* 29 Debug-register accesses. */
4066 case VMX_EXIT_PORT_IO: /* 30 I/O instruction. */
4067 case VMX_EXIT_RDPMC: /* 15 Guest software attempted to execute RDPMC. */
4068 /* already handled above */
4069 AssertMsg( rc == VINF_PGM_CHANGE_MODE
4070 || rc == VINF_EM_RAW_INTERRUPT
4071 || rc == VERR_EM_INTERPRETER
4072 || rc == VINF_EM_RAW_EMULATE_INSTR
4073 || rc == VINF_PGM_SYNC_CR3
4074 || rc == VINF_IOM_HC_IOPORT_READ
4075 || rc == VINF_IOM_HC_IOPORT_WRITE
4076 || rc == VINF_EM_RAW_GUEST_TRAP
4077 || rc == VINF_TRPM_XCPT_DISPATCHED
4078 || rc == VINF_EM_RESCHEDULE_REM,
4079 ("rc = %d\n", VBOXSTRICTRC_VAL(rc)));
4080 break;
4081
4082 case VMX_EXIT_TPR: /* 43 TPR below threshold. Guest software executed MOV to CR8. */
4083 case VMX_EXIT_APIC_ACCESS: /* 44 APIC access. Guest software attempted to access memory at a physical address on the APIC-access page. */
4084 case VMX_EXIT_RDMSR: /* 31 RDMSR. Guest software attempted to execute RDMSR. */
4085 case VMX_EXIT_WRMSR: /* 32 WRMSR. Guest software attempted to execute WRMSR. */
4086 case VMX_EXIT_PAUSE: /* 40 Guest software attempted to execute PAUSE. */
4087 case VMX_EXIT_MONITOR: /* 39 Guest software attempted to execute MONITOR. */
4088 /* Note: If we decide to emulate them here, then we must sync the MSRs that could have been changed (sysenter, fs/gs base)!!! */
4089 rc = VERR_EM_INTERPRETER;
4090 break;
4091
4092 case VMX_EXIT_IRQ_WINDOW: /* 7 Interrupt window. */
4093 Assert(rc == VINF_EM_RAW_INTERRUPT);
4094 break;
4095
4096 case VMX_EXIT_ERR_INVALID_GUEST_STATE: /* 33 VM-entry failure due to invalid guest state. */
4097 {
4098#ifdef VBOX_STRICT
4099 RTCCUINTREG val2 = 0;
4100
4101 Log(("VMX_EXIT_ERR_INVALID_GUEST_STATE\n"));
4102
4103 VMXReadVMCS(VMX_VMCS64_GUEST_RIP, &val2);
4104 Log(("Old eip %RGv new %RGv\n", (RTGCPTR)pCtx->rip, (RTGCPTR)val2));
4105
4106 VMXReadVMCS(VMX_VMCS64_GUEST_CR0, &val2);
4107 Log(("VMX_VMCS_GUEST_CR0 %RX64\n", (uint64_t)val2));
4108
4109 VMXReadVMCS(VMX_VMCS64_GUEST_CR3, &val2);
4110 Log(("VMX_VMCS_GUEST_CR3 %RX64\n", (uint64_t)val2));
4111
4112 VMXReadVMCS(VMX_VMCS64_GUEST_CR4, &val2);
4113 Log(("VMX_VMCS_GUEST_CR4 %RX64\n", (uint64_t)val2));
4114
4115 VMXReadVMCS(VMX_VMCS_GUEST_RFLAGS, &val2);
4116 Log(("VMX_VMCS_GUEST_RFLAGS %08x\n", val2));
4117
4118 VMX_LOG_SELREG(CS, "CS", val2);
4119 VMX_LOG_SELREG(DS, "DS", val2);
4120 VMX_LOG_SELREG(ES, "ES", val2);
4121 VMX_LOG_SELREG(FS, "FS", val2);
4122 VMX_LOG_SELREG(GS, "GS", val2);
4123 VMX_LOG_SELREG(SS, "SS", val2);
4124 VMX_LOG_SELREG(TR, "TR", val2);
4125 VMX_LOG_SELREG(LDTR, "LDTR", val2);
4126
4127 VMXReadVMCS(VMX_VMCS64_GUEST_GDTR_BASE, &val2);
4128 Log(("VMX_VMCS_GUEST_GDTR_BASE %RX64\n", (uint64_t)val2));
4129 VMXReadVMCS(VMX_VMCS64_GUEST_IDTR_BASE, &val2);
4130 Log(("VMX_VMCS_GUEST_IDTR_BASE %RX64\n", (uint64_t)val2));
4131#endif /* VBOX_STRICT */
4132 rc = VERR_VMX_INVALID_GUEST_STATE;
4133 break;
4134 }
4135
4136 case VMX_EXIT_ERR_MSR_LOAD: /* 34 VM-entry failure due to MSR loading. */
4137 case VMX_EXIT_ERR_MACHINE_CHECK: /* 41 VM-entry failure due to machine-check. */
4138 default:
4139 rc = VERR_VMX_UNEXPECTED_EXIT_CODE;
4140 AssertMsgFailed(("Unexpected exit code %d\n", exitReason)); /* Can't happen. */
4141 break;
4142
4143 }
4144end:
4145
4146 /* Signal changes for the recompiler. */
4147 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR | CPUM_CHANGED_LDTR | CPUM_CHANGED_GDTR | CPUM_CHANGED_IDTR | CPUM_CHANGED_TR | CPUM_CHANGED_HIDDEN_SEL_REGS);
4148
4149 /* If we executed vmlaunch/vmresume and an external irq was pending, then we don't have to do a full sync the next time. */
4150 if ( exitReason == VMX_EXIT_EXTERNAL_IRQ
4151 && !VMX_EXIT_INTERRUPTION_INFO_VALID(intInfo))
4152 {
4153 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatPendingHostIrq);
4154 /* On the next entry we'll only sync the host context. */
4155 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_HOST_CONTEXT;
4156 }
4157 else
4158 {
4159 /* On the next entry we'll sync everything. */
4160 /** @todo we can do better than this */
4161 /* Not in the VINF_PGM_CHANGE_MODE though! */
4162 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_ALL;
4163 }
4164
4165 /* translate into a less severe return code */
4166 if (rc == VERR_EM_INTERPRETER)
4167 rc = VINF_EM_RAW_EMULATE_INSTR;
4168 else
4169 /* Try to extract more information about what might have gone wrong here. */
4170 if (rc == VERR_VMX_INVALID_VMCS_PTR)
4171 {
4172 VMXGetActivateVMCS(&pVCpu->hwaccm.s.vmx.lasterror.u64VMCSPhys);
4173 pVCpu->hwaccm.s.vmx.lasterror.ulVMCSRevision = *(uint32_t *)pVCpu->hwaccm.s.vmx.pVMCS;
4174 pVCpu->hwaccm.s.vmx.lasterror.idEnteredCpu = pVCpu->hwaccm.s.idEnteredCpu;
4175 pVCpu->hwaccm.s.vmx.lasterror.idCurrentCpu = RTMpCpuId();
4176 }
4177
4178 /* Just set the correct state here instead of trying to catch every goto above. */
4179 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED, VMCPUSTATE_STARTED_EXEC);
4180
4181#ifdef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
4182 /* Restore interrupts if we exitted after disabling them. */
4183 if (uOldEFlags != ~(RTCCUINTREG)0)
4184 ASMSetFlags(uOldEFlags);
4185#endif
4186
4187 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2, x);
4188 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit1, x);
4189 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatEntry, x);
4190 Log2(("X"));
4191 return VBOXSTRICTRC_TODO(rc);
4192}
4193
4194
4195/**
4196 * Enters the VT-x session
4197 *
4198 * @returns VBox status code.
4199 * @param pVM The VM to operate on.
4200 * @param pVCpu The VMCPU to operate on.
4201 * @param pCpu CPU info struct
4202 */
4203VMMR0DECL(int) VMXR0Enter(PVM pVM, PVMCPU pVCpu, PHWACCM_CPUINFO pCpu)
4204{
4205 Assert(pVM->hwaccm.s.vmx.fSupported);
4206
4207 unsigned cr4 = ASMGetCR4();
4208 if (!(cr4 & X86_CR4_VMXE))
4209 {
4210 AssertMsgFailed(("X86_CR4_VMXE should be set!\n"));
4211 return VERR_VMX_X86_CR4_VMXE_CLEARED;
4212 }
4213
4214 /* Activate the VM Control Structure. */
4215 int rc = VMXActivateVMCS(pVCpu->hwaccm.s.vmx.pVMCSPhys);
4216 if (RT_FAILURE(rc))
4217 return rc;
4218
4219 pVCpu->hwaccm.s.fResumeVM = false;
4220 return VINF_SUCCESS;
4221}
4222
4223
4224/**
4225 * Leaves the VT-x session
4226 *
4227 * @returns VBox status code.
4228 * @param pVM The VM to operate on.
4229 * @param pVCpu The VMCPU to operate on.
4230 * @param pCtx CPU context
4231 */
4232VMMR0DECL(int) VMXR0Leave(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
4233{
4234 Assert(pVM->hwaccm.s.vmx.fSupported);
4235
4236#ifdef DEBUG
4237 if (CPUMIsHyperDebugStateActive(pVCpu))
4238 {
4239 CPUMR0LoadHostDebugState(pVM, pVCpu);
4240 Assert(pVCpu->hwaccm.s.vmx.proc_ctls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT);
4241 }
4242 else
4243#endif
4244 /* Save the guest debug state if necessary. */
4245 if (CPUMIsGuestDebugStateActive(pVCpu))
4246 {
4247 CPUMR0SaveGuestDebugState(pVM, pVCpu, pCtx, true /* save DR6 */);
4248
4249 /* Enable drx move intercepts again. */
4250 pVCpu->hwaccm.s.vmx.proc_ctls |= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT;
4251 int rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
4252 AssertRC(rc);
4253
4254 /* Resync the debug registers the next time. */
4255 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_GUEST_DEBUG;
4256 }
4257 else
4258 Assert(pVCpu->hwaccm.s.vmx.proc_ctls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT);
4259
4260 /* Clear VM Control Structure. Marking it inactive, clearing implementation specific data and writing back VMCS data to memory. */
4261 int rc = VMXClearVMCS(pVCpu->hwaccm.s.vmx.pVMCSPhys);
4262 AssertRC(rc);
4263
4264 return VINF_SUCCESS;
4265}
4266
4267/**
4268 * Flush the TLB (EPT)
4269 *
4270 * @returns VBox status code.
4271 * @param pVM The VM to operate on.
4272 * @param pVCpu The VM CPU to operate on.
4273 * @param enmFlush Type of flush
4274 * @param GCPhys Physical address of the page to flush
4275 */
4276static void vmxR0FlushEPT(PVM pVM, PVMCPU pVCpu, VMX_FLUSH enmFlush, RTGCPHYS GCPhys)
4277{
4278 uint64_t descriptor[2];
4279
4280 LogFlow(("vmxR0FlushEPT %d %RGv\n", enmFlush, GCPhys));
4281 Assert(pVM->hwaccm.s.fNestedPaging);
4282 descriptor[0] = pVCpu->hwaccm.s.vmx.GCPhysEPTP;
4283 descriptor[1] = GCPhys;
4284 int rc = VMXR0InvEPT(enmFlush, &descriptor[0]);
4285 AssertRC(rc);
4286}
4287
4288#ifdef HWACCM_VTX_WITH_VPID
4289/**
4290 * Flush the TLB (EPT)
4291 *
4292 * @returns VBox status code.
4293 * @param pVM The VM to operate on.
4294 * @param pVCpu The VM CPU to operate on.
4295 * @param enmFlush Type of flush
4296 * @param GCPtr Virtual address of the page to flush
4297 */
4298static void vmxR0FlushVPID(PVM pVM, PVMCPU pVCpu, VMX_FLUSH enmFlush, RTGCPTR GCPtr)
4299{
4300#if HC_ARCH_BITS == 32
4301 /* If we get a flush in 64 bits guest mode, then force a full TLB flush. Invvpid probably takes only 32 bits addresses. (@todo) */
4302 if ( CPUMIsGuestInLongMode(pVCpu)
4303 && !VMX_IS_64BIT_HOST_MODE())
4304 {
4305 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
4306 }
4307 else
4308#endif
4309 {
4310 uint64_t descriptor[2];
4311
4312 Assert(pVM->hwaccm.s.vmx.fVPID);
4313 descriptor[0] = pVCpu->hwaccm.s.uCurrentASID;
4314 descriptor[1] = GCPtr;
4315 int rc = VMXR0InvVPID(enmFlush, &descriptor[0]);
4316 AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvVPID %x %x %RGv failed with %d\n", enmFlush, pVCpu->hwaccm.s.uCurrentASID, GCPtr, rc));
4317 }
4318}
4319#endif /* HWACCM_VTX_WITH_VPID */
4320
4321/**
4322 * Invalidates a guest page
4323 *
4324 * @returns VBox status code.
4325 * @param pVM The VM to operate on.
4326 * @param pVCpu The VM CPU to operate on.
4327 * @param GCVirt Page to invalidate
4328 */
4329VMMR0DECL(int) VMXR0InvalidatePage(PVM pVM, PVMCPU pVCpu, RTGCPTR GCVirt)
4330{
4331 bool fFlushPending = VMCPU_FF_ISSET(pVCpu, VMCPU_FF_TLB_FLUSH);
4332
4333 Log2(("VMXR0InvalidatePage %RGv\n", GCVirt));
4334
4335 /* Only relevant if we want to use VPID.
4336 * In the nested paging case we still see such calls, but
4337 * can safely ignore them. (e.g. after cr3 updates)
4338 */
4339#ifdef HWACCM_VTX_WITH_VPID
4340 /* Skip it if a TLB flush is already pending. */
4341 if ( !fFlushPending
4342 && pVM->hwaccm.s.vmx.fVPID)
4343 vmxR0FlushVPID(pVM, pVCpu, pVM->hwaccm.s.vmx.enmFlushPage, GCVirt);
4344#endif /* HWACCM_VTX_WITH_VPID */
4345
4346 return VINF_SUCCESS;
4347}
4348
4349/**
4350 * Invalidates a guest page by physical address
4351 *
4352 * NOTE: Assumes the current instruction references this physical page though a virtual address!!
4353 *
4354 * @returns VBox status code.
4355 * @param pVM The VM to operate on.
4356 * @param pVCpu The VM CPU to operate on.
4357 * @param GCPhys Page to invalidate
4358 */
4359VMMR0DECL(int) VMXR0InvalidatePhysPage(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys)
4360{
4361 bool fFlushPending = VMCPU_FF_ISSET(pVCpu, VMCPU_FF_TLB_FLUSH);
4362
4363 Assert(pVM->hwaccm.s.fNestedPaging);
4364
4365 LogFlow(("VMXR0InvalidatePhysPage %RGp\n", GCPhys));
4366
4367 /* Skip it if a TLB flush is already pending. */
4368 if (!fFlushPending)
4369 vmxR0FlushEPT(pVM, pVCpu, pVM->hwaccm.s.vmx.enmFlushPage, GCPhys);
4370
4371 return VINF_SUCCESS;
4372}
4373
4374/**
4375 * Report world switch error and dump some useful debug info
4376 *
4377 * @param pVM The VM to operate on.
4378 * @param pVCpu The VMCPU to operate on.
4379 * @param rc Return code
4380 * @param pCtx Current CPU context (not updated)
4381 */
4382static void VMXR0ReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, VBOXSTRICTRC rc, PCPUMCTX pCtx)
4383{
4384 switch (VBOXSTRICTRC_VAL(rc))
4385 {
4386 case VERR_VMX_INVALID_VMXON_PTR:
4387 AssertFailed();
4388 break;
4389
4390 case VERR_VMX_UNABLE_TO_START_VM:
4391 case VERR_VMX_UNABLE_TO_RESUME_VM:
4392 {
4393 int rc2;
4394 RTCCUINTREG exitReason, instrError;
4395
4396 rc2 = VMXReadVMCS(VMX_VMCS32_RO_EXIT_REASON, &exitReason);
4397 rc2 |= VMXReadVMCS(VMX_VMCS32_RO_VM_INSTR_ERROR, &instrError);
4398 AssertRC(rc2);
4399 if (rc2 == VINF_SUCCESS)
4400 {
4401 Log(("Unable to start/resume VM for reason: %x. Instruction error %x\n", (uint32_t)exitReason, (uint32_t)instrError));
4402 Log(("Current stack %08x\n", &rc2));
4403
4404 pVCpu->hwaccm.s.vmx.lasterror.ulInstrError = instrError;
4405 pVCpu->hwaccm.s.vmx.lasterror.ulExitReason = exitReason;
4406
4407#ifdef VBOX_STRICT
4408 RTGDTR gdtr;
4409 PCX86DESCHC pDesc;
4410 RTCCUINTREG val;
4411
4412 ASMGetGDTR(&gdtr);
4413
4414 VMXReadVMCS(VMX_VMCS64_GUEST_RIP, &val);
4415 Log(("Old eip %RGv new %RGv\n", (RTGCPTR)pCtx->rip, (RTGCPTR)val));
4416 VMXReadVMCS(VMX_VMCS_CTRL_PIN_EXEC_CONTROLS, &val);
4417 Log(("VMX_VMCS_CTRL_PIN_EXEC_CONTROLS %08x\n", val));
4418 VMXReadVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, &val);
4419 Log(("VMX_VMCS_CTRL_PROC_EXEC_CONTROLS %08x\n", val));
4420 VMXReadVMCS(VMX_VMCS_CTRL_ENTRY_CONTROLS, &val);
4421 Log(("VMX_VMCS_CTRL_ENTRY_CONTROLS %08x\n", val));
4422 VMXReadVMCS(VMX_VMCS_CTRL_EXIT_CONTROLS, &val);
4423 Log(("VMX_VMCS_CTRL_EXIT_CONTROLS %08x\n", val));
4424
4425 VMXReadVMCS(VMX_VMCS_HOST_CR0, &val);
4426 Log(("VMX_VMCS_HOST_CR0 %08x\n", val));
4427
4428 VMXReadVMCS(VMX_VMCS_HOST_CR3, &val);
4429 Log(("VMX_VMCS_HOST_CR3 %08x\n", val));
4430
4431 VMXReadVMCS(VMX_VMCS_HOST_CR4, &val);
4432 Log(("VMX_VMCS_HOST_CR4 %08x\n", val));
4433
4434 VMXReadVMCS(VMX_VMCS16_HOST_FIELD_CS, &val);
4435 Log(("VMX_VMCS_HOST_FIELD_CS %08x\n", val));
4436
4437 VMXReadVMCS(VMX_VMCS_GUEST_RFLAGS, &val);
4438 Log(("VMX_VMCS_GUEST_RFLAGS %08x\n", val));
4439
4440 if (val < gdtr.cbGdt)
4441 {
4442 pDesc = (PCX86DESCHC)(gdtr.pGdt + (val & X86_SEL_MASK));
4443 HWACCMR0DumpDescriptor(pDesc, val, "CS: ");
4444 }
4445
4446 VMXReadVMCS(VMX_VMCS16_HOST_FIELD_DS, &val);
4447 Log(("VMX_VMCS_HOST_FIELD_DS %08x\n", val));
4448 if (val < gdtr.cbGdt)
4449 {
4450 pDesc = (PCX86DESCHC)(gdtr.pGdt + (val & X86_SEL_MASK));
4451 HWACCMR0DumpDescriptor(pDesc, val, "DS: ");
4452 }
4453
4454 VMXReadVMCS(VMX_VMCS16_HOST_FIELD_ES, &val);
4455 Log(("VMX_VMCS_HOST_FIELD_ES %08x\n", val));
4456 if (val < gdtr.cbGdt)
4457 {
4458 pDesc = (PCX86DESCHC)(gdtr.pGdt + (val & X86_SEL_MASK));
4459 HWACCMR0DumpDescriptor(pDesc, val, "ES: ");
4460 }
4461
4462 VMXReadVMCS(VMX_VMCS16_HOST_FIELD_FS, &val);
4463 Log(("VMX_VMCS16_HOST_FIELD_FS %08x\n", val));
4464 if (val < gdtr.cbGdt)
4465 {
4466 pDesc = (PCX86DESCHC)(gdtr.pGdt + (val & X86_SEL_MASK));
4467 HWACCMR0DumpDescriptor(pDesc, val, "FS: ");
4468 }
4469
4470 VMXReadVMCS(VMX_VMCS16_HOST_FIELD_GS, &val);
4471 Log(("VMX_VMCS16_HOST_FIELD_GS %08x\n", val));
4472 if (val < gdtr.cbGdt)
4473 {
4474 pDesc = (PCX86DESCHC)(gdtr.pGdt + (val & X86_SEL_MASK));
4475 HWACCMR0DumpDescriptor(pDesc, val, "GS: ");
4476 }
4477
4478 VMXReadVMCS(VMX_VMCS16_HOST_FIELD_SS, &val);
4479 Log(("VMX_VMCS16_HOST_FIELD_SS %08x\n", val));
4480 if (val < gdtr.cbGdt)
4481 {
4482 pDesc = (PCX86DESCHC)(gdtr.pGdt + (val & X86_SEL_MASK));
4483 HWACCMR0DumpDescriptor(pDesc, val, "SS: ");
4484 }
4485
4486 VMXReadVMCS(VMX_VMCS16_HOST_FIELD_TR, &val);
4487 Log(("VMX_VMCS16_HOST_FIELD_TR %08x\n", val));
4488 if (val < gdtr.cbGdt)
4489 {
4490 pDesc = (PCX86DESCHC)(gdtr.pGdt + (val & X86_SEL_MASK));
4491 HWACCMR0DumpDescriptor(pDesc, val, "TR: ");
4492 }
4493
4494 VMXReadVMCS(VMX_VMCS_HOST_TR_BASE, &val);
4495 Log(("VMX_VMCS_HOST_TR_BASE %RHv\n", val));
4496
4497 VMXReadVMCS(VMX_VMCS_HOST_GDTR_BASE, &val);
4498 Log(("VMX_VMCS_HOST_GDTR_BASE %RHv\n", val));
4499 VMXReadVMCS(VMX_VMCS_HOST_IDTR_BASE, &val);
4500 Log(("VMX_VMCS_HOST_IDTR_BASE %RHv\n", val));
4501
4502 VMXReadVMCS(VMX_VMCS32_HOST_SYSENTER_CS, &val);
4503 Log(("VMX_VMCS_HOST_SYSENTER_CS %08x\n", val));
4504
4505 VMXReadVMCS(VMX_VMCS_HOST_SYSENTER_EIP, &val);
4506 Log(("VMX_VMCS_HOST_SYSENTER_EIP %RHv\n", val));
4507
4508 VMXReadVMCS(VMX_VMCS_HOST_SYSENTER_ESP, &val);
4509 Log(("VMX_VMCS_HOST_SYSENTER_ESP %RHv\n", val));
4510
4511 VMXReadVMCS(VMX_VMCS_HOST_RSP, &val);
4512 Log(("VMX_VMCS_HOST_RSP %RHv\n", val));
4513 VMXReadVMCS(VMX_VMCS_HOST_RIP, &val);
4514 Log(("VMX_VMCS_HOST_RIP %RHv\n", val));
4515
4516# if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4517 if (VMX_IS_64BIT_HOST_MODE())
4518 {
4519 Log(("MSR_K6_EFER = %RX64\n", ASMRdMsr(MSR_K6_EFER)));
4520 Log(("MSR_K6_STAR = %RX64\n", ASMRdMsr(MSR_K6_STAR)));
4521 Log(("MSR_K8_LSTAR = %RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
4522 Log(("MSR_K8_CSTAR = %RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
4523 Log(("MSR_K8_SF_MASK = %RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
4524 }
4525# endif
4526#endif /* VBOX_STRICT */
4527 }
4528 break;
4529 }
4530
4531 default:
4532 /* impossible */
4533 AssertMsgFailed(("%Rrc (%#x)\n", VBOXSTRICTRC_VAL(rc), VBOXSTRICTRC_VAL(rc)));
4534 break;
4535 }
4536}
4537
4538#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4539/**
4540 * Prepares for and executes VMLAUNCH (64 bits guest mode)
4541 *
4542 * @returns VBox status code
4543 * @param fResume vmlauch/vmresume
4544 * @param pCtx Guest context
4545 * @param pCache VMCS cache
4546 * @param pVM The VM to operate on.
4547 * @param pVCpu The VMCPU to operate on.
4548 */
4549DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
4550{
4551 uint32_t aParam[6];
4552 PHWACCM_CPUINFO pCpu;
4553 RTHCPHYS pPageCpuPhys;
4554 int rc;
4555
4556 pCpu = HWACCMR0GetCurrentCpu();
4557 pPageCpuPhys = RTR0MemObjGetPagePhysAddr(pCpu->pMemObj, 0);
4558
4559#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4560 pCache->uPos = 1;
4561 pCache->interPD = PGMGetInterPaeCR3(pVM);
4562 pCache->pSwitcher = (uint64_t)pVM->hwaccm.s.pfnHost32ToGuest64R0;
4563#endif
4564
4565#ifdef DEBUG
4566 pCache->TestIn.pPageCpuPhys = 0;
4567 pCache->TestIn.pVMCSPhys = 0;
4568 pCache->TestIn.pCache = 0;
4569 pCache->TestOut.pVMCSPhys = 0;
4570 pCache->TestOut.pCache = 0;
4571 pCache->TestOut.pCtx = 0;
4572 pCache->TestOut.eflags = 0;
4573#endif
4574
4575 aParam[0] = (uint32_t)(pPageCpuPhys); /* Param 1: VMXON physical address - Lo. */
4576 aParam[1] = (uint32_t)(pPageCpuPhys >> 32); /* Param 1: VMXON physical address - Hi. */
4577 aParam[2] = (uint32_t)(pVCpu->hwaccm.s.vmx.pVMCSPhys); /* Param 2: VMCS physical address - Lo. */
4578 aParam[3] = (uint32_t)(pVCpu->hwaccm.s.vmx.pVMCSPhys >> 32); /* Param 2: VMCS physical address - Hi. */
4579 aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hwaccm.s.vmx.VMCSCache);
4580 aParam[5] = 0;
4581
4582#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4583 pCtx->dr[4] = pVM->hwaccm.s.vmx.pScratchPhys + 16 + 8;
4584 *(uint32_t *)(pVM->hwaccm.s.vmx.pScratch + 16 + 8) = 1;
4585#endif
4586 rc = VMXR0Execute64BitsHandler(pVM, pVCpu, pCtx, pVM->hwaccm.s.pfnVMXGCStartVM64, 6, &aParam[0]);
4587
4588#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4589 Assert(*(uint32_t *)(pVM->hwaccm.s.vmx.pScratch + 16 + 8) == 5);
4590 Assert(pCtx->dr[4] == 10);
4591 *(uint32_t *)(pVM->hwaccm.s.vmx.pScratch + 16 + 8) = 0xff;
4592#endif
4593
4594#ifdef DEBUG
4595 AssertMsg(pCache->TestIn.pPageCpuPhys == pPageCpuPhys, ("%RHp vs %RHp\n", pCache->TestIn.pPageCpuPhys, pPageCpuPhys));
4596 AssertMsg(pCache->TestIn.pVMCSPhys == pVCpu->hwaccm.s.vmx.pVMCSPhys, ("%RHp vs %RHp\n", pCache->TestIn.pVMCSPhys, pVCpu->hwaccm.s.vmx.pVMCSPhys));
4597 AssertMsg(pCache->TestIn.pVMCSPhys == pCache->TestOut.pVMCSPhys, ("%RHp vs %RHp\n", pCache->TestIn.pVMCSPhys, pCache->TestOut.pVMCSPhys));
4598 AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache, pCache->TestOut.pCache));
4599 AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hwaccm.s.vmx.VMCSCache), ("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hwaccm.s.vmx.VMCSCache)));
4600 AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx, pCache->TestOut.pCtx));
4601 Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
4602#endif
4603 return rc;
4604}
4605
4606/**
4607 * Executes the specified handler in 64 mode
4608 *
4609 * @returns VBox status code.
4610 * @param pVM The VM to operate on.
4611 * @param pVCpu The VMCPU to operate on.
4612 * @param pCtx Guest context
4613 * @param pfnHandler RC handler
4614 * @param cbParam Number of parameters
4615 * @param paParam Array of 32 bits parameters
4616 */
4617VMMR0DECL(int) VMXR0Execute64BitsHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, RTRCPTR pfnHandler, uint32_t cbParam, uint32_t *paParam)
4618{
4619 int rc, rc2;
4620 PHWACCM_CPUINFO pCpu;
4621 RTHCPHYS pPageCpuPhys;
4622 RTHCUINTREG uOldEFlags;
4623
4624 AssertReturn(pVM->hwaccm.s.pfnHost32ToGuest64R0, VERR_INTERNAL_ERROR);
4625 Assert(pfnHandler);
4626 Assert(pVCpu->hwaccm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hwaccm.s.vmx.VMCSCache.Write.aField));
4627 Assert(pVCpu->hwaccm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hwaccm.s.vmx.VMCSCache.Read.aField));
4628
4629#ifdef VBOX_STRICT
4630 for (unsigned i=0;i<pVCpu->hwaccm.s.vmx.VMCSCache.Write.cValidEntries;i++)
4631 Assert(vmxR0IsValidWriteField(pVCpu->hwaccm.s.vmx.VMCSCache.Write.aField[i]));
4632
4633 for (unsigned i=0;i<pVCpu->hwaccm.s.vmx.VMCSCache.Read.cValidEntries;i++)
4634 Assert(vmxR0IsValidReadField(pVCpu->hwaccm.s.vmx.VMCSCache.Read.aField[i]));
4635#endif
4636
4637 /* Disable interrupts. */
4638 uOldEFlags = ASMIntDisableFlags();
4639
4640 pCpu = HWACCMR0GetCurrentCpu();
4641 pPageCpuPhys = RTR0MemObjGetPagePhysAddr(pCpu->pMemObj, 0);
4642
4643 /* Clear VM Control Structure. Marking it inactive, clearing implementation specific data and writing back VMCS data to memory. */
4644 VMXClearVMCS(pVCpu->hwaccm.s.vmx.pVMCSPhys);
4645
4646 /* Leave VMX Root Mode. */
4647 VMXDisable();
4648
4649 ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
4650
4651 CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
4652 CPUMSetHyperEIP(pVCpu, pfnHandler);
4653 for (int i=(int)cbParam-1;i>=0;i--)
4654 CPUMPushHyper(pVCpu, paParam[i]);
4655
4656 STAM_PROFILE_ADV_START(&pVCpu->hwaccm.s.StatWorldSwitch3264, z);
4657 /* Call switcher. */
4658 rc = pVM->hwaccm.s.pfnHost32ToGuest64R0(pVM, RT_OFFSETOF(VM, aCpus[pVCpu->idCpu].cpum) - RT_OFFSETOF(VM, cpum));
4659 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatWorldSwitch3264, z);
4660
4661 /* Make sure the VMX instructions don't cause #UD faults. */
4662 ASMSetCR4(ASMGetCR4() | X86_CR4_VMXE);
4663
4664 /* Enter VMX Root Mode */
4665 rc2 = VMXEnable(pPageCpuPhys);
4666 if (RT_FAILURE(rc2))
4667 {
4668 ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
4669 ASMSetFlags(uOldEFlags);
4670 return VERR_VMX_VMXON_FAILED;
4671 }
4672
4673 rc2 = VMXActivateVMCS(pVCpu->hwaccm.s.vmx.pVMCSPhys);
4674 AssertRC(rc2);
4675 Assert(!(ASMGetFlags() & X86_EFL_IF));
4676 ASMSetFlags(uOldEFlags);
4677 return rc;
4678}
4679
4680#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL) */
4681
4682
4683#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
4684/**
4685 * Executes VMWRITE
4686 *
4687 * @returns VBox status code
4688 * @param pVCpu The VMCPU to operate on.
4689 * @param idxField VMCS index
4690 * @param u64Val 16, 32 or 64 bits value
4691 */
4692VMMR0DECL(int) VMXWriteVMCS64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
4693{
4694 int rc;
4695
4696 switch (idxField)
4697 {
4698 case VMX_VMCS_CTRL_TSC_OFFSET_FULL:
4699 case VMX_VMCS_CTRL_IO_BITMAP_A_FULL:
4700 case VMX_VMCS_CTRL_IO_BITMAP_B_FULL:
4701 case VMX_VMCS_CTRL_MSR_BITMAP_FULL:
4702 case VMX_VMCS_CTRL_VMEXIT_MSR_STORE_FULL:
4703 case VMX_VMCS_CTRL_VMEXIT_MSR_LOAD_FULL:
4704 case VMX_VMCS_CTRL_VMENTRY_MSR_LOAD_FULL:
4705 case VMX_VMCS_CTRL_VAPIC_PAGEADDR_FULL:
4706 case VMX_VMCS_CTRL_APIC_ACCESSADDR_FULL:
4707 case VMX_VMCS_GUEST_LINK_PTR_FULL:
4708 case VMX_VMCS_GUEST_PDPTR0_FULL:
4709 case VMX_VMCS_GUEST_PDPTR1_FULL:
4710 case VMX_VMCS_GUEST_PDPTR2_FULL:
4711 case VMX_VMCS_GUEST_PDPTR3_FULL:
4712 case VMX_VMCS_GUEST_DEBUGCTL_FULL:
4713 case VMX_VMCS_GUEST_EFER_FULL:
4714 case VMX_VMCS_CTRL_EPTP_FULL:
4715 /* These fields consist of two parts, which are both writable in 32 bits mode. */
4716 rc = VMXWriteVMCS32(idxField, u64Val);
4717 rc |= VMXWriteVMCS32(idxField + 1, (uint32_t)(u64Val >> 32ULL));
4718 AssertRC(rc);
4719 return rc;
4720
4721 case VMX_VMCS64_GUEST_LDTR_BASE:
4722 case VMX_VMCS64_GUEST_TR_BASE:
4723 case VMX_VMCS64_GUEST_GDTR_BASE:
4724 case VMX_VMCS64_GUEST_IDTR_BASE:
4725 case VMX_VMCS64_GUEST_SYSENTER_EIP:
4726 case VMX_VMCS64_GUEST_SYSENTER_ESP:
4727 case VMX_VMCS64_GUEST_CR0:
4728 case VMX_VMCS64_GUEST_CR4:
4729 case VMX_VMCS64_GUEST_CR3:
4730 case VMX_VMCS64_GUEST_DR7:
4731 case VMX_VMCS64_GUEST_RIP:
4732 case VMX_VMCS64_GUEST_RSP:
4733 case VMX_VMCS64_GUEST_CS_BASE:
4734 case VMX_VMCS64_GUEST_DS_BASE:
4735 case VMX_VMCS64_GUEST_ES_BASE:
4736 case VMX_VMCS64_GUEST_FS_BASE:
4737 case VMX_VMCS64_GUEST_GS_BASE:
4738 case VMX_VMCS64_GUEST_SS_BASE:
4739 /* Queue a 64 bits value as we can't set it in 32 bits host mode. */
4740 if (u64Val >> 32ULL)
4741 rc = VMXWriteCachedVMCSEx(pVCpu, idxField, u64Val);
4742 else
4743 rc = VMXWriteVMCS32(idxField, (uint32_t)u64Val);
4744
4745 return rc;
4746
4747 default:
4748 AssertMsgFailed(("Unexpected field %x\n", idxField));
4749 return VERR_INVALID_PARAMETER;
4750 }
4751}
4752
4753/**
4754 * Cache VMCS writes for performance reasons (Darwin) and for running 64 bits guests on 32 bits hosts.
4755 *
4756 * @param pVCpu The VMCPU to operate on.
4757 * @param idxField VMCS field
4758 * @param u64Val Value
4759 */
4760VMMR0DECL(int) VMXWriteCachedVMCSEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
4761{
4762 PVMCSCACHE pCache = &pVCpu->hwaccm.s.vmx.VMCSCache;
4763
4764 AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1, ("entries=%x\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
4765
4766 /* Make sure there are no duplicates. */
4767 for (unsigned i=0;i<pCache->Write.cValidEntries;i++)
4768 {
4769 if (pCache->Write.aField[i] == idxField)
4770 {
4771 pCache->Write.aFieldVal[i] = u64Val;
4772 return VINF_SUCCESS;
4773 }
4774 }
4775
4776 pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
4777 pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
4778 pCache->Write.cValidEntries++;
4779 return VINF_SUCCESS;
4780}
4781
4782#endif /* HC_ARCH_BITS == 32 && !VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0 */
4783
4784#ifdef VBOX_STRICT
4785static bool vmxR0IsValidReadField(uint32_t idxField)
4786{
4787 switch(idxField)
4788 {
4789 case VMX_VMCS64_GUEST_RIP:
4790 case VMX_VMCS64_GUEST_RSP:
4791 case VMX_VMCS_GUEST_RFLAGS:
4792 case VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE:
4793 case VMX_VMCS_CTRL_CR0_READ_SHADOW:
4794 case VMX_VMCS64_GUEST_CR0:
4795 case VMX_VMCS_CTRL_CR4_READ_SHADOW:
4796 case VMX_VMCS64_GUEST_CR4:
4797 case VMX_VMCS64_GUEST_DR7:
4798 case VMX_VMCS32_GUEST_SYSENTER_CS:
4799 case VMX_VMCS64_GUEST_SYSENTER_EIP:
4800 case VMX_VMCS64_GUEST_SYSENTER_ESP:
4801 case VMX_VMCS32_GUEST_GDTR_LIMIT:
4802 case VMX_VMCS64_GUEST_GDTR_BASE:
4803 case VMX_VMCS32_GUEST_IDTR_LIMIT:
4804 case VMX_VMCS64_GUEST_IDTR_BASE:
4805 case VMX_VMCS16_GUEST_FIELD_CS:
4806 case VMX_VMCS32_GUEST_CS_LIMIT:
4807 case VMX_VMCS64_GUEST_CS_BASE:
4808 case VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS:
4809 case VMX_VMCS16_GUEST_FIELD_DS:
4810 case VMX_VMCS32_GUEST_DS_LIMIT:
4811 case VMX_VMCS64_GUEST_DS_BASE:
4812 case VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS:
4813 case VMX_VMCS16_GUEST_FIELD_ES:
4814 case VMX_VMCS32_GUEST_ES_LIMIT:
4815 case VMX_VMCS64_GUEST_ES_BASE:
4816 case VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS:
4817 case VMX_VMCS16_GUEST_FIELD_FS:
4818 case VMX_VMCS32_GUEST_FS_LIMIT:
4819 case VMX_VMCS64_GUEST_FS_BASE:
4820 case VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS:
4821 case VMX_VMCS16_GUEST_FIELD_GS:
4822 case VMX_VMCS32_GUEST_GS_LIMIT:
4823 case VMX_VMCS64_GUEST_GS_BASE:
4824 case VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS:
4825 case VMX_VMCS16_GUEST_FIELD_SS:
4826 case VMX_VMCS32_GUEST_SS_LIMIT:
4827 case VMX_VMCS64_GUEST_SS_BASE:
4828 case VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS:
4829 case VMX_VMCS16_GUEST_FIELD_LDTR:
4830 case VMX_VMCS32_GUEST_LDTR_LIMIT:
4831 case VMX_VMCS64_GUEST_LDTR_BASE:
4832 case VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS:
4833 case VMX_VMCS16_GUEST_FIELD_TR:
4834 case VMX_VMCS32_GUEST_TR_LIMIT:
4835 case VMX_VMCS64_GUEST_TR_BASE:
4836 case VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS:
4837 case VMX_VMCS32_RO_EXIT_REASON:
4838 case VMX_VMCS32_RO_VM_INSTR_ERROR:
4839 case VMX_VMCS32_RO_EXIT_INSTR_LENGTH:
4840 case VMX_VMCS32_RO_EXIT_INTERRUPTION_ERRCODE:
4841 case VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO:
4842 case VMX_VMCS32_RO_EXIT_INSTR_INFO:
4843 case VMX_VMCS_RO_EXIT_QUALIFICATION:
4844 case VMX_VMCS32_RO_IDT_INFO:
4845 case VMX_VMCS32_RO_IDT_ERRCODE:
4846 case VMX_VMCS64_GUEST_CR3:
4847 case VMX_VMCS_EXIT_PHYS_ADDR_FULL:
4848 return true;
4849 }
4850 return false;
4851}
4852
4853static bool vmxR0IsValidWriteField(uint32_t idxField)
4854{
4855 switch(idxField)
4856 {
4857 case VMX_VMCS64_GUEST_LDTR_BASE:
4858 case VMX_VMCS64_GUEST_TR_BASE:
4859 case VMX_VMCS64_GUEST_GDTR_BASE:
4860 case VMX_VMCS64_GUEST_IDTR_BASE:
4861 case VMX_VMCS64_GUEST_SYSENTER_EIP:
4862 case VMX_VMCS64_GUEST_SYSENTER_ESP:
4863 case VMX_VMCS64_GUEST_CR0:
4864 case VMX_VMCS64_GUEST_CR4:
4865 case VMX_VMCS64_GUEST_CR3:
4866 case VMX_VMCS64_GUEST_DR7:
4867 case VMX_VMCS64_GUEST_RIP:
4868 case VMX_VMCS64_GUEST_RSP:
4869 case VMX_VMCS64_GUEST_CS_BASE:
4870 case VMX_VMCS64_GUEST_DS_BASE:
4871 case VMX_VMCS64_GUEST_ES_BASE:
4872 case VMX_VMCS64_GUEST_FS_BASE:
4873 case VMX_VMCS64_GUEST_GS_BASE:
4874 case VMX_VMCS64_GUEST_SS_BASE:
4875 return true;
4876 }
4877 return false;
4878}
4879
4880#endif
Note: See TracBrowser for help on using the repository browser.

© 2024 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette