VirtualBox

source: vbox/trunk/src/VBox/VMM/VMMR0/HWVMXR0.cpp@ 40651

Last change on this file since 40651 was 40561, checked in by vboxsync, 13 years ago

Framework for handling VT-x MTF exits.
  • Property svn:eol-style set to native
  • Property svn:keywords set to Id
File size: 200.0 KB
Line 
1/* $Id: HWVMXR0.cpp 40561 2012-03-21 11:57:10Z vboxsync $ */
2/** @file
3 * HM VMX (VT-x) - Host Context Ring 0.
4 */
5
6/*
7 * Copyright (C) 2006-2011 Oracle Corporation
8 *
9 * This file is part of VirtualBox Open Source Edition (OSE), as
10 * available from http://www.virtualbox.org. This file is free software;
11 * you can redistribute it and/or modify it under the terms of the GNU
12 * General Public License (GPL) as published by the Free Software
13 * Foundation, in version 2 as it comes in the "COPYING" file of the
14 * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15 * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16 */
17
18
19/*******************************************************************************
20* Header Files *
21*******************************************************************************/
22#define LOG_GROUP LOG_GROUP_HWACCM
23#include <iprt/asm-amd64-x86.h>
24#include <VBox/vmm/hwaccm.h>
25#include <VBox/vmm/pgm.h>
26#include <VBox/vmm/dbgf.h>
27#include <VBox/vmm/dbgftrace.h>
28#include <VBox/vmm/selm.h>
29#include <VBox/vmm/iom.h>
30#ifdef VBOX_WITH_REM
31# include <VBox/vmm/rem.h>
32#endif
33#include <VBox/vmm/tm.h>
34#include "HWACCMInternal.h"
35#include <VBox/vmm/vm.h>
36#include <VBox/vmm/pdmapi.h>
37#include <VBox/err.h>
38#include <VBox/log.h>
39#include <iprt/assert.h>
40#include <iprt/param.h>
41#include <iprt/string.h>
42#include <iprt/time.h>
43#ifdef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
44# include <iprt/thread.h>
45#endif
46#include <iprt/x86.h>
47#include "HWVMXR0.h"
48
49/*******************************************************************************
50* Defined Constants And Macros *
51*******************************************************************************/
52#if defined(RT_ARCH_AMD64)
53# define VMX_IS_64BIT_HOST_MODE() (true)
54#elif defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
55# define VMX_IS_64BIT_HOST_MODE() (g_fVMXIs64bitHost != 0)
56#else
57# define VMX_IS_64BIT_HOST_MODE() (false)
58#endif
59
60/*******************************************************************************
61* Global Variables *
62*******************************************************************************/
63/* IO operation lookup arrays. */
64static uint32_t const g_aIOSize[4] = {1, 2, 0, 4};
65static uint32_t const g_aIOOpAnd[4] = {0xff, 0xffff, 0, 0xffffffff};
66
67#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
68/** See HWACCMR0A.asm. */
69extern "C" uint32_t g_fVMXIs64bitHost;
70#endif
71
72/*******************************************************************************
73* Local Functions *
74*******************************************************************************/
75static void hmR0VmxReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, VBOXSTRICTRC rc, PCPUMCTX pCtx);
76static DECLCALLBACK(void) hmR0VmxSetupTLBEPT(PVM pVM, PVMCPU pVCpu);
77static DECLCALLBACK(void) hmR0VmxSetupTLBVPID(PVM pVM, PVMCPU pVCpu);
78static DECLCALLBACK(void) hmR0VmxSetupTLBDummy(PVM pVM, PVMCPU pVCpu);
79static void hmR0VmxFlushEPT(PVM pVM, PVMCPU pVCpu, VMX_FLUSH enmFlush, RTGCPHYS GCPhys);
80static void hmR0VmxFlushVPID(PVM pVM, PVMCPU pVCpu, VMX_FLUSH enmFlush, RTGCPTR GCPtr);
81static void hmR0VmxUpdateExceptionBitmap(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx);
82static void hmR0VmxSetMSRPermission(PVMCPU pVCpu, unsigned ulMSR, bool fRead, bool fWrite);
83
84
85static void hmR0VmxCheckError(PVM pVM, PVMCPU pVCpu, int rc)
86{
87 if (rc == VERR_VMX_GENERIC)
88 {
89 RTCCUINTREG instrError;
90
91 VMXReadVMCS(VMX_VMCS32_RO_VM_INSTR_ERROR, &instrError);
92 pVCpu->hwaccm.s.vmx.lasterror.ulInstrError = instrError;
93 }
94 pVM->hwaccm.s.lLastError = rc;
95}
96
97/**
98 * Sets up and activates VT-x on the current CPU
99 *
100 * @returns VBox status code.
101 * @param pCpu CPU info struct
102 * @param pVM The VM to operate on. (can be NULL after a resume!!)
103 * @param pvCpuPage Pointer to the global cpu page.
104 * @param HCPhysCpuPage Physical address of the global cpu page.
105 */
106VMMR0DECL(int) VMXR0EnableCpu(PHMGLOBLCPUINFO pCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
107{
108 AssertReturn(HCPhysCpuPage != 0 && HCPhysCpuPage != NIL_RTHCPHYS, VERR_INVALID_PARAMETER);
109 AssertReturn(pvCpuPage, VERR_INVALID_PARAMETER);
110 NOREF(pCpu);
111
112 if (pVM)
113 {
114 /* Set revision dword at the beginning of the VMXON structure. */
115 *(uint32_t *)pvCpuPage = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hwaccm.s.vmx.msr.vmx_basic_info);
116 }
117
118 /** @todo we should unmap the two pages from the virtual address space in order to prevent accidental corruption.
119 * (which can have very bad consequences!!!)
120 */
121
122 if (ASMGetCR4() & X86_CR4_VMXE)
123 return VERR_VMX_IN_VMX_ROOT_MODE;
124
125 /* Make sure the VMX instructions don't cause #UD faults. */
126 ASMSetCR4(ASMGetCR4() | X86_CR4_VMXE);
127
128 /* Enter VMX Root Mode. */
129 int rc = VMXEnable(HCPhysCpuPage);
130 if (RT_FAILURE(rc))
131 {
132 ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
133 return VERR_VMX_VMXON_FAILED;
134 }
135 return VINF_SUCCESS;
136}
137
138/**
139 * Deactivates VT-x on the current CPU
140 *
141 * @returns VBox status code.
142 * @param pCpu CPU info struct
143 * @param pvCpuPage Pointer to the global cpu page.
144 * @param HCPhysCpuPage Physical address of the global cpu page.
145 */
146VMMR0DECL(int) VMXR0DisableCpu(PHMGLOBLCPUINFO pCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
147{
148 AssertReturn(HCPhysCpuPage != 0 && HCPhysCpuPage != NIL_RTHCPHYS, VERR_INVALID_PARAMETER);
149 AssertReturn(pvCpuPage, VERR_INVALID_PARAMETER);
150 NOREF(pCpu);
151
152 /* If we're somehow not in VMX root mode, then we shouldn't dare leaving it. */
153 if (!(ASMGetCR4() & X86_CR4_VMXE))
154 return VERR_VMX_NOT_IN_VMX_ROOT_MODE;
155
156 /* Leave VMX Root Mode. */
157 VMXDisable();
158
159 /* And clear the X86_CR4_VMXE bit. */
160 ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
161 return VINF_SUCCESS;
162}
163
164/**
165 * Does Ring-0 per VM VT-x init.
166 *
167 * @returns VBox status code.
168 * @param pVM The VM to operate on.
169 */
170VMMR0DECL(int) VMXR0InitVM(PVM pVM)
171{
172 int rc;
173
174#ifdef LOG_ENABLED
175 SUPR0Printf("VMXR0InitVM %x\n", pVM);
176#endif
177
178 pVM->hwaccm.s.vmx.pMemObjAPIC = NIL_RTR0MEMOBJ;
179
180 if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
181 {
182 /* Allocate one page for the APIC physical page (serves for filtering accesses). */
183 rc = RTR0MemObjAllocCont(&pVM->hwaccm.s.vmx.pMemObjAPIC, PAGE_SIZE, true /* executable R0 mapping */);
184 AssertRC(rc);
185 if (RT_FAILURE(rc))
186 return rc;
187
188 pVM->hwaccm.s.vmx.pAPIC = (uint8_t *)RTR0MemObjAddress(pVM->hwaccm.s.vmx.pMemObjAPIC);
189 pVM->hwaccm.s.vmx.pAPICPhys = RTR0MemObjGetPagePhysAddr(pVM->hwaccm.s.vmx.pMemObjAPIC, 0);
190 ASMMemZero32(pVM->hwaccm.s.vmx.pAPIC, PAGE_SIZE);
191 }
192 else
193 {
194 pVM->hwaccm.s.vmx.pMemObjAPIC = 0;
195 pVM->hwaccm.s.vmx.pAPIC = 0;
196 pVM->hwaccm.s.vmx.pAPICPhys = 0;
197 }
198
199#ifdef VBOX_WITH_CRASHDUMP_MAGIC
200 {
201 rc = RTR0MemObjAllocCont(&pVM->hwaccm.s.vmx.pMemObjScratch, PAGE_SIZE, true /* executable R0 mapping */);
202 AssertRC(rc);
203 if (RT_FAILURE(rc))
204 return rc;
205
206 pVM->hwaccm.s.vmx.pScratch = (uint8_t *)RTR0MemObjAddress(pVM->hwaccm.s.vmx.pMemObjScratch);
207 pVM->hwaccm.s.vmx.pScratchPhys = RTR0MemObjGetPagePhysAddr(pVM->hwaccm.s.vmx.pMemObjScratch, 0);
208
209 ASMMemZero32(pVM->hwaccm.s.vmx.pScratch, PAGE_SIZE);
210 strcpy((char *)pVM->hwaccm.s.vmx.pScratch, "SCRATCH Magic");
211 *(uint64_t *)(pVM->hwaccm.s.vmx.pScratch + 16) = UINT64_C(0xDEADBEEFDEADBEEF);
212 }
213#endif
214
215 /* Allocate VMCBs for all guest CPUs. */
216 for (VMCPUID i = 0; i < pVM->cCpus; i++)
217 {
218 PVMCPU pVCpu = &pVM->aCpus[i];
219
220 pVCpu->hwaccm.s.vmx.hMemObjVMCS = NIL_RTR0MEMOBJ;
221
222 /* Allocate one page for the VM control structure (VMCS). */
223 rc = RTR0MemObjAllocCont(&pVCpu->hwaccm.s.vmx.hMemObjVMCS, PAGE_SIZE, true /* executable R0 mapping */);
224 AssertRC(rc);
225 if (RT_FAILURE(rc))
226 return rc;
227
228 pVCpu->hwaccm.s.vmx.pvVMCS = RTR0MemObjAddress(pVCpu->hwaccm.s.vmx.hMemObjVMCS);
229 pVCpu->hwaccm.s.vmx.HCPhysVMCS = RTR0MemObjGetPagePhysAddr(pVCpu->hwaccm.s.vmx.hMemObjVMCS, 0);
230 ASMMemZeroPage(pVCpu->hwaccm.s.vmx.pvVMCS);
231
232 pVCpu->hwaccm.s.vmx.cr0_mask = 0;
233 pVCpu->hwaccm.s.vmx.cr4_mask = 0;
234
235 /* Allocate one page for the virtual APIC page for TPR caching. */
236 rc = RTR0MemObjAllocCont(&pVCpu->hwaccm.s.vmx.hMemObjVAPIC, PAGE_SIZE, true /* executable R0 mapping */);
237 AssertRC(rc);
238 if (RT_FAILURE(rc))
239 return rc;
240
241 pVCpu->hwaccm.s.vmx.pbVAPIC = (uint8_t *)RTR0MemObjAddress(pVCpu->hwaccm.s.vmx.hMemObjVAPIC);
242 pVCpu->hwaccm.s.vmx.HCPhysVAPIC = RTR0MemObjGetPagePhysAddr(pVCpu->hwaccm.s.vmx.hMemObjVAPIC, 0);
243 ASMMemZeroPage(pVCpu->hwaccm.s.vmx.pbVAPIC);
244
245 /* Allocate the MSR bitmap if this feature is supported. */
246 if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
247 {
248 rc = RTR0MemObjAllocCont(&pVCpu->hwaccm.s.vmx.pMemObjMSRBitmap, PAGE_SIZE, true /* executable R0 mapping */);
249 AssertRC(rc);
250 if (RT_FAILURE(rc))
251 return rc;
252
253 pVCpu->hwaccm.s.vmx.pMSRBitmap = (uint8_t *)RTR0MemObjAddress(pVCpu->hwaccm.s.vmx.pMemObjMSRBitmap);
254 pVCpu->hwaccm.s.vmx.pMSRBitmapPhys = RTR0MemObjGetPagePhysAddr(pVCpu->hwaccm.s.vmx.pMemObjMSRBitmap, 0);
255 memset(pVCpu->hwaccm.s.vmx.pMSRBitmap, 0xff, PAGE_SIZE);
256 }
257
258#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
259 /* Allocate one page for the guest MSR load area (for preloading guest MSRs during the world switch). */
260 rc = RTR0MemObjAllocCont(&pVCpu->hwaccm.s.vmx.pMemObjGuestMSR, PAGE_SIZE, true /* executable R0 mapping */);
261 AssertRC(rc);
262 if (RT_FAILURE(rc))
263 return rc;
264
265 pVCpu->hwaccm.s.vmx.pGuestMSR = (uint8_t *)RTR0MemObjAddress(pVCpu->hwaccm.s.vmx.pMemObjGuestMSR);
266 pVCpu->hwaccm.s.vmx.pGuestMSRPhys = RTR0MemObjGetPagePhysAddr(pVCpu->hwaccm.s.vmx.pMemObjGuestMSR, 0);
267 memset(pVCpu->hwaccm.s.vmx.pGuestMSR, 0, PAGE_SIZE);
268
269 /* Allocate one page for the host MSR load area (for restoring host MSRs after the world switch back). */
270 rc = RTR0MemObjAllocCont(&pVCpu->hwaccm.s.vmx.pMemObjHostMSR, PAGE_SIZE, true /* executable R0 mapping */);
271 AssertRC(rc);
272 if (RT_FAILURE(rc))
273 return rc;
274
275 pVCpu->hwaccm.s.vmx.pHostMSR = (uint8_t *)RTR0MemObjAddress(pVCpu->hwaccm.s.vmx.pMemObjHostMSR);
276 pVCpu->hwaccm.s.vmx.pHostMSRPhys = RTR0MemObjGetPagePhysAddr(pVCpu->hwaccm.s.vmx.pMemObjHostMSR, 0);
277 memset(pVCpu->hwaccm.s.vmx.pHostMSR, 0, PAGE_SIZE);
278#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
279
280 /* Current guest paging mode. */
281 pVCpu->hwaccm.s.vmx.enmLastSeenGuestMode = PGMMODE_REAL;
282
283#ifdef LOG_ENABLED
284 SUPR0Printf("VMXR0InitVM %x VMCS=%x (%x)\n", pVM, pVCpu->hwaccm.s.vmx.pvVMCS, (uint32_t)pVCpu->hwaccm.s.vmx.HCPhysVMCS);
285#endif
286 }
287
288 return VINF_SUCCESS;
289}
290
291/**
292 * Does Ring-0 per VM VT-x termination.
293 *
294 * @returns VBox status code.
295 * @param pVM The VM to operate on.
296 */
297VMMR0DECL(int) VMXR0TermVM(PVM pVM)
298{
299 for (VMCPUID i = 0; i < pVM->cCpus; i++)
300 {
301 PVMCPU pVCpu = &pVM->aCpus[i];
302
303 if (pVCpu->hwaccm.s.vmx.hMemObjVMCS != NIL_RTR0MEMOBJ)
304 {
305 RTR0MemObjFree(pVCpu->hwaccm.s.vmx.hMemObjVMCS, false);
306 pVCpu->hwaccm.s.vmx.hMemObjVMCS = NIL_RTR0MEMOBJ;
307 pVCpu->hwaccm.s.vmx.pvVMCS = 0;
308 pVCpu->hwaccm.s.vmx.HCPhysVMCS = 0;
309 }
310 if (pVCpu->hwaccm.s.vmx.hMemObjVAPIC != NIL_RTR0MEMOBJ)
311 {
312 RTR0MemObjFree(pVCpu->hwaccm.s.vmx.hMemObjVAPIC, false);
313 pVCpu->hwaccm.s.vmx.hMemObjVAPIC = NIL_RTR0MEMOBJ;
314 pVCpu->hwaccm.s.vmx.pbVAPIC = 0;
315 pVCpu->hwaccm.s.vmx.HCPhysVAPIC = 0;
316 }
317 if (pVCpu->hwaccm.s.vmx.pMemObjMSRBitmap != NIL_RTR0MEMOBJ)
318 {
319 RTR0MemObjFree(pVCpu->hwaccm.s.vmx.pMemObjMSRBitmap, false);
320 pVCpu->hwaccm.s.vmx.pMemObjMSRBitmap = NIL_RTR0MEMOBJ;
321 pVCpu->hwaccm.s.vmx.pMSRBitmap = 0;
322 pVCpu->hwaccm.s.vmx.pMSRBitmapPhys = 0;
323 }
324#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
325 if (pVCpu->hwaccm.s.vmx.pMemObjHostMSR != NIL_RTR0MEMOBJ)
326 {
327 RTR0MemObjFree(pVCpu->hwaccm.s.vmx.pMemObjHostMSR, false);
328 pVCpu->hwaccm.s.vmx.pMemObjHostMSR = NIL_RTR0MEMOBJ;
329 pVCpu->hwaccm.s.vmx.pHostMSR = 0;
330 pVCpu->hwaccm.s.vmx.pHostMSRPhys = 0;
331 }
332 if (pVCpu->hwaccm.s.vmx.pMemObjGuestMSR != NIL_RTR0MEMOBJ)
333 {
334 RTR0MemObjFree(pVCpu->hwaccm.s.vmx.pMemObjGuestMSR, false);
335 pVCpu->hwaccm.s.vmx.pMemObjGuestMSR = NIL_RTR0MEMOBJ;
336 pVCpu->hwaccm.s.vmx.pGuestMSR = 0;
337 pVCpu->hwaccm.s.vmx.pGuestMSRPhys = 0;
338 }
339#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
340 }
341 if (pVM->hwaccm.s.vmx.pMemObjAPIC != NIL_RTR0MEMOBJ)
342 {
343 RTR0MemObjFree(pVM->hwaccm.s.vmx.pMemObjAPIC, false);
344 pVM->hwaccm.s.vmx.pMemObjAPIC = NIL_RTR0MEMOBJ;
345 pVM->hwaccm.s.vmx.pAPIC = 0;
346 pVM->hwaccm.s.vmx.pAPICPhys = 0;
347 }
348#ifdef VBOX_WITH_CRASHDUMP_MAGIC
349 if (pVM->hwaccm.s.vmx.pMemObjScratch != NIL_RTR0MEMOBJ)
350 {
351 ASMMemZero32(pVM->hwaccm.s.vmx.pScratch, PAGE_SIZE);
352 RTR0MemObjFree(pVM->hwaccm.s.vmx.pMemObjScratch, false);
353 pVM->hwaccm.s.vmx.pMemObjScratch = NIL_RTR0MEMOBJ;
354 pVM->hwaccm.s.vmx.pScratch = 0;
355 pVM->hwaccm.s.vmx.pScratchPhys = 0;
356 }
357#endif
358 return VINF_SUCCESS;
359}
360
361/**
362 * Sets up VT-x for the specified VM
363 *
364 * @returns VBox status code.
365 * @param pVM The VM to operate on.
366 */
367VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
368{
369 int rc = VINF_SUCCESS;
370 uint32_t val;
371
372 AssertReturn(pVM, VERR_INVALID_PARAMETER);
373
374 for (VMCPUID i = 0; i < pVM->cCpus; i++)
375 {
376 PVMCPU pVCpu = &pVM->aCpus[i];
377
378 AssertPtr(pVCpu->hwaccm.s.vmx.pvVMCS);
379
380 /* Set revision dword at the beginning of the VMCS structure. */
381 *(uint32_t *)pVCpu->hwaccm.s.vmx.pvVMCS = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hwaccm.s.vmx.msr.vmx_basic_info);
382
383 /* Clear VM Control Structure. */
384 Log(("HCPhysVMCS = %RHp\n", pVCpu->hwaccm.s.vmx.HCPhysVMCS));
385 rc = VMXClearVMCS(pVCpu->hwaccm.s.vmx.HCPhysVMCS);
386 if (RT_FAILURE(rc))
387 goto vmx_end;
388
389 /* Activate the VM Control Structure. */
390 rc = VMXActivateVMCS(pVCpu->hwaccm.s.vmx.HCPhysVMCS);
391 if (RT_FAILURE(rc))
392 goto vmx_end;
393
394 /* VMX_VMCS_CTRL_PIN_EXEC_CONTROLS
395 * Set required bits to one and zero according to the MSR capabilities.
396 */
397 val = pVM->hwaccm.s.vmx.msr.vmx_pin_ctls.n.disallowed0;
398 /* External and non-maskable interrupts cause VM-exits. */
399 val |= VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_EXT_INT_EXIT | VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_NMI_EXIT;
400 /* enable the preemption timer. */
401 if (pVM->hwaccm.s.vmx.fUsePreemptTimer)
402 val |= VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_PREEMPT_TIMER;
403 val &= pVM->hwaccm.s.vmx.msr.vmx_pin_ctls.n.allowed1;
404
405 rc = VMXWriteVMCS(VMX_VMCS_CTRL_PIN_EXEC_CONTROLS, val);
406 AssertRC(rc);
407
408 /* VMX_VMCS_CTRL_PROC_EXEC_CONTROLS
409 * Set required bits to one and zero according to the MSR capabilities.
410 */
411 val = pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.disallowed0;
412 /* Program which event cause VM-exits and which features we want to use. */
413 val = val | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_HLT_EXIT
414 | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_TSC_OFFSET
415 | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT
416 | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_UNCOND_IO_EXIT
417 | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDPMC_EXIT
418 | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_EXIT
419 | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MWAIT_EXIT; /* don't execute mwait or else we'll idle inside the guest (host thinks the cpu load is high) */
420
421 /* Without nested paging we should intercept invlpg and cr3 mov instructions. */
422 if (!pVM->hwaccm.s.fNestedPaging)
423 val |= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INVLPG_EXIT
424 | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
425 | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT;
426
427 /* Note: VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MWAIT_EXIT might cause a vmlaunch failure with an invalid control fields error. (combined with some other exit reasons) */
428 if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
429 {
430 /* CR8 reads from the APIC shadow page; writes cause an exit is they lower the TPR below the threshold */
431 val |= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW;
432 Assert(pVM->hwaccm.s.vmx.pAPIC);
433 }
434 else
435 /* Exit on CR8 reads & writes in case the TPR shadow feature isn't present. */
436 val |= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR8_STORE_EXIT | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR8_LOAD_EXIT;
437
438 if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
439 {
440 Assert(pVCpu->hwaccm.s.vmx.pMSRBitmapPhys);
441 val |= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS;
442 }
443
444 /* We will use the secondary control if it's present. */
445 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
446
447 /* Mask away the bits that the CPU doesn't support */
448 /** @todo make sure they don't conflict with the above requirements. */
449 val &= pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1;
450 pVCpu->hwaccm.s.vmx.proc_ctls = val;
451
452 rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, val);
453 AssertRC(rc);
454
455 if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
456 {
457 /* VMX_VMCS_CTRL_PROC_EXEC_CONTROLS2
458 * Set required bits to one and zero according to the MSR capabilities.
459 */
460 val = pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2.n.disallowed0;
461 val |= VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT;
462
463#ifdef HWACCM_VTX_WITH_EPT
464 if (pVM->hwaccm.s.fNestedPaging)
465 val |= VMX_VMCS_CTRL_PROC_EXEC2_EPT;
466#endif /* HWACCM_VTX_WITH_EPT */
467#ifdef HWACCM_VTX_WITH_VPID
468 else
469 if (pVM->hwaccm.s.vmx.fVPID)
470 val |= VMX_VMCS_CTRL_PROC_EXEC2_VPID;
471#endif /* HWACCM_VTX_WITH_VPID */
472
473 if (pVM->hwaccm.s.fHasIoApic)
474 val |= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC;
475
476 if (pVM->hwaccm.s.vmx.fUnrestrictedGuest)
477 val |= VMX_VMCS_CTRL_PROC_EXEC2_REAL_MODE;
478
479 /* Mask away the bits that the CPU doesn't support */
480 /** @todo make sure they don't conflict with the above requirements. */
481 val &= pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2.n.allowed1;
482 pVCpu->hwaccm.s.vmx.proc_ctls2 = val;
483 rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS2, val);
484 AssertRC(rc);
485 }
486
487 /* VMX_VMCS_CTRL_CR3_TARGET_COUNT
488 * Set required bits to one and zero according to the MSR capabilities.
489 */
490 rc = VMXWriteVMCS(VMX_VMCS_CTRL_CR3_TARGET_COUNT, 0);
491 AssertRC(rc);
492
493 /* Forward all exception except #NM & #PF to the guest.
494 * We always need to check pagefaults since our shadow page table can be out of sync.
495 * And we always lazily sync the FPU & XMM state.
496 */
497
498 /** @todo Possible optimization:
499 * Keep the FPU and XMM state current in the EM thread. That way there's no need to
500 * lazily sync anything, but the downside is that we can't use the FPU stack or XMM
501 * registers ourselves of course.
502 *
503 * Note: only possible if the current state is actually ours (X86_CR0_TS flag)
504 */
505
506 /* Don't filter page faults; all of them should cause a switch. */
507 rc = VMXWriteVMCS(VMX_VMCS_CTRL_PAGEFAULT_ERROR_MASK, 0);
508 rc |= VMXWriteVMCS(VMX_VMCS_CTRL_PAGEFAULT_ERROR_MATCH, 0);
509 AssertRC(rc);
510
511 /* Init TSC offset to zero. */
512 rc = VMXWriteVMCS64(VMX_VMCS_CTRL_TSC_OFFSET_FULL, 0);
513 AssertRC(rc);
514
515 rc = VMXWriteVMCS64(VMX_VMCS_CTRL_IO_BITMAP_A_FULL, 0);
516 AssertRC(rc);
517
518 rc = VMXWriteVMCS64(VMX_VMCS_CTRL_IO_BITMAP_B_FULL, 0);
519 AssertRC(rc);
520
521 /* Set the MSR bitmap address. */
522 if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
523 {
524 Assert(pVCpu->hwaccm.s.vmx.pMSRBitmapPhys);
525
526 rc = VMXWriteVMCS64(VMX_VMCS_CTRL_MSR_BITMAP_FULL, pVCpu->hwaccm.s.vmx.pMSRBitmapPhys);
527 AssertRC(rc);
528
529 /* Allow the guest to directly modify these MSRs; they are restored and saved automatically. */
530 hmR0VmxSetMSRPermission(pVCpu, MSR_IA32_SYSENTER_CS, true, true);
531 hmR0VmxSetMSRPermission(pVCpu, MSR_IA32_SYSENTER_ESP, true, true);
532 hmR0VmxSetMSRPermission(pVCpu, MSR_IA32_SYSENTER_EIP, true, true);
533 hmR0VmxSetMSRPermission(pVCpu, MSR_K8_LSTAR, true, true);
534 hmR0VmxSetMSRPermission(pVCpu, MSR_K6_STAR, true, true);
535 hmR0VmxSetMSRPermission(pVCpu, MSR_K8_SF_MASK, true, true);
536 hmR0VmxSetMSRPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, true, true);
537 hmR0VmxSetMSRPermission(pVCpu, MSR_K8_GS_BASE, true, true);
538 hmR0VmxSetMSRPermission(pVCpu, MSR_K8_FS_BASE, true, true);
539 }
540
541#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
542 /* Set the guest & host MSR load/store physical addresses. */
543 Assert(pVCpu->hwaccm.s.vmx.pGuestMSRPhys);
544 rc = VMXWriteVMCS64(VMX_VMCS_CTRL_VMENTRY_MSR_LOAD_FULL, pVCpu->hwaccm.s.vmx.pGuestMSRPhys);
545 AssertRC(rc);
546 rc = VMXWriteVMCS64(VMX_VMCS_CTRL_VMEXIT_MSR_STORE_FULL, pVCpu->hwaccm.s.vmx.pGuestMSRPhys);
547 AssertRC(rc);
548
549 Assert(pVCpu->hwaccm.s.vmx.pHostMSRPhys);
550 rc = VMXWriteVMCS64(VMX_VMCS_CTRL_VMEXIT_MSR_LOAD_FULL, pVCpu->hwaccm.s.vmx.pHostMSRPhys);
551 AssertRC(rc);
552#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
553
554 rc = VMXWriteVMCS(VMX_VMCS_CTRL_ENTRY_MSR_LOAD_COUNT, 0);
555 AssertRC(rc);
556
557 rc = VMXWriteVMCS(VMX_VMCS_CTRL_EXIT_MSR_STORE_COUNT, 0);
558 AssertRC(rc);
559
560 if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
561 {
562 Assert(pVM->hwaccm.s.vmx.pMemObjAPIC);
563 /* Optional */
564 rc = VMXWriteVMCS(VMX_VMCS_CTRL_TPR_THRESHOLD, 0);
565 rc |= VMXWriteVMCS64(VMX_VMCS_CTRL_VAPIC_PAGEADDR_FULL, pVCpu->hwaccm.s.vmx.HCPhysVAPIC);
566
567 if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
568 rc |= VMXWriteVMCS64(VMX_VMCS_CTRL_APIC_ACCESSADDR_FULL, pVM->hwaccm.s.vmx.pAPICPhys);
569
570 AssertRC(rc);
571 }
572
573 /* Set link pointer to -1. Not currently used. */
574 rc = VMXWriteVMCS64(VMX_VMCS_GUEST_LINK_PTR_FULL, 0xFFFFFFFFFFFFFFFFULL);
575 AssertRC(rc);
576
577 /* Clear VM Control Structure. Marking it inactive, clearing implementation specific data and writing back VMCS data to memory. */
578 rc = VMXClearVMCS(pVCpu->hwaccm.s.vmx.HCPhysVMCS);
579 AssertRC(rc);
580
581 /* Configure the VMCS read cache. */
582 PVMCSCACHE pCache = &pVCpu->hwaccm.s.vmx.VMCSCache;
583
584 VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_RIP);
585 VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_RSP);
586 VMXSetupCachedReadVMCS(pCache, VMX_VMCS_GUEST_RFLAGS);
587 VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE);
588 VMXSetupCachedReadVMCS(pCache, VMX_VMCS_CTRL_CR0_READ_SHADOW);
589 VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_CR0);
590 VMXSetupCachedReadVMCS(pCache, VMX_VMCS_CTRL_CR4_READ_SHADOW);
591 VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_CR4);
592 VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_DR7);
593 VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_GUEST_SYSENTER_CS);
594 VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_SYSENTER_EIP);
595 VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_SYSENTER_ESP);
596 VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_GUEST_GDTR_LIMIT);
597 VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_GDTR_BASE);
598 VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_GUEST_IDTR_LIMIT);
599 VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_IDTR_BASE);
600
601 VMX_SETUP_SELREG(ES, pCache);
602 VMX_SETUP_SELREG(SS, pCache);
603 VMX_SETUP_SELREG(CS, pCache);
604 VMX_SETUP_SELREG(DS, pCache);
605 VMX_SETUP_SELREG(FS, pCache);
606 VMX_SETUP_SELREG(GS, pCache);
607 VMX_SETUP_SELREG(LDTR, pCache);
608 VMX_SETUP_SELREG(TR, pCache);
609
610 /* Status code VMCS reads. */
611 VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_EXIT_REASON);
612 VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_VM_INSTR_ERROR);
613 VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_EXIT_INSTR_LENGTH);
614 VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_EXIT_INTERRUPTION_ERRCODE);
615 VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO);
616 VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_EXIT_INSTR_INFO);
617 VMXSetupCachedReadVMCS(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
618 VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_IDT_INFO);
619 VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_IDT_ERRCODE);
620
621 if (pVM->hwaccm.s.fNestedPaging)
622 {
623 VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_CR3);
624 VMXSetupCachedReadVMCS(pCache, VMX_VMCS_EXIT_PHYS_ADDR_FULL);
625 pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
626 }
627 else
628 pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
629 } /* for each VMCPU */
630
631 /* Choose the right TLB setup function. */
632 if (pVM->hwaccm.s.fNestedPaging)
633 {
634 pVM->hwaccm.s.vmx.pfnSetupTaggedTLB = hmR0VmxSetupTLBEPT;
635
636 /* Default values for flushing. */
637 pVM->hwaccm.s.vmx.enmFlushPage = VMX_FLUSH_ALL_CONTEXTS;
638 pVM->hwaccm.s.vmx.enmFlushContext = VMX_FLUSH_ALL_CONTEXTS;
639
640 /* If the capabilities specify we can do more, then make use of it. */
641 if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVEPT_CAPS_INDIV)
642 pVM->hwaccm.s.vmx.enmFlushPage = VMX_FLUSH_PAGE;
643 else
644 if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVEPT_CAPS_CONTEXT)
645 pVM->hwaccm.s.vmx.enmFlushPage = VMX_FLUSH_SINGLE_CONTEXT;
646
647 if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVEPT_CAPS_CONTEXT)
648 pVM->hwaccm.s.vmx.enmFlushContext = VMX_FLUSH_SINGLE_CONTEXT;
649 }
650#ifdef HWACCM_VTX_WITH_VPID
651 else
652 if (pVM->hwaccm.s.vmx.fVPID)
653 {
654 pVM->hwaccm.s.vmx.pfnSetupTaggedTLB = hmR0VmxSetupTLBVPID;
655
656 /* Default values for flushing. */
657 pVM->hwaccm.s.vmx.enmFlushPage = VMX_FLUSH_ALL_CONTEXTS;
658 pVM->hwaccm.s.vmx.enmFlushContext = VMX_FLUSH_ALL_CONTEXTS;
659
660 /* If the capabilities specify we can do more, then make use of it. */
661 if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVVPID_CAPS_INDIV)
662 pVM->hwaccm.s.vmx.enmFlushPage = VMX_FLUSH_PAGE;
663 else
664 if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVVPID_CAPS_CONTEXT)
665 pVM->hwaccm.s.vmx.enmFlushPage = VMX_FLUSH_SINGLE_CONTEXT;
666
667 if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVVPID_CAPS_CONTEXT)
668 pVM->hwaccm.s.vmx.enmFlushContext = VMX_FLUSH_SINGLE_CONTEXT;
669 }
670#endif /* HWACCM_VTX_WITH_VPID */
671 else
672 pVM->hwaccm.s.vmx.pfnSetupTaggedTLB = hmR0VmxSetupTLBDummy;
673
674vmx_end:
675 hmR0VmxCheckError(pVM, &pVM->aCpus[0], rc);
676 return rc;
677}
678
679/**
680 * Sets the permission bits for the specified MSR
681 *
682 * @param pVCpu The VMCPU to operate on.
683 * @param ulMSR MSR value
684 * @param fRead Reading allowed/disallowed
685 * @param fWrite Writing allowed/disallowed
686 */
687static void hmR0VmxSetMSRPermission(PVMCPU pVCpu, unsigned ulMSR, bool fRead, bool fWrite)
688{
689 unsigned ulBit;
690 uint8_t *pMSRBitmap = (uint8_t *)pVCpu->hwaccm.s.vmx.pMSRBitmap;
691
692 /* Layout:
693 * 0x000 - 0x3ff - Low MSR read bits
694 * 0x400 - 0x7ff - High MSR read bits
695 * 0x800 - 0xbff - Low MSR write bits
696 * 0xc00 - 0xfff - High MSR write bits
697 */
698 if (ulMSR <= 0x00001FFF)
699 {
700 /* Pentium-compatible MSRs */
701 ulBit = ulMSR;
702 }
703 else
704 if ( ulMSR >= 0xC0000000
705 && ulMSR <= 0xC0001FFF)
706 {
707 /* AMD Sixth Generation x86 Processor MSRs */
708 ulBit = (ulMSR - 0xC0000000);
709 pMSRBitmap += 0x400;
710 }
711 else
712 {
713 AssertFailed();
714 return;
715 }
716
717 Assert(ulBit <= 0x1fff);
718 if (fRead)
719 ASMBitClear(pMSRBitmap, ulBit);
720 else
721 ASMBitSet(pMSRBitmap, ulBit);
722
723 if (fWrite)
724 ASMBitClear(pMSRBitmap + 0x800, ulBit);
725 else
726 ASMBitSet(pMSRBitmap + 0x800, ulBit);
727}
728
729
730/**
731 * Injects an event (trap or external interrupt)
732 *
733 * @returns VBox status code. Note that it may return VINF_EM_RESET to
734 * indicate a triple fault when injecting X86_XCPT_DF.
735 *
736 * @param pVM The VM to operate on.
737 * @param pVCpu The VMCPU to operate on.
738 * @param pCtx CPU Context
739 * @param intInfo VMX interrupt info
740 * @param cbInstr Opcode length of faulting instruction
741 * @param errCode Error code (optional)
742 */
743static int hmR0VmxInjectEvent(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, uint32_t intInfo, uint32_t cbInstr, uint32_t errCode)
744{
745 int rc;
746 uint32_t iGate = VMX_EXIT_INTERRUPTION_INFO_VECTOR(intInfo);
747
748#ifdef VBOX_WITH_STATISTICS
749 STAM_COUNTER_INC(&pVCpu->hwaccm.s.paStatInjectedIrqsR0[iGate & MASK_INJECT_IRQ_STAT]);
750#endif
751
752#ifdef VBOX_STRICT
753 if (iGate == 0xE)
754 LogFlow(("hmR0VmxInjectEvent: Injecting interrupt %d at %RGv error code=%08x CR2=%RGv intInfo=%08x\n", iGate, (RTGCPTR)pCtx->rip, errCode, pCtx->cr2, intInfo));
755 else
756 if (iGate < 0x20)
757 LogFlow(("hmR0VmxInjectEvent: Injecting interrupt %d at %RGv error code=%08x\n", iGate, (RTGCPTR)pCtx->rip, errCode));
758 else
759 {
760 LogFlow(("INJ-EI: %x at %RGv\n", iGate, (RTGCPTR)pCtx->rip));
761 Assert( VMX_EXIT_INTERRUPTION_INFO_TYPE(intInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW
762 || !VMCPU_FF_ISSET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
763 Assert( VMX_EXIT_INTERRUPTION_INFO_TYPE(intInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW
764 || pCtx->eflags.u32 & X86_EFL_IF);
765 }
766#endif
767
768 if ( CPUMIsGuestInRealModeEx(pCtx)
769 && pVM->hwaccm.s.vmx.pRealModeTSS)
770 {
771 RTGCPHYS GCPhysHandler;
772 uint16_t offset, ip;
773 RTSEL sel;
774
775 /* Injecting events doesn't work right with real mode emulation.
776 * (#GP if we try to inject external hardware interrupts)
777 * Inject the interrupt or trap directly instead.
778 *
779 * ASSUMES no access handlers for the bits we read or write below (should be safe).
780 */
781 Log(("Manual interrupt/trap '%x' inject (real mode)\n", iGate));
782
783 /* Check if the interrupt handler is present. */
784 if (iGate * 4 + 3 > pCtx->idtr.cbIdt)
785 {
786 Log(("IDT cbIdt violation\n"));
787 if (iGate != X86_XCPT_DF)
788 {
789 uint32_t intInfo2;
790
791 intInfo2 = (iGate == X86_XCPT_GP) ? (uint32_t)X86_XCPT_DF : iGate;
792 intInfo2 |= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
793 intInfo2 |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
794 intInfo2 |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HWEXCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
795
796 return hmR0VmxInjectEvent(pVM, pVCpu, pCtx, intInfo2, 0, 0 /* no error code according to the Intel docs */);
797 }
798 Log(("Triple fault -> reset the VM!\n"));
799 return VINF_EM_RESET;
800 }
801 if ( VMX_EXIT_INTERRUPTION_INFO_TYPE(intInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW
802 || iGate == 3 /* Both #BP and #OF point to the instruction after. */
803 || iGate == 4)
804 {
805 ip = pCtx->ip + cbInstr;
806 }
807 else
808 ip = pCtx->ip;
809
810 /* Read the selector:offset pair of the interrupt handler. */
811 GCPhysHandler = (RTGCPHYS)pCtx->idtr.pIdt + iGate * 4;
812 rc = PGMPhysSimpleReadGCPhys(pVM, &offset, GCPhysHandler, sizeof(offset)); AssertRC(rc);
813 rc = PGMPhysSimpleReadGCPhys(pVM, &sel, GCPhysHandler + 2, sizeof(sel)); AssertRC(rc);
814
815 LogFlow(("IDT handler %04X:%04X\n", sel, offset));
816
817 /* Construct the stack frame. */
818 /** @todo should check stack limit. */
819 pCtx->sp -= 2;
820 LogFlow(("ss:sp %04X:%04X eflags=%x\n", pCtx->ss, pCtx->sp, pCtx->eflags.u));
821 rc = PGMPhysSimpleWriteGCPhys(pVM, pCtx->ssHid.u64Base + pCtx->sp, &pCtx->eflags, sizeof(uint16_t)); AssertRC(rc);
822 pCtx->sp -= 2;
823 LogFlow(("ss:sp %04X:%04X cs=%x\n", pCtx->ss, pCtx->sp, pCtx->cs));
824 rc = PGMPhysSimpleWriteGCPhys(pVM, pCtx->ssHid.u64Base + pCtx->sp, &pCtx->cs, sizeof(uint16_t)); AssertRC(rc);
825 pCtx->sp -= 2;
826 LogFlow(("ss:sp %04X:%04X ip=%x\n", pCtx->ss, pCtx->sp, ip));
827 rc = PGMPhysSimpleWriteGCPhys(pVM, pCtx->ssHid.u64Base + pCtx->sp, &ip, sizeof(ip)); AssertRC(rc);
828
829 /* Update the CPU state for executing the handler. */
830 pCtx->rip = offset;
831 pCtx->cs = sel;
832 pCtx->csHid.u64Base = sel << 4;
833 pCtx->eflags.u &= ~(X86_EFL_IF|X86_EFL_TF|X86_EFL_RF|X86_EFL_AC);
834
835 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_GUEST_SEGMENT_REGS;
836 return VINF_SUCCESS;
837 }
838
839 /* Set event injection state. */
840 rc = VMXWriteVMCS(VMX_VMCS_CTRL_ENTRY_IRQ_INFO, intInfo | (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT));
841
842 rc |= VMXWriteVMCS(VMX_VMCS_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
843 rc |= VMXWriteVMCS(VMX_VMCS_CTRL_ENTRY_EXCEPTION_ERRCODE, errCode);
844
845 AssertRC(rc);
846 return rc;
847}
848
849
850/**
851 * Checks for pending guest interrupts and injects them
852 *
853 * @returns VBox status code.
854 * @param pVM The VM to operate on.
855 * @param pVCpu The VMCPU to operate on.
856 * @param pCtx CPU Context
857 */
858static int hmR0VmxCheckPendingInterrupt(PVM pVM, PVMCPU pVCpu, CPUMCTX *pCtx)
859{
860 int rc;
861
862 /* Dispatch any pending interrupts. (injected before, but a VM exit occurred prematurely) */
863 if (pVCpu->hwaccm.s.Event.fPending)
864 {
865 Log(("CPU%d: Reinjecting event %RX64 %08x at %RGv cr2=%RX64\n", pVCpu->idCpu, pVCpu->hwaccm.s.Event.intInfo, pVCpu->hwaccm.s.Event.errCode, (RTGCPTR)pCtx->rip, pCtx->cr2));
866 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatIntReinject);
867 rc = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, pVCpu->hwaccm.s.Event.intInfo, 0, pVCpu->hwaccm.s.Event.errCode);
868 AssertRC(rc);
869
870 pVCpu->hwaccm.s.Event.fPending = false;
871 return VINF_SUCCESS;
872 }
873
874 /* If an active trap is already pending, then we must forward it first! */
875 if (!TRPMHasTrap(pVCpu))
876 {
877 if (VMCPU_FF_TESTANDCLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI))
878 {
879 RTGCUINTPTR intInfo;
880
881 Log(("CPU%d: injecting #NMI\n", pVCpu->idCpu));
882
883 intInfo = X86_XCPT_NMI;
884 intInfo |= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
885 intInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
886
887 rc = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, intInfo, 0, 0);
888 AssertRC(rc);
889
890 return VINF_SUCCESS;
891 }
892
893 /* @todo SMI interrupts. */
894
895 /* When external interrupts are pending, we should exit the VM when IF is set. */
896 if (VMCPU_FF_ISPENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC|VMCPU_FF_INTERRUPT_PIC)))
897 {
898 if (!(pCtx->eflags.u32 & X86_EFL_IF))
899 {
900 if (!(pVCpu->hwaccm.s.vmx.proc_ctls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_IRQ_WINDOW_EXIT))
901 {
902 LogFlow(("Enable irq window exit!\n"));
903 pVCpu->hwaccm.s.vmx.proc_ctls |= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_IRQ_WINDOW_EXIT;
904 rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
905 AssertRC(rc);
906 }
907 /* else nothing to do but wait */
908 }
909 else
910 if (!VMCPU_FF_ISSET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
911 {
912 uint8_t u8Interrupt;
913
914 rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
915 Log(("CPU%d: Dispatch interrupt: u8Interrupt=%x (%d) rc=%Rrc cs:rip=%04X:%RGv\n", pVCpu->idCpu, u8Interrupt, u8Interrupt, rc, pCtx->cs, (RTGCPTR)pCtx->rip));
916 if (RT_SUCCESS(rc))
917 {
918 rc = TRPMAssertTrap(pVCpu, u8Interrupt, TRPM_HARDWARE_INT);
919 AssertRC(rc);
920 }
921 else
922 {
923 /* Can only happen in rare cases where a pending interrupt is cleared behind our back */
924 Assert(!VMCPU_FF_ISPENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC|VMCPU_FF_INTERRUPT_PIC)));
925 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatSwitchGuestIrq);
926 /* Just continue */
927 }
928 }
929 else
930 Log(("Pending interrupt blocked at %RGv by VM_FF_INHIBIT_INTERRUPTS!!\n", (RTGCPTR)pCtx->rip));
931 }
932 }
933
934#ifdef VBOX_STRICT
935 if (TRPMHasTrap(pVCpu))
936 {
937 uint8_t u8Vector;
938 rc = TRPMQueryTrapAll(pVCpu, &u8Vector, 0, 0, 0);
939 AssertRC(rc);
940 }
941#endif
942
943 if ( (pCtx->eflags.u32 & X86_EFL_IF)
944 && (!VMCPU_FF_ISSET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
945 && TRPMHasTrap(pVCpu)
946 )
947 {
948 uint8_t u8Vector;
949 TRPMEVENT enmType;
950 RTGCUINTPTR intInfo;
951 RTGCUINT errCode;
952
953 /* If a new event is pending, then dispatch it now. */
954 rc = TRPMQueryTrapAll(pVCpu, &u8Vector, &enmType, &errCode, 0);
955 AssertRC(rc);
956 Assert(pCtx->eflags.Bits.u1IF == 1 || enmType == TRPM_TRAP);
957 Assert(enmType != TRPM_SOFTWARE_INT);
958
959 /* Clear the pending trap. */
960 rc = TRPMResetTrap(pVCpu);
961 AssertRC(rc);
962
963 intInfo = u8Vector;
964 intInfo |= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
965
966 if (enmType == TRPM_TRAP)
967 {
968 switch (u8Vector) {
969 case X86_XCPT_DF:
970 case X86_XCPT_TS:
971 case X86_XCPT_NP:
972 case X86_XCPT_SS:
973 case X86_XCPT_GP:
974 case X86_XCPT_PF:
975 case X86_XCPT_AC:
976 /* Valid error codes. */
977 intInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
978 break;
979 default:
980 break;
981 }
982 if (u8Vector == X86_XCPT_BP || u8Vector == X86_XCPT_OF)
983 intInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SWEXCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
984 else
985 intInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HWEXCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
986 }
987 else
988 intInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
989
990 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatIntInject);
991 rc = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, intInfo, 0, errCode);
992 AssertRC(rc);
993 } /* if (interrupts can be dispatched) */
994
995 return VINF_SUCCESS;
996}
997
998/**
999 * Save the host state
1000 *
1001 * @returns VBox status code.
1002 * @param pVM The VM to operate on.
1003 * @param pVCpu The VMCPU to operate on.
1004 */
1005VMMR0DECL(int) VMXR0SaveHostState(PVM pVM, PVMCPU pVCpu)
1006{
1007 int rc = VINF_SUCCESS;
1008 NOREF(pVM);
1009
1010 /*
1011 * Host CPU Context
1012 */
1013 if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_HOST_CONTEXT)
1014 {
1015 RTIDTR idtr;
1016 RTGDTR gdtr;
1017 RTSEL SelTR;
1018 PCX86DESCHC pDesc;
1019 uintptr_t trBase;
1020 RTSEL cs;
1021 RTSEL ss;
1022 uint64_t cr3;
1023
1024 /* Control registers */
1025 rc = VMXWriteVMCS(VMX_VMCS_HOST_CR0, ASMGetCR0());
1026#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
1027 if (VMX_IS_64BIT_HOST_MODE())
1028 {
1029 cr3 = hwaccmR0Get64bitCR3();
1030 rc |= VMXWriteVMCS64(VMX_VMCS_HOST_CR3, cr3);
1031 }
1032 else
1033#endif
1034 {
1035 cr3 = ASMGetCR3();
1036 rc |= VMXWriteVMCS(VMX_VMCS_HOST_CR3, cr3);
1037 }
1038 rc |= VMXWriteVMCS(VMX_VMCS_HOST_CR4, ASMGetCR4());
1039 AssertRC(rc);
1040 Log2(("VMX_VMCS_HOST_CR0 %08x\n", ASMGetCR0()));
1041 Log2(("VMX_VMCS_HOST_CR3 %08RX64\n", cr3));
1042 Log2(("VMX_VMCS_HOST_CR4 %08x\n", ASMGetCR4()));
1043
1044 /* Selector registers. */
1045#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
1046 if (VMX_IS_64BIT_HOST_MODE())
1047 {
1048 cs = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelCS;
1049 ss = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelSS;
1050 }
1051 else
1052 {
1053 /* sysenter loads LDT cs & ss, VMX doesn't like this. Load the GDT ones (safe). */
1054 cs = (RTSEL)(uintptr_t)&SUPR0AbsKernelCS;
1055 ss = (RTSEL)(uintptr_t)&SUPR0AbsKernelSS;
1056 }
1057#else
1058 cs = ASMGetCS();
1059 ss = ASMGetSS();
1060#endif
1061 Assert(!(cs & X86_SEL_LDT)); Assert((cs & X86_SEL_RPL) == 0);
1062 Assert(!(ss & X86_SEL_LDT)); Assert((ss & X86_SEL_RPL) == 0);
1063 rc = VMXWriteVMCS(VMX_VMCS16_HOST_FIELD_CS, cs);
1064 /* Note: VMX is (again) very picky about the RPL of the selectors here; we'll restore them manually. */
1065 rc |= VMXWriteVMCS(VMX_VMCS16_HOST_FIELD_DS, 0);
1066 rc |= VMXWriteVMCS(VMX_VMCS16_HOST_FIELD_ES, 0);
1067#if HC_ARCH_BITS == 32
1068 if (!VMX_IS_64BIT_HOST_MODE())
1069 {
1070 rc |= VMXWriteVMCS(VMX_VMCS16_HOST_FIELD_FS, 0);
1071 rc |= VMXWriteVMCS(VMX_VMCS16_HOST_FIELD_GS, 0);
1072 }
1073#endif
1074 rc |= VMXWriteVMCS(VMX_VMCS16_HOST_FIELD_SS, ss);
1075 SelTR = ASMGetTR();
1076 rc |= VMXWriteVMCS(VMX_VMCS16_HOST_FIELD_TR, SelTR);
1077 AssertRC(rc);
1078 Log2(("VMX_VMCS_HOST_FIELD_CS %08x (%08x)\n", cs, ASMGetSS()));
1079 Log2(("VMX_VMCS_HOST_FIELD_DS 00000000 (%08x)\n", ASMGetDS()));
1080 Log2(("VMX_VMCS_HOST_FIELD_ES 00000000 (%08x)\n", ASMGetES()));
1081 Log2(("VMX_VMCS_HOST_FIELD_FS 00000000 (%08x)\n", ASMGetFS()));
1082 Log2(("VMX_VMCS_HOST_FIELD_GS 00000000 (%08x)\n", ASMGetGS()));
1083 Log2(("VMX_VMCS_HOST_FIELD_SS %08x (%08x)\n", ss, ASMGetSS()));
1084 Log2(("VMX_VMCS_HOST_FIELD_TR %08x\n", ASMGetTR()));
1085
1086 /* GDTR & IDTR */
1087#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
1088 if (VMX_IS_64BIT_HOST_MODE())
1089 {
1090 X86XDTR64 gdtr64, idtr64;
1091 hwaccmR0Get64bitGDTRandIDTR(&gdtr64, &idtr64);
1092 rc = VMXWriteVMCS64(VMX_VMCS_HOST_GDTR_BASE, gdtr64.uAddr);
1093 rc |= VMXWriteVMCS64(VMX_VMCS_HOST_IDTR_BASE, gdtr64.uAddr);
1094 AssertRC(rc);
1095 Log2(("VMX_VMCS_HOST_GDTR_BASE %RX64\n", gdtr64.uAddr));
1096 Log2(("VMX_VMCS_HOST_IDTR_BASE %RX64\n", idtr64.uAddr));
1097 gdtr.cbGdt = gdtr64.cb;
1098 gdtr.pGdt = (uintptr_t)gdtr64.uAddr;
1099 }
1100 else
1101#endif
1102 {
1103 ASMGetGDTR(&gdtr);
1104 rc = VMXWriteVMCS(VMX_VMCS_HOST_GDTR_BASE, gdtr.pGdt);
1105 ASMGetIDTR(&idtr);
1106 rc |= VMXWriteVMCS(VMX_VMCS_HOST_IDTR_BASE, idtr.pIdt);
1107 AssertRC(rc);
1108 Log2(("VMX_VMCS_HOST_GDTR_BASE %RHv\n", gdtr.pGdt));
1109 Log2(("VMX_VMCS_HOST_IDTR_BASE %RHv\n", idtr.pIdt));
1110 }
1111
1112 /* Save the base address of the TR selector. */
1113 if (SelTR > gdtr.cbGdt)
1114 {
1115 AssertMsgFailed(("Invalid TR selector %x. GDTR.cbGdt=%x\n", SelTR, gdtr.cbGdt));
1116 return VERR_VMX_INVALID_HOST_STATE;
1117 }
1118
1119 pDesc = (PCX86DESCHC)(gdtr.pGdt + (SelTR & X86_SEL_MASK));
1120#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
1121 if (VMX_IS_64BIT_HOST_MODE())
1122 {
1123 uint64_t trBase64 = X86DESC64_BASE(*(PX86DESC64)pDesc);
1124 rc = VMXWriteVMCS64(VMX_VMCS_HOST_TR_BASE, trBase64);
1125 Log2(("VMX_VMCS_HOST_TR_BASE %RX64\n", trBase64));
1126 AssertRC(rc);
1127 }
1128 else
1129#endif
1130 {
1131#if HC_ARCH_BITS == 64
1132 trBase = X86DESC64_BASE(*pDesc);
1133#else
1134 trBase = X86DESC_BASE(*pDesc);
1135#endif
1136 rc = VMXWriteVMCS(VMX_VMCS_HOST_TR_BASE, trBase);
1137 AssertRC(rc);
1138 Log2(("VMX_VMCS_HOST_TR_BASE %RHv\n", trBase));
1139 }
1140
1141 /* FS and GS base. */
1142#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
1143 if (VMX_IS_64BIT_HOST_MODE())
1144 {
1145 Log2(("MSR_K8_FS_BASE = %RX64\n", ASMRdMsr(MSR_K8_FS_BASE)));
1146 Log2(("MSR_K8_GS_BASE = %RX64\n", ASMRdMsr(MSR_K8_GS_BASE)));
1147 rc = VMXWriteVMCS64(VMX_VMCS_HOST_FS_BASE, ASMRdMsr(MSR_K8_FS_BASE));
1148 rc |= VMXWriteVMCS64(VMX_VMCS_HOST_GS_BASE, ASMRdMsr(MSR_K8_GS_BASE));
1149 }
1150#endif
1151 AssertRC(rc);
1152
1153 /* Sysenter MSRs. */
1154 /** @todo expensive!! */
1155 rc = VMXWriteVMCS(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
1156 Log2(("VMX_VMCS_HOST_SYSENTER_CS %08x\n", ASMRdMsr_Low(MSR_IA32_SYSENTER_CS)));
1157#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
1158 if (VMX_IS_64BIT_HOST_MODE())
1159 {
1160 Log2(("VMX_VMCS_HOST_SYSENTER_EIP %RX64\n", ASMRdMsr(MSR_IA32_SYSENTER_EIP)));
1161 Log2(("VMX_VMCS_HOST_SYSENTER_ESP %RX64\n", ASMRdMsr(MSR_IA32_SYSENTER_ESP)));
1162 rc |= VMXWriteVMCS64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
1163 rc |= VMXWriteVMCS64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
1164 }
1165 else
1166 {
1167 rc |= VMXWriteVMCS(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
1168 rc |= VMXWriteVMCS(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
1169 Log2(("VMX_VMCS_HOST_SYSENTER_EIP %RX32\n", ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP)));
1170 Log2(("VMX_VMCS_HOST_SYSENTER_ESP %RX32\n", ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP)));
1171 }
1172#elif HC_ARCH_BITS == 32
1173 rc |= VMXWriteVMCS(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
1174 rc |= VMXWriteVMCS(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
1175 Log2(("VMX_VMCS_HOST_SYSENTER_EIP %RX32\n", ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP)));
1176 Log2(("VMX_VMCS_HOST_SYSENTER_ESP %RX32\n", ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP)));
1177#else
1178 Log2(("VMX_VMCS_HOST_SYSENTER_EIP %RX64\n", ASMRdMsr(MSR_IA32_SYSENTER_EIP)));
1179 Log2(("VMX_VMCS_HOST_SYSENTER_ESP %RX64\n", ASMRdMsr(MSR_IA32_SYSENTER_ESP)));
1180 rc |= VMXWriteVMCS64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
1181 rc |= VMXWriteVMCS64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
1182#endif
1183 AssertRC(rc);
1184
1185#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
1186 /* Store all host MSRs in the VM-Exit load area, so they will be reloaded after the world switch back to the host. */
1187 PVMXMSR pMsr = (PVMXMSR)pVCpu->hwaccm.s.vmx.pHostMSR;
1188 unsigned idxMsr = 0;
1189
1190 /* EFER MSR present? */
1191 if (ASMCpuId_EDX(0x80000001) & (X86_CPUID_AMD_FEATURE_EDX_NX|X86_CPUID_AMD_FEATURE_EDX_LONG_MODE))
1192 {
1193 if (ASMCpuId_EDX(0x80000001) & X86_CPUID_AMD_FEATURE_EDX_SEP)
1194 {
1195 pMsr->u32IndexMSR = MSR_K6_STAR;
1196 pMsr->u32Reserved = 0;
1197 pMsr->u64Value = ASMRdMsr(MSR_K6_STAR); /* legacy syscall eip, cs & ss */
1198 pMsr++; idxMsr++;
1199 }
1200
1201 pMsr->u32IndexMSR = MSR_K6_EFER;
1202 pMsr->u32Reserved = 0;
1203# if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
1204 if (CPUMIsGuestInLongMode(pVCpu))
1205 {
1206 /* Must match the efer value in our 64 bits switcher. */
1207 pMsr->u64Value = ASMRdMsr(MSR_K6_EFER) | MSR_K6_EFER_LME | MSR_K6_EFER_SCE | MSR_K6_EFER_NXE;
1208 }
1209 else
1210# endif
1211 pMsr->u64Value = ASMRdMsr(MSR_K6_EFER);
1212 pMsr++; idxMsr++;
1213 }
1214
1215# if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
1216 if (VMX_IS_64BIT_HOST_MODE())
1217 {
1218 pMsr->u32IndexMSR = MSR_K8_LSTAR;
1219 pMsr->u32Reserved = 0;
1220 pMsr->u64Value = ASMRdMsr(MSR_K8_LSTAR); /* 64 bits mode syscall rip */
1221 pMsr++; idxMsr++;
1222 pMsr->u32IndexMSR = MSR_K8_SF_MASK;
1223 pMsr->u32Reserved = 0;
1224 pMsr->u64Value = ASMRdMsr(MSR_K8_SF_MASK); /* syscall flag mask */
1225 pMsr++; idxMsr++;
1226 pMsr->u32IndexMSR = MSR_K8_KERNEL_GS_BASE;
1227 pMsr->u32Reserved = 0;
1228 pMsr->u64Value = ASMRdMsr(MSR_K8_KERNEL_GS_BASE); /* swapgs exchange value */
1229 pMsr++; idxMsr++;
1230 }
1231# endif
1232 rc = VMXWriteVMCS(VMX_VMCS_CTRL_EXIT_MSR_LOAD_COUNT, idxMsr);
1233 AssertRC(rc);
1234#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
1235
1236 pVCpu->hwaccm.s.fContextUseFlags &= ~HWACCM_CHANGED_HOST_CONTEXT;
1237 }
1238 return rc;
1239}
1240
1241/**
1242 * Loads the 4 PDPEs into the guest state when nested paging is used and the
1243 * guest operates in PAE mode.
1244 *
1245 * @returns VINF_SUCCESS or fatal error.
1246 * @param pVCpu The VMCPU to operate on.
1247 * @param pCtx Guest context
1248 */
1249static int hmR0VmxLoadPaePdpes(PVMCPU pVCpu, PCPUMCTX pCtx)
1250{
1251 if (CPUMIsGuestInPAEModeEx(pCtx))
1252 {
1253 X86PDPE aPdpes[4];
1254 int rc = PGMGstGetPaePdpes(pVCpu, &aPdpes[0]);
1255 AssertRCReturn(rc, rc);
1256
1257 rc = VMXWriteVMCS64(VMX_VMCS_GUEST_PDPTR0_FULL, aPdpes[0].u); AssertRCReturn(rc, rc);
1258 rc = VMXWriteVMCS64(VMX_VMCS_GUEST_PDPTR1_FULL, aPdpes[1].u); AssertRCReturn(rc, rc);
1259 rc = VMXWriteVMCS64(VMX_VMCS_GUEST_PDPTR2_FULL, aPdpes[2].u); AssertRCReturn(rc, rc);
1260 rc = VMXWriteVMCS64(VMX_VMCS_GUEST_PDPTR3_FULL, aPdpes[3].u); AssertRCReturn(rc, rc);
1261 }
1262 return VINF_SUCCESS;
1263}
1264
1265/**
1266 * Saves the 4 PDPEs into the guest state when nested paging is used and the
1267 * guest operates in PAE mode.
1268 *
1269 * @returns VINF_SUCCESS or fatal error.
1270 * @param pVCpu The VMCPU to operate on.
1271 * @param pCtx Guest context
1272 *
1273 * @remarks Tell PGM about CR3 changes before calling this helper.
1274 */
1275static int hmR0VmxSavePaePdpes(PVMCPU pVCpu, PCPUMCTX pCtx)
1276{
1277 if (CPUMIsGuestInPAEModeEx(pCtx))
1278 {
1279 int rc;
1280 X86PDPE aPdpes[4];
1281 rc = VMXReadVMCS64(VMX_VMCS_GUEST_PDPTR0_FULL, &aPdpes[0].u); AssertRCReturn(rc, rc);
1282 rc = VMXReadVMCS64(VMX_VMCS_GUEST_PDPTR1_FULL, &aPdpes[1].u); AssertRCReturn(rc, rc);
1283 rc = VMXReadVMCS64(VMX_VMCS_GUEST_PDPTR2_FULL, &aPdpes[2].u); AssertRCReturn(rc, rc);
1284 rc = VMXReadVMCS64(VMX_VMCS_GUEST_PDPTR3_FULL, &aPdpes[3].u); AssertRCReturn(rc, rc);
1285
1286 rc = PGMGstUpdatePaePdpes(pVCpu, &aPdpes[0]);
1287 AssertRCReturn(rc, rc);
1288 }
1289 return VINF_SUCCESS;
1290}
1291
1292
1293/**
1294 * Update the exception bitmap according to the current CPU state
1295 *
1296 * @param pVM The VM to operate on.
1297 * @param pVCpu The VMCPU to operate on.
1298 * @param pCtx Guest context
1299 */
1300static void hmR0VmxUpdateExceptionBitmap(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
1301{
1302 uint32_t u32TrapMask;
1303 Assert(pCtx);
1304
1305 u32TrapMask = HWACCM_VMX_TRAP_MASK;
1306#ifndef DEBUG
1307 if (pVM->hwaccm.s.fNestedPaging)
1308 u32TrapMask &= ~RT_BIT(X86_XCPT_PF); /* no longer need to intercept #PF. */
1309#endif
1310
1311 /* Also catch floating point exceptions if we need to report them to the guest in a different way. */
1312 if (!(pCtx->cr0 & X86_CR0_NE))
1313 {
1314 u32TrapMask |= RT_BIT(X86_XCPT_MF);
1315 }
1316
1317#ifdef VBOX_STRICT
1318 Assert(u32TrapMask & RT_BIT(X86_XCPT_GP));
1319#endif
1320
1321 /* Intercept all exceptions in real mode as none of them can be injected directly (#GP otherwise). */
1322 if ( CPUMIsGuestInRealModeEx(pCtx)
1323 && pVM->hwaccm.s.vmx.pRealModeTSS)
1324 u32TrapMask |= HWACCM_VMX_TRAP_MASK_REALMODE;
1325
1326 int rc = VMXWriteVMCS(VMX_VMCS_CTRL_EXCEPTION_BITMAP, u32TrapMask);
1327 AssertRC(rc);
1328}
1329
1330/**
1331 * Loads a minimal guest state
1332 *
1333 * NOTE: Don't do anything here that can cause a jump back to ring 3!!!!!
1334 *
1335 * @param pVM The VM to operate on.
1336 * @param pVCpu The VMCPU to operate on.
1337 * @param pCtx Guest context
1338 */
1339VMMR0DECL(void) VMXR0LoadMinimalGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
1340{
1341 int rc;
1342 X86EFLAGS eflags;
1343
1344 Assert(!(pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_ALL_GUEST));
1345
1346 /* EIP, ESP and EFLAGS */
1347 rc = VMXWriteVMCS64(VMX_VMCS64_GUEST_RIP, pCtx->rip);
1348 rc |= VMXWriteVMCS64(VMX_VMCS64_GUEST_RSP, pCtx->rsp);
1349 AssertRC(rc);
1350
1351 /* Bits 22-31, 15, 5 & 3 must be zero. Bit 1 must be 1. */
1352 eflags = pCtx->eflags;
1353 eflags.u32 &= VMX_EFLAGS_RESERVED_0;
1354 eflags.u32 |= VMX_EFLAGS_RESERVED_1;
1355
1356 /* Real mode emulation using v86 mode. */
1357 if ( CPUMIsGuestInRealModeEx(pCtx)
1358 && pVM->hwaccm.s.vmx.pRealModeTSS)
1359 {
1360 pVCpu->hwaccm.s.vmx.RealMode.eflags = eflags;
1361
1362 eflags.Bits.u1VM = 1;
1363 eflags.Bits.u2IOPL = 0; /* must always be 0 or else certain instructions won't cause faults. */
1364 }
1365 rc = VMXWriteVMCS(VMX_VMCS_GUEST_RFLAGS, eflags.u32);
1366 AssertRC(rc);
1367}
1368
1369/**
1370 * Loads the guest state
1371 *
1372 * NOTE: Don't do anything here that can cause a jump back to ring 3!!!!!
1373 *
1374 * @returns VBox status code.
1375 * @param pVM The VM to operate on.
1376 * @param pVCpu The VMCPU to operate on.
1377 * @param pCtx Guest context
1378 */
1379VMMR0DECL(int) VMXR0LoadGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
1380{
1381 int rc = VINF_SUCCESS;
1382 RTGCUINTPTR val;
1383
1384 /* VMX_VMCS_CTRL_ENTRY_CONTROLS
1385 * Set required bits to one and zero according to the MSR capabilities.
1386 */
1387 val = pVM->hwaccm.s.vmx.msr.vmx_entry.n.disallowed0;
1388 /* Load guest debug controls (dr7 & IA32_DEBUGCTL_MSR) (forced to 1 on the 'first' VT-x capable CPUs; this actually includes the newest Nehalem CPUs) */
1389 val |= VMX_VMCS_CTRL_ENTRY_CONTROLS_LOAD_DEBUG;
1390 /* 64 bits guest mode? */
1391 if (CPUMIsGuestInLongModeEx(pCtx))
1392 val |= VMX_VMCS_CTRL_ENTRY_CONTROLS_IA64_MODE;
1393 /* else Must be zero when AMD64 is not available. */
1394
1395 /* Mask away the bits that the CPU doesn't support */
1396 val &= pVM->hwaccm.s.vmx.msr.vmx_entry.n.allowed1;
1397 rc = VMXWriteVMCS(VMX_VMCS_CTRL_ENTRY_CONTROLS, val);
1398 AssertRC(rc);
1399
1400 /* VMX_VMCS_CTRL_EXIT_CONTROLS
1401 * Set required bits to one and zero according to the MSR capabilities.
1402 */
1403 val = pVM->hwaccm.s.vmx.msr.vmx_exit.n.disallowed0;
1404
1405 /* Save debug controls (dr7 & IA32_DEBUGCTL_MSR) (forced to 1 on the 'first' VT-x capable CPUs; this actually includes the newest Nehalem CPUs) */
1406 val |= VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_DEBUG;
1407
1408#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
1409 if (VMX_IS_64BIT_HOST_MODE())
1410 val |= VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_AMD64;
1411 /* else: Must be zero when AMD64 is not available. */
1412#elif HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
1413 if (CPUMIsGuestInLongModeEx(pCtx))
1414 val |= VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_AMD64; /* our switcher goes to long mode */
1415 else
1416 Assert(!(val & VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_AMD64));
1417#endif
1418 val &= pVM->hwaccm.s.vmx.msr.vmx_exit.n.allowed1;
1419 /* Don't acknowledge external interrupts on VM-exit. */
1420 rc = VMXWriteVMCS(VMX_VMCS_CTRL_EXIT_CONTROLS, val);
1421 AssertRC(rc);
1422
1423 /* Guest CPU context: ES, CS, SS, DS, FS, GS. */
1424 if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_SEGMENT_REGS)
1425 {
1426 if (pVM->hwaccm.s.vmx.pRealModeTSS)
1427 {
1428 PGMMODE enmGuestMode = PGMGetGuestMode(pVCpu);
1429 if (pVCpu->hwaccm.s.vmx.enmLastSeenGuestMode != enmGuestMode)
1430 {
1431 /* Correct weird requirements for switching to protected mode. */
1432 if ( pVCpu->hwaccm.s.vmx.enmLastSeenGuestMode == PGMMODE_REAL
1433 && enmGuestMode >= PGMMODE_PROTECTED)
1434 {
1435#ifdef VBOX_WITH_REM
1436 /* Flush the recompiler code cache as it's not unlikely
1437 * the guest will rewrite code it will later execute in real
1438 * mode (OpenBSD 4.0 is one such example)
1439 */
1440 REMFlushTBs(pVM);
1441#endif
1442
1443 /* DPL of all hidden selector registers must match the current CPL (0). */
1444 pCtx->csHid.Attr.n.u2Dpl = 0;
1445 pCtx->csHid.Attr.n.u4Type = X86_SEL_TYPE_CODE | X86_SEL_TYPE_RW_ACC;
1446
1447 pCtx->dsHid.Attr.n.u2Dpl = 0;
1448 pCtx->esHid.Attr.n.u2Dpl = 0;
1449 pCtx->fsHid.Attr.n.u2Dpl = 0;
1450 pCtx->gsHid.Attr.n.u2Dpl = 0;
1451 pCtx->ssHid.Attr.n.u2Dpl = 0;
1452 }
1453 pVCpu->hwaccm.s.vmx.enmLastSeenGuestMode = enmGuestMode;
1454 }
1455 else
1456 /* VT-x will fail with a guest invalid state otherwise... (CPU state after a reset) */
1457 if ( CPUMIsGuestInRealModeEx(pCtx)
1458 && pCtx->csHid.u64Base == 0xffff0000)
1459 {
1460 pCtx->csHid.u64Base = 0xf0000;
1461 pCtx->cs = 0xf000;
1462 }
1463 }
1464
1465 VMX_WRITE_SELREG(ES, es);
1466 AssertRC(rc);
1467
1468 VMX_WRITE_SELREG(CS, cs);
1469 AssertRC(rc);
1470
1471 VMX_WRITE_SELREG(SS, ss);
1472 AssertRC(rc);
1473
1474 VMX_WRITE_SELREG(DS, ds);
1475 AssertRC(rc);
1476
1477 VMX_WRITE_SELREG(FS, fs);
1478 AssertRC(rc);
1479
1480 VMX_WRITE_SELREG(GS, gs);
1481 AssertRC(rc);
1482 }
1483
1484 /* Guest CPU context: LDTR. */
1485 if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_LDTR)
1486 {
1487 if (pCtx->ldtr == 0)
1488 {
1489 rc = VMXWriteVMCS(VMX_VMCS16_GUEST_FIELD_LDTR, 0);
1490 rc |= VMXWriteVMCS(VMX_VMCS32_GUEST_LDTR_LIMIT, 0);
1491 rc |= VMXWriteVMCS64(VMX_VMCS64_GUEST_LDTR_BASE, 0);
1492 /* Note: vmlaunch will fail with 0 or just 0x02. No idea why. */
1493 rc |= VMXWriteVMCS(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, 0x82 /* present, LDT */);
1494 }
1495 else
1496 {
1497 rc = VMXWriteVMCS(VMX_VMCS16_GUEST_FIELD_LDTR, pCtx->ldtr);
1498 rc |= VMXWriteVMCS(VMX_VMCS32_GUEST_LDTR_LIMIT, pCtx->ldtrHid.u32Limit);
1499 rc |= VMXWriteVMCS64(VMX_VMCS64_GUEST_LDTR_BASE, pCtx->ldtrHid.u64Base);
1500 rc |= VMXWriteVMCS(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, pCtx->ldtrHid.Attr.u);
1501 }
1502 AssertRC(rc);
1503 }
1504 /* Guest CPU context: TR. */
1505 if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_TR)
1506 {
1507 /* Real mode emulation using v86 mode with CR4.VME (interrupt redirection using the int bitmap in the TSS) */
1508 if ( CPUMIsGuestInRealModeEx(pCtx)
1509 && pVM->hwaccm.s.vmx.pRealModeTSS)
1510 {
1511 RTGCPHYS GCPhys;
1512
1513 /* We convert it here every time as pci regions could be reconfigured. */
1514 rc = PDMVMMDevHeapR3ToGCPhys(pVM, pVM->hwaccm.s.vmx.pRealModeTSS, &GCPhys);
1515 AssertRC(rc);
1516
1517 rc = VMXWriteVMCS(VMX_VMCS16_GUEST_FIELD_TR, 0);
1518 rc |= VMXWriteVMCS(VMX_VMCS32_GUEST_TR_LIMIT, HWACCM_VTX_TSS_SIZE);
1519 rc |= VMXWriteVMCS64(VMX_VMCS64_GUEST_TR_BASE, GCPhys /* phys = virt in this mode */);
1520
1521 X86DESCATTR attr;
1522
1523 attr.u = 0;
1524 attr.n.u1Present = 1;
1525 attr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
1526 val = attr.u;
1527 }
1528 else
1529 {
1530 rc = VMXWriteVMCS(VMX_VMCS16_GUEST_FIELD_TR, pCtx->tr);
1531 rc |= VMXWriteVMCS(VMX_VMCS32_GUEST_TR_LIMIT, pCtx->trHid.u32Limit);
1532 rc |= VMXWriteVMCS64(VMX_VMCS64_GUEST_TR_BASE, pCtx->trHid.u64Base);
1533
1534 val = pCtx->trHid.Attr.u;
1535
1536 /* The TSS selector must be busy (REM bugs? see defect #XXXX). */
1537 if (!(val & X86_SEL_TYPE_SYS_TSS_BUSY_MASK))
1538 {
1539 if (val & 0xf)
1540 val |= X86_SEL_TYPE_SYS_TSS_BUSY_MASK;
1541 else
1542 /* Default if no TR selector has been set (otherwise vmlaunch will fail!) */
1543 val = (val & ~0xF) | X86_SEL_TYPE_SYS_386_TSS_BUSY;
1544 }
1545 AssertMsg((val & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY || (val & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("%#x\n", val));
1546 }
1547 rc |= VMXWriteVMCS(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, val);
1548 AssertRC(rc);
1549 }
1550 /* Guest CPU context: GDTR. */
1551 if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_GDTR)
1552 {
1553 rc = VMXWriteVMCS(VMX_VMCS32_GUEST_GDTR_LIMIT, pCtx->gdtr.cbGdt);
1554 rc |= VMXWriteVMCS64(VMX_VMCS64_GUEST_GDTR_BASE, pCtx->gdtr.pGdt);
1555 AssertRC(rc);
1556 }
1557 /* Guest CPU context: IDTR. */
1558 if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_IDTR)
1559 {
1560 rc = VMXWriteVMCS(VMX_VMCS32_GUEST_IDTR_LIMIT, pCtx->idtr.cbIdt);
1561 rc |= VMXWriteVMCS64(VMX_VMCS64_GUEST_IDTR_BASE, pCtx->idtr.pIdt);
1562 AssertRC(rc);
1563 }
1564
1565 /*
1566 * Sysenter MSRs
1567 */
1568 if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_MSR)
1569 {
1570 rc = VMXWriteVMCS(VMX_VMCS32_GUEST_SYSENTER_CS, pCtx->SysEnter.cs);
1571 rc |= VMXWriteVMCS64(VMX_VMCS64_GUEST_SYSENTER_EIP, pCtx->SysEnter.eip);
1572 rc |= VMXWriteVMCS64(VMX_VMCS64_GUEST_SYSENTER_ESP, pCtx->SysEnter.esp);
1573 AssertRC(rc);
1574 }
1575
1576 /* Control registers */
1577 if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_CR0)
1578 {
1579 val = pCtx->cr0;
1580 rc = VMXWriteVMCS(VMX_VMCS_CTRL_CR0_READ_SHADOW, val);
1581 Log2(("Guest CR0-shadow %08x\n", val));
1582 if (CPUMIsGuestFPUStateActive(pVCpu) == false)
1583 {
1584 /* Always use #NM exceptions to load the FPU/XMM state on demand. */
1585 val |= X86_CR0_TS | X86_CR0_ET | X86_CR0_NE | X86_CR0_MP;
1586 }
1587 else
1588 {
1589 /** @todo check if we support the old style mess correctly. */
1590 if (!(val & X86_CR0_NE))
1591 Log(("Forcing X86_CR0_NE!!!\n"));
1592
1593 val |= X86_CR0_NE; /* always turn on the native mechanism to report FPU errors (old style uses interrupts) */
1594 }
1595 /* Note: protected mode & paging are always enabled; we use them for emulating real and protected mode without paging too. */
1596 if (!pVM->hwaccm.s.vmx.fUnrestrictedGuest)
1597 val |= X86_CR0_PE | X86_CR0_PG;
1598
1599 if (pVM->hwaccm.s.fNestedPaging)
1600 {
1601 if (CPUMIsGuestInPagedProtectedModeEx(pCtx))
1602 {
1603 /* Disable cr3 read/write monitoring as we don't need it for EPT. */
1604 pVCpu->hwaccm.s.vmx.proc_ctls &= ~( VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
1605 | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT);
1606 }
1607 else
1608 {
1609 /* Reenable cr3 read/write monitoring as our identity mapped page table is active. */
1610 pVCpu->hwaccm.s.vmx.proc_ctls |= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
1611 | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT;
1612 }
1613 rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
1614 AssertRC(rc);
1615 }
1616 else
1617 {
1618 /* Note: We must also set this as we rely on protecting various pages for which supervisor writes must be caught. */
1619 val |= X86_CR0_WP;
1620 }
1621
1622 /* Always enable caching. */
1623 val &= ~(X86_CR0_CD|X86_CR0_NW);
1624
1625 rc |= VMXWriteVMCS64(VMX_VMCS64_GUEST_CR0, val);
1626 Log2(("Guest CR0 %08x\n", val));
1627 /* CR0 flags owned by the host; if the guests attempts to change them, then
1628 * the VM will exit.
1629 */
1630 val = X86_CR0_PE /* Must monitor this bit (assumptions are made for real mode emulation) */
1631 | X86_CR0_WP /* Must monitor this bit (it must always be enabled). */
1632 | X86_CR0_PG /* Must monitor this bit (assumptions are made for real mode & protected mode without paging emulation) */
1633 | X86_CR0_CD /* Bit not restored during VM-exit! */
1634 | X86_CR0_NW /* Bit not restored during VM-exit! */
1635 | X86_CR0_NE;
1636
1637 /* When the guest's FPU state is active, then we no longer care about
1638 * the FPU related bits.
1639 */
1640 if (CPUMIsGuestFPUStateActive(pVCpu) == false)
1641 val |= X86_CR0_TS | X86_CR0_ET | X86_CR0_MP;
1642
1643 pVCpu->hwaccm.s.vmx.cr0_mask = val;
1644
1645 rc |= VMXWriteVMCS(VMX_VMCS_CTRL_CR0_MASK, val);
1646 Log2(("Guest CR0-mask %08x\n", val));
1647 AssertRC(rc);
1648 }
1649 if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_CR4)
1650 {
1651 /* CR4 */
1652 rc = VMXWriteVMCS(VMX_VMCS_CTRL_CR4_READ_SHADOW, pCtx->cr4);
1653 Log2(("Guest CR4-shadow %08x\n", pCtx->cr4));
1654 /* Set the required bits in cr4 too (currently X86_CR4_VMXE). */
1655 val = pCtx->cr4 | (uint32_t)pVM->hwaccm.s.vmx.msr.vmx_cr4_fixed0;
1656
1657 if (!pVM->hwaccm.s.fNestedPaging)
1658 {
1659 switch(pVCpu->hwaccm.s.enmShadowMode)
1660 {
1661 case PGMMODE_REAL: /* Real mode -> emulated using v86 mode */
1662 case PGMMODE_PROTECTED: /* Protected mode, no paging -> emulated using identity mapping. */
1663 case PGMMODE_32_BIT: /* 32-bit paging. */
1664 val &= ~X86_CR4_PAE;
1665 break;
1666
1667 case PGMMODE_PAE: /* PAE paging. */
1668 case PGMMODE_PAE_NX: /* PAE paging with NX enabled. */
1669 /** Must use PAE paging as we could use physical memory > 4 GB */
1670 val |= X86_CR4_PAE;
1671 break;
1672
1673 case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
1674 case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
1675#ifdef VBOX_ENABLE_64_BITS_GUESTS
1676 break;
1677#else
1678 AssertFailed();
1679 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
1680#endif
1681 default: /* shut up gcc */
1682 AssertFailed();
1683 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
1684 }
1685 }
1686 else
1687 if ( !CPUMIsGuestInPagedProtectedModeEx(pCtx)
1688 && !pVM->hwaccm.s.vmx.fUnrestrictedGuest)
1689 {
1690 /* We use 4 MB pages in our identity mapping page table for real and protected mode without paging. */
1691 val |= X86_CR4_PSE;
1692 /* Our identity mapping is a 32 bits page directory. */
1693 val &= ~X86_CR4_PAE;
1694 }
1695
1696 /* Turn off VME if we're in emulated real mode. */
1697 if ( CPUMIsGuestInRealModeEx(pCtx)
1698 && pVM->hwaccm.s.vmx.pRealModeTSS)
1699 val &= ~X86_CR4_VME;
1700
1701 rc |= VMXWriteVMCS64(VMX_VMCS64_GUEST_CR4, val);
1702 Log2(("Guest CR4 %08x\n", val));
1703 /* CR4 flags owned by the host; if the guests attempts to change them, then
1704 * the VM will exit.
1705 */
1706 val = 0
1707 | X86_CR4_VME
1708 | X86_CR4_PAE
1709 | X86_CR4_PGE
1710 | X86_CR4_PSE
1711 | X86_CR4_VMXE;
1712 pVCpu->hwaccm.s.vmx.cr4_mask = val;
1713
1714 rc |= VMXWriteVMCS(VMX_VMCS_CTRL_CR4_MASK, val);
1715 Log2(("Guest CR4-mask %08x\n", val));
1716 AssertRC(rc);
1717 }
1718
1719#if 0
1720 /* Enable single stepping if requested and CPU supports it. */
1721 if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_TRAP_FLAG)
1722 if (DBGFIsStepping(pVCpu))
1723 {
1724 pVCpu->hwaccm.s.vmx.proc_ctls |= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_TRAP_FLAG;
1725 rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
1726 AssertRC(rc);
1727 }
1728#endif
1729
1730 if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_CR3)
1731 {
1732 if (pVM->hwaccm.s.fNestedPaging)
1733 {
1734 Assert(PGMGetHyperCR3(pVCpu));
1735 pVCpu->hwaccm.s.vmx.GCPhysEPTP = PGMGetHyperCR3(pVCpu);
1736
1737 Assert(!(pVCpu->hwaccm.s.vmx.GCPhysEPTP & 0xfff));
1738 /** @todo Check the IA32_VMX_EPT_VPID_CAP MSR for other supported memory types. */
1739 pVCpu->hwaccm.s.vmx.GCPhysEPTP |= VMX_EPT_MEMTYPE_WB
1740 | (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
1741
1742 rc = VMXWriteVMCS64(VMX_VMCS_CTRL_EPTP_FULL, pVCpu->hwaccm.s.vmx.GCPhysEPTP);
1743 AssertRC(rc);
1744
1745 if ( !CPUMIsGuestInPagedProtectedModeEx(pCtx)
1746 && !pVM->hwaccm.s.vmx.fUnrestrictedGuest)
1747 {
1748 RTGCPHYS GCPhys;
1749
1750 /* We convert it here every time as pci regions could be reconfigured. */
1751 rc = PDMVMMDevHeapR3ToGCPhys(pVM, pVM->hwaccm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
1752 AssertMsgRC(rc, ("pNonPagingModeEPTPageTable = %RGv\n", pVM->hwaccm.s.vmx.pNonPagingModeEPTPageTable));
1753
1754 /* We use our identity mapping page table here as we need to map guest virtual to guest physical addresses; EPT will
1755 * take care of the translation to host physical addresses.
1756 */
1757 val = GCPhys;
1758 }
1759 else
1760 {
1761 /* Save the real guest CR3 in VMX_VMCS_GUEST_CR3 */
1762 val = pCtx->cr3;
1763 rc = hmR0VmxLoadPaePdpes(pVCpu, pCtx);
1764 AssertRCReturn(rc, rc);
1765 }
1766 }
1767 else
1768 {
1769 val = PGMGetHyperCR3(pVCpu);
1770 Assert(val || VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL));
1771 }
1772
1773 /* Save our shadow CR3 register. */
1774 rc = VMXWriteVMCS64(VMX_VMCS64_GUEST_CR3, val);
1775 AssertRC(rc);
1776 }
1777
1778 /* Debug registers. */
1779 if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_DEBUG)
1780 {
1781 pCtx->dr[6] |= X86_DR6_INIT_VAL; /* set all reserved bits to 1. */
1782 pCtx->dr[6] &= ~RT_BIT(12); /* must be zero. */
1783
1784 pCtx->dr[7] &= 0xffffffff; /* upper 32 bits reserved */
1785 pCtx->dr[7] &= ~(RT_BIT(11) | RT_BIT(12) | RT_BIT(14) | RT_BIT(15)); /* must be zero */
1786 pCtx->dr[7] |= 0x400; /* must be one */
1787
1788 /* Resync DR7 */
1789 rc = VMXWriteVMCS64(VMX_VMCS64_GUEST_DR7, pCtx->dr[7]);
1790 AssertRC(rc);
1791
1792#ifdef DEBUG
1793 /* Sync the hypervisor debug state now if any breakpoint is armed. */
1794 if ( CPUMGetHyperDR7(pVCpu) & (X86_DR7_ENABLED_MASK|X86_DR7_GD)
1795 && !CPUMIsHyperDebugStateActive(pVCpu)
1796 && !DBGFIsStepping(pVCpu))
1797 {
1798 /* Save the host and load the hypervisor debug state. */
1799 rc = CPUMR0LoadHyperDebugState(pVM, pVCpu, pCtx, true /* include DR6 */);
1800 AssertRC(rc);
1801
1802 /* DRx intercepts remain enabled. */
1803
1804 /* Override dr7 with the hypervisor value. */
1805 rc = VMXWriteVMCS64(VMX_VMCS64_GUEST_DR7, CPUMGetHyperDR7(pVCpu));
1806 AssertRC(rc);
1807 }
1808 else
1809#endif
1810 /* Sync the debug state now if any breakpoint is armed. */
1811 if ( (pCtx->dr[7] & (X86_DR7_ENABLED_MASK|X86_DR7_GD))
1812 && !CPUMIsGuestDebugStateActive(pVCpu)
1813 && !DBGFIsStepping(pVCpu))
1814 {
1815 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatDRxArmed);
1816
1817 /* Disable drx move intercepts. */
1818 pVCpu->hwaccm.s.vmx.proc_ctls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT;
1819 rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
1820 AssertRC(rc);
1821
1822 /* Save the host and load the guest debug state. */
1823 rc = CPUMR0LoadGuestDebugState(pVM, pVCpu, pCtx, true /* include DR6 */);
1824 AssertRC(rc);
1825 }
1826
1827 /* IA32_DEBUGCTL MSR. */
1828 rc = VMXWriteVMCS64(VMX_VMCS_GUEST_DEBUGCTL_FULL, 0);
1829 AssertRC(rc);
1830
1831 /** @todo do we really ever need this? */
1832 rc |= VMXWriteVMCS(VMX_VMCS_GUEST_DEBUG_EXCEPTIONS, 0);
1833 AssertRC(rc);
1834 }
1835
1836 /* 64 bits guest mode? */
1837 if (CPUMIsGuestInLongModeEx(pCtx))
1838 {
1839#if !defined(VBOX_ENABLE_64_BITS_GUESTS)
1840 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
1841#elif HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
1842 pVCpu->hwaccm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
1843#else
1844# ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
1845 if (!pVM->hwaccm.s.fAllow64BitGuests)
1846 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
1847# endif
1848 pVCpu->hwaccm.s.vmx.pfnStartVM = VMXR0StartVM64;
1849#endif
1850 if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_MSR)
1851 {
1852 /* Update these as wrmsr might have changed them. */
1853 rc = VMXWriteVMCS64(VMX_VMCS64_GUEST_FS_BASE, pCtx->fsHid.u64Base);
1854 AssertRC(rc);
1855 rc = VMXWriteVMCS64(VMX_VMCS64_GUEST_GS_BASE, pCtx->gsHid.u64Base);
1856 AssertRC(rc);
1857 }
1858 }
1859 else
1860 {
1861 pVCpu->hwaccm.s.vmx.pfnStartVM = VMXR0StartVM32;
1862 }
1863
1864 hmR0VmxUpdateExceptionBitmap(pVM, pVCpu, pCtx);
1865
1866#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
1867 /* Store all guest MSRs in the VM-Entry load area, so they will be loaded during the world switch. */
1868 PVMXMSR pMsr = (PVMXMSR)pVCpu->hwaccm.s.vmx.pGuestMSR;
1869 unsigned idxMsr = 0;
1870
1871 uint32_t ulEdx;
1872 uint32_t ulTemp;
1873 CPUMGetGuestCpuId(pVCpu, 0x80000001, &ulTemp, &ulTemp, &ulTemp, &ulEdx);
1874 /* EFER MSR present? */
1875 if (ulEdx & (X86_CPUID_AMD_FEATURE_EDX_NX|X86_CPUID_AMD_FEATURE_EDX_LONG_MODE))
1876 {
1877 pMsr->u32IndexMSR = MSR_K6_EFER;
1878 pMsr->u32Reserved = 0;
1879 pMsr->u64Value = pCtx->msrEFER;
1880 /* VT-x will complain if only MSR_K6_EFER_LME is set. */
1881 if (!CPUMIsGuestInLongModeEx(pCtx))
1882 pMsr->u64Value &= ~(MSR_K6_EFER_LMA|MSR_K6_EFER_LME);
1883 pMsr++; idxMsr++;
1884
1885 if (ulEdx & X86_CPUID_AMD_FEATURE_EDX_LONG_MODE)
1886 {
1887 pMsr->u32IndexMSR = MSR_K8_LSTAR;
1888 pMsr->u32Reserved = 0;
1889 pMsr->u64Value = pCtx->msrLSTAR; /* 64 bits mode syscall rip */
1890 pMsr++; idxMsr++;
1891 pMsr->u32IndexMSR = MSR_K6_STAR;
1892 pMsr->u32Reserved = 0;
1893 pMsr->u64Value = pCtx->msrSTAR; /* legacy syscall eip, cs & ss */
1894 pMsr++; idxMsr++;
1895 pMsr->u32IndexMSR = MSR_K8_SF_MASK;
1896 pMsr->u32Reserved = 0;
1897 pMsr->u64Value = pCtx->msrSFMASK; /* syscall flag mask */
1898 pMsr++; idxMsr++;
1899 pMsr->u32IndexMSR = MSR_K8_KERNEL_GS_BASE;
1900 pMsr->u32Reserved = 0;
1901 pMsr->u64Value = pCtx->msrKERNELGSBASE; /* swapgs exchange value */
1902 pMsr++; idxMsr++;
1903 }
1904 }
1905 pVCpu->hwaccm.s.vmx.cCachedMSRs = idxMsr;
1906
1907 rc = VMXWriteVMCS(VMX_VMCS_CTRL_ENTRY_MSR_LOAD_COUNT, idxMsr);
1908 AssertRC(rc);
1909
1910 rc = VMXWriteVMCS(VMX_VMCS_CTRL_EXIT_MSR_STORE_COUNT, idxMsr);
1911 AssertRC(rc);
1912#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
1913
1914 bool fOffsettedTsc;
1915 if (pVM->hwaccm.s.vmx.fUsePreemptTimer)
1916 {
1917 uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVCpu, &fOffsettedTsc, &pVCpu->hwaccm.s.vmx.u64TSCOffset);
1918
1919 /* Make sure the returned values have sane upper and lower boundaries. */
1920 uint64_t u64CpuHz = SUPGetCpuHzFromGIP(g_pSUPGlobalInfoPage);
1921
1922 cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64 of a second */
1923 cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
1924
1925 cTicksToDeadline >>= pVM->hwaccm.s.vmx.cPreemptTimerShift;
1926 uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
1927 rc = VMXWriteVMCS(VMX_VMCS32_GUEST_PREEMPTION_TIMER_VALUE, cPreemptionTickCount);
1928 AssertRC(rc);
1929 }
1930 else
1931 fOffsettedTsc = TMCpuTickCanUseRealTSC(pVCpu, &pVCpu->hwaccm.s.vmx.u64TSCOffset);
1932 if (fOffsettedTsc)
1933 {
1934 uint64_t u64CurTSC = ASMReadTSC();
1935 if (u64CurTSC + pVCpu->hwaccm.s.vmx.u64TSCOffset >= TMCpuTickGetLastSeen(pVCpu))
1936 {
1937 /* Note: VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT takes precedence over TSC_OFFSET */
1938 rc = VMXWriteVMCS64(VMX_VMCS_CTRL_TSC_OFFSET_FULL, pVCpu->hwaccm.s.vmx.u64TSCOffset);
1939 AssertRC(rc);
1940
1941 pVCpu->hwaccm.s.vmx.proc_ctls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT;
1942 rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
1943 AssertRC(rc);
1944 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatTSCOffset);
1945 }
1946 else
1947 {
1948 /* Fall back to rdtsc emulation as we would otherwise pass decreasing tsc values to the guest. */
1949 LogFlow(("TSC %RX64 offset %RX64 time=%RX64 last=%RX64 (diff=%RX64, virt_tsc=%RX64)\n", u64CurTSC, pVCpu->hwaccm.s.vmx.u64TSCOffset, u64CurTSC + pVCpu->hwaccm.s.vmx.u64TSCOffset, TMCpuTickGetLastSeen(pVCpu), TMCpuTickGetLastSeen(pVCpu) - u64CurTSC - pVCpu->hwaccm.s.vmx.u64TSCOffset, TMCpuTickGet(pVCpu)));
1950 pVCpu->hwaccm.s.vmx.proc_ctls |= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT;
1951 rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
1952 AssertRC(rc);
1953 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatTSCInterceptOverFlow);
1954 }
1955 }
1956 else
1957 {
1958 pVCpu->hwaccm.s.vmx.proc_ctls |= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT;
1959 rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
1960 AssertRC(rc);
1961 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatTSCIntercept);
1962 }
1963
1964 /* Done with the major changes */
1965 pVCpu->hwaccm.s.fContextUseFlags &= ~HWACCM_CHANGED_ALL_GUEST;
1966
1967 /* Minimal guest state update (esp, eip, eflags mostly) */
1968 VMXR0LoadMinimalGuestState(pVM, pVCpu, pCtx);
1969 return rc;
1970}
1971
1972/**
1973 * Syncs back the guest state
1974 *
1975 * @returns VBox status code.
1976 * @param pVM The VM to operate on.
1977 * @param pVCpu The VMCPU to operate on.
1978 * @param pCtx Guest context
1979 */
1980DECLINLINE(int) VMXR0SaveGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
1981{
1982 RTGCUINTREG val, valShadow;
1983 RTGCUINTPTR uInterruptState;
1984 int rc;
1985
1986 /* Let's first sync back eip, esp, and eflags. */
1987 rc = VMXReadCachedVMCS(VMX_VMCS64_GUEST_RIP, &val);
1988 AssertRC(rc);
1989 pCtx->rip = val;
1990 rc = VMXReadCachedVMCS(VMX_VMCS64_GUEST_RSP, &val);
1991 AssertRC(rc);
1992 pCtx->rsp = val;
1993 rc = VMXReadCachedVMCS(VMX_VMCS_GUEST_RFLAGS, &val);
1994 AssertRC(rc);
1995 pCtx->eflags.u32 = val;
1996
1997 /* Take care of instruction fusing (sti, mov ss) */
1998 rc |= VMXReadCachedVMCS(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &val);
1999 uInterruptState = val;
2000 if (uInterruptState != 0)
2001 {
2002 Assert(uInterruptState <= 2); /* only sti & mov ss */
2003 Log(("uInterruptState %x eip=%RGv\n", (uint32_t)uInterruptState, pCtx->rip));
2004 EMSetInhibitInterruptsPC(pVCpu, pCtx->rip);
2005 }
2006 else
2007 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
2008
2009 /* Control registers. */
2010 VMXReadCachedVMCS(VMX_VMCS_CTRL_CR0_READ_SHADOW, &valShadow);
2011 VMXReadCachedVMCS(VMX_VMCS64_GUEST_CR0, &val);
2012 val = (valShadow & pVCpu->hwaccm.s.vmx.cr0_mask) | (val & ~pVCpu->hwaccm.s.vmx.cr0_mask);
2013 CPUMSetGuestCR0(pVCpu, val);
2014
2015 VMXReadCachedVMCS(VMX_VMCS_CTRL_CR4_READ_SHADOW, &valShadow);
2016 VMXReadCachedVMCS(VMX_VMCS64_GUEST_CR4, &val);
2017 val = (valShadow & pVCpu->hwaccm.s.vmx.cr4_mask) | (val & ~pVCpu->hwaccm.s.vmx.cr4_mask);
2018 CPUMSetGuestCR4(pVCpu, val);
2019
2020 /* Note: no reason to sync back the CRx registers. They can't be changed by the guest. */
2021 /* Note: only in the nested paging case can CR3 & CR4 be changed by the guest. */
2022 if ( pVM->hwaccm.s.fNestedPaging
2023 && CPUMIsGuestInPagedProtectedModeEx(pCtx)) /** @todo check if we will always catch mode switches and such... */
2024 {
2025 PVMCSCACHE pCache = &pVCpu->hwaccm.s.vmx.VMCSCache;
2026
2027 /* Can be updated behind our back in the nested paging case. */
2028 CPUMSetGuestCR2(pVCpu, pCache->cr2);
2029
2030 VMXReadCachedVMCS(VMX_VMCS64_GUEST_CR3, &val);
2031
2032 if (val != pCtx->cr3)
2033 {
2034 CPUMSetGuestCR3(pVCpu, val);
2035 PGMUpdateCR3(pVCpu, val);
2036 }
2037 rc = hmR0VmxSavePaePdpes(pVCpu, pCtx);
2038 AssertRCReturn(rc, rc);
2039 }
2040
2041 /* Sync back DR7 here. */
2042 VMXReadCachedVMCS(VMX_VMCS64_GUEST_DR7, &val);
2043 pCtx->dr[7] = val;
2044
2045 /* Guest CPU context: ES, CS, SS, DS, FS, GS. */
2046 VMX_READ_SELREG(ES, es);
2047 VMX_READ_SELREG(SS, ss);
2048 VMX_READ_SELREG(CS, cs);
2049 VMX_READ_SELREG(DS, ds);
2050 VMX_READ_SELREG(FS, fs);
2051 VMX_READ_SELREG(GS, gs);
2052
2053 /*
2054 * System MSRs
2055 */
2056 VMXReadCachedVMCS(VMX_VMCS32_GUEST_SYSENTER_CS, &val);
2057 pCtx->SysEnter.cs = val;
2058 VMXReadCachedVMCS(VMX_VMCS64_GUEST_SYSENTER_EIP, &val);
2059 pCtx->SysEnter.eip = val;
2060 VMXReadCachedVMCS(VMX_VMCS64_GUEST_SYSENTER_ESP, &val);
2061 pCtx->SysEnter.esp = val;
2062
2063 /* Misc. registers; must sync everything otherwise we can get out of sync when jumping to ring 3. */
2064 VMX_READ_SELREG(LDTR, ldtr);
2065
2066 VMXReadCachedVMCS(VMX_VMCS32_GUEST_GDTR_LIMIT, &val);
2067 pCtx->gdtr.cbGdt = val;
2068 VMXReadCachedVMCS(VMX_VMCS64_GUEST_GDTR_BASE, &val);
2069 pCtx->gdtr.pGdt = val;
2070
2071 VMXReadCachedVMCS(VMX_VMCS32_GUEST_IDTR_LIMIT, &val);
2072 pCtx->idtr.cbIdt = val;
2073 VMXReadCachedVMCS(VMX_VMCS64_GUEST_IDTR_BASE, &val);
2074 pCtx->idtr.pIdt = val;
2075
2076 /* Real mode emulation using v86 mode. */
2077 if ( CPUMIsGuestInRealModeEx(pCtx)
2078 && pVM->hwaccm.s.vmx.pRealModeTSS)
2079 {
2080 /* Hide our emulation flags */
2081 pCtx->eflags.Bits.u1VM = 0;
2082
2083 /* Restore original IOPL setting as we always use 0. */
2084 pCtx->eflags.Bits.u2IOPL = pVCpu->hwaccm.s.vmx.RealMode.eflags.Bits.u2IOPL;
2085
2086 /* Force a TR resync every time in case we switch modes. */
2087 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_GUEST_TR;
2088 }
2089 else
2090 {
2091 /* In real mode we have a fake TSS, so only sync it back when it's supposed to be valid. */
2092 VMX_READ_SELREG(TR, tr);
2093 }
2094
2095#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
2096 /* Save the possibly changed MSRs that we automatically restore and save during a world switch. */
2097 for (unsigned i = 0; i < pVCpu->hwaccm.s.vmx.cCachedMSRs; i++)
2098 {
2099 PVMXMSR pMsr = (PVMXMSR)pVCpu->hwaccm.s.vmx.pGuestMSR;
2100 pMsr += i;
2101
2102 switch (pMsr->u32IndexMSR)
2103 {
2104 case MSR_K8_LSTAR:
2105 pCtx->msrLSTAR = pMsr->u64Value;
2106 break;
2107 case MSR_K6_STAR:
2108 pCtx->msrSTAR = pMsr->u64Value;
2109 break;
2110 case MSR_K8_SF_MASK:
2111 pCtx->msrSFMASK = pMsr->u64Value;
2112 break;
2113 case MSR_K8_KERNEL_GS_BASE:
2114 pCtx->msrKERNELGSBASE = pMsr->u64Value;
2115 break;
2116 case MSR_K6_EFER:
2117 /* EFER can't be changed without causing a VM-exit. */
2118// Assert(pCtx->msrEFER == pMsr->u64Value);
2119 break;
2120 default:
2121 AssertFailed();
2122 return VERR_HM_UNEXPECTED_LD_ST_MSR;
2123 }
2124 }
2125#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
2126 return VINF_SUCCESS;
2127}
2128
2129/**
2130 * Dummy placeholder
2131 *
2132 * @param pVM The VM to operate on.
2133 * @param pVCpu The VMCPU to operate on.
2134 */
2135static DECLCALLBACK(void) hmR0VmxSetupTLBDummy(PVM pVM, PVMCPU pVCpu)
2136{
2137 NOREF(pVM);
2138 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
2139 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
2140 pVCpu->hwaccm.s.TlbShootdown.cPages = 0;
2141 return;
2142}
2143
2144/**
2145 * Setup the tagged TLB for EPT
2146 *
2147 * @returns VBox status code.
2148 * @param pVM The VM to operate on.
2149 * @param pVCpu The VMCPU to operate on.
2150 */
2151static DECLCALLBACK(void) hmR0VmxSetupTLBEPT(PVM pVM, PVMCPU pVCpu)
2152{
2153 PHMGLOBLCPUINFO pCpu;
2154
2155 Assert(pVM->hwaccm.s.fNestedPaging);
2156 Assert(!pVM->hwaccm.s.vmx.fVPID);
2157
2158 /* Deal with tagged TLBs if VPID or EPT is supported. */
2159 pCpu = HWACCMR0GetCurrentCpu();
2160 /* Force a TLB flush for the first world switch if the current cpu differs from the one we ran on last. */
2161 /* Note that this can happen both for start and resume due to long jumps back to ring 3. */
2162 if ( pVCpu->hwaccm.s.idLastCpu != pCpu->idCpu
2163 /* if the tlb flush count has changed, another VM has flushed the TLB of this cpu, so we can't use our current ASID anymore. */
2164 || pVCpu->hwaccm.s.cTLBFlushes != pCpu->cTLBFlushes)
2165 {
2166 /* Force a TLB flush on VM entry. */
2167 pVCpu->hwaccm.s.fForceTLBFlush = true;
2168 }
2169 /* Disabled because this has triggered every time I have suspended my
2170 * laptop with a VM running for the past three months or more. */
2171 // else
2172 // Assert(!pCpu->fFlushTLB);
2173
2174 /* Check for tlb shootdown flushes. */
2175 if (VMCPU_FF_TESTANDCLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2176 pVCpu->hwaccm.s.fForceTLBFlush = true;
2177
2178 pVCpu->hwaccm.s.idLastCpu = pCpu->idCpu;
2179 pCpu->fFlushTLB = false;
2180
2181 if (pVCpu->hwaccm.s.fForceTLBFlush)
2182 {
2183 hmR0VmxFlushEPT(pVM, pVCpu, pVM->hwaccm.s.vmx.enmFlushContext, 0);
2184 }
2185 else
2186 if (VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
2187 {
2188 /* Deal with pending TLB shootdown actions which were queued when we were not executing code. */
2189 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatTlbShootdown);
2190
2191 for (unsigned i=0;i<pVCpu->hwaccm.s.TlbShootdown.cPages;i++)
2192 {
2193 /* aTlbShootdownPages contains physical addresses in this case. */
2194 hmR0VmxFlushEPT(pVM, pVCpu, pVM->hwaccm.s.vmx.enmFlushPage, pVCpu->hwaccm.s.TlbShootdown.aPages[i]);
2195 }
2196 }
2197 pVCpu->hwaccm.s.TlbShootdown.cPages= 0;
2198 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
2199
2200#ifdef VBOX_WITH_STATISTICS
2201 if (pVCpu->hwaccm.s.fForceTLBFlush)
2202 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatFlushTLBWorldSwitch);
2203 else
2204 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatNoFlushTLBWorldSwitch);
2205#endif
2206}
2207
2208#ifdef HWACCM_VTX_WITH_VPID
2209/**
2210 * Setup the tagged TLB for VPID
2211 *
2212 * @returns VBox status code.
2213 * @param pVM The VM to operate on.
2214 * @param pVCpu The VMCPU to operate on.
2215 */
2216static DECLCALLBACK(void) hmR0VmxSetupTLBVPID(PVM pVM, PVMCPU pVCpu)
2217{
2218 PHMGLOBLCPUINFO pCpu;
2219
2220 Assert(pVM->hwaccm.s.vmx.fVPID);
2221 Assert(!pVM->hwaccm.s.fNestedPaging);
2222
2223 /* Deal with tagged TLBs if VPID or EPT is supported. */
2224 pCpu = HWACCMR0GetCurrentCpu();
2225 /* Force a TLB flush for the first world switch if the current cpu differs from the one we ran on last. */
2226 /* Note that this can happen both for start and resume due to long jumps back to ring 3. */
2227 if ( pVCpu->hwaccm.s.idLastCpu != pCpu->idCpu
2228 /* if the tlb flush count has changed, another VM has flushed the TLB of this cpu, so we can't use our current ASID anymore. */
2229 || pVCpu->hwaccm.s.cTLBFlushes != pCpu->cTLBFlushes)
2230 {
2231 /* Force a TLB flush on VM entry. */
2232 pVCpu->hwaccm.s.fForceTLBFlush = true;
2233 }
2234 else
2235 Assert(!pCpu->fFlushTLB);
2236
2237 pVCpu->hwaccm.s.idLastCpu = pCpu->idCpu;
2238
2239 /* Check for tlb shootdown flushes. */
2240 if (VMCPU_FF_TESTANDCLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2241 pVCpu->hwaccm.s.fForceTLBFlush = true;
2242
2243 /* Make sure we flush the TLB when required. Switch ASID to achieve the same thing, but without actually flushing the whole TLB (which is expensive). */
2244 if (pVCpu->hwaccm.s.fForceTLBFlush)
2245 {
2246 if ( ++pCpu->uCurrentASID >= pVM->hwaccm.s.uMaxASID
2247 || pCpu->fFlushTLB)
2248 {
2249 pCpu->fFlushTLB = false;
2250 pCpu->uCurrentASID = 1; /* start at 1; host uses 0 */
2251 pCpu->cTLBFlushes++;
2252 hmR0VmxFlushVPID(pVM, pVCpu, VMX_FLUSH_ALL_CONTEXTS, 0);
2253 }
2254 else
2255 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatFlushASID);
2256
2257 pVCpu->hwaccm.s.fForceTLBFlush = false;
2258 pVCpu->hwaccm.s.cTLBFlushes = pCpu->cTLBFlushes;
2259 pVCpu->hwaccm.s.uCurrentASID = pCpu->uCurrentASID;
2260 }
2261 else
2262 {
2263 Assert(!pCpu->fFlushTLB);
2264 Assert(pVCpu->hwaccm.s.uCurrentASID && pCpu->uCurrentASID);
2265
2266 if (VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
2267 {
2268 /* Deal with pending TLB shootdown actions which were queued when we were not executing code. */
2269 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatTlbShootdown);
2270 for (unsigned i = 0; i < pVCpu->hwaccm.s.TlbShootdown.cPages; i++)
2271 hmR0VmxFlushVPID(pVM, pVCpu, pVM->hwaccm.s.vmx.enmFlushPage, pVCpu->hwaccm.s.TlbShootdown.aPages[i]);
2272 }
2273 }
2274 pVCpu->hwaccm.s.TlbShootdown.cPages = 0;
2275 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
2276
2277 AssertMsg(pVCpu->hwaccm.s.cTLBFlushes == pCpu->cTLBFlushes, ("Flush count mismatch for cpu %d (%x vs %x)\n", pCpu->idCpu, pVCpu->hwaccm.s.cTLBFlushes, pCpu->cTLBFlushes));
2278 AssertMsg(pCpu->uCurrentASID >= 1 && pCpu->uCurrentASID < pVM->hwaccm.s.uMaxASID, ("cpu%d uCurrentASID = %x\n", pCpu->idCpu, pCpu->uCurrentASID));
2279 AssertMsg(pVCpu->hwaccm.s.uCurrentASID >= 1 && pVCpu->hwaccm.s.uCurrentASID < pVM->hwaccm.s.uMaxASID, ("cpu%d VM uCurrentASID = %x\n", pCpu->idCpu, pVCpu->hwaccm.s.uCurrentASID));
2280
2281 int rc = VMXWriteVMCS(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hwaccm.s.uCurrentASID);
2282 AssertRC(rc);
2283
2284 if (pVCpu->hwaccm.s.fForceTLBFlush)
2285 hmR0VmxFlushVPID(pVM, pVCpu, pVM->hwaccm.s.vmx.enmFlushContext, 0);
2286
2287# ifdef VBOX_WITH_STATISTICS
2288 if (pVCpu->hwaccm.s.fForceTLBFlush)
2289 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatFlushTLBWorldSwitch);
2290 else
2291 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatNoFlushTLBWorldSwitch);
2292# endif
2293}
2294#endif /* HWACCM_VTX_WITH_VPID */
2295
2296/**
2297 * Runs guest code in a VT-x VM.
2298 *
2299 * @returns VBox status code.
2300 * @param pVM The VM to operate on.
2301 * @param pVCpu The VMCPU to operate on.
2302 * @param pCtx Guest context
2303 */
2304VMMR0DECL(int) VMXR0RunGuestCode(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
2305{
2306 STAM_PROFILE_ADV_START(&pVCpu->hwaccm.s.StatEntry, x);
2307 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hwaccm.s.StatExit1);
2308 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hwaccm.s.StatExit2);
2309
2310 VBOXSTRICTRC rc = VINF_SUCCESS;
2311 int rc2;
2312 RTGCUINTREG val;
2313 RTGCUINTREG exitReason = (RTGCUINTREG)VMX_EXIT_INVALID;
2314 RTGCUINTREG instrError, cbInstr;
2315 RTGCUINTPTR exitQualification = 0;
2316 RTGCUINTPTR intInfo = 0; /* shut up buggy gcc 4 */
2317 RTGCUINTPTR errCode, instrInfo;
2318 bool fSetupTPRCaching = false;
2319 uint64_t u64OldLSTAR = 0;
2320 uint8_t u8LastTPR = 0;
2321 RTCCUINTREG uOldEFlags = ~(RTCCUINTREG)0;
2322 unsigned cResume = 0;
2323#ifdef VBOX_STRICT
2324 RTCPUID idCpuCheck;
2325 bool fWasInLongMode = false;
2326#endif
2327#ifdef VBOX_HIGH_RES_TIMERS_HACK_IN_RING0
2328 uint64_t u64LastTime = RTTimeMilliTS();
2329#endif
2330
2331 Assert(!(pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC) || (pVCpu->hwaccm.s.vmx.pbVAPIC && pVM->hwaccm.s.vmx.pAPIC));
2332
2333 /* Check if we need to use TPR shadowing. */
2334 if ( CPUMIsGuestInLongModeEx(pCtx)
2335 || ( ((pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC) || pVM->hwaccm.s.fTRPPatchingAllowed)
2336 && pVM->hwaccm.s.fHasIoApic)
2337 )
2338 {
2339 fSetupTPRCaching = true;
2340 }
2341
2342 Log2(("\nE"));
2343
2344#ifdef VBOX_STRICT
2345 {
2346 RTCCUINTREG val2;
2347
2348 rc2 = VMXReadVMCS(VMX_VMCS_CTRL_PIN_EXEC_CONTROLS, &val2);
2349 AssertRC(rc2);
2350 Log2(("VMX_VMCS_CTRL_PIN_EXEC_CONTROLS = %08x\n", val2));
2351
2352 /* allowed zero */
2353 if ((val2 & pVM->hwaccm.s.vmx.msr.vmx_pin_ctls.n.disallowed0) != pVM->hwaccm.s.vmx.msr.vmx_pin_ctls.n.disallowed0)
2354 Log(("Invalid VMX_VMCS_CTRL_PIN_EXEC_CONTROLS: zero\n"));
2355
2356 /* allowed one */
2357 if ((val2 & ~pVM->hwaccm.s.vmx.msr.vmx_pin_ctls.n.allowed1) != 0)
2358 Log(("Invalid VMX_VMCS_CTRL_PIN_EXEC_CONTROLS: one\n"));
2359
2360 rc2 = VMXReadVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, &val2);
2361 AssertRC(rc2);
2362 Log2(("VMX_VMCS_CTRL_PROC_EXEC_CONTROLS = %08x\n", val2));
2363
2364 /* Must be set according to the MSR, but can be cleared in case of EPT. */
2365 if (pVM->hwaccm.s.fNestedPaging)
2366 val2 |= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INVLPG_EXIT
2367 | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
2368 | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT;
2369
2370 /* allowed zero */
2371 if ((val2 & pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.disallowed0) != pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.disallowed0)
2372 Log(("Invalid VMX_VMCS_CTRL_PROC_EXEC_CONTROLS: zero\n"));
2373
2374 /* allowed one */
2375 if ((val2 & ~pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1) != 0)
2376 Log(("Invalid VMX_VMCS_CTRL_PROC_EXEC_CONTROLS: one\n"));
2377
2378 rc2 = VMXReadVMCS(VMX_VMCS_CTRL_ENTRY_CONTROLS, &val2);
2379 AssertRC(rc2);
2380 Log2(("VMX_VMCS_CTRL_ENTRY_CONTROLS = %08x\n", val2));
2381
2382 /* allowed zero */
2383 if ((val2 & pVM->hwaccm.s.vmx.msr.vmx_entry.n.disallowed0) != pVM->hwaccm.s.vmx.msr.vmx_entry.n.disallowed0)
2384 Log(("Invalid VMX_VMCS_CTRL_ENTRY_CONTROLS: zero\n"));
2385
2386 /* allowed one */
2387 if ((val2 & ~pVM->hwaccm.s.vmx.msr.vmx_entry.n.allowed1) != 0)
2388 Log(("Invalid VMX_VMCS_CTRL_ENTRY_CONTROLS: one\n"));
2389
2390 rc2 = VMXReadVMCS(VMX_VMCS_CTRL_EXIT_CONTROLS, &val2);
2391 AssertRC(rc2);
2392 Log2(("VMX_VMCS_CTRL_EXIT_CONTROLS = %08x\n", val2));
2393
2394 /* allowed zero */
2395 if ((val2 & pVM->hwaccm.s.vmx.msr.vmx_exit.n.disallowed0) != pVM->hwaccm.s.vmx.msr.vmx_exit.n.disallowed0)
2396 Log(("Invalid VMX_VMCS_CTRL_EXIT_CONTROLS: zero\n"));
2397
2398 /* allowed one */
2399 if ((val2 & ~pVM->hwaccm.s.vmx.msr.vmx_exit.n.allowed1) != 0)
2400 Log(("Invalid VMX_VMCS_CTRL_EXIT_CONTROLS: one\n"));
2401 }
2402 fWasInLongMode = CPUMIsGuestInLongModeEx(pCtx);
2403#endif /* VBOX_STRICT */
2404
2405#ifdef VBOX_WITH_CRASHDUMP_MAGIC
2406 pVCpu->hwaccm.s.vmx.VMCSCache.u64TimeEntry = RTTimeNanoTS();
2407#endif
2408
2409 /* We can jump to this point to resume execution after determining that a VM-exit is innocent.
2410 */
2411ResumeExecution:
2412 if (!STAM_REL_PROFILE_ADV_IS_RUNNING(&pVCpu->hwaccm.s.StatEntry))
2413 STAM_REL_PROFILE_ADV_STOP_START(&pVCpu->hwaccm.s.StatExit2, &pVCpu->hwaccm.s.StatEntry, x);
2414 AssertMsg(pVCpu->hwaccm.s.idEnteredCpu == RTMpCpuId(),
2415 ("Expected %d, I'm %d; cResume=%d exitReason=%RGv exitQualification=%RGv\n",
2416 (int)pVCpu->hwaccm.s.idEnteredCpu, (int)RTMpCpuId(), cResume, exitReason, exitQualification));
2417 Assert(!HWACCMR0SuspendPending());
2418 /* Not allowed to switch modes without reloading the host state (32->64 switcher)!! */
2419 Assert(fWasInLongMode == CPUMIsGuestInLongModeEx(pCtx));
2420
2421 /* Safety precaution; looping for too long here can have a very bad effect on the host */
2422 if (RT_UNLIKELY(++cResume > pVM->hwaccm.s.cMaxResumeLoops))
2423 {
2424 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitMaxResume);
2425 rc = VINF_EM_RAW_INTERRUPT;
2426 goto end;
2427 }
2428
2429 /* Check for irq inhibition due to instruction fusing (sti, mov ss). */
2430 if (VMCPU_FF_ISSET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
2431 {
2432 Log(("VM_FF_INHIBIT_INTERRUPTS at %RGv successor %RGv\n", (RTGCPTR)pCtx->rip, EMGetInhibitInterruptsPC(pVCpu)));
2433 if (pCtx->rip != EMGetInhibitInterruptsPC(pVCpu))
2434 {
2435 /* Note: we intentionally don't clear VM_FF_INHIBIT_INTERRUPTS here.
2436 * Before we are able to execute this instruction in raw mode (iret to guest code) an external interrupt might
2437 * force a world switch again. Possibly allowing a guest interrupt to be dispatched in the process. This could
2438 * break the guest. Sounds very unlikely, but such timing sensitive problems are not as rare as you might think.
2439 */
2440 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
2441 /* Irq inhibition is no longer active; clear the corresponding VMX state. */
2442 rc2 = VMXWriteVMCS(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, 0);
2443 AssertRC(rc2);
2444 }
2445 }
2446 else
2447 {
2448 /* Irq inhibition is no longer active; clear the corresponding VMX state. */
2449 rc2 = VMXWriteVMCS(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, 0);
2450 AssertRC(rc2);
2451 }
2452
2453#ifdef VBOX_HIGH_RES_TIMERS_HACK_IN_RING0
2454 if (RT_UNLIKELY((cResume & 0xf) == 0))
2455 {
2456 uint64_t u64CurTime = RTTimeMilliTS();
2457
2458 if (RT_UNLIKELY(u64CurTime > u64LastTime))
2459 {
2460 u64LastTime = u64CurTime;
2461 TMTimerPollVoid(pVM, pVCpu);
2462 }
2463 }
2464#endif
2465
2466 /* Check for pending actions that force us to go back to ring 3. */
2467 if ( VM_FF_ISPENDING(pVM, VM_FF_HWACCM_TO_R3_MASK | VM_FF_REQUEST | VM_FF_PGM_POOL_FLUSH_PENDING | VM_FF_PDM_DMA)
2468 || VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_HWACCM_TO_R3_MASK | VMCPU_FF_PGM_SYNC_CR3 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL | VMCPU_FF_REQUEST))
2469 {
2470 /* Check if a sync operation is pending. */
2471 if (VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
2472 {
2473 rc = PGMSyncCR3(pVCpu, pCtx->cr0, pCtx->cr3, pCtx->cr4, VMCPU_FF_ISSET(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
2474 if (rc != VINF_SUCCESS)
2475 {
2476 AssertRC(VBOXSTRICTRC_VAL(rc));
2477 Log(("Pending pool sync is forcing us back to ring 3; rc=%d\n", VBOXSTRICTRC_VAL(rc)));
2478 goto end;
2479 }
2480 }
2481
2482#ifdef DEBUG
2483 /* Intercept X86_XCPT_DB if stepping is enabled */
2484 if (!DBGFIsStepping(pVCpu))
2485#endif
2486 {
2487 if ( VM_FF_ISPENDING(pVM, VM_FF_HWACCM_TO_R3_MASK)
2488 || VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_HWACCM_TO_R3_MASK))
2489 {
2490 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatSwitchToR3);
2491 rc = RT_UNLIKELY(VM_FF_ISPENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
2492 goto end;
2493 }
2494 }
2495
2496 /* Pending request packets might contain actions that need immediate attention, such as pending hardware interrupts. */
2497 if ( VM_FF_ISPENDING(pVM, VM_FF_REQUEST)
2498 || VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_REQUEST))
2499 {
2500 rc = VINF_EM_PENDING_REQUEST;
2501 goto end;
2502 }
2503
2504 /* Check if a pgm pool flush is in progress. */
2505 if (VM_FF_ISPENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
2506 {
2507 rc = VINF_PGM_POOL_FLUSH_PENDING;
2508 goto end;
2509 }
2510
2511 /* Check if DMA work is pending (2nd+ run). */
2512 if (VM_FF_ISPENDING(pVM, VM_FF_PDM_DMA) && cResume > 1)
2513 {
2514 rc = VINF_EM_RAW_TO_R3;
2515 goto end;
2516 }
2517 }
2518
2519#ifdef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
2520 /*
2521 * Exit to ring-3 preemption/work is pending.
2522 *
2523 * Interrupts are disabled before the call to make sure we don't miss any interrupt
2524 * that would flag preemption (IPI, timer tick, ++). (Would've been nice to do this
2525 * further down, but hmR0VmxCheckPendingInterrupt makes that impossible.)
2526 *
2527 * Note! Interrupts must be disabled done *before* we check for TLB flushes; TLB
2528 * shootdowns rely on this.
2529 */
2530 uOldEFlags = ASMIntDisableFlags();
2531 if (RTThreadPreemptIsPending(NIL_RTTHREAD))
2532 {
2533 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitPreemptPending);
2534 rc = VINF_EM_RAW_INTERRUPT;
2535 goto end;
2536 }
2537 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
2538#endif
2539
2540 /* When external interrupts are pending, we should exit the VM when IF is set. */
2541 /* Note! *After* VM_FF_INHIBIT_INTERRUPTS check!!! */
2542 rc = hmR0VmxCheckPendingInterrupt(pVM, pVCpu, pCtx);
2543 if (RT_FAILURE(rc))
2544 goto end;
2545
2546 /** @todo check timers?? */
2547
2548 /* TPR caching using CR8 is only available in 64 bits mode */
2549 /* Note the 32 bits exception for AMD (X86_CPUID_AMD_FEATURE_ECX_CR8L), but that appears missing in Intel CPUs */
2550 /* Note: we can't do this in LoadGuestState as PDMApicGetTPR can jump back to ring 3 (lock)!!!!! (no longer true) */
2551 /**
2552 * @todo query and update the TPR only when it could have been changed (mmio access & wrmsr (x2apic))
2553 */
2554 if (fSetupTPRCaching)
2555 {
2556 /* TPR caching in CR8 */
2557 bool fPending;
2558
2559 rc2 = PDMApicGetTPR(pVCpu, &u8LastTPR, &fPending);
2560 AssertRC(rc2);
2561 /* The TPR can be found at offset 0x80 in the APIC mmio page. */
2562 pVCpu->hwaccm.s.vmx.pbVAPIC[0x80] = u8LastTPR;
2563
2564 /* Two options here:
2565 * - external interrupt pending, but masked by the TPR value.
2566 * -> a CR8 update that lower the current TPR value should cause an exit
2567 * - no pending interrupts
2568 * -> We don't need to be explicitely notified. There are enough world switches for detecting pending interrupts.
2569 */
2570 rc = VMXWriteVMCS(VMX_VMCS_CTRL_TPR_THRESHOLD, (fPending) ? (u8LastTPR >> 4) : 0); /* cr8 bits 3-0 correspond to bits 7-4 of the task priority mmio register. */
2571 AssertRC(VBOXSTRICTRC_VAL(rc));
2572
2573 if (pVM->hwaccm.s.fTPRPatchingActive)
2574 {
2575 Assert(!CPUMIsGuestInLongModeEx(pCtx));
2576 /* Our patch code uses LSTAR for TPR caching. */
2577 pCtx->msrLSTAR = u8LastTPR;
2578
2579 if (fPending)
2580 {
2581 /* A TPR change could activate a pending interrupt, so catch lstar writes. */
2582 hmR0VmxSetMSRPermission(pVCpu, MSR_K8_LSTAR, true, false);
2583 }
2584 else
2585 {
2586 /* No interrupts are pending, so we don't need to be explicitely notified.
2587 * There are enough world switches for detecting pending interrupts.
2588 */
2589 hmR0VmxSetMSRPermission(pVCpu, MSR_K8_LSTAR, true, true);
2590 }
2591 }
2592 }
2593
2594#if defined(HWACCM_VTX_WITH_EPT) && defined(LOG_ENABLED)
2595 if ( pVM->hwaccm.s.fNestedPaging
2596# ifdef HWACCM_VTX_WITH_VPID
2597 || pVM->hwaccm.s.vmx.fVPID
2598# endif /* HWACCM_VTX_WITH_VPID */
2599 )
2600 {
2601 PHMGLOBLCPUINFO pCpu;
2602
2603 pCpu = HWACCMR0GetCurrentCpu();
2604 if ( pVCpu->hwaccm.s.idLastCpu != pCpu->idCpu
2605 || pVCpu->hwaccm.s.cTLBFlushes != pCpu->cTLBFlushes)
2606 {
2607 if (pVCpu->hwaccm.s.idLastCpu != pCpu->idCpu)
2608 LogFlow(("Force TLB flush due to rescheduling to a different cpu (%d vs %d)\n", pVCpu->hwaccm.s.idLastCpu, pCpu->idCpu));
2609 else
2610 LogFlow(("Force TLB flush due to changed TLB flush count (%x vs %x)\n", pVCpu->hwaccm.s.cTLBFlushes, pCpu->cTLBFlushes));
2611 }
2612 if (pCpu->fFlushTLB)
2613 LogFlow(("Force TLB flush: first time cpu %d is used -> flush\n", pCpu->idCpu));
2614 else
2615 if (pVCpu->hwaccm.s.fForceTLBFlush)
2616 LogFlow(("Manual TLB flush\n"));
2617 }
2618#endif
2619#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
2620 PGMRZDynMapFlushAutoSet(pVCpu);
2621#endif
2622
2623 /*
2624 * NOTE: DO NOT DO ANYTHING AFTER THIS POINT THAT MIGHT JUMP BACK TO RING 3!
2625 * (until the actual world switch)
2626 */
2627#ifdef VBOX_STRICT
2628 idCpuCheck = RTMpCpuId();
2629#endif
2630#ifdef LOG_ENABLED
2631 VMMR0LogFlushDisable(pVCpu);
2632#endif
2633 /* Save the host state first. */
2634 if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_HOST_CONTEXT)
2635 {
2636 rc = VMXR0SaveHostState(pVM, pVCpu);
2637 if (RT_UNLIKELY(rc != VINF_SUCCESS))
2638 {
2639 VMMR0LogFlushEnable(pVCpu);
2640 goto end;
2641 }
2642 }
2643
2644 /* Load the guest state */
2645 if (!pVCpu->hwaccm.s.fContextUseFlags)
2646 {
2647 VMXR0LoadMinimalGuestState(pVM, pVCpu, pCtx);
2648 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatLoadMinimal);
2649 }
2650 else
2651 {
2652 rc = VMXR0LoadGuestState(pVM, pVCpu, pCtx);
2653 if (RT_UNLIKELY(rc != VINF_SUCCESS))
2654 {
2655 VMMR0LogFlushEnable(pVCpu);
2656 goto end;
2657 }
2658 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatLoadFull);
2659 }
2660
2661#ifndef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
2662 /* Disable interrupts to make sure a poke will interrupt execution.
2663 * This must be done *before* we check for TLB flushes; TLB shootdowns rely on this.
2664 */
2665 uOldEFlags = ASMIntDisableFlags();
2666 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
2667#endif
2668
2669 /* Non-register state Guest Context */
2670 /** @todo change me according to cpu state */
2671 rc2 = VMXWriteVMCS(VMX_VMCS32_GUEST_ACTIVITY_STATE, VMX_CMS_GUEST_ACTIVITY_ACTIVE);
2672 AssertRC(rc2);
2673
2674 /* Set TLB flush state as checked until we return from the world switch. */
2675 ASMAtomicWriteBool(&pVCpu->hwaccm.s.fCheckedTLBFlush, true);
2676 /* Deal with tagged TLB setup and invalidation. */
2677 pVM->hwaccm.s.vmx.pfnSetupTaggedTLB(pVM, pVCpu);
2678
2679 /* Manual save and restore:
2680 * - General purpose registers except RIP, RSP
2681 *
2682 * Trashed:
2683 * - CR2 (we don't care)
2684 * - LDTR (reset to 0)
2685 * - DRx (presumably not changed at all)
2686 * - DR7 (reset to 0x400)
2687 * - EFLAGS (reset to RT_BIT(1); not relevant)
2688 *
2689 */
2690
2691 /* All done! Let's start VM execution. */
2692 STAM_PROFILE_ADV_STOP_START(&pVCpu->hwaccm.s.StatEntry, &pVCpu->hwaccm.s.StatInGC, x);
2693 Assert(idCpuCheck == RTMpCpuId());
2694
2695#ifdef VBOX_WITH_CRASHDUMP_MAGIC
2696 pVCpu->hwaccm.s.vmx.VMCSCache.cResume = cResume;
2697 pVCpu->hwaccm.s.vmx.VMCSCache.u64TimeSwitch = RTTimeNanoTS();
2698#endif
2699
2700 /* Save the current TPR value in the LSTAR msr so our patches can access it. */
2701 if (pVM->hwaccm.s.fTPRPatchingActive)
2702 {
2703 Assert(pVM->hwaccm.s.fTPRPatchingActive);
2704 u64OldLSTAR = ASMRdMsr(MSR_K8_LSTAR);
2705 ASMWrMsr(MSR_K8_LSTAR, u8LastTPR);
2706 }
2707
2708 TMNotifyStartOfExecution(pVCpu);
2709#ifdef VBOX_WITH_KERNEL_USING_XMM
2710 rc = hwaccmR0VMXStartVMWrapXMM(pVCpu->hwaccm.s.fResumeVM, pCtx, &pVCpu->hwaccm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hwaccm.s.vmx.pfnStartVM);
2711#else
2712 rc = pVCpu->hwaccm.s.vmx.pfnStartVM(pVCpu->hwaccm.s.fResumeVM, pCtx, &pVCpu->hwaccm.s.vmx.VMCSCache, pVM, pVCpu);
2713#endif
2714 ASMAtomicWriteBool(&pVCpu->hwaccm.s.fCheckedTLBFlush, false);
2715 ASMAtomicIncU32(&pVCpu->hwaccm.s.cWorldSwitchExits);
2716 /* Possibly the last TSC value seen by the guest (too high) (only when we're in tsc offset mode). */
2717 if (!(pVCpu->hwaccm.s.vmx.proc_ctls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT))
2718 TMCpuTickSetLastSeen(pVCpu, ASMReadTSC() + pVCpu->hwaccm.s.vmx.u64TSCOffset - 0x400 /* guestimate of world switch overhead in clock ticks */);
2719
2720 TMNotifyEndOfExecution(pVCpu);
2721 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED);
2722 Assert(!(ASMGetFlags() & X86_EFL_IF));
2723
2724 /* Restore the host LSTAR msr if the guest could have changed it. */
2725 if (pVM->hwaccm.s.fTPRPatchingActive)
2726 {
2727 Assert(pVM->hwaccm.s.fTPRPatchingActive);
2728 pVCpu->hwaccm.s.vmx.pbVAPIC[0x80] = pCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR);
2729 ASMWrMsr(MSR_K8_LSTAR, u64OldLSTAR);
2730 }
2731
2732 STAM_PROFILE_ADV_STOP_START(&pVCpu->hwaccm.s.StatInGC, &pVCpu->hwaccm.s.StatExit1, x);
2733 ASMSetFlags(uOldEFlags);
2734#ifdef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
2735 uOldEFlags = ~(RTCCUINTREG)0;
2736#endif
2737
2738 AssertMsg(!pVCpu->hwaccm.s.vmx.VMCSCache.Write.cValidEntries, ("pVCpu->hwaccm.s.vmx.VMCSCache.Write.cValidEntries=%d\n", pVCpu->hwaccm.s.vmx.VMCSCache.Write.cValidEntries));
2739
2740 /* In case we execute a goto ResumeExecution later on. */
2741 pVCpu->hwaccm.s.fResumeVM = true;
2742 pVCpu->hwaccm.s.fForceTLBFlush = false;
2743
2744 /*
2745 * !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
2746 * IMPORTANT: WE CAN'T DO ANY LOGGING OR OPERATIONS THAT CAN DO A LONGJMP BACK TO RING 3 *BEFORE* WE'VE SYNCED BACK (MOST OF) THE GUEST STATE
2747 * !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
2748 */
2749
2750 if (RT_UNLIKELY(rc != VINF_SUCCESS))
2751 {
2752 hmR0VmxReportWorldSwitchError(pVM, pVCpu, rc, pCtx);
2753 VMMR0LogFlushEnable(pVCpu);
2754 goto end;
2755 }
2756
2757 /* Success. Query the guest state and figure out what has happened. */
2758
2759 /* Investigate why there was a VM-exit. */
2760 rc2 = VMXReadCachedVMCS(VMX_VMCS32_RO_EXIT_REASON, &exitReason);
2761 STAM_COUNTER_INC(&pVCpu->hwaccm.s.paStatExitReasonR0[exitReason & MASK_EXITREASON_STAT]);
2762
2763 exitReason &= 0xffff; /* bit 0-15 contain the exit code. */
2764 rc2 |= VMXReadCachedVMCS(VMX_VMCS32_RO_VM_INSTR_ERROR, &instrError);
2765 rc2 |= VMXReadCachedVMCS(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &cbInstr);
2766 rc2 |= VMXReadCachedVMCS(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &intInfo);
2767 /* might not be valid; depends on VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID. */
2768 rc2 |= VMXReadCachedVMCS(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERRCODE, &errCode);
2769 rc2 |= VMXReadCachedVMCS(VMX_VMCS32_RO_EXIT_INSTR_INFO, &instrInfo);
2770 rc2 |= VMXReadCachedVMCS(VMX_VMCS_RO_EXIT_QUALIFICATION, &exitQualification);
2771 AssertRC(rc2);
2772
2773 /* Sync back the guest state */
2774 rc2 = VMXR0SaveGuestState(pVM, pVCpu, pCtx);
2775 AssertRC(rc2);
2776
2777 /* Note! NOW IT'S SAFE FOR LOGGING! */
2778 VMMR0LogFlushEnable(pVCpu);
2779 Log2(("Raw exit reason %08x\n", exitReason));
2780
2781 /* Check if an injected event was interrupted prematurely. */
2782 rc2 = VMXReadCachedVMCS(VMX_VMCS32_RO_IDT_INFO, &val);
2783 AssertRC(rc2);
2784 pVCpu->hwaccm.s.Event.intInfo = VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(val);
2785 if ( VMX_EXIT_INTERRUPTION_INFO_VALID(pVCpu->hwaccm.s.Event.intInfo)
2786 /* Ignore 'int xx' as they'll be restarted anyway. */
2787 && VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hwaccm.s.Event.intInfo) != VMX_EXIT_INTERRUPTION_INFO_TYPE_SW
2788 /* Ignore software exceptions (such as int3) as they'll reoccur when we restart the instruction anyway. */
2789 && VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hwaccm.s.Event.intInfo) != VMX_EXIT_INTERRUPTION_INFO_TYPE_SWEXCPT)
2790 {
2791 Assert(!pVCpu->hwaccm.s.Event.fPending);
2792 pVCpu->hwaccm.s.Event.fPending = true;
2793 /* Error code present? */
2794 if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(pVCpu->hwaccm.s.Event.intInfo))
2795 {
2796 rc2 = VMXReadCachedVMCS(VMX_VMCS32_RO_IDT_ERRCODE, &val);
2797 AssertRC(rc2);
2798 pVCpu->hwaccm.s.Event.errCode = val;
2799 Log(("Pending inject %RX64 at %RGv exit=%08x intInfo=%08x exitQualification=%RGv pending error=%RX64\n", pVCpu->hwaccm.s.Event.intInfo, (RTGCPTR)pCtx->rip, exitReason, intInfo, exitQualification, val));
2800 }
2801 else
2802 {
2803 Log(("Pending inject %RX64 at %RGv exit=%08x intInfo=%08x exitQualification=%RGv\n", pVCpu->hwaccm.s.Event.intInfo, (RTGCPTR)pCtx->rip, exitReason, intInfo, exitQualification));
2804 pVCpu->hwaccm.s.Event.errCode = 0;
2805 }
2806 }
2807#ifdef VBOX_STRICT
2808 else
2809 if ( VMX_EXIT_INTERRUPTION_INFO_VALID(pVCpu->hwaccm.s.Event.intInfo)
2810 /* Ignore software exceptions (such as int3) as they're reoccur when we restart the instruction anyway. */
2811 && VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hwaccm.s.Event.intInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_SWEXCPT)
2812 {
2813 Log(("Ignore pending inject %RX64 at %RGv exit=%08x intInfo=%08x exitQualification=%RGv\n", pVCpu->hwaccm.s.Event.intInfo, (RTGCPTR)pCtx->rip, exitReason, intInfo, exitQualification));
2814 }
2815
2816 if (exitReason == VMX_EXIT_ERR_INVALID_GUEST_STATE)
2817 HWACCMDumpRegs(pVM, pVCpu, pCtx);
2818#endif
2819
2820 Log2(("E%d: New EIP=%x:%RGv\n", (uint32_t)exitReason, pCtx->cs, (RTGCPTR)pCtx->rip));
2821 Log2(("Exit reason %d, exitQualification %RGv\n", (uint32_t)exitReason, exitQualification));
2822 Log2(("instrInfo=%d instrError=%d instr length=%d\n", (uint32_t)instrInfo, (uint32_t)instrError, (uint32_t)cbInstr));
2823 Log2(("Interruption error code %d\n", (uint32_t)errCode));
2824 Log2(("IntInfo = %08x\n", (uint32_t)intInfo));
2825
2826 /* Sync back the TPR if it was changed. */
2827 if ( fSetupTPRCaching
2828 && u8LastTPR != pVCpu->hwaccm.s.vmx.pbVAPIC[0x80])
2829 {
2830 rc2 = PDMApicSetTPR(pVCpu, pVCpu->hwaccm.s.vmx.pbVAPIC[0x80]);
2831 AssertRC(rc2);
2832 }
2833
2834#ifdef DBGFTRACE_ENABLED /** @todo DTrace later. */
2835 RTTraceBufAddMsgF(pVM->CTX_SUFF(hTraceBuf), "vmexit %08x %016RX64 at %04:%08RX64 %RX64",
2836 exitReason, (uint64_t)exitQualification, pCtx->cs, pCtx->rip, (uint64_t)intInfo);
2837#endif
2838 STAM_PROFILE_ADV_STOP_START(&pVCpu->hwaccm.s.StatExit1, &pVCpu->hwaccm.s.StatExit2, x);
2839
2840 /* Some cases don't need a complete resync of the guest CPU state; handle them here. */
2841 Assert(rc == VINF_SUCCESS); /* might consider VERR_IPE_UNINITIALIZED_STATUS here later... */
2842 switch (exitReason)
2843 {
2844 case VMX_EXIT_EXCEPTION: /* 0 Exception or non-maskable interrupt (NMI). */
2845 case VMX_EXIT_EXTERNAL_IRQ: /* 1 External interrupt. */
2846 {
2847 uint32_t vector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(intInfo);
2848
2849 if (!VMX_EXIT_INTERRUPTION_INFO_VALID(intInfo))
2850 {
2851 Assert(exitReason == VMX_EXIT_EXTERNAL_IRQ);
2852#if 0 //def VBOX_WITH_VMMR0_DISABLE_PREEMPTION
2853 if ( RTThreadPreemptIsPendingTrusty()
2854 && !RTThreadPreemptIsPending(NIL_RTTHREAD))
2855 goto ResumeExecution;
2856#endif
2857 /* External interrupt; leave to allow it to be dispatched again. */
2858 rc = VINF_EM_RAW_INTERRUPT;
2859 break;
2860 }
2861 STAM_PROFILE_ADV_START(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
2862 switch (VMX_EXIT_INTERRUPTION_INFO_TYPE(intInfo))
2863 {
2864 case VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI: /* Non-maskable interrupt. */
2865 /* External interrupt; leave to allow it to be dispatched again. */
2866 rc = VINF_EM_RAW_INTERRUPT;
2867 break;
2868
2869 case VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT: /* External hardware interrupt. */
2870 AssertFailed(); /* can't come here; fails the first check. */
2871 break;
2872
2873 case VMX_EXIT_INTERRUPTION_INFO_TYPE_DBEXCPT: /* Unknown why we get this type for #DB */
2874 case VMX_EXIT_INTERRUPTION_INFO_TYPE_SWEXCPT: /* Software exception. (#BP or #OF) */
2875 Assert(vector == 1 || vector == 3 || vector == 4);
2876 /* no break */
2877 case VMX_EXIT_INTERRUPTION_INFO_TYPE_HWEXCPT: /* Hardware exception. */
2878 Log2(("Hardware/software interrupt %d\n", vector));
2879 switch (vector)
2880 {
2881 case X86_XCPT_NM:
2882 {
2883 Log(("#NM fault at %RGv error code %x\n", (RTGCPTR)pCtx->rip, errCode));
2884
2885 /** @todo don't intercept #NM exceptions anymore when we've activated the guest FPU state. */
2886 /* If we sync the FPU/XMM state on-demand, then we can continue execution as if nothing has happened. */
2887 rc = CPUMR0LoadGuestFPU(pVM, pVCpu, pCtx);
2888 if (rc == VINF_SUCCESS)
2889 {
2890 Assert(CPUMIsGuestFPUStateActive(pVCpu));
2891
2892 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitShadowNM);
2893
2894 /* Continue execution. */
2895 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_GUEST_CR0;
2896
2897 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
2898 goto ResumeExecution;
2899 }
2900
2901 Log(("Forward #NM fault to the guest\n"));
2902 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestNM);
2903 rc2 = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), cbInstr, 0);
2904 AssertRC(rc2);
2905 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
2906 goto ResumeExecution;
2907 }
2908
2909 case X86_XCPT_PF: /* Page fault */
2910 {
2911#ifdef DEBUG
2912 if (pVM->hwaccm.s.fNestedPaging)
2913 { /* A genuine pagefault.
2914 * Forward the trap to the guest by injecting the exception and resuming execution.
2915 */
2916 Log(("Guest page fault at %RGv cr2=%RGv error code %RGv rsp=%RGv\n", (RTGCPTR)pCtx->rip, exitQualification, errCode, (RTGCPTR)pCtx->rsp));
2917
2918 Assert(CPUMIsGuestInPagedProtectedModeEx(pCtx));
2919
2920 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestPF);
2921
2922 /* Now we must update CR2. */
2923 pCtx->cr2 = exitQualification;
2924 rc2 = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), cbInstr, errCode);
2925 AssertRC(rc2);
2926
2927 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
2928 goto ResumeExecution;
2929 }
2930#endif
2931 Assert(!pVM->hwaccm.s.fNestedPaging);
2932
2933#ifdef VBOX_HWACCM_WITH_GUEST_PATCHING
2934 /* Shortcut for APIC TPR reads and writes; 32 bits guests only */
2935 if ( pVM->hwaccm.s.fTRPPatchingAllowed
2936 && pVM->hwaccm.s.pGuestPatchMem
2937 && (exitQualification & 0xfff) == 0x080
2938 && !(errCode & X86_TRAP_PF_P) /* not present */
2939 && CPUMGetGuestCPL(pVCpu, CPUMCTX2CORE(pCtx)) == 0
2940 && !CPUMIsGuestInLongModeEx(pCtx)
2941 && pVM->hwaccm.s.cPatches < RT_ELEMENTS(pVM->hwaccm.s.aPatches))
2942 {
2943 RTGCPHYS GCPhysApicBase, GCPhys;
2944 PDMApicGetBase(pVM, &GCPhysApicBase); /* @todo cache this */
2945 GCPhysApicBase &= PAGE_BASE_GC_MASK;
2946
2947 rc = PGMGstGetPage(pVCpu, (RTGCPTR)exitQualification, NULL, &GCPhys);
2948 if ( rc == VINF_SUCCESS
2949 && GCPhys == GCPhysApicBase)
2950 {
2951 /* Only attempt to patch the instruction once. */
2952 PHWACCMTPRPATCH pPatch = (PHWACCMTPRPATCH)RTAvloU32Get(&pVM->hwaccm.s.PatchTree, (AVLOU32KEY)pCtx->eip);
2953 if (!pPatch)
2954 {
2955 rc = VINF_EM_HWACCM_PATCH_TPR_INSTR;
2956 break;
2957 }
2958 }
2959 }
2960#endif
2961
2962 Log2(("Page fault at %RGv error code %x\n", exitQualification, errCode));
2963 /* Exit qualification contains the linear address of the page fault. */
2964 TRPMAssertTrap(pVCpu, X86_XCPT_PF, TRPM_TRAP);
2965 TRPMSetErrorCode(pVCpu, errCode);
2966 TRPMSetFaultAddress(pVCpu, exitQualification);
2967
2968 /* Shortcut for APIC TPR reads and writes. */
2969 if ( (exitQualification & 0xfff) == 0x080
2970 && !(errCode & X86_TRAP_PF_P) /* not present */
2971 && fSetupTPRCaching
2972 && (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC))
2973 {
2974 RTGCPHYS GCPhysApicBase, GCPhys;
2975 PDMApicGetBase(pVM, &GCPhysApicBase); /* @todo cache this */
2976 GCPhysApicBase &= PAGE_BASE_GC_MASK;
2977
2978 rc = PGMGstGetPage(pVCpu, (RTGCPTR)exitQualification, NULL, &GCPhys);
2979 if ( rc == VINF_SUCCESS
2980 && GCPhys == GCPhysApicBase)
2981 {
2982 Log(("Enable VT-x virtual APIC access filtering\n"));
2983 rc2 = IOMMMIOMapMMIOHCPage(pVM, GCPhysApicBase, pVM->hwaccm.s.vmx.pAPICPhys, X86_PTE_RW | X86_PTE_P);
2984 AssertRC(rc2);
2985 }
2986 }
2987
2988 /* Forward it to our trap handler first, in case our shadow pages are out of sync. */
2989 rc = PGMTrap0eHandler(pVCpu, errCode, CPUMCTX2CORE(pCtx), (RTGCPTR)exitQualification);
2990 Log2(("PGMTrap0eHandler %RGv returned %Rrc\n", (RTGCPTR)pCtx->rip, VBOXSTRICTRC_VAL(rc)));
2991
2992 if (rc == VINF_SUCCESS)
2993 { /* We've successfully synced our shadow pages, so let's just continue execution. */
2994 Log2(("Shadow page fault at %RGv cr2=%RGv error code %x\n", (RTGCPTR)pCtx->rip, exitQualification ,errCode));
2995 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitShadowPF);
2996
2997 TRPMResetTrap(pVCpu);
2998 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
2999 goto ResumeExecution;
3000 }
3001 else
3002 if (rc == VINF_EM_RAW_GUEST_TRAP)
3003 { /* A genuine pagefault.
3004 * Forward the trap to the guest by injecting the exception and resuming execution.
3005 */
3006 Log2(("Forward page fault to the guest\n"));
3007
3008 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestPF);
3009 /* The error code might have been changed. */
3010 errCode = TRPMGetErrorCode(pVCpu);
3011
3012 TRPMResetTrap(pVCpu);
3013
3014 /* Now we must update CR2. */
3015 pCtx->cr2 = exitQualification;
3016 rc2 = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), cbInstr, errCode);
3017 AssertRC(rc2);
3018
3019 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3020 goto ResumeExecution;
3021 }
3022#ifdef VBOX_STRICT
3023 if (rc != VINF_EM_RAW_EMULATE_INSTR && rc != VINF_EM_RAW_EMULATE_IO_BLOCK)
3024 Log2(("PGMTrap0eHandler failed with %d\n", VBOXSTRICTRC_VAL(rc)));
3025#endif
3026 /* Need to go back to the recompiler to emulate the instruction. */
3027 TRPMResetTrap(pVCpu);
3028 break;
3029 }
3030
3031 case X86_XCPT_MF: /* Floating point exception. */
3032 {
3033 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestMF);
3034 if (!(pCtx->cr0 & X86_CR0_NE))
3035 {
3036 /* old style FPU error reporting needs some extra work. */
3037 /** @todo don't fall back to the recompiler, but do it manually. */
3038 rc = VINF_EM_RAW_EMULATE_INSTR;
3039 break;
3040 }
3041 Log(("Trap %x at %04X:%RGv\n", vector, pCtx->cs, (RTGCPTR)pCtx->rip));
3042 rc2 = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), cbInstr, errCode);
3043 AssertRC(rc2);
3044
3045 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3046 goto ResumeExecution;
3047 }
3048
3049 case X86_XCPT_DB: /* Debug exception. */
3050 {
3051 uint64_t uDR6;
3052
3053 /* DR6, DR7.GD and IA32_DEBUGCTL.LBR are not updated yet.
3054 *
3055 * Exit qualification bits:
3056 * 3:0 B0-B3 which breakpoint condition was met
3057 * 12:4 Reserved (0)
3058 * 13 BD - debug register access detected
3059 * 14 BS - single step execution or branch taken
3060 * 63:15 Reserved (0)
3061 */
3062 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestDB);
3063
3064 /* Note that we don't support guest and host-initiated debugging at the same time. */
3065
3066 uDR6 = X86_DR6_INIT_VAL;
3067 uDR6 |= (exitQualification & (X86_DR6_B0|X86_DR6_B1|X86_DR6_B2|X86_DR6_B3|X86_DR6_BD|X86_DR6_BS));
3068 rc = DBGFRZTrap01Handler(pVM, pVCpu, CPUMCTX2CORE(pCtx), uDR6);
3069 if (rc == VINF_EM_RAW_GUEST_TRAP)
3070 {
3071 /* Update DR6 here. */
3072 pCtx->dr[6] = uDR6;
3073
3074 /* Resync DR6 if the debug state is active. */
3075 if (CPUMIsGuestDebugStateActive(pVCpu))
3076 ASMSetDR6(pCtx->dr[6]);
3077
3078 /* X86_DR7_GD will be cleared if drx accesses should be trapped inside the guest. */
3079 pCtx->dr[7] &= ~X86_DR7_GD;
3080
3081 /* Paranoia. */
3082 pCtx->dr[7] &= 0xffffffff; /* upper 32 bits reserved */
3083 pCtx->dr[7] &= ~(RT_BIT(11) | RT_BIT(12) | RT_BIT(14) | RT_BIT(15)); /* must be zero */
3084 pCtx->dr[7] |= 0x400; /* must be one */
3085
3086 /* Resync DR7 */
3087 rc2 = VMXWriteVMCS64(VMX_VMCS64_GUEST_DR7, pCtx->dr[7]);
3088 AssertRC(rc2);
3089
3090 Log(("Trap %x (debug) at %RGv exit qualification %RX64 dr6=%x dr7=%x\n", vector, (RTGCPTR)pCtx->rip, exitQualification, (uint32_t)pCtx->dr[6], (uint32_t)pCtx->dr[7]));
3091 rc2 = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), cbInstr, errCode);
3092 AssertRC(rc2);
3093
3094 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3095 goto ResumeExecution;
3096 }
3097 /* Return to ring 3 to deal with the debug exit code. */
3098 Log(("Debugger hardware BP at %04x:%RGv (rc=%Rrc)\n", pCtx->cs, pCtx->rip, VBOXSTRICTRC_VAL(rc)));
3099 break;
3100 }
3101
3102 case X86_XCPT_BP: /* Breakpoint. */
3103 {
3104 rc = DBGFRZTrap03Handler(pVM, pVCpu, CPUMCTX2CORE(pCtx));
3105 if (rc == VINF_EM_RAW_GUEST_TRAP)
3106 {
3107 Log(("Guest #BP at %04x:%RGv\n", pCtx->cs, pCtx->rip));
3108 rc2 = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), cbInstr, errCode);
3109 AssertRC(rc2);
3110 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3111 goto ResumeExecution;
3112 }
3113 if (rc == VINF_SUCCESS)
3114 {
3115 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3116 goto ResumeExecution;
3117 }
3118 Log(("Debugger BP at %04x:%RGv (rc=%Rrc)\n", pCtx->cs, pCtx->rip, VBOXSTRICTRC_VAL(rc)));
3119 break;
3120 }
3121
3122 case X86_XCPT_GP: /* General protection failure exception.*/
3123 {
3124 uint32_t cbOp;
3125 PDISCPUSTATE pDis = &pVCpu->hwaccm.s.DisState;
3126
3127 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestGP);
3128#ifdef VBOX_STRICT
3129 if ( !CPUMIsGuestInRealModeEx(pCtx)
3130 || !pVM->hwaccm.s.vmx.pRealModeTSS)
3131 {
3132 Log(("Trap %x at %04X:%RGv errorCode=%RGv\n", vector, pCtx->cs, (RTGCPTR)pCtx->rip, errCode));
3133 rc2 = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), cbInstr, errCode);
3134 AssertRC(rc2);
3135 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3136 goto ResumeExecution;
3137 }
3138#endif
3139 Assert(CPUMIsGuestInRealModeEx(pCtx));
3140
3141 LogFlow(("Real mode X86_XCPT_GP instruction emulation at %x:%RGv\n", pCtx->cs, (RTGCPTR)pCtx->rip));
3142
3143 rc2 = EMInterpretDisasOne(pVM, pVCpu, CPUMCTX2CORE(pCtx), pDis, &cbOp);
3144 if (RT_SUCCESS(rc2))
3145 {
3146 bool fUpdateRIP = true;
3147
3148 rc = VINF_SUCCESS;
3149 Assert(cbOp == pDis->opsize);
3150 switch (pDis->pCurInstr->opcode)
3151 {
3152 case OP_CLI:
3153 pCtx->eflags.Bits.u1IF = 0;
3154 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitCli);
3155 break;
3156
3157 case OP_STI:
3158 pCtx->eflags.Bits.u1IF = 1;
3159 EMSetInhibitInterruptsPC(pVCpu, pCtx->rip + pDis->opsize);
3160 Assert(VMCPU_FF_ISSET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
3161 rc2 = VMXWriteVMCS(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
3162 AssertRC(rc2);
3163 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitSti);
3164 break;
3165
3166 case OP_HLT:
3167 fUpdateRIP = false;
3168 rc = VINF_EM_HALT;
3169 pCtx->rip += pDis->opsize;
3170 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitHlt);
3171 break;
3172
3173 case OP_POPF:
3174 {
3175 RTGCPTR GCPtrStack;
3176 uint32_t cbParm;
3177 uint32_t uMask;
3178 X86EFLAGS eflags;
3179
3180 if (pDis->prefix & PREFIX_OPSIZE)
3181 {
3182 cbParm = 4;
3183 uMask = 0xffffffff;
3184 }
3185 else
3186 {
3187 cbParm = 2;
3188 uMask = 0xffff;
3189 }
3190
3191 rc2 = SELMToFlatEx(pVCpu, DIS_SELREG_SS, CPUMCTX2CORE(pCtx), pCtx->esp & uMask, 0, &GCPtrStack);
3192 if (RT_FAILURE(rc2))
3193 {
3194 rc = VERR_EM_INTERPRETER;
3195 break;
3196 }
3197 eflags.u = 0;
3198 rc2 = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &eflags.u, cbParm);
3199 if (RT_FAILURE(rc2))
3200 {
3201 rc = VERR_EM_INTERPRETER;
3202 break;
3203 }
3204 LogFlow(("POPF %x -> %RGv mask=%x\n", eflags.u, pCtx->rsp, uMask));
3205 pCtx->eflags.u = (pCtx->eflags.u & ~(X86_EFL_POPF_BITS & uMask)) | (eflags.u & X86_EFL_POPF_BITS & uMask);
3206 /* RF cleared when popped in real mode; see pushf description in AMD manual. */
3207 pCtx->eflags.Bits.u1RF = 0;
3208 pCtx->esp += cbParm;
3209 pCtx->esp &= uMask;
3210
3211 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitPopf);
3212 break;
3213 }
3214
3215 case OP_PUSHF:
3216 {
3217 RTGCPTR GCPtrStack;
3218 uint32_t cbParm;
3219 uint32_t uMask;
3220 X86EFLAGS eflags;
3221
3222 if (pDis->prefix & PREFIX_OPSIZE)
3223 {
3224 cbParm = 4;
3225 uMask = 0xffffffff;
3226 }
3227 else
3228 {
3229 cbParm = 2;
3230 uMask = 0xffff;
3231 }
3232
3233 rc2 = SELMToFlatEx(pVCpu, DIS_SELREG_SS, CPUMCTX2CORE(pCtx), (pCtx->esp - cbParm) & uMask, 0, &GCPtrStack);
3234 if (RT_FAILURE(rc2))
3235 {
3236 rc = VERR_EM_INTERPRETER;
3237 break;
3238 }
3239 eflags = pCtx->eflags;
3240 /* RF & VM cleared when pushed in real mode; see pushf description in AMD manual. */
3241 eflags.Bits.u1RF = 0;
3242 eflags.Bits.u1VM = 0;
3243
3244 rc2 = PGMPhysWrite(pVM, (RTGCPHYS)GCPtrStack, &eflags.u, cbParm);
3245 if (RT_FAILURE(rc2))
3246 {
3247 rc = VERR_EM_INTERPRETER;
3248 break;
3249 }
3250 LogFlow(("PUSHF %x -> %RGv\n", eflags.u, GCPtrStack));
3251 pCtx->esp -= cbParm;
3252 pCtx->esp &= uMask;
3253 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitPushf);
3254 break;
3255 }
3256
3257 case OP_IRET:
3258 {
3259 RTGCPTR GCPtrStack;
3260 uint32_t uMask = 0xffff;
3261 uint16_t aIretFrame[3];
3262
3263 if (pDis->prefix & (PREFIX_OPSIZE | PREFIX_ADDRSIZE))
3264 {
3265 rc = VERR_EM_INTERPRETER;
3266 break;
3267 }
3268
3269 rc2 = SELMToFlatEx(pVCpu, DIS_SELREG_SS, CPUMCTX2CORE(pCtx), pCtx->esp & uMask, 0, &GCPtrStack);
3270 if (RT_FAILURE(rc2))
3271 {
3272 rc = VERR_EM_INTERPRETER;
3273 break;
3274 }
3275 rc2 = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &aIretFrame[0], sizeof(aIretFrame));
3276 if (RT_FAILURE(rc2))
3277 {
3278 rc = VERR_EM_INTERPRETER;
3279 break;
3280 }
3281 pCtx->ip = aIretFrame[0];
3282 pCtx->cs = aIretFrame[1];
3283 pCtx->csHid.u64Base = pCtx->cs << 4;
3284 pCtx->eflags.u = (pCtx->eflags.u & ~(X86_EFL_POPF_BITS & uMask)) | (aIretFrame[2] & X86_EFL_POPF_BITS & uMask);
3285 pCtx->sp += sizeof(aIretFrame);
3286
3287 LogFlow(("iret to %04x:%x\n", pCtx->cs, pCtx->ip));
3288 fUpdateRIP = false;
3289 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitIret);
3290 break;
3291 }
3292
3293 case OP_INT:
3294 {
3295 uint32_t intInfo2;
3296
3297 LogFlow(("Realmode: INT %x\n", pDis->param1.parval & 0xff));
3298 intInfo2 = pDis->param1.parval & 0xff;
3299 intInfo2 |= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
3300 intInfo2 |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
3301
3302 rc = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, intInfo2, cbOp, 0);
3303 AssertRC(VBOXSTRICTRC_VAL(rc));
3304 fUpdateRIP = false;
3305 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitInt);
3306 break;
3307 }
3308
3309 case OP_INTO:
3310 {
3311 if (pCtx->eflags.Bits.u1OF)
3312 {
3313 uint32_t intInfo2;
3314
3315 LogFlow(("Realmode: INTO\n"));
3316 intInfo2 = X86_XCPT_OF;
3317 intInfo2 |= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
3318 intInfo2 |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
3319
3320 rc = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, intInfo2, cbOp, 0);
3321 AssertRC(VBOXSTRICTRC_VAL(rc));
3322 fUpdateRIP = false;
3323 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitInt);
3324 }
3325 break;
3326 }
3327
3328 case OP_INT3:
3329 {
3330 uint32_t intInfo2;
3331
3332 LogFlow(("Realmode: INT 3\n"));
3333 intInfo2 = 3;
3334 intInfo2 |= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
3335 intInfo2 |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
3336
3337 rc = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, intInfo2, cbOp, 0);
3338 AssertRC(VBOXSTRICTRC_VAL(rc));
3339 fUpdateRIP = false;
3340 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitInt);
3341 break;
3342 }
3343
3344 default:
3345 rc = EMInterpretInstructionDisasState(pVCpu, pDis, CPUMCTX2CORE(pCtx), 0, EMCODETYPE_SUPERVISOR);
3346 fUpdateRIP = false;
3347 break;
3348 }
3349
3350 if (rc == VINF_SUCCESS)
3351 {
3352 if (fUpdateRIP)
3353 pCtx->rip += cbOp; /* Move on to the next instruction. */
3354
3355 /* lidt, lgdt can end up here. In the future crx changes as well. Just reload the whole context to be done with it. */
3356 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_ALL;
3357
3358 /* Only resume if successful. */
3359 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3360 goto ResumeExecution;
3361 }
3362 }
3363 else
3364 rc = VERR_EM_INTERPRETER;
3365
3366 AssertMsg(rc == VERR_EM_INTERPRETER || rc == VINF_PGM_CHANGE_MODE || rc == VINF_EM_HALT, ("Unexpected rc=%Rrc\n", VBOXSTRICTRC_VAL(rc)));
3367 break;
3368 }
3369
3370#ifdef VBOX_STRICT
3371 case X86_XCPT_XF: /* SIMD exception. */
3372 case X86_XCPT_DE: /* Divide error. */
3373 case X86_XCPT_UD: /* Unknown opcode exception. */
3374 case X86_XCPT_SS: /* Stack segment exception. */
3375 case X86_XCPT_NP: /* Segment not present exception. */
3376 {
3377 switch(vector)
3378 {
3379 case X86_XCPT_DE:
3380 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestDE);
3381 break;
3382 case X86_XCPT_UD:
3383 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestUD);
3384 break;
3385 case X86_XCPT_SS:
3386 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestSS);
3387 break;
3388 case X86_XCPT_NP:
3389 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestNP);
3390 break;
3391 }
3392
3393 Log(("Trap %x at %04X:%RGv\n", vector, pCtx->cs, (RTGCPTR)pCtx->rip));
3394 rc2 = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), cbInstr, errCode);
3395 AssertRC(rc2);
3396
3397 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3398 goto ResumeExecution;
3399 }
3400#endif
3401 default:
3402 if ( CPUMIsGuestInRealModeEx(pCtx)
3403 && pVM->hwaccm.s.vmx.pRealModeTSS)
3404 {
3405 Log(("Real Mode Trap %x at %04x:%04X error code %x\n", vector, pCtx->cs, pCtx->eip, errCode));
3406 rc = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), cbInstr, errCode);
3407 AssertRC(VBOXSTRICTRC_VAL(rc)); /* Strict RC check below. */
3408
3409 /* Go back to ring 3 in case of a triple fault. */
3410 if ( vector == X86_XCPT_DF
3411 && rc == VINF_EM_RESET)
3412 break;
3413
3414 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3415 goto ResumeExecution;
3416 }
3417 AssertMsgFailed(("Unexpected vm-exit caused by exception %x\n", vector));
3418 rc = VERR_VMX_UNEXPECTED_EXCEPTION;
3419 break;
3420 } /* switch (vector) */
3421
3422 break;
3423
3424 default:
3425 rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_CODE;
3426 AssertMsgFailed(("Unexpected interruption code %x\n", intInfo));
3427 break;
3428 }
3429
3430 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3431 break;
3432 }
3433
3434 case VMX_EXIT_EPT_VIOLATION: /* 48 EPT violation. An attempt to access memory with a guest-physical address was disallowed by the configuration of the EPT paging structures. */
3435 {
3436 RTGCPHYS GCPhys;
3437
3438 Assert(pVM->hwaccm.s.fNestedPaging);
3439
3440 rc2 = VMXReadVMCS64(VMX_VMCS_EXIT_PHYS_ADDR_FULL, &GCPhys);
3441 AssertRC(rc2);
3442 Assert(((exitQualification >> 7) & 3) != 2);
3443
3444 /* Determine the kind of violation. */
3445 errCode = 0;
3446 if (exitQualification & VMX_EXIT_QUALIFICATION_EPT_INSTR_FETCH)
3447 errCode |= X86_TRAP_PF_ID;
3448
3449 if (exitQualification & VMX_EXIT_QUALIFICATION_EPT_DATA_WRITE)
3450 errCode |= X86_TRAP_PF_RW;
3451
3452 /* If the page is present, then it's a page level protection fault. */
3453 if (exitQualification & VMX_EXIT_QUALIFICATION_EPT_ENTRY_PRESENT)
3454 {
3455 errCode |= X86_TRAP_PF_P;
3456 }
3457 else
3458 {
3459 /* Shortcut for APIC TPR reads and writes. */
3460 if ( (GCPhys & 0xfff) == 0x080
3461 && GCPhys > 0x1000000 /* to skip VGA frame buffer accesses */
3462 && fSetupTPRCaching
3463 && (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC))
3464 {
3465 RTGCPHYS GCPhysApicBase;
3466 PDMApicGetBase(pVM, &GCPhysApicBase); /* @todo cache this */
3467 GCPhysApicBase &= PAGE_BASE_GC_MASK;
3468 if (GCPhys == GCPhysApicBase + 0x80)
3469 {
3470 Log(("Enable VT-x virtual APIC access filtering\n"));
3471 rc2 = IOMMMIOMapMMIOHCPage(pVM, GCPhysApicBase, pVM->hwaccm.s.vmx.pAPICPhys, X86_PTE_RW | X86_PTE_P);
3472 AssertRC(rc2);
3473 }
3474 }
3475 }
3476 Log(("EPT Page fault %x at %RGp error code %x\n", (uint32_t)exitQualification, GCPhys, errCode));
3477
3478 /* GCPhys contains the guest physical address of the page fault. */
3479 TRPMAssertTrap(pVCpu, X86_XCPT_PF, TRPM_TRAP);
3480 TRPMSetErrorCode(pVCpu, errCode);
3481 TRPMSetFaultAddress(pVCpu, GCPhys);
3482
3483 /* Handle the pagefault trap for the nested shadow table. */
3484 rc = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, errCode, CPUMCTX2CORE(pCtx), GCPhys);
3485 Log2(("PGMR0Trap0eHandlerNestedPaging %RGv returned %Rrc\n", (RTGCPTR)pCtx->rip, VBOXSTRICTRC_VAL(rc)));
3486 if (rc == VINF_SUCCESS)
3487 { /* We've successfully synced our shadow pages, so let's just continue execution. */
3488 Log2(("Shadow page fault at %RGv cr2=%RGp error code %x\n", (RTGCPTR)pCtx->rip, exitQualification , errCode));
3489 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitReasonNPF);
3490
3491 TRPMResetTrap(pVCpu);
3492 goto ResumeExecution;
3493 }
3494
3495#ifdef VBOX_STRICT
3496 if (rc != VINF_EM_RAW_EMULATE_INSTR)
3497 LogFlow(("PGMTrap0eHandlerNestedPaging failed with %d\n", VBOXSTRICTRC_VAL(rc)));
3498#endif
3499 /* Need to go back to the recompiler to emulate the instruction. */
3500 TRPMResetTrap(pVCpu);
3501 break;
3502 }
3503
3504 case VMX_EXIT_EPT_MISCONFIG:
3505 {
3506 RTGCPHYS GCPhys;
3507
3508 Assert(pVM->hwaccm.s.fNestedPaging);
3509
3510 rc2 = VMXReadVMCS64(VMX_VMCS_EXIT_PHYS_ADDR_FULL, &GCPhys);
3511 AssertRC(rc2);
3512 Log(("VMX_EXIT_EPT_MISCONFIG for %RGp\n", GCPhys));
3513
3514 /* Shortcut for APIC TPR reads and writes. */
3515 if ( (GCPhys & 0xfff) == 0x080
3516 && GCPhys > 0x1000000 /* to skip VGA frame buffer accesses */
3517 && fSetupTPRCaching
3518 && (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC))
3519 {
3520 RTGCPHYS GCPhysApicBase;
3521 PDMApicGetBase(pVM, &GCPhysApicBase); /* @todo cache this */
3522 GCPhysApicBase &= PAGE_BASE_GC_MASK;
3523 if (GCPhys == GCPhysApicBase + 0x80)
3524 {
3525 Log(("Enable VT-x virtual APIC access filtering\n"));
3526 rc2 = IOMMMIOMapMMIOHCPage(pVM, GCPhysApicBase, pVM->hwaccm.s.vmx.pAPICPhys, X86_PTE_RW | X86_PTE_P);
3527 AssertRC(rc2);
3528 }
3529 }
3530
3531 rc = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pCtx), GCPhys, UINT32_MAX);
3532 if (rc == VINF_SUCCESS)
3533 {
3534 Log2(("PGMR0Trap0eHandlerNPMisconfig(,,,%RGp) at %RGv -> resume\n", GCPhys, (RTGCPTR)pCtx->rip));
3535 goto ResumeExecution;
3536 }
3537
3538 Log2(("PGMR0Trap0eHandlerNPMisconfig(,,,%RGp) at %RGv -> %Rrc\n", GCPhys, (RTGCPTR)pCtx->rip, VBOXSTRICTRC_VAL(rc)));
3539 break;
3540 }
3541
3542 case VMX_EXIT_IRQ_WINDOW: /* 7 Interrupt window. */
3543 /* Clear VM-exit on IF=1 change. */
3544 LogFlow(("VMX_EXIT_IRQ_WINDOW %RGv pending=%d IF=%d\n", (RTGCPTR)pCtx->rip, VMCPU_FF_ISPENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC|VMCPU_FF_INTERRUPT_PIC)), pCtx->eflags.Bits.u1IF));
3545 pVCpu->hwaccm.s.vmx.proc_ctls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_IRQ_WINDOW_EXIT;
3546 rc2 = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
3547 AssertRC(rc2);
3548 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitIrqWindow);
3549 goto ResumeExecution; /* we check for pending guest interrupts there */
3550
3551 case VMX_EXIT_WBINVD: /* 54 Guest software attempted to execute WBINVD. (conditional) */
3552 case VMX_EXIT_INVD: /* 13 Guest software attempted to execute INVD. (unconditional) */
3553 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitInvd);
3554 /* Skip instruction and continue directly. */
3555 pCtx->rip += cbInstr;
3556 /* Continue execution.*/
3557 goto ResumeExecution;
3558
3559 case VMX_EXIT_CPUID: /* 10 Guest software attempted to execute CPUID. */
3560 {
3561 Log2(("VMX: Cpuid %x\n", pCtx->eax));
3562 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitCpuid);
3563 rc = EMInterpretCpuId(pVM, pVCpu, CPUMCTX2CORE(pCtx));
3564 if (rc == VINF_SUCCESS)
3565 {
3566 /* Update EIP and continue execution. */
3567 Assert(cbInstr == 2);
3568 pCtx->rip += cbInstr;
3569 goto ResumeExecution;
3570 }
3571 AssertMsgFailed(("EMU: cpuid failed with %Rrc\n", VBOXSTRICTRC_VAL(rc)));
3572 rc = VINF_EM_RAW_EMULATE_INSTR;
3573 break;
3574 }
3575
3576 case VMX_EXIT_RDPMC: /* 15 Guest software attempted to execute RDPMC. */
3577 {
3578 Log2(("VMX: Rdpmc %x\n", pCtx->ecx));
3579 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitRdpmc);
3580 rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pCtx));
3581 if (rc == VINF_SUCCESS)
3582 {
3583 /* Update EIP and continue execution. */
3584 Assert(cbInstr == 2);
3585 pCtx->rip += cbInstr;
3586 goto ResumeExecution;
3587 }
3588 rc = VINF_EM_RAW_EMULATE_INSTR;
3589 break;
3590 }
3591
3592 case VMX_EXIT_RDTSC: /* 16 Guest software attempted to execute RDTSC. */
3593 {
3594 Log2(("VMX: Rdtsc\n"));
3595 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitRdtsc);
3596 rc = EMInterpretRdtsc(pVM, pVCpu, CPUMCTX2CORE(pCtx));
3597 if (rc == VINF_SUCCESS)
3598 {
3599 /* Update EIP and continue execution. */
3600 Assert(cbInstr == 2);
3601 pCtx->rip += cbInstr;
3602 goto ResumeExecution;
3603 }
3604 rc = VINF_EM_RAW_EMULATE_INSTR;
3605 break;
3606 }
3607
3608 case VMX_EXIT_INVPG: /* 14 Guest software attempted to execute INVPG. */
3609 {
3610 Log2(("VMX: invlpg\n"));
3611 Assert(!pVM->hwaccm.s.fNestedPaging);
3612
3613 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitInvpg);
3614 rc = EMInterpretInvlpg(pVM, pVCpu, CPUMCTX2CORE(pCtx), exitQualification);
3615 if (rc == VINF_SUCCESS)
3616 {
3617 /* Update EIP and continue execution. */
3618 pCtx->rip += cbInstr;
3619 goto ResumeExecution;
3620 }
3621 AssertMsg(rc == VERR_EM_INTERPRETER, ("EMU: invlpg %RGv failed with %Rrc\n", exitQualification, VBOXSTRICTRC_VAL(rc)));
3622 break;
3623 }
3624
3625 case VMX_EXIT_MONITOR: /* 39 Guest software attempted to execute MONITOR. */
3626 {
3627 Log2(("VMX: monitor\n"));
3628
3629 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitMonitor);
3630 rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pCtx));
3631 if (rc == VINF_SUCCESS)
3632 {
3633 /* Update EIP and continue execution. */
3634 pCtx->rip += cbInstr;
3635 goto ResumeExecution;
3636 }
3637 AssertMsg(rc == VERR_EM_INTERPRETER, ("EMU: monitor failed with %Rrc\n", VBOXSTRICTRC_VAL(rc)));
3638 break;
3639 }
3640
3641 case VMX_EXIT_WRMSR: /* 32 WRMSR. Guest software attempted to execute WRMSR. */
3642 /* When an interrupt is pending, we'll let MSR_K8_LSTAR writes fault in our TPR patch code. */
3643 if ( pVM->hwaccm.s.fTPRPatchingActive
3644 && pCtx->ecx == MSR_K8_LSTAR)
3645 {
3646 Assert(!CPUMIsGuestInLongModeEx(pCtx));
3647 if ((pCtx->eax & 0xff) != u8LastTPR)
3648 {
3649 Log(("VMX: Faulting MSR_K8_LSTAR write with new TPR value %x\n", pCtx->eax & 0xff));
3650
3651 /* Our patch code uses LSTAR for TPR caching. */
3652 rc2 = PDMApicSetTPR(pVCpu, pCtx->eax & 0xff);
3653 AssertRC(rc2);
3654 }
3655
3656 /* Skip the instruction and continue. */
3657 pCtx->rip += cbInstr; /* wrmsr = [0F 30] */
3658
3659 /* Only resume if successful. */
3660 goto ResumeExecution;
3661 }
3662 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_GUEST_MSR;
3663 /* no break */
3664 case VMX_EXIT_RDMSR: /* 31 RDMSR. Guest software attempted to execute RDMSR. */
3665 {
3666 STAM_COUNTER_INC((exitReason == VMX_EXIT_RDMSR) ? &pVCpu->hwaccm.s.StatExitRdmsr : &pVCpu->hwaccm.s.StatExitWrmsr);
3667
3668 /* Note: the intel manual claims there's a REX version of RDMSR that's slightly different, so we play safe by completely disassembling the instruction. */
3669 Log2(("VMX: %s\n", (exitReason == VMX_EXIT_RDMSR) ? "rdmsr" : "wrmsr"));
3670 rc = EMInterpretInstruction(pVCpu, CPUMCTX2CORE(pCtx), 0);
3671 if (rc == VINF_SUCCESS)
3672 {
3673 /* EIP has been updated already. */
3674
3675 /* Only resume if successful. */
3676 goto ResumeExecution;
3677 }
3678 AssertMsg(rc == VERR_EM_INTERPRETER, ("EMU: %s failed with %Rrc\n", (exitReason == VMX_EXIT_RDMSR) ? "rdmsr" : "wrmsr", VBOXSTRICTRC_VAL(rc)));
3679 break;
3680 }
3681
3682 case VMX_EXIT_CRX_MOVE: /* 28 Control-register accesses. */
3683 {
3684 STAM_PROFILE_ADV_START(&pVCpu->hwaccm.s.StatExit2Sub2, y2);
3685
3686 switch (VMX_EXIT_QUALIFICATION_CRX_ACCESS(exitQualification))
3687 {
3688 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_WRITE:
3689 Log2(("VMX: %RGv mov cr%d, x\n", (RTGCPTR)pCtx->rip, VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification)));
3690 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitCRxWrite[VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification)]);
3691 rc = EMInterpretCRxWrite(pVM, pVCpu, CPUMCTX2CORE(pCtx),
3692 VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification),
3693 VMX_EXIT_QUALIFICATION_CRX_GENREG(exitQualification));
3694
3695 switch (VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification))
3696 {
3697 case 0:
3698 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_GUEST_CR0 | HWACCM_CHANGED_GUEST_CR3;
3699 break;
3700 case 2:
3701 break;
3702 case 3:
3703 Assert(!pVM->hwaccm.s.fNestedPaging || !CPUMIsGuestInPagedProtectedModeEx(pCtx));
3704 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_GUEST_CR3;
3705 break;
3706 case 4:
3707 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_GUEST_CR4;
3708 break;
3709 case 8:
3710 /* CR8 contains the APIC TPR */
3711 Assert(!(pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW));
3712 break;
3713
3714 default:
3715 AssertFailed();
3716 break;
3717 }
3718 break;
3719
3720 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ:
3721 Log2(("VMX: mov x, crx\n"));
3722 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitCRxRead[VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification)]);
3723
3724 Assert(!pVM->hwaccm.s.fNestedPaging || !CPUMIsGuestInPagedProtectedModeEx(pCtx) || VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification) != USE_REG_CR3);
3725
3726 /* CR8 reads only cause an exit when the TPR shadow feature isn't present. */
3727 Assert(VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification) != 8 || !(pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW));
3728
3729 rc = EMInterpretCRxRead(pVM, pVCpu, CPUMCTX2CORE(pCtx),
3730 VMX_EXIT_QUALIFICATION_CRX_GENREG(exitQualification),
3731 VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification));
3732 break;
3733
3734 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_CLTS:
3735 Log2(("VMX: clts\n"));
3736 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitCLTS);
3737 rc = EMInterpretCLTS(pVM, pVCpu);
3738 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_GUEST_CR0;
3739 break;
3740
3741 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_LMSW:
3742 Log2(("VMX: lmsw %x\n", VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(exitQualification)));
3743 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitLMSW);
3744 rc = EMInterpretLMSW(pVM, pVCpu, CPUMCTX2CORE(pCtx), VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(exitQualification));
3745 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_GUEST_CR0;
3746 break;
3747 }
3748
3749 /* Update EIP if no error occurred. */
3750 if (RT_SUCCESS(rc))
3751 pCtx->rip += cbInstr;
3752
3753 if (rc == VINF_SUCCESS)
3754 {
3755 /* Only resume if successful. */
3756 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub2, y2);
3757 goto ResumeExecution;
3758 }
3759 Assert(rc == VERR_EM_INTERPRETER || rc == VINF_PGM_CHANGE_MODE || rc == VINF_PGM_SYNC_CR3);
3760 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub2, y2);
3761 break;
3762 }
3763
3764 case VMX_EXIT_DRX_MOVE: /* 29 Debug-register accesses. */
3765 {
3766 if ( !DBGFIsStepping(pVCpu)
3767 && !CPUMIsHyperDebugStateActive(pVCpu))
3768 {
3769 /* Disable drx move intercepts. */
3770 pVCpu->hwaccm.s.vmx.proc_ctls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT;
3771 rc2 = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
3772 AssertRC(rc2);
3773
3774 /* Save the host and load the guest debug state. */
3775 rc2 = CPUMR0LoadGuestDebugState(pVM, pVCpu, pCtx, true /* include DR6 */);
3776 AssertRC(rc2);
3777
3778#ifdef LOG_ENABLED
3779 if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(exitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
3780 Log(("VMX_EXIT_DRX_MOVE: write DR%d genreg %d\n", VMX_EXIT_QUALIFICATION_DRX_REGISTER(exitQualification), VMX_EXIT_QUALIFICATION_DRX_GENREG(exitQualification)));
3781 else
3782 Log(("VMX_EXIT_DRX_MOVE: read DR%d\n", VMX_EXIT_QUALIFICATION_DRX_REGISTER(exitQualification)));
3783#endif
3784
3785#ifdef VBOX_WITH_STATISTICS
3786 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatDRxContextSwitch);
3787 if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(exitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
3788 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitDRxWrite);
3789 else
3790 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitDRxRead);
3791#endif
3792
3793 goto ResumeExecution;
3794 }
3795
3796 /** @todo clear VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT after the first time and restore drx registers afterwards */
3797 if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(exitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
3798 {
3799 Log2(("VMX: mov drx%d, genreg%d\n", VMX_EXIT_QUALIFICATION_DRX_REGISTER(exitQualification), VMX_EXIT_QUALIFICATION_DRX_GENREG(exitQualification)));
3800 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitDRxWrite);
3801 rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pCtx),
3802 VMX_EXIT_QUALIFICATION_DRX_REGISTER(exitQualification),
3803 VMX_EXIT_QUALIFICATION_DRX_GENREG(exitQualification));
3804 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_GUEST_DEBUG;
3805 Log2(("DR7=%08x\n", pCtx->dr[7]));
3806 }
3807 else
3808 {
3809 Log2(("VMX: mov x, drx\n"));
3810 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitDRxRead);
3811 rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pCtx),
3812 VMX_EXIT_QUALIFICATION_DRX_GENREG(exitQualification),
3813 VMX_EXIT_QUALIFICATION_DRX_REGISTER(exitQualification));
3814 }
3815 /* Update EIP if no error occurred. */
3816 if (RT_SUCCESS(rc))
3817 pCtx->rip += cbInstr;
3818
3819 if (rc == VINF_SUCCESS)
3820 {
3821 /* Only resume if successful. */
3822 goto ResumeExecution;
3823 }
3824 Assert(rc == VERR_EM_INTERPRETER);
3825 break;
3826 }
3827
3828 /* Note: We'll get a #GP if the IO instruction isn't allowed (IOPL or TSS bitmap); no need to double check. */
3829 case VMX_EXIT_PORT_IO: /* 30 I/O instruction. */
3830 {
3831 STAM_PROFILE_ADV_START(&pVCpu->hwaccm.s.StatExit2Sub1, y1);
3832 uint32_t uIOWidth = VMX_EXIT_QUALIFICATION_IO_WIDTH(exitQualification);
3833 uint32_t uPort;
3834 bool fIOWrite = (VMX_EXIT_QUALIFICATION_IO_DIRECTION(exitQualification) == VMX_EXIT_QUALIFICATION_IO_DIRECTION_OUT);
3835
3836 /** @todo necessary to make the distinction? */
3837 if (VMX_EXIT_QUALIFICATION_IO_ENCODING(exitQualification) == VMX_EXIT_QUALIFICATION_IO_ENCODING_DX)
3838 {
3839 uPort = pCtx->edx & 0xffff;
3840 }
3841 else
3842 uPort = VMX_EXIT_QUALIFICATION_IO_PORT(exitQualification); /* Immediate encoding. */
3843
3844 /* paranoia */
3845 if (RT_UNLIKELY(uIOWidth == 2 || uIOWidth >= 4))
3846 {
3847 rc = fIOWrite ? VINF_IOM_R3_IOPORT_WRITE : VINF_IOM_R3_IOPORT_READ;
3848 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub1, y1);
3849 break;
3850 }
3851
3852 uint32_t cbSize = g_aIOSize[uIOWidth];
3853
3854 if (VMX_EXIT_QUALIFICATION_IO_STRING(exitQualification))
3855 {
3856 /* ins/outs */
3857 PDISCPUSTATE pDis = &pVCpu->hwaccm.s.DisState;
3858
3859 /* Disassemble manually to deal with segment prefixes. */
3860 /** @todo VMX_VMCS_EXIT_GUEST_LINEAR_ADDR contains the flat pointer operand of the instruction. */
3861 /** @todo VMX_VMCS32_RO_EXIT_INSTR_INFO also contains segment prefix info. */
3862 rc2 = EMInterpretDisasOne(pVM, pVCpu, CPUMCTX2CORE(pCtx), pDis, NULL);
3863 if (RT_SUCCESS(rc))
3864 {
3865 if (fIOWrite)
3866 {
3867 Log2(("IOMInterpretOUTSEx %RGv %x size=%d\n", (RTGCPTR)pCtx->rip, uPort, cbSize));
3868 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitIOStringWrite);
3869 rc = IOMInterpretOUTSEx(pVM, CPUMCTX2CORE(pCtx), uPort, pDis->prefix, pDis->addrmode, cbSize);
3870 }
3871 else
3872 {
3873 Log2(("IOMInterpretINSEx %RGv %x size=%d\n", (RTGCPTR)pCtx->rip, uPort, cbSize));
3874 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitIOStringRead);
3875 rc = IOMInterpretINSEx(pVM, CPUMCTX2CORE(pCtx), uPort, pDis->prefix, pDis->addrmode, cbSize);
3876 }
3877 }
3878 else
3879 rc = VINF_EM_RAW_EMULATE_INSTR;
3880 }
3881 else
3882 {
3883 /* normal in/out */
3884 uint32_t uAndVal = g_aIOOpAnd[uIOWidth];
3885
3886 Assert(!VMX_EXIT_QUALIFICATION_IO_REP(exitQualification));
3887
3888 if (fIOWrite)
3889 {
3890 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitIOWrite);
3891 rc = IOMIOPortWrite(pVM, uPort, pCtx->eax & uAndVal, cbSize);
3892 if (rc == VINF_IOM_R3_IOPORT_WRITE)
3893 HWACCMR0SavePendingIOPortWrite(pVCpu, pCtx->rip, pCtx->rip + cbInstr, uPort, uAndVal, cbSize);
3894 }
3895 else
3896 {
3897 uint32_t u32Val = 0;
3898
3899 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitIORead);
3900 rc = IOMIOPortRead(pVM, uPort, &u32Val, cbSize);
3901 if (IOM_SUCCESS(rc))
3902 {
3903 /* Write back to the EAX register. */
3904 pCtx->eax = (pCtx->eax & ~uAndVal) | (u32Val & uAndVal);
3905 }
3906 else
3907 if (rc == VINF_IOM_R3_IOPORT_READ)
3908 HWACCMR0SavePendingIOPortRead(pVCpu, pCtx->rip, pCtx->rip + cbInstr, uPort, uAndVal, cbSize);
3909 }
3910 }
3911 /*
3912 * Handled the I/O return codes.
3913 * (The unhandled cases end up with rc == VINF_EM_RAW_EMULATE_INSTR.)
3914 */
3915 if (IOM_SUCCESS(rc))
3916 {
3917 /* Update EIP and continue execution. */
3918 pCtx->rip += cbInstr;
3919 if (RT_LIKELY(rc == VINF_SUCCESS))
3920 {
3921 /* If any IO breakpoints are armed, then we should check if a debug trap needs to be generated. */
3922 if (pCtx->dr[7] & X86_DR7_ENABLED_MASK)
3923 {
3924 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatDRxIOCheck);
3925 for (unsigned i=0;i<4;i++)
3926 {
3927 unsigned uBPLen = g_aIOSize[X86_DR7_GET_LEN(pCtx->dr[7], i)];
3928
3929 if ( (uPort >= pCtx->dr[i] && uPort < pCtx->dr[i] + uBPLen)
3930 && (pCtx->dr[7] & (X86_DR7_L(i) | X86_DR7_G(i)))
3931 && (pCtx->dr[7] & X86_DR7_RW(i, X86_DR7_RW_IO)) == X86_DR7_RW(i, X86_DR7_RW_IO))
3932 {
3933 uint64_t uDR6;
3934
3935 Assert(CPUMIsGuestDebugStateActive(pVCpu));
3936
3937 uDR6 = ASMGetDR6();
3938
3939 /* Clear all breakpoint status flags and set the one we just hit. */
3940 uDR6 &= ~(X86_DR6_B0|X86_DR6_B1|X86_DR6_B2|X86_DR6_B3);
3941 uDR6 |= (uint64_t)RT_BIT(i);
3942
3943 /* Note: AMD64 Architecture Programmer's Manual 13.1:
3944 * Bits 15:13 of the DR6 register is never cleared by the processor and must be cleared by software after
3945 * the contents have been read.
3946 */
3947 ASMSetDR6(uDR6);
3948
3949 /* X86_DR7_GD will be cleared if drx accesses should be trapped inside the guest. */
3950 pCtx->dr[7] &= ~X86_DR7_GD;
3951
3952 /* Paranoia. */
3953 pCtx->dr[7] &= 0xffffffff; /* upper 32 bits reserved */
3954 pCtx->dr[7] &= ~(RT_BIT(11) | RT_BIT(12) | RT_BIT(14) | RT_BIT(15)); /* must be zero */
3955 pCtx->dr[7] |= 0x400; /* must be one */
3956
3957 /* Resync DR7 */
3958 rc2 = VMXWriteVMCS64(VMX_VMCS64_GUEST_DR7, pCtx->dr[7]);
3959 AssertRC(rc2);
3960
3961 /* Construct inject info. */
3962 intInfo = X86_XCPT_DB;
3963 intInfo |= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
3964 intInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HWEXCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
3965
3966 Log(("Inject IO debug trap at %RGv\n", (RTGCPTR)pCtx->rip));
3967 rc2 = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), 0, 0);
3968 AssertRC(rc2);
3969
3970 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub1, y1);
3971 goto ResumeExecution;
3972 }
3973 }
3974 }
3975 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub1, y1);
3976 goto ResumeExecution;
3977 }
3978 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub1, y1);
3979 break;
3980 }
3981
3982#ifdef VBOX_STRICT
3983 if (rc == VINF_IOM_R3_IOPORT_READ)
3984 Assert(!fIOWrite);
3985 else if (rc == VINF_IOM_R3_IOPORT_WRITE)
3986 Assert(fIOWrite);
3987 else
3988 AssertMsg(RT_FAILURE(rc) || rc == VINF_EM_RAW_EMULATE_INSTR || rc == VINF_EM_RAW_GUEST_TRAP || rc == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", VBOXSTRICTRC_VAL(rc)));
3989#endif
3990 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub1, y1);
3991 break;
3992 }
3993
3994 case VMX_EXIT_TPR: /* 43 TPR below threshold. Guest software executed MOV to CR8. */
3995 LogFlow(("VMX_EXIT_TPR\n"));
3996 /* RIP is already set to the next instruction and the TPR has been synced back. Just resume. */
3997 goto ResumeExecution;
3998
3999 case VMX_EXIT_APIC_ACCESS: /* 44 APIC access. Guest software attempted to access memory at a physical address on the APIC-access page. */
4000 {
4001 LogFlow(("VMX_EXIT_APIC_ACCESS\n"));
4002 unsigned uAccessType = VMX_EXIT_QUALIFICATION_APIC_ACCESS_TYPE(exitQualification);
4003
4004 switch(uAccessType)
4005 {
4006 case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
4007 case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
4008 {
4009 RTGCPHYS GCPhys;
4010 PDMApicGetBase(pVM, &GCPhys);
4011 GCPhys &= PAGE_BASE_GC_MASK;
4012 GCPhys += VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(exitQualification);
4013
4014 LogFlow(("Apic access at %RGp\n", GCPhys));
4015 rc = IOMMMIOPhysHandler(pVM, (uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ) ? 0 : X86_TRAP_PF_RW, CPUMCTX2CORE(pCtx), GCPhys);
4016 if (rc == VINF_SUCCESS)
4017 goto ResumeExecution; /* rip already updated */
4018 break;
4019 }
4020
4021 default:
4022 rc = VINF_EM_RAW_EMULATE_INSTR;
4023 break;
4024 }
4025 break;
4026 }
4027
4028 case VMX_EXIT_PREEMPTION_TIMER: /* 52 VMX-preemption timer expired. The preemption timer counted down to zero. */
4029 if (!TMTimerPollBool(pVM, pVCpu))
4030 goto ResumeExecution;
4031 rc = VINF_EM_RAW_TIMER_PENDING;
4032 break;
4033
4034 default:
4035 /* The rest is handled after syncing the entire CPU state. */
4036 break;
4037 }
4038
4039 /* Note: the guest state isn't entirely synced back at this stage. */
4040
4041 /* Investigate why there was a VM-exit. (part 2) */
4042 switch (exitReason)
4043 {
4044 case VMX_EXIT_EXCEPTION: /* 0 Exception or non-maskable interrupt (NMI). */
4045 case VMX_EXIT_EXTERNAL_IRQ: /* 1 External interrupt. */
4046 case VMX_EXIT_EPT_VIOLATION:
4047 case VMX_EXIT_EPT_MISCONFIG: /* 49 EPT misconfig is used by the PGM/MMIO optimizations. */
4048 case VMX_EXIT_PREEMPTION_TIMER: /* 52 VMX-preemption timer expired. The preemption timer counted down to zero. */
4049 /* Already handled above. */
4050 break;
4051
4052 case VMX_EXIT_TRIPLE_FAULT: /* 2 Triple fault. */
4053 rc = VINF_EM_RESET; /* Triple fault equals a reset. */
4054 break;
4055
4056 case VMX_EXIT_INIT_SIGNAL: /* 3 INIT signal. */
4057 case VMX_EXIT_SIPI: /* 4 Start-up IPI (SIPI). */
4058 rc = VINF_EM_RAW_INTERRUPT;
4059 AssertFailed(); /* Can't happen. Yet. */
4060 break;
4061
4062 case VMX_EXIT_IO_SMI_IRQ: /* 5 I/O system-management interrupt (SMI). */
4063 case VMX_EXIT_SMI_IRQ: /* 6 Other SMI. */
4064 rc = VINF_EM_RAW_INTERRUPT;
4065 AssertFailed(); /* Can't happen afaik. */
4066 break;
4067
4068 case VMX_EXIT_TASK_SWITCH: /* 9 Task switch: too complicated to emulate, so fall back to the recompiler */
4069 Log(("VMX_EXIT_TASK_SWITCH: exit=%RX64\n", exitQualification));
4070 if ( (VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE(exitQualification) == VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE_IDT)
4071 && pVCpu->hwaccm.s.Event.fPending)
4072 {
4073 /* Caused by an injected interrupt. */
4074 pVCpu->hwaccm.s.Event.fPending = false;
4075
4076 Log(("VMX_EXIT_TASK_SWITCH: reassert trap %d\n", VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVCpu->hwaccm.s.Event.intInfo)));
4077 Assert(!VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(pVCpu->hwaccm.s.Event.intInfo));
4078 rc2 = TRPMAssertTrap(pVCpu, VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVCpu->hwaccm.s.Event.intInfo), TRPM_HARDWARE_INT);
4079 AssertRC(rc2);
4080 }
4081 /* else Exceptions and software interrupts can just be restarted. */
4082 rc = VERR_EM_INTERPRETER;
4083 break;
4084
4085 case VMX_EXIT_HLT: /* 12 Guest software attempted to execute HLT. */
4086 /* Check if external interrupts are pending; if so, don't switch back. */
4087 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitHlt);
4088 pCtx->rip++; /* skip hlt */
4089 if (EMShouldContinueAfterHalt(pVCpu, pCtx))
4090 goto ResumeExecution;
4091
4092 rc = VINF_EM_HALT;
4093 break;
4094
4095 case VMX_EXIT_MWAIT: /* 36 Guest software executed MWAIT. */
4096 Log2(("VMX: mwait\n"));
4097 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitMwait);
4098 rc = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pCtx));
4099 if ( rc == VINF_EM_HALT
4100 || rc == VINF_SUCCESS)
4101 {
4102 /* Update EIP and continue execution. */
4103 pCtx->rip += cbInstr;
4104
4105 /* Check if external interrupts are pending; if so, don't switch back. */
4106 if ( rc == VINF_SUCCESS
4107 || ( rc == VINF_EM_HALT
4108 && EMShouldContinueAfterHalt(pVCpu, pCtx))
4109 )
4110 goto ResumeExecution;
4111 }
4112 AssertMsg(rc == VERR_EM_INTERPRETER || rc == VINF_EM_HALT, ("EMU: mwait failed with %Rrc\n", VBOXSTRICTRC_VAL(rc)));
4113 break;
4114
4115 case VMX_EXIT_RSM: /* 17 Guest software attempted to execute RSM in SMM. */
4116 AssertFailed(); /* can't happen. */
4117 rc = VERR_EM_INTERPRETER;
4118 break;
4119
4120 case VMX_EXIT_MTF: /* 37 Exit due to Monitor Trap Flag. */
4121 LogFlow(("VMX_EXIT_MTF at %RGv\n", (RTGCPTR)pCtx->rip));
4122 pVCpu->hwaccm.s.vmx.proc_ctls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_TRAP_FLAG;
4123 rc2 = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
4124 AssertRC(rc2);
4125 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitMTF);
4126#if 0
4127 DBGFDoneStepping(pVCpu);
4128#endif
4129 rc = VINF_EM_DBG_STOP;
4130 break;
4131
4132 case VMX_EXIT_VMCALL: /* 18 Guest software executed VMCALL. */
4133 case VMX_EXIT_VMCLEAR: /* 19 Guest software executed VMCLEAR. */
4134 case VMX_EXIT_VMLAUNCH: /* 20 Guest software executed VMLAUNCH. */
4135 case VMX_EXIT_VMPTRLD: /* 21 Guest software executed VMPTRLD. */
4136 case VMX_EXIT_VMPTRST: /* 22 Guest software executed VMPTRST. */
4137 case VMX_EXIT_VMREAD: /* 23 Guest software executed VMREAD. */
4138 case VMX_EXIT_VMRESUME: /* 24 Guest software executed VMRESUME. */
4139 case VMX_EXIT_VMWRITE: /* 25 Guest software executed VMWRITE. */
4140 case VMX_EXIT_VMXOFF: /* 26 Guest software executed VMXOFF. */
4141 case VMX_EXIT_VMXON: /* 27 Guest software executed VMXON. */
4142 /** @todo inject #UD immediately */
4143 rc = VERR_EM_INTERPRETER;
4144 break;
4145
4146 case VMX_EXIT_CPUID: /* 10 Guest software attempted to execute CPUID. */
4147 case VMX_EXIT_RDTSC: /* 16 Guest software attempted to execute RDTSC. */
4148 case VMX_EXIT_INVPG: /* 14 Guest software attempted to execute INVPG. */
4149 case VMX_EXIT_CRX_MOVE: /* 28 Control-register accesses. */
4150 case VMX_EXIT_DRX_MOVE: /* 29 Debug-register accesses. */
4151 case VMX_EXIT_PORT_IO: /* 30 I/O instruction. */
4152 case VMX_EXIT_RDPMC: /* 15 Guest software attempted to execute RDPMC. */
4153 /* already handled above */
4154 AssertMsg( rc == VINF_PGM_CHANGE_MODE
4155 || rc == VINF_EM_RAW_INTERRUPT
4156 || rc == VERR_EM_INTERPRETER
4157 || rc == VINF_EM_RAW_EMULATE_INSTR
4158 || rc == VINF_PGM_SYNC_CR3
4159 || rc == VINF_IOM_R3_IOPORT_READ
4160 || rc == VINF_IOM_R3_IOPORT_WRITE
4161 || rc == VINF_EM_RAW_GUEST_TRAP
4162 || rc == VINF_TRPM_XCPT_DISPATCHED
4163 || rc == VINF_EM_RESCHEDULE_REM,
4164 ("rc = %d\n", VBOXSTRICTRC_VAL(rc)));
4165 break;
4166
4167 case VMX_EXIT_TPR: /* 43 TPR below threshold. Guest software executed MOV to CR8. */
4168 case VMX_EXIT_APIC_ACCESS: /* 44 APIC access. Guest software attempted to access memory at a physical address on the APIC-access page. */
4169 case VMX_EXIT_RDMSR: /* 31 RDMSR. Guest software attempted to execute RDMSR. */
4170 case VMX_EXIT_WRMSR: /* 32 WRMSR. Guest software attempted to execute WRMSR. */
4171 case VMX_EXIT_PAUSE: /* 40 Guest software attempted to execute PAUSE. */
4172 case VMX_EXIT_MONITOR: /* 39 Guest software attempted to execute MONITOR. */
4173 /* Note: If we decide to emulate them here, then we must sync the MSRs that could have been changed (sysenter, fs/gs base)!!! */
4174 rc = VERR_EM_INTERPRETER;
4175 break;
4176
4177 case VMX_EXIT_IRQ_WINDOW: /* 7 Interrupt window. */
4178 Assert(rc == VINF_EM_RAW_INTERRUPT);
4179 break;
4180
4181 case VMX_EXIT_ERR_INVALID_GUEST_STATE: /* 33 VM-entry failure due to invalid guest state. */
4182 {
4183#ifdef VBOX_STRICT
4184 RTCCUINTREG val2 = 0;
4185
4186 Log(("VMX_EXIT_ERR_INVALID_GUEST_STATE\n"));
4187
4188 VMXReadVMCS(VMX_VMCS64_GUEST_RIP, &val2);
4189 Log(("Old eip %RGv new %RGv\n", (RTGCPTR)pCtx->rip, (RTGCPTR)val2));
4190
4191 VMXReadVMCS(VMX_VMCS64_GUEST_CR0, &val2);
4192 Log(("VMX_VMCS_GUEST_CR0 %RX64\n", (uint64_t)val2));
4193
4194 VMXReadVMCS(VMX_VMCS64_GUEST_CR3, &val2);
4195 Log(("VMX_VMCS_GUEST_CR3 %RX64\n", (uint64_t)val2));
4196
4197 VMXReadVMCS(VMX_VMCS64_GUEST_CR4, &val2);
4198 Log(("VMX_VMCS_GUEST_CR4 %RX64\n", (uint64_t)val2));
4199
4200 VMXReadVMCS(VMX_VMCS_GUEST_RFLAGS, &val2);
4201 Log(("VMX_VMCS_GUEST_RFLAGS %08x\n", val2));
4202
4203 VMX_LOG_SELREG(CS, "CS", val2);
4204 VMX_LOG_SELREG(DS, "DS", val2);
4205 VMX_LOG_SELREG(ES, "ES", val2);
4206 VMX_LOG_SELREG(FS, "FS", val2);
4207 VMX_LOG_SELREG(GS, "GS", val2);
4208 VMX_LOG_SELREG(SS, "SS", val2);
4209 VMX_LOG_SELREG(TR, "TR", val2);
4210 VMX_LOG_SELREG(LDTR, "LDTR", val2);
4211
4212 VMXReadVMCS(VMX_VMCS64_GUEST_GDTR_BASE, &val2);
4213 Log(("VMX_VMCS_GUEST_GDTR_BASE %RX64\n", (uint64_t)val2));
4214 VMXReadVMCS(VMX_VMCS64_GUEST_IDTR_BASE, &val2);
4215 Log(("VMX_VMCS_GUEST_IDTR_BASE %RX64\n", (uint64_t)val2));
4216#endif /* VBOX_STRICT */
4217 rc = VERR_VMX_INVALID_GUEST_STATE;
4218 break;
4219 }
4220
4221 case VMX_EXIT_ERR_MSR_LOAD: /* 34 VM-entry failure due to MSR loading. */
4222 case VMX_EXIT_ERR_MACHINE_CHECK: /* 41 VM-entry failure due to machine-check. */
4223 default:
4224 rc = VERR_VMX_UNEXPECTED_EXIT_CODE;
4225 AssertMsgFailed(("Unexpected exit code %d\n", exitReason)); /* Can't happen. */
4226 break;
4227
4228 }
4229end:
4230
4231 /* We now going back to ring-3, so clear the action flag. */
4232 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
4233
4234 /* Signal changes for the recompiler. */
4235 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR | CPUM_CHANGED_LDTR | CPUM_CHANGED_GDTR | CPUM_CHANGED_IDTR | CPUM_CHANGED_TR | CPUM_CHANGED_HIDDEN_SEL_REGS);
4236
4237 /* If we executed vmlaunch/vmresume and an external irq was pending, then we don't have to do a full sync the next time. */
4238 if ( exitReason == VMX_EXIT_EXTERNAL_IRQ
4239 && !VMX_EXIT_INTERRUPTION_INFO_VALID(intInfo))
4240 {
4241 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatPendingHostIrq);
4242 /* On the next entry we'll only sync the host context. */
4243 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_HOST_CONTEXT;
4244 }
4245 else
4246 {
4247 /* On the next entry we'll sync everything. */
4248 /** @todo we can do better than this */
4249 /* Not in the VINF_PGM_CHANGE_MODE though! */
4250 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_ALL;
4251 }
4252
4253 /* translate into a less severe return code */
4254 if (rc == VERR_EM_INTERPRETER)
4255 rc = VINF_EM_RAW_EMULATE_INSTR;
4256 else
4257 /* Try to extract more information about what might have gone wrong here. */
4258 if (rc == VERR_VMX_INVALID_VMCS_PTR)
4259 {
4260 VMXGetActivateVMCS(&pVCpu->hwaccm.s.vmx.lasterror.u64VMCSPhys);
4261 pVCpu->hwaccm.s.vmx.lasterror.ulVMCSRevision = *(uint32_t *)pVCpu->hwaccm.s.vmx.pvVMCS;
4262 pVCpu->hwaccm.s.vmx.lasterror.idEnteredCpu = pVCpu->hwaccm.s.idEnteredCpu;
4263 pVCpu->hwaccm.s.vmx.lasterror.idCurrentCpu = RTMpCpuId();
4264 }
4265
4266 /* Just set the correct state here instead of trying to catch every goto above. */
4267 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED, VMCPUSTATE_STARTED_EXEC);
4268
4269#ifdef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
4270 /* Restore interrupts if we exitted after disabling them. */
4271 if (uOldEFlags != ~(RTCCUINTREG)0)
4272 ASMSetFlags(uOldEFlags);
4273#endif
4274
4275 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2, x);
4276 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit1, x);
4277 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatEntry, x);
4278 Log2(("X"));
4279 return VBOXSTRICTRC_TODO(rc);
4280}
4281
4282
4283/**
4284 * Enters the VT-x session
4285 *
4286 * @returns VBox status code.
4287 * @param pVM The VM to operate on.
4288 * @param pVCpu The VMCPU to operate on.
4289 * @param pCpu CPU info struct
4290 */
4291VMMR0DECL(int) VMXR0Enter(PVM pVM, PVMCPU pVCpu, PHMGLOBLCPUINFO pCpu)
4292{
4293 Assert(pVM->hwaccm.s.vmx.fSupported);
4294 NOREF(pCpu);
4295
4296 unsigned cr4 = ASMGetCR4();
4297 if (!(cr4 & X86_CR4_VMXE))
4298 {
4299 AssertMsgFailed(("X86_CR4_VMXE should be set!\n"));
4300 return VERR_VMX_X86_CR4_VMXE_CLEARED;
4301 }
4302
4303 /* Activate the VM Control Structure. */
4304 int rc = VMXActivateVMCS(pVCpu->hwaccm.s.vmx.HCPhysVMCS);
4305 if (RT_FAILURE(rc))
4306 return rc;
4307
4308 pVCpu->hwaccm.s.fResumeVM = false;
4309 return VINF_SUCCESS;
4310}
4311
4312
4313/**
4314 * Leaves the VT-x session
4315 *
4316 * @returns VBox status code.
4317 * @param pVM The VM to operate on.
4318 * @param pVCpu The VMCPU to operate on.
4319 * @param pCtx CPU context
4320 */
4321VMMR0DECL(int) VMXR0Leave(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
4322{
4323 Assert(pVM->hwaccm.s.vmx.fSupported);
4324
4325#ifdef DEBUG
4326 if (CPUMIsHyperDebugStateActive(pVCpu))
4327 {
4328 CPUMR0LoadHostDebugState(pVM, pVCpu);
4329 Assert(pVCpu->hwaccm.s.vmx.proc_ctls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT);
4330 }
4331 else
4332#endif
4333 /* Save the guest debug state if necessary. */
4334 if (CPUMIsGuestDebugStateActive(pVCpu))
4335 {
4336 CPUMR0SaveGuestDebugState(pVM, pVCpu, pCtx, true /* save DR6 */);
4337
4338 /* Enable drx move intercepts again. */
4339 pVCpu->hwaccm.s.vmx.proc_ctls |= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT;
4340 int rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
4341 AssertRC(rc);
4342
4343 /* Resync the debug registers the next time. */
4344 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_GUEST_DEBUG;
4345 }
4346 else
4347 Assert(pVCpu->hwaccm.s.vmx.proc_ctls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT);
4348
4349 /* Clear VM Control Structure. Marking it inactive, clearing implementation specific data and writing back VMCS data to memory. */
4350 int rc = VMXClearVMCS(pVCpu->hwaccm.s.vmx.HCPhysVMCS);
4351 AssertRC(rc);
4352
4353 return VINF_SUCCESS;
4354}
4355
4356/**
4357 * Flush the TLB (EPT)
4358 *
4359 * @returns VBox status code.
4360 * @param pVM The VM to operate on.
4361 * @param pVCpu The VM CPU to operate on.
4362 * @param enmFlush Type of flush
4363 * @param GCPhys Physical address of the page to flush
4364 */
4365static void hmR0VmxFlushEPT(PVM pVM, PVMCPU pVCpu, VMX_FLUSH enmFlush, RTGCPHYS GCPhys)
4366{
4367 uint64_t descriptor[2];
4368
4369 LogFlow(("hmR0VmxFlushEPT %d %RGv\n", enmFlush, GCPhys));
4370 Assert(pVM->hwaccm.s.fNestedPaging);
4371 descriptor[0] = pVCpu->hwaccm.s.vmx.GCPhysEPTP;
4372 descriptor[1] = GCPhys;
4373 int rc = VMXR0InvEPT(enmFlush, &descriptor[0]);
4374 AssertRC(rc);
4375}
4376
4377#ifdef HWACCM_VTX_WITH_VPID
4378/**
4379 * Flush the TLB (EPT)
4380 *
4381 * @returns VBox status code.
4382 * @param pVM The VM to operate on.
4383 * @param pVCpu The VM CPU to operate on.
4384 * @param enmFlush Type of flush
4385 * @param GCPtr Virtual address of the page to flush
4386 */
4387static void hmR0VmxFlushVPID(PVM pVM, PVMCPU pVCpu, VMX_FLUSH enmFlush, RTGCPTR GCPtr)
4388{
4389#if HC_ARCH_BITS == 32
4390 /* If we get a flush in 64 bits guest mode, then force a full TLB flush. Invvpid probably takes only 32 bits addresses. (@todo) */
4391 if ( CPUMIsGuestInLongMode(pVCpu)
4392 && !VMX_IS_64BIT_HOST_MODE())
4393 {
4394 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
4395 }
4396 else
4397#endif
4398 {
4399 uint64_t descriptor[2];
4400
4401 Assert(pVM->hwaccm.s.vmx.fVPID);
4402 descriptor[0] = pVCpu->hwaccm.s.uCurrentASID;
4403 descriptor[1] = GCPtr;
4404 int rc = VMXR0InvVPID(enmFlush, &descriptor[0]); NOREF(rc);
4405 AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvVPID %x %x %RGv failed with %d\n", enmFlush, pVCpu->hwaccm.s.uCurrentASID, GCPtr, rc));
4406 }
4407}
4408#endif /* HWACCM_VTX_WITH_VPID */
4409
4410/**
4411 * Invalidates a guest page
4412 *
4413 * @returns VBox status code.
4414 * @param pVM The VM to operate on.
4415 * @param pVCpu The VM CPU to operate on.
4416 * @param GCVirt Page to invalidate
4417 */
4418VMMR0DECL(int) VMXR0InvalidatePage(PVM pVM, PVMCPU pVCpu, RTGCPTR GCVirt)
4419{
4420 bool fFlushPending = VMCPU_FF_ISSET(pVCpu, VMCPU_FF_TLB_FLUSH);
4421
4422 Log2(("VMXR0InvalidatePage %RGv\n", GCVirt));
4423
4424 /* Only relevant if we want to use VPID.
4425 * In the nested paging case we still see such calls, but
4426 * can safely ignore them. (e.g. after cr3 updates)
4427 */
4428#ifdef HWACCM_VTX_WITH_VPID
4429 /* Skip it if a TLB flush is already pending. */
4430 if ( !fFlushPending
4431 && pVM->hwaccm.s.vmx.fVPID)
4432 hmR0VmxFlushVPID(pVM, pVCpu, pVM->hwaccm.s.vmx.enmFlushPage, GCVirt);
4433#endif /* HWACCM_VTX_WITH_VPID */
4434
4435 return VINF_SUCCESS;
4436}
4437
4438/**
4439 * Invalidates a guest page by physical address
4440 *
4441 * NOTE: Assumes the current instruction references this physical page though a virtual address!!
4442 *
4443 * @returns VBox status code.
4444 * @param pVM The VM to operate on.
4445 * @param pVCpu The VM CPU to operate on.
4446 * @param GCPhys Page to invalidate
4447 */
4448VMMR0DECL(int) VMXR0InvalidatePhysPage(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys)
4449{
4450 bool fFlushPending = VMCPU_FF_ISSET(pVCpu, VMCPU_FF_TLB_FLUSH);
4451
4452 Assert(pVM->hwaccm.s.fNestedPaging);
4453
4454 LogFlow(("VMXR0InvalidatePhysPage %RGp\n", GCPhys));
4455
4456 /* Skip it if a TLB flush is already pending. */
4457 if (!fFlushPending)
4458 hmR0VmxFlushEPT(pVM, pVCpu, pVM->hwaccm.s.vmx.enmFlushPage, GCPhys);
4459
4460 return VINF_SUCCESS;
4461}
4462
4463/**
4464 * Report world switch error and dump some useful debug info
4465 *
4466 * @param pVM The VM to operate on.
4467 * @param pVCpu The VMCPU to operate on.
4468 * @param rc Return code
4469 * @param pCtx Current CPU context (not updated)
4470 */
4471static void hmR0VmxReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, VBOXSTRICTRC rc, PCPUMCTX pCtx)
4472{
4473 NOREF(pVM);
4474
4475 switch (VBOXSTRICTRC_VAL(rc))
4476 {
4477 case VERR_VMX_INVALID_VMXON_PTR:
4478 AssertFailed();
4479 break;
4480
4481 case VERR_VMX_UNABLE_TO_START_VM:
4482 case VERR_VMX_UNABLE_TO_RESUME_VM:
4483 {
4484 int rc2;
4485 RTCCUINTREG exitReason, instrError;
4486
4487 rc2 = VMXReadVMCS(VMX_VMCS32_RO_EXIT_REASON, &exitReason);
4488 rc2 |= VMXReadVMCS(VMX_VMCS32_RO_VM_INSTR_ERROR, &instrError);
4489 AssertRC(rc2);
4490 if (rc2 == VINF_SUCCESS)
4491 {
4492 Log(("Unable to start/resume VM for reason: %x. Instruction error %x\n", (uint32_t)exitReason, (uint32_t)instrError));
4493 Log(("Current stack %08x\n", &rc2));
4494
4495 pVCpu->hwaccm.s.vmx.lasterror.ulInstrError = instrError;
4496 pVCpu->hwaccm.s.vmx.lasterror.ulExitReason = exitReason;
4497
4498#ifdef VBOX_STRICT
4499 RTGDTR gdtr;
4500 PCX86DESCHC pDesc;
4501 RTCCUINTREG val;
4502
4503 ASMGetGDTR(&gdtr);
4504
4505 VMXReadVMCS(VMX_VMCS64_GUEST_RIP, &val);
4506 Log(("Old eip %RGv new %RGv\n", (RTGCPTR)pCtx->rip, (RTGCPTR)val));
4507 VMXReadVMCS(VMX_VMCS_CTRL_PIN_EXEC_CONTROLS, &val);
4508 Log(("VMX_VMCS_CTRL_PIN_EXEC_CONTROLS %08x\n", val));
4509 VMXReadVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, &val);
4510 Log(("VMX_VMCS_CTRL_PROC_EXEC_CONTROLS %08x\n", val));
4511 VMXReadVMCS(VMX_VMCS_CTRL_ENTRY_CONTROLS, &val);
4512 Log(("VMX_VMCS_CTRL_ENTRY_CONTROLS %08x\n", val));
4513 VMXReadVMCS(VMX_VMCS_CTRL_EXIT_CONTROLS, &val);
4514 Log(("VMX_VMCS_CTRL_EXIT_CONTROLS %08x\n", val));
4515
4516 VMXReadVMCS(VMX_VMCS_HOST_CR0, &val);
4517 Log(("VMX_VMCS_HOST_CR0 %08x\n", val));
4518
4519 VMXReadVMCS(VMX_VMCS_HOST_CR3, &val);
4520 Log(("VMX_VMCS_HOST_CR3 %08x\n", val));
4521
4522 VMXReadVMCS(VMX_VMCS_HOST_CR4, &val);
4523 Log(("VMX_VMCS_HOST_CR4 %08x\n", val));
4524
4525 VMXReadVMCS(VMX_VMCS16_HOST_FIELD_CS, &val);
4526 Log(("VMX_VMCS_HOST_FIELD_CS %08x\n", val));
4527
4528 VMXReadVMCS(VMX_VMCS_GUEST_RFLAGS, &val);
4529 Log(("VMX_VMCS_GUEST_RFLAGS %08x\n", val));
4530
4531 if (val < gdtr.cbGdt)
4532 {
4533 pDesc = (PCX86DESCHC)(gdtr.pGdt + (val & X86_SEL_MASK));
4534 HWACCMR0DumpDescriptor(pDesc, val, "CS: ");
4535 }
4536
4537 VMXReadVMCS(VMX_VMCS16_HOST_FIELD_DS, &val);
4538 Log(("VMX_VMCS_HOST_FIELD_DS %08x\n", val));
4539 if (val < gdtr.cbGdt)
4540 {
4541 pDesc = (PCX86DESCHC)(gdtr.pGdt + (val & X86_SEL_MASK));
4542 HWACCMR0DumpDescriptor(pDesc, val, "DS: ");
4543 }
4544
4545 VMXReadVMCS(VMX_VMCS16_HOST_FIELD_ES, &val);
4546 Log(("VMX_VMCS_HOST_FIELD_ES %08x\n", val));
4547 if (val < gdtr.cbGdt)
4548 {
4549 pDesc = (PCX86DESCHC)(gdtr.pGdt + (val & X86_SEL_MASK));
4550 HWACCMR0DumpDescriptor(pDesc, val, "ES: ");
4551 }
4552
4553 VMXReadVMCS(VMX_VMCS16_HOST_FIELD_FS, &val);
4554 Log(("VMX_VMCS16_HOST_FIELD_FS %08x\n", val));
4555 if (val < gdtr.cbGdt)
4556 {
4557 pDesc = (PCX86DESCHC)(gdtr.pGdt + (val & X86_SEL_MASK));
4558 HWACCMR0DumpDescriptor(pDesc, val, "FS: ");
4559 }
4560
4561 VMXReadVMCS(VMX_VMCS16_HOST_FIELD_GS, &val);
4562 Log(("VMX_VMCS16_HOST_FIELD_GS %08x\n", val));
4563 if (val < gdtr.cbGdt)
4564 {
4565 pDesc = (PCX86DESCHC)(gdtr.pGdt + (val & X86_SEL_MASK));
4566 HWACCMR0DumpDescriptor(pDesc, val, "GS: ");
4567 }
4568
4569 VMXReadVMCS(VMX_VMCS16_HOST_FIELD_SS, &val);
4570 Log(("VMX_VMCS16_HOST_FIELD_SS %08x\n", val));
4571 if (val < gdtr.cbGdt)
4572 {
4573 pDesc = (PCX86DESCHC)(gdtr.pGdt + (val & X86_SEL_MASK));
4574 HWACCMR0DumpDescriptor(pDesc, val, "SS: ");
4575 }
4576
4577 VMXReadVMCS(VMX_VMCS16_HOST_FIELD_TR, &val);
4578 Log(("VMX_VMCS16_HOST_FIELD_TR %08x\n", val));
4579 if (val < gdtr.cbGdt)
4580 {
4581 pDesc = (PCX86DESCHC)(gdtr.pGdt + (val & X86_SEL_MASK));
4582 HWACCMR0DumpDescriptor(pDesc, val, "TR: ");
4583 }
4584
4585 VMXReadVMCS(VMX_VMCS_HOST_TR_BASE, &val);
4586 Log(("VMX_VMCS_HOST_TR_BASE %RHv\n", val));
4587
4588 VMXReadVMCS(VMX_VMCS_HOST_GDTR_BASE, &val);
4589 Log(("VMX_VMCS_HOST_GDTR_BASE %RHv\n", val));
4590 VMXReadVMCS(VMX_VMCS_HOST_IDTR_BASE, &val);
4591 Log(("VMX_VMCS_HOST_IDTR_BASE %RHv\n", val));
4592
4593 VMXReadVMCS(VMX_VMCS32_HOST_SYSENTER_CS, &val);
4594 Log(("VMX_VMCS_HOST_SYSENTER_CS %08x\n", val));
4595
4596 VMXReadVMCS(VMX_VMCS_HOST_SYSENTER_EIP, &val);
4597 Log(("VMX_VMCS_HOST_SYSENTER_EIP %RHv\n", val));
4598
4599 VMXReadVMCS(VMX_VMCS_HOST_SYSENTER_ESP, &val);
4600 Log(("VMX_VMCS_HOST_SYSENTER_ESP %RHv\n", val));
4601
4602 VMXReadVMCS(VMX_VMCS_HOST_RSP, &val);
4603 Log(("VMX_VMCS_HOST_RSP %RHv\n", val));
4604 VMXReadVMCS(VMX_VMCS_HOST_RIP, &val);
4605 Log(("VMX_VMCS_HOST_RIP %RHv\n", val));
4606
4607# if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4608 if (VMX_IS_64BIT_HOST_MODE())
4609 {
4610 Log(("MSR_K6_EFER = %RX64\n", ASMRdMsr(MSR_K6_EFER)));
4611 Log(("MSR_K6_STAR = %RX64\n", ASMRdMsr(MSR_K6_STAR)));
4612 Log(("MSR_K8_LSTAR = %RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
4613 Log(("MSR_K8_CSTAR = %RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
4614 Log(("MSR_K8_SF_MASK = %RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
4615 }
4616# endif
4617#endif /* VBOX_STRICT */
4618 }
4619 break;
4620 }
4621
4622 default:
4623 /* impossible */
4624 AssertMsgFailed(("%Rrc (%#x)\n", VBOXSTRICTRC_VAL(rc), VBOXSTRICTRC_VAL(rc)));
4625 break;
4626 }
4627}
4628
4629#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4630
4631/**
4632 * Prepares for and executes VMLAUNCH (64 bits guest mode)
4633 *
4634 * @returns VBox status code
4635 * @param fResume vmlauch/vmresume
4636 * @param pCtx Guest context
4637 * @param pCache VMCS cache
4638 * @param pVM The VM to operate on.
4639 * @param pVCpu The VMCPU to operate on.
4640 */
4641DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
4642{
4643 uint32_t aParam[6];
4644 PHMGLOBLCPUINFO pCpu;
4645 RTHCPHYS HCPhysCpuPage;
4646 int rc;
4647
4648 pCpu = HWACCMR0GetCurrentCpu();
4649 HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
4650
4651#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4652 pCache->uPos = 1;
4653 pCache->interPD = PGMGetInterPaeCR3(pVM);
4654 pCache->pSwitcher = (uint64_t)pVM->hwaccm.s.pfnHost32ToGuest64R0;
4655#endif
4656
4657#ifdef DEBUG
4658 pCache->TestIn.HCPhysCpuPage= 0;
4659 pCache->TestIn.HCPhysVMCS = 0;
4660 pCache->TestIn.pCache = 0;
4661 pCache->TestOut.HCPhysVMCS = 0;
4662 pCache->TestOut.pCache = 0;
4663 pCache->TestOut.pCtx = 0;
4664 pCache->TestOut.eflags = 0;
4665#endif
4666
4667 aParam[0] = (uint32_t)(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
4668 aParam[1] = (uint32_t)(HCPhysCpuPage >> 32); /* Param 1: VMXON physical address - Hi. */
4669 aParam[2] = (uint32_t)(pVCpu->hwaccm.s.vmx.HCPhysVMCS); /* Param 2: VMCS physical address - Lo. */
4670 aParam[3] = (uint32_t)(pVCpu->hwaccm.s.vmx.HCPhysVMCS >> 32); /* Param 2: VMCS physical address - Hi. */
4671 aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hwaccm.s.vmx.VMCSCache);
4672 aParam[5] = 0;
4673
4674#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4675 pCtx->dr[4] = pVM->hwaccm.s.vmx.pScratchPhys + 16 + 8;
4676 *(uint32_t *)(pVM->hwaccm.s.vmx.pScratch + 16 + 8) = 1;
4677#endif
4678 rc = VMXR0Execute64BitsHandler(pVM, pVCpu, pCtx, pVM->hwaccm.s.pfnVMXGCStartVM64, 6, &aParam[0]);
4679
4680#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4681 Assert(*(uint32_t *)(pVM->hwaccm.s.vmx.pScratch + 16 + 8) == 5);
4682 Assert(pCtx->dr[4] == 10);
4683 *(uint32_t *)(pVM->hwaccm.s.vmx.pScratch + 16 + 8) = 0xff;
4684#endif
4685
4686#ifdef DEBUG
4687 AssertMsg(pCache->TestIn.HCPhysCpuPage== HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
4688 AssertMsg(pCache->TestIn.HCPhysVMCS == pVCpu->hwaccm.s.vmx.HCPhysVMCS, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVMCS, pVCpu->hwaccm.s.vmx.HCPhysVMCS));
4689 AssertMsg(pCache->TestIn.HCPhysVMCS == pCache->TestOut.HCPhysVMCS, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVMCS, pCache->TestOut.HCPhysVMCS));
4690 AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache, pCache->TestOut.pCache));
4691 AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hwaccm.s.vmx.VMCSCache), ("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hwaccm.s.vmx.VMCSCache)));
4692 AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx, pCache->TestOut.pCtx));
4693 Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
4694#endif
4695 return rc;
4696}
4697
4698# ifdef VBOX_STRICT
4699
4700static bool hmR0VmxIsValidReadField(uint32_t idxField)
4701{
4702 switch(idxField)
4703 {
4704 case VMX_VMCS64_GUEST_RIP:
4705 case VMX_VMCS64_GUEST_RSP:
4706 case VMX_VMCS_GUEST_RFLAGS:
4707 case VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE:
4708 case VMX_VMCS_CTRL_CR0_READ_SHADOW:
4709 case VMX_VMCS64_GUEST_CR0:
4710 case VMX_VMCS_CTRL_CR4_READ_SHADOW:
4711 case VMX_VMCS64_GUEST_CR4:
4712 case VMX_VMCS64_GUEST_DR7:
4713 case VMX_VMCS32_GUEST_SYSENTER_CS:
4714 case VMX_VMCS64_GUEST_SYSENTER_EIP:
4715 case VMX_VMCS64_GUEST_SYSENTER_ESP:
4716 case VMX_VMCS32_GUEST_GDTR_LIMIT:
4717 case VMX_VMCS64_GUEST_GDTR_BASE:
4718 case VMX_VMCS32_GUEST_IDTR_LIMIT:
4719 case VMX_VMCS64_GUEST_IDTR_BASE:
4720 case VMX_VMCS16_GUEST_FIELD_CS:
4721 case VMX_VMCS32_GUEST_CS_LIMIT:
4722 case VMX_VMCS64_GUEST_CS_BASE:
4723 case VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS:
4724 case VMX_VMCS16_GUEST_FIELD_DS:
4725 case VMX_VMCS32_GUEST_DS_LIMIT:
4726 case VMX_VMCS64_GUEST_DS_BASE:
4727 case VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS:
4728 case VMX_VMCS16_GUEST_FIELD_ES:
4729 case VMX_VMCS32_GUEST_ES_LIMIT:
4730 case VMX_VMCS64_GUEST_ES_BASE:
4731 case VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS:
4732 case VMX_VMCS16_GUEST_FIELD_FS:
4733 case VMX_VMCS32_GUEST_FS_LIMIT:
4734 case VMX_VMCS64_GUEST_FS_BASE:
4735 case VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS:
4736 case VMX_VMCS16_GUEST_FIELD_GS:
4737 case VMX_VMCS32_GUEST_GS_LIMIT:
4738 case VMX_VMCS64_GUEST_GS_BASE:
4739 case VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS:
4740 case VMX_VMCS16_GUEST_FIELD_SS:
4741 case VMX_VMCS32_GUEST_SS_LIMIT:
4742 case VMX_VMCS64_GUEST_SS_BASE:
4743 case VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS:
4744 case VMX_VMCS16_GUEST_FIELD_LDTR:
4745 case VMX_VMCS32_GUEST_LDTR_LIMIT:
4746 case VMX_VMCS64_GUEST_LDTR_BASE:
4747 case VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS:
4748 case VMX_VMCS16_GUEST_FIELD_TR:
4749 case VMX_VMCS32_GUEST_TR_LIMIT:
4750 case VMX_VMCS64_GUEST_TR_BASE:
4751 case VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS:
4752 case VMX_VMCS32_RO_EXIT_REASON:
4753 case VMX_VMCS32_RO_VM_INSTR_ERROR:
4754 case VMX_VMCS32_RO_EXIT_INSTR_LENGTH:
4755 case VMX_VMCS32_RO_EXIT_INTERRUPTION_ERRCODE:
4756 case VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO:
4757 case VMX_VMCS32_RO_EXIT_INSTR_INFO:
4758 case VMX_VMCS_RO_EXIT_QUALIFICATION:
4759 case VMX_VMCS32_RO_IDT_INFO:
4760 case VMX_VMCS32_RO_IDT_ERRCODE:
4761 case VMX_VMCS64_GUEST_CR3:
4762 case VMX_VMCS_EXIT_PHYS_ADDR_FULL:
4763 return true;
4764 }
4765 return false;
4766}
4767
4768static bool hmR0VmxIsValidWriteField(uint32_t idxField)
4769{
4770 switch(idxField)
4771 {
4772 case VMX_VMCS64_GUEST_LDTR_BASE:
4773 case VMX_VMCS64_GUEST_TR_BASE:
4774 case VMX_VMCS64_GUEST_GDTR_BASE:
4775 case VMX_VMCS64_GUEST_IDTR_BASE:
4776 case VMX_VMCS64_GUEST_SYSENTER_EIP:
4777 case VMX_VMCS64_GUEST_SYSENTER_ESP:
4778 case VMX_VMCS64_GUEST_CR0:
4779 case VMX_VMCS64_GUEST_CR4:
4780 case VMX_VMCS64_GUEST_CR3:
4781 case VMX_VMCS64_GUEST_DR7:
4782 case VMX_VMCS64_GUEST_RIP:
4783 case VMX_VMCS64_GUEST_RSP:
4784 case VMX_VMCS64_GUEST_CS_BASE:
4785 case VMX_VMCS64_GUEST_DS_BASE:
4786 case VMX_VMCS64_GUEST_ES_BASE:
4787 case VMX_VMCS64_GUEST_FS_BASE:
4788 case VMX_VMCS64_GUEST_GS_BASE:
4789 case VMX_VMCS64_GUEST_SS_BASE:
4790 return true;
4791 }
4792 return false;
4793}
4794
4795# endif /* VBOX_STRICT */
4796
4797/**
4798 * Executes the specified handler in 64 mode
4799 *
4800 * @returns VBox status code.
4801 * @param pVM The VM to operate on.
4802 * @param pVCpu The VMCPU to operate on.
4803 * @param pCtx Guest context
4804 * @param pfnHandler RC handler
4805 * @param cbParam Number of parameters
4806 * @param paParam Array of 32 bits parameters
4807 */
4808VMMR0DECL(int) VMXR0Execute64BitsHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, RTRCPTR pfnHandler, uint32_t cbParam, uint32_t *paParam)
4809{
4810 int rc, rc2;
4811 PHMGLOBLCPUINFO pCpu;
4812 RTHCPHYS HCPhysCpuPage;
4813 RTHCUINTREG uOldEFlags;
4814
4815 AssertReturn(pVM->hwaccm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
4816 Assert(pfnHandler);
4817 Assert(pVCpu->hwaccm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hwaccm.s.vmx.VMCSCache.Write.aField));
4818 Assert(pVCpu->hwaccm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hwaccm.s.vmx.VMCSCache.Read.aField));
4819
4820#ifdef VBOX_STRICT
4821 for (unsigned i=0;i<pVCpu->hwaccm.s.vmx.VMCSCache.Write.cValidEntries;i++)
4822 Assert(hmR0VmxIsValidWriteField(pVCpu->hwaccm.s.vmx.VMCSCache.Write.aField[i]));
4823
4824 for (unsigned i=0;i<pVCpu->hwaccm.s.vmx.VMCSCache.Read.cValidEntries;i++)
4825 Assert(hmR0VmxIsValidReadField(pVCpu->hwaccm.s.vmx.VMCSCache.Read.aField[i]));
4826#endif
4827
4828 /* Disable interrupts. */
4829 uOldEFlags = ASMIntDisableFlags();
4830
4831 pCpu = HWACCMR0GetCurrentCpu();
4832 HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
4833
4834 /* Clear VM Control Structure. Marking it inactive, clearing implementation specific data and writing back VMCS data to memory. */
4835 VMXClearVMCS(pVCpu->hwaccm.s.vmx.HCPhysVMCS);
4836
4837 /* Leave VMX Root Mode. */
4838 VMXDisable();
4839
4840 ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
4841
4842 CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
4843 CPUMSetHyperEIP(pVCpu, pfnHandler);
4844 for (int i=(int)cbParam-1;i>=0;i--)
4845 CPUMPushHyper(pVCpu, paParam[i]);
4846
4847 STAM_PROFILE_ADV_START(&pVCpu->hwaccm.s.StatWorldSwitch3264, z);
4848 /* Call switcher. */
4849 rc = pVM->hwaccm.s.pfnHost32ToGuest64R0(pVM, RT_OFFSETOF(VM, aCpus[pVCpu->idCpu].cpum) - RT_OFFSETOF(VM, cpum));
4850 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatWorldSwitch3264, z);
4851
4852 /* Make sure the VMX instructions don't cause #UD faults. */
4853 ASMSetCR4(ASMGetCR4() | X86_CR4_VMXE);
4854
4855 /* Enter VMX Root Mode */
4856 rc2 = VMXEnable(HCPhysCpuPage);
4857 if (RT_FAILURE(rc2))
4858 {
4859 ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
4860 ASMSetFlags(uOldEFlags);
4861 return VERR_VMX_VMXON_FAILED;
4862 }
4863
4864 rc2 = VMXActivateVMCS(pVCpu->hwaccm.s.vmx.HCPhysVMCS);
4865 AssertRC(rc2);
4866 Assert(!(ASMGetFlags() & X86_EFL_IF));
4867 ASMSetFlags(uOldEFlags);
4868 return rc;
4869}
4870
4871#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL) */
4872
4873
4874#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
4875/**
4876 * Executes VMWRITE
4877 *
4878 * @returns VBox status code
4879 * @param pVCpu The VMCPU to operate on.
4880 * @param idxField VMCS index
4881 * @param u64Val 16, 32 or 64 bits value
4882 */
4883VMMR0DECL(int) VMXWriteVMCS64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
4884{
4885 int rc;
4886
4887 switch (idxField)
4888 {
4889 case VMX_VMCS_CTRL_TSC_OFFSET_FULL:
4890 case VMX_VMCS_CTRL_IO_BITMAP_A_FULL:
4891 case VMX_VMCS_CTRL_IO_BITMAP_B_FULL:
4892 case VMX_VMCS_CTRL_MSR_BITMAP_FULL:
4893 case VMX_VMCS_CTRL_VMEXIT_MSR_STORE_FULL:
4894 case VMX_VMCS_CTRL_VMEXIT_MSR_LOAD_FULL:
4895 case VMX_VMCS_CTRL_VMENTRY_MSR_LOAD_FULL:
4896 case VMX_VMCS_CTRL_VAPIC_PAGEADDR_FULL:
4897 case VMX_VMCS_CTRL_APIC_ACCESSADDR_FULL:
4898 case VMX_VMCS_GUEST_LINK_PTR_FULL:
4899 case VMX_VMCS_GUEST_PDPTR0_FULL:
4900 case VMX_VMCS_GUEST_PDPTR1_FULL:
4901 case VMX_VMCS_GUEST_PDPTR2_FULL:
4902 case VMX_VMCS_GUEST_PDPTR3_FULL:
4903 case VMX_VMCS_GUEST_DEBUGCTL_FULL:
4904 case VMX_VMCS_GUEST_EFER_FULL:
4905 case VMX_VMCS_CTRL_EPTP_FULL:
4906 /* These fields consist of two parts, which are both writable in 32 bits mode. */
4907 rc = VMXWriteVMCS32(idxField, u64Val);
4908 rc |= VMXWriteVMCS32(idxField + 1, (uint32_t)(u64Val >> 32ULL));
4909 AssertRC(rc);
4910 return rc;
4911
4912 case VMX_VMCS64_GUEST_LDTR_BASE:
4913 case VMX_VMCS64_GUEST_TR_BASE:
4914 case VMX_VMCS64_GUEST_GDTR_BASE:
4915 case VMX_VMCS64_GUEST_IDTR_BASE:
4916 case VMX_VMCS64_GUEST_SYSENTER_EIP:
4917 case VMX_VMCS64_GUEST_SYSENTER_ESP:
4918 case VMX_VMCS64_GUEST_CR0:
4919 case VMX_VMCS64_GUEST_CR4:
4920 case VMX_VMCS64_GUEST_CR3:
4921 case VMX_VMCS64_GUEST_DR7:
4922 case VMX_VMCS64_GUEST_RIP:
4923 case VMX_VMCS64_GUEST_RSP:
4924 case VMX_VMCS64_GUEST_CS_BASE:
4925 case VMX_VMCS64_GUEST_DS_BASE:
4926 case VMX_VMCS64_GUEST_ES_BASE:
4927 case VMX_VMCS64_GUEST_FS_BASE:
4928 case VMX_VMCS64_GUEST_GS_BASE:
4929 case VMX_VMCS64_GUEST_SS_BASE:
4930 /* Queue a 64 bits value as we can't set it in 32 bits host mode. */
4931 if (u64Val >> 32ULL)
4932 rc = VMXWriteCachedVMCSEx(pVCpu, idxField, u64Val);
4933 else
4934 rc = VMXWriteVMCS32(idxField, (uint32_t)u64Val);
4935
4936 return rc;
4937
4938 default:
4939 AssertMsgFailed(("Unexpected field %x\n", idxField));
4940 return VERR_INVALID_PARAMETER;
4941 }
4942}
4943
4944/**
4945 * Cache VMCS writes for performance reasons (Darwin) and for running 64 bits guests on 32 bits hosts.
4946 *
4947 * @param pVCpu The VMCPU to operate on.
4948 * @param idxField VMCS field
4949 * @param u64Val Value
4950 */
4951VMMR0DECL(int) VMXWriteCachedVMCSEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
4952{
4953 PVMCSCACHE pCache = &pVCpu->hwaccm.s.vmx.VMCSCache;
4954
4955 AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1, ("entries=%x\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
4956
4957 /* Make sure there are no duplicates. */
4958 for (unsigned i=0;i<pCache->Write.cValidEntries;i++)
4959 {
4960 if (pCache->Write.aField[i] == idxField)
4961 {
4962 pCache->Write.aFieldVal[i] = u64Val;
4963 return VINF_SUCCESS;
4964 }
4965 }
4966
4967 pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
4968 pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
4969 pCache->Write.cValidEntries++;
4970 return VINF_SUCCESS;
4971}
4972
4973#endif /* HC_ARCH_BITS == 32 && !VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0 */
4974
Note: See TracBrowser for help on using the repository browser.

© 2025 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette