VirtualBox

source: vbox/trunk/src/VBox/VMM/VMMR0/HWVMXR0.cpp@ 40832

Last change on this file since 40832 was 40832, checked in by vboxsync, 13 years ago

HWVMXR0: added a exit reason probe for testing dtrace on windows.
  • Property svn:eol-style set to native
  • Property svn:keywords set to Id
File size: 200.4 KB
Line 
1/* $Id: HWVMXR0.cpp 40832 2012-04-08 19:55:07Z vboxsync $ */
2/** @file
3 * HM VMX (VT-x) - Host Context Ring 0.
4 */
5
6/*
7 * Copyright (C) 2006-2011 Oracle Corporation
8 *
9 * This file is part of VirtualBox Open Source Edition (OSE), as
10 * available from http://www.virtualbox.org. This file is free software;
11 * you can redistribute it and/or modify it under the terms of the GNU
12 * General Public License (GPL) as published by the Free Software
13 * Foundation, in version 2 as it comes in the "COPYING" file of the
14 * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15 * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16 */
17
18
19/*******************************************************************************
20* Header Files *
21*******************************************************************************/
22#define LOG_GROUP LOG_GROUP_HWACCM
23#include <iprt/asm-amd64-x86.h>
24#include <VBox/vmm/hwaccm.h>
25#include <VBox/vmm/pgm.h>
26#include <VBox/vmm/dbgf.h>
27#include <VBox/vmm/dbgftrace.h>
28#include <VBox/vmm/selm.h>
29#include <VBox/vmm/iom.h>
30#ifdef VBOX_WITH_REM
31# include <VBox/vmm/rem.h>
32#endif
33#include <VBox/vmm/tm.h>
34#include "HWACCMInternal.h"
35#include <VBox/vmm/vm.h>
36#include <VBox/vmm/pdmapi.h>
37#include <VBox/err.h>
38#include <VBox/log.h>
39#include <iprt/assert.h>
40#include <iprt/param.h>
41#include <iprt/string.h>
42#include <iprt/time.h>
43#ifdef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
44# include <iprt/thread.h>
45#endif
46#include <iprt/x86.h>
47#include "HWVMXR0.h"
48
49#include "dtrace/VBoxVMM.h"
50
51
52/*******************************************************************************
53* Defined Constants And Macros *
54*******************************************************************************/
55#if defined(RT_ARCH_AMD64)
56# define VMX_IS_64BIT_HOST_MODE() (true)
57#elif defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
58# define VMX_IS_64BIT_HOST_MODE() (g_fVMXIs64bitHost != 0)
59#else
60# define VMX_IS_64BIT_HOST_MODE() (false)
61#endif
62
63/*******************************************************************************
64* Global Variables *
65*******************************************************************************/
66/* IO operation lookup arrays. */
67static uint32_t const g_aIOSize[4] = {1, 2, 0, 4};
68static uint32_t const g_aIOOpAnd[4] = {0xff, 0xffff, 0, 0xffffffff};
69
70#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
71/** See HWACCMR0A.asm. */
72extern "C" uint32_t g_fVMXIs64bitHost;
73#endif
74
75/*******************************************************************************
76* Local Functions *
77*******************************************************************************/
78static void hmR0VmxReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, VBOXSTRICTRC rc, PCPUMCTX pCtx);
79static DECLCALLBACK(void) hmR0VmxSetupTLBEPT(PVM pVM, PVMCPU pVCpu);
80static DECLCALLBACK(void) hmR0VmxSetupTLBVPID(PVM pVM, PVMCPU pVCpu);
81static DECLCALLBACK(void) hmR0VmxSetupTLBDummy(PVM pVM, PVMCPU pVCpu);
82static void hmR0VmxFlushEPT(PVM pVM, PVMCPU pVCpu, VMX_FLUSH enmFlush, RTGCPHYS GCPhys);
83static void hmR0VmxFlushVPID(PVM pVM, PVMCPU pVCpu, VMX_FLUSH enmFlush, RTGCPTR GCPtr);
84static void hmR0VmxUpdateExceptionBitmap(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx);
85static void hmR0VmxSetMSRPermission(PVMCPU pVCpu, unsigned ulMSR, bool fRead, bool fWrite);
86
87
88static void hmR0VmxCheckError(PVM pVM, PVMCPU pVCpu, int rc)
89{
90 if (rc == VERR_VMX_GENERIC)
91 {
92 RTCCUINTREG instrError;
93
94 VMXReadVMCS(VMX_VMCS32_RO_VM_INSTR_ERROR, &instrError);
95 pVCpu->hwaccm.s.vmx.lasterror.ulInstrError = instrError;
96 }
97 pVM->hwaccm.s.lLastError = rc;
98}
99
100/**
101 * Sets up and activates VT-x on the current CPU
102 *
103 * @returns VBox status code.
104 * @param pCpu CPU info struct
105 * @param pVM The VM to operate on. (can be NULL after a resume!!)
106 * @param pvCpuPage Pointer to the global cpu page.
107 * @param HCPhysCpuPage Physical address of the global cpu page.
108 */
109VMMR0DECL(int) VMXR0EnableCpu(PHMGLOBLCPUINFO pCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
110{
111 AssertReturn(HCPhysCpuPage != 0 && HCPhysCpuPage != NIL_RTHCPHYS, VERR_INVALID_PARAMETER);
112 AssertReturn(pvCpuPage, VERR_INVALID_PARAMETER);
113 NOREF(pCpu);
114
115 if (pVM)
116 {
117 /* Set revision dword at the beginning of the VMXON structure. */
118 *(uint32_t *)pvCpuPage = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hwaccm.s.vmx.msr.vmx_basic_info);
119 }
120
121 /** @todo we should unmap the two pages from the virtual address space in order to prevent accidental corruption.
122 * (which can have very bad consequences!!!)
123 */
124
125 if (ASMGetCR4() & X86_CR4_VMXE)
126 return VERR_VMX_IN_VMX_ROOT_MODE;
127
128 /* Make sure the VMX instructions don't cause #UD faults. */
129 ASMSetCR4(ASMGetCR4() | X86_CR4_VMXE);
130
131 /* Enter VMX Root Mode. */
132 int rc = VMXEnable(HCPhysCpuPage);
133 if (RT_FAILURE(rc))
134 {
135 ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
136 return VERR_VMX_VMXON_FAILED;
137 }
138 return VINF_SUCCESS;
139}
140
141/**
142 * Deactivates VT-x on the current CPU
143 *
144 * @returns VBox status code.
145 * @param pCpu CPU info struct
146 * @param pvCpuPage Pointer to the global cpu page.
147 * @param HCPhysCpuPage Physical address of the global cpu page.
148 */
149VMMR0DECL(int) VMXR0DisableCpu(PHMGLOBLCPUINFO pCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
150{
151 AssertReturn(HCPhysCpuPage != 0 && HCPhysCpuPage != NIL_RTHCPHYS, VERR_INVALID_PARAMETER);
152 AssertReturn(pvCpuPage, VERR_INVALID_PARAMETER);
153 NOREF(pCpu);
154
155 /* If we're somehow not in VMX root mode, then we shouldn't dare leaving it. */
156 if (!(ASMGetCR4() & X86_CR4_VMXE))
157 return VERR_VMX_NOT_IN_VMX_ROOT_MODE;
158
159 /* Leave VMX Root Mode. */
160 VMXDisable();
161
162 /* And clear the X86_CR4_VMXE bit. */
163 ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
164 return VINF_SUCCESS;
165}
166
167/**
168 * Does Ring-0 per VM VT-x init.
169 *
170 * @returns VBox status code.
171 * @param pVM The VM to operate on.
172 */
173VMMR0DECL(int) VMXR0InitVM(PVM pVM)
174{
175 int rc;
176
177#ifdef LOG_ENABLED
178 SUPR0Printf("VMXR0InitVM %x\n", pVM);
179#endif
180
181 pVM->hwaccm.s.vmx.pMemObjAPIC = NIL_RTR0MEMOBJ;
182
183 if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
184 {
185 /* Allocate one page for the APIC physical page (serves for filtering accesses). */
186 rc = RTR0MemObjAllocCont(&pVM->hwaccm.s.vmx.pMemObjAPIC, PAGE_SIZE, true /* executable R0 mapping */);
187 AssertRC(rc);
188 if (RT_FAILURE(rc))
189 return rc;
190
191 pVM->hwaccm.s.vmx.pAPIC = (uint8_t *)RTR0MemObjAddress(pVM->hwaccm.s.vmx.pMemObjAPIC);
192 pVM->hwaccm.s.vmx.pAPICPhys = RTR0MemObjGetPagePhysAddr(pVM->hwaccm.s.vmx.pMemObjAPIC, 0);
193 ASMMemZero32(pVM->hwaccm.s.vmx.pAPIC, PAGE_SIZE);
194 }
195 else
196 {
197 pVM->hwaccm.s.vmx.pMemObjAPIC = 0;
198 pVM->hwaccm.s.vmx.pAPIC = 0;
199 pVM->hwaccm.s.vmx.pAPICPhys = 0;
200 }
201
202#ifdef VBOX_WITH_CRASHDUMP_MAGIC
203 {
204 rc = RTR0MemObjAllocCont(&pVM->hwaccm.s.vmx.pMemObjScratch, PAGE_SIZE, true /* executable R0 mapping */);
205 AssertRC(rc);
206 if (RT_FAILURE(rc))
207 return rc;
208
209 pVM->hwaccm.s.vmx.pScratch = (uint8_t *)RTR0MemObjAddress(pVM->hwaccm.s.vmx.pMemObjScratch);
210 pVM->hwaccm.s.vmx.pScratchPhys = RTR0MemObjGetPagePhysAddr(pVM->hwaccm.s.vmx.pMemObjScratch, 0);
211
212 ASMMemZero32(pVM->hwaccm.s.vmx.pScratch, PAGE_SIZE);
213 strcpy((char *)pVM->hwaccm.s.vmx.pScratch, "SCRATCH Magic");
214 *(uint64_t *)(pVM->hwaccm.s.vmx.pScratch + 16) = UINT64_C(0xDEADBEEFDEADBEEF);
215 }
216#endif
217
218 /* Allocate VMCBs for all guest CPUs. */
219 for (VMCPUID i = 0; i < pVM->cCpus; i++)
220 {
221 PVMCPU pVCpu = &pVM->aCpus[i];
222
223 pVCpu->hwaccm.s.vmx.hMemObjVMCS = NIL_RTR0MEMOBJ;
224
225 /* Allocate one page for the VM control structure (VMCS). */
226 rc = RTR0MemObjAllocCont(&pVCpu->hwaccm.s.vmx.hMemObjVMCS, PAGE_SIZE, true /* executable R0 mapping */);
227 AssertRC(rc);
228 if (RT_FAILURE(rc))
229 return rc;
230
231 pVCpu->hwaccm.s.vmx.pvVMCS = RTR0MemObjAddress(pVCpu->hwaccm.s.vmx.hMemObjVMCS);
232 pVCpu->hwaccm.s.vmx.HCPhysVMCS = RTR0MemObjGetPagePhysAddr(pVCpu->hwaccm.s.vmx.hMemObjVMCS, 0);
233 ASMMemZeroPage(pVCpu->hwaccm.s.vmx.pvVMCS);
234
235 pVCpu->hwaccm.s.vmx.cr0_mask = 0;
236 pVCpu->hwaccm.s.vmx.cr4_mask = 0;
237
238 /* Allocate one page for the virtual APIC page for TPR caching. */
239 rc = RTR0MemObjAllocCont(&pVCpu->hwaccm.s.vmx.hMemObjVAPIC, PAGE_SIZE, true /* executable R0 mapping */);
240 AssertRC(rc);
241 if (RT_FAILURE(rc))
242 return rc;
243
244 pVCpu->hwaccm.s.vmx.pbVAPIC = (uint8_t *)RTR0MemObjAddress(pVCpu->hwaccm.s.vmx.hMemObjVAPIC);
245 pVCpu->hwaccm.s.vmx.HCPhysVAPIC = RTR0MemObjGetPagePhysAddr(pVCpu->hwaccm.s.vmx.hMemObjVAPIC, 0);
246 ASMMemZeroPage(pVCpu->hwaccm.s.vmx.pbVAPIC);
247
248 /* Allocate the MSR bitmap if this feature is supported. */
249 if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
250 {
251 rc = RTR0MemObjAllocCont(&pVCpu->hwaccm.s.vmx.pMemObjMSRBitmap, PAGE_SIZE, true /* executable R0 mapping */);
252 AssertRC(rc);
253 if (RT_FAILURE(rc))
254 return rc;
255
256 pVCpu->hwaccm.s.vmx.pMSRBitmap = (uint8_t *)RTR0MemObjAddress(pVCpu->hwaccm.s.vmx.pMemObjMSRBitmap);
257 pVCpu->hwaccm.s.vmx.pMSRBitmapPhys = RTR0MemObjGetPagePhysAddr(pVCpu->hwaccm.s.vmx.pMemObjMSRBitmap, 0);
258 memset(pVCpu->hwaccm.s.vmx.pMSRBitmap, 0xff, PAGE_SIZE);
259 }
260
261#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
262 /* Allocate one page for the guest MSR load area (for preloading guest MSRs during the world switch). */
263 rc = RTR0MemObjAllocCont(&pVCpu->hwaccm.s.vmx.pMemObjGuestMSR, PAGE_SIZE, true /* executable R0 mapping */);
264 AssertRC(rc);
265 if (RT_FAILURE(rc))
266 return rc;
267
268 pVCpu->hwaccm.s.vmx.pGuestMSR = (uint8_t *)RTR0MemObjAddress(pVCpu->hwaccm.s.vmx.pMemObjGuestMSR);
269 pVCpu->hwaccm.s.vmx.pGuestMSRPhys = RTR0MemObjGetPagePhysAddr(pVCpu->hwaccm.s.vmx.pMemObjGuestMSR, 0);
270 memset(pVCpu->hwaccm.s.vmx.pGuestMSR, 0, PAGE_SIZE);
271
272 /* Allocate one page for the host MSR load area (for restoring host MSRs after the world switch back). */
273 rc = RTR0MemObjAllocCont(&pVCpu->hwaccm.s.vmx.pMemObjHostMSR, PAGE_SIZE, true /* executable R0 mapping */);
274 AssertRC(rc);
275 if (RT_FAILURE(rc))
276 return rc;
277
278 pVCpu->hwaccm.s.vmx.pHostMSR = (uint8_t *)RTR0MemObjAddress(pVCpu->hwaccm.s.vmx.pMemObjHostMSR);
279 pVCpu->hwaccm.s.vmx.pHostMSRPhys = RTR0MemObjGetPagePhysAddr(pVCpu->hwaccm.s.vmx.pMemObjHostMSR, 0);
280 memset(pVCpu->hwaccm.s.vmx.pHostMSR, 0, PAGE_SIZE);
281#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
282
283 /* Current guest paging mode. */
284 pVCpu->hwaccm.s.vmx.enmLastSeenGuestMode = PGMMODE_REAL;
285
286#ifdef LOG_ENABLED
287 SUPR0Printf("VMXR0InitVM %x VMCS=%x (%x)\n", pVM, pVCpu->hwaccm.s.vmx.pvVMCS, (uint32_t)pVCpu->hwaccm.s.vmx.HCPhysVMCS);
288#endif
289 }
290
291 return VINF_SUCCESS;
292}
293
294/**
295 * Does Ring-0 per VM VT-x termination.
296 *
297 * @returns VBox status code.
298 * @param pVM The VM to operate on.
299 */
300VMMR0DECL(int) VMXR0TermVM(PVM pVM)
301{
302 for (VMCPUID i = 0; i < pVM->cCpus; i++)
303 {
304 PVMCPU pVCpu = &pVM->aCpus[i];
305
306 if (pVCpu->hwaccm.s.vmx.hMemObjVMCS != NIL_RTR0MEMOBJ)
307 {
308 RTR0MemObjFree(pVCpu->hwaccm.s.vmx.hMemObjVMCS, false);
309 pVCpu->hwaccm.s.vmx.hMemObjVMCS = NIL_RTR0MEMOBJ;
310 pVCpu->hwaccm.s.vmx.pvVMCS = 0;
311 pVCpu->hwaccm.s.vmx.HCPhysVMCS = 0;
312 }
313 if (pVCpu->hwaccm.s.vmx.hMemObjVAPIC != NIL_RTR0MEMOBJ)
314 {
315 RTR0MemObjFree(pVCpu->hwaccm.s.vmx.hMemObjVAPIC, false);
316 pVCpu->hwaccm.s.vmx.hMemObjVAPIC = NIL_RTR0MEMOBJ;
317 pVCpu->hwaccm.s.vmx.pbVAPIC = 0;
318 pVCpu->hwaccm.s.vmx.HCPhysVAPIC = 0;
319 }
320 if (pVCpu->hwaccm.s.vmx.pMemObjMSRBitmap != NIL_RTR0MEMOBJ)
321 {
322 RTR0MemObjFree(pVCpu->hwaccm.s.vmx.pMemObjMSRBitmap, false);
323 pVCpu->hwaccm.s.vmx.pMemObjMSRBitmap = NIL_RTR0MEMOBJ;
324 pVCpu->hwaccm.s.vmx.pMSRBitmap = 0;
325 pVCpu->hwaccm.s.vmx.pMSRBitmapPhys = 0;
326 }
327#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
328 if (pVCpu->hwaccm.s.vmx.pMemObjHostMSR != NIL_RTR0MEMOBJ)
329 {
330 RTR0MemObjFree(pVCpu->hwaccm.s.vmx.pMemObjHostMSR, false);
331 pVCpu->hwaccm.s.vmx.pMemObjHostMSR = NIL_RTR0MEMOBJ;
332 pVCpu->hwaccm.s.vmx.pHostMSR = 0;
333 pVCpu->hwaccm.s.vmx.pHostMSRPhys = 0;
334 }
335 if (pVCpu->hwaccm.s.vmx.pMemObjGuestMSR != NIL_RTR0MEMOBJ)
336 {
337 RTR0MemObjFree(pVCpu->hwaccm.s.vmx.pMemObjGuestMSR, false);
338 pVCpu->hwaccm.s.vmx.pMemObjGuestMSR = NIL_RTR0MEMOBJ;
339 pVCpu->hwaccm.s.vmx.pGuestMSR = 0;
340 pVCpu->hwaccm.s.vmx.pGuestMSRPhys = 0;
341 }
342#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
343 }
344 if (pVM->hwaccm.s.vmx.pMemObjAPIC != NIL_RTR0MEMOBJ)
345 {
346 RTR0MemObjFree(pVM->hwaccm.s.vmx.pMemObjAPIC, false);
347 pVM->hwaccm.s.vmx.pMemObjAPIC = NIL_RTR0MEMOBJ;
348 pVM->hwaccm.s.vmx.pAPIC = 0;
349 pVM->hwaccm.s.vmx.pAPICPhys = 0;
350 }
351#ifdef VBOX_WITH_CRASHDUMP_MAGIC
352 if (pVM->hwaccm.s.vmx.pMemObjScratch != NIL_RTR0MEMOBJ)
353 {
354 ASMMemZero32(pVM->hwaccm.s.vmx.pScratch, PAGE_SIZE);
355 RTR0MemObjFree(pVM->hwaccm.s.vmx.pMemObjScratch, false);
356 pVM->hwaccm.s.vmx.pMemObjScratch = NIL_RTR0MEMOBJ;
357 pVM->hwaccm.s.vmx.pScratch = 0;
358 pVM->hwaccm.s.vmx.pScratchPhys = 0;
359 }
360#endif
361 return VINF_SUCCESS;
362}
363
364/**
365 * Sets up VT-x for the specified VM
366 *
367 * @returns VBox status code.
368 * @param pVM The VM to operate on.
369 */
370VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
371{
372 int rc = VINF_SUCCESS;
373 uint32_t val;
374
375 AssertReturn(pVM, VERR_INVALID_PARAMETER);
376
377 for (VMCPUID i = 0; i < pVM->cCpus; i++)
378 {
379 PVMCPU pVCpu = &pVM->aCpus[i];
380
381 AssertPtr(pVCpu->hwaccm.s.vmx.pvVMCS);
382
383 /* Set revision dword at the beginning of the VMCS structure. */
384 *(uint32_t *)pVCpu->hwaccm.s.vmx.pvVMCS = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hwaccm.s.vmx.msr.vmx_basic_info);
385
386 /* Clear VM Control Structure. */
387 Log(("HCPhysVMCS = %RHp\n", pVCpu->hwaccm.s.vmx.HCPhysVMCS));
388 rc = VMXClearVMCS(pVCpu->hwaccm.s.vmx.HCPhysVMCS);
389 if (RT_FAILURE(rc))
390 goto vmx_end;
391
392 /* Activate the VM Control Structure. */
393 rc = VMXActivateVMCS(pVCpu->hwaccm.s.vmx.HCPhysVMCS);
394 if (RT_FAILURE(rc))
395 goto vmx_end;
396
397 /* VMX_VMCS_CTRL_PIN_EXEC_CONTROLS
398 * Set required bits to one and zero according to the MSR capabilities.
399 */
400 val = pVM->hwaccm.s.vmx.msr.vmx_pin_ctls.n.disallowed0;
401 /* External and non-maskable interrupts cause VM-exits. */
402 val |= VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_EXT_INT_EXIT | VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_NMI_EXIT;
403 /* enable the preemption timer. */
404 if (pVM->hwaccm.s.vmx.fUsePreemptTimer)
405 val |= VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_PREEMPT_TIMER;
406 val &= pVM->hwaccm.s.vmx.msr.vmx_pin_ctls.n.allowed1;
407
408 rc = VMXWriteVMCS(VMX_VMCS_CTRL_PIN_EXEC_CONTROLS, val);
409 AssertRC(rc);
410
411 /* VMX_VMCS_CTRL_PROC_EXEC_CONTROLS
412 * Set required bits to one and zero according to the MSR capabilities.
413 */
414 val = pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.disallowed0;
415 /* Program which event cause VM-exits and which features we want to use. */
416 val = val | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_HLT_EXIT
417 | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_TSC_OFFSET
418 | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT
419 | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_UNCOND_IO_EXIT
420 | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDPMC_EXIT
421 | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_EXIT
422 | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MWAIT_EXIT; /* don't execute mwait or else we'll idle inside the guest (host thinks the cpu load is high) */
423
424 /* Without nested paging we should intercept invlpg and cr3 mov instructions. */
425 if (!pVM->hwaccm.s.fNestedPaging)
426 val |= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INVLPG_EXIT
427 | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
428 | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT;
429
430 /* Note: VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MWAIT_EXIT might cause a vmlaunch failure with an invalid control fields error. (combined with some other exit reasons) */
431 if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
432 {
433 /* CR8 reads from the APIC shadow page; writes cause an exit is they lower the TPR below the threshold */
434 val |= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW;
435 Assert(pVM->hwaccm.s.vmx.pAPIC);
436 }
437 else
438 /* Exit on CR8 reads & writes in case the TPR shadow feature isn't present. */
439 val |= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR8_STORE_EXIT | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR8_LOAD_EXIT;
440
441 if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
442 {
443 Assert(pVCpu->hwaccm.s.vmx.pMSRBitmapPhys);
444 val |= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS;
445 }
446
447 /* We will use the secondary control if it's present. */
448 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
449
450 /* Mask away the bits that the CPU doesn't support */
451 /** @todo make sure they don't conflict with the above requirements. */
452 val &= pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1;
453 pVCpu->hwaccm.s.vmx.proc_ctls = val;
454
455 rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, val);
456 AssertRC(rc);
457
458 if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
459 {
460 /* VMX_VMCS_CTRL_PROC_EXEC_CONTROLS2
461 * Set required bits to one and zero according to the MSR capabilities.
462 */
463 val = pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2.n.disallowed0;
464 val |= VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT;
465
466#ifdef HWACCM_VTX_WITH_EPT
467 if (pVM->hwaccm.s.fNestedPaging)
468 val |= VMX_VMCS_CTRL_PROC_EXEC2_EPT;
469#endif /* HWACCM_VTX_WITH_EPT */
470#ifdef HWACCM_VTX_WITH_VPID
471 else
472 if (pVM->hwaccm.s.vmx.fVPID)
473 val |= VMX_VMCS_CTRL_PROC_EXEC2_VPID;
474#endif /* HWACCM_VTX_WITH_VPID */
475
476 if (pVM->hwaccm.s.fHasIoApic)
477 val |= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC;
478
479 if (pVM->hwaccm.s.vmx.fUnrestrictedGuest)
480 val |= VMX_VMCS_CTRL_PROC_EXEC2_REAL_MODE;
481
482 /* Mask away the bits that the CPU doesn't support */
483 /** @todo make sure they don't conflict with the above requirements. */
484 val &= pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2.n.allowed1;
485 pVCpu->hwaccm.s.vmx.proc_ctls2 = val;
486 rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS2, val);
487 AssertRC(rc);
488 }
489
490 /* VMX_VMCS_CTRL_CR3_TARGET_COUNT
491 * Set required bits to one and zero according to the MSR capabilities.
492 */
493 rc = VMXWriteVMCS(VMX_VMCS_CTRL_CR3_TARGET_COUNT, 0);
494 AssertRC(rc);
495
496 /* Forward all exception except #NM & #PF to the guest.
497 * We always need to check pagefaults since our shadow page table can be out of sync.
498 * And we always lazily sync the FPU & XMM state.
499 */
500
501 /** @todo Possible optimization:
502 * Keep the FPU and XMM state current in the EM thread. That way there's no need to
503 * lazily sync anything, but the downside is that we can't use the FPU stack or XMM
504 * registers ourselves of course.
505 *
506 * Note: only possible if the current state is actually ours (X86_CR0_TS flag)
507 */
508
509 /* Don't filter page faults; all of them should cause a switch. */
510 rc = VMXWriteVMCS(VMX_VMCS_CTRL_PAGEFAULT_ERROR_MASK, 0);
511 rc |= VMXWriteVMCS(VMX_VMCS_CTRL_PAGEFAULT_ERROR_MATCH, 0);
512 AssertRC(rc);
513
514 /* Init TSC offset to zero. */
515 rc = VMXWriteVMCS64(VMX_VMCS_CTRL_TSC_OFFSET_FULL, 0);
516 AssertRC(rc);
517
518 rc = VMXWriteVMCS64(VMX_VMCS_CTRL_IO_BITMAP_A_FULL, 0);
519 AssertRC(rc);
520
521 rc = VMXWriteVMCS64(VMX_VMCS_CTRL_IO_BITMAP_B_FULL, 0);
522 AssertRC(rc);
523
524 /* Set the MSR bitmap address. */
525 if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
526 {
527 Assert(pVCpu->hwaccm.s.vmx.pMSRBitmapPhys);
528
529 rc = VMXWriteVMCS64(VMX_VMCS_CTRL_MSR_BITMAP_FULL, pVCpu->hwaccm.s.vmx.pMSRBitmapPhys);
530 AssertRC(rc);
531
532 /* Allow the guest to directly modify these MSRs; they are restored and saved automatically. */
533 hmR0VmxSetMSRPermission(pVCpu, MSR_IA32_SYSENTER_CS, true, true);
534 hmR0VmxSetMSRPermission(pVCpu, MSR_IA32_SYSENTER_ESP, true, true);
535 hmR0VmxSetMSRPermission(pVCpu, MSR_IA32_SYSENTER_EIP, true, true);
536 hmR0VmxSetMSRPermission(pVCpu, MSR_K8_LSTAR, true, true);
537 hmR0VmxSetMSRPermission(pVCpu, MSR_K6_STAR, true, true);
538 hmR0VmxSetMSRPermission(pVCpu, MSR_K8_SF_MASK, true, true);
539 hmR0VmxSetMSRPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, true, true);
540 hmR0VmxSetMSRPermission(pVCpu, MSR_K8_GS_BASE, true, true);
541 hmR0VmxSetMSRPermission(pVCpu, MSR_K8_FS_BASE, true, true);
542 }
543
544#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
545 /* Set the guest & host MSR load/store physical addresses. */
546 Assert(pVCpu->hwaccm.s.vmx.pGuestMSRPhys);
547 rc = VMXWriteVMCS64(VMX_VMCS_CTRL_VMENTRY_MSR_LOAD_FULL, pVCpu->hwaccm.s.vmx.pGuestMSRPhys);
548 AssertRC(rc);
549 rc = VMXWriteVMCS64(VMX_VMCS_CTRL_VMEXIT_MSR_STORE_FULL, pVCpu->hwaccm.s.vmx.pGuestMSRPhys);
550 AssertRC(rc);
551
552 Assert(pVCpu->hwaccm.s.vmx.pHostMSRPhys);
553 rc = VMXWriteVMCS64(VMX_VMCS_CTRL_VMEXIT_MSR_LOAD_FULL, pVCpu->hwaccm.s.vmx.pHostMSRPhys);
554 AssertRC(rc);
555#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
556
557 rc = VMXWriteVMCS(VMX_VMCS_CTRL_ENTRY_MSR_LOAD_COUNT, 0);
558 AssertRC(rc);
559
560 rc = VMXWriteVMCS(VMX_VMCS_CTRL_EXIT_MSR_STORE_COUNT, 0);
561 AssertRC(rc);
562
563 if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
564 {
565 Assert(pVM->hwaccm.s.vmx.pMemObjAPIC);
566 /* Optional */
567 rc = VMXWriteVMCS(VMX_VMCS_CTRL_TPR_THRESHOLD, 0);
568 rc |= VMXWriteVMCS64(VMX_VMCS_CTRL_VAPIC_PAGEADDR_FULL, pVCpu->hwaccm.s.vmx.HCPhysVAPIC);
569
570 if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
571 rc |= VMXWriteVMCS64(VMX_VMCS_CTRL_APIC_ACCESSADDR_FULL, pVM->hwaccm.s.vmx.pAPICPhys);
572
573 AssertRC(rc);
574 }
575
576 /* Set link pointer to -1. Not currently used. */
577 rc = VMXWriteVMCS64(VMX_VMCS_GUEST_LINK_PTR_FULL, 0xFFFFFFFFFFFFFFFFULL);
578 AssertRC(rc);
579
580 /* Clear VM Control Structure. Marking it inactive, clearing implementation specific data and writing back VMCS data to memory. */
581 rc = VMXClearVMCS(pVCpu->hwaccm.s.vmx.HCPhysVMCS);
582 AssertRC(rc);
583
584 /* Configure the VMCS read cache. */
585 PVMCSCACHE pCache = &pVCpu->hwaccm.s.vmx.VMCSCache;
586
587 VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_RIP);
588 VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_RSP);
589 VMXSetupCachedReadVMCS(pCache, VMX_VMCS_GUEST_RFLAGS);
590 VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE);
591 VMXSetupCachedReadVMCS(pCache, VMX_VMCS_CTRL_CR0_READ_SHADOW);
592 VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_CR0);
593 VMXSetupCachedReadVMCS(pCache, VMX_VMCS_CTRL_CR4_READ_SHADOW);
594 VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_CR4);
595 VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_DR7);
596 VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_GUEST_SYSENTER_CS);
597 VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_SYSENTER_EIP);
598 VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_SYSENTER_ESP);
599 VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_GUEST_GDTR_LIMIT);
600 VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_GDTR_BASE);
601 VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_GUEST_IDTR_LIMIT);
602 VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_IDTR_BASE);
603
604 VMX_SETUP_SELREG(ES, pCache);
605 VMX_SETUP_SELREG(SS, pCache);
606 VMX_SETUP_SELREG(CS, pCache);
607 VMX_SETUP_SELREG(DS, pCache);
608 VMX_SETUP_SELREG(FS, pCache);
609 VMX_SETUP_SELREG(GS, pCache);
610 VMX_SETUP_SELREG(LDTR, pCache);
611 VMX_SETUP_SELREG(TR, pCache);
612
613 /* Status code VMCS reads. */
614 VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_EXIT_REASON);
615 VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_VM_INSTR_ERROR);
616 VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_EXIT_INSTR_LENGTH);
617 VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_EXIT_INTERRUPTION_ERRCODE);
618 VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO);
619 VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_EXIT_INSTR_INFO);
620 VMXSetupCachedReadVMCS(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
621 VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_IDT_INFO);
622 VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_IDT_ERRCODE);
623
624 if (pVM->hwaccm.s.fNestedPaging)
625 {
626 VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_CR3);
627 VMXSetupCachedReadVMCS(pCache, VMX_VMCS_EXIT_PHYS_ADDR_FULL);
628 pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
629 }
630 else
631 pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
632 } /* for each VMCPU */
633
634 /* Choose the right TLB setup function. */
635 if (pVM->hwaccm.s.fNestedPaging)
636 {
637 pVM->hwaccm.s.vmx.pfnSetupTaggedTLB = hmR0VmxSetupTLBEPT;
638
639 /* Default values for flushing. */
640 pVM->hwaccm.s.vmx.enmFlushPage = VMX_FLUSH_ALL_CONTEXTS;
641 pVM->hwaccm.s.vmx.enmFlushContext = VMX_FLUSH_ALL_CONTEXTS;
642
643 /* If the capabilities specify we can do more, then make use of it. */
644 if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVEPT_CAPS_INDIV)
645 pVM->hwaccm.s.vmx.enmFlushPage = VMX_FLUSH_PAGE;
646 else
647 if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVEPT_CAPS_CONTEXT)
648 pVM->hwaccm.s.vmx.enmFlushPage = VMX_FLUSH_SINGLE_CONTEXT;
649
650 if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVEPT_CAPS_CONTEXT)
651 pVM->hwaccm.s.vmx.enmFlushContext = VMX_FLUSH_SINGLE_CONTEXT;
652 }
653#ifdef HWACCM_VTX_WITH_VPID
654 else
655 if (pVM->hwaccm.s.vmx.fVPID)
656 {
657 pVM->hwaccm.s.vmx.pfnSetupTaggedTLB = hmR0VmxSetupTLBVPID;
658
659 /* Default values for flushing. */
660 pVM->hwaccm.s.vmx.enmFlushPage = VMX_FLUSH_ALL_CONTEXTS;
661 pVM->hwaccm.s.vmx.enmFlushContext = VMX_FLUSH_ALL_CONTEXTS;
662
663 /* If the capabilities specify we can do more, then make use of it. */
664 if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVVPID_CAPS_INDIV)
665 pVM->hwaccm.s.vmx.enmFlushPage = VMX_FLUSH_PAGE;
666 else
667 if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVVPID_CAPS_CONTEXT)
668 pVM->hwaccm.s.vmx.enmFlushPage = VMX_FLUSH_SINGLE_CONTEXT;
669
670 if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVVPID_CAPS_CONTEXT)
671 pVM->hwaccm.s.vmx.enmFlushContext = VMX_FLUSH_SINGLE_CONTEXT;
672 }
673#endif /* HWACCM_VTX_WITH_VPID */
674 else
675 pVM->hwaccm.s.vmx.pfnSetupTaggedTLB = hmR0VmxSetupTLBDummy;
676
677vmx_end:
678 hmR0VmxCheckError(pVM, &pVM->aCpus[0], rc);
679 return rc;
680}
681
682/**
683 * Sets the permission bits for the specified MSR
684 *
685 * @param pVCpu The VMCPU to operate on.
686 * @param ulMSR MSR value
687 * @param fRead Reading allowed/disallowed
688 * @param fWrite Writing allowed/disallowed
689 */
690static void hmR0VmxSetMSRPermission(PVMCPU pVCpu, unsigned ulMSR, bool fRead, bool fWrite)
691{
692 unsigned ulBit;
693 uint8_t *pMSRBitmap = (uint8_t *)pVCpu->hwaccm.s.vmx.pMSRBitmap;
694
695 /* Layout:
696 * 0x000 - 0x3ff - Low MSR read bits
697 * 0x400 - 0x7ff - High MSR read bits
698 * 0x800 - 0xbff - Low MSR write bits
699 * 0xc00 - 0xfff - High MSR write bits
700 */
701 if (ulMSR <= 0x00001FFF)
702 {
703 /* Pentium-compatible MSRs */
704 ulBit = ulMSR;
705 }
706 else
707 if ( ulMSR >= 0xC0000000
708 && ulMSR <= 0xC0001FFF)
709 {
710 /* AMD Sixth Generation x86 Processor MSRs */
711 ulBit = (ulMSR - 0xC0000000);
712 pMSRBitmap += 0x400;
713 }
714 else
715 {
716 AssertFailed();
717 return;
718 }
719
720 Assert(ulBit <= 0x1fff);
721 if (fRead)
722 ASMBitClear(pMSRBitmap, ulBit);
723 else
724 ASMBitSet(pMSRBitmap, ulBit);
725
726 if (fWrite)
727 ASMBitClear(pMSRBitmap + 0x800, ulBit);
728 else
729 ASMBitSet(pMSRBitmap + 0x800, ulBit);
730}
731
732
733/**
734 * Injects an event (trap or external interrupt)
735 *
736 * @returns VBox status code. Note that it may return VINF_EM_RESET to
737 * indicate a triple fault when injecting X86_XCPT_DF.
738 *
739 * @param pVM The VM to operate on.
740 * @param pVCpu The VMCPU to operate on.
741 * @param pCtx CPU Context
742 * @param intInfo VMX interrupt info
743 * @param cbInstr Opcode length of faulting instruction
744 * @param errCode Error code (optional)
745 */
746static int hmR0VmxInjectEvent(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, uint32_t intInfo, uint32_t cbInstr, uint32_t errCode)
747{
748 int rc;
749 uint32_t iGate = VMX_EXIT_INTERRUPTION_INFO_VECTOR(intInfo);
750
751#ifdef VBOX_WITH_STATISTICS
752 STAM_COUNTER_INC(&pVCpu->hwaccm.s.paStatInjectedIrqsR0[iGate & MASK_INJECT_IRQ_STAT]);
753#endif
754
755#ifdef VBOX_STRICT
756 if (iGate == 0xE)
757 LogFlow(("hmR0VmxInjectEvent: Injecting interrupt %d at %RGv error code=%08x CR2=%RGv intInfo=%08x\n", iGate, (RTGCPTR)pCtx->rip, errCode, pCtx->cr2, intInfo));
758 else
759 if (iGate < 0x20)
760 LogFlow(("hmR0VmxInjectEvent: Injecting interrupt %d at %RGv error code=%08x\n", iGate, (RTGCPTR)pCtx->rip, errCode));
761 else
762 {
763 LogFlow(("INJ-EI: %x at %RGv\n", iGate, (RTGCPTR)pCtx->rip));
764 Assert( VMX_EXIT_INTERRUPTION_INFO_TYPE(intInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW
765 || !VMCPU_FF_ISSET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
766 Assert( VMX_EXIT_INTERRUPTION_INFO_TYPE(intInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW
767 || pCtx->eflags.u32 & X86_EFL_IF);
768 }
769#endif
770
771 if ( CPUMIsGuestInRealModeEx(pCtx)
772 && pVM->hwaccm.s.vmx.pRealModeTSS)
773 {
774 RTGCPHYS GCPhysHandler;
775 uint16_t offset, ip;
776 RTSEL sel;
777
778 /* Injecting events doesn't work right with real mode emulation.
779 * (#GP if we try to inject external hardware interrupts)
780 * Inject the interrupt or trap directly instead.
781 *
782 * ASSUMES no access handlers for the bits we read or write below (should be safe).
783 */
784 Log(("Manual interrupt/trap '%x' inject (real mode)\n", iGate));
785
786 /* Check if the interrupt handler is present. */
787 if (iGate * 4 + 3 > pCtx->idtr.cbIdt)
788 {
789 Log(("IDT cbIdt violation\n"));
790 if (iGate != X86_XCPT_DF)
791 {
792 uint32_t intInfo2;
793
794 intInfo2 = (iGate == X86_XCPT_GP) ? (uint32_t)X86_XCPT_DF : iGate;
795 intInfo2 |= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
796 intInfo2 |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
797 intInfo2 |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HWEXCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
798
799 return hmR0VmxInjectEvent(pVM, pVCpu, pCtx, intInfo2, 0, 0 /* no error code according to the Intel docs */);
800 }
801 Log(("Triple fault -> reset the VM!\n"));
802 return VINF_EM_RESET;
803 }
804 if ( VMX_EXIT_INTERRUPTION_INFO_TYPE(intInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW
805 || iGate == 3 /* Both #BP and #OF point to the instruction after. */
806 || iGate == 4)
807 {
808 ip = pCtx->ip + cbInstr;
809 }
810 else
811 ip = pCtx->ip;
812
813 /* Read the selector:offset pair of the interrupt handler. */
814 GCPhysHandler = (RTGCPHYS)pCtx->idtr.pIdt + iGate * 4;
815 rc = PGMPhysSimpleReadGCPhys(pVM, &offset, GCPhysHandler, sizeof(offset)); AssertRC(rc);
816 rc = PGMPhysSimpleReadGCPhys(pVM, &sel, GCPhysHandler + 2, sizeof(sel)); AssertRC(rc);
817
818 LogFlow(("IDT handler %04X:%04X\n", sel, offset));
819
820 /* Construct the stack frame. */
821 /** @todo should check stack limit. */
822 pCtx->sp -= 2;
823 LogFlow(("ss:sp %04X:%04X eflags=%x\n", pCtx->ss, pCtx->sp, pCtx->eflags.u));
824 rc = PGMPhysSimpleWriteGCPhys(pVM, pCtx->ssHid.u64Base + pCtx->sp, &pCtx->eflags, sizeof(uint16_t)); AssertRC(rc);
825 pCtx->sp -= 2;
826 LogFlow(("ss:sp %04X:%04X cs=%x\n", pCtx->ss, pCtx->sp, pCtx->cs));
827 rc = PGMPhysSimpleWriteGCPhys(pVM, pCtx->ssHid.u64Base + pCtx->sp, &pCtx->cs, sizeof(uint16_t)); AssertRC(rc);
828 pCtx->sp -= 2;
829 LogFlow(("ss:sp %04X:%04X ip=%x\n", pCtx->ss, pCtx->sp, ip));
830 rc = PGMPhysSimpleWriteGCPhys(pVM, pCtx->ssHid.u64Base + pCtx->sp, &ip, sizeof(ip)); AssertRC(rc);
831
832 /* Update the CPU state for executing the handler. */
833 pCtx->rip = offset;
834 pCtx->cs = sel;
835 pCtx->csHid.u64Base = sel << 4;
836 pCtx->eflags.u &= ~(X86_EFL_IF|X86_EFL_TF|X86_EFL_RF|X86_EFL_AC);
837
838 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_GUEST_SEGMENT_REGS;
839 return VINF_SUCCESS;
840 }
841
842 /* Set event injection state. */
843 rc = VMXWriteVMCS(VMX_VMCS_CTRL_ENTRY_IRQ_INFO, intInfo | (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT));
844
845 rc |= VMXWriteVMCS(VMX_VMCS_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
846 rc |= VMXWriteVMCS(VMX_VMCS_CTRL_ENTRY_EXCEPTION_ERRCODE, errCode);
847
848 AssertRC(rc);
849 return rc;
850}
851
852
853/**
854 * Checks for pending guest interrupts and injects them
855 *
856 * @returns VBox status code.
857 * @param pVM The VM to operate on.
858 * @param pVCpu The VMCPU to operate on.
859 * @param pCtx CPU Context
860 */
861static int hmR0VmxCheckPendingInterrupt(PVM pVM, PVMCPU pVCpu, CPUMCTX *pCtx)
862{
863 int rc;
864
865 /* Dispatch any pending interrupts. (injected before, but a VM exit occurred prematurely) */
866 if (pVCpu->hwaccm.s.Event.fPending)
867 {
868 Log(("CPU%d: Reinjecting event %RX64 %08x at %RGv cr2=%RX64\n", pVCpu->idCpu, pVCpu->hwaccm.s.Event.intInfo, pVCpu->hwaccm.s.Event.errCode, (RTGCPTR)pCtx->rip, pCtx->cr2));
869 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatIntReinject);
870 rc = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, pVCpu->hwaccm.s.Event.intInfo, 0, pVCpu->hwaccm.s.Event.errCode);
871 AssertRC(rc);
872
873 pVCpu->hwaccm.s.Event.fPending = false;
874 return VINF_SUCCESS;
875 }
876
877 /* If an active trap is already pending, then we must forward it first! */
878 if (!TRPMHasTrap(pVCpu))
879 {
880 if (VMCPU_FF_TESTANDCLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI))
881 {
882 RTGCUINTPTR intInfo;
883
884 Log(("CPU%d: injecting #NMI\n", pVCpu->idCpu));
885
886 intInfo = X86_XCPT_NMI;
887 intInfo |= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
888 intInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
889
890 rc = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, intInfo, 0, 0);
891 AssertRC(rc);
892
893 return VINF_SUCCESS;
894 }
895
896 /* @todo SMI interrupts. */
897
898 /* When external interrupts are pending, we should exit the VM when IF is set. */
899 if (VMCPU_FF_ISPENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC|VMCPU_FF_INTERRUPT_PIC)))
900 {
901 if (!(pCtx->eflags.u32 & X86_EFL_IF))
902 {
903 if (!(pVCpu->hwaccm.s.vmx.proc_ctls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_IRQ_WINDOW_EXIT))
904 {
905 LogFlow(("Enable irq window exit!\n"));
906 pVCpu->hwaccm.s.vmx.proc_ctls |= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_IRQ_WINDOW_EXIT;
907 rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
908 AssertRC(rc);
909 }
910 /* else nothing to do but wait */
911 }
912 else
913 if (!VMCPU_FF_ISSET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
914 {
915 uint8_t u8Interrupt;
916
917 rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
918 Log(("CPU%d: Dispatch interrupt: u8Interrupt=%x (%d) rc=%Rrc cs:rip=%04X:%RGv\n", pVCpu->idCpu, u8Interrupt, u8Interrupt, rc, pCtx->cs, (RTGCPTR)pCtx->rip));
919 if (RT_SUCCESS(rc))
920 {
921 rc = TRPMAssertTrap(pVCpu, u8Interrupt, TRPM_HARDWARE_INT);
922 AssertRC(rc);
923 }
924 else
925 {
926 /* Can only happen in rare cases where a pending interrupt is cleared behind our back */
927 Assert(!VMCPU_FF_ISPENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC|VMCPU_FF_INTERRUPT_PIC)));
928 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatSwitchGuestIrq);
929 /* Just continue */
930 }
931 }
932 else
933 Log(("Pending interrupt blocked at %RGv by VM_FF_INHIBIT_INTERRUPTS!!\n", (RTGCPTR)pCtx->rip));
934 }
935 }
936
937#ifdef VBOX_STRICT
938 if (TRPMHasTrap(pVCpu))
939 {
940 uint8_t u8Vector;
941 rc = TRPMQueryTrapAll(pVCpu, &u8Vector, 0, 0, 0);
942 AssertRC(rc);
943 }
944#endif
945
946 if ( (pCtx->eflags.u32 & X86_EFL_IF)
947 && (!VMCPU_FF_ISSET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
948 && TRPMHasTrap(pVCpu)
949 )
950 {
951 uint8_t u8Vector;
952 TRPMEVENT enmType;
953 RTGCUINTPTR intInfo;
954 RTGCUINT errCode;
955
956 /* If a new event is pending, then dispatch it now. */
957 rc = TRPMQueryTrapAll(pVCpu, &u8Vector, &enmType, &errCode, 0);
958 AssertRC(rc);
959 Assert(pCtx->eflags.Bits.u1IF == 1 || enmType == TRPM_TRAP);
960 Assert(enmType != TRPM_SOFTWARE_INT);
961
962 /* Clear the pending trap. */
963 rc = TRPMResetTrap(pVCpu);
964 AssertRC(rc);
965
966 intInfo = u8Vector;
967 intInfo |= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
968
969 if (enmType == TRPM_TRAP)
970 {
971 switch (u8Vector) {
972 case X86_XCPT_DF:
973 case X86_XCPT_TS:
974 case X86_XCPT_NP:
975 case X86_XCPT_SS:
976 case X86_XCPT_GP:
977 case X86_XCPT_PF:
978 case X86_XCPT_AC:
979 /* Valid error codes. */
980 intInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
981 break;
982 default:
983 break;
984 }
985 if (u8Vector == X86_XCPT_BP || u8Vector == X86_XCPT_OF)
986 intInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SWEXCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
987 else
988 intInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HWEXCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
989 }
990 else
991 intInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
992
993 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatIntInject);
994 rc = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, intInfo, 0, errCode);
995 AssertRC(rc);
996 } /* if (interrupts can be dispatched) */
997
998 return VINF_SUCCESS;
999}
1000
1001/**
1002 * Save the host state
1003 *
1004 * @returns VBox status code.
1005 * @param pVM The VM to operate on.
1006 * @param pVCpu The VMCPU to operate on.
1007 */
1008VMMR0DECL(int) VMXR0SaveHostState(PVM pVM, PVMCPU pVCpu)
1009{
1010 int rc = VINF_SUCCESS;
1011 NOREF(pVM);
1012
1013 /*
1014 * Host CPU Context
1015 */
1016 if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_HOST_CONTEXT)
1017 {
1018 RTIDTR idtr;
1019 RTGDTR gdtr;
1020 RTSEL SelTR;
1021 PCX86DESCHC pDesc;
1022 uintptr_t trBase;
1023 RTSEL cs;
1024 RTSEL ss;
1025 uint64_t cr3;
1026
1027 /* Control registers */
1028 rc = VMXWriteVMCS(VMX_VMCS_HOST_CR0, ASMGetCR0());
1029#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
1030 if (VMX_IS_64BIT_HOST_MODE())
1031 {
1032 cr3 = hwaccmR0Get64bitCR3();
1033 rc |= VMXWriteVMCS64(VMX_VMCS_HOST_CR3, cr3);
1034 }
1035 else
1036#endif
1037 {
1038 cr3 = ASMGetCR3();
1039 rc |= VMXWriteVMCS(VMX_VMCS_HOST_CR3, cr3);
1040 }
1041 rc |= VMXWriteVMCS(VMX_VMCS_HOST_CR4, ASMGetCR4());
1042 AssertRC(rc);
1043 Log2(("VMX_VMCS_HOST_CR0 %08x\n", ASMGetCR0()));
1044 Log2(("VMX_VMCS_HOST_CR3 %08RX64\n", cr3));
1045 Log2(("VMX_VMCS_HOST_CR4 %08x\n", ASMGetCR4()));
1046
1047 /* Selector registers. */
1048#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
1049 if (VMX_IS_64BIT_HOST_MODE())
1050 {
1051 cs = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelCS;
1052 ss = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelSS;
1053 }
1054 else
1055 {
1056 /* sysenter loads LDT cs & ss, VMX doesn't like this. Load the GDT ones (safe). */
1057 cs = (RTSEL)(uintptr_t)&SUPR0AbsKernelCS;
1058 ss = (RTSEL)(uintptr_t)&SUPR0AbsKernelSS;
1059 }
1060#else
1061 cs = ASMGetCS();
1062 ss = ASMGetSS();
1063#endif
1064 Assert(!(cs & X86_SEL_LDT)); Assert((cs & X86_SEL_RPL) == 0);
1065 Assert(!(ss & X86_SEL_LDT)); Assert((ss & X86_SEL_RPL) == 0);
1066 rc = VMXWriteVMCS(VMX_VMCS16_HOST_FIELD_CS, cs);
1067 /* Note: VMX is (again) very picky about the RPL of the selectors here; we'll restore them manually. */
1068 rc |= VMXWriteVMCS(VMX_VMCS16_HOST_FIELD_DS, 0);
1069 rc |= VMXWriteVMCS(VMX_VMCS16_HOST_FIELD_ES, 0);
1070#if HC_ARCH_BITS == 32
1071 if (!VMX_IS_64BIT_HOST_MODE())
1072 {
1073 rc |= VMXWriteVMCS(VMX_VMCS16_HOST_FIELD_FS, 0);
1074 rc |= VMXWriteVMCS(VMX_VMCS16_HOST_FIELD_GS, 0);
1075 }
1076#endif
1077 rc |= VMXWriteVMCS(VMX_VMCS16_HOST_FIELD_SS, ss);
1078 SelTR = ASMGetTR();
1079 rc |= VMXWriteVMCS(VMX_VMCS16_HOST_FIELD_TR, SelTR);
1080 AssertRC(rc);
1081 Log2(("VMX_VMCS_HOST_FIELD_CS %08x (%08x)\n", cs, ASMGetSS()));
1082 Log2(("VMX_VMCS_HOST_FIELD_DS 00000000 (%08x)\n", ASMGetDS()));
1083 Log2(("VMX_VMCS_HOST_FIELD_ES 00000000 (%08x)\n", ASMGetES()));
1084 Log2(("VMX_VMCS_HOST_FIELD_FS 00000000 (%08x)\n", ASMGetFS()));
1085 Log2(("VMX_VMCS_HOST_FIELD_GS 00000000 (%08x)\n", ASMGetGS()));
1086 Log2(("VMX_VMCS_HOST_FIELD_SS %08x (%08x)\n", ss, ASMGetSS()));
1087 Log2(("VMX_VMCS_HOST_FIELD_TR %08x\n", ASMGetTR()));
1088
1089 /* GDTR & IDTR */
1090#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
1091 if (VMX_IS_64BIT_HOST_MODE())
1092 {
1093 X86XDTR64 gdtr64, idtr64;
1094 hwaccmR0Get64bitGDTRandIDTR(&gdtr64, &idtr64);
1095 rc = VMXWriteVMCS64(VMX_VMCS_HOST_GDTR_BASE, gdtr64.uAddr);
1096 rc |= VMXWriteVMCS64(VMX_VMCS_HOST_IDTR_BASE, gdtr64.uAddr);
1097 AssertRC(rc);
1098 Log2(("VMX_VMCS_HOST_GDTR_BASE %RX64\n", gdtr64.uAddr));
1099 Log2(("VMX_VMCS_HOST_IDTR_BASE %RX64\n", idtr64.uAddr));
1100 gdtr.cbGdt = gdtr64.cb;
1101 gdtr.pGdt = (uintptr_t)gdtr64.uAddr;
1102 }
1103 else
1104#endif
1105 {
1106 ASMGetGDTR(&gdtr);
1107 rc = VMXWriteVMCS(VMX_VMCS_HOST_GDTR_BASE, gdtr.pGdt);
1108 ASMGetIDTR(&idtr);
1109 rc |= VMXWriteVMCS(VMX_VMCS_HOST_IDTR_BASE, idtr.pIdt);
1110 AssertRC(rc);
1111 Log2(("VMX_VMCS_HOST_GDTR_BASE %RHv\n", gdtr.pGdt));
1112 Log2(("VMX_VMCS_HOST_IDTR_BASE %RHv\n", idtr.pIdt));
1113 }
1114
1115 /* Save the base address of the TR selector. */
1116 if (SelTR > gdtr.cbGdt)
1117 {
1118 AssertMsgFailed(("Invalid TR selector %x. GDTR.cbGdt=%x\n", SelTR, gdtr.cbGdt));
1119 return VERR_VMX_INVALID_HOST_STATE;
1120 }
1121
1122 pDesc = (PCX86DESCHC)(gdtr.pGdt + (SelTR & X86_SEL_MASK));
1123#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
1124 if (VMX_IS_64BIT_HOST_MODE())
1125 {
1126 uint64_t trBase64 = X86DESC64_BASE(*(PX86DESC64)pDesc);
1127 rc = VMXWriteVMCS64(VMX_VMCS_HOST_TR_BASE, trBase64);
1128 Log2(("VMX_VMCS_HOST_TR_BASE %RX64\n", trBase64));
1129 AssertRC(rc);
1130 }
1131 else
1132#endif
1133 {
1134#if HC_ARCH_BITS == 64
1135 trBase = X86DESC64_BASE(*pDesc);
1136#else
1137 trBase = X86DESC_BASE(*pDesc);
1138#endif
1139 rc = VMXWriteVMCS(VMX_VMCS_HOST_TR_BASE, trBase);
1140 AssertRC(rc);
1141 Log2(("VMX_VMCS_HOST_TR_BASE %RHv\n", trBase));
1142 }
1143
1144 /* FS and GS base. */
1145#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
1146 if (VMX_IS_64BIT_HOST_MODE())
1147 {
1148 Log2(("MSR_K8_FS_BASE = %RX64\n", ASMRdMsr(MSR_K8_FS_BASE)));
1149 Log2(("MSR_K8_GS_BASE = %RX64\n", ASMRdMsr(MSR_K8_GS_BASE)));
1150 rc = VMXWriteVMCS64(VMX_VMCS_HOST_FS_BASE, ASMRdMsr(MSR_K8_FS_BASE));
1151 rc |= VMXWriteVMCS64(VMX_VMCS_HOST_GS_BASE, ASMRdMsr(MSR_K8_GS_BASE));
1152 }
1153#endif
1154 AssertRC(rc);
1155
1156 /* Sysenter MSRs. */
1157 /** @todo expensive!! */
1158 rc = VMXWriteVMCS(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
1159 Log2(("VMX_VMCS_HOST_SYSENTER_CS %08x\n", ASMRdMsr_Low(MSR_IA32_SYSENTER_CS)));
1160#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
1161 if (VMX_IS_64BIT_HOST_MODE())
1162 {
1163 Log2(("VMX_VMCS_HOST_SYSENTER_EIP %RX64\n", ASMRdMsr(MSR_IA32_SYSENTER_EIP)));
1164 Log2(("VMX_VMCS_HOST_SYSENTER_ESP %RX64\n", ASMRdMsr(MSR_IA32_SYSENTER_ESP)));
1165 rc |= VMXWriteVMCS64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
1166 rc |= VMXWriteVMCS64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
1167 }
1168 else
1169 {
1170 rc |= VMXWriteVMCS(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
1171 rc |= VMXWriteVMCS(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
1172 Log2(("VMX_VMCS_HOST_SYSENTER_EIP %RX32\n", ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP)));
1173 Log2(("VMX_VMCS_HOST_SYSENTER_ESP %RX32\n", ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP)));
1174 }
1175#elif HC_ARCH_BITS == 32
1176 rc |= VMXWriteVMCS(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
1177 rc |= VMXWriteVMCS(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
1178 Log2(("VMX_VMCS_HOST_SYSENTER_EIP %RX32\n", ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP)));
1179 Log2(("VMX_VMCS_HOST_SYSENTER_ESP %RX32\n", ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP)));
1180#else
1181 Log2(("VMX_VMCS_HOST_SYSENTER_EIP %RX64\n", ASMRdMsr(MSR_IA32_SYSENTER_EIP)));
1182 Log2(("VMX_VMCS_HOST_SYSENTER_ESP %RX64\n", ASMRdMsr(MSR_IA32_SYSENTER_ESP)));
1183 rc |= VMXWriteVMCS64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
1184 rc |= VMXWriteVMCS64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
1185#endif
1186 AssertRC(rc);
1187
1188#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
1189 /* Store all host MSRs in the VM-Exit load area, so they will be reloaded after the world switch back to the host. */
1190 PVMXMSR pMsr = (PVMXMSR)pVCpu->hwaccm.s.vmx.pHostMSR;
1191 unsigned idxMsr = 0;
1192
1193 /* EFER MSR present? */
1194 if (ASMCpuId_EDX(0x80000001) & (X86_CPUID_AMD_FEATURE_EDX_NX|X86_CPUID_AMD_FEATURE_EDX_LONG_MODE))
1195 {
1196 if (ASMCpuId_EDX(0x80000001) & X86_CPUID_AMD_FEATURE_EDX_SEP)
1197 {
1198 pMsr->u32IndexMSR = MSR_K6_STAR;
1199 pMsr->u32Reserved = 0;
1200 pMsr->u64Value = ASMRdMsr(MSR_K6_STAR); /* legacy syscall eip, cs & ss */
1201 pMsr++; idxMsr++;
1202 }
1203
1204 pMsr->u32IndexMSR = MSR_K6_EFER;
1205 pMsr->u32Reserved = 0;
1206# if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
1207 if (CPUMIsGuestInLongMode(pVCpu))
1208 {
1209 /* Must match the efer value in our 64 bits switcher. */
1210 pMsr->u64Value = ASMRdMsr(MSR_K6_EFER) | MSR_K6_EFER_LME | MSR_K6_EFER_SCE | MSR_K6_EFER_NXE;
1211 }
1212 else
1213# endif
1214 pMsr->u64Value = ASMRdMsr(MSR_K6_EFER);
1215 pMsr++; idxMsr++;
1216 }
1217
1218# if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
1219 if (VMX_IS_64BIT_HOST_MODE())
1220 {
1221 pMsr->u32IndexMSR = MSR_K8_LSTAR;
1222 pMsr->u32Reserved = 0;
1223 pMsr->u64Value = ASMRdMsr(MSR_K8_LSTAR); /* 64 bits mode syscall rip */
1224 pMsr++; idxMsr++;
1225 pMsr->u32IndexMSR = MSR_K8_SF_MASK;
1226 pMsr->u32Reserved = 0;
1227 pMsr->u64Value = ASMRdMsr(MSR_K8_SF_MASK); /* syscall flag mask */
1228 pMsr++; idxMsr++;
1229 pMsr->u32IndexMSR = MSR_K8_KERNEL_GS_BASE;
1230 pMsr->u32Reserved = 0;
1231 pMsr->u64Value = ASMRdMsr(MSR_K8_KERNEL_GS_BASE); /* swapgs exchange value */
1232 pMsr++; idxMsr++;
1233 }
1234# endif
1235 rc = VMXWriteVMCS(VMX_VMCS_CTRL_EXIT_MSR_LOAD_COUNT, idxMsr);
1236 AssertRC(rc);
1237#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
1238
1239 pVCpu->hwaccm.s.fContextUseFlags &= ~HWACCM_CHANGED_HOST_CONTEXT;
1240 }
1241 return rc;
1242}
1243
1244/**
1245 * Loads the 4 PDPEs into the guest state when nested paging is used and the
1246 * guest operates in PAE mode.
1247 *
1248 * @returns VINF_SUCCESS or fatal error.
1249 * @param pVCpu The VMCPU to operate on.
1250 * @param pCtx Guest context
1251 */
1252static int hmR0VmxLoadPaePdpes(PVMCPU pVCpu, PCPUMCTX pCtx)
1253{
1254 if (CPUMIsGuestInPAEModeEx(pCtx))
1255 {
1256 X86PDPE aPdpes[4];
1257 int rc = PGMGstGetPaePdpes(pVCpu, &aPdpes[0]);
1258 AssertRCReturn(rc, rc);
1259
1260 rc = VMXWriteVMCS64(VMX_VMCS_GUEST_PDPTR0_FULL, aPdpes[0].u); AssertRCReturn(rc, rc);
1261 rc = VMXWriteVMCS64(VMX_VMCS_GUEST_PDPTR1_FULL, aPdpes[1].u); AssertRCReturn(rc, rc);
1262 rc = VMXWriteVMCS64(VMX_VMCS_GUEST_PDPTR2_FULL, aPdpes[2].u); AssertRCReturn(rc, rc);
1263 rc = VMXWriteVMCS64(VMX_VMCS_GUEST_PDPTR3_FULL, aPdpes[3].u); AssertRCReturn(rc, rc);
1264 }
1265 return VINF_SUCCESS;
1266}
1267
1268/**
1269 * Saves the 4 PDPEs into the guest state when nested paging is used and the
1270 * guest operates in PAE mode.
1271 *
1272 * @returns VINF_SUCCESS or fatal error.
1273 * @param pVCpu The VMCPU to operate on.
1274 * @param pCtx Guest context
1275 *
1276 * @remarks Tell PGM about CR3 changes before calling this helper.
1277 */
1278static int hmR0VmxSavePaePdpes(PVMCPU pVCpu, PCPUMCTX pCtx)
1279{
1280 if (CPUMIsGuestInPAEModeEx(pCtx))
1281 {
1282 int rc;
1283 X86PDPE aPdpes[4];
1284 rc = VMXReadVMCS64(VMX_VMCS_GUEST_PDPTR0_FULL, &aPdpes[0].u); AssertRCReturn(rc, rc);
1285 rc = VMXReadVMCS64(VMX_VMCS_GUEST_PDPTR1_FULL, &aPdpes[1].u); AssertRCReturn(rc, rc);
1286 rc = VMXReadVMCS64(VMX_VMCS_GUEST_PDPTR2_FULL, &aPdpes[2].u); AssertRCReturn(rc, rc);
1287 rc = VMXReadVMCS64(VMX_VMCS_GUEST_PDPTR3_FULL, &aPdpes[3].u); AssertRCReturn(rc, rc);
1288
1289 rc = PGMGstUpdatePaePdpes(pVCpu, &aPdpes[0]);
1290 AssertRCReturn(rc, rc);
1291 }
1292 return VINF_SUCCESS;
1293}
1294
1295
1296/**
1297 * Update the exception bitmap according to the current CPU state
1298 *
1299 * @param pVM The VM to operate on.
1300 * @param pVCpu The VMCPU to operate on.
1301 * @param pCtx Guest context
1302 */
1303static void hmR0VmxUpdateExceptionBitmap(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
1304{
1305 uint32_t u32TrapMask;
1306 Assert(pCtx);
1307
1308 u32TrapMask = HWACCM_VMX_TRAP_MASK;
1309#ifndef DEBUG
1310 if (pVM->hwaccm.s.fNestedPaging)
1311 u32TrapMask &= ~RT_BIT(X86_XCPT_PF); /* no longer need to intercept #PF. */
1312#endif
1313
1314 /* Also catch floating point exceptions if we need to report them to the guest in a different way. */
1315 if (!(pCtx->cr0 & X86_CR0_NE))
1316 {
1317 u32TrapMask |= RT_BIT(X86_XCPT_MF);
1318 }
1319
1320#ifdef VBOX_STRICT
1321 Assert(u32TrapMask & RT_BIT(X86_XCPT_GP));
1322#endif
1323
1324 /* Intercept all exceptions in real mode as none of them can be injected directly (#GP otherwise). */
1325 if ( CPUMIsGuestInRealModeEx(pCtx)
1326 && pVM->hwaccm.s.vmx.pRealModeTSS)
1327 u32TrapMask |= HWACCM_VMX_TRAP_MASK_REALMODE;
1328
1329 int rc = VMXWriteVMCS(VMX_VMCS_CTRL_EXCEPTION_BITMAP, u32TrapMask);
1330 AssertRC(rc);
1331}
1332
1333/**
1334 * Loads a minimal guest state
1335 *
1336 * NOTE: Don't do anything here that can cause a jump back to ring 3!!!!!
1337 *
1338 * @param pVM The VM to operate on.
1339 * @param pVCpu The VMCPU to operate on.
1340 * @param pCtx Guest context
1341 */
1342VMMR0DECL(void) VMXR0LoadMinimalGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
1343{
1344 int rc;
1345 X86EFLAGS eflags;
1346
1347 Assert(!(pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_ALL_GUEST));
1348
1349 /* EIP, ESP and EFLAGS */
1350 rc = VMXWriteVMCS64(VMX_VMCS64_GUEST_RIP, pCtx->rip);
1351 rc |= VMXWriteVMCS64(VMX_VMCS64_GUEST_RSP, pCtx->rsp);
1352 AssertRC(rc);
1353
1354 /* Bits 22-31, 15, 5 & 3 must be zero. Bit 1 must be 1. */
1355 eflags = pCtx->eflags;
1356 eflags.u32 &= VMX_EFLAGS_RESERVED_0;
1357 eflags.u32 |= VMX_EFLAGS_RESERVED_1;
1358
1359 /* Real mode emulation using v86 mode. */
1360 if ( CPUMIsGuestInRealModeEx(pCtx)
1361 && pVM->hwaccm.s.vmx.pRealModeTSS)
1362 {
1363 pVCpu->hwaccm.s.vmx.RealMode.eflags = eflags;
1364
1365 eflags.Bits.u1VM = 1;
1366 eflags.Bits.u2IOPL = 0; /* must always be 0 or else certain instructions won't cause faults. */
1367 }
1368 rc = VMXWriteVMCS(VMX_VMCS_GUEST_RFLAGS, eflags.u32);
1369 AssertRC(rc);
1370}
1371
1372/**
1373 * Loads the guest state
1374 *
1375 * NOTE: Don't do anything here that can cause a jump back to ring 3!!!!!
1376 *
1377 * @returns VBox status code.
1378 * @param pVM The VM to operate on.
1379 * @param pVCpu The VMCPU to operate on.
1380 * @param pCtx Guest context
1381 */
1382VMMR0DECL(int) VMXR0LoadGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
1383{
1384 int rc = VINF_SUCCESS;
1385 RTGCUINTPTR val;
1386
1387 /* VMX_VMCS_CTRL_ENTRY_CONTROLS
1388 * Set required bits to one and zero according to the MSR capabilities.
1389 */
1390 val = pVM->hwaccm.s.vmx.msr.vmx_entry.n.disallowed0;
1391 /* Load guest debug controls (dr7 & IA32_DEBUGCTL_MSR) (forced to 1 on the 'first' VT-x capable CPUs; this actually includes the newest Nehalem CPUs) */
1392 val |= VMX_VMCS_CTRL_ENTRY_CONTROLS_LOAD_DEBUG;
1393 /* 64 bits guest mode? */
1394 if (CPUMIsGuestInLongModeEx(pCtx))
1395 val |= VMX_VMCS_CTRL_ENTRY_CONTROLS_IA64_MODE;
1396 /* else Must be zero when AMD64 is not available. */
1397
1398 /* Mask away the bits that the CPU doesn't support */
1399 val &= pVM->hwaccm.s.vmx.msr.vmx_entry.n.allowed1;
1400 rc = VMXWriteVMCS(VMX_VMCS_CTRL_ENTRY_CONTROLS, val);
1401 AssertRC(rc);
1402
1403 /* VMX_VMCS_CTRL_EXIT_CONTROLS
1404 * Set required bits to one and zero according to the MSR capabilities.
1405 */
1406 val = pVM->hwaccm.s.vmx.msr.vmx_exit.n.disallowed0;
1407
1408 /* Save debug controls (dr7 & IA32_DEBUGCTL_MSR) (forced to 1 on the 'first' VT-x capable CPUs; this actually includes the newest Nehalem CPUs) */
1409 val |= VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_DEBUG;
1410
1411#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
1412 if (VMX_IS_64BIT_HOST_MODE())
1413 val |= VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_AMD64;
1414 /* else: Must be zero when AMD64 is not available. */
1415#elif HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
1416 if (CPUMIsGuestInLongModeEx(pCtx))
1417 val |= VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_AMD64; /* our switcher goes to long mode */
1418 else
1419 Assert(!(val & VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_AMD64));
1420#endif
1421 val &= pVM->hwaccm.s.vmx.msr.vmx_exit.n.allowed1;
1422 /* Don't acknowledge external interrupts on VM-exit. */
1423 rc = VMXWriteVMCS(VMX_VMCS_CTRL_EXIT_CONTROLS, val);
1424 AssertRC(rc);
1425
1426 /* Guest CPU context: ES, CS, SS, DS, FS, GS. */
1427 if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_SEGMENT_REGS)
1428 {
1429 if (pVM->hwaccm.s.vmx.pRealModeTSS)
1430 {
1431 PGMMODE enmGuestMode = PGMGetGuestMode(pVCpu);
1432 if (pVCpu->hwaccm.s.vmx.enmLastSeenGuestMode != enmGuestMode)
1433 {
1434 /* Correct weird requirements for switching to protected mode. */
1435 if ( pVCpu->hwaccm.s.vmx.enmLastSeenGuestMode == PGMMODE_REAL
1436 && enmGuestMode >= PGMMODE_PROTECTED)
1437 {
1438#ifdef VBOX_WITH_REM
1439 /* Flush the recompiler code cache as it's not unlikely
1440 * the guest will rewrite code it will later execute in real
1441 * mode (OpenBSD 4.0 is one such example)
1442 */
1443 REMFlushTBs(pVM);
1444#endif
1445
1446 /* DPL of all hidden selector registers must match the current CPL (0). */
1447 pCtx->csHid.Attr.n.u2Dpl = 0;
1448 pCtx->csHid.Attr.n.u4Type = X86_SEL_TYPE_CODE | X86_SEL_TYPE_RW_ACC;
1449
1450 pCtx->dsHid.Attr.n.u2Dpl = 0;
1451 pCtx->esHid.Attr.n.u2Dpl = 0;
1452 pCtx->fsHid.Attr.n.u2Dpl = 0;
1453 pCtx->gsHid.Attr.n.u2Dpl = 0;
1454 pCtx->ssHid.Attr.n.u2Dpl = 0;
1455 }
1456 pVCpu->hwaccm.s.vmx.enmLastSeenGuestMode = enmGuestMode;
1457 }
1458 else
1459 /* VT-x will fail with a guest invalid state otherwise... (CPU state after a reset) */
1460 if ( CPUMIsGuestInRealModeEx(pCtx)
1461 && pCtx->csHid.u64Base == 0xffff0000)
1462 {
1463 pCtx->csHid.u64Base = 0xf0000;
1464 pCtx->cs = 0xf000;
1465 }
1466 }
1467
1468 VMX_WRITE_SELREG(ES, es);
1469 AssertRC(rc);
1470
1471 VMX_WRITE_SELREG(CS, cs);
1472 AssertRC(rc);
1473
1474 VMX_WRITE_SELREG(SS, ss);
1475 AssertRC(rc);
1476
1477 VMX_WRITE_SELREG(DS, ds);
1478 AssertRC(rc);
1479
1480 VMX_WRITE_SELREG(FS, fs);
1481 AssertRC(rc);
1482
1483 VMX_WRITE_SELREG(GS, gs);
1484 AssertRC(rc);
1485 }
1486
1487 /* Guest CPU context: LDTR. */
1488 if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_LDTR)
1489 {
1490 if (pCtx->ldtr == 0)
1491 {
1492 rc = VMXWriteVMCS(VMX_VMCS16_GUEST_FIELD_LDTR, 0);
1493 rc |= VMXWriteVMCS(VMX_VMCS32_GUEST_LDTR_LIMIT, 0);
1494 rc |= VMXWriteVMCS64(VMX_VMCS64_GUEST_LDTR_BASE, 0);
1495 /* Note: vmlaunch will fail with 0 or just 0x02. No idea why. */
1496 rc |= VMXWriteVMCS(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, 0x82 /* present, LDT */);
1497 }
1498 else
1499 {
1500 rc = VMXWriteVMCS(VMX_VMCS16_GUEST_FIELD_LDTR, pCtx->ldtr);
1501 rc |= VMXWriteVMCS(VMX_VMCS32_GUEST_LDTR_LIMIT, pCtx->ldtrHid.u32Limit);
1502 rc |= VMXWriteVMCS64(VMX_VMCS64_GUEST_LDTR_BASE, pCtx->ldtrHid.u64Base);
1503 rc |= VMXWriteVMCS(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, pCtx->ldtrHid.Attr.u);
1504 }
1505 AssertRC(rc);
1506 }
1507 /* Guest CPU context: TR. */
1508 if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_TR)
1509 {
1510 /* Real mode emulation using v86 mode with CR4.VME (interrupt redirection using the int bitmap in the TSS) */
1511 if ( CPUMIsGuestInRealModeEx(pCtx)
1512 && pVM->hwaccm.s.vmx.pRealModeTSS)
1513 {
1514 RTGCPHYS GCPhys;
1515
1516 /* We convert it here every time as pci regions could be reconfigured. */
1517 rc = PDMVMMDevHeapR3ToGCPhys(pVM, pVM->hwaccm.s.vmx.pRealModeTSS, &GCPhys);
1518 AssertRC(rc);
1519
1520 rc = VMXWriteVMCS(VMX_VMCS16_GUEST_FIELD_TR, 0);
1521 rc |= VMXWriteVMCS(VMX_VMCS32_GUEST_TR_LIMIT, HWACCM_VTX_TSS_SIZE);
1522 rc |= VMXWriteVMCS64(VMX_VMCS64_GUEST_TR_BASE, GCPhys /* phys = virt in this mode */);
1523
1524 X86DESCATTR attr;
1525
1526 attr.u = 0;
1527 attr.n.u1Present = 1;
1528 attr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
1529 val = attr.u;
1530 }
1531 else
1532 {
1533 rc = VMXWriteVMCS(VMX_VMCS16_GUEST_FIELD_TR, pCtx->tr);
1534 rc |= VMXWriteVMCS(VMX_VMCS32_GUEST_TR_LIMIT, pCtx->trHid.u32Limit);
1535 rc |= VMXWriteVMCS64(VMX_VMCS64_GUEST_TR_BASE, pCtx->trHid.u64Base);
1536
1537 val = pCtx->trHid.Attr.u;
1538
1539 /* The TSS selector must be busy (REM bugs? see defect #XXXX). */
1540 if (!(val & X86_SEL_TYPE_SYS_TSS_BUSY_MASK))
1541 {
1542 if (val & 0xf)
1543 val |= X86_SEL_TYPE_SYS_TSS_BUSY_MASK;
1544 else
1545 /* Default if no TR selector has been set (otherwise vmlaunch will fail!) */
1546 val = (val & ~0xF) | X86_SEL_TYPE_SYS_386_TSS_BUSY;
1547 }
1548 AssertMsg((val & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY || (val & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("%#x\n", val));
1549 }
1550 rc |= VMXWriteVMCS(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, val);
1551 AssertRC(rc);
1552 }
1553 /* Guest CPU context: GDTR. */
1554 if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_GDTR)
1555 {
1556 rc = VMXWriteVMCS(VMX_VMCS32_GUEST_GDTR_LIMIT, pCtx->gdtr.cbGdt);
1557 rc |= VMXWriteVMCS64(VMX_VMCS64_GUEST_GDTR_BASE, pCtx->gdtr.pGdt);
1558 AssertRC(rc);
1559 }
1560 /* Guest CPU context: IDTR. */
1561 if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_IDTR)
1562 {
1563 rc = VMXWriteVMCS(VMX_VMCS32_GUEST_IDTR_LIMIT, pCtx->idtr.cbIdt);
1564 rc |= VMXWriteVMCS64(VMX_VMCS64_GUEST_IDTR_BASE, pCtx->idtr.pIdt);
1565 AssertRC(rc);
1566 }
1567
1568 /*
1569 * Sysenter MSRs
1570 */
1571 if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_MSR)
1572 {
1573 rc = VMXWriteVMCS(VMX_VMCS32_GUEST_SYSENTER_CS, pCtx->SysEnter.cs);
1574 rc |= VMXWriteVMCS64(VMX_VMCS64_GUEST_SYSENTER_EIP, pCtx->SysEnter.eip);
1575 rc |= VMXWriteVMCS64(VMX_VMCS64_GUEST_SYSENTER_ESP, pCtx->SysEnter.esp);
1576 AssertRC(rc);
1577 }
1578
1579 /* Control registers */
1580 if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_CR0)
1581 {
1582 val = pCtx->cr0;
1583 rc = VMXWriteVMCS(VMX_VMCS_CTRL_CR0_READ_SHADOW, val);
1584 Log2(("Guest CR0-shadow %08x\n", val));
1585 if (CPUMIsGuestFPUStateActive(pVCpu) == false)
1586 {
1587 /* Always use #NM exceptions to load the FPU/XMM state on demand. */
1588 val |= X86_CR0_TS | X86_CR0_ET | X86_CR0_NE | X86_CR0_MP;
1589 }
1590 else
1591 {
1592 /** @todo check if we support the old style mess correctly. */
1593 if (!(val & X86_CR0_NE))
1594 Log(("Forcing X86_CR0_NE!!!\n"));
1595
1596 val |= X86_CR0_NE; /* always turn on the native mechanism to report FPU errors (old style uses interrupts) */
1597 }
1598 /* Note: protected mode & paging are always enabled; we use them for emulating real and protected mode without paging too. */
1599 if (!pVM->hwaccm.s.vmx.fUnrestrictedGuest)
1600 val |= X86_CR0_PE | X86_CR0_PG;
1601
1602 if (pVM->hwaccm.s.fNestedPaging)
1603 {
1604 if (CPUMIsGuestInPagedProtectedModeEx(pCtx))
1605 {
1606 /* Disable cr3 read/write monitoring as we don't need it for EPT. */
1607 pVCpu->hwaccm.s.vmx.proc_ctls &= ~( VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
1608 | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT);
1609 }
1610 else
1611 {
1612 /* Reenable cr3 read/write monitoring as our identity mapped page table is active. */
1613 pVCpu->hwaccm.s.vmx.proc_ctls |= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
1614 | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT;
1615 }
1616 rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
1617 AssertRC(rc);
1618 }
1619 else
1620 {
1621 /* Note: We must also set this as we rely on protecting various pages for which supervisor writes must be caught. */
1622 val |= X86_CR0_WP;
1623 }
1624
1625 /* Always enable caching. */
1626 val &= ~(X86_CR0_CD|X86_CR0_NW);
1627
1628 rc |= VMXWriteVMCS64(VMX_VMCS64_GUEST_CR0, val);
1629 Log2(("Guest CR0 %08x\n", val));
1630 /* CR0 flags owned by the host; if the guests attempts to change them, then
1631 * the VM will exit.
1632 */
1633 val = X86_CR0_PE /* Must monitor this bit (assumptions are made for real mode emulation) */
1634 | X86_CR0_WP /* Must monitor this bit (it must always be enabled). */
1635 | X86_CR0_PG /* Must monitor this bit (assumptions are made for real mode & protected mode without paging emulation) */
1636 | X86_CR0_CD /* Bit not restored during VM-exit! */
1637 | X86_CR0_NW /* Bit not restored during VM-exit! */
1638 | X86_CR0_NE;
1639
1640 /* When the guest's FPU state is active, then we no longer care about
1641 * the FPU related bits.
1642 */
1643 if (CPUMIsGuestFPUStateActive(pVCpu) == false)
1644 val |= X86_CR0_TS | X86_CR0_ET | X86_CR0_MP;
1645
1646 pVCpu->hwaccm.s.vmx.cr0_mask = val;
1647
1648 rc |= VMXWriteVMCS(VMX_VMCS_CTRL_CR0_MASK, val);
1649 Log2(("Guest CR0-mask %08x\n", val));
1650 AssertRC(rc);
1651 }
1652 if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_CR4)
1653 {
1654 /* CR4 */
1655 rc = VMXWriteVMCS(VMX_VMCS_CTRL_CR4_READ_SHADOW, pCtx->cr4);
1656 Log2(("Guest CR4-shadow %08x\n", pCtx->cr4));
1657 /* Set the required bits in cr4 too (currently X86_CR4_VMXE). */
1658 val = pCtx->cr4 | (uint32_t)pVM->hwaccm.s.vmx.msr.vmx_cr4_fixed0;
1659
1660 if (!pVM->hwaccm.s.fNestedPaging)
1661 {
1662 switch(pVCpu->hwaccm.s.enmShadowMode)
1663 {
1664 case PGMMODE_REAL: /* Real mode -> emulated using v86 mode */
1665 case PGMMODE_PROTECTED: /* Protected mode, no paging -> emulated using identity mapping. */
1666 case PGMMODE_32_BIT: /* 32-bit paging. */
1667 val &= ~X86_CR4_PAE;
1668 break;
1669
1670 case PGMMODE_PAE: /* PAE paging. */
1671 case PGMMODE_PAE_NX: /* PAE paging with NX enabled. */
1672 /** Must use PAE paging as we could use physical memory > 4 GB */
1673 val |= X86_CR4_PAE;
1674 break;
1675
1676 case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
1677 case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
1678#ifdef VBOX_ENABLE_64_BITS_GUESTS
1679 break;
1680#else
1681 AssertFailed();
1682 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
1683#endif
1684 default: /* shut up gcc */
1685 AssertFailed();
1686 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
1687 }
1688 }
1689 else
1690 if ( !CPUMIsGuestInPagedProtectedModeEx(pCtx)
1691 && !pVM->hwaccm.s.vmx.fUnrestrictedGuest)
1692 {
1693 /* We use 4 MB pages in our identity mapping page table for real and protected mode without paging. */
1694 val |= X86_CR4_PSE;
1695 /* Our identity mapping is a 32 bits page directory. */
1696 val &= ~X86_CR4_PAE;
1697 }
1698
1699 /* Turn off VME if we're in emulated real mode. */
1700 if ( CPUMIsGuestInRealModeEx(pCtx)
1701 && pVM->hwaccm.s.vmx.pRealModeTSS)
1702 val &= ~X86_CR4_VME;
1703
1704 rc |= VMXWriteVMCS64(VMX_VMCS64_GUEST_CR4, val);
1705 Log2(("Guest CR4 %08x\n", val));
1706 /* CR4 flags owned by the host; if the guests attempts to change them, then
1707 * the VM will exit.
1708 */
1709 val = 0
1710 | X86_CR4_VME
1711 | X86_CR4_PAE
1712 | X86_CR4_PGE
1713 | X86_CR4_PSE
1714 | X86_CR4_VMXE;
1715 pVCpu->hwaccm.s.vmx.cr4_mask = val;
1716
1717 rc |= VMXWriteVMCS(VMX_VMCS_CTRL_CR4_MASK, val);
1718 Log2(("Guest CR4-mask %08x\n", val));
1719 AssertRC(rc);
1720 }
1721
1722#if 0
1723 /* Enable single stepping if requested and CPU supports it. */
1724 if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_TRAP_FLAG)
1725 if (DBGFIsStepping(pVCpu))
1726 {
1727 pVCpu->hwaccm.s.vmx.proc_ctls |= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_TRAP_FLAG;
1728 rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
1729 AssertRC(rc);
1730 }
1731#endif
1732
1733 if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_CR3)
1734 {
1735 if (pVM->hwaccm.s.fNestedPaging)
1736 {
1737 Assert(PGMGetHyperCR3(pVCpu));
1738 pVCpu->hwaccm.s.vmx.GCPhysEPTP = PGMGetHyperCR3(pVCpu);
1739
1740 Assert(!(pVCpu->hwaccm.s.vmx.GCPhysEPTP & 0xfff));
1741 /** @todo Check the IA32_VMX_EPT_VPID_CAP MSR for other supported memory types. */
1742 pVCpu->hwaccm.s.vmx.GCPhysEPTP |= VMX_EPT_MEMTYPE_WB
1743 | (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
1744
1745 rc = VMXWriteVMCS64(VMX_VMCS_CTRL_EPTP_FULL, pVCpu->hwaccm.s.vmx.GCPhysEPTP);
1746 AssertRC(rc);
1747
1748 if ( !CPUMIsGuestInPagedProtectedModeEx(pCtx)
1749 && !pVM->hwaccm.s.vmx.fUnrestrictedGuest)
1750 {
1751 RTGCPHYS GCPhys;
1752
1753 /* We convert it here every time as pci regions could be reconfigured. */
1754 rc = PDMVMMDevHeapR3ToGCPhys(pVM, pVM->hwaccm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
1755 AssertMsgRC(rc, ("pNonPagingModeEPTPageTable = %RGv\n", pVM->hwaccm.s.vmx.pNonPagingModeEPTPageTable));
1756
1757 /* We use our identity mapping page table here as we need to map guest virtual to guest physical addresses; EPT will
1758 * take care of the translation to host physical addresses.
1759 */
1760 val = GCPhys;
1761 }
1762 else
1763 {
1764 /* Save the real guest CR3 in VMX_VMCS_GUEST_CR3 */
1765 val = pCtx->cr3;
1766 rc = hmR0VmxLoadPaePdpes(pVCpu, pCtx);
1767 AssertRCReturn(rc, rc);
1768 }
1769 }
1770 else
1771 {
1772 val = PGMGetHyperCR3(pVCpu);
1773 Assert(val || VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL));
1774 }
1775
1776 /* Save our shadow CR3 register. */
1777 rc = VMXWriteVMCS64(VMX_VMCS64_GUEST_CR3, val);
1778 AssertRC(rc);
1779 }
1780
1781 /* Debug registers. */
1782 if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_DEBUG)
1783 {
1784 pCtx->dr[6] |= X86_DR6_INIT_VAL; /* set all reserved bits to 1. */
1785 pCtx->dr[6] &= ~RT_BIT(12); /* must be zero. */
1786
1787 pCtx->dr[7] &= 0xffffffff; /* upper 32 bits reserved */
1788 pCtx->dr[7] &= ~(RT_BIT(11) | RT_BIT(12) | RT_BIT(14) | RT_BIT(15)); /* must be zero */
1789 pCtx->dr[7] |= 0x400; /* must be one */
1790
1791 /* Resync DR7 */
1792 rc = VMXWriteVMCS64(VMX_VMCS64_GUEST_DR7, pCtx->dr[7]);
1793 AssertRC(rc);
1794
1795#ifdef DEBUG
1796 /* Sync the hypervisor debug state now if any breakpoint is armed. */
1797 if ( CPUMGetHyperDR7(pVCpu) & (X86_DR7_ENABLED_MASK|X86_DR7_GD)
1798 && !CPUMIsHyperDebugStateActive(pVCpu)
1799 && !DBGFIsStepping(pVCpu))
1800 {
1801 /* Save the host and load the hypervisor debug state. */
1802 rc = CPUMR0LoadHyperDebugState(pVM, pVCpu, pCtx, true /* include DR6 */);
1803 AssertRC(rc);
1804
1805 /* DRx intercepts remain enabled. */
1806
1807 /* Override dr7 with the hypervisor value. */
1808 rc = VMXWriteVMCS64(VMX_VMCS64_GUEST_DR7, CPUMGetHyperDR7(pVCpu));
1809 AssertRC(rc);
1810 }
1811 else
1812#endif
1813 /* Sync the debug state now if any breakpoint is armed. */
1814 if ( (pCtx->dr[7] & (X86_DR7_ENABLED_MASK|X86_DR7_GD))
1815 && !CPUMIsGuestDebugStateActive(pVCpu)
1816 && !DBGFIsStepping(pVCpu))
1817 {
1818 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatDRxArmed);
1819
1820 /* Disable drx move intercepts. */
1821 pVCpu->hwaccm.s.vmx.proc_ctls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT;
1822 rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
1823 AssertRC(rc);
1824
1825 /* Save the host and load the guest debug state. */
1826 rc = CPUMR0LoadGuestDebugState(pVM, pVCpu, pCtx, true /* include DR6 */);
1827 AssertRC(rc);
1828 }
1829
1830 /* IA32_DEBUGCTL MSR. */
1831 rc = VMXWriteVMCS64(VMX_VMCS_GUEST_DEBUGCTL_FULL, 0);
1832 AssertRC(rc);
1833
1834 /** @todo do we really ever need this? */
1835 rc |= VMXWriteVMCS(VMX_VMCS_GUEST_DEBUG_EXCEPTIONS, 0);
1836 AssertRC(rc);
1837 }
1838
1839 /* 64 bits guest mode? */
1840 if (CPUMIsGuestInLongModeEx(pCtx))
1841 {
1842#if !defined(VBOX_ENABLE_64_BITS_GUESTS)
1843 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
1844#elif HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
1845 pVCpu->hwaccm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
1846#else
1847# ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
1848 if (!pVM->hwaccm.s.fAllow64BitGuests)
1849 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
1850# endif
1851 pVCpu->hwaccm.s.vmx.pfnStartVM = VMXR0StartVM64;
1852#endif
1853 if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_MSR)
1854 {
1855 /* Update these as wrmsr might have changed them. */
1856 rc = VMXWriteVMCS64(VMX_VMCS64_GUEST_FS_BASE, pCtx->fsHid.u64Base);
1857 AssertRC(rc);
1858 rc = VMXWriteVMCS64(VMX_VMCS64_GUEST_GS_BASE, pCtx->gsHid.u64Base);
1859 AssertRC(rc);
1860 }
1861 }
1862 else
1863 {
1864 pVCpu->hwaccm.s.vmx.pfnStartVM = VMXR0StartVM32;
1865 }
1866
1867 hmR0VmxUpdateExceptionBitmap(pVM, pVCpu, pCtx);
1868
1869#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
1870 /* Store all guest MSRs in the VM-Entry load area, so they will be loaded during the world switch. */
1871 PVMXMSR pMsr = (PVMXMSR)pVCpu->hwaccm.s.vmx.pGuestMSR;
1872 unsigned idxMsr = 0;
1873
1874 uint32_t ulEdx;
1875 uint32_t ulTemp;
1876 CPUMGetGuestCpuId(pVCpu, 0x80000001, &ulTemp, &ulTemp, &ulTemp, &ulEdx);
1877 /* EFER MSR present? */
1878 if (ulEdx & (X86_CPUID_AMD_FEATURE_EDX_NX|X86_CPUID_AMD_FEATURE_EDX_LONG_MODE))
1879 {
1880 pMsr->u32IndexMSR = MSR_K6_EFER;
1881 pMsr->u32Reserved = 0;
1882 pMsr->u64Value = pCtx->msrEFER;
1883 /* VT-x will complain if only MSR_K6_EFER_LME is set. */
1884 if (!CPUMIsGuestInLongModeEx(pCtx))
1885 pMsr->u64Value &= ~(MSR_K6_EFER_LMA|MSR_K6_EFER_LME);
1886 pMsr++; idxMsr++;
1887
1888 if (ulEdx & X86_CPUID_AMD_FEATURE_EDX_LONG_MODE)
1889 {
1890 pMsr->u32IndexMSR = MSR_K8_LSTAR;
1891 pMsr->u32Reserved = 0;
1892 pMsr->u64Value = pCtx->msrLSTAR; /* 64 bits mode syscall rip */
1893 pMsr++; idxMsr++;
1894 pMsr->u32IndexMSR = MSR_K6_STAR;
1895 pMsr->u32Reserved = 0;
1896 pMsr->u64Value = pCtx->msrSTAR; /* legacy syscall eip, cs & ss */
1897 pMsr++; idxMsr++;
1898 pMsr->u32IndexMSR = MSR_K8_SF_MASK;
1899 pMsr->u32Reserved = 0;
1900 pMsr->u64Value = pCtx->msrSFMASK; /* syscall flag mask */
1901 pMsr++; idxMsr++;
1902 pMsr->u32IndexMSR = MSR_K8_KERNEL_GS_BASE;
1903 pMsr->u32Reserved = 0;
1904 pMsr->u64Value = pCtx->msrKERNELGSBASE; /* swapgs exchange value */
1905 pMsr++; idxMsr++;
1906 }
1907 }
1908 pVCpu->hwaccm.s.vmx.cCachedMSRs = idxMsr;
1909
1910 rc = VMXWriteVMCS(VMX_VMCS_CTRL_ENTRY_MSR_LOAD_COUNT, idxMsr);
1911 AssertRC(rc);
1912
1913 rc = VMXWriteVMCS(VMX_VMCS_CTRL_EXIT_MSR_STORE_COUNT, idxMsr);
1914 AssertRC(rc);
1915#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
1916
1917 bool fOffsettedTsc;
1918 if (pVM->hwaccm.s.vmx.fUsePreemptTimer)
1919 {
1920 uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVCpu, &fOffsettedTsc, &pVCpu->hwaccm.s.vmx.u64TSCOffset);
1921
1922 /* Make sure the returned values have sane upper and lower boundaries. */
1923 uint64_t u64CpuHz = SUPGetCpuHzFromGIP(g_pSUPGlobalInfoPage);
1924
1925 cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64 of a second */
1926 cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
1927
1928 cTicksToDeadline >>= pVM->hwaccm.s.vmx.cPreemptTimerShift;
1929 uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
1930 rc = VMXWriteVMCS(VMX_VMCS32_GUEST_PREEMPTION_TIMER_VALUE, cPreemptionTickCount);
1931 AssertRC(rc);
1932 }
1933 else
1934 fOffsettedTsc = TMCpuTickCanUseRealTSC(pVCpu, &pVCpu->hwaccm.s.vmx.u64TSCOffset);
1935 if (fOffsettedTsc)
1936 {
1937 uint64_t u64CurTSC = ASMReadTSC();
1938 if (u64CurTSC + pVCpu->hwaccm.s.vmx.u64TSCOffset >= TMCpuTickGetLastSeen(pVCpu))
1939 {
1940 /* Note: VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT takes precedence over TSC_OFFSET */
1941 rc = VMXWriteVMCS64(VMX_VMCS_CTRL_TSC_OFFSET_FULL, pVCpu->hwaccm.s.vmx.u64TSCOffset);
1942 AssertRC(rc);
1943
1944 pVCpu->hwaccm.s.vmx.proc_ctls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT;
1945 rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
1946 AssertRC(rc);
1947 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatTSCOffset);
1948 }
1949 else
1950 {
1951 /* Fall back to rdtsc emulation as we would otherwise pass decreasing tsc values to the guest. */
1952 LogFlow(("TSC %RX64 offset %RX64 time=%RX64 last=%RX64 (diff=%RX64, virt_tsc=%RX64)\n", u64CurTSC, pVCpu->hwaccm.s.vmx.u64TSCOffset, u64CurTSC + pVCpu->hwaccm.s.vmx.u64TSCOffset, TMCpuTickGetLastSeen(pVCpu), TMCpuTickGetLastSeen(pVCpu) - u64CurTSC - pVCpu->hwaccm.s.vmx.u64TSCOffset, TMCpuTickGet(pVCpu)));
1953 pVCpu->hwaccm.s.vmx.proc_ctls |= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT;
1954 rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
1955 AssertRC(rc);
1956 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatTSCInterceptOverFlow);
1957 }
1958 }
1959 else
1960 {
1961 pVCpu->hwaccm.s.vmx.proc_ctls |= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT;
1962 rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
1963 AssertRC(rc);
1964 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatTSCIntercept);
1965 }
1966
1967 /* Done with the major changes */
1968 pVCpu->hwaccm.s.fContextUseFlags &= ~HWACCM_CHANGED_ALL_GUEST;
1969
1970 /* Minimal guest state update (esp, eip, eflags mostly) */
1971 VMXR0LoadMinimalGuestState(pVM, pVCpu, pCtx);
1972 return rc;
1973}
1974
1975/**
1976 * Syncs back the guest state
1977 *
1978 * @returns VBox status code.
1979 * @param pVM The VM to operate on.
1980 * @param pVCpu The VMCPU to operate on.
1981 * @param pCtx Guest context
1982 */
1983DECLINLINE(int) VMXR0SaveGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
1984{
1985 RTGCUINTREG val, valShadow;
1986 RTGCUINTPTR uInterruptState;
1987 int rc;
1988
1989 /* Let's first sync back eip, esp, and eflags. */
1990 rc = VMXReadCachedVMCS(VMX_VMCS64_GUEST_RIP, &val);
1991 AssertRC(rc);
1992 pCtx->rip = val;
1993 rc = VMXReadCachedVMCS(VMX_VMCS64_GUEST_RSP, &val);
1994 AssertRC(rc);
1995 pCtx->rsp = val;
1996 rc = VMXReadCachedVMCS(VMX_VMCS_GUEST_RFLAGS, &val);
1997 AssertRC(rc);
1998 pCtx->eflags.u32 = val;
1999
2000 /* Take care of instruction fusing (sti, mov ss) */
2001 rc |= VMXReadCachedVMCS(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &val);
2002 uInterruptState = val;
2003 if (uInterruptState != 0)
2004 {
2005 Assert(uInterruptState <= 2); /* only sti & mov ss */
2006 Log(("uInterruptState %x eip=%RGv\n", (uint32_t)uInterruptState, pCtx->rip));
2007 EMSetInhibitInterruptsPC(pVCpu, pCtx->rip);
2008 }
2009 else
2010 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
2011
2012 /* Control registers. */
2013 VMXReadCachedVMCS(VMX_VMCS_CTRL_CR0_READ_SHADOW, &valShadow);
2014 VMXReadCachedVMCS(VMX_VMCS64_GUEST_CR0, &val);
2015 val = (valShadow & pVCpu->hwaccm.s.vmx.cr0_mask) | (val & ~pVCpu->hwaccm.s.vmx.cr0_mask);
2016 CPUMSetGuestCR0(pVCpu, val);
2017
2018 VMXReadCachedVMCS(VMX_VMCS_CTRL_CR4_READ_SHADOW, &valShadow);
2019 VMXReadCachedVMCS(VMX_VMCS64_GUEST_CR4, &val);
2020 val = (valShadow & pVCpu->hwaccm.s.vmx.cr4_mask) | (val & ~pVCpu->hwaccm.s.vmx.cr4_mask);
2021 CPUMSetGuestCR4(pVCpu, val);
2022
2023 /* Note: no reason to sync back the CRx registers. They can't be changed by the guest. */
2024 /* Note: only in the nested paging case can CR3 & CR4 be changed by the guest. */
2025 if ( pVM->hwaccm.s.fNestedPaging
2026 && CPUMIsGuestInPagedProtectedModeEx(pCtx)) /** @todo check if we will always catch mode switches and such... */
2027 {
2028 PVMCSCACHE pCache = &pVCpu->hwaccm.s.vmx.VMCSCache;
2029
2030 /* Can be updated behind our back in the nested paging case. */
2031 CPUMSetGuestCR2(pVCpu, pCache->cr2);
2032
2033 VMXReadCachedVMCS(VMX_VMCS64_GUEST_CR3, &val);
2034
2035 if (val != pCtx->cr3)
2036 {
2037 CPUMSetGuestCR3(pVCpu, val);
2038 PGMUpdateCR3(pVCpu, val);
2039 }
2040 rc = hmR0VmxSavePaePdpes(pVCpu, pCtx);
2041 AssertRCReturn(rc, rc);
2042 }
2043
2044 /* Sync back DR7 here. */
2045 VMXReadCachedVMCS(VMX_VMCS64_GUEST_DR7, &val);
2046 pCtx->dr[7] = val;
2047
2048 /* Guest CPU context: ES, CS, SS, DS, FS, GS. */
2049 VMX_READ_SELREG(ES, es);
2050 VMX_READ_SELREG(SS, ss);
2051 VMX_READ_SELREG(CS, cs);
2052 VMX_READ_SELREG(DS, ds);
2053 VMX_READ_SELREG(FS, fs);
2054 VMX_READ_SELREG(GS, gs);
2055
2056 /*
2057 * System MSRs
2058 */
2059 VMXReadCachedVMCS(VMX_VMCS32_GUEST_SYSENTER_CS, &val);
2060 pCtx->SysEnter.cs = val;
2061 VMXReadCachedVMCS(VMX_VMCS64_GUEST_SYSENTER_EIP, &val);
2062 pCtx->SysEnter.eip = val;
2063 VMXReadCachedVMCS(VMX_VMCS64_GUEST_SYSENTER_ESP, &val);
2064 pCtx->SysEnter.esp = val;
2065
2066 /* Misc. registers; must sync everything otherwise we can get out of sync when jumping to ring 3. */
2067 VMX_READ_SELREG(LDTR, ldtr);
2068
2069 VMXReadCachedVMCS(VMX_VMCS32_GUEST_GDTR_LIMIT, &val);
2070 pCtx->gdtr.cbGdt = val;
2071 VMXReadCachedVMCS(VMX_VMCS64_GUEST_GDTR_BASE, &val);
2072 pCtx->gdtr.pGdt = val;
2073
2074 VMXReadCachedVMCS(VMX_VMCS32_GUEST_IDTR_LIMIT, &val);
2075 pCtx->idtr.cbIdt = val;
2076 VMXReadCachedVMCS(VMX_VMCS64_GUEST_IDTR_BASE, &val);
2077 pCtx->idtr.pIdt = val;
2078
2079 /* Real mode emulation using v86 mode. */
2080 if ( CPUMIsGuestInRealModeEx(pCtx)
2081 && pVM->hwaccm.s.vmx.pRealModeTSS)
2082 {
2083 /* Hide our emulation flags */
2084 pCtx->eflags.Bits.u1VM = 0;
2085
2086 /* Restore original IOPL setting as we always use 0. */
2087 pCtx->eflags.Bits.u2IOPL = pVCpu->hwaccm.s.vmx.RealMode.eflags.Bits.u2IOPL;
2088
2089 /* Force a TR resync every time in case we switch modes. */
2090 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_GUEST_TR;
2091 }
2092 else
2093 {
2094 /* In real mode we have a fake TSS, so only sync it back when it's supposed to be valid. */
2095 VMX_READ_SELREG(TR, tr);
2096 }
2097
2098#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
2099 /* Save the possibly changed MSRs that we automatically restore and save during a world switch. */
2100 for (unsigned i = 0; i < pVCpu->hwaccm.s.vmx.cCachedMSRs; i++)
2101 {
2102 PVMXMSR pMsr = (PVMXMSR)pVCpu->hwaccm.s.vmx.pGuestMSR;
2103 pMsr += i;
2104
2105 switch (pMsr->u32IndexMSR)
2106 {
2107 case MSR_K8_LSTAR:
2108 pCtx->msrLSTAR = pMsr->u64Value;
2109 break;
2110 case MSR_K6_STAR:
2111 pCtx->msrSTAR = pMsr->u64Value;
2112 break;
2113 case MSR_K8_SF_MASK:
2114 pCtx->msrSFMASK = pMsr->u64Value;
2115 break;
2116 case MSR_K8_KERNEL_GS_BASE:
2117 pCtx->msrKERNELGSBASE = pMsr->u64Value;
2118 break;
2119 case MSR_K6_EFER:
2120 /* EFER can't be changed without causing a VM-exit. */
2121// Assert(pCtx->msrEFER == pMsr->u64Value);
2122 break;
2123 default:
2124 AssertFailed();
2125 return VERR_HM_UNEXPECTED_LD_ST_MSR;
2126 }
2127 }
2128#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
2129 return VINF_SUCCESS;
2130}
2131
2132/**
2133 * Dummy placeholder
2134 *
2135 * @param pVM The VM to operate on.
2136 * @param pVCpu The VMCPU to operate on.
2137 */
2138static DECLCALLBACK(void) hmR0VmxSetupTLBDummy(PVM pVM, PVMCPU pVCpu)
2139{
2140 NOREF(pVM);
2141 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
2142 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
2143 pVCpu->hwaccm.s.TlbShootdown.cPages = 0;
2144 return;
2145}
2146
2147/**
2148 * Setup the tagged TLB for EPT
2149 *
2150 * @returns VBox status code.
2151 * @param pVM The VM to operate on.
2152 * @param pVCpu The VMCPU to operate on.
2153 */
2154static DECLCALLBACK(void) hmR0VmxSetupTLBEPT(PVM pVM, PVMCPU pVCpu)
2155{
2156 PHMGLOBLCPUINFO pCpu;
2157
2158 Assert(pVM->hwaccm.s.fNestedPaging);
2159 Assert(!pVM->hwaccm.s.vmx.fVPID);
2160
2161 /* Deal with tagged TLBs if VPID or EPT is supported. */
2162 pCpu = HWACCMR0GetCurrentCpu();
2163 /* Force a TLB flush for the first world switch if the current cpu differs from the one we ran on last. */
2164 /* Note that this can happen both for start and resume due to long jumps back to ring 3. */
2165 if ( pVCpu->hwaccm.s.idLastCpu != pCpu->idCpu
2166 /* if the tlb flush count has changed, another VM has flushed the TLB of this cpu, so we can't use our current ASID anymore. */
2167 || pVCpu->hwaccm.s.cTLBFlushes != pCpu->cTLBFlushes)
2168 {
2169 /* Force a TLB flush on VM entry. */
2170 pVCpu->hwaccm.s.fForceTLBFlush = true;
2171 }
2172 /* Disabled because this has triggered every time I have suspended my
2173 * laptop with a VM running for the past three months or more. */
2174 // else
2175 // Assert(!pCpu->fFlushTLB);
2176
2177 /* Check for tlb shootdown flushes. */
2178 if (VMCPU_FF_TESTANDCLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2179 pVCpu->hwaccm.s.fForceTLBFlush = true;
2180
2181 pVCpu->hwaccm.s.idLastCpu = pCpu->idCpu;
2182 pCpu->fFlushTLB = false;
2183
2184 if (pVCpu->hwaccm.s.fForceTLBFlush)
2185 {
2186 hmR0VmxFlushEPT(pVM, pVCpu, pVM->hwaccm.s.vmx.enmFlushContext, 0);
2187 }
2188 else
2189 if (VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
2190 {
2191 /* Deal with pending TLB shootdown actions which were queued when we were not executing code. */
2192 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatTlbShootdown);
2193
2194 for (unsigned i=0;i<pVCpu->hwaccm.s.TlbShootdown.cPages;i++)
2195 {
2196 /* aTlbShootdownPages contains physical addresses in this case. */
2197 hmR0VmxFlushEPT(pVM, pVCpu, pVM->hwaccm.s.vmx.enmFlushPage, pVCpu->hwaccm.s.TlbShootdown.aPages[i]);
2198 }
2199 }
2200 pVCpu->hwaccm.s.TlbShootdown.cPages= 0;
2201 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
2202
2203#ifdef VBOX_WITH_STATISTICS
2204 if (pVCpu->hwaccm.s.fForceTLBFlush)
2205 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatFlushTLBWorldSwitch);
2206 else
2207 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatNoFlushTLBWorldSwitch);
2208#endif
2209}
2210
2211#ifdef HWACCM_VTX_WITH_VPID
2212/**
2213 * Setup the tagged TLB for VPID
2214 *
2215 * @returns VBox status code.
2216 * @param pVM The VM to operate on.
2217 * @param pVCpu The VMCPU to operate on.
2218 */
2219static DECLCALLBACK(void) hmR0VmxSetupTLBVPID(PVM pVM, PVMCPU pVCpu)
2220{
2221 PHMGLOBLCPUINFO pCpu;
2222
2223 Assert(pVM->hwaccm.s.vmx.fVPID);
2224 Assert(!pVM->hwaccm.s.fNestedPaging);
2225
2226 /* Deal with tagged TLBs if VPID or EPT is supported. */
2227 pCpu = HWACCMR0GetCurrentCpu();
2228 /* Force a TLB flush for the first world switch if the current cpu differs from the one we ran on last. */
2229 /* Note that this can happen both for start and resume due to long jumps back to ring 3. */
2230 if ( pVCpu->hwaccm.s.idLastCpu != pCpu->idCpu
2231 /* if the tlb flush count has changed, another VM has flushed the TLB of this cpu, so we can't use our current ASID anymore. */
2232 || pVCpu->hwaccm.s.cTLBFlushes != pCpu->cTLBFlushes)
2233 {
2234 /* Force a TLB flush on VM entry. */
2235 pVCpu->hwaccm.s.fForceTLBFlush = true;
2236 }
2237 else
2238 Assert(!pCpu->fFlushTLB);
2239
2240 pVCpu->hwaccm.s.idLastCpu = pCpu->idCpu;
2241
2242 /* Check for tlb shootdown flushes. */
2243 if (VMCPU_FF_TESTANDCLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2244 pVCpu->hwaccm.s.fForceTLBFlush = true;
2245
2246 /* Make sure we flush the TLB when required. Switch ASID to achieve the same thing, but without actually flushing the whole TLB (which is expensive). */
2247 if (pVCpu->hwaccm.s.fForceTLBFlush)
2248 {
2249 if ( ++pCpu->uCurrentASID >= pVM->hwaccm.s.uMaxASID
2250 || pCpu->fFlushTLB)
2251 {
2252 pCpu->fFlushTLB = false;
2253 pCpu->uCurrentASID = 1; /* start at 1; host uses 0 */
2254 pCpu->cTLBFlushes++;
2255 hmR0VmxFlushVPID(pVM, pVCpu, VMX_FLUSH_ALL_CONTEXTS, 0);
2256 }
2257 else
2258 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatFlushASID);
2259
2260 pVCpu->hwaccm.s.fForceTLBFlush = false;
2261 pVCpu->hwaccm.s.cTLBFlushes = pCpu->cTLBFlushes;
2262 pVCpu->hwaccm.s.uCurrentASID = pCpu->uCurrentASID;
2263 }
2264 else
2265 {
2266 Assert(!pCpu->fFlushTLB);
2267 Assert(pVCpu->hwaccm.s.uCurrentASID && pCpu->uCurrentASID);
2268
2269 if (VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
2270 {
2271 /* Deal with pending TLB shootdown actions which were queued when we were not executing code. */
2272 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatTlbShootdown);
2273 for (unsigned i = 0; i < pVCpu->hwaccm.s.TlbShootdown.cPages; i++)
2274 hmR0VmxFlushVPID(pVM, pVCpu, pVM->hwaccm.s.vmx.enmFlushPage, pVCpu->hwaccm.s.TlbShootdown.aPages[i]);
2275 }
2276 }
2277 pVCpu->hwaccm.s.TlbShootdown.cPages = 0;
2278 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
2279
2280 AssertMsg(pVCpu->hwaccm.s.cTLBFlushes == pCpu->cTLBFlushes, ("Flush count mismatch for cpu %d (%x vs %x)\n", pCpu->idCpu, pVCpu->hwaccm.s.cTLBFlushes, pCpu->cTLBFlushes));
2281 AssertMsg(pCpu->uCurrentASID >= 1 && pCpu->uCurrentASID < pVM->hwaccm.s.uMaxASID, ("cpu%d uCurrentASID = %x\n", pCpu->idCpu, pCpu->uCurrentASID));
2282 AssertMsg(pVCpu->hwaccm.s.uCurrentASID >= 1 && pVCpu->hwaccm.s.uCurrentASID < pVM->hwaccm.s.uMaxASID, ("cpu%d VM uCurrentASID = %x\n", pCpu->idCpu, pVCpu->hwaccm.s.uCurrentASID));
2283
2284 int rc = VMXWriteVMCS(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hwaccm.s.uCurrentASID);
2285 AssertRC(rc);
2286
2287 if (pVCpu->hwaccm.s.fForceTLBFlush)
2288 hmR0VmxFlushVPID(pVM, pVCpu, pVM->hwaccm.s.vmx.enmFlushContext, 0);
2289
2290# ifdef VBOX_WITH_STATISTICS
2291 if (pVCpu->hwaccm.s.fForceTLBFlush)
2292 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatFlushTLBWorldSwitch);
2293 else
2294 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatNoFlushTLBWorldSwitch);
2295# endif
2296}
2297#endif /* HWACCM_VTX_WITH_VPID */
2298
2299/**
2300 * Runs guest code in a VT-x VM.
2301 *
2302 * @returns VBox status code.
2303 * @param pVM The VM to operate on.
2304 * @param pVCpu The VMCPU to operate on.
2305 * @param pCtx Guest context
2306 */
2307VMMR0DECL(int) VMXR0RunGuestCode(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
2308{
2309 STAM_PROFILE_ADV_START(&pVCpu->hwaccm.s.StatEntry, x);
2310 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hwaccm.s.StatExit1);
2311 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hwaccm.s.StatExit2);
2312
2313 VBOXSTRICTRC rc = VINF_SUCCESS;
2314 int rc2;
2315 RTGCUINTREG val;
2316 RTGCUINTREG exitReason = (RTGCUINTREG)VMX_EXIT_INVALID;
2317 RTGCUINTREG instrError, cbInstr;
2318 RTGCUINTPTR exitQualification = 0;
2319 RTGCUINTPTR intInfo = 0; /* shut up buggy gcc 4 */
2320 RTGCUINTPTR errCode, instrInfo;
2321 bool fSetupTPRCaching = false;
2322 uint64_t u64OldLSTAR = 0;
2323 uint8_t u8LastTPR = 0;
2324 RTCCUINTREG uOldEFlags = ~(RTCCUINTREG)0;
2325 unsigned cResume = 0;
2326#ifdef VBOX_STRICT
2327 RTCPUID idCpuCheck;
2328 bool fWasInLongMode = false;
2329#endif
2330#ifdef VBOX_HIGH_RES_TIMERS_HACK_IN_RING0
2331 uint64_t u64LastTime = RTTimeMilliTS();
2332#endif
2333
2334 Assert(!(pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC) || (pVCpu->hwaccm.s.vmx.pbVAPIC && pVM->hwaccm.s.vmx.pAPIC));
2335
2336 /* Check if we need to use TPR shadowing. */
2337 if ( CPUMIsGuestInLongModeEx(pCtx)
2338 || ( ((pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC) || pVM->hwaccm.s.fTRPPatchingAllowed)
2339 && pVM->hwaccm.s.fHasIoApic)
2340 )
2341 {
2342 fSetupTPRCaching = true;
2343 }
2344
2345 Log2(("\nE"));
2346
2347#ifdef VBOX_STRICT
2348 {
2349 RTCCUINTREG val2;
2350
2351 rc2 = VMXReadVMCS(VMX_VMCS_CTRL_PIN_EXEC_CONTROLS, &val2);
2352 AssertRC(rc2);
2353 Log2(("VMX_VMCS_CTRL_PIN_EXEC_CONTROLS = %08x\n", val2));
2354
2355 /* allowed zero */
2356 if ((val2 & pVM->hwaccm.s.vmx.msr.vmx_pin_ctls.n.disallowed0) != pVM->hwaccm.s.vmx.msr.vmx_pin_ctls.n.disallowed0)
2357 Log(("Invalid VMX_VMCS_CTRL_PIN_EXEC_CONTROLS: zero\n"));
2358
2359 /* allowed one */
2360 if ((val2 & ~pVM->hwaccm.s.vmx.msr.vmx_pin_ctls.n.allowed1) != 0)
2361 Log(("Invalid VMX_VMCS_CTRL_PIN_EXEC_CONTROLS: one\n"));
2362
2363 rc2 = VMXReadVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, &val2);
2364 AssertRC(rc2);
2365 Log2(("VMX_VMCS_CTRL_PROC_EXEC_CONTROLS = %08x\n", val2));
2366
2367 /* Must be set according to the MSR, but can be cleared in case of EPT. */
2368 if (pVM->hwaccm.s.fNestedPaging)
2369 val2 |= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INVLPG_EXIT
2370 | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
2371 | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT;
2372
2373 /* allowed zero */
2374 if ((val2 & pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.disallowed0) != pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.disallowed0)
2375 Log(("Invalid VMX_VMCS_CTRL_PROC_EXEC_CONTROLS: zero\n"));
2376
2377 /* allowed one */
2378 if ((val2 & ~pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1) != 0)
2379 Log(("Invalid VMX_VMCS_CTRL_PROC_EXEC_CONTROLS: one\n"));
2380
2381 rc2 = VMXReadVMCS(VMX_VMCS_CTRL_ENTRY_CONTROLS, &val2);
2382 AssertRC(rc2);
2383 Log2(("VMX_VMCS_CTRL_ENTRY_CONTROLS = %08x\n", val2));
2384
2385 /* allowed zero */
2386 if ((val2 & pVM->hwaccm.s.vmx.msr.vmx_entry.n.disallowed0) != pVM->hwaccm.s.vmx.msr.vmx_entry.n.disallowed0)
2387 Log(("Invalid VMX_VMCS_CTRL_ENTRY_CONTROLS: zero\n"));
2388
2389 /* allowed one */
2390 if ((val2 & ~pVM->hwaccm.s.vmx.msr.vmx_entry.n.allowed1) != 0)
2391 Log(("Invalid VMX_VMCS_CTRL_ENTRY_CONTROLS: one\n"));
2392
2393 rc2 = VMXReadVMCS(VMX_VMCS_CTRL_EXIT_CONTROLS, &val2);
2394 AssertRC(rc2);
2395 Log2(("VMX_VMCS_CTRL_EXIT_CONTROLS = %08x\n", val2));
2396
2397 /* allowed zero */
2398 if ((val2 & pVM->hwaccm.s.vmx.msr.vmx_exit.n.disallowed0) != pVM->hwaccm.s.vmx.msr.vmx_exit.n.disallowed0)
2399 Log(("Invalid VMX_VMCS_CTRL_EXIT_CONTROLS: zero\n"));
2400
2401 /* allowed one */
2402 if ((val2 & ~pVM->hwaccm.s.vmx.msr.vmx_exit.n.allowed1) != 0)
2403 Log(("Invalid VMX_VMCS_CTRL_EXIT_CONTROLS: one\n"));
2404 }
2405 fWasInLongMode = CPUMIsGuestInLongModeEx(pCtx);
2406#endif /* VBOX_STRICT */
2407
2408#ifdef VBOX_WITH_CRASHDUMP_MAGIC
2409 pVCpu->hwaccm.s.vmx.VMCSCache.u64TimeEntry = RTTimeNanoTS();
2410#endif
2411
2412 /* We can jump to this point to resume execution after determining that a VM-exit is innocent.
2413 */
2414ResumeExecution:
2415 if (!STAM_REL_PROFILE_ADV_IS_RUNNING(&pVCpu->hwaccm.s.StatEntry))
2416 STAM_REL_PROFILE_ADV_STOP_START(&pVCpu->hwaccm.s.StatExit2, &pVCpu->hwaccm.s.StatEntry, x);
2417 AssertMsg(pVCpu->hwaccm.s.idEnteredCpu == RTMpCpuId(),
2418 ("Expected %d, I'm %d; cResume=%d exitReason=%RGv exitQualification=%RGv\n",
2419 (int)pVCpu->hwaccm.s.idEnteredCpu, (int)RTMpCpuId(), cResume, exitReason, exitQualification));
2420 Assert(!HWACCMR0SuspendPending());
2421 /* Not allowed to switch modes without reloading the host state (32->64 switcher)!! */
2422 Assert(fWasInLongMode == CPUMIsGuestInLongModeEx(pCtx));
2423
2424 /* Safety precaution; looping for too long here can have a very bad effect on the host */
2425 if (RT_UNLIKELY(++cResume > pVM->hwaccm.s.cMaxResumeLoops))
2426 {
2427 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitMaxResume);
2428 rc = VINF_EM_RAW_INTERRUPT;
2429 goto end;
2430 }
2431
2432 /* Check for irq inhibition due to instruction fusing (sti, mov ss). */
2433 if (VMCPU_FF_ISSET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
2434 {
2435 Log(("VM_FF_INHIBIT_INTERRUPTS at %RGv successor %RGv\n", (RTGCPTR)pCtx->rip, EMGetInhibitInterruptsPC(pVCpu)));
2436 if (pCtx->rip != EMGetInhibitInterruptsPC(pVCpu))
2437 {
2438 /* Note: we intentionally don't clear VM_FF_INHIBIT_INTERRUPTS here.
2439 * Before we are able to execute this instruction in raw mode (iret to guest code) an external interrupt might
2440 * force a world switch again. Possibly allowing a guest interrupt to be dispatched in the process. This could
2441 * break the guest. Sounds very unlikely, but such timing sensitive problems are not as rare as you might think.
2442 */
2443 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
2444 /* Irq inhibition is no longer active; clear the corresponding VMX state. */
2445 rc2 = VMXWriteVMCS(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, 0);
2446 AssertRC(rc2);
2447 }
2448 }
2449 else
2450 {
2451 /* Irq inhibition is no longer active; clear the corresponding VMX state. */
2452 rc2 = VMXWriteVMCS(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, 0);
2453 AssertRC(rc2);
2454 }
2455
2456#ifdef VBOX_HIGH_RES_TIMERS_HACK_IN_RING0
2457 if (RT_UNLIKELY((cResume & 0xf) == 0))
2458 {
2459 uint64_t u64CurTime = RTTimeMilliTS();
2460
2461 if (RT_UNLIKELY(u64CurTime > u64LastTime))
2462 {
2463 u64LastTime = u64CurTime;
2464 TMTimerPollVoid(pVM, pVCpu);
2465 }
2466 }
2467#endif
2468
2469 /* Check for pending actions that force us to go back to ring 3. */
2470 if ( VM_FF_ISPENDING(pVM, VM_FF_HWACCM_TO_R3_MASK | VM_FF_REQUEST | VM_FF_PGM_POOL_FLUSH_PENDING | VM_FF_PDM_DMA)
2471 || VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_HWACCM_TO_R3_MASK | VMCPU_FF_PGM_SYNC_CR3 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL | VMCPU_FF_REQUEST))
2472 {
2473 /* Check if a sync operation is pending. */
2474 if (VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
2475 {
2476 rc = PGMSyncCR3(pVCpu, pCtx->cr0, pCtx->cr3, pCtx->cr4, VMCPU_FF_ISSET(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
2477 if (rc != VINF_SUCCESS)
2478 {
2479 AssertRC(VBOXSTRICTRC_VAL(rc));
2480 Log(("Pending pool sync is forcing us back to ring 3; rc=%d\n", VBOXSTRICTRC_VAL(rc)));
2481 goto end;
2482 }
2483 }
2484
2485#ifdef DEBUG
2486 /* Intercept X86_XCPT_DB if stepping is enabled */
2487 if (!DBGFIsStepping(pVCpu))
2488#endif
2489 {
2490 if ( VM_FF_ISPENDING(pVM, VM_FF_HWACCM_TO_R3_MASK)
2491 || VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_HWACCM_TO_R3_MASK))
2492 {
2493 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatSwitchToR3);
2494 rc = RT_UNLIKELY(VM_FF_ISPENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
2495 goto end;
2496 }
2497 }
2498
2499 /* Pending request packets might contain actions that need immediate attention, such as pending hardware interrupts. */
2500 if ( VM_FF_ISPENDING(pVM, VM_FF_REQUEST)
2501 || VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_REQUEST))
2502 {
2503 rc = VINF_EM_PENDING_REQUEST;
2504 goto end;
2505 }
2506
2507 /* Check if a pgm pool flush is in progress. */
2508 if (VM_FF_ISPENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
2509 {
2510 rc = VINF_PGM_POOL_FLUSH_PENDING;
2511 goto end;
2512 }
2513
2514 /* Check if DMA work is pending (2nd+ run). */
2515 if (VM_FF_ISPENDING(pVM, VM_FF_PDM_DMA) && cResume > 1)
2516 {
2517 rc = VINF_EM_RAW_TO_R3;
2518 goto end;
2519 }
2520 }
2521
2522#ifdef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
2523 /*
2524 * Exit to ring-3 preemption/work is pending.
2525 *
2526 * Interrupts are disabled before the call to make sure we don't miss any interrupt
2527 * that would flag preemption (IPI, timer tick, ++). (Would've been nice to do this
2528 * further down, but hmR0VmxCheckPendingInterrupt makes that impossible.)
2529 *
2530 * Note! Interrupts must be disabled done *before* we check for TLB flushes; TLB
2531 * shootdowns rely on this.
2532 */
2533 uOldEFlags = ASMIntDisableFlags();
2534 if (RTThreadPreemptIsPending(NIL_RTTHREAD))
2535 {
2536 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitPreemptPending);
2537 rc = VINF_EM_RAW_INTERRUPT;
2538 goto end;
2539 }
2540 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
2541#endif
2542
2543 /* When external interrupts are pending, we should exit the VM when IF is set. */
2544 /* Note! *After* VM_FF_INHIBIT_INTERRUPTS check!!! */
2545 rc = hmR0VmxCheckPendingInterrupt(pVM, pVCpu, pCtx);
2546 if (RT_FAILURE(rc))
2547 goto end;
2548
2549 /** @todo check timers?? */
2550
2551 /* TPR caching using CR8 is only available in 64 bits mode */
2552 /* Note the 32 bits exception for AMD (X86_CPUID_AMD_FEATURE_ECX_CR8L), but that appears missing in Intel CPUs */
2553 /* Note: we can't do this in LoadGuestState as PDMApicGetTPR can jump back to ring 3 (lock)!!!!! (no longer true) */
2554 /**
2555 * @todo query and update the TPR only when it could have been changed (mmio access & wrmsr (x2apic))
2556 */
2557 if (fSetupTPRCaching)
2558 {
2559 /* TPR caching in CR8 */
2560 bool fPending;
2561
2562 rc2 = PDMApicGetTPR(pVCpu, &u8LastTPR, &fPending);
2563 AssertRC(rc2);
2564 /* The TPR can be found at offset 0x80 in the APIC mmio page. */
2565 pVCpu->hwaccm.s.vmx.pbVAPIC[0x80] = u8LastTPR;
2566
2567 /* Two options here:
2568 * - external interrupt pending, but masked by the TPR value.
2569 * -> a CR8 update that lower the current TPR value should cause an exit
2570 * - no pending interrupts
2571 * -> We don't need to be explicitely notified. There are enough world switches for detecting pending interrupts.
2572 */
2573 rc = VMXWriteVMCS(VMX_VMCS_CTRL_TPR_THRESHOLD, (fPending) ? (u8LastTPR >> 4) : 0); /* cr8 bits 3-0 correspond to bits 7-4 of the task priority mmio register. */
2574 AssertRC(VBOXSTRICTRC_VAL(rc));
2575
2576 if (pVM->hwaccm.s.fTPRPatchingActive)
2577 {
2578 Assert(!CPUMIsGuestInLongModeEx(pCtx));
2579 /* Our patch code uses LSTAR for TPR caching. */
2580 pCtx->msrLSTAR = u8LastTPR;
2581
2582 if (fPending)
2583 {
2584 /* A TPR change could activate a pending interrupt, so catch lstar writes. */
2585 hmR0VmxSetMSRPermission(pVCpu, MSR_K8_LSTAR, true, false);
2586 }
2587 else
2588 {
2589 /* No interrupts are pending, so we don't need to be explicitely notified.
2590 * There are enough world switches for detecting pending interrupts.
2591 */
2592 hmR0VmxSetMSRPermission(pVCpu, MSR_K8_LSTAR, true, true);
2593 }
2594 }
2595 }
2596
2597#if defined(HWACCM_VTX_WITH_EPT) && defined(LOG_ENABLED)
2598 if ( pVM->hwaccm.s.fNestedPaging
2599# ifdef HWACCM_VTX_WITH_VPID
2600 || pVM->hwaccm.s.vmx.fVPID
2601# endif /* HWACCM_VTX_WITH_VPID */
2602 )
2603 {
2604 PHMGLOBLCPUINFO pCpu;
2605
2606 pCpu = HWACCMR0GetCurrentCpu();
2607 if ( pVCpu->hwaccm.s.idLastCpu != pCpu->idCpu
2608 || pVCpu->hwaccm.s.cTLBFlushes != pCpu->cTLBFlushes)
2609 {
2610 if (pVCpu->hwaccm.s.idLastCpu != pCpu->idCpu)
2611 LogFlow(("Force TLB flush due to rescheduling to a different cpu (%d vs %d)\n", pVCpu->hwaccm.s.idLastCpu, pCpu->idCpu));
2612 else
2613 LogFlow(("Force TLB flush due to changed TLB flush count (%x vs %x)\n", pVCpu->hwaccm.s.cTLBFlushes, pCpu->cTLBFlushes));
2614 }
2615 if (pCpu->fFlushTLB)
2616 LogFlow(("Force TLB flush: first time cpu %d is used -> flush\n", pCpu->idCpu));
2617 else
2618 if (pVCpu->hwaccm.s.fForceTLBFlush)
2619 LogFlow(("Manual TLB flush\n"));
2620 }
2621#endif
2622#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
2623 PGMRZDynMapFlushAutoSet(pVCpu);
2624#endif
2625
2626 /*
2627 * NOTE: DO NOT DO ANYTHING AFTER THIS POINT THAT MIGHT JUMP BACK TO RING 3!
2628 * (until the actual world switch)
2629 */
2630#ifdef VBOX_STRICT
2631 idCpuCheck = RTMpCpuId();
2632#endif
2633#ifdef LOG_ENABLED
2634 VMMR0LogFlushDisable(pVCpu);
2635#endif
2636 /* Save the host state first. */
2637 if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_HOST_CONTEXT)
2638 {
2639 rc = VMXR0SaveHostState(pVM, pVCpu);
2640 if (RT_UNLIKELY(rc != VINF_SUCCESS))
2641 {
2642 VMMR0LogFlushEnable(pVCpu);
2643 goto end;
2644 }
2645 }
2646
2647 /* Load the guest state */
2648 if (!pVCpu->hwaccm.s.fContextUseFlags)
2649 {
2650 VMXR0LoadMinimalGuestState(pVM, pVCpu, pCtx);
2651 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatLoadMinimal);
2652 }
2653 else
2654 {
2655 rc = VMXR0LoadGuestState(pVM, pVCpu, pCtx);
2656 if (RT_UNLIKELY(rc != VINF_SUCCESS))
2657 {
2658 VMMR0LogFlushEnable(pVCpu);
2659 goto end;
2660 }
2661 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatLoadFull);
2662 }
2663
2664#ifndef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
2665 /* Disable interrupts to make sure a poke will interrupt execution.
2666 * This must be done *before* we check for TLB flushes; TLB shootdowns rely on this.
2667 */
2668 uOldEFlags = ASMIntDisableFlags();
2669 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
2670#endif
2671
2672 /* Non-register state Guest Context */
2673 /** @todo change me according to cpu state */
2674 rc2 = VMXWriteVMCS(VMX_VMCS32_GUEST_ACTIVITY_STATE, VMX_CMS_GUEST_ACTIVITY_ACTIVE);
2675 AssertRC(rc2);
2676
2677 /* Set TLB flush state as checked until we return from the world switch. */
2678 ASMAtomicWriteBool(&pVCpu->hwaccm.s.fCheckedTLBFlush, true);
2679 /* Deal with tagged TLB setup and invalidation. */
2680 pVM->hwaccm.s.vmx.pfnSetupTaggedTLB(pVM, pVCpu);
2681
2682 /* Manual save and restore:
2683 * - General purpose registers except RIP, RSP
2684 *
2685 * Trashed:
2686 * - CR2 (we don't care)
2687 * - LDTR (reset to 0)
2688 * - DRx (presumably not changed at all)
2689 * - DR7 (reset to 0x400)
2690 * - EFLAGS (reset to RT_BIT(1); not relevant)
2691 *
2692 */
2693
2694 /* All done! Let's start VM execution. */
2695 STAM_PROFILE_ADV_STOP_START(&pVCpu->hwaccm.s.StatEntry, &pVCpu->hwaccm.s.StatInGC, x);
2696 Assert(idCpuCheck == RTMpCpuId());
2697
2698#ifdef VBOX_WITH_CRASHDUMP_MAGIC
2699 pVCpu->hwaccm.s.vmx.VMCSCache.cResume = cResume;
2700 pVCpu->hwaccm.s.vmx.VMCSCache.u64TimeSwitch = RTTimeNanoTS();
2701#endif
2702
2703 /* Save the current TPR value in the LSTAR msr so our patches can access it. */
2704 if (pVM->hwaccm.s.fTPRPatchingActive)
2705 {
2706 Assert(pVM->hwaccm.s.fTPRPatchingActive);
2707 u64OldLSTAR = ASMRdMsr(MSR_K8_LSTAR);
2708 ASMWrMsr(MSR_K8_LSTAR, u8LastTPR);
2709 }
2710
2711 TMNotifyStartOfExecution(pVCpu);
2712#ifdef VBOX_WITH_KERNEL_USING_XMM
2713 rc = hwaccmR0VMXStartVMWrapXMM(pVCpu->hwaccm.s.fResumeVM, pCtx, &pVCpu->hwaccm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hwaccm.s.vmx.pfnStartVM);
2714#else
2715 rc = pVCpu->hwaccm.s.vmx.pfnStartVM(pVCpu->hwaccm.s.fResumeVM, pCtx, &pVCpu->hwaccm.s.vmx.VMCSCache, pVM, pVCpu);
2716#endif
2717 ASMAtomicWriteBool(&pVCpu->hwaccm.s.fCheckedTLBFlush, false);
2718 ASMAtomicIncU32(&pVCpu->hwaccm.s.cWorldSwitchExits);
2719 /* Possibly the last TSC value seen by the guest (too high) (only when we're in tsc offset mode). */
2720 if (!(pVCpu->hwaccm.s.vmx.proc_ctls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT))
2721 TMCpuTickSetLastSeen(pVCpu, ASMReadTSC() + pVCpu->hwaccm.s.vmx.u64TSCOffset - 0x400 /* guestimate of world switch overhead in clock ticks */);
2722
2723 TMNotifyEndOfExecution(pVCpu);
2724 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED);
2725 Assert(!(ASMGetFlags() & X86_EFL_IF));
2726
2727 /* Restore the host LSTAR msr if the guest could have changed it. */
2728 if (pVM->hwaccm.s.fTPRPatchingActive)
2729 {
2730 Assert(pVM->hwaccm.s.fTPRPatchingActive);
2731 pVCpu->hwaccm.s.vmx.pbVAPIC[0x80] = pCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR);
2732 ASMWrMsr(MSR_K8_LSTAR, u64OldLSTAR);
2733 }
2734
2735 STAM_PROFILE_ADV_STOP_START(&pVCpu->hwaccm.s.StatInGC, &pVCpu->hwaccm.s.StatExit1, x);
2736 ASMSetFlags(uOldEFlags);
2737#ifdef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
2738 uOldEFlags = ~(RTCCUINTREG)0;
2739#endif
2740
2741 AssertMsg(!pVCpu->hwaccm.s.vmx.VMCSCache.Write.cValidEntries, ("pVCpu->hwaccm.s.vmx.VMCSCache.Write.cValidEntries=%d\n", pVCpu->hwaccm.s.vmx.VMCSCache.Write.cValidEntries));
2742
2743 /* In case we execute a goto ResumeExecution later on. */
2744 pVCpu->hwaccm.s.fResumeVM = true;
2745 pVCpu->hwaccm.s.fForceTLBFlush = false;
2746
2747 /*
2748 * !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
2749 * IMPORTANT: WE CAN'T DO ANY LOGGING OR OPERATIONS THAT CAN DO A LONGJMP BACK TO RING 3 *BEFORE* WE'VE SYNCED BACK (MOST OF) THE GUEST STATE
2750 * !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
2751 */
2752
2753 if (RT_UNLIKELY(rc != VINF_SUCCESS))
2754 {
2755 hmR0VmxReportWorldSwitchError(pVM, pVCpu, rc, pCtx);
2756 VMMR0LogFlushEnable(pVCpu);
2757 goto end;
2758 }
2759
2760 /* Success. Query the guest state and figure out what has happened. */
2761
2762 /* Investigate why there was a VM-exit. */
2763 rc2 = VMXReadCachedVMCS(VMX_VMCS32_RO_EXIT_REASON, &exitReason);
2764 STAM_COUNTER_INC(&pVCpu->hwaccm.s.paStatExitReasonR0[exitReason & MASK_EXITREASON_STAT]);
2765
2766 exitReason &= 0xffff; /* bit 0-15 contain the exit code. */
2767 rc2 |= VMXReadCachedVMCS(VMX_VMCS32_RO_VM_INSTR_ERROR, &instrError);
2768 rc2 |= VMXReadCachedVMCS(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &cbInstr);
2769 rc2 |= VMXReadCachedVMCS(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &intInfo);
2770 /* might not be valid; depends on VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID. */
2771 rc2 |= VMXReadCachedVMCS(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERRCODE, &errCode);
2772 rc2 |= VMXReadCachedVMCS(VMX_VMCS32_RO_EXIT_INSTR_INFO, &instrInfo);
2773 rc2 |= VMXReadCachedVMCS(VMX_VMCS_RO_EXIT_QUALIFICATION, &exitQualification);
2774 AssertRC(rc2);
2775
2776 /* Sync back the guest state */
2777 rc2 = VMXR0SaveGuestState(pVM, pVCpu, pCtx);
2778 AssertRC(rc2);
2779
2780 /* Note! NOW IT'S SAFE FOR LOGGING! */
2781 VMMR0LogFlushEnable(pVCpu);
2782 Log2(("Raw exit reason %08x\n", exitReason));
2783#if ARCH_BITS == 64 /* for the time being */
2784 VBOXVMM_R0_HMVMX_VMEXIT(pVCpu, pCtx, exitReason);
2785#endif
2786
2787 /* Check if an injected event was interrupted prematurely. */
2788 rc2 = VMXReadCachedVMCS(VMX_VMCS32_RO_IDT_INFO, &val);
2789 AssertRC(rc2);
2790 pVCpu->hwaccm.s.Event.intInfo = VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(val);
2791 if ( VMX_EXIT_INTERRUPTION_INFO_VALID(pVCpu->hwaccm.s.Event.intInfo)
2792 /* Ignore 'int xx' as they'll be restarted anyway. */
2793 && VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hwaccm.s.Event.intInfo) != VMX_EXIT_INTERRUPTION_INFO_TYPE_SW
2794 /* Ignore software exceptions (such as int3) as they'll reoccur when we restart the instruction anyway. */
2795 && VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hwaccm.s.Event.intInfo) != VMX_EXIT_INTERRUPTION_INFO_TYPE_SWEXCPT)
2796 {
2797 Assert(!pVCpu->hwaccm.s.Event.fPending);
2798 pVCpu->hwaccm.s.Event.fPending = true;
2799 /* Error code present? */
2800 if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(pVCpu->hwaccm.s.Event.intInfo))
2801 {
2802 rc2 = VMXReadCachedVMCS(VMX_VMCS32_RO_IDT_ERRCODE, &val);
2803 AssertRC(rc2);
2804 pVCpu->hwaccm.s.Event.errCode = val;
2805 Log(("Pending inject %RX64 at %RGv exit=%08x intInfo=%08x exitQualification=%RGv pending error=%RX64\n", pVCpu->hwaccm.s.Event.intInfo, (RTGCPTR)pCtx->rip, exitReason, intInfo, exitQualification, val));
2806 }
2807 else
2808 {
2809 Log(("Pending inject %RX64 at %RGv exit=%08x intInfo=%08x exitQualification=%RGv\n", pVCpu->hwaccm.s.Event.intInfo, (RTGCPTR)pCtx->rip, exitReason, intInfo, exitQualification));
2810 pVCpu->hwaccm.s.Event.errCode = 0;
2811 }
2812 }
2813#ifdef VBOX_STRICT
2814 else
2815 if ( VMX_EXIT_INTERRUPTION_INFO_VALID(pVCpu->hwaccm.s.Event.intInfo)
2816 /* Ignore software exceptions (such as int3) as they're reoccur when we restart the instruction anyway. */
2817 && VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hwaccm.s.Event.intInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_SWEXCPT)
2818 {
2819 Log(("Ignore pending inject %RX64 at %RGv exit=%08x intInfo=%08x exitQualification=%RGv\n", pVCpu->hwaccm.s.Event.intInfo, (RTGCPTR)pCtx->rip, exitReason, intInfo, exitQualification));
2820 }
2821
2822 if (exitReason == VMX_EXIT_ERR_INVALID_GUEST_STATE)
2823 HWACCMDumpRegs(pVM, pVCpu, pCtx);
2824#endif
2825
2826 Log2(("E%d: New EIP=%x:%RGv\n", (uint32_t)exitReason, pCtx->cs, (RTGCPTR)pCtx->rip));
2827 Log2(("Exit reason %d, exitQualification %RGv\n", (uint32_t)exitReason, exitQualification));
2828 Log2(("instrInfo=%d instrError=%d instr length=%d\n", (uint32_t)instrInfo, (uint32_t)instrError, (uint32_t)cbInstr));
2829 Log2(("Interruption error code %d\n", (uint32_t)errCode));
2830 Log2(("IntInfo = %08x\n", (uint32_t)intInfo));
2831
2832 /* Sync back the TPR if it was changed. */
2833 if ( fSetupTPRCaching
2834 && u8LastTPR != pVCpu->hwaccm.s.vmx.pbVAPIC[0x80])
2835 {
2836 rc2 = PDMApicSetTPR(pVCpu, pVCpu->hwaccm.s.vmx.pbVAPIC[0x80]);
2837 AssertRC(rc2);
2838 }
2839
2840#ifdef DBGFTRACE_ENABLED /** @todo DTrace later. */
2841 RTTraceBufAddMsgF(pVM->CTX_SUFF(hTraceBuf), "vmexit %08x %016RX64 at %04:%08RX64 %RX64",
2842 exitReason, (uint64_t)exitQualification, pCtx->cs, pCtx->rip, (uint64_t)intInfo);
2843#endif
2844 STAM_PROFILE_ADV_STOP_START(&pVCpu->hwaccm.s.StatExit1, &pVCpu->hwaccm.s.StatExit2, x);
2845
2846 /* Some cases don't need a complete resync of the guest CPU state; handle them here. */
2847 Assert(rc == VINF_SUCCESS); /* might consider VERR_IPE_UNINITIALIZED_STATUS here later... */
2848 switch (exitReason)
2849 {
2850 case VMX_EXIT_EXCEPTION: /* 0 Exception or non-maskable interrupt (NMI). */
2851 case VMX_EXIT_EXTERNAL_IRQ: /* 1 External interrupt. */
2852 {
2853 uint32_t vector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(intInfo);
2854
2855 if (!VMX_EXIT_INTERRUPTION_INFO_VALID(intInfo))
2856 {
2857 Assert(exitReason == VMX_EXIT_EXTERNAL_IRQ);
2858#if 0 //def VBOX_WITH_VMMR0_DISABLE_PREEMPTION
2859 if ( RTThreadPreemptIsPendingTrusty()
2860 && !RTThreadPreemptIsPending(NIL_RTTHREAD))
2861 goto ResumeExecution;
2862#endif
2863 /* External interrupt; leave to allow it to be dispatched again. */
2864 rc = VINF_EM_RAW_INTERRUPT;
2865 break;
2866 }
2867 STAM_PROFILE_ADV_START(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
2868 switch (VMX_EXIT_INTERRUPTION_INFO_TYPE(intInfo))
2869 {
2870 case VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI: /* Non-maskable interrupt. */
2871 /* External interrupt; leave to allow it to be dispatched again. */
2872 rc = VINF_EM_RAW_INTERRUPT;
2873 break;
2874
2875 case VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT: /* External hardware interrupt. */
2876 AssertFailed(); /* can't come here; fails the first check. */
2877 break;
2878
2879 case VMX_EXIT_INTERRUPTION_INFO_TYPE_DBEXCPT: /* Unknown why we get this type for #DB */
2880 case VMX_EXIT_INTERRUPTION_INFO_TYPE_SWEXCPT: /* Software exception. (#BP or #OF) */
2881 Assert(vector == 1 || vector == 3 || vector == 4);
2882 /* no break */
2883 case VMX_EXIT_INTERRUPTION_INFO_TYPE_HWEXCPT: /* Hardware exception. */
2884 Log2(("Hardware/software interrupt %d\n", vector));
2885 switch (vector)
2886 {
2887 case X86_XCPT_NM:
2888 {
2889 Log(("#NM fault at %RGv error code %x\n", (RTGCPTR)pCtx->rip, errCode));
2890
2891 /** @todo don't intercept #NM exceptions anymore when we've activated the guest FPU state. */
2892 /* If we sync the FPU/XMM state on-demand, then we can continue execution as if nothing has happened. */
2893 rc = CPUMR0LoadGuestFPU(pVM, pVCpu, pCtx);
2894 if (rc == VINF_SUCCESS)
2895 {
2896 Assert(CPUMIsGuestFPUStateActive(pVCpu));
2897
2898 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitShadowNM);
2899
2900 /* Continue execution. */
2901 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_GUEST_CR0;
2902
2903 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
2904 goto ResumeExecution;
2905 }
2906
2907 Log(("Forward #NM fault to the guest\n"));
2908 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestNM);
2909 rc2 = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), cbInstr, 0);
2910 AssertRC(rc2);
2911 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
2912 goto ResumeExecution;
2913 }
2914
2915 case X86_XCPT_PF: /* Page fault */
2916 {
2917#ifdef DEBUG
2918 if (pVM->hwaccm.s.fNestedPaging)
2919 { /* A genuine pagefault.
2920 * Forward the trap to the guest by injecting the exception and resuming execution.
2921 */
2922 Log(("Guest page fault at %RGv cr2=%RGv error code %RGv rsp=%RGv\n", (RTGCPTR)pCtx->rip, exitQualification, errCode, (RTGCPTR)pCtx->rsp));
2923
2924 Assert(CPUMIsGuestInPagedProtectedModeEx(pCtx));
2925
2926 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestPF);
2927
2928 /* Now we must update CR2. */
2929 pCtx->cr2 = exitQualification;
2930 rc2 = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), cbInstr, errCode);
2931 AssertRC(rc2);
2932
2933 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
2934 goto ResumeExecution;
2935 }
2936#endif
2937 Assert(!pVM->hwaccm.s.fNestedPaging);
2938
2939#ifdef VBOX_HWACCM_WITH_GUEST_PATCHING
2940 /* Shortcut for APIC TPR reads and writes; 32 bits guests only */
2941 if ( pVM->hwaccm.s.fTRPPatchingAllowed
2942 && pVM->hwaccm.s.pGuestPatchMem
2943 && (exitQualification & 0xfff) == 0x080
2944 && !(errCode & X86_TRAP_PF_P) /* not present */
2945 && CPUMGetGuestCPL(pVCpu, CPUMCTX2CORE(pCtx)) == 0
2946 && !CPUMIsGuestInLongModeEx(pCtx)
2947 && pVM->hwaccm.s.cPatches < RT_ELEMENTS(pVM->hwaccm.s.aPatches))
2948 {
2949 RTGCPHYS GCPhysApicBase, GCPhys;
2950 PDMApicGetBase(pVM, &GCPhysApicBase); /* @todo cache this */
2951 GCPhysApicBase &= PAGE_BASE_GC_MASK;
2952
2953 rc = PGMGstGetPage(pVCpu, (RTGCPTR)exitQualification, NULL, &GCPhys);
2954 if ( rc == VINF_SUCCESS
2955 && GCPhys == GCPhysApicBase)
2956 {
2957 /* Only attempt to patch the instruction once. */
2958 PHWACCMTPRPATCH pPatch = (PHWACCMTPRPATCH)RTAvloU32Get(&pVM->hwaccm.s.PatchTree, (AVLOU32KEY)pCtx->eip);
2959 if (!pPatch)
2960 {
2961 rc = VINF_EM_HWACCM_PATCH_TPR_INSTR;
2962 break;
2963 }
2964 }
2965 }
2966#endif
2967
2968 Log2(("Page fault at %RGv error code %x\n", exitQualification, errCode));
2969 /* Exit qualification contains the linear address of the page fault. */
2970 TRPMAssertTrap(pVCpu, X86_XCPT_PF, TRPM_TRAP);
2971 TRPMSetErrorCode(pVCpu, errCode);
2972 TRPMSetFaultAddress(pVCpu, exitQualification);
2973
2974 /* Shortcut for APIC TPR reads and writes. */
2975 if ( (exitQualification & 0xfff) == 0x080
2976 && !(errCode & X86_TRAP_PF_P) /* not present */
2977 && fSetupTPRCaching
2978 && (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC))
2979 {
2980 RTGCPHYS GCPhysApicBase, GCPhys;
2981 PDMApicGetBase(pVM, &GCPhysApicBase); /* @todo cache this */
2982 GCPhysApicBase &= PAGE_BASE_GC_MASK;
2983
2984 rc = PGMGstGetPage(pVCpu, (RTGCPTR)exitQualification, NULL, &GCPhys);
2985 if ( rc == VINF_SUCCESS
2986 && GCPhys == GCPhysApicBase)
2987 {
2988 Log(("Enable VT-x virtual APIC access filtering\n"));
2989 rc2 = IOMMMIOMapMMIOHCPage(pVM, GCPhysApicBase, pVM->hwaccm.s.vmx.pAPICPhys, X86_PTE_RW | X86_PTE_P);
2990 AssertRC(rc2);
2991 }
2992 }
2993
2994 /* Forward it to our trap handler first, in case our shadow pages are out of sync. */
2995 rc = PGMTrap0eHandler(pVCpu, errCode, CPUMCTX2CORE(pCtx), (RTGCPTR)exitQualification);
2996 Log2(("PGMTrap0eHandler %RGv returned %Rrc\n", (RTGCPTR)pCtx->rip, VBOXSTRICTRC_VAL(rc)));
2997
2998 if (rc == VINF_SUCCESS)
2999 { /* We've successfully synced our shadow pages, so let's just continue execution. */
3000 Log2(("Shadow page fault at %RGv cr2=%RGv error code %x\n", (RTGCPTR)pCtx->rip, exitQualification ,errCode));
3001 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitShadowPF);
3002
3003 TRPMResetTrap(pVCpu);
3004 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3005 goto ResumeExecution;
3006 }
3007 else
3008 if (rc == VINF_EM_RAW_GUEST_TRAP)
3009 { /* A genuine pagefault.
3010 * Forward the trap to the guest by injecting the exception and resuming execution.
3011 */
3012 Log2(("Forward page fault to the guest\n"));
3013
3014 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestPF);
3015 /* The error code might have been changed. */
3016 errCode = TRPMGetErrorCode(pVCpu);
3017
3018 TRPMResetTrap(pVCpu);
3019
3020 /* Now we must update CR2. */
3021 pCtx->cr2 = exitQualification;
3022 rc2 = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), cbInstr, errCode);
3023 AssertRC(rc2);
3024
3025 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3026 goto ResumeExecution;
3027 }
3028#ifdef VBOX_STRICT
3029 if (rc != VINF_EM_RAW_EMULATE_INSTR && rc != VINF_EM_RAW_EMULATE_IO_BLOCK)
3030 Log2(("PGMTrap0eHandler failed with %d\n", VBOXSTRICTRC_VAL(rc)));
3031#endif
3032 /* Need to go back to the recompiler to emulate the instruction. */
3033 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitShadowPFEM);
3034 TRPMResetTrap(pVCpu);
3035 break;
3036 }
3037
3038 case X86_XCPT_MF: /* Floating point exception. */
3039 {
3040 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestMF);
3041 if (!(pCtx->cr0 & X86_CR0_NE))
3042 {
3043 /* old style FPU error reporting needs some extra work. */
3044 /** @todo don't fall back to the recompiler, but do it manually. */
3045 rc = VINF_EM_RAW_EMULATE_INSTR;
3046 break;
3047 }
3048 Log(("Trap %x at %04X:%RGv\n", vector, pCtx->cs, (RTGCPTR)pCtx->rip));
3049 rc2 = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), cbInstr, errCode);
3050 AssertRC(rc2);
3051
3052 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3053 goto ResumeExecution;
3054 }
3055
3056 case X86_XCPT_DB: /* Debug exception. */
3057 {
3058 uint64_t uDR6;
3059
3060 /* DR6, DR7.GD and IA32_DEBUGCTL.LBR are not updated yet.
3061 *
3062 * Exit qualification bits:
3063 * 3:0 B0-B3 which breakpoint condition was met
3064 * 12:4 Reserved (0)
3065 * 13 BD - debug register access detected
3066 * 14 BS - single step execution or branch taken
3067 * 63:15 Reserved (0)
3068 */
3069 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestDB);
3070
3071 /* Note that we don't support guest and host-initiated debugging at the same time. */
3072
3073 uDR6 = X86_DR6_INIT_VAL;
3074 uDR6 |= (exitQualification & (X86_DR6_B0|X86_DR6_B1|X86_DR6_B2|X86_DR6_B3|X86_DR6_BD|X86_DR6_BS));
3075 rc = DBGFRZTrap01Handler(pVM, pVCpu, CPUMCTX2CORE(pCtx), uDR6);
3076 if (rc == VINF_EM_RAW_GUEST_TRAP)
3077 {
3078 /* Update DR6 here. */
3079 pCtx->dr[6] = uDR6;
3080
3081 /* Resync DR6 if the debug state is active. */
3082 if (CPUMIsGuestDebugStateActive(pVCpu))
3083 ASMSetDR6(pCtx->dr[6]);
3084
3085 /* X86_DR7_GD will be cleared if drx accesses should be trapped inside the guest. */
3086 pCtx->dr[7] &= ~X86_DR7_GD;
3087
3088 /* Paranoia. */
3089 pCtx->dr[7] &= 0xffffffff; /* upper 32 bits reserved */
3090 pCtx->dr[7] &= ~(RT_BIT(11) | RT_BIT(12) | RT_BIT(14) | RT_BIT(15)); /* must be zero */
3091 pCtx->dr[7] |= 0x400; /* must be one */
3092
3093 /* Resync DR7 */
3094 rc2 = VMXWriteVMCS64(VMX_VMCS64_GUEST_DR7, pCtx->dr[7]);
3095 AssertRC(rc2);
3096
3097 Log(("Trap %x (debug) at %RGv exit qualification %RX64 dr6=%x dr7=%x\n", vector, (RTGCPTR)pCtx->rip, exitQualification, (uint32_t)pCtx->dr[6], (uint32_t)pCtx->dr[7]));
3098 rc2 = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), cbInstr, errCode);
3099 AssertRC(rc2);
3100
3101 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3102 goto ResumeExecution;
3103 }
3104 /* Return to ring 3 to deal with the debug exit code. */
3105 Log(("Debugger hardware BP at %04x:%RGv (rc=%Rrc)\n", pCtx->cs, pCtx->rip, VBOXSTRICTRC_VAL(rc)));
3106 break;
3107 }
3108
3109 case X86_XCPT_BP: /* Breakpoint. */
3110 {
3111 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestBP);
3112 rc = DBGFRZTrap03Handler(pVM, pVCpu, CPUMCTX2CORE(pCtx));
3113 if (rc == VINF_EM_RAW_GUEST_TRAP)
3114 {
3115 Log(("Guest #BP at %04x:%RGv\n", pCtx->cs, pCtx->rip));
3116 rc2 = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), cbInstr, errCode);
3117 AssertRC(rc2);
3118 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3119 goto ResumeExecution;
3120 }
3121 if (rc == VINF_SUCCESS)
3122 {
3123 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3124 goto ResumeExecution;
3125 }
3126 Log(("Debugger BP at %04x:%RGv (rc=%Rrc)\n", pCtx->cs, pCtx->rip, VBOXSTRICTRC_VAL(rc)));
3127 break;
3128 }
3129
3130 case X86_XCPT_GP: /* General protection failure exception.*/
3131 {
3132 uint32_t cbOp;
3133 PDISCPUSTATE pDis = &pVCpu->hwaccm.s.DisState;
3134
3135 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestGP);
3136#ifdef VBOX_STRICT
3137 if ( !CPUMIsGuestInRealModeEx(pCtx)
3138 || !pVM->hwaccm.s.vmx.pRealModeTSS)
3139 {
3140 Log(("Trap %x at %04X:%RGv errorCode=%RGv\n", vector, pCtx->cs, (RTGCPTR)pCtx->rip, errCode));
3141 rc2 = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), cbInstr, errCode);
3142 AssertRC(rc2);
3143 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3144 goto ResumeExecution;
3145 }
3146#endif
3147 Assert(CPUMIsGuestInRealModeEx(pCtx));
3148
3149 LogFlow(("Real mode X86_XCPT_GP instruction emulation at %x:%RGv\n", pCtx->cs, (RTGCPTR)pCtx->rip));
3150
3151 rc2 = EMInterpretDisasOne(pVM, pVCpu, CPUMCTX2CORE(pCtx), pDis, &cbOp);
3152 if (RT_SUCCESS(rc2))
3153 {
3154 bool fUpdateRIP = true;
3155
3156 rc = VINF_SUCCESS;
3157 Assert(cbOp == pDis->opsize);
3158 switch (pDis->pCurInstr->opcode)
3159 {
3160 case OP_CLI:
3161 pCtx->eflags.Bits.u1IF = 0;
3162 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitCli);
3163 break;
3164
3165 case OP_STI:
3166 pCtx->eflags.Bits.u1IF = 1;
3167 EMSetInhibitInterruptsPC(pVCpu, pCtx->rip + pDis->opsize);
3168 Assert(VMCPU_FF_ISSET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
3169 rc2 = VMXWriteVMCS(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
3170 AssertRC(rc2);
3171 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitSti);
3172 break;
3173
3174 case OP_HLT:
3175 fUpdateRIP = false;
3176 rc = VINF_EM_HALT;
3177 pCtx->rip += pDis->opsize;
3178 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitHlt);
3179 break;
3180
3181 case OP_POPF:
3182 {
3183 RTGCPTR GCPtrStack;
3184 uint32_t cbParm;
3185 uint32_t uMask;
3186 X86EFLAGS eflags;
3187
3188 if (pDis->prefix & PREFIX_OPSIZE)
3189 {
3190 cbParm = 4;
3191 uMask = 0xffffffff;
3192 }
3193 else
3194 {
3195 cbParm = 2;
3196 uMask = 0xffff;
3197 }
3198
3199 rc2 = SELMToFlatEx(pVCpu, DIS_SELREG_SS, CPUMCTX2CORE(pCtx), pCtx->esp & uMask, 0, &GCPtrStack);
3200 if (RT_FAILURE(rc2))
3201 {
3202 rc = VERR_EM_INTERPRETER;
3203 break;
3204 }
3205 eflags.u = 0;
3206 rc2 = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &eflags.u, cbParm);
3207 if (RT_FAILURE(rc2))
3208 {
3209 rc = VERR_EM_INTERPRETER;
3210 break;
3211 }
3212 LogFlow(("POPF %x -> %RGv mask=%x\n", eflags.u, pCtx->rsp, uMask));
3213 pCtx->eflags.u = (pCtx->eflags.u & ~(X86_EFL_POPF_BITS & uMask)) | (eflags.u & X86_EFL_POPF_BITS & uMask);
3214 /* RF cleared when popped in real mode; see pushf description in AMD manual. */
3215 pCtx->eflags.Bits.u1RF = 0;
3216 pCtx->esp += cbParm;
3217 pCtx->esp &= uMask;
3218
3219 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitPopf);
3220 break;
3221 }
3222
3223 case OP_PUSHF:
3224 {
3225 RTGCPTR GCPtrStack;
3226 uint32_t cbParm;
3227 uint32_t uMask;
3228 X86EFLAGS eflags;
3229
3230 if (pDis->prefix & PREFIX_OPSIZE)
3231 {
3232 cbParm = 4;
3233 uMask = 0xffffffff;
3234 }
3235 else
3236 {
3237 cbParm = 2;
3238 uMask = 0xffff;
3239 }
3240
3241 rc2 = SELMToFlatEx(pVCpu, DIS_SELREG_SS, CPUMCTX2CORE(pCtx), (pCtx->esp - cbParm) & uMask, 0, &GCPtrStack);
3242 if (RT_FAILURE(rc2))
3243 {
3244 rc = VERR_EM_INTERPRETER;
3245 break;
3246 }
3247 eflags = pCtx->eflags;
3248 /* RF & VM cleared when pushed in real mode; see pushf description in AMD manual. */
3249 eflags.Bits.u1RF = 0;
3250 eflags.Bits.u1VM = 0;
3251
3252 rc2 = PGMPhysWrite(pVM, (RTGCPHYS)GCPtrStack, &eflags.u, cbParm);
3253 if (RT_FAILURE(rc2))
3254 {
3255 rc = VERR_EM_INTERPRETER;
3256 break;
3257 }
3258 LogFlow(("PUSHF %x -> %RGv\n", eflags.u, GCPtrStack));
3259 pCtx->esp -= cbParm;
3260 pCtx->esp &= uMask;
3261 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitPushf);
3262 break;
3263 }
3264
3265 case OP_IRET:
3266 {
3267 RTGCPTR GCPtrStack;
3268 uint32_t uMask = 0xffff;
3269 uint16_t aIretFrame[3];
3270
3271 if (pDis->prefix & (PREFIX_OPSIZE | PREFIX_ADDRSIZE))
3272 {
3273 rc = VERR_EM_INTERPRETER;
3274 break;
3275 }
3276
3277 rc2 = SELMToFlatEx(pVCpu, DIS_SELREG_SS, CPUMCTX2CORE(pCtx), pCtx->esp & uMask, 0, &GCPtrStack);
3278 if (RT_FAILURE(rc2))
3279 {
3280 rc = VERR_EM_INTERPRETER;
3281 break;
3282 }
3283 rc2 = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &aIretFrame[0], sizeof(aIretFrame));
3284 if (RT_FAILURE(rc2))
3285 {
3286 rc = VERR_EM_INTERPRETER;
3287 break;
3288 }
3289 pCtx->ip = aIretFrame[0];
3290 pCtx->cs = aIretFrame[1];
3291 pCtx->csHid.u64Base = pCtx->cs << 4;
3292 pCtx->eflags.u = (pCtx->eflags.u & ~(X86_EFL_POPF_BITS & uMask)) | (aIretFrame[2] & X86_EFL_POPF_BITS & uMask);
3293 pCtx->sp += sizeof(aIretFrame);
3294
3295 LogFlow(("iret to %04x:%x\n", pCtx->cs, pCtx->ip));
3296 fUpdateRIP = false;
3297 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitIret);
3298 break;
3299 }
3300
3301 case OP_INT:
3302 {
3303 uint32_t intInfo2;
3304
3305 LogFlow(("Realmode: INT %x\n", pDis->param1.parval & 0xff));
3306 intInfo2 = pDis->param1.parval & 0xff;
3307 intInfo2 |= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
3308 intInfo2 |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
3309
3310 rc = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, intInfo2, cbOp, 0);
3311 AssertRC(VBOXSTRICTRC_VAL(rc));
3312 fUpdateRIP = false;
3313 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitInt);
3314 break;
3315 }
3316
3317 case OP_INTO:
3318 {
3319 if (pCtx->eflags.Bits.u1OF)
3320 {
3321 uint32_t intInfo2;
3322
3323 LogFlow(("Realmode: INTO\n"));
3324 intInfo2 = X86_XCPT_OF;
3325 intInfo2 |= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
3326 intInfo2 |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
3327
3328 rc = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, intInfo2, cbOp, 0);
3329 AssertRC(VBOXSTRICTRC_VAL(rc));
3330 fUpdateRIP = false;
3331 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitInt);
3332 }
3333 break;
3334 }
3335
3336 case OP_INT3:
3337 {
3338 uint32_t intInfo2;
3339
3340 LogFlow(("Realmode: INT 3\n"));
3341 intInfo2 = 3;
3342 intInfo2 |= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
3343 intInfo2 |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
3344
3345 rc = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, intInfo2, cbOp, 0);
3346 AssertRC(VBOXSTRICTRC_VAL(rc));
3347 fUpdateRIP = false;
3348 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitInt);
3349 break;
3350 }
3351
3352 default:
3353 rc = EMInterpretInstructionDisasState(pVCpu, pDis, CPUMCTX2CORE(pCtx), 0, EMCODETYPE_SUPERVISOR);
3354 fUpdateRIP = false;
3355 break;
3356 }
3357
3358 if (rc == VINF_SUCCESS)
3359 {
3360 if (fUpdateRIP)
3361 pCtx->rip += cbOp; /* Move on to the next instruction. */
3362
3363 /* lidt, lgdt can end up here. In the future crx changes as well. Just reload the whole context to be done with it. */
3364 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_ALL;
3365
3366 /* Only resume if successful. */
3367 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3368 goto ResumeExecution;
3369 }
3370 }
3371 else
3372 rc = VERR_EM_INTERPRETER;
3373
3374 AssertMsg(rc == VERR_EM_INTERPRETER || rc == VINF_PGM_CHANGE_MODE || rc == VINF_EM_HALT, ("Unexpected rc=%Rrc\n", VBOXSTRICTRC_VAL(rc)));
3375 break;
3376 }
3377
3378#ifdef VBOX_STRICT
3379 case X86_XCPT_XF: /* SIMD exception. */
3380 case X86_XCPT_DE: /* Divide error. */
3381 case X86_XCPT_UD: /* Unknown opcode exception. */
3382 case X86_XCPT_SS: /* Stack segment exception. */
3383 case X86_XCPT_NP: /* Segment not present exception. */
3384 {
3385 switch(vector)
3386 {
3387 case X86_XCPT_DE:
3388 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestDE);
3389 break;
3390 case X86_XCPT_UD:
3391 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestUD);
3392 break;
3393 case X86_XCPT_SS:
3394 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestSS);
3395 break;
3396 case X86_XCPT_NP:
3397 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestNP);
3398 break;
3399 case X86_XCPT_XF:
3400 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestXF);
3401 break;
3402 }
3403
3404 Log(("Trap %x at %04X:%RGv\n", vector, pCtx->cs, (RTGCPTR)pCtx->rip));
3405 rc2 = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), cbInstr, errCode);
3406 AssertRC(rc2);
3407
3408 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3409 goto ResumeExecution;
3410 }
3411#endif
3412 default:
3413 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestXcpUnk);
3414 if ( CPUMIsGuestInRealModeEx(pCtx)
3415 && pVM->hwaccm.s.vmx.pRealModeTSS)
3416 {
3417 Log(("Real Mode Trap %x at %04x:%04X error code %x\n", vector, pCtx->cs, pCtx->eip, errCode));
3418 rc = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), cbInstr, errCode);
3419 AssertRC(VBOXSTRICTRC_VAL(rc)); /* Strict RC check below. */
3420
3421 /* Go back to ring 3 in case of a triple fault. */
3422 if ( vector == X86_XCPT_DF
3423 && rc == VINF_EM_RESET)
3424 break;
3425
3426 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3427 goto ResumeExecution;
3428 }
3429 AssertMsgFailed(("Unexpected vm-exit caused by exception %x\n", vector));
3430 rc = VERR_VMX_UNEXPECTED_EXCEPTION;
3431 break;
3432 } /* switch (vector) */
3433
3434 break;
3435
3436 default:
3437 rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_CODE;
3438 AssertMsgFailed(("Unexpected interruption code %x\n", intInfo));
3439 break;
3440 }
3441
3442 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3443 break;
3444 }
3445
3446 case VMX_EXIT_EPT_VIOLATION: /* 48 EPT violation. An attempt to access memory with a guest-physical address was disallowed by the configuration of the EPT paging structures. */
3447 {
3448 RTGCPHYS GCPhys;
3449
3450 Assert(pVM->hwaccm.s.fNestedPaging);
3451
3452 rc2 = VMXReadVMCS64(VMX_VMCS_EXIT_PHYS_ADDR_FULL, &GCPhys);
3453 AssertRC(rc2);
3454 Assert(((exitQualification >> 7) & 3) != 2);
3455
3456 /* Determine the kind of violation. */
3457 errCode = 0;
3458 if (exitQualification & VMX_EXIT_QUALIFICATION_EPT_INSTR_FETCH)
3459 errCode |= X86_TRAP_PF_ID;
3460
3461 if (exitQualification & VMX_EXIT_QUALIFICATION_EPT_DATA_WRITE)
3462 errCode |= X86_TRAP_PF_RW;
3463
3464 /* If the page is present, then it's a page level protection fault. */
3465 if (exitQualification & VMX_EXIT_QUALIFICATION_EPT_ENTRY_PRESENT)
3466 {
3467 errCode |= X86_TRAP_PF_P;
3468 }
3469 else
3470 {
3471 /* Shortcut for APIC TPR reads and writes. */
3472 if ( (GCPhys & 0xfff) == 0x080
3473 && GCPhys > 0x1000000 /* to skip VGA frame buffer accesses */
3474 && fSetupTPRCaching
3475 && (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC))
3476 {
3477 RTGCPHYS GCPhysApicBase;
3478 PDMApicGetBase(pVM, &GCPhysApicBase); /* @todo cache this */
3479 GCPhysApicBase &= PAGE_BASE_GC_MASK;
3480 if (GCPhys == GCPhysApicBase + 0x80)
3481 {
3482 Log(("Enable VT-x virtual APIC access filtering\n"));
3483 rc2 = IOMMMIOMapMMIOHCPage(pVM, GCPhysApicBase, pVM->hwaccm.s.vmx.pAPICPhys, X86_PTE_RW | X86_PTE_P);
3484 AssertRC(rc2);
3485 }
3486 }
3487 }
3488 Log(("EPT Page fault %x at %RGp error code %x\n", (uint32_t)exitQualification, GCPhys, errCode));
3489
3490 /* GCPhys contains the guest physical address of the page fault. */
3491 TRPMAssertTrap(pVCpu, X86_XCPT_PF, TRPM_TRAP);
3492 TRPMSetErrorCode(pVCpu, errCode);
3493 TRPMSetFaultAddress(pVCpu, GCPhys);
3494
3495 /* Handle the pagefault trap for the nested shadow table. */
3496 rc = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, errCode, CPUMCTX2CORE(pCtx), GCPhys);
3497 Log2(("PGMR0Trap0eHandlerNestedPaging %RGv returned %Rrc\n", (RTGCPTR)pCtx->rip, VBOXSTRICTRC_VAL(rc)));
3498 if (rc == VINF_SUCCESS)
3499 { /* We've successfully synced our shadow pages, so let's just continue execution. */
3500 Log2(("Shadow page fault at %RGv cr2=%RGp error code %x\n", (RTGCPTR)pCtx->rip, exitQualification , errCode));
3501 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitReasonNPF);
3502
3503 TRPMResetTrap(pVCpu);
3504 goto ResumeExecution;
3505 }
3506
3507#ifdef VBOX_STRICT
3508 if (rc != VINF_EM_RAW_EMULATE_INSTR)
3509 LogFlow(("PGMTrap0eHandlerNestedPaging failed with %d\n", VBOXSTRICTRC_VAL(rc)));
3510#endif
3511 /* Need to go back to the recompiler to emulate the instruction. */
3512 TRPMResetTrap(pVCpu);
3513 break;
3514 }
3515
3516 case VMX_EXIT_EPT_MISCONFIG:
3517 {
3518 RTGCPHYS GCPhys;
3519
3520 Assert(pVM->hwaccm.s.fNestedPaging);
3521
3522 rc2 = VMXReadVMCS64(VMX_VMCS_EXIT_PHYS_ADDR_FULL, &GCPhys);
3523 AssertRC(rc2);
3524 Log(("VMX_EXIT_EPT_MISCONFIG for %RGp\n", GCPhys));
3525
3526 /* Shortcut for APIC TPR reads and writes. */
3527 if ( (GCPhys & 0xfff) == 0x080
3528 && GCPhys > 0x1000000 /* to skip VGA frame buffer accesses */
3529 && fSetupTPRCaching
3530 && (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC))
3531 {
3532 RTGCPHYS GCPhysApicBase;
3533 PDMApicGetBase(pVM, &GCPhysApicBase); /* @todo cache this */
3534 GCPhysApicBase &= PAGE_BASE_GC_MASK;
3535 if (GCPhys == GCPhysApicBase + 0x80)
3536 {
3537 Log(("Enable VT-x virtual APIC access filtering\n"));
3538 rc2 = IOMMMIOMapMMIOHCPage(pVM, GCPhysApicBase, pVM->hwaccm.s.vmx.pAPICPhys, X86_PTE_RW | X86_PTE_P);
3539 AssertRC(rc2);
3540 }
3541 }
3542
3543 rc = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pCtx), GCPhys, UINT32_MAX);
3544 if (rc == VINF_SUCCESS)
3545 {
3546 Log2(("PGMR0Trap0eHandlerNPMisconfig(,,,%RGp) at %RGv -> resume\n", GCPhys, (RTGCPTR)pCtx->rip));
3547 goto ResumeExecution;
3548 }
3549
3550 Log2(("PGMR0Trap0eHandlerNPMisconfig(,,,%RGp) at %RGv -> %Rrc\n", GCPhys, (RTGCPTR)pCtx->rip, VBOXSTRICTRC_VAL(rc)));
3551 break;
3552 }
3553
3554 case VMX_EXIT_IRQ_WINDOW: /* 7 Interrupt window. */
3555 /* Clear VM-exit on IF=1 change. */
3556 LogFlow(("VMX_EXIT_IRQ_WINDOW %RGv pending=%d IF=%d\n", (RTGCPTR)pCtx->rip, VMCPU_FF_ISPENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC|VMCPU_FF_INTERRUPT_PIC)), pCtx->eflags.Bits.u1IF));
3557 pVCpu->hwaccm.s.vmx.proc_ctls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_IRQ_WINDOW_EXIT;
3558 rc2 = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
3559 AssertRC(rc2);
3560 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitIrqWindow);
3561 goto ResumeExecution; /* we check for pending guest interrupts there */
3562
3563 case VMX_EXIT_WBINVD: /* 54 Guest software attempted to execute WBINVD. (conditional) */
3564 case VMX_EXIT_INVD: /* 13 Guest software attempted to execute INVD. (unconditional) */
3565 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitInvd);
3566 /* Skip instruction and continue directly. */
3567 pCtx->rip += cbInstr;
3568 /* Continue execution.*/
3569 goto ResumeExecution;
3570
3571 case VMX_EXIT_CPUID: /* 10 Guest software attempted to execute CPUID. */
3572 {
3573 Log2(("VMX: Cpuid %x\n", pCtx->eax));
3574 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitCpuid);
3575 rc = EMInterpretCpuId(pVM, pVCpu, CPUMCTX2CORE(pCtx));
3576 if (rc == VINF_SUCCESS)
3577 {
3578 /* Update EIP and continue execution. */
3579 Assert(cbInstr == 2);
3580 pCtx->rip += cbInstr;
3581 goto ResumeExecution;
3582 }
3583 AssertMsgFailed(("EMU: cpuid failed with %Rrc\n", VBOXSTRICTRC_VAL(rc)));
3584 rc = VINF_EM_RAW_EMULATE_INSTR;
3585 break;
3586 }
3587
3588 case VMX_EXIT_RDPMC: /* 15 Guest software attempted to execute RDPMC. */
3589 {
3590 Log2(("VMX: Rdpmc %x\n", pCtx->ecx));
3591 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitRdpmc);
3592 rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pCtx));
3593 if (rc == VINF_SUCCESS)
3594 {
3595 /* Update EIP and continue execution. */
3596 Assert(cbInstr == 2);
3597 pCtx->rip += cbInstr;
3598 goto ResumeExecution;
3599 }
3600 rc = VINF_EM_RAW_EMULATE_INSTR;
3601 break;
3602 }
3603
3604 case VMX_EXIT_RDTSC: /* 16 Guest software attempted to execute RDTSC. */
3605 {
3606 Log2(("VMX: Rdtsc\n"));
3607 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitRdtsc);
3608 rc = EMInterpretRdtsc(pVM, pVCpu, CPUMCTX2CORE(pCtx));
3609 if (rc == VINF_SUCCESS)
3610 {
3611 /* Update EIP and continue execution. */
3612 Assert(cbInstr == 2);
3613 pCtx->rip += cbInstr;
3614 goto ResumeExecution;
3615 }
3616 rc = VINF_EM_RAW_EMULATE_INSTR;
3617 break;
3618 }
3619
3620 case VMX_EXIT_INVPG: /* 14 Guest software attempted to execute INVPG. */
3621 {
3622 Log2(("VMX: invlpg\n"));
3623 Assert(!pVM->hwaccm.s.fNestedPaging);
3624
3625 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitInvpg);
3626 rc = EMInterpretInvlpg(pVM, pVCpu, CPUMCTX2CORE(pCtx), exitQualification);
3627 if (rc == VINF_SUCCESS)
3628 {
3629 /* Update EIP and continue execution. */
3630 pCtx->rip += cbInstr;
3631 goto ResumeExecution;
3632 }
3633 AssertMsg(rc == VERR_EM_INTERPRETER, ("EMU: invlpg %RGv failed with %Rrc\n", exitQualification, VBOXSTRICTRC_VAL(rc)));
3634 break;
3635 }
3636
3637 case VMX_EXIT_MONITOR: /* 39 Guest software attempted to execute MONITOR. */
3638 {
3639 Log2(("VMX: monitor\n"));
3640
3641 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitMonitor);
3642 rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pCtx));
3643 if (rc == VINF_SUCCESS)
3644 {
3645 /* Update EIP and continue execution. */
3646 pCtx->rip += cbInstr;
3647 goto ResumeExecution;
3648 }
3649 AssertMsg(rc == VERR_EM_INTERPRETER, ("EMU: monitor failed with %Rrc\n", VBOXSTRICTRC_VAL(rc)));
3650 break;
3651 }
3652
3653 case VMX_EXIT_WRMSR: /* 32 WRMSR. Guest software attempted to execute WRMSR. */
3654 /* When an interrupt is pending, we'll let MSR_K8_LSTAR writes fault in our TPR patch code. */
3655 if ( pVM->hwaccm.s.fTPRPatchingActive
3656 && pCtx->ecx == MSR_K8_LSTAR)
3657 {
3658 Assert(!CPUMIsGuestInLongModeEx(pCtx));
3659 if ((pCtx->eax & 0xff) != u8LastTPR)
3660 {
3661 Log(("VMX: Faulting MSR_K8_LSTAR write with new TPR value %x\n", pCtx->eax & 0xff));
3662
3663 /* Our patch code uses LSTAR for TPR caching. */
3664 rc2 = PDMApicSetTPR(pVCpu, pCtx->eax & 0xff);
3665 AssertRC(rc2);
3666 }
3667
3668 /* Skip the instruction and continue. */
3669 pCtx->rip += cbInstr; /* wrmsr = [0F 30] */
3670
3671 /* Only resume if successful. */
3672 goto ResumeExecution;
3673 }
3674 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_GUEST_MSR;
3675 /* no break */
3676 case VMX_EXIT_RDMSR: /* 31 RDMSR. Guest software attempted to execute RDMSR. */
3677 {
3678 STAM_COUNTER_INC((exitReason == VMX_EXIT_RDMSR) ? &pVCpu->hwaccm.s.StatExitRdmsr : &pVCpu->hwaccm.s.StatExitWrmsr);
3679
3680 /* Note: the intel manual claims there's a REX version of RDMSR that's slightly different, so we play safe by completely disassembling the instruction. */
3681 Log2(("VMX: %s\n", (exitReason == VMX_EXIT_RDMSR) ? "rdmsr" : "wrmsr"));
3682 rc = EMInterpretInstruction(pVCpu, CPUMCTX2CORE(pCtx), 0);
3683 if (rc == VINF_SUCCESS)
3684 {
3685 /* EIP has been updated already. */
3686
3687 /* Only resume if successful. */
3688 goto ResumeExecution;
3689 }
3690 AssertMsg(rc == VERR_EM_INTERPRETER, ("EMU: %s failed with %Rrc\n", (exitReason == VMX_EXIT_RDMSR) ? "rdmsr" : "wrmsr", VBOXSTRICTRC_VAL(rc)));
3691 break;
3692 }
3693
3694 case VMX_EXIT_CRX_MOVE: /* 28 Control-register accesses. */
3695 {
3696 STAM_PROFILE_ADV_START(&pVCpu->hwaccm.s.StatExit2Sub2, y2);
3697
3698 switch (VMX_EXIT_QUALIFICATION_CRX_ACCESS(exitQualification))
3699 {
3700 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_WRITE:
3701 Log2(("VMX: %RGv mov cr%d, x\n", (RTGCPTR)pCtx->rip, VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification)));
3702 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitCRxWrite[VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification)]);
3703 rc = EMInterpretCRxWrite(pVM, pVCpu, CPUMCTX2CORE(pCtx),
3704 VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification),
3705 VMX_EXIT_QUALIFICATION_CRX_GENREG(exitQualification));
3706
3707 switch (VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification))
3708 {
3709 case 0:
3710 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_GUEST_CR0 | HWACCM_CHANGED_GUEST_CR3;
3711 break;
3712 case 2:
3713 break;
3714 case 3:
3715 Assert(!pVM->hwaccm.s.fNestedPaging || !CPUMIsGuestInPagedProtectedModeEx(pCtx));
3716 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_GUEST_CR3;
3717 break;
3718 case 4:
3719 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_GUEST_CR4;
3720 break;
3721 case 8:
3722 /* CR8 contains the APIC TPR */
3723 Assert(!(pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW));
3724 break;
3725
3726 default:
3727 AssertFailed();
3728 break;
3729 }
3730 break;
3731
3732 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ:
3733 Log2(("VMX: mov x, crx\n"));
3734 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitCRxRead[VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification)]);
3735
3736 Assert(!pVM->hwaccm.s.fNestedPaging || !CPUMIsGuestInPagedProtectedModeEx(pCtx) || VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification) != USE_REG_CR3);
3737
3738 /* CR8 reads only cause an exit when the TPR shadow feature isn't present. */
3739 Assert(VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification) != 8 || !(pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW));
3740
3741 rc = EMInterpretCRxRead(pVM, pVCpu, CPUMCTX2CORE(pCtx),
3742 VMX_EXIT_QUALIFICATION_CRX_GENREG(exitQualification),
3743 VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification));
3744 break;
3745
3746 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_CLTS:
3747 Log2(("VMX: clts\n"));
3748 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitCLTS);
3749 rc = EMInterpretCLTS(pVM, pVCpu);
3750 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_GUEST_CR0;
3751 break;
3752
3753 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_LMSW:
3754 Log2(("VMX: lmsw %x\n", VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(exitQualification)));
3755 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitLMSW);
3756 rc = EMInterpretLMSW(pVM, pVCpu, CPUMCTX2CORE(pCtx), VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(exitQualification));
3757 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_GUEST_CR0;
3758 break;
3759 }
3760
3761 /* Update EIP if no error occurred. */
3762 if (RT_SUCCESS(rc))
3763 pCtx->rip += cbInstr;
3764
3765 if (rc == VINF_SUCCESS)
3766 {
3767 /* Only resume if successful. */
3768 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub2, y2);
3769 goto ResumeExecution;
3770 }
3771 Assert(rc == VERR_EM_INTERPRETER || rc == VINF_PGM_CHANGE_MODE || rc == VINF_PGM_SYNC_CR3);
3772 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub2, y2);
3773 break;
3774 }
3775
3776 case VMX_EXIT_DRX_MOVE: /* 29 Debug-register accesses. */
3777 {
3778 if ( !DBGFIsStepping(pVCpu)
3779 && !CPUMIsHyperDebugStateActive(pVCpu))
3780 {
3781 /* Disable drx move intercepts. */
3782 pVCpu->hwaccm.s.vmx.proc_ctls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT;
3783 rc2 = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
3784 AssertRC(rc2);
3785
3786 /* Save the host and load the guest debug state. */
3787 rc2 = CPUMR0LoadGuestDebugState(pVM, pVCpu, pCtx, true /* include DR6 */);
3788 AssertRC(rc2);
3789
3790#ifdef LOG_ENABLED
3791 if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(exitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
3792 Log(("VMX_EXIT_DRX_MOVE: write DR%d genreg %d\n", VMX_EXIT_QUALIFICATION_DRX_REGISTER(exitQualification), VMX_EXIT_QUALIFICATION_DRX_GENREG(exitQualification)));
3793 else
3794 Log(("VMX_EXIT_DRX_MOVE: read DR%d\n", VMX_EXIT_QUALIFICATION_DRX_REGISTER(exitQualification)));
3795#endif
3796
3797#ifdef VBOX_WITH_STATISTICS
3798 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatDRxContextSwitch);
3799 if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(exitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
3800 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitDRxWrite);
3801 else
3802 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitDRxRead);
3803#endif
3804
3805 goto ResumeExecution;
3806 }
3807
3808 /** @todo clear VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT after the first time and restore drx registers afterwards */
3809 if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(exitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
3810 {
3811 Log2(("VMX: mov drx%d, genreg%d\n", VMX_EXIT_QUALIFICATION_DRX_REGISTER(exitQualification), VMX_EXIT_QUALIFICATION_DRX_GENREG(exitQualification)));
3812 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitDRxWrite);
3813 rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pCtx),
3814 VMX_EXIT_QUALIFICATION_DRX_REGISTER(exitQualification),
3815 VMX_EXIT_QUALIFICATION_DRX_GENREG(exitQualification));
3816 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_GUEST_DEBUG;
3817 Log2(("DR7=%08x\n", pCtx->dr[7]));
3818 }
3819 else
3820 {
3821 Log2(("VMX: mov x, drx\n"));
3822 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitDRxRead);
3823 rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pCtx),
3824 VMX_EXIT_QUALIFICATION_DRX_GENREG(exitQualification),
3825 VMX_EXIT_QUALIFICATION_DRX_REGISTER(exitQualification));
3826 }
3827 /* Update EIP if no error occurred. */
3828 if (RT_SUCCESS(rc))
3829 pCtx->rip += cbInstr;
3830
3831 if (rc == VINF_SUCCESS)
3832 {
3833 /* Only resume if successful. */
3834 goto ResumeExecution;
3835 }
3836 Assert(rc == VERR_EM_INTERPRETER);
3837 break;
3838 }
3839
3840 /* Note: We'll get a #GP if the IO instruction isn't allowed (IOPL or TSS bitmap); no need to double check. */
3841 case VMX_EXIT_PORT_IO: /* 30 I/O instruction. */
3842 {
3843 STAM_PROFILE_ADV_START(&pVCpu->hwaccm.s.StatExit2Sub1, y1);
3844 uint32_t uIOWidth = VMX_EXIT_QUALIFICATION_IO_WIDTH(exitQualification);
3845 uint32_t uPort;
3846 bool fIOWrite = (VMX_EXIT_QUALIFICATION_IO_DIRECTION(exitQualification) == VMX_EXIT_QUALIFICATION_IO_DIRECTION_OUT);
3847
3848 /** @todo necessary to make the distinction? */
3849 if (VMX_EXIT_QUALIFICATION_IO_ENCODING(exitQualification) == VMX_EXIT_QUALIFICATION_IO_ENCODING_DX)
3850 {
3851 uPort = pCtx->edx & 0xffff;
3852 }
3853 else
3854 uPort = VMX_EXIT_QUALIFICATION_IO_PORT(exitQualification); /* Immediate encoding. */
3855
3856 /* paranoia */
3857 if (RT_UNLIKELY(uIOWidth == 2 || uIOWidth >= 4))
3858 {
3859 rc = fIOWrite ? VINF_IOM_R3_IOPORT_WRITE : VINF_IOM_R3_IOPORT_READ;
3860 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub1, y1);
3861 break;
3862 }
3863
3864 uint32_t cbSize = g_aIOSize[uIOWidth];
3865
3866 if (VMX_EXIT_QUALIFICATION_IO_STRING(exitQualification))
3867 {
3868 /* ins/outs */
3869 PDISCPUSTATE pDis = &pVCpu->hwaccm.s.DisState;
3870
3871 /* Disassemble manually to deal with segment prefixes. */
3872 /** @todo VMX_VMCS_EXIT_GUEST_LINEAR_ADDR contains the flat pointer operand of the instruction. */
3873 /** @todo VMX_VMCS32_RO_EXIT_INSTR_INFO also contains segment prefix info. */
3874 rc2 = EMInterpretDisasOne(pVM, pVCpu, CPUMCTX2CORE(pCtx), pDis, NULL);
3875 if (RT_SUCCESS(rc))
3876 {
3877 if (fIOWrite)
3878 {
3879 Log2(("IOMInterpretOUTSEx %RGv %x size=%d\n", (RTGCPTR)pCtx->rip, uPort, cbSize));
3880 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitIOStringWrite);
3881 rc = IOMInterpretOUTSEx(pVM, CPUMCTX2CORE(pCtx), uPort, pDis->prefix, pDis->addrmode, cbSize);
3882 }
3883 else
3884 {
3885 Log2(("IOMInterpretINSEx %RGv %x size=%d\n", (RTGCPTR)pCtx->rip, uPort, cbSize));
3886 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitIOStringRead);
3887 rc = IOMInterpretINSEx(pVM, CPUMCTX2CORE(pCtx), uPort, pDis->prefix, pDis->addrmode, cbSize);
3888 }
3889 }
3890 else
3891 rc = VINF_EM_RAW_EMULATE_INSTR;
3892 }
3893 else
3894 {
3895 /* normal in/out */
3896 uint32_t uAndVal = g_aIOOpAnd[uIOWidth];
3897
3898 Assert(!VMX_EXIT_QUALIFICATION_IO_REP(exitQualification));
3899
3900 if (fIOWrite)
3901 {
3902 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitIOWrite);
3903 rc = IOMIOPortWrite(pVM, uPort, pCtx->eax & uAndVal, cbSize);
3904 if (rc == VINF_IOM_R3_IOPORT_WRITE)
3905 HWACCMR0SavePendingIOPortWrite(pVCpu, pCtx->rip, pCtx->rip + cbInstr, uPort, uAndVal, cbSize);
3906 }
3907 else
3908 {
3909 uint32_t u32Val = 0;
3910
3911 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitIORead);
3912 rc = IOMIOPortRead(pVM, uPort, &u32Val, cbSize);
3913 if (IOM_SUCCESS(rc))
3914 {
3915 /* Write back to the EAX register. */
3916 pCtx->eax = (pCtx->eax & ~uAndVal) | (u32Val & uAndVal);
3917 }
3918 else
3919 if (rc == VINF_IOM_R3_IOPORT_READ)
3920 HWACCMR0SavePendingIOPortRead(pVCpu, pCtx->rip, pCtx->rip + cbInstr, uPort, uAndVal, cbSize);
3921 }
3922 }
3923 /*
3924 * Handled the I/O return codes.
3925 * (The unhandled cases end up with rc == VINF_EM_RAW_EMULATE_INSTR.)
3926 */
3927 if (IOM_SUCCESS(rc))
3928 {
3929 /* Update EIP and continue execution. */
3930 pCtx->rip += cbInstr;
3931 if (RT_LIKELY(rc == VINF_SUCCESS))
3932 {
3933 /* If any IO breakpoints are armed, then we should check if a debug trap needs to be generated. */
3934 if (pCtx->dr[7] & X86_DR7_ENABLED_MASK)
3935 {
3936 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatDRxIOCheck);
3937 for (unsigned i=0;i<4;i++)
3938 {
3939 unsigned uBPLen = g_aIOSize[X86_DR7_GET_LEN(pCtx->dr[7], i)];
3940
3941 if ( (uPort >= pCtx->dr[i] && uPort < pCtx->dr[i] + uBPLen)
3942 && (pCtx->dr[7] & (X86_DR7_L(i) | X86_DR7_G(i)))
3943 && (pCtx->dr[7] & X86_DR7_RW(i, X86_DR7_RW_IO)) == X86_DR7_RW(i, X86_DR7_RW_IO))
3944 {
3945 uint64_t uDR6;
3946
3947 Assert(CPUMIsGuestDebugStateActive(pVCpu));
3948
3949 uDR6 = ASMGetDR6();
3950
3951 /* Clear all breakpoint status flags and set the one we just hit. */
3952 uDR6 &= ~(X86_DR6_B0|X86_DR6_B1|X86_DR6_B2|X86_DR6_B3);
3953 uDR6 |= (uint64_t)RT_BIT(i);
3954
3955 /* Note: AMD64 Architecture Programmer's Manual 13.1:
3956 * Bits 15:13 of the DR6 register is never cleared by the processor and must be cleared by software after
3957 * the contents have been read.
3958 */
3959 ASMSetDR6(uDR6);
3960
3961 /* X86_DR7_GD will be cleared if drx accesses should be trapped inside the guest. */
3962 pCtx->dr[7] &= ~X86_DR7_GD;
3963
3964 /* Paranoia. */
3965 pCtx->dr[7] &= 0xffffffff; /* upper 32 bits reserved */
3966 pCtx->dr[7] &= ~(RT_BIT(11) | RT_BIT(12) | RT_BIT(14) | RT_BIT(15)); /* must be zero */
3967 pCtx->dr[7] |= 0x400; /* must be one */
3968
3969 /* Resync DR7 */
3970 rc2 = VMXWriteVMCS64(VMX_VMCS64_GUEST_DR7, pCtx->dr[7]);
3971 AssertRC(rc2);
3972
3973 /* Construct inject info. */
3974 intInfo = X86_XCPT_DB;
3975 intInfo |= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
3976 intInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HWEXCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
3977
3978 Log(("Inject IO debug trap at %RGv\n", (RTGCPTR)pCtx->rip));
3979 rc2 = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), 0, 0);
3980 AssertRC(rc2);
3981
3982 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub1, y1);
3983 goto ResumeExecution;
3984 }
3985 }
3986 }
3987 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub1, y1);
3988 goto ResumeExecution;
3989 }
3990 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub1, y1);
3991 break;
3992 }
3993
3994#ifdef VBOX_STRICT
3995 if (rc == VINF_IOM_R3_IOPORT_READ)
3996 Assert(!fIOWrite);
3997 else if (rc == VINF_IOM_R3_IOPORT_WRITE)
3998 Assert(fIOWrite);
3999 else
4000 AssertMsg(RT_FAILURE(rc) || rc == VINF_EM_RAW_EMULATE_INSTR || rc == VINF_EM_RAW_GUEST_TRAP || rc == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", VBOXSTRICTRC_VAL(rc)));
4001#endif
4002 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub1, y1);
4003 break;
4004 }
4005
4006 case VMX_EXIT_TPR: /* 43 TPR below threshold. Guest software executed MOV to CR8. */
4007 LogFlow(("VMX_EXIT_TPR\n"));
4008 /* RIP is already set to the next instruction and the TPR has been synced back. Just resume. */
4009 goto ResumeExecution;
4010
4011 case VMX_EXIT_APIC_ACCESS: /* 44 APIC access. Guest software attempted to access memory at a physical address on the APIC-access page. */
4012 {
4013 LogFlow(("VMX_EXIT_APIC_ACCESS\n"));
4014 unsigned uAccessType = VMX_EXIT_QUALIFICATION_APIC_ACCESS_TYPE(exitQualification);
4015
4016 switch(uAccessType)
4017 {
4018 case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
4019 case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
4020 {
4021 RTGCPHYS GCPhys;
4022 PDMApicGetBase(pVM, &GCPhys);
4023 GCPhys &= PAGE_BASE_GC_MASK;
4024 GCPhys += VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(exitQualification);
4025
4026 LogFlow(("Apic access at %RGp\n", GCPhys));
4027 rc = IOMMMIOPhysHandler(pVM, (uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ) ? 0 : X86_TRAP_PF_RW, CPUMCTX2CORE(pCtx), GCPhys);
4028 if (rc == VINF_SUCCESS)
4029 goto ResumeExecution; /* rip already updated */
4030 break;
4031 }
4032
4033 default:
4034 rc = VINF_EM_RAW_EMULATE_INSTR;
4035 break;
4036 }
4037 break;
4038 }
4039
4040 case VMX_EXIT_PREEMPTION_TIMER: /* 52 VMX-preemption timer expired. The preemption timer counted down to zero. */
4041 if (!TMTimerPollBool(pVM, pVCpu))
4042 goto ResumeExecution;
4043 rc = VINF_EM_RAW_TIMER_PENDING;
4044 break;
4045
4046 default:
4047 /* The rest is handled after syncing the entire CPU state. */
4048 break;
4049 }
4050
4051 /* Note: the guest state isn't entirely synced back at this stage. */
4052
4053 /* Investigate why there was a VM-exit. (part 2) */
4054 switch (exitReason)
4055 {
4056 case VMX_EXIT_EXCEPTION: /* 0 Exception or non-maskable interrupt (NMI). */
4057 case VMX_EXIT_EXTERNAL_IRQ: /* 1 External interrupt. */
4058 case VMX_EXIT_EPT_VIOLATION:
4059 case VMX_EXIT_EPT_MISCONFIG: /* 49 EPT misconfig is used by the PGM/MMIO optimizations. */
4060 case VMX_EXIT_PREEMPTION_TIMER: /* 52 VMX-preemption timer expired. The preemption timer counted down to zero. */
4061 /* Already handled above. */
4062 break;
4063
4064 case VMX_EXIT_TRIPLE_FAULT: /* 2 Triple fault. */
4065 rc = VINF_EM_RESET; /* Triple fault equals a reset. */
4066 break;
4067
4068 case VMX_EXIT_INIT_SIGNAL: /* 3 INIT signal. */
4069 case VMX_EXIT_SIPI: /* 4 Start-up IPI (SIPI). */
4070 rc = VINF_EM_RAW_INTERRUPT;
4071 AssertFailed(); /* Can't happen. Yet. */
4072 break;
4073
4074 case VMX_EXIT_IO_SMI_IRQ: /* 5 I/O system-management interrupt (SMI). */
4075 case VMX_EXIT_SMI_IRQ: /* 6 Other SMI. */
4076 rc = VINF_EM_RAW_INTERRUPT;
4077 AssertFailed(); /* Can't happen afaik. */
4078 break;
4079
4080 case VMX_EXIT_TASK_SWITCH: /* 9 Task switch: too complicated to emulate, so fall back to the recompiler */
4081 Log(("VMX_EXIT_TASK_SWITCH: exit=%RX64\n", exitQualification));
4082 if ( (VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE(exitQualification) == VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE_IDT)
4083 && pVCpu->hwaccm.s.Event.fPending)
4084 {
4085 /* Caused by an injected interrupt. */
4086 pVCpu->hwaccm.s.Event.fPending = false;
4087
4088 Log(("VMX_EXIT_TASK_SWITCH: reassert trap %d\n", VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVCpu->hwaccm.s.Event.intInfo)));
4089 Assert(!VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(pVCpu->hwaccm.s.Event.intInfo));
4090 rc2 = TRPMAssertTrap(pVCpu, VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVCpu->hwaccm.s.Event.intInfo), TRPM_HARDWARE_INT);
4091 AssertRC(rc2);
4092 }
4093 /* else Exceptions and software interrupts can just be restarted. */
4094 rc = VERR_EM_INTERPRETER;
4095 break;
4096
4097 case VMX_EXIT_HLT: /* 12 Guest software attempted to execute HLT. */
4098 /* Check if external interrupts are pending; if so, don't switch back. */
4099 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitHlt);
4100 pCtx->rip++; /* skip hlt */
4101 if (EMShouldContinueAfterHalt(pVCpu, pCtx))
4102 goto ResumeExecution;
4103
4104 rc = VINF_EM_HALT;
4105 break;
4106
4107 case VMX_EXIT_MWAIT: /* 36 Guest software executed MWAIT. */
4108 Log2(("VMX: mwait\n"));
4109 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitMwait);
4110 rc = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pCtx));
4111 if ( rc == VINF_EM_HALT
4112 || rc == VINF_SUCCESS)
4113 {
4114 /* Update EIP and continue execution. */
4115 pCtx->rip += cbInstr;
4116
4117 /* Check if external interrupts are pending; if so, don't switch back. */
4118 if ( rc == VINF_SUCCESS
4119 || ( rc == VINF_EM_HALT
4120 && EMShouldContinueAfterHalt(pVCpu, pCtx))
4121 )
4122 goto ResumeExecution;
4123 }
4124 AssertMsg(rc == VERR_EM_INTERPRETER || rc == VINF_EM_HALT, ("EMU: mwait failed with %Rrc\n", VBOXSTRICTRC_VAL(rc)));
4125 break;
4126
4127 case VMX_EXIT_RSM: /* 17 Guest software attempted to execute RSM in SMM. */
4128 AssertFailed(); /* can't happen. */
4129 rc = VERR_EM_INTERPRETER;
4130 break;
4131
4132 case VMX_EXIT_MTF: /* 37 Exit due to Monitor Trap Flag. */
4133 LogFlow(("VMX_EXIT_MTF at %RGv\n", (RTGCPTR)pCtx->rip));
4134 pVCpu->hwaccm.s.vmx.proc_ctls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_TRAP_FLAG;
4135 rc2 = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
4136 AssertRC(rc2);
4137 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitMTF);
4138#if 0
4139 DBGFDoneStepping(pVCpu);
4140#endif
4141 rc = VINF_EM_DBG_STOP;
4142 break;
4143
4144 case VMX_EXIT_VMCALL: /* 18 Guest software executed VMCALL. */
4145 case VMX_EXIT_VMCLEAR: /* 19 Guest software executed VMCLEAR. */
4146 case VMX_EXIT_VMLAUNCH: /* 20 Guest software executed VMLAUNCH. */
4147 case VMX_EXIT_VMPTRLD: /* 21 Guest software executed VMPTRLD. */
4148 case VMX_EXIT_VMPTRST: /* 22 Guest software executed VMPTRST. */
4149 case VMX_EXIT_VMREAD: /* 23 Guest software executed VMREAD. */
4150 case VMX_EXIT_VMRESUME: /* 24 Guest software executed VMRESUME. */
4151 case VMX_EXIT_VMWRITE: /* 25 Guest software executed VMWRITE. */
4152 case VMX_EXIT_VMXOFF: /* 26 Guest software executed VMXOFF. */
4153 case VMX_EXIT_VMXON: /* 27 Guest software executed VMXON. */
4154 /** @todo inject #UD immediately */
4155 rc = VERR_EM_INTERPRETER;
4156 break;
4157
4158 case VMX_EXIT_CPUID: /* 10 Guest software attempted to execute CPUID. */
4159 case VMX_EXIT_RDTSC: /* 16 Guest software attempted to execute RDTSC. */
4160 case VMX_EXIT_INVPG: /* 14 Guest software attempted to execute INVPG. */
4161 case VMX_EXIT_CRX_MOVE: /* 28 Control-register accesses. */
4162 case VMX_EXIT_DRX_MOVE: /* 29 Debug-register accesses. */
4163 case VMX_EXIT_PORT_IO: /* 30 I/O instruction. */
4164 case VMX_EXIT_RDPMC: /* 15 Guest software attempted to execute RDPMC. */
4165 /* already handled above */
4166 AssertMsg( rc == VINF_PGM_CHANGE_MODE
4167 || rc == VINF_EM_RAW_INTERRUPT
4168 || rc == VERR_EM_INTERPRETER
4169 || rc == VINF_EM_RAW_EMULATE_INSTR
4170 || rc == VINF_PGM_SYNC_CR3
4171 || rc == VINF_IOM_R3_IOPORT_READ
4172 || rc == VINF_IOM_R3_IOPORT_WRITE
4173 || rc == VINF_EM_RAW_GUEST_TRAP
4174 || rc == VINF_TRPM_XCPT_DISPATCHED
4175 || rc == VINF_EM_RESCHEDULE_REM,
4176 ("rc = %d\n", VBOXSTRICTRC_VAL(rc)));
4177 break;
4178
4179 case VMX_EXIT_TPR: /* 43 TPR below threshold. Guest software executed MOV to CR8. */
4180 case VMX_EXIT_APIC_ACCESS: /* 44 APIC access. Guest software attempted to access memory at a physical address on the APIC-access page. */
4181 case VMX_EXIT_RDMSR: /* 31 RDMSR. Guest software attempted to execute RDMSR. */
4182 case VMX_EXIT_WRMSR: /* 32 WRMSR. Guest software attempted to execute WRMSR. */
4183 case VMX_EXIT_PAUSE: /* 40 Guest software attempted to execute PAUSE. */
4184 case VMX_EXIT_MONITOR: /* 39 Guest software attempted to execute MONITOR. */
4185 /* Note: If we decide to emulate them here, then we must sync the MSRs that could have been changed (sysenter, fs/gs base)!!! */
4186 rc = VERR_EM_INTERPRETER;
4187 break;
4188
4189 case VMX_EXIT_IRQ_WINDOW: /* 7 Interrupt window. */
4190 Assert(rc == VINF_EM_RAW_INTERRUPT);
4191 break;
4192
4193 case VMX_EXIT_ERR_INVALID_GUEST_STATE: /* 33 VM-entry failure due to invalid guest state. */
4194 {
4195#ifdef VBOX_STRICT
4196 RTCCUINTREG val2 = 0;
4197
4198 Log(("VMX_EXIT_ERR_INVALID_GUEST_STATE\n"));
4199
4200 VMXReadVMCS(VMX_VMCS64_GUEST_RIP, &val2);
4201 Log(("Old eip %RGv new %RGv\n", (RTGCPTR)pCtx->rip, (RTGCPTR)val2));
4202
4203 VMXReadVMCS(VMX_VMCS64_GUEST_CR0, &val2);
4204 Log(("VMX_VMCS_GUEST_CR0 %RX64\n", (uint64_t)val2));
4205
4206 VMXReadVMCS(VMX_VMCS64_GUEST_CR3, &val2);
4207 Log(("VMX_VMCS_GUEST_CR3 %RX64\n", (uint64_t)val2));
4208
4209 VMXReadVMCS(VMX_VMCS64_GUEST_CR4, &val2);
4210 Log(("VMX_VMCS_GUEST_CR4 %RX64\n", (uint64_t)val2));
4211
4212 VMXReadVMCS(VMX_VMCS_GUEST_RFLAGS, &val2);
4213 Log(("VMX_VMCS_GUEST_RFLAGS %08x\n", val2));
4214
4215 VMX_LOG_SELREG(CS, "CS", val2);
4216 VMX_LOG_SELREG(DS, "DS", val2);
4217 VMX_LOG_SELREG(ES, "ES", val2);
4218 VMX_LOG_SELREG(FS, "FS", val2);
4219 VMX_LOG_SELREG(GS, "GS", val2);
4220 VMX_LOG_SELREG(SS, "SS", val2);
4221 VMX_LOG_SELREG(TR, "TR", val2);
4222 VMX_LOG_SELREG(LDTR, "LDTR", val2);
4223
4224 VMXReadVMCS(VMX_VMCS64_GUEST_GDTR_BASE, &val2);
4225 Log(("VMX_VMCS_GUEST_GDTR_BASE %RX64\n", (uint64_t)val2));
4226 VMXReadVMCS(VMX_VMCS64_GUEST_IDTR_BASE, &val2);
4227 Log(("VMX_VMCS_GUEST_IDTR_BASE %RX64\n", (uint64_t)val2));
4228#endif /* VBOX_STRICT */
4229 rc = VERR_VMX_INVALID_GUEST_STATE;
4230 break;
4231 }
4232
4233 case VMX_EXIT_ERR_MSR_LOAD: /* 34 VM-entry failure due to MSR loading. */
4234 case VMX_EXIT_ERR_MACHINE_CHECK: /* 41 VM-entry failure due to machine-check. */
4235 default:
4236 rc = VERR_VMX_UNEXPECTED_EXIT_CODE;
4237 AssertMsgFailed(("Unexpected exit code %d\n", exitReason)); /* Can't happen. */
4238 break;
4239
4240 }
4241end:
4242
4243 /* We now going back to ring-3, so clear the action flag. */
4244 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
4245
4246 /* Signal changes for the recompiler. */
4247 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR | CPUM_CHANGED_LDTR | CPUM_CHANGED_GDTR | CPUM_CHANGED_IDTR | CPUM_CHANGED_TR | CPUM_CHANGED_HIDDEN_SEL_REGS);
4248
4249 /* If we executed vmlaunch/vmresume and an external irq was pending, then we don't have to do a full sync the next time. */
4250 if ( exitReason == VMX_EXIT_EXTERNAL_IRQ
4251 && !VMX_EXIT_INTERRUPTION_INFO_VALID(intInfo))
4252 {
4253 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatPendingHostIrq);
4254 /* On the next entry we'll only sync the host context. */
4255 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_HOST_CONTEXT;
4256 }
4257 else
4258 {
4259 /* On the next entry we'll sync everything. */
4260 /** @todo we can do better than this */
4261 /* Not in the VINF_PGM_CHANGE_MODE though! */
4262 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_ALL;
4263 }
4264
4265 /* translate into a less severe return code */
4266 if (rc == VERR_EM_INTERPRETER)
4267 rc = VINF_EM_RAW_EMULATE_INSTR;
4268 else
4269 /* Try to extract more information about what might have gone wrong here. */
4270 if (rc == VERR_VMX_INVALID_VMCS_PTR)
4271 {
4272 VMXGetActivateVMCS(&pVCpu->hwaccm.s.vmx.lasterror.u64VMCSPhys);
4273 pVCpu->hwaccm.s.vmx.lasterror.ulVMCSRevision = *(uint32_t *)pVCpu->hwaccm.s.vmx.pvVMCS;
4274 pVCpu->hwaccm.s.vmx.lasterror.idEnteredCpu = pVCpu->hwaccm.s.idEnteredCpu;
4275 pVCpu->hwaccm.s.vmx.lasterror.idCurrentCpu = RTMpCpuId();
4276 }
4277
4278 /* Just set the correct state here instead of trying to catch every goto above. */
4279 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED, VMCPUSTATE_STARTED_EXEC);
4280
4281#ifdef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
4282 /* Restore interrupts if we exitted after disabling them. */
4283 if (uOldEFlags != ~(RTCCUINTREG)0)
4284 ASMSetFlags(uOldEFlags);
4285#endif
4286
4287 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2, x);
4288 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit1, x);
4289 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatEntry, x);
4290 Log2(("X"));
4291 return VBOXSTRICTRC_TODO(rc);
4292}
4293
4294
4295/**
4296 * Enters the VT-x session
4297 *
4298 * @returns VBox status code.
4299 * @param pVM The VM to operate on.
4300 * @param pVCpu The VMCPU to operate on.
4301 * @param pCpu CPU info struct
4302 */
4303VMMR0DECL(int) VMXR0Enter(PVM pVM, PVMCPU pVCpu, PHMGLOBLCPUINFO pCpu)
4304{
4305 Assert(pVM->hwaccm.s.vmx.fSupported);
4306 NOREF(pCpu);
4307
4308 unsigned cr4 = ASMGetCR4();
4309 if (!(cr4 & X86_CR4_VMXE))
4310 {
4311 AssertMsgFailed(("X86_CR4_VMXE should be set!\n"));
4312 return VERR_VMX_X86_CR4_VMXE_CLEARED;
4313 }
4314
4315 /* Activate the VM Control Structure. */
4316 int rc = VMXActivateVMCS(pVCpu->hwaccm.s.vmx.HCPhysVMCS);
4317 if (RT_FAILURE(rc))
4318 return rc;
4319
4320 pVCpu->hwaccm.s.fResumeVM = false;
4321 return VINF_SUCCESS;
4322}
4323
4324
4325/**
4326 * Leaves the VT-x session
4327 *
4328 * @returns VBox status code.
4329 * @param pVM The VM to operate on.
4330 * @param pVCpu The VMCPU to operate on.
4331 * @param pCtx CPU context
4332 */
4333VMMR0DECL(int) VMXR0Leave(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
4334{
4335 Assert(pVM->hwaccm.s.vmx.fSupported);
4336
4337#ifdef DEBUG
4338 if (CPUMIsHyperDebugStateActive(pVCpu))
4339 {
4340 CPUMR0LoadHostDebugState(pVM, pVCpu);
4341 Assert(pVCpu->hwaccm.s.vmx.proc_ctls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT);
4342 }
4343 else
4344#endif
4345 /* Save the guest debug state if necessary. */
4346 if (CPUMIsGuestDebugStateActive(pVCpu))
4347 {
4348 CPUMR0SaveGuestDebugState(pVM, pVCpu, pCtx, true /* save DR6 */);
4349
4350 /* Enable drx move intercepts again. */
4351 pVCpu->hwaccm.s.vmx.proc_ctls |= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT;
4352 int rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
4353 AssertRC(rc);
4354
4355 /* Resync the debug registers the next time. */
4356 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_GUEST_DEBUG;
4357 }
4358 else
4359 Assert(pVCpu->hwaccm.s.vmx.proc_ctls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT);
4360
4361 /* Clear VM Control Structure. Marking it inactive, clearing implementation specific data and writing back VMCS data to memory. */
4362 int rc = VMXClearVMCS(pVCpu->hwaccm.s.vmx.HCPhysVMCS);
4363 AssertRC(rc);
4364
4365 return VINF_SUCCESS;
4366}
4367
4368/**
4369 * Flush the TLB (EPT)
4370 *
4371 * @returns VBox status code.
4372 * @param pVM The VM to operate on.
4373 * @param pVCpu The VM CPU to operate on.
4374 * @param enmFlush Type of flush
4375 * @param GCPhys Physical address of the page to flush
4376 */
4377static void hmR0VmxFlushEPT(PVM pVM, PVMCPU pVCpu, VMX_FLUSH enmFlush, RTGCPHYS GCPhys)
4378{
4379 uint64_t descriptor[2];
4380
4381 LogFlow(("hmR0VmxFlushEPT %d %RGv\n", enmFlush, GCPhys));
4382 Assert(pVM->hwaccm.s.fNestedPaging);
4383 descriptor[0] = pVCpu->hwaccm.s.vmx.GCPhysEPTP;
4384 descriptor[1] = GCPhys;
4385 int rc = VMXR0InvEPT(enmFlush, &descriptor[0]);
4386 AssertRC(rc);
4387}
4388
4389#ifdef HWACCM_VTX_WITH_VPID
4390/**
4391 * Flush the TLB (EPT)
4392 *
4393 * @returns VBox status code.
4394 * @param pVM The VM to operate on.
4395 * @param pVCpu The VM CPU to operate on.
4396 * @param enmFlush Type of flush
4397 * @param GCPtr Virtual address of the page to flush
4398 */
4399static void hmR0VmxFlushVPID(PVM pVM, PVMCPU pVCpu, VMX_FLUSH enmFlush, RTGCPTR GCPtr)
4400{
4401#if HC_ARCH_BITS == 32
4402 /* If we get a flush in 64 bits guest mode, then force a full TLB flush. Invvpid probably takes only 32 bits addresses. (@todo) */
4403 if ( CPUMIsGuestInLongMode(pVCpu)
4404 && !VMX_IS_64BIT_HOST_MODE())
4405 {
4406 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
4407 }
4408 else
4409#endif
4410 {
4411 uint64_t descriptor[2];
4412
4413 Assert(pVM->hwaccm.s.vmx.fVPID);
4414 descriptor[0] = pVCpu->hwaccm.s.uCurrentASID;
4415 descriptor[1] = GCPtr;
4416 int rc = VMXR0InvVPID(enmFlush, &descriptor[0]); NOREF(rc);
4417 AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvVPID %x %x %RGv failed with %d\n", enmFlush, pVCpu->hwaccm.s.uCurrentASID, GCPtr, rc));
4418 }
4419}
4420#endif /* HWACCM_VTX_WITH_VPID */
4421
4422/**
4423 * Invalidates a guest page
4424 *
4425 * @returns VBox status code.
4426 * @param pVM The VM to operate on.
4427 * @param pVCpu The VM CPU to operate on.
4428 * @param GCVirt Page to invalidate
4429 */
4430VMMR0DECL(int) VMXR0InvalidatePage(PVM pVM, PVMCPU pVCpu, RTGCPTR GCVirt)
4431{
4432 bool fFlushPending = VMCPU_FF_ISSET(pVCpu, VMCPU_FF_TLB_FLUSH);
4433
4434 Log2(("VMXR0InvalidatePage %RGv\n", GCVirt));
4435
4436 /* Only relevant if we want to use VPID.
4437 * In the nested paging case we still see such calls, but
4438 * can safely ignore them. (e.g. after cr3 updates)
4439 */
4440#ifdef HWACCM_VTX_WITH_VPID
4441 /* Skip it if a TLB flush is already pending. */
4442 if ( !fFlushPending
4443 && pVM->hwaccm.s.vmx.fVPID)
4444 hmR0VmxFlushVPID(pVM, pVCpu, pVM->hwaccm.s.vmx.enmFlushPage, GCVirt);
4445#endif /* HWACCM_VTX_WITH_VPID */
4446
4447 return VINF_SUCCESS;
4448}
4449
4450/**
4451 * Invalidates a guest page by physical address
4452 *
4453 * NOTE: Assumes the current instruction references this physical page though a virtual address!!
4454 *
4455 * @returns VBox status code.
4456 * @param pVM The VM to operate on.
4457 * @param pVCpu The VM CPU to operate on.
4458 * @param GCPhys Page to invalidate
4459 */
4460VMMR0DECL(int) VMXR0InvalidatePhysPage(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys)
4461{
4462 bool fFlushPending = VMCPU_FF_ISSET(pVCpu, VMCPU_FF_TLB_FLUSH);
4463
4464 Assert(pVM->hwaccm.s.fNestedPaging);
4465
4466 LogFlow(("VMXR0InvalidatePhysPage %RGp\n", GCPhys));
4467
4468 /* Skip it if a TLB flush is already pending. */
4469 if (!fFlushPending)
4470 hmR0VmxFlushEPT(pVM, pVCpu, pVM->hwaccm.s.vmx.enmFlushPage, GCPhys);
4471
4472 return VINF_SUCCESS;
4473}
4474
4475/**
4476 * Report world switch error and dump some useful debug info
4477 *
4478 * @param pVM The VM to operate on.
4479 * @param pVCpu The VMCPU to operate on.
4480 * @param rc Return code
4481 * @param pCtx Current CPU context (not updated)
4482 */
4483static void hmR0VmxReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, VBOXSTRICTRC rc, PCPUMCTX pCtx)
4484{
4485 NOREF(pVM);
4486
4487 switch (VBOXSTRICTRC_VAL(rc))
4488 {
4489 case VERR_VMX_INVALID_VMXON_PTR:
4490 AssertFailed();
4491 break;
4492
4493 case VERR_VMX_UNABLE_TO_START_VM:
4494 case VERR_VMX_UNABLE_TO_RESUME_VM:
4495 {
4496 int rc2;
4497 RTCCUINTREG exitReason, instrError;
4498
4499 rc2 = VMXReadVMCS(VMX_VMCS32_RO_EXIT_REASON, &exitReason);
4500 rc2 |= VMXReadVMCS(VMX_VMCS32_RO_VM_INSTR_ERROR, &instrError);
4501 AssertRC(rc2);
4502 if (rc2 == VINF_SUCCESS)
4503 {
4504 Log(("Unable to start/resume VM for reason: %x. Instruction error %x\n", (uint32_t)exitReason, (uint32_t)instrError));
4505 Log(("Current stack %08x\n", &rc2));
4506
4507 pVCpu->hwaccm.s.vmx.lasterror.ulInstrError = instrError;
4508 pVCpu->hwaccm.s.vmx.lasterror.ulExitReason = exitReason;
4509
4510#ifdef VBOX_STRICT
4511 RTGDTR gdtr;
4512 PCX86DESCHC pDesc;
4513 RTCCUINTREG val;
4514
4515 ASMGetGDTR(&gdtr);
4516
4517 VMXReadVMCS(VMX_VMCS64_GUEST_RIP, &val);
4518 Log(("Old eip %RGv new %RGv\n", (RTGCPTR)pCtx->rip, (RTGCPTR)val));
4519 VMXReadVMCS(VMX_VMCS_CTRL_PIN_EXEC_CONTROLS, &val);
4520 Log(("VMX_VMCS_CTRL_PIN_EXEC_CONTROLS %08x\n", val));
4521 VMXReadVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, &val);
4522 Log(("VMX_VMCS_CTRL_PROC_EXEC_CONTROLS %08x\n", val));
4523 VMXReadVMCS(VMX_VMCS_CTRL_ENTRY_CONTROLS, &val);
4524 Log(("VMX_VMCS_CTRL_ENTRY_CONTROLS %08x\n", val));
4525 VMXReadVMCS(VMX_VMCS_CTRL_EXIT_CONTROLS, &val);
4526 Log(("VMX_VMCS_CTRL_EXIT_CONTROLS %08x\n", val));
4527
4528 VMXReadVMCS(VMX_VMCS_HOST_CR0, &val);
4529 Log(("VMX_VMCS_HOST_CR0 %08x\n", val));
4530
4531 VMXReadVMCS(VMX_VMCS_HOST_CR3, &val);
4532 Log(("VMX_VMCS_HOST_CR3 %08x\n", val));
4533
4534 VMXReadVMCS(VMX_VMCS_HOST_CR4, &val);
4535 Log(("VMX_VMCS_HOST_CR4 %08x\n", val));
4536
4537 VMXReadVMCS(VMX_VMCS16_HOST_FIELD_CS, &val);
4538 Log(("VMX_VMCS_HOST_FIELD_CS %08x\n", val));
4539
4540 VMXReadVMCS(VMX_VMCS_GUEST_RFLAGS, &val);
4541 Log(("VMX_VMCS_GUEST_RFLAGS %08x\n", val));
4542
4543 if (val < gdtr.cbGdt)
4544 {
4545 pDesc = (PCX86DESCHC)(gdtr.pGdt + (val & X86_SEL_MASK));
4546 HWACCMR0DumpDescriptor(pDesc, val, "CS: ");
4547 }
4548
4549 VMXReadVMCS(VMX_VMCS16_HOST_FIELD_DS, &val);
4550 Log(("VMX_VMCS_HOST_FIELD_DS %08x\n", val));
4551 if (val < gdtr.cbGdt)
4552 {
4553 pDesc = (PCX86DESCHC)(gdtr.pGdt + (val & X86_SEL_MASK));
4554 HWACCMR0DumpDescriptor(pDesc, val, "DS: ");
4555 }
4556
4557 VMXReadVMCS(VMX_VMCS16_HOST_FIELD_ES, &val);
4558 Log(("VMX_VMCS_HOST_FIELD_ES %08x\n", val));
4559 if (val < gdtr.cbGdt)
4560 {
4561 pDesc = (PCX86DESCHC)(gdtr.pGdt + (val & X86_SEL_MASK));
4562 HWACCMR0DumpDescriptor(pDesc, val, "ES: ");
4563 }
4564
4565 VMXReadVMCS(VMX_VMCS16_HOST_FIELD_FS, &val);
4566 Log(("VMX_VMCS16_HOST_FIELD_FS %08x\n", val));
4567 if (val < gdtr.cbGdt)
4568 {
4569 pDesc = (PCX86DESCHC)(gdtr.pGdt + (val & X86_SEL_MASK));
4570 HWACCMR0DumpDescriptor(pDesc, val, "FS: ");
4571 }
4572
4573 VMXReadVMCS(VMX_VMCS16_HOST_FIELD_GS, &val);
4574 Log(("VMX_VMCS16_HOST_FIELD_GS %08x\n", val));
4575 if (val < gdtr.cbGdt)
4576 {
4577 pDesc = (PCX86DESCHC)(gdtr.pGdt + (val & X86_SEL_MASK));
4578 HWACCMR0DumpDescriptor(pDesc, val, "GS: ");
4579 }
4580
4581 VMXReadVMCS(VMX_VMCS16_HOST_FIELD_SS, &val);
4582 Log(("VMX_VMCS16_HOST_FIELD_SS %08x\n", val));
4583 if (val < gdtr.cbGdt)
4584 {
4585 pDesc = (PCX86DESCHC)(gdtr.pGdt + (val & X86_SEL_MASK));
4586 HWACCMR0DumpDescriptor(pDesc, val, "SS: ");
4587 }
4588
4589 VMXReadVMCS(VMX_VMCS16_HOST_FIELD_TR, &val);
4590 Log(("VMX_VMCS16_HOST_FIELD_TR %08x\n", val));
4591 if (val < gdtr.cbGdt)
4592 {
4593 pDesc = (PCX86DESCHC)(gdtr.pGdt + (val & X86_SEL_MASK));
4594 HWACCMR0DumpDescriptor(pDesc, val, "TR: ");
4595 }
4596
4597 VMXReadVMCS(VMX_VMCS_HOST_TR_BASE, &val);
4598 Log(("VMX_VMCS_HOST_TR_BASE %RHv\n", val));
4599
4600 VMXReadVMCS(VMX_VMCS_HOST_GDTR_BASE, &val);
4601 Log(("VMX_VMCS_HOST_GDTR_BASE %RHv\n", val));
4602 VMXReadVMCS(VMX_VMCS_HOST_IDTR_BASE, &val);
4603 Log(("VMX_VMCS_HOST_IDTR_BASE %RHv\n", val));
4604
4605 VMXReadVMCS(VMX_VMCS32_HOST_SYSENTER_CS, &val);
4606 Log(("VMX_VMCS_HOST_SYSENTER_CS %08x\n", val));
4607
4608 VMXReadVMCS(VMX_VMCS_HOST_SYSENTER_EIP, &val);
4609 Log(("VMX_VMCS_HOST_SYSENTER_EIP %RHv\n", val));
4610
4611 VMXReadVMCS(VMX_VMCS_HOST_SYSENTER_ESP, &val);
4612 Log(("VMX_VMCS_HOST_SYSENTER_ESP %RHv\n", val));
4613
4614 VMXReadVMCS(VMX_VMCS_HOST_RSP, &val);
4615 Log(("VMX_VMCS_HOST_RSP %RHv\n", val));
4616 VMXReadVMCS(VMX_VMCS_HOST_RIP, &val);
4617 Log(("VMX_VMCS_HOST_RIP %RHv\n", val));
4618
4619# if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4620 if (VMX_IS_64BIT_HOST_MODE())
4621 {
4622 Log(("MSR_K6_EFER = %RX64\n", ASMRdMsr(MSR_K6_EFER)));
4623 Log(("MSR_K6_STAR = %RX64\n", ASMRdMsr(MSR_K6_STAR)));
4624 Log(("MSR_K8_LSTAR = %RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
4625 Log(("MSR_K8_CSTAR = %RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
4626 Log(("MSR_K8_SF_MASK = %RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
4627 }
4628# endif
4629#endif /* VBOX_STRICT */
4630 }
4631 break;
4632 }
4633
4634 default:
4635 /* impossible */
4636 AssertMsgFailed(("%Rrc (%#x)\n", VBOXSTRICTRC_VAL(rc), VBOXSTRICTRC_VAL(rc)));
4637 break;
4638 }
4639}
4640
4641#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4642
4643/**
4644 * Prepares for and executes VMLAUNCH (64 bits guest mode)
4645 *
4646 * @returns VBox status code
4647 * @param fResume vmlauch/vmresume
4648 * @param pCtx Guest context
4649 * @param pCache VMCS cache
4650 * @param pVM The VM to operate on.
4651 * @param pVCpu The VMCPU to operate on.
4652 */
4653DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
4654{
4655 uint32_t aParam[6];
4656 PHMGLOBLCPUINFO pCpu;
4657 RTHCPHYS HCPhysCpuPage;
4658 int rc;
4659
4660 pCpu = HWACCMR0GetCurrentCpu();
4661 HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
4662
4663#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4664 pCache->uPos = 1;
4665 pCache->interPD = PGMGetInterPaeCR3(pVM);
4666 pCache->pSwitcher = (uint64_t)pVM->hwaccm.s.pfnHost32ToGuest64R0;
4667#endif
4668
4669#ifdef DEBUG
4670 pCache->TestIn.HCPhysCpuPage= 0;
4671 pCache->TestIn.HCPhysVMCS = 0;
4672 pCache->TestIn.pCache = 0;
4673 pCache->TestOut.HCPhysVMCS = 0;
4674 pCache->TestOut.pCache = 0;
4675 pCache->TestOut.pCtx = 0;
4676 pCache->TestOut.eflags = 0;
4677#endif
4678
4679 aParam[0] = (uint32_t)(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
4680 aParam[1] = (uint32_t)(HCPhysCpuPage >> 32); /* Param 1: VMXON physical address - Hi. */
4681 aParam[2] = (uint32_t)(pVCpu->hwaccm.s.vmx.HCPhysVMCS); /* Param 2: VMCS physical address - Lo. */
4682 aParam[3] = (uint32_t)(pVCpu->hwaccm.s.vmx.HCPhysVMCS >> 32); /* Param 2: VMCS physical address - Hi. */
4683 aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hwaccm.s.vmx.VMCSCache);
4684 aParam[5] = 0;
4685
4686#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4687 pCtx->dr[4] = pVM->hwaccm.s.vmx.pScratchPhys + 16 + 8;
4688 *(uint32_t *)(pVM->hwaccm.s.vmx.pScratch + 16 + 8) = 1;
4689#endif
4690 rc = VMXR0Execute64BitsHandler(pVM, pVCpu, pCtx, pVM->hwaccm.s.pfnVMXGCStartVM64, 6, &aParam[0]);
4691
4692#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4693 Assert(*(uint32_t *)(pVM->hwaccm.s.vmx.pScratch + 16 + 8) == 5);
4694 Assert(pCtx->dr[4] == 10);
4695 *(uint32_t *)(pVM->hwaccm.s.vmx.pScratch + 16 + 8) = 0xff;
4696#endif
4697
4698#ifdef DEBUG
4699 AssertMsg(pCache->TestIn.HCPhysCpuPage== HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
4700 AssertMsg(pCache->TestIn.HCPhysVMCS == pVCpu->hwaccm.s.vmx.HCPhysVMCS, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVMCS, pVCpu->hwaccm.s.vmx.HCPhysVMCS));
4701 AssertMsg(pCache->TestIn.HCPhysVMCS == pCache->TestOut.HCPhysVMCS, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVMCS, pCache->TestOut.HCPhysVMCS));
4702 AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache, pCache->TestOut.pCache));
4703 AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hwaccm.s.vmx.VMCSCache), ("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hwaccm.s.vmx.VMCSCache)));
4704 AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx, pCache->TestOut.pCtx));
4705 Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
4706#endif
4707 return rc;
4708}
4709
4710# ifdef VBOX_STRICT
4711
4712static bool hmR0VmxIsValidReadField(uint32_t idxField)
4713{
4714 switch(idxField)
4715 {
4716 case VMX_VMCS64_GUEST_RIP:
4717 case VMX_VMCS64_GUEST_RSP:
4718 case VMX_VMCS_GUEST_RFLAGS:
4719 case VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE:
4720 case VMX_VMCS_CTRL_CR0_READ_SHADOW:
4721 case VMX_VMCS64_GUEST_CR0:
4722 case VMX_VMCS_CTRL_CR4_READ_SHADOW:
4723 case VMX_VMCS64_GUEST_CR4:
4724 case VMX_VMCS64_GUEST_DR7:
4725 case VMX_VMCS32_GUEST_SYSENTER_CS:
4726 case VMX_VMCS64_GUEST_SYSENTER_EIP:
4727 case VMX_VMCS64_GUEST_SYSENTER_ESP:
4728 case VMX_VMCS32_GUEST_GDTR_LIMIT:
4729 case VMX_VMCS64_GUEST_GDTR_BASE:
4730 case VMX_VMCS32_GUEST_IDTR_LIMIT:
4731 case VMX_VMCS64_GUEST_IDTR_BASE:
4732 case VMX_VMCS16_GUEST_FIELD_CS:
4733 case VMX_VMCS32_GUEST_CS_LIMIT:
4734 case VMX_VMCS64_GUEST_CS_BASE:
4735 case VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS:
4736 case VMX_VMCS16_GUEST_FIELD_DS:
4737 case VMX_VMCS32_GUEST_DS_LIMIT:
4738 case VMX_VMCS64_GUEST_DS_BASE:
4739 case VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS:
4740 case VMX_VMCS16_GUEST_FIELD_ES:
4741 case VMX_VMCS32_GUEST_ES_LIMIT:
4742 case VMX_VMCS64_GUEST_ES_BASE:
4743 case VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS:
4744 case VMX_VMCS16_GUEST_FIELD_FS:
4745 case VMX_VMCS32_GUEST_FS_LIMIT:
4746 case VMX_VMCS64_GUEST_FS_BASE:
4747 case VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS:
4748 case VMX_VMCS16_GUEST_FIELD_GS:
4749 case VMX_VMCS32_GUEST_GS_LIMIT:
4750 case VMX_VMCS64_GUEST_GS_BASE:
4751 case VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS:
4752 case VMX_VMCS16_GUEST_FIELD_SS:
4753 case VMX_VMCS32_GUEST_SS_LIMIT:
4754 case VMX_VMCS64_GUEST_SS_BASE:
4755 case VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS:
4756 case VMX_VMCS16_GUEST_FIELD_LDTR:
4757 case VMX_VMCS32_GUEST_LDTR_LIMIT:
4758 case VMX_VMCS64_GUEST_LDTR_BASE:
4759 case VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS:
4760 case VMX_VMCS16_GUEST_FIELD_TR:
4761 case VMX_VMCS32_GUEST_TR_LIMIT:
4762 case VMX_VMCS64_GUEST_TR_BASE:
4763 case VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS:
4764 case VMX_VMCS32_RO_EXIT_REASON:
4765 case VMX_VMCS32_RO_VM_INSTR_ERROR:
4766 case VMX_VMCS32_RO_EXIT_INSTR_LENGTH:
4767 case VMX_VMCS32_RO_EXIT_INTERRUPTION_ERRCODE:
4768 case VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO:
4769 case VMX_VMCS32_RO_EXIT_INSTR_INFO:
4770 case VMX_VMCS_RO_EXIT_QUALIFICATION:
4771 case VMX_VMCS32_RO_IDT_INFO:
4772 case VMX_VMCS32_RO_IDT_ERRCODE:
4773 case VMX_VMCS64_GUEST_CR3:
4774 case VMX_VMCS_EXIT_PHYS_ADDR_FULL:
4775 return true;
4776 }
4777 return false;
4778}
4779
4780static bool hmR0VmxIsValidWriteField(uint32_t idxField)
4781{
4782 switch(idxField)
4783 {
4784 case VMX_VMCS64_GUEST_LDTR_BASE:
4785 case VMX_VMCS64_GUEST_TR_BASE:
4786 case VMX_VMCS64_GUEST_GDTR_BASE:
4787 case VMX_VMCS64_GUEST_IDTR_BASE:
4788 case VMX_VMCS64_GUEST_SYSENTER_EIP:
4789 case VMX_VMCS64_GUEST_SYSENTER_ESP:
4790 case VMX_VMCS64_GUEST_CR0:
4791 case VMX_VMCS64_GUEST_CR4:
4792 case VMX_VMCS64_GUEST_CR3:
4793 case VMX_VMCS64_GUEST_DR7:
4794 case VMX_VMCS64_GUEST_RIP:
4795 case VMX_VMCS64_GUEST_RSP:
4796 case VMX_VMCS64_GUEST_CS_BASE:
4797 case VMX_VMCS64_GUEST_DS_BASE:
4798 case VMX_VMCS64_GUEST_ES_BASE:
4799 case VMX_VMCS64_GUEST_FS_BASE:
4800 case VMX_VMCS64_GUEST_GS_BASE:
4801 case VMX_VMCS64_GUEST_SS_BASE:
4802 return true;
4803 }
4804 return false;
4805}
4806
4807# endif /* VBOX_STRICT */
4808
4809/**
4810 * Executes the specified handler in 64 mode
4811 *
4812 * @returns VBox status code.
4813 * @param pVM The VM to operate on.
4814 * @param pVCpu The VMCPU to operate on.
4815 * @param pCtx Guest context
4816 * @param pfnHandler RC handler
4817 * @param cbParam Number of parameters
4818 * @param paParam Array of 32 bits parameters
4819 */
4820VMMR0DECL(int) VMXR0Execute64BitsHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, RTRCPTR pfnHandler, uint32_t cbParam, uint32_t *paParam)
4821{
4822 int rc, rc2;
4823 PHMGLOBLCPUINFO pCpu;
4824 RTHCPHYS HCPhysCpuPage;
4825 RTHCUINTREG uOldEFlags;
4826
4827 AssertReturn(pVM->hwaccm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
4828 Assert(pfnHandler);
4829 Assert(pVCpu->hwaccm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hwaccm.s.vmx.VMCSCache.Write.aField));
4830 Assert(pVCpu->hwaccm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hwaccm.s.vmx.VMCSCache.Read.aField));
4831
4832#ifdef VBOX_STRICT
4833 for (unsigned i=0;i<pVCpu->hwaccm.s.vmx.VMCSCache.Write.cValidEntries;i++)
4834 Assert(hmR0VmxIsValidWriteField(pVCpu->hwaccm.s.vmx.VMCSCache.Write.aField[i]));
4835
4836 for (unsigned i=0;i<pVCpu->hwaccm.s.vmx.VMCSCache.Read.cValidEntries;i++)
4837 Assert(hmR0VmxIsValidReadField(pVCpu->hwaccm.s.vmx.VMCSCache.Read.aField[i]));
4838#endif
4839
4840 /* Disable interrupts. */
4841 uOldEFlags = ASMIntDisableFlags();
4842
4843 pCpu = HWACCMR0GetCurrentCpu();
4844 HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
4845
4846 /* Clear VM Control Structure. Marking it inactive, clearing implementation specific data and writing back VMCS data to memory. */
4847 VMXClearVMCS(pVCpu->hwaccm.s.vmx.HCPhysVMCS);
4848
4849 /* Leave VMX Root Mode. */
4850 VMXDisable();
4851
4852 ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
4853
4854 CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
4855 CPUMSetHyperEIP(pVCpu, pfnHandler);
4856 for (int i=(int)cbParam-1;i>=0;i--)
4857 CPUMPushHyper(pVCpu, paParam[i]);
4858
4859 STAM_PROFILE_ADV_START(&pVCpu->hwaccm.s.StatWorldSwitch3264, z);
4860 /* Call switcher. */
4861 rc = pVM->hwaccm.s.pfnHost32ToGuest64R0(pVM, RT_OFFSETOF(VM, aCpus[pVCpu->idCpu].cpum) - RT_OFFSETOF(VM, cpum));
4862 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatWorldSwitch3264, z);
4863
4864 /* Make sure the VMX instructions don't cause #UD faults. */
4865 ASMSetCR4(ASMGetCR4() | X86_CR4_VMXE);
4866
4867 /* Enter VMX Root Mode */
4868 rc2 = VMXEnable(HCPhysCpuPage);
4869 if (RT_FAILURE(rc2))
4870 {
4871 ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
4872 ASMSetFlags(uOldEFlags);
4873 return VERR_VMX_VMXON_FAILED;
4874 }
4875
4876 rc2 = VMXActivateVMCS(pVCpu->hwaccm.s.vmx.HCPhysVMCS);
4877 AssertRC(rc2);
4878 Assert(!(ASMGetFlags() & X86_EFL_IF));
4879 ASMSetFlags(uOldEFlags);
4880 return rc;
4881}
4882
4883#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL) */
4884
4885
4886#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
4887/**
4888 * Executes VMWRITE
4889 *
4890 * @returns VBox status code
4891 * @param pVCpu The VMCPU to operate on.
4892 * @param idxField VMCS index
4893 * @param u64Val 16, 32 or 64 bits value
4894 */
4895VMMR0DECL(int) VMXWriteVMCS64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
4896{
4897 int rc;
4898
4899 switch (idxField)
4900 {
4901 case VMX_VMCS_CTRL_TSC_OFFSET_FULL:
4902 case VMX_VMCS_CTRL_IO_BITMAP_A_FULL:
4903 case VMX_VMCS_CTRL_IO_BITMAP_B_FULL:
4904 case VMX_VMCS_CTRL_MSR_BITMAP_FULL:
4905 case VMX_VMCS_CTRL_VMEXIT_MSR_STORE_FULL:
4906 case VMX_VMCS_CTRL_VMEXIT_MSR_LOAD_FULL:
4907 case VMX_VMCS_CTRL_VMENTRY_MSR_LOAD_FULL:
4908 case VMX_VMCS_CTRL_VAPIC_PAGEADDR_FULL:
4909 case VMX_VMCS_CTRL_APIC_ACCESSADDR_FULL:
4910 case VMX_VMCS_GUEST_LINK_PTR_FULL:
4911 case VMX_VMCS_GUEST_PDPTR0_FULL:
4912 case VMX_VMCS_GUEST_PDPTR1_FULL:
4913 case VMX_VMCS_GUEST_PDPTR2_FULL:
4914 case VMX_VMCS_GUEST_PDPTR3_FULL:
4915 case VMX_VMCS_GUEST_DEBUGCTL_FULL:
4916 case VMX_VMCS_GUEST_EFER_FULL:
4917 case VMX_VMCS_CTRL_EPTP_FULL:
4918 /* These fields consist of two parts, which are both writable in 32 bits mode. */
4919 rc = VMXWriteVMCS32(idxField, u64Val);
4920 rc |= VMXWriteVMCS32(idxField + 1, (uint32_t)(u64Val >> 32ULL));
4921 AssertRC(rc);
4922 return rc;
4923
4924 case VMX_VMCS64_GUEST_LDTR_BASE:
4925 case VMX_VMCS64_GUEST_TR_BASE:
4926 case VMX_VMCS64_GUEST_GDTR_BASE:
4927 case VMX_VMCS64_GUEST_IDTR_BASE:
4928 case VMX_VMCS64_GUEST_SYSENTER_EIP:
4929 case VMX_VMCS64_GUEST_SYSENTER_ESP:
4930 case VMX_VMCS64_GUEST_CR0:
4931 case VMX_VMCS64_GUEST_CR4:
4932 case VMX_VMCS64_GUEST_CR3:
4933 case VMX_VMCS64_GUEST_DR7:
4934 case VMX_VMCS64_GUEST_RIP:
4935 case VMX_VMCS64_GUEST_RSP:
4936 case VMX_VMCS64_GUEST_CS_BASE:
4937 case VMX_VMCS64_GUEST_DS_BASE:
4938 case VMX_VMCS64_GUEST_ES_BASE:
4939 case VMX_VMCS64_GUEST_FS_BASE:
4940 case VMX_VMCS64_GUEST_GS_BASE:
4941 case VMX_VMCS64_GUEST_SS_BASE:
4942 /* Queue a 64 bits value as we can't set it in 32 bits host mode. */
4943 if (u64Val >> 32ULL)
4944 rc = VMXWriteCachedVMCSEx(pVCpu, idxField, u64Val);
4945 else
4946 rc = VMXWriteVMCS32(idxField, (uint32_t)u64Val);
4947
4948 return rc;
4949
4950 default:
4951 AssertMsgFailed(("Unexpected field %x\n", idxField));
4952 return VERR_INVALID_PARAMETER;
4953 }
4954}
4955
4956/**
4957 * Cache VMCS writes for performance reasons (Darwin) and for running 64 bits guests on 32 bits hosts.
4958 *
4959 * @param pVCpu The VMCPU to operate on.
4960 * @param idxField VMCS field
4961 * @param u64Val Value
4962 */
4963VMMR0DECL(int) VMXWriteCachedVMCSEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
4964{
4965 PVMCSCACHE pCache = &pVCpu->hwaccm.s.vmx.VMCSCache;
4966
4967 AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1, ("entries=%x\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
4968
4969 /* Make sure there are no duplicates. */
4970 for (unsigned i=0;i<pCache->Write.cValidEntries;i++)
4971 {
4972 if (pCache->Write.aField[i] == idxField)
4973 {
4974 pCache->Write.aFieldVal[i] = u64Val;
4975 return VINF_SUCCESS;
4976 }
4977 }
4978
4979 pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
4980 pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
4981 pCache->Write.cValidEntries++;
4982 return VINF_SUCCESS;
4983}
4984
4985#endif /* HC_ARCH_BITS == 32 && !VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0 */
4986
Note: See TracBrowser for help on using the repository browser.

© 2024 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette