VirtualBox

source: vbox/trunk/src/VBox/VMM/VMMR0/HWVMXR0.cpp@ 38677

Last change on this file since 38677 was 38677, checked in by vboxsync, 13 years ago

IOM: MMIO instruction interpreter bug fixes.
  • Property svn:eol-style set to native
  • Property svn:keywords set to Id
File size: 199.1 KB
Line 
1/* $Id: HWVMXR0.cpp 38677 2011-09-07 14:45:22Z vboxsync $ */
2/** @file
3 * HWACCM VMX - Host Context Ring 0.
4 */
5
6/*
7 * Copyright (C) 2006-2007 Oracle Corporation
8 *
9 * This file is part of VirtualBox Open Source Edition (OSE), as
10 * available from http://www.virtualbox.org. This file is free software;
11 * you can redistribute it and/or modify it under the terms of the GNU
12 * General Public License (GPL) as published by the Free Software
13 * Foundation, in version 2 as it comes in the "COPYING" file of the
14 * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15 * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16 */
17
18
19/*******************************************************************************
20* Header Files *
21*******************************************************************************/
22#define LOG_GROUP LOG_GROUP_HWACCM
23#include <iprt/asm-amd64-x86.h>
24#include <VBox/vmm/hwaccm.h>
25#include <VBox/vmm/pgm.h>
26#include <VBox/vmm/dbgf.h>
27#include <VBox/vmm/selm.h>
28#include <VBox/vmm/iom.h>
29#include <VBox/vmm/rem.h>
30#include <VBox/vmm/tm.h>
31#include "HWACCMInternal.h"
32#include <VBox/vmm/vm.h>
33#include <VBox/vmm/pdmapi.h>
34#include <VBox/err.h>
35#include <VBox/log.h>
36#include <iprt/assert.h>
37#include <iprt/param.h>
38#include <iprt/string.h>
39#include <iprt/time.h>
40#ifdef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
41# include <iprt/thread.h>
42#endif
43#include <iprt/x86.h>
44#include "HWVMXR0.h"
45
46/*******************************************************************************
47* Defined Constants And Macros *
48*******************************************************************************/
49#if defined(RT_ARCH_AMD64)
50# define VMX_IS_64BIT_HOST_MODE() (true)
51#elif defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
52# define VMX_IS_64BIT_HOST_MODE() (g_fVMXIs64bitHost != 0)
53#else
54# define VMX_IS_64BIT_HOST_MODE() (false)
55#endif
56
57/*******************************************************************************
58* Global Variables *
59*******************************************************************************/
60/* IO operation lookup arrays. */
61static uint32_t const g_aIOSize[4] = {1, 2, 0, 4};
62static uint32_t const g_aIOOpAnd[4] = {0xff, 0xffff, 0, 0xffffffff};
63
64#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
65/** See HWACCMR0A.asm. */
66extern "C" uint32_t g_fVMXIs64bitHost;
67#endif
68
69/*******************************************************************************
70* Local Functions *
71*******************************************************************************/
72static void VMXR0ReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, VBOXSTRICTRC rc, PCPUMCTX pCtx);
73static void vmxR0SetupTLBEPT(PVM pVM, PVMCPU pVCpu);
74static void vmxR0SetupTLBVPID(PVM pVM, PVMCPU pVCpu);
75static void vmxR0SetupTLBDummy(PVM pVM, PVMCPU pVCpu);
76static void vmxR0FlushEPT(PVM pVM, PVMCPU pVCpu, VMX_FLUSH enmFlush, RTGCPHYS GCPhys);
77static void vmxR0FlushVPID(PVM pVM, PVMCPU pVCpu, VMX_FLUSH enmFlush, RTGCPTR GCPtr);
78static void vmxR0UpdateExceptionBitmap(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx);
79#ifdef VBOX_STRICT
80static bool vmxR0IsValidReadField(uint32_t idxField);
81static bool vmxR0IsValidWriteField(uint32_t idxField);
82#endif
83static void vmxR0SetMSRPermission(PVMCPU pVCpu, unsigned ulMSR, bool fRead, bool fWrite);
84
85static void VMXR0CheckError(PVM pVM, PVMCPU pVCpu, int rc)
86{
87 if (rc == VERR_VMX_GENERIC)
88 {
89 RTCCUINTREG instrError;
90
91 VMXReadVMCS(VMX_VMCS32_RO_VM_INSTR_ERROR, &instrError);
92 pVCpu->hwaccm.s.vmx.lasterror.ulInstrError = instrError;
93 }
94 pVM->hwaccm.s.lLastError = rc;
95}
96
97/**
98 * Sets up and activates VT-x on the current CPU
99 *
100 * @returns VBox status code.
101 * @param pCpu CPU info struct
102 * @param pVM The VM to operate on. (can be NULL after a resume!!)
103 * @param pvCpuPage Pointer to the global cpu page.
104 * @param HCPhysCpuPage Physical address of the global cpu page.
105 */
106VMMR0DECL(int) VMXR0EnableCpu(PHMGLOBLCPUINFO pCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
107{
108 AssertReturn(HCPhysCpuPage != 0 && HCPhysCpuPage != NIL_RTHCPHYS, VERR_INVALID_PARAMETER);
109 AssertReturn(pvCpuPage, VERR_INVALID_PARAMETER);
110
111 if (pVM)
112 {
113 /* Set revision dword at the beginning of the VMXON structure. */
114 *(uint32_t *)pvCpuPage = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hwaccm.s.vmx.msr.vmx_basic_info);
115 }
116
117 /** @todo we should unmap the two pages from the virtual address space in order to prevent accidental corruption.
118 * (which can have very bad consequences!!!)
119 */
120
121 if (ASMGetCR4() & X86_CR4_VMXE)
122 return VERR_VMX_IN_VMX_ROOT_MODE;
123
124 /* Make sure the VMX instructions don't cause #UD faults. */
125 ASMSetCR4(ASMGetCR4() | X86_CR4_VMXE);
126
127 /* Enter VMX Root Mode. */
128 int rc = VMXEnable(HCPhysCpuPage);
129 if (RT_FAILURE(rc))
130 {
131 ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
132 return VERR_VMX_VMXON_FAILED;
133 }
134 return VINF_SUCCESS;
135}
136
137/**
138 * Deactivates VT-x on the current CPU
139 *
140 * @returns VBox status code.
141 * @param pCpu CPU info struct
142 * @param pvCpuPage Pointer to the global cpu page.
143 * @param HCPhysCpuPage Physical address of the global cpu page.
144 */
145VMMR0DECL(int) VMXR0DisableCpu(PHMGLOBLCPUINFO pCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
146{
147 AssertReturn(HCPhysCpuPage != 0 && HCPhysCpuPage != NIL_RTHCPHYS, VERR_INVALID_PARAMETER);
148 AssertReturn(pvCpuPage, VERR_INVALID_PARAMETER);
149
150 /* If we're somehow not in VMX root mode, then we shouldn't dare leaving it. */
151 if (!(ASMGetCR4() & X86_CR4_VMXE))
152 return VERR_VMX_NOT_IN_VMX_ROOT_MODE;
153
154 /* Leave VMX Root Mode. */
155 VMXDisable();
156
157 /* And clear the X86_CR4_VMXE bit. */
158 ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
159 return VINF_SUCCESS;
160}
161
162/**
163 * Does Ring-0 per VM VT-x init.
164 *
165 * @returns VBox status code.
166 * @param pVM The VM to operate on.
167 */
168VMMR0DECL(int) VMXR0InitVM(PVM pVM)
169{
170 int rc;
171
172#ifdef LOG_ENABLED
173 SUPR0Printf("VMXR0InitVM %x\n", pVM);
174#endif
175
176 pVM->hwaccm.s.vmx.pMemObjAPIC = NIL_RTR0MEMOBJ;
177
178 if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
179 {
180 /* Allocate one page for the APIC physical page (serves for filtering accesses). */
181 rc = RTR0MemObjAllocCont(&pVM->hwaccm.s.vmx.pMemObjAPIC, PAGE_SIZE, true /* executable R0 mapping */);
182 AssertRC(rc);
183 if (RT_FAILURE(rc))
184 return rc;
185
186 pVM->hwaccm.s.vmx.pAPIC = (uint8_t *)RTR0MemObjAddress(pVM->hwaccm.s.vmx.pMemObjAPIC);
187 pVM->hwaccm.s.vmx.pAPICPhys = RTR0MemObjGetPagePhysAddr(pVM->hwaccm.s.vmx.pMemObjAPIC, 0);
188 ASMMemZero32(pVM->hwaccm.s.vmx.pAPIC, PAGE_SIZE);
189 }
190 else
191 {
192 pVM->hwaccm.s.vmx.pMemObjAPIC = 0;
193 pVM->hwaccm.s.vmx.pAPIC = 0;
194 pVM->hwaccm.s.vmx.pAPICPhys = 0;
195 }
196
197#ifdef VBOX_WITH_CRASHDUMP_MAGIC
198 {
199 rc = RTR0MemObjAllocCont(&pVM->hwaccm.s.vmx.pMemObjScratch, PAGE_SIZE, true /* executable R0 mapping */);
200 AssertRC(rc);
201 if (RT_FAILURE(rc))
202 return rc;
203
204 pVM->hwaccm.s.vmx.pScratch = (uint8_t *)RTR0MemObjAddress(pVM->hwaccm.s.vmx.pMemObjScratch);
205 pVM->hwaccm.s.vmx.pScratchPhys = RTR0MemObjGetPagePhysAddr(pVM->hwaccm.s.vmx.pMemObjScratch, 0);
206
207 ASMMemZero32(pVM->hwaccm.s.vmx.pScratch, PAGE_SIZE);
208 strcpy((char *)pVM->hwaccm.s.vmx.pScratch, "SCRATCH Magic");
209 *(uint64_t *)(pVM->hwaccm.s.vmx.pScratch + 16) = UINT64_C(0xDEADBEEFDEADBEEF);
210 }
211#endif
212
213 /* Allocate VMCBs for all guest CPUs. */
214 for (VMCPUID i = 0; i < pVM->cCpus; i++)
215 {
216 PVMCPU pVCpu = &pVM->aCpus[i];
217
218 pVCpu->hwaccm.s.vmx.hMemObjVMCS = NIL_RTR0MEMOBJ;
219
220 /* Allocate one page for the VM control structure (VMCS). */
221 rc = RTR0MemObjAllocCont(&pVCpu->hwaccm.s.vmx.hMemObjVMCS, PAGE_SIZE, true /* executable R0 mapping */);
222 AssertRC(rc);
223 if (RT_FAILURE(rc))
224 return rc;
225
226 pVCpu->hwaccm.s.vmx.pvVMCS = RTR0MemObjAddress(pVCpu->hwaccm.s.vmx.hMemObjVMCS);
227 pVCpu->hwaccm.s.vmx.HCPhysVMCS = RTR0MemObjGetPagePhysAddr(pVCpu->hwaccm.s.vmx.hMemObjVMCS, 0);
228 ASMMemZeroPage(pVCpu->hwaccm.s.vmx.pvVMCS);
229
230 pVCpu->hwaccm.s.vmx.cr0_mask = 0;
231 pVCpu->hwaccm.s.vmx.cr4_mask = 0;
232
233 /* Allocate one page for the virtual APIC page for TPR caching. */
234 rc = RTR0MemObjAllocCont(&pVCpu->hwaccm.s.vmx.hMemObjVAPIC, PAGE_SIZE, true /* executable R0 mapping */);
235 AssertRC(rc);
236 if (RT_FAILURE(rc))
237 return rc;
238
239 pVCpu->hwaccm.s.vmx.pbVAPIC = (uint8_t *)RTR0MemObjAddress(pVCpu->hwaccm.s.vmx.hMemObjVAPIC);
240 pVCpu->hwaccm.s.vmx.HCPhysVAPIC = RTR0MemObjGetPagePhysAddr(pVCpu->hwaccm.s.vmx.hMemObjVAPIC, 0);
241 ASMMemZeroPage(pVCpu->hwaccm.s.vmx.pbVAPIC);
242
243 /* Allocate the MSR bitmap if this feature is supported. */
244 if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
245 {
246 rc = RTR0MemObjAllocCont(&pVCpu->hwaccm.s.vmx.pMemObjMSRBitmap, PAGE_SIZE, true /* executable R0 mapping */);
247 AssertRC(rc);
248 if (RT_FAILURE(rc))
249 return rc;
250
251 pVCpu->hwaccm.s.vmx.pMSRBitmap = (uint8_t *)RTR0MemObjAddress(pVCpu->hwaccm.s.vmx.pMemObjMSRBitmap);
252 pVCpu->hwaccm.s.vmx.pMSRBitmapPhys = RTR0MemObjGetPagePhysAddr(pVCpu->hwaccm.s.vmx.pMemObjMSRBitmap, 0);
253 memset(pVCpu->hwaccm.s.vmx.pMSRBitmap, 0xff, PAGE_SIZE);
254 }
255
256#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
257 /* Allocate one page for the guest MSR load area (for preloading guest MSRs during the world switch). */
258 rc = RTR0MemObjAllocCont(&pVCpu->hwaccm.s.vmx.pMemObjGuestMSR, PAGE_SIZE, true /* executable R0 mapping */);
259 AssertRC(rc);
260 if (RT_FAILURE(rc))
261 return rc;
262
263 pVCpu->hwaccm.s.vmx.pGuestMSR = (uint8_t *)RTR0MemObjAddress(pVCpu->hwaccm.s.vmx.pMemObjGuestMSR);
264 pVCpu->hwaccm.s.vmx.pGuestMSRPhys = RTR0MemObjGetPagePhysAddr(pVCpu->hwaccm.s.vmx.pMemObjGuestMSR, 0);
265 memset(pVCpu->hwaccm.s.vmx.pGuestMSR, 0, PAGE_SIZE);
266
267 /* Allocate one page for the host MSR load area (for restoring host MSRs after the world switch back). */
268 rc = RTR0MemObjAllocCont(&pVCpu->hwaccm.s.vmx.pMemObjHostMSR, PAGE_SIZE, true /* executable R0 mapping */);
269 AssertRC(rc);
270 if (RT_FAILURE(rc))
271 return rc;
272
273 pVCpu->hwaccm.s.vmx.pHostMSR = (uint8_t *)RTR0MemObjAddress(pVCpu->hwaccm.s.vmx.pMemObjHostMSR);
274 pVCpu->hwaccm.s.vmx.pHostMSRPhys = RTR0MemObjGetPagePhysAddr(pVCpu->hwaccm.s.vmx.pMemObjHostMSR, 0);
275 memset(pVCpu->hwaccm.s.vmx.pHostMSR, 0, PAGE_SIZE);
276#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
277
278 /* Current guest paging mode. */
279 pVCpu->hwaccm.s.vmx.enmLastSeenGuestMode = PGMMODE_REAL;
280
281#ifdef LOG_ENABLED
282 SUPR0Printf("VMXR0InitVM %x VMCS=%x (%x)\n", pVM, pVCpu->hwaccm.s.vmx.pvVMCS, (uint32_t)pVCpu->hwaccm.s.vmx.HCPhysVMCS);
283#endif
284 }
285
286 return VINF_SUCCESS;
287}
288
289/**
290 * Does Ring-0 per VM VT-x termination.
291 *
292 * @returns VBox status code.
293 * @param pVM The VM to operate on.
294 */
295VMMR0DECL(int) VMXR0TermVM(PVM pVM)
296{
297 for (VMCPUID i = 0; i < pVM->cCpus; i++)
298 {
299 PVMCPU pVCpu = &pVM->aCpus[i];
300
301 if (pVCpu->hwaccm.s.vmx.hMemObjVMCS != NIL_RTR0MEMOBJ)
302 {
303 RTR0MemObjFree(pVCpu->hwaccm.s.vmx.hMemObjVMCS, false);
304 pVCpu->hwaccm.s.vmx.hMemObjVMCS = NIL_RTR0MEMOBJ;
305 pVCpu->hwaccm.s.vmx.pvVMCS = 0;
306 pVCpu->hwaccm.s.vmx.HCPhysVMCS = 0;
307 }
308 if (pVCpu->hwaccm.s.vmx.hMemObjVAPIC != NIL_RTR0MEMOBJ)
309 {
310 RTR0MemObjFree(pVCpu->hwaccm.s.vmx.hMemObjVAPIC, false);
311 pVCpu->hwaccm.s.vmx.hMemObjVAPIC = NIL_RTR0MEMOBJ;
312 pVCpu->hwaccm.s.vmx.pbVAPIC = 0;
313 pVCpu->hwaccm.s.vmx.HCPhysVAPIC = 0;
314 }
315 if (pVCpu->hwaccm.s.vmx.pMemObjMSRBitmap != NIL_RTR0MEMOBJ)
316 {
317 RTR0MemObjFree(pVCpu->hwaccm.s.vmx.pMemObjMSRBitmap, false);
318 pVCpu->hwaccm.s.vmx.pMemObjMSRBitmap = NIL_RTR0MEMOBJ;
319 pVCpu->hwaccm.s.vmx.pMSRBitmap = 0;
320 pVCpu->hwaccm.s.vmx.pMSRBitmapPhys = 0;
321 }
322#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
323 if (pVCpu->hwaccm.s.vmx.pMemObjHostMSR != NIL_RTR0MEMOBJ)
324 {
325 RTR0MemObjFree(pVCpu->hwaccm.s.vmx.pMemObjHostMSR, false);
326 pVCpu->hwaccm.s.vmx.pMemObjHostMSR = NIL_RTR0MEMOBJ;
327 pVCpu->hwaccm.s.vmx.pHostMSR = 0;
328 pVCpu->hwaccm.s.vmx.pHostMSRPhys = 0;
329 }
330 if (pVCpu->hwaccm.s.vmx.pMemObjGuestMSR != NIL_RTR0MEMOBJ)
331 {
332 RTR0MemObjFree(pVCpu->hwaccm.s.vmx.pMemObjGuestMSR, false);
333 pVCpu->hwaccm.s.vmx.pMemObjGuestMSR = NIL_RTR0MEMOBJ;
334 pVCpu->hwaccm.s.vmx.pGuestMSR = 0;
335 pVCpu->hwaccm.s.vmx.pGuestMSRPhys = 0;
336 }
337#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
338 }
339 if (pVM->hwaccm.s.vmx.pMemObjAPIC != NIL_RTR0MEMOBJ)
340 {
341 RTR0MemObjFree(pVM->hwaccm.s.vmx.pMemObjAPIC, false);
342 pVM->hwaccm.s.vmx.pMemObjAPIC = NIL_RTR0MEMOBJ;
343 pVM->hwaccm.s.vmx.pAPIC = 0;
344 pVM->hwaccm.s.vmx.pAPICPhys = 0;
345 }
346#ifdef VBOX_WITH_CRASHDUMP_MAGIC
347 if (pVM->hwaccm.s.vmx.pMemObjScratch != NIL_RTR0MEMOBJ)
348 {
349 ASMMemZero32(pVM->hwaccm.s.vmx.pScratch, PAGE_SIZE);
350 RTR0MemObjFree(pVM->hwaccm.s.vmx.pMemObjScratch, false);
351 pVM->hwaccm.s.vmx.pMemObjScratch = NIL_RTR0MEMOBJ;
352 pVM->hwaccm.s.vmx.pScratch = 0;
353 pVM->hwaccm.s.vmx.pScratchPhys = 0;
354 }
355#endif
356 return VINF_SUCCESS;
357}
358
359/**
360 * Sets up VT-x for the specified VM
361 *
362 * @returns VBox status code.
363 * @param pVM The VM to operate on.
364 */
365VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
366{
367 int rc = VINF_SUCCESS;
368 uint32_t val;
369
370 AssertReturn(pVM, VERR_INVALID_PARAMETER);
371
372 for (VMCPUID i = 0; i < pVM->cCpus; i++)
373 {
374 PVMCPU pVCpu = &pVM->aCpus[i];
375
376 AssertPtr(pVCpu->hwaccm.s.vmx.pvVMCS);
377
378 /* Set revision dword at the beginning of the VMCS structure. */
379 *(uint32_t *)pVCpu->hwaccm.s.vmx.pvVMCS = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hwaccm.s.vmx.msr.vmx_basic_info);
380
381 /* Clear VM Control Structure. */
382 Log(("HCPhysVMCS = %RHp\n", pVCpu->hwaccm.s.vmx.HCPhysVMCS));
383 rc = VMXClearVMCS(pVCpu->hwaccm.s.vmx.HCPhysVMCS);
384 if (RT_FAILURE(rc))
385 goto vmx_end;
386
387 /* Activate the VM Control Structure. */
388 rc = VMXActivateVMCS(pVCpu->hwaccm.s.vmx.HCPhysVMCS);
389 if (RT_FAILURE(rc))
390 goto vmx_end;
391
392 /* VMX_VMCS_CTRL_PIN_EXEC_CONTROLS
393 * Set required bits to one and zero according to the MSR capabilities.
394 */
395 val = pVM->hwaccm.s.vmx.msr.vmx_pin_ctls.n.disallowed0;
396 /* External and non-maskable interrupts cause VM-exits. */
397 val |= VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_EXT_INT_EXIT | VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_NMI_EXIT;
398 /* enable the preemption timer. */
399 if (pVM->hwaccm.s.vmx.fUsePreemptTimer)
400 val |= VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_PREEMPT_TIMER;
401 val &= pVM->hwaccm.s.vmx.msr.vmx_pin_ctls.n.allowed1;
402
403 rc = VMXWriteVMCS(VMX_VMCS_CTRL_PIN_EXEC_CONTROLS, val);
404 AssertRC(rc);
405
406 /* VMX_VMCS_CTRL_PROC_EXEC_CONTROLS
407 * Set required bits to one and zero according to the MSR capabilities.
408 */
409 val = pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.disallowed0;
410 /* Program which event cause VM-exits and which features we want to use. */
411 val = val | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_HLT_EXIT
412 | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_TSC_OFFSET
413 | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT
414 | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_UNCOND_IO_EXIT
415 | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDPMC_EXIT
416 | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_EXIT
417 | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MWAIT_EXIT; /* don't execute mwait or else we'll idle inside the guest (host thinks the cpu load is high) */
418
419 /* Without nested paging we should intercept invlpg and cr3 mov instructions. */
420 if (!pVM->hwaccm.s.fNestedPaging)
421 val |= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INVLPG_EXIT
422 | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
423 | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT;
424
425 /* Note: VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MWAIT_EXIT might cause a vmlaunch failure with an invalid control fields error. (combined with some other exit reasons) */
426 if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
427 {
428 /* CR8 reads from the APIC shadow page; writes cause an exit is they lower the TPR below the threshold */
429 val |= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW;
430 Assert(pVM->hwaccm.s.vmx.pAPIC);
431 }
432 else
433 /* Exit on CR8 reads & writes in case the TPR shadow feature isn't present. */
434 val |= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR8_STORE_EXIT | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR8_LOAD_EXIT;
435
436 if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
437 {
438 Assert(pVCpu->hwaccm.s.vmx.pMSRBitmapPhys);
439 val |= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS;
440 }
441
442 /* We will use the secondary control if it's present. */
443 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
444
445 /* Mask away the bits that the CPU doesn't support */
446 /** @todo make sure they don't conflict with the above requirements. */
447 val &= pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1;
448 pVCpu->hwaccm.s.vmx.proc_ctls = val;
449
450 rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, val);
451 AssertRC(rc);
452
453 if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
454 {
455 /* VMX_VMCS_CTRL_PROC_EXEC_CONTROLS2
456 * Set required bits to one and zero according to the MSR capabilities.
457 */
458 val = pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2.n.disallowed0;
459 val |= VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT;
460
461#ifdef HWACCM_VTX_WITH_EPT
462 if (pVM->hwaccm.s.fNestedPaging)
463 val |= VMX_VMCS_CTRL_PROC_EXEC2_EPT;
464#endif /* HWACCM_VTX_WITH_EPT */
465#ifdef HWACCM_VTX_WITH_VPID
466 else
467 if (pVM->hwaccm.s.vmx.fVPID)
468 val |= VMX_VMCS_CTRL_PROC_EXEC2_VPID;
469#endif /* HWACCM_VTX_WITH_VPID */
470
471 if (pVM->hwaccm.s.fHasIoApic)
472 val |= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC;
473
474 if (pVM->hwaccm.s.vmx.fUnrestrictedGuest)
475 val |= VMX_VMCS_CTRL_PROC_EXEC2_REAL_MODE;
476
477 /* Mask away the bits that the CPU doesn't support */
478 /** @todo make sure they don't conflict with the above requirements. */
479 val &= pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2.n.allowed1;
480 pVCpu->hwaccm.s.vmx.proc_ctls2 = val;
481 rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS2, val);
482 AssertRC(rc);
483 }
484
485 /* VMX_VMCS_CTRL_CR3_TARGET_COUNT
486 * Set required bits to one and zero according to the MSR capabilities.
487 */
488 rc = VMXWriteVMCS(VMX_VMCS_CTRL_CR3_TARGET_COUNT, 0);
489 AssertRC(rc);
490
491 /* Forward all exception except #NM & #PF to the guest.
492 * We always need to check pagefaults since our shadow page table can be out of sync.
493 * And we always lazily sync the FPU & XMM state.
494 */
495
496 /** @todo Possible optimization:
497 * Keep the FPU and XMM state current in the EM thread. That way there's no need to
498 * lazily sync anything, but the downside is that we can't use the FPU stack or XMM
499 * registers ourselves of course.
500 *
501 * Note: only possible if the current state is actually ours (X86_CR0_TS flag)
502 */
503
504 /* Don't filter page faults; all of them should cause a switch. */
505 rc = VMXWriteVMCS(VMX_VMCS_CTRL_PAGEFAULT_ERROR_MASK, 0);
506 rc |= VMXWriteVMCS(VMX_VMCS_CTRL_PAGEFAULT_ERROR_MATCH, 0);
507 AssertRC(rc);
508
509 /* Init TSC offset to zero. */
510 rc = VMXWriteVMCS64(VMX_VMCS_CTRL_TSC_OFFSET_FULL, 0);
511 AssertRC(rc);
512
513 rc = VMXWriteVMCS64(VMX_VMCS_CTRL_IO_BITMAP_A_FULL, 0);
514 AssertRC(rc);
515
516 rc = VMXWriteVMCS64(VMX_VMCS_CTRL_IO_BITMAP_B_FULL, 0);
517 AssertRC(rc);
518
519 /* Set the MSR bitmap address. */
520 if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
521 {
522 Assert(pVCpu->hwaccm.s.vmx.pMSRBitmapPhys);
523
524 rc = VMXWriteVMCS64(VMX_VMCS_CTRL_MSR_BITMAP_FULL, pVCpu->hwaccm.s.vmx.pMSRBitmapPhys);
525 AssertRC(rc);
526
527 /* Allow the guest to directly modify these MSRs; they are restored and saved automatically. */
528 vmxR0SetMSRPermission(pVCpu, MSR_IA32_SYSENTER_CS, true, true);
529 vmxR0SetMSRPermission(pVCpu, MSR_IA32_SYSENTER_ESP, true, true);
530 vmxR0SetMSRPermission(pVCpu, MSR_IA32_SYSENTER_EIP, true, true);
531 vmxR0SetMSRPermission(pVCpu, MSR_K8_LSTAR, true, true);
532 vmxR0SetMSRPermission(pVCpu, MSR_K6_STAR, true, true);
533 vmxR0SetMSRPermission(pVCpu, MSR_K8_SF_MASK, true, true);
534 vmxR0SetMSRPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, true, true);
535 vmxR0SetMSRPermission(pVCpu, MSR_K8_GS_BASE, true, true);
536 vmxR0SetMSRPermission(pVCpu, MSR_K8_FS_BASE, true, true);
537 }
538
539#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
540 /* Set the guest & host MSR load/store physical addresses. */
541 Assert(pVCpu->hwaccm.s.vmx.pGuestMSRPhys);
542 rc = VMXWriteVMCS64(VMX_VMCS_CTRL_VMENTRY_MSR_LOAD_FULL, pVCpu->hwaccm.s.vmx.pGuestMSRPhys);
543 AssertRC(rc);
544 rc = VMXWriteVMCS64(VMX_VMCS_CTRL_VMEXIT_MSR_STORE_FULL, pVCpu->hwaccm.s.vmx.pGuestMSRPhys);
545 AssertRC(rc);
546
547 Assert(pVCpu->hwaccm.s.vmx.pHostMSRPhys);
548 rc = VMXWriteVMCS64(VMX_VMCS_CTRL_VMEXIT_MSR_LOAD_FULL, pVCpu->hwaccm.s.vmx.pHostMSRPhys);
549 AssertRC(rc);
550#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
551
552 rc = VMXWriteVMCS(VMX_VMCS_CTRL_ENTRY_MSR_LOAD_COUNT, 0);
553 AssertRC(rc);
554
555 rc = VMXWriteVMCS(VMX_VMCS_CTRL_EXIT_MSR_STORE_COUNT, 0);
556 AssertRC(rc);
557
558 if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
559 {
560 Assert(pVM->hwaccm.s.vmx.pMemObjAPIC);
561 /* Optional */
562 rc = VMXWriteVMCS(VMX_VMCS_CTRL_TPR_THRESHOLD, 0);
563 rc |= VMXWriteVMCS64(VMX_VMCS_CTRL_VAPIC_PAGEADDR_FULL, pVCpu->hwaccm.s.vmx.HCPhysVAPIC);
564
565 if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
566 rc |= VMXWriteVMCS64(VMX_VMCS_CTRL_APIC_ACCESSADDR_FULL, pVM->hwaccm.s.vmx.pAPICPhys);
567
568 AssertRC(rc);
569 }
570
571 /* Set link pointer to -1. Not currently used. */
572 rc = VMXWriteVMCS64(VMX_VMCS_GUEST_LINK_PTR_FULL, 0xFFFFFFFFFFFFFFFFULL);
573 AssertRC(rc);
574
575 /* Clear VM Control Structure. Marking it inactive, clearing implementation specific data and writing back VMCS data to memory. */
576 rc = VMXClearVMCS(pVCpu->hwaccm.s.vmx.HCPhysVMCS);
577 AssertRC(rc);
578
579 /* Configure the VMCS read cache. */
580 PVMCSCACHE pCache = &pVCpu->hwaccm.s.vmx.VMCSCache;
581
582 VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_RIP);
583 VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_RSP);
584 VMXSetupCachedReadVMCS(pCache, VMX_VMCS_GUEST_RFLAGS);
585 VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE);
586 VMXSetupCachedReadVMCS(pCache, VMX_VMCS_CTRL_CR0_READ_SHADOW);
587 VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_CR0);
588 VMXSetupCachedReadVMCS(pCache, VMX_VMCS_CTRL_CR4_READ_SHADOW);
589 VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_CR4);
590 VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_DR7);
591 VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_GUEST_SYSENTER_CS);
592 VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_SYSENTER_EIP);
593 VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_SYSENTER_ESP);
594 VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_GUEST_GDTR_LIMIT);
595 VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_GDTR_BASE);
596 VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_GUEST_IDTR_LIMIT);
597 VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_IDTR_BASE);
598
599 VMX_SETUP_SELREG(ES, pCache);
600 VMX_SETUP_SELREG(SS, pCache);
601 VMX_SETUP_SELREG(CS, pCache);
602 VMX_SETUP_SELREG(DS, pCache);
603 VMX_SETUP_SELREG(FS, pCache);
604 VMX_SETUP_SELREG(GS, pCache);
605 VMX_SETUP_SELREG(LDTR, pCache);
606 VMX_SETUP_SELREG(TR, pCache);
607
608 /* Status code VMCS reads. */
609 VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_EXIT_REASON);
610 VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_VM_INSTR_ERROR);
611 VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_EXIT_INSTR_LENGTH);
612 VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_EXIT_INTERRUPTION_ERRCODE);
613 VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO);
614 VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_EXIT_INSTR_INFO);
615 VMXSetupCachedReadVMCS(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
616 VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_IDT_INFO);
617 VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_IDT_ERRCODE);
618
619 if (pVM->hwaccm.s.fNestedPaging)
620 {
621 VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_CR3);
622 VMXSetupCachedReadVMCS(pCache, VMX_VMCS_EXIT_PHYS_ADDR_FULL);
623 pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
624 }
625 else
626 pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
627 } /* for each VMCPU */
628
629 /* Choose the right TLB setup function. */
630 if (pVM->hwaccm.s.fNestedPaging)
631 {
632 pVM->hwaccm.s.vmx.pfnSetupTaggedTLB = vmxR0SetupTLBEPT;
633
634 /* Default values for flushing. */
635 pVM->hwaccm.s.vmx.enmFlushPage = VMX_FLUSH_ALL_CONTEXTS;
636 pVM->hwaccm.s.vmx.enmFlushContext = VMX_FLUSH_ALL_CONTEXTS;
637
638 /* If the capabilities specify we can do more, then make use of it. */
639 if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVEPT_CAPS_INDIV)
640 pVM->hwaccm.s.vmx.enmFlushPage = VMX_FLUSH_PAGE;
641 else
642 if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVEPT_CAPS_CONTEXT)
643 pVM->hwaccm.s.vmx.enmFlushPage = VMX_FLUSH_SINGLE_CONTEXT;
644
645 if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVEPT_CAPS_CONTEXT)
646 pVM->hwaccm.s.vmx.enmFlushContext = VMX_FLUSH_SINGLE_CONTEXT;
647 }
648#ifdef HWACCM_VTX_WITH_VPID
649 else
650 if (pVM->hwaccm.s.vmx.fVPID)
651 {
652 pVM->hwaccm.s.vmx.pfnSetupTaggedTLB = vmxR0SetupTLBVPID;
653
654 /* Default values for flushing. */
655 pVM->hwaccm.s.vmx.enmFlushPage = VMX_FLUSH_ALL_CONTEXTS;
656 pVM->hwaccm.s.vmx.enmFlushContext = VMX_FLUSH_ALL_CONTEXTS;
657
658 /* If the capabilities specify we can do more, then make use of it. */
659 if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVVPID_CAPS_INDIV)
660 pVM->hwaccm.s.vmx.enmFlushPage = VMX_FLUSH_PAGE;
661 else
662 if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVVPID_CAPS_CONTEXT)
663 pVM->hwaccm.s.vmx.enmFlushPage = VMX_FLUSH_SINGLE_CONTEXT;
664
665 if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVVPID_CAPS_CONTEXT)
666 pVM->hwaccm.s.vmx.enmFlushContext = VMX_FLUSH_SINGLE_CONTEXT;
667 }
668#endif /* HWACCM_VTX_WITH_VPID */
669 else
670 pVM->hwaccm.s.vmx.pfnSetupTaggedTLB = vmxR0SetupTLBDummy;
671
672vmx_end:
673 VMXR0CheckError(pVM, &pVM->aCpus[0], rc);
674 return rc;
675}
676
677/**
678 * Sets the permission bits for the specified MSR
679 *
680 * @param pVCpu The VMCPU to operate on.
681 * @param ulMSR MSR value
682 * @param fRead Reading allowed/disallowed
683 * @param fWrite Writing allowed/disallowed
684 */
685static void vmxR0SetMSRPermission(PVMCPU pVCpu, unsigned ulMSR, bool fRead, bool fWrite)
686{
687 unsigned ulBit;
688 uint8_t *pMSRBitmap = (uint8_t *)pVCpu->hwaccm.s.vmx.pMSRBitmap;
689
690 /* Layout:
691 * 0x000 - 0x3ff - Low MSR read bits
692 * 0x400 - 0x7ff - High MSR read bits
693 * 0x800 - 0xbff - Low MSR write bits
694 * 0xc00 - 0xfff - High MSR write bits
695 */
696 if (ulMSR <= 0x00001FFF)
697 {
698 /* Pentium-compatible MSRs */
699 ulBit = ulMSR;
700 }
701 else
702 if ( ulMSR >= 0xC0000000
703 && ulMSR <= 0xC0001FFF)
704 {
705 /* AMD Sixth Generation x86 Processor MSRs */
706 ulBit = (ulMSR - 0xC0000000);
707 pMSRBitmap += 0x400;
708 }
709 else
710 {
711 AssertFailed();
712 return;
713 }
714
715 Assert(ulBit <= 0x1fff);
716 if (fRead)
717 ASMBitClear(pMSRBitmap, ulBit);
718 else
719 ASMBitSet(pMSRBitmap, ulBit);
720
721 if (fWrite)
722 ASMBitClear(pMSRBitmap + 0x800, ulBit);
723 else
724 ASMBitSet(pMSRBitmap + 0x800, ulBit);
725}
726
727
728/**
729 * Injects an event (trap or external interrupt)
730 *
731 * @returns VBox status code. Note that it may return VINF_EM_RESET to
732 * indicate a triple fault when injecting X86_XCPT_DF.
733 *
734 * @param pVM The VM to operate on.
735 * @param pVCpu The VMCPU to operate on.
736 * @param pCtx CPU Context
737 * @param intInfo VMX interrupt info
738 * @param cbInstr Opcode length of faulting instruction
739 * @param errCode Error code (optional)
740 */
741static int VMXR0InjectEvent(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, uint32_t intInfo, uint32_t cbInstr, uint32_t errCode)
742{
743 int rc;
744 uint32_t iGate = VMX_EXIT_INTERRUPTION_INFO_VECTOR(intInfo);
745
746#ifdef VBOX_WITH_STATISTICS
747 STAM_COUNTER_INC(&pVCpu->hwaccm.s.paStatInjectedIrqsR0[iGate & MASK_INJECT_IRQ_STAT]);
748#endif
749
750#ifdef VBOX_STRICT
751 if (iGate == 0xE)
752 LogFlow(("VMXR0InjectEvent: Injecting interrupt %d at %RGv error code=%08x CR2=%RGv intInfo=%08x\n", iGate, (RTGCPTR)pCtx->rip, errCode, pCtx->cr2, intInfo));
753 else
754 if (iGate < 0x20)
755 LogFlow(("VMXR0InjectEvent: Injecting interrupt %d at %RGv error code=%08x\n", iGate, (RTGCPTR)pCtx->rip, errCode));
756 else
757 {
758 LogFlow(("INJ-EI: %x at %RGv\n", iGate, (RTGCPTR)pCtx->rip));
759 Assert(VMX_EXIT_INTERRUPTION_INFO_TYPE(intInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW || !VMCPU_FF_ISSET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
760 Assert(VMX_EXIT_INTERRUPTION_INFO_TYPE(intInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW || pCtx->eflags.u32 & X86_EFL_IF);
761 }
762#endif
763
764 if ( CPUMIsGuestInRealModeEx(pCtx)
765 && pVM->hwaccm.s.vmx.pRealModeTSS)
766 {
767 RTGCPHYS GCPhysHandler;
768 uint16_t offset, ip;
769 RTSEL sel;
770
771 /* Injecting events doesn't work right with real mode emulation.
772 * (#GP if we try to inject external hardware interrupts)
773 * Inject the interrupt or trap directly instead.
774 *
775 * ASSUMES no access handlers for the bits we read or write below (should be safe).
776 */
777 Log(("Manual interrupt/trap '%x' inject (real mode)\n", iGate));
778
779 /* Check if the interrupt handler is present. */
780 if (iGate * 4 + 3 > pCtx->idtr.cbIdt)
781 {
782 Log(("IDT cbIdt violation\n"));
783 if (iGate != X86_XCPT_DF)
784 {
785 uint32_t intInfo2;
786
787 intInfo2 = (iGate == X86_XCPT_GP) ? (uint32_t)X86_XCPT_DF : iGate;
788 intInfo2 |= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
789 intInfo2 |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
790 intInfo2 |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HWEXCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
791
792 return VMXR0InjectEvent(pVM, pVCpu, pCtx, intInfo2, 0, 0 /* no error code according to the Intel docs */);
793 }
794 Log(("Triple fault -> reset the VM!\n"));
795 return VINF_EM_RESET;
796 }
797 if ( VMX_EXIT_INTERRUPTION_INFO_TYPE(intInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW
798 || iGate == 3 /* Both #BP and #OF point to the instruction after. */
799 || iGate == 4)
800 {
801 ip = pCtx->ip + cbInstr;
802 }
803 else
804 ip = pCtx->ip;
805
806 /* Read the selector:offset pair of the interrupt handler. */
807 GCPhysHandler = (RTGCPHYS)pCtx->idtr.pIdt + iGate * 4;
808 rc = PGMPhysSimpleReadGCPhys(pVM, &offset, GCPhysHandler, sizeof(offset)); AssertRC(rc);
809 rc = PGMPhysSimpleReadGCPhys(pVM, &sel, GCPhysHandler + 2, sizeof(sel)); AssertRC(rc);
810
811 LogFlow(("IDT handler %04X:%04X\n", sel, offset));
812
813 /* Construct the stack frame. */
814 /** @todo should check stack limit. */
815 pCtx->sp -= 2;
816 LogFlow(("ss:sp %04X:%04X eflags=%x\n", pCtx->ss, pCtx->sp, pCtx->eflags.u));
817 rc = PGMPhysSimpleWriteGCPhys(pVM, pCtx->ssHid.u64Base + pCtx->sp, &pCtx->eflags, sizeof(uint16_t)); AssertRC(rc);
818 pCtx->sp -= 2;
819 LogFlow(("ss:sp %04X:%04X cs=%x\n", pCtx->ss, pCtx->sp, pCtx->cs));
820 rc = PGMPhysSimpleWriteGCPhys(pVM, pCtx->ssHid.u64Base + pCtx->sp, &pCtx->cs, sizeof(uint16_t)); AssertRC(rc);
821 pCtx->sp -= 2;
822 LogFlow(("ss:sp %04X:%04X ip=%x\n", pCtx->ss, pCtx->sp, ip));
823 rc = PGMPhysSimpleWriteGCPhys(pVM, pCtx->ssHid.u64Base + pCtx->sp, &ip, sizeof(ip)); AssertRC(rc);
824
825 /* Update the CPU state for executing the handler. */
826 pCtx->rip = offset;
827 pCtx->cs = sel;
828 pCtx->csHid.u64Base = sel << 4;
829 pCtx->eflags.u &= ~(X86_EFL_IF|X86_EFL_TF|X86_EFL_RF|X86_EFL_AC);
830
831 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_GUEST_SEGMENT_REGS;
832 return VINF_SUCCESS;
833 }
834
835 /* Set event injection state. */
836 rc = VMXWriteVMCS(VMX_VMCS_CTRL_ENTRY_IRQ_INFO, intInfo | (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT));
837
838 rc |= VMXWriteVMCS(VMX_VMCS_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
839 rc |= VMXWriteVMCS(VMX_VMCS_CTRL_ENTRY_EXCEPTION_ERRCODE, errCode);
840
841 AssertRC(rc);
842 return rc;
843}
844
845
846/**
847 * Checks for pending guest interrupts and injects them
848 *
849 * @returns VBox status code.
850 * @param pVM The VM to operate on.
851 * @param pVCpu The VMCPU to operate on.
852 * @param pCtx CPU Context
853 */
854static int VMXR0CheckPendingInterrupt(PVM pVM, PVMCPU pVCpu, CPUMCTX *pCtx)
855{
856 int rc;
857
858 /* Dispatch any pending interrupts. (injected before, but a VM exit occurred prematurely) */
859 if (pVCpu->hwaccm.s.Event.fPending)
860 {
861 Log(("CPU%d: Reinjecting event %RX64 %08x at %RGv cr2=%RX64\n", pVCpu->idCpu, pVCpu->hwaccm.s.Event.intInfo, pVCpu->hwaccm.s.Event.errCode, (RTGCPTR)pCtx->rip, pCtx->cr2));
862 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatIntReinject);
863 rc = VMXR0InjectEvent(pVM, pVCpu, pCtx, pVCpu->hwaccm.s.Event.intInfo, 0, pVCpu->hwaccm.s.Event.errCode);
864 AssertRC(rc);
865
866 pVCpu->hwaccm.s.Event.fPending = false;
867 return VINF_SUCCESS;
868 }
869
870 /* If an active trap is already pending, then we must forward it first! */
871 if (!TRPMHasTrap(pVCpu))
872 {
873 if (VMCPU_FF_TESTANDCLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI))
874 {
875 RTGCUINTPTR intInfo;
876
877 Log(("CPU%d: injecting #NMI\n", pVCpu->idCpu));
878
879 intInfo = X86_XCPT_NMI;
880 intInfo |= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
881 intInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
882
883 rc = VMXR0InjectEvent(pVM, pVCpu, pCtx, intInfo, 0, 0);
884 AssertRC(rc);
885
886 return VINF_SUCCESS;
887 }
888
889 /* @todo SMI interrupts. */
890
891 /* When external interrupts are pending, we should exit the VM when IF is set. */
892 if (VMCPU_FF_ISPENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC|VMCPU_FF_INTERRUPT_PIC)))
893 {
894 if (!(pCtx->eflags.u32 & X86_EFL_IF))
895 {
896 if (!(pVCpu->hwaccm.s.vmx.proc_ctls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_IRQ_WINDOW_EXIT))
897 {
898 LogFlow(("Enable irq window exit!\n"));
899 pVCpu->hwaccm.s.vmx.proc_ctls |= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_IRQ_WINDOW_EXIT;
900 rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
901 AssertRC(rc);
902 }
903 /* else nothing to do but wait */
904 }
905 else
906 if (!VMCPU_FF_ISSET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
907 {
908 uint8_t u8Interrupt;
909
910 rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
911 Log(("CPU%d: Dispatch interrupt: u8Interrupt=%x (%d) rc=%Rrc cs:rip=%04X:%RGv\n", pVCpu->idCpu, u8Interrupt, u8Interrupt, rc, pCtx->cs, (RTGCPTR)pCtx->rip));
912 if (RT_SUCCESS(rc))
913 {
914 rc = TRPMAssertTrap(pVCpu, u8Interrupt, TRPM_HARDWARE_INT);
915 AssertRC(rc);
916 }
917 else
918 {
919 /* Can only happen in rare cases where a pending interrupt is cleared behind our back */
920 Assert(!VMCPU_FF_ISPENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC|VMCPU_FF_INTERRUPT_PIC)));
921 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatSwitchGuestIrq);
922 /* Just continue */
923 }
924 }
925 else
926 Log(("Pending interrupt blocked at %RGv by VM_FF_INHIBIT_INTERRUPTS!!\n", (RTGCPTR)pCtx->rip));
927 }
928 }
929
930#ifdef VBOX_STRICT
931 if (TRPMHasTrap(pVCpu))
932 {
933 uint8_t u8Vector;
934 rc = TRPMQueryTrapAll(pVCpu, &u8Vector, 0, 0, 0);
935 AssertRC(rc);
936 }
937#endif
938
939 if ( (pCtx->eflags.u32 & X86_EFL_IF)
940 && (!VMCPU_FF_ISSET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
941 && TRPMHasTrap(pVCpu)
942 )
943 {
944 uint8_t u8Vector;
945 TRPMEVENT enmType;
946 RTGCUINTPTR intInfo;
947 RTGCUINT errCode;
948
949 /* If a new event is pending, then dispatch it now. */
950 rc = TRPMQueryTrapAll(pVCpu, &u8Vector, &enmType, &errCode, 0);
951 AssertRC(rc);
952 Assert(pCtx->eflags.Bits.u1IF == 1 || enmType == TRPM_TRAP);
953 Assert(enmType != TRPM_SOFTWARE_INT);
954
955 /* Clear the pending trap. */
956 rc = TRPMResetTrap(pVCpu);
957 AssertRC(rc);
958
959 intInfo = u8Vector;
960 intInfo |= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
961
962 if (enmType == TRPM_TRAP)
963 {
964 switch (u8Vector) {
965 case 8:
966 case 10:
967 case 11:
968 case 12:
969 case 13:
970 case 14:
971 case 17:
972 /* Valid error codes. */
973 intInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
974 break;
975 default:
976 break;
977 }
978 if (u8Vector == X86_XCPT_BP || u8Vector == X86_XCPT_OF)
979 intInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SWEXCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
980 else
981 intInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HWEXCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
982 }
983 else
984 intInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
985
986 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatIntInject);
987 rc = VMXR0InjectEvent(pVM, pVCpu, pCtx, intInfo, 0, errCode);
988 AssertRC(rc);
989 } /* if (interrupts can be dispatched) */
990
991 return VINF_SUCCESS;
992}
993
994/**
995 * Save the host state
996 *
997 * @returns VBox status code.
998 * @param pVM The VM to operate on.
999 * @param pVCpu The VMCPU to operate on.
1000 */
1001VMMR0DECL(int) VMXR0SaveHostState(PVM pVM, PVMCPU pVCpu)
1002{
1003 int rc = VINF_SUCCESS;
1004
1005 /*
1006 * Host CPU Context
1007 */
1008 if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_HOST_CONTEXT)
1009 {
1010 RTIDTR idtr;
1011 RTGDTR gdtr;
1012 RTSEL SelTR;
1013 PCX86DESCHC pDesc;
1014 uintptr_t trBase;
1015 RTSEL cs;
1016 RTSEL ss;
1017 uint64_t cr3;
1018
1019 /* Control registers */
1020 rc = VMXWriteVMCS(VMX_VMCS_HOST_CR0, ASMGetCR0());
1021#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
1022 if (VMX_IS_64BIT_HOST_MODE())
1023 {
1024 cr3 = hwaccmR0Get64bitCR3();
1025 rc |= VMXWriteVMCS64(VMX_VMCS_HOST_CR3, cr3);
1026 }
1027 else
1028#endif
1029 {
1030 cr3 = ASMGetCR3();
1031 rc |= VMXWriteVMCS(VMX_VMCS_HOST_CR3, cr3);
1032 }
1033 rc |= VMXWriteVMCS(VMX_VMCS_HOST_CR4, ASMGetCR4());
1034 AssertRC(rc);
1035 Log2(("VMX_VMCS_HOST_CR0 %08x\n", ASMGetCR0()));
1036 Log2(("VMX_VMCS_HOST_CR3 %08RX64\n", cr3));
1037 Log2(("VMX_VMCS_HOST_CR4 %08x\n", ASMGetCR4()));
1038
1039 /* Selector registers. */
1040#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
1041 if (VMX_IS_64BIT_HOST_MODE())
1042 {
1043 cs = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelCS;
1044 ss = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelSS;
1045 }
1046 else
1047 {
1048 /* sysenter loads LDT cs & ss, VMX doesn't like this. Load the GDT ones (safe). */
1049 cs = (RTSEL)(uintptr_t)&SUPR0AbsKernelCS;
1050 ss = (RTSEL)(uintptr_t)&SUPR0AbsKernelSS;
1051 }
1052#else
1053 cs = ASMGetCS();
1054 ss = ASMGetSS();
1055#endif
1056 Assert(!(cs & X86_SEL_LDT)); Assert((cs & X86_SEL_RPL) == 0);
1057 Assert(!(ss & X86_SEL_LDT)); Assert((ss & X86_SEL_RPL) == 0);
1058 rc = VMXWriteVMCS(VMX_VMCS16_HOST_FIELD_CS, cs);
1059 /* Note: VMX is (again) very picky about the RPL of the selectors here; we'll restore them manually. */
1060 rc |= VMXWriteVMCS(VMX_VMCS16_HOST_FIELD_DS, 0);
1061 rc |= VMXWriteVMCS(VMX_VMCS16_HOST_FIELD_ES, 0);
1062#if HC_ARCH_BITS == 32
1063 if (!VMX_IS_64BIT_HOST_MODE())
1064 {
1065 rc |= VMXWriteVMCS(VMX_VMCS16_HOST_FIELD_FS, 0);
1066 rc |= VMXWriteVMCS(VMX_VMCS16_HOST_FIELD_GS, 0);
1067 }
1068#endif
1069 rc |= VMXWriteVMCS(VMX_VMCS16_HOST_FIELD_SS, ss);
1070 SelTR = ASMGetTR();
1071 rc |= VMXWriteVMCS(VMX_VMCS16_HOST_FIELD_TR, SelTR);
1072 AssertRC(rc);
1073 Log2(("VMX_VMCS_HOST_FIELD_CS %08x (%08x)\n", cs, ASMGetSS()));
1074 Log2(("VMX_VMCS_HOST_FIELD_DS 00000000 (%08x)\n", ASMGetDS()));
1075 Log2(("VMX_VMCS_HOST_FIELD_ES 00000000 (%08x)\n", ASMGetES()));
1076 Log2(("VMX_VMCS_HOST_FIELD_FS 00000000 (%08x)\n", ASMGetFS()));
1077 Log2(("VMX_VMCS_HOST_FIELD_GS 00000000 (%08x)\n", ASMGetGS()));
1078 Log2(("VMX_VMCS_HOST_FIELD_SS %08x (%08x)\n", ss, ASMGetSS()));
1079 Log2(("VMX_VMCS_HOST_FIELD_TR %08x\n", ASMGetTR()));
1080
1081 /* GDTR & IDTR */
1082#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
1083 if (VMX_IS_64BIT_HOST_MODE())
1084 {
1085 X86XDTR64 gdtr64, idtr64;
1086 hwaccmR0Get64bitGDTRandIDTR(&gdtr64, &idtr64);
1087 rc = VMXWriteVMCS64(VMX_VMCS_HOST_GDTR_BASE, gdtr64.uAddr);
1088 rc |= VMXWriteVMCS64(VMX_VMCS_HOST_IDTR_BASE, gdtr64.uAddr);
1089 AssertRC(rc);
1090 Log2(("VMX_VMCS_HOST_GDTR_BASE %RX64\n", gdtr64.uAddr));
1091 Log2(("VMX_VMCS_HOST_IDTR_BASE %RX64\n", idtr64.uAddr));
1092 gdtr.cbGdt = gdtr64.cb;
1093 gdtr.pGdt = (uintptr_t)gdtr64.uAddr;
1094 }
1095 else
1096#endif
1097 {
1098 ASMGetGDTR(&gdtr);
1099 rc = VMXWriteVMCS(VMX_VMCS_HOST_GDTR_BASE, gdtr.pGdt);
1100 ASMGetIDTR(&idtr);
1101 rc |= VMXWriteVMCS(VMX_VMCS_HOST_IDTR_BASE, idtr.pIdt);
1102 AssertRC(rc);
1103 Log2(("VMX_VMCS_HOST_GDTR_BASE %RHv\n", gdtr.pGdt));
1104 Log2(("VMX_VMCS_HOST_IDTR_BASE %RHv\n", idtr.pIdt));
1105 }
1106
1107 /* Save the base address of the TR selector. */
1108 if (SelTR > gdtr.cbGdt)
1109 {
1110 AssertMsgFailed(("Invalid TR selector %x. GDTR.cbGdt=%x\n", SelTR, gdtr.cbGdt));
1111 return VERR_VMX_INVALID_HOST_STATE;
1112 }
1113
1114 pDesc = (PCX86DESCHC)(gdtr.pGdt + (SelTR & X86_SEL_MASK));
1115#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
1116 if (VMX_IS_64BIT_HOST_MODE())
1117 {
1118 uint64_t trBase64 = X86DESC64_BASE(*(PX86DESC64)pDesc);
1119 rc = VMXWriteVMCS64(VMX_VMCS_HOST_TR_BASE, trBase64);
1120 Log2(("VMX_VMCS_HOST_TR_BASE %RX64\n", trBase64));
1121 AssertRC(rc);
1122 }
1123 else
1124#endif
1125 {
1126#if HC_ARCH_BITS == 64
1127 trBase = X86DESC64_BASE(*pDesc);
1128#else
1129 trBase = X86DESC_BASE(*pDesc);
1130#endif
1131 rc = VMXWriteVMCS(VMX_VMCS_HOST_TR_BASE, trBase);
1132 AssertRC(rc);
1133 Log2(("VMX_VMCS_HOST_TR_BASE %RHv\n", trBase));
1134 }
1135
1136 /* FS and GS base. */
1137#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
1138 if (VMX_IS_64BIT_HOST_MODE())
1139 {
1140 Log2(("MSR_K8_FS_BASE = %RX64\n", ASMRdMsr(MSR_K8_FS_BASE)));
1141 Log2(("MSR_K8_GS_BASE = %RX64\n", ASMRdMsr(MSR_K8_GS_BASE)));
1142 rc = VMXWriteVMCS64(VMX_VMCS_HOST_FS_BASE, ASMRdMsr(MSR_K8_FS_BASE));
1143 rc |= VMXWriteVMCS64(VMX_VMCS_HOST_GS_BASE, ASMRdMsr(MSR_K8_GS_BASE));
1144 }
1145#endif
1146 AssertRC(rc);
1147
1148 /* Sysenter MSRs. */
1149 /** @todo expensive!! */
1150 rc = VMXWriteVMCS(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
1151 Log2(("VMX_VMCS_HOST_SYSENTER_CS %08x\n", ASMRdMsr_Low(MSR_IA32_SYSENTER_CS)));
1152#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
1153 if (VMX_IS_64BIT_HOST_MODE())
1154 {
1155 Log2(("VMX_VMCS_HOST_SYSENTER_EIP %RX64\n", ASMRdMsr(MSR_IA32_SYSENTER_EIP)));
1156 Log2(("VMX_VMCS_HOST_SYSENTER_ESP %RX64\n", ASMRdMsr(MSR_IA32_SYSENTER_ESP)));
1157 rc |= VMXWriteVMCS64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
1158 rc |= VMXWriteVMCS64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
1159 }
1160 else
1161 {
1162 rc |= VMXWriteVMCS(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
1163 rc |= VMXWriteVMCS(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
1164 Log2(("VMX_VMCS_HOST_SYSENTER_EIP %RX32\n", ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP)));
1165 Log2(("VMX_VMCS_HOST_SYSENTER_ESP %RX32\n", ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP)));
1166 }
1167#elif HC_ARCH_BITS == 32
1168 rc |= VMXWriteVMCS(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
1169 rc |= VMXWriteVMCS(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
1170 Log2(("VMX_VMCS_HOST_SYSENTER_EIP %RX32\n", ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP)));
1171 Log2(("VMX_VMCS_HOST_SYSENTER_ESP %RX32\n", ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP)));
1172#else
1173 Log2(("VMX_VMCS_HOST_SYSENTER_EIP %RX64\n", ASMRdMsr(MSR_IA32_SYSENTER_EIP)));
1174 Log2(("VMX_VMCS_HOST_SYSENTER_ESP %RX64\n", ASMRdMsr(MSR_IA32_SYSENTER_ESP)));
1175 rc |= VMXWriteVMCS64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
1176 rc |= VMXWriteVMCS64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
1177#endif
1178 AssertRC(rc);
1179
1180#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
1181 /* Store all host MSRs in the VM-Exit load area, so they will be reloaded after the world switch back to the host. */
1182 PVMXMSR pMsr = (PVMXMSR)pVCpu->hwaccm.s.vmx.pHostMSR;
1183 unsigned idxMsr = 0;
1184
1185 /* EFER MSR present? */
1186 if (ASMCpuId_EDX(0x80000001) & (X86_CPUID_AMD_FEATURE_EDX_NX|X86_CPUID_AMD_FEATURE_EDX_LONG_MODE))
1187 {
1188 if (ASMCpuId_EDX(0x80000001) & X86_CPUID_AMD_FEATURE_EDX_SEP)
1189 {
1190 pMsr->u32IndexMSR = MSR_K6_STAR;
1191 pMsr->u32Reserved = 0;
1192 pMsr->u64Value = ASMRdMsr(MSR_K6_STAR); /* legacy syscall eip, cs & ss */
1193 pMsr++; idxMsr++;
1194 }
1195
1196 pMsr->u32IndexMSR = MSR_K6_EFER;
1197 pMsr->u32Reserved = 0;
1198# if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
1199 if (CPUMIsGuestInLongMode(pVCpu))
1200 {
1201 /* Must match the efer value in our 64 bits switcher. */
1202 pMsr->u64Value = ASMRdMsr(MSR_K6_EFER) | MSR_K6_EFER_LME | MSR_K6_EFER_SCE | MSR_K6_EFER_NXE;
1203 }
1204 else
1205# endif
1206 pMsr->u64Value = ASMRdMsr(MSR_K6_EFER);
1207 pMsr++; idxMsr++;
1208 }
1209
1210# if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
1211 if (VMX_IS_64BIT_HOST_MODE())
1212 {
1213 pMsr->u32IndexMSR = MSR_K8_LSTAR;
1214 pMsr->u32Reserved = 0;
1215 pMsr->u64Value = ASMRdMsr(MSR_K8_LSTAR); /* 64 bits mode syscall rip */
1216 pMsr++; idxMsr++;
1217 pMsr->u32IndexMSR = MSR_K8_SF_MASK;
1218 pMsr->u32Reserved = 0;
1219 pMsr->u64Value = ASMRdMsr(MSR_K8_SF_MASK); /* syscall flag mask */
1220 pMsr++; idxMsr++;
1221 pMsr->u32IndexMSR = MSR_K8_KERNEL_GS_BASE;
1222 pMsr->u32Reserved = 0;
1223 pMsr->u64Value = ASMRdMsr(MSR_K8_KERNEL_GS_BASE); /* swapgs exchange value */
1224 pMsr++; idxMsr++;
1225 }
1226# endif
1227 rc = VMXWriteVMCS(VMX_VMCS_CTRL_EXIT_MSR_LOAD_COUNT, idxMsr);
1228 AssertRC(rc);
1229#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
1230
1231 pVCpu->hwaccm.s.fContextUseFlags &= ~HWACCM_CHANGED_HOST_CONTEXT;
1232 }
1233 return rc;
1234}
1235
1236/**
1237 * Prefetch the 4 PDPT pointers (PAE and nested paging only)
1238 *
1239 * @returns VINF_SUCCESS or fatal error.
1240 * @param pVM The VM to operate on.
1241 * @param pVCpu The VMCPU to operate on.
1242 * @param pCtx Guest context
1243 */
1244static int vmxR0PrefetchPAEPdptrs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
1245{
1246 if (CPUMIsGuestInPAEModeEx(pCtx))
1247 {
1248 for (unsigned i=0;i<4;i++)
1249 {
1250 X86PDPE Pdpe;
1251 int rc = PGMGstQueryPaePDPtr(pVCpu, i, &Pdpe);
1252 AssertRCReturn(rc, rc);
1253
1254 rc = VMXWriteVMCS64(VMX_VMCS_GUEST_PDPTR0_FULL + i*2, Pdpe.u);
1255 AssertRC(rc);
1256 }
1257 }
1258 return VINF_SUCCESS;
1259}
1260
1261/**
1262 * Update the exception bitmap according to the current CPU state
1263 *
1264 * @param pVM The VM to operate on.
1265 * @param pVCpu The VMCPU to operate on.
1266 * @param pCtx Guest context
1267 */
1268static void vmxR0UpdateExceptionBitmap(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
1269{
1270 uint32_t u32TrapMask;
1271 Assert(pCtx);
1272
1273 u32TrapMask = HWACCM_VMX_TRAP_MASK;
1274#ifndef DEBUG
1275 if (pVM->hwaccm.s.fNestedPaging)
1276 u32TrapMask &= ~RT_BIT(X86_XCPT_PF); /* no longer need to intercept #PF. */
1277#endif
1278
1279 /* Also catch floating point exceptions as we need to report them to the guest in a different way. */
1280 if ( CPUMIsGuestFPUStateActive(pVCpu) == true
1281 && !(pCtx->cr0 & X86_CR0_NE)
1282 && !pVCpu->hwaccm.s.fFPUOldStyleOverride)
1283 {
1284 u32TrapMask |= RT_BIT(X86_XCPT_MF);
1285 pVCpu->hwaccm.s.fFPUOldStyleOverride = true;
1286 }
1287
1288#ifdef VBOX_STRICT
1289 Assert(u32TrapMask & RT_BIT(X86_XCPT_GP));
1290#endif
1291
1292 /* Intercept all exceptions in real mode as none of them can be injected directly (#GP otherwise). */
1293 if ( CPUMIsGuestInRealModeEx(pCtx)
1294 && pVM->hwaccm.s.vmx.pRealModeTSS)
1295 u32TrapMask |= HWACCM_VMX_TRAP_MASK_REALMODE;
1296
1297 int rc = VMXWriteVMCS(VMX_VMCS_CTRL_EXCEPTION_BITMAP, u32TrapMask);
1298 AssertRC(rc);
1299}
1300
1301/**
1302 * Loads a minimal guest state
1303 *
1304 * NOTE: Don't do anything here that can cause a jump back to ring 3!!!!!
1305 *
1306 * @param pVM The VM to operate on.
1307 * @param pVCpu The VMCPU to operate on.
1308 * @param pCtx Guest context
1309 */
1310VMMR0DECL(void) VMXR0LoadMinimalGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
1311{
1312 int rc;
1313 X86EFLAGS eflags;
1314
1315 Assert(!(pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_ALL_GUEST));
1316
1317 /* EIP, ESP and EFLAGS */
1318 rc = VMXWriteVMCS64(VMX_VMCS64_GUEST_RIP, pCtx->rip);
1319 rc |= VMXWriteVMCS64(VMX_VMCS64_GUEST_RSP, pCtx->rsp);
1320 AssertRC(rc);
1321
1322 /* Bits 22-31, 15, 5 & 3 must be zero. Bit 1 must be 1. */
1323 eflags = pCtx->eflags;
1324 eflags.u32 &= VMX_EFLAGS_RESERVED_0;
1325 eflags.u32 |= VMX_EFLAGS_RESERVED_1;
1326
1327 /* Real mode emulation using v86 mode. */
1328 if ( CPUMIsGuestInRealModeEx(pCtx)
1329 && pVM->hwaccm.s.vmx.pRealModeTSS)
1330 {
1331 pVCpu->hwaccm.s.vmx.RealMode.eflags = eflags;
1332
1333 eflags.Bits.u1VM = 1;
1334 eflags.Bits.u2IOPL = 0; /* must always be 0 or else certain instructions won't cause faults. */
1335 }
1336 rc = VMXWriteVMCS(VMX_VMCS_GUEST_RFLAGS, eflags.u32);
1337 AssertRC(rc);
1338}
1339
1340/**
1341 * Loads the guest state
1342 *
1343 * NOTE: Don't do anything here that can cause a jump back to ring 3!!!!!
1344 *
1345 * @returns VBox status code.
1346 * @param pVM The VM to operate on.
1347 * @param pVCpu The VMCPU to operate on.
1348 * @param pCtx Guest context
1349 */
1350VMMR0DECL(int) VMXR0LoadGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
1351{
1352 int rc = VINF_SUCCESS;
1353 RTGCUINTPTR val;
1354
1355 /* VMX_VMCS_CTRL_ENTRY_CONTROLS
1356 * Set required bits to one and zero according to the MSR capabilities.
1357 */
1358 val = pVM->hwaccm.s.vmx.msr.vmx_entry.n.disallowed0;
1359 /* Load guest debug controls (dr7 & IA32_DEBUGCTL_MSR) (forced to 1 on the 'first' VT-x capable CPUs; this actually includes the newest Nehalem CPUs) */
1360 val |= VMX_VMCS_CTRL_ENTRY_CONTROLS_LOAD_DEBUG;
1361 /* 64 bits guest mode? */
1362 if (CPUMIsGuestInLongModeEx(pCtx))
1363 val |= VMX_VMCS_CTRL_ENTRY_CONTROLS_IA64_MODE;
1364 /* else Must be zero when AMD64 is not available. */
1365
1366 /* Mask away the bits that the CPU doesn't support */
1367 val &= pVM->hwaccm.s.vmx.msr.vmx_entry.n.allowed1;
1368 rc = VMXWriteVMCS(VMX_VMCS_CTRL_ENTRY_CONTROLS, val);
1369 AssertRC(rc);
1370
1371 /* VMX_VMCS_CTRL_EXIT_CONTROLS
1372 * Set required bits to one and zero according to the MSR capabilities.
1373 */
1374 val = pVM->hwaccm.s.vmx.msr.vmx_exit.n.disallowed0;
1375
1376 /* Save debug controls (dr7 & IA32_DEBUGCTL_MSR) (forced to 1 on the 'first' VT-x capable CPUs; this actually includes the newest Nehalem CPUs) */
1377 val |= VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_DEBUG;
1378
1379#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
1380 if (VMX_IS_64BIT_HOST_MODE())
1381 val |= VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_AMD64;
1382 /* else: Must be zero when AMD64 is not available. */
1383#elif HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
1384 if (CPUMIsGuestInLongModeEx(pCtx))
1385 val |= VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_AMD64; /* our switcher goes to long mode */
1386 else
1387 Assert(!(val & VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_AMD64));
1388#endif
1389 val &= pVM->hwaccm.s.vmx.msr.vmx_exit.n.allowed1;
1390 /* Don't acknowledge external interrupts on VM-exit. */
1391 rc = VMXWriteVMCS(VMX_VMCS_CTRL_EXIT_CONTROLS, val);
1392 AssertRC(rc);
1393
1394 /* Guest CPU context: ES, CS, SS, DS, FS, GS. */
1395 if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_SEGMENT_REGS)
1396 {
1397 if (pVM->hwaccm.s.vmx.pRealModeTSS)
1398 {
1399 PGMMODE enmGuestMode = PGMGetGuestMode(pVCpu);
1400 if (pVCpu->hwaccm.s.vmx.enmLastSeenGuestMode != enmGuestMode)
1401 {
1402 /* Correct weird requirements for switching to protected mode. */
1403 if ( pVCpu->hwaccm.s.vmx.enmLastSeenGuestMode == PGMMODE_REAL
1404 && enmGuestMode >= PGMMODE_PROTECTED)
1405 {
1406 /* Flush the recompiler code cache as it's not unlikely
1407 * the guest will rewrite code it will later execute in real
1408 * mode (OpenBSD 4.0 is one such example)
1409 */
1410 REMFlushTBs(pVM);
1411
1412 /* DPL of all hidden selector registers must match the current CPL (0). */
1413 pCtx->csHid.Attr.n.u2Dpl = 0;
1414 pCtx->csHid.Attr.n.u4Type = X86_SEL_TYPE_CODE | X86_SEL_TYPE_RW_ACC;
1415
1416 pCtx->dsHid.Attr.n.u2Dpl = 0;
1417 pCtx->esHid.Attr.n.u2Dpl = 0;
1418 pCtx->fsHid.Attr.n.u2Dpl = 0;
1419 pCtx->gsHid.Attr.n.u2Dpl = 0;
1420 pCtx->ssHid.Attr.n.u2Dpl = 0;
1421
1422 /* The limit must correspond to the 32 bits setting. */
1423 if (!pCtx->csHid.Attr.n.u1DefBig)
1424 pCtx->csHid.u32Limit &= 0xffff;
1425 if (!pCtx->dsHid.Attr.n.u1DefBig)
1426 pCtx->dsHid.u32Limit &= 0xffff;
1427 if (!pCtx->esHid.Attr.n.u1DefBig)
1428 pCtx->esHid.u32Limit &= 0xffff;
1429 if (!pCtx->fsHid.Attr.n.u1DefBig)
1430 pCtx->fsHid.u32Limit &= 0xffff;
1431 if (!pCtx->gsHid.Attr.n.u1DefBig)
1432 pCtx->gsHid.u32Limit &= 0xffff;
1433 if (!pCtx->ssHid.Attr.n.u1DefBig)
1434 pCtx->ssHid.u32Limit &= 0xffff;
1435 }
1436 else
1437 /* Switching from protected mode to real mode. */
1438 if ( pVCpu->hwaccm.s.vmx.enmLastSeenGuestMode >= PGMMODE_PROTECTED
1439 && enmGuestMode == PGMMODE_REAL)
1440 {
1441 /* The limit must also be set to 0xffff. */
1442 pCtx->csHid.u32Limit = 0xffff;
1443 pCtx->dsHid.u32Limit = 0xffff;
1444 pCtx->esHid.u32Limit = 0xffff;
1445 pCtx->fsHid.u32Limit = 0xffff;
1446 pCtx->gsHid.u32Limit = 0xffff;
1447 pCtx->ssHid.u32Limit = 0xffff;
1448
1449 Assert(pCtx->csHid.u64Base <= 0xfffff);
1450 Assert(pCtx->dsHid.u64Base <= 0xfffff);
1451 Assert(pCtx->esHid.u64Base <= 0xfffff);
1452 Assert(pCtx->fsHid.u64Base <= 0xfffff);
1453 Assert(pCtx->gsHid.u64Base <= 0xfffff);
1454 }
1455 pVCpu->hwaccm.s.vmx.enmLastSeenGuestMode = enmGuestMode;
1456 }
1457 else
1458 /* VT-x will fail with a guest invalid state otherwise... (CPU state after a reset) */
1459 if ( CPUMIsGuestInRealModeEx(pCtx)
1460 && pCtx->csHid.u64Base == 0xffff0000)
1461 {
1462 pCtx->csHid.u64Base = 0xf0000;
1463 pCtx->cs = 0xf000;
1464 }
1465 }
1466
1467 VMX_WRITE_SELREG(ES, es);
1468 AssertRC(rc);
1469
1470 VMX_WRITE_SELREG(CS, cs);
1471 AssertRC(rc);
1472
1473 VMX_WRITE_SELREG(SS, ss);
1474 AssertRC(rc);
1475
1476 VMX_WRITE_SELREG(DS, ds);
1477 AssertRC(rc);
1478
1479 VMX_WRITE_SELREG(FS, fs);
1480 AssertRC(rc);
1481
1482 VMX_WRITE_SELREG(GS, gs);
1483 AssertRC(rc);
1484 }
1485
1486 /* Guest CPU context: LDTR. */
1487 if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_LDTR)
1488 {
1489 if (pCtx->ldtr == 0)
1490 {
1491 rc = VMXWriteVMCS(VMX_VMCS16_GUEST_FIELD_LDTR, 0);
1492 rc |= VMXWriteVMCS(VMX_VMCS32_GUEST_LDTR_LIMIT, 0);
1493 rc |= VMXWriteVMCS64(VMX_VMCS64_GUEST_LDTR_BASE, 0);
1494 /* Note: vmlaunch will fail with 0 or just 0x02. No idea why. */
1495 rc |= VMXWriteVMCS(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, 0x82 /* present, LDT */);
1496 }
1497 else
1498 {
1499 rc = VMXWriteVMCS(VMX_VMCS16_GUEST_FIELD_LDTR, pCtx->ldtr);
1500 rc |= VMXWriteVMCS(VMX_VMCS32_GUEST_LDTR_LIMIT, pCtx->ldtrHid.u32Limit);
1501 rc |= VMXWriteVMCS64(VMX_VMCS64_GUEST_LDTR_BASE, pCtx->ldtrHid.u64Base);
1502 rc |= VMXWriteVMCS(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, pCtx->ldtrHid.Attr.u);
1503 }
1504 AssertRC(rc);
1505 }
1506 /* Guest CPU context: TR. */
1507 if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_TR)
1508 {
1509 /* Real mode emulation using v86 mode with CR4.VME (interrupt redirection using the int bitmap in the TSS) */
1510 if ( CPUMIsGuestInRealModeEx(pCtx)
1511 && pVM->hwaccm.s.vmx.pRealModeTSS)
1512 {
1513 RTGCPHYS GCPhys;
1514
1515 /* We convert it here every time as pci regions could be reconfigured. */
1516 rc = PDMVMMDevHeapR3ToGCPhys(pVM, pVM->hwaccm.s.vmx.pRealModeTSS, &GCPhys);
1517 AssertRC(rc);
1518
1519 rc = VMXWriteVMCS(VMX_VMCS16_GUEST_FIELD_TR, 0);
1520 rc |= VMXWriteVMCS(VMX_VMCS32_GUEST_TR_LIMIT, HWACCM_VTX_TSS_SIZE);
1521 rc |= VMXWriteVMCS64(VMX_VMCS64_GUEST_TR_BASE, GCPhys /* phys = virt in this mode */);
1522
1523 X86DESCATTR attr;
1524
1525 attr.u = 0;
1526 attr.n.u1Present = 1;
1527 attr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
1528 val = attr.u;
1529 }
1530 else
1531 {
1532 rc = VMXWriteVMCS(VMX_VMCS16_GUEST_FIELD_TR, pCtx->tr);
1533 rc |= VMXWriteVMCS(VMX_VMCS32_GUEST_TR_LIMIT, pCtx->trHid.u32Limit);
1534 rc |= VMXWriteVMCS64(VMX_VMCS64_GUEST_TR_BASE, pCtx->trHid.u64Base);
1535
1536 val = pCtx->trHid.Attr.u;
1537
1538 /* The TSS selector must be busy (REM bugs? see defect #XXXX). */
1539 if (!(val & X86_SEL_TYPE_SYS_TSS_BUSY_MASK))
1540 {
1541 if (val & 0xf)
1542 val |= X86_SEL_TYPE_SYS_TSS_BUSY_MASK;
1543 else
1544 /* Default if no TR selector has been set (otherwise vmlaunch will fail!) */
1545 val = (val & ~0xF) | X86_SEL_TYPE_SYS_386_TSS_BUSY;
1546 }
1547 AssertMsg((val & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY || (val & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("%#x\n", val));
1548 }
1549 rc |= VMXWriteVMCS(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, val);
1550 AssertRC(rc);
1551 }
1552 /* Guest CPU context: GDTR. */
1553 if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_GDTR)
1554 {
1555 rc = VMXWriteVMCS(VMX_VMCS32_GUEST_GDTR_LIMIT, pCtx->gdtr.cbGdt);
1556 rc |= VMXWriteVMCS64(VMX_VMCS64_GUEST_GDTR_BASE, pCtx->gdtr.pGdt);
1557 AssertRC(rc);
1558 }
1559 /* Guest CPU context: IDTR. */
1560 if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_IDTR)
1561 {
1562 rc = VMXWriteVMCS(VMX_VMCS32_GUEST_IDTR_LIMIT, pCtx->idtr.cbIdt);
1563 rc |= VMXWriteVMCS64(VMX_VMCS64_GUEST_IDTR_BASE, pCtx->idtr.pIdt);
1564 AssertRC(rc);
1565 }
1566
1567 /*
1568 * Sysenter MSRs
1569 */
1570 if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_MSR)
1571 {
1572 rc = VMXWriteVMCS(VMX_VMCS32_GUEST_SYSENTER_CS, pCtx->SysEnter.cs);
1573 rc |= VMXWriteVMCS64(VMX_VMCS64_GUEST_SYSENTER_EIP, pCtx->SysEnter.eip);
1574 rc |= VMXWriteVMCS64(VMX_VMCS64_GUEST_SYSENTER_ESP, pCtx->SysEnter.esp);
1575 AssertRC(rc);
1576 }
1577
1578 /* Control registers */
1579 if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_CR0)
1580 {
1581 val = pCtx->cr0;
1582 rc = VMXWriteVMCS(VMX_VMCS_CTRL_CR0_READ_SHADOW, val);
1583 Log2(("Guest CR0-shadow %08x\n", val));
1584 if (CPUMIsGuestFPUStateActive(pVCpu) == false)
1585 {
1586 /* Always use #NM exceptions to load the FPU/XMM state on demand. */
1587 val |= X86_CR0_TS | X86_CR0_ET | X86_CR0_NE | X86_CR0_MP;
1588 }
1589 else
1590 {
1591 /** @todo check if we support the old style mess correctly. */
1592 if (!(val & X86_CR0_NE))
1593 Log(("Forcing X86_CR0_NE!!!\n"));
1594
1595 val |= X86_CR0_NE; /* always turn on the native mechanism to report FPU errors (old style uses interrupts) */
1596 }
1597 /* Note: protected mode & paging are always enabled; we use them for emulating real and protected mode without paging too. */
1598 if (!pVM->hwaccm.s.vmx.fUnrestrictedGuest)
1599 val |= X86_CR0_PE | X86_CR0_PG;
1600
1601 if (pVM->hwaccm.s.fNestedPaging)
1602 {
1603 if (CPUMIsGuestInPagedProtectedModeEx(pCtx))
1604 {
1605 /* Disable cr3 read/write monitoring as we don't need it for EPT. */
1606 pVCpu->hwaccm.s.vmx.proc_ctls &= ~( VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
1607 | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT);
1608 }
1609 else
1610 {
1611 /* Reenable cr3 read/write monitoring as our identity mapped page table is active. */
1612 pVCpu->hwaccm.s.vmx.proc_ctls |= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
1613 | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT;
1614 }
1615 rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
1616 AssertRC(rc);
1617 }
1618 else
1619 {
1620 /* Note: We must also set this as we rely on protecting various pages for which supervisor writes must be caught. */
1621 val |= X86_CR0_WP;
1622 }
1623
1624 /* Always enable caching. */
1625 val &= ~(X86_CR0_CD|X86_CR0_NW);
1626
1627 rc |= VMXWriteVMCS64(VMX_VMCS64_GUEST_CR0, val);
1628 Log2(("Guest CR0 %08x\n", val));
1629 /* CR0 flags owned by the host; if the guests attempts to change them, then
1630 * the VM will exit.
1631 */
1632 val = X86_CR0_PE /* Must monitor this bit (assumptions are made for real mode emulation) */
1633 | X86_CR0_WP /* Must monitor this bit (it must always be enabled). */
1634 | X86_CR0_PG /* Must monitor this bit (assumptions are made for real mode & protected mode without paging emulation) */
1635 | X86_CR0_CD /* Bit not restored during VM-exit! */
1636 | X86_CR0_NW /* Bit not restored during VM-exit! */
1637 | X86_CR0_NE;
1638
1639 /* When the guest's FPU state is active, then we no longer care about
1640 * the FPU related bits.
1641 */
1642 if (CPUMIsGuestFPUStateActive(pVCpu) == false)
1643 val |= X86_CR0_TS | X86_CR0_ET | X86_CR0_MP;
1644
1645 pVCpu->hwaccm.s.vmx.cr0_mask = val;
1646
1647 rc |= VMXWriteVMCS(VMX_VMCS_CTRL_CR0_MASK, val);
1648 Log2(("Guest CR0-mask %08x\n", val));
1649 AssertRC(rc);
1650 }
1651 if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_CR4)
1652 {
1653 /* CR4 */
1654 rc = VMXWriteVMCS(VMX_VMCS_CTRL_CR4_READ_SHADOW, pCtx->cr4);
1655 Log2(("Guest CR4-shadow %08x\n", pCtx->cr4));
1656 /* Set the required bits in cr4 too (currently X86_CR4_VMXE). */
1657 val = pCtx->cr4 | (uint32_t)pVM->hwaccm.s.vmx.msr.vmx_cr4_fixed0;
1658
1659 if (!pVM->hwaccm.s.fNestedPaging)
1660 {
1661 switch(pVCpu->hwaccm.s.enmShadowMode)
1662 {
1663 case PGMMODE_REAL: /* Real mode -> emulated using v86 mode */
1664 case PGMMODE_PROTECTED: /* Protected mode, no paging -> emulated using identity mapping. */
1665 case PGMMODE_32_BIT: /* 32-bit paging. */
1666 val &= ~X86_CR4_PAE;
1667 break;
1668
1669 case PGMMODE_PAE: /* PAE paging. */
1670 case PGMMODE_PAE_NX: /* PAE paging with NX enabled. */
1671 /** Must use PAE paging as we could use physical memory > 4 GB */
1672 val |= X86_CR4_PAE;
1673 break;
1674
1675 case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
1676 case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
1677#ifdef VBOX_ENABLE_64_BITS_GUESTS
1678 break;
1679#else
1680 AssertFailed();
1681 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
1682#endif
1683 default: /* shut up gcc */
1684 AssertFailed();
1685 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
1686 }
1687 }
1688 else
1689 if ( !CPUMIsGuestInPagedProtectedModeEx(pCtx)
1690 && !pVM->hwaccm.s.vmx.fUnrestrictedGuest)
1691 {
1692 /* We use 4 MB pages in our identity mapping page table for real and protected mode without paging. */
1693 val |= X86_CR4_PSE;
1694 /* Our identity mapping is a 32 bits page directory. */
1695 val &= ~X86_CR4_PAE;
1696 }
1697
1698 /* Turn off VME if we're in emulated real mode. */
1699 if ( CPUMIsGuestInRealModeEx(pCtx)
1700 && pVM->hwaccm.s.vmx.pRealModeTSS)
1701 val &= ~X86_CR4_VME;
1702
1703 rc |= VMXWriteVMCS64(VMX_VMCS64_GUEST_CR4, val);
1704 Log2(("Guest CR4 %08x\n", val));
1705 /* CR4 flags owned by the host; if the guests attempts to change them, then
1706 * the VM will exit.
1707 */
1708 val = 0
1709 | X86_CR4_VME
1710 | X86_CR4_PAE
1711 | X86_CR4_PGE
1712 | X86_CR4_PSE
1713 | X86_CR4_VMXE;
1714 pVCpu->hwaccm.s.vmx.cr4_mask = val;
1715
1716 rc |= VMXWriteVMCS(VMX_VMCS_CTRL_CR4_MASK, val);
1717 Log2(("Guest CR4-mask %08x\n", val));
1718 AssertRC(rc);
1719 }
1720
1721 if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_CR3)
1722 {
1723 if (pVM->hwaccm.s.fNestedPaging)
1724 {
1725 Assert(PGMGetHyperCR3(pVCpu));
1726 pVCpu->hwaccm.s.vmx.GCPhysEPTP = PGMGetHyperCR3(pVCpu);
1727
1728 Assert(!(pVCpu->hwaccm.s.vmx.GCPhysEPTP & 0xfff));
1729 /** @todo Check the IA32_VMX_EPT_VPID_CAP MSR for other supported memory types. */
1730 pVCpu->hwaccm.s.vmx.GCPhysEPTP |= VMX_EPT_MEMTYPE_WB
1731 | (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
1732
1733 rc = VMXWriteVMCS64(VMX_VMCS_CTRL_EPTP_FULL, pVCpu->hwaccm.s.vmx.GCPhysEPTP);
1734 AssertRC(rc);
1735
1736 if ( !CPUMIsGuestInPagedProtectedModeEx(pCtx)
1737 && !pVM->hwaccm.s.vmx.fUnrestrictedGuest)
1738 {
1739 RTGCPHYS GCPhys;
1740
1741 /* We convert it here every time as pci regions could be reconfigured. */
1742 rc = PDMVMMDevHeapR3ToGCPhys(pVM, pVM->hwaccm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
1743 AssertMsgRC(rc, ("pNonPagingModeEPTPageTable = %RGv\n", pVM->hwaccm.s.vmx.pNonPagingModeEPTPageTable));
1744
1745 /* We use our identity mapping page table here as we need to map guest virtual to guest physical addresses; EPT will
1746 * take care of the translation to host physical addresses.
1747 */
1748 val = GCPhys;
1749 }
1750 else
1751 {
1752 /* Save the real guest CR3 in VMX_VMCS_GUEST_CR3 */
1753 val = pCtx->cr3;
1754 /* Prefetch the four PDPT entries in PAE mode. */
1755 rc = vmxR0PrefetchPAEPdptrs(pVM, pVCpu, pCtx);
1756 AssertRCReturn(rc, rc);
1757 }
1758 }
1759 else
1760 {
1761 val = PGMGetHyperCR3(pVCpu);
1762 Assert(val || VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL));
1763 }
1764
1765 /* Save our shadow CR3 register. */
1766 rc = VMXWriteVMCS64(VMX_VMCS64_GUEST_CR3, val);
1767 AssertRC(rc);
1768 }
1769
1770 /* Debug registers. */
1771 if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_DEBUG)
1772 {
1773 pCtx->dr[6] |= X86_DR6_INIT_VAL; /* set all reserved bits to 1. */
1774 pCtx->dr[6] &= ~RT_BIT(12); /* must be zero. */
1775
1776 pCtx->dr[7] &= 0xffffffff; /* upper 32 bits reserved */
1777 pCtx->dr[7] &= ~(RT_BIT(11) | RT_BIT(12) | RT_BIT(14) | RT_BIT(15)); /* must be zero */
1778 pCtx->dr[7] |= 0x400; /* must be one */
1779
1780 /* Resync DR7 */
1781 rc = VMXWriteVMCS64(VMX_VMCS64_GUEST_DR7, pCtx->dr[7]);
1782 AssertRC(rc);
1783
1784#ifdef DEBUG
1785 /* Sync the hypervisor debug state now if any breakpoint is armed. */
1786 if ( CPUMGetHyperDR7(pVCpu) & (X86_DR7_ENABLED_MASK|X86_DR7_GD)
1787 && !CPUMIsHyperDebugStateActive(pVCpu)
1788 && !DBGFIsStepping(pVCpu))
1789 {
1790 /* Save the host and load the hypervisor debug state. */
1791 rc = CPUMR0LoadHyperDebugState(pVM, pVCpu, pCtx, true /* include DR6 */);
1792 AssertRC(rc);
1793
1794 /* DRx intercepts remain enabled. */
1795
1796 /* Override dr7 with the hypervisor value. */
1797 rc = VMXWriteVMCS64(VMX_VMCS64_GUEST_DR7, CPUMGetHyperDR7(pVCpu));
1798 AssertRC(rc);
1799 }
1800 else
1801#endif
1802 /* Sync the debug state now if any breakpoint is armed. */
1803 if ( (pCtx->dr[7] & (X86_DR7_ENABLED_MASK|X86_DR7_GD))
1804 && !CPUMIsGuestDebugStateActive(pVCpu)
1805 && !DBGFIsStepping(pVCpu))
1806 {
1807 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatDRxArmed);
1808
1809 /* Disable drx move intercepts. */
1810 pVCpu->hwaccm.s.vmx.proc_ctls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT;
1811 rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
1812 AssertRC(rc);
1813
1814 /* Save the host and load the guest debug state. */
1815 rc = CPUMR0LoadGuestDebugState(pVM, pVCpu, pCtx, true /* include DR6 */);
1816 AssertRC(rc);
1817 }
1818
1819 /* IA32_DEBUGCTL MSR. */
1820 rc = VMXWriteVMCS64(VMX_VMCS_GUEST_DEBUGCTL_FULL, 0);
1821 AssertRC(rc);
1822
1823 /** @todo do we really ever need this? */
1824 rc |= VMXWriteVMCS(VMX_VMCS_GUEST_DEBUG_EXCEPTIONS, 0);
1825 AssertRC(rc);
1826 }
1827
1828 /* 64 bits guest mode? */
1829 if (CPUMIsGuestInLongModeEx(pCtx))
1830 {
1831#if !defined(VBOX_ENABLE_64_BITS_GUESTS)
1832 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
1833#elif HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
1834 pVCpu->hwaccm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
1835#else
1836# ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
1837 if (!pVM->hwaccm.s.fAllow64BitGuests)
1838 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
1839# endif
1840 pVCpu->hwaccm.s.vmx.pfnStartVM = VMXR0StartVM64;
1841#endif
1842 if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_MSR)
1843 {
1844 /* Update these as wrmsr might have changed them. */
1845 rc = VMXWriteVMCS64(VMX_VMCS64_GUEST_FS_BASE, pCtx->fsHid.u64Base);
1846 AssertRC(rc);
1847 rc = VMXWriteVMCS64(VMX_VMCS64_GUEST_GS_BASE, pCtx->gsHid.u64Base);
1848 AssertRC(rc);
1849 }
1850 }
1851 else
1852 {
1853 pVCpu->hwaccm.s.vmx.pfnStartVM = VMXR0StartVM32;
1854 }
1855
1856 vmxR0UpdateExceptionBitmap(pVM, pVCpu, pCtx);
1857
1858#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
1859 /* Store all guest MSRs in the VM-Entry load area, so they will be loaded during the world switch. */
1860 PVMXMSR pMsr = (PVMXMSR)pVCpu->hwaccm.s.vmx.pGuestMSR;
1861 unsigned idxMsr = 0;
1862
1863 uint32_t ulEdx;
1864 uint32_t ulTemp;
1865 CPUMGetGuestCpuId(pVCpu, 0x80000001, &ulTemp, &ulTemp, &ulTemp, &ulEdx);
1866 /* EFER MSR present? */
1867 if (ulEdx & (X86_CPUID_AMD_FEATURE_EDX_NX|X86_CPUID_AMD_FEATURE_EDX_LONG_MODE))
1868 {
1869 pMsr->u32IndexMSR = MSR_K6_EFER;
1870 pMsr->u32Reserved = 0;
1871 pMsr->u64Value = pCtx->msrEFER;
1872 /* VT-x will complain if only MSR_K6_EFER_LME is set. */
1873 if (!CPUMIsGuestInLongModeEx(pCtx))
1874 pMsr->u64Value &= ~(MSR_K6_EFER_LMA|MSR_K6_EFER_LME);
1875 pMsr++; idxMsr++;
1876
1877 if (ulEdx & X86_CPUID_AMD_FEATURE_EDX_LONG_MODE)
1878 {
1879 pMsr->u32IndexMSR = MSR_K8_LSTAR;
1880 pMsr->u32Reserved = 0;
1881 pMsr->u64Value = pCtx->msrLSTAR; /* 64 bits mode syscall rip */
1882 pMsr++; idxMsr++;
1883 pMsr->u32IndexMSR = MSR_K6_STAR;
1884 pMsr->u32Reserved = 0;
1885 pMsr->u64Value = pCtx->msrSTAR; /* legacy syscall eip, cs & ss */
1886 pMsr++; idxMsr++;
1887 pMsr->u32IndexMSR = MSR_K8_SF_MASK;
1888 pMsr->u32Reserved = 0;
1889 pMsr->u64Value = pCtx->msrSFMASK; /* syscall flag mask */
1890 pMsr++; idxMsr++;
1891 pMsr->u32IndexMSR = MSR_K8_KERNEL_GS_BASE;
1892 pMsr->u32Reserved = 0;
1893 pMsr->u64Value = pCtx->msrKERNELGSBASE; /* swapgs exchange value */
1894 pMsr++; idxMsr++;
1895 }
1896 }
1897 pVCpu->hwaccm.s.vmx.cCachedMSRs = idxMsr;
1898
1899 rc = VMXWriteVMCS(VMX_VMCS_CTRL_ENTRY_MSR_LOAD_COUNT, idxMsr);
1900 AssertRC(rc);
1901
1902 rc = VMXWriteVMCS(VMX_VMCS_CTRL_EXIT_MSR_STORE_COUNT, idxMsr);
1903 AssertRC(rc);
1904#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
1905
1906 bool fOffsettedTsc;
1907 if (pVM->hwaccm.s.vmx.fUsePreemptTimer)
1908 {
1909 uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVCpu, &fOffsettedTsc, &pVCpu->hwaccm.s.vmx.u64TSCOffset);
1910
1911 /* Make sure the returned values have sane upper and lower boundaries. */
1912 uint64_t u64CpuHz = SUPGetCpuHzFromGIP(g_pSUPGlobalInfoPage);
1913
1914 cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64 of a second */
1915 cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
1916
1917 cTicksToDeadline >>= pVM->hwaccm.s.vmx.cPreemptTimerShift;
1918 uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
1919 rc = VMXWriteVMCS(VMX_VMCS32_GUEST_PREEMPTION_TIMER_VALUE, cPreemptionTickCount);
1920 AssertRC(rc);
1921 }
1922 else
1923 fOffsettedTsc = TMCpuTickCanUseRealTSC(pVCpu, &pVCpu->hwaccm.s.vmx.u64TSCOffset);
1924 if (fOffsettedTsc)
1925 {
1926 uint64_t u64CurTSC = ASMReadTSC();
1927 if (u64CurTSC + pVCpu->hwaccm.s.vmx.u64TSCOffset >= TMCpuTickGetLastSeen(pVCpu))
1928 {
1929 /* Note: VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT takes precedence over TSC_OFFSET */
1930 rc = VMXWriteVMCS64(VMX_VMCS_CTRL_TSC_OFFSET_FULL, pVCpu->hwaccm.s.vmx.u64TSCOffset);
1931 AssertRC(rc);
1932
1933 pVCpu->hwaccm.s.vmx.proc_ctls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT;
1934 rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
1935 AssertRC(rc);
1936 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatTSCOffset);
1937 }
1938 else
1939 {
1940 /* Fall back to rdtsc emulation as we would otherwise pass decreasing tsc values to the guest. */
1941 LogFlow(("TSC %RX64 offset %RX64 time=%RX64 last=%RX64 (diff=%RX64, virt_tsc=%RX64)\n", u64CurTSC, pVCpu->hwaccm.s.vmx.u64TSCOffset, u64CurTSC + pVCpu->hwaccm.s.vmx.u64TSCOffset, TMCpuTickGetLastSeen(pVCpu), TMCpuTickGetLastSeen(pVCpu) - u64CurTSC - pVCpu->hwaccm.s.vmx.u64TSCOffset, TMCpuTickGet(pVCpu)));
1942 pVCpu->hwaccm.s.vmx.proc_ctls |= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT;
1943 rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
1944 AssertRC(rc);
1945 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatTSCInterceptOverFlow);
1946 }
1947 }
1948 else
1949 {
1950 pVCpu->hwaccm.s.vmx.proc_ctls |= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT;
1951 rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
1952 AssertRC(rc);
1953 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatTSCIntercept);
1954 }
1955
1956 /* Done with the major changes */
1957 pVCpu->hwaccm.s.fContextUseFlags &= ~HWACCM_CHANGED_ALL_GUEST;
1958
1959 /* Minimal guest state update (esp, eip, eflags mostly) */
1960 VMXR0LoadMinimalGuestState(pVM, pVCpu, pCtx);
1961 return rc;
1962}
1963
1964/**
1965 * Syncs back the guest state
1966 *
1967 * @returns VBox status code.
1968 * @param pVM The VM to operate on.
1969 * @param pVCpu The VMCPU to operate on.
1970 * @param pCtx Guest context
1971 */
1972DECLINLINE(int) VMXR0SaveGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
1973{
1974 RTGCUINTREG val, valShadow;
1975 RTGCUINTPTR uInterruptState;
1976 int rc;
1977
1978 /* Let's first sync back eip, esp, and eflags. */
1979 rc = VMXReadCachedVMCS(VMX_VMCS64_GUEST_RIP, &val);
1980 AssertRC(rc);
1981 pCtx->rip = val;
1982 rc = VMXReadCachedVMCS(VMX_VMCS64_GUEST_RSP, &val);
1983 AssertRC(rc);
1984 pCtx->rsp = val;
1985 rc = VMXReadCachedVMCS(VMX_VMCS_GUEST_RFLAGS, &val);
1986 AssertRC(rc);
1987 pCtx->eflags.u32 = val;
1988
1989 /* Take care of instruction fusing (sti, mov ss) */
1990 rc |= VMXReadCachedVMCS(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &val);
1991 uInterruptState = val;
1992 if (uInterruptState != 0)
1993 {
1994 Assert(uInterruptState <= 2); /* only sti & mov ss */
1995 Log(("uInterruptState %x eip=%RGv\n", (uint32_t)uInterruptState, pCtx->rip));
1996 EMSetInhibitInterruptsPC(pVCpu, pCtx->rip);
1997 }
1998 else
1999 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
2000
2001 /* Control registers. */
2002 VMXReadCachedVMCS(VMX_VMCS_CTRL_CR0_READ_SHADOW, &valShadow);
2003 VMXReadCachedVMCS(VMX_VMCS64_GUEST_CR0, &val);
2004 val = (valShadow & pVCpu->hwaccm.s.vmx.cr0_mask) | (val & ~pVCpu->hwaccm.s.vmx.cr0_mask);
2005 CPUMSetGuestCR0(pVCpu, val);
2006
2007 VMXReadCachedVMCS(VMX_VMCS_CTRL_CR4_READ_SHADOW, &valShadow);
2008 VMXReadCachedVMCS(VMX_VMCS64_GUEST_CR4, &val);
2009 val = (valShadow & pVCpu->hwaccm.s.vmx.cr4_mask) | (val & ~pVCpu->hwaccm.s.vmx.cr4_mask);
2010 CPUMSetGuestCR4(pVCpu, val);
2011
2012 /* Note: no reason to sync back the CRx registers. They can't be changed by the guest. */
2013 /* Note: only in the nested paging case can CR3 & CR4 be changed by the guest. */
2014 if ( pVM->hwaccm.s.fNestedPaging
2015 && CPUMIsGuestInPagedProtectedModeEx(pCtx))
2016 {
2017 PVMCSCACHE pCache = &pVCpu->hwaccm.s.vmx.VMCSCache;
2018
2019 /* Can be updated behind our back in the nested paging case. */
2020 CPUMSetGuestCR2(pVCpu, pCache->cr2);
2021
2022 VMXReadCachedVMCS(VMX_VMCS64_GUEST_CR3, &val);
2023
2024 if (val != pCtx->cr3)
2025 {
2026 CPUMSetGuestCR3(pVCpu, val);
2027 PGMUpdateCR3(pVCpu, val);
2028 }
2029 /* Prefetch the four PDPT entries in PAE mode. */
2030 rc = vmxR0PrefetchPAEPdptrs(pVM, pVCpu, pCtx);
2031 AssertRCReturn(rc, rc);
2032 }
2033
2034 /* Sync back DR7 here. */
2035 VMXReadCachedVMCS(VMX_VMCS64_GUEST_DR7, &val);
2036 pCtx->dr[7] = val;
2037
2038 /* Guest CPU context: ES, CS, SS, DS, FS, GS. */
2039 VMX_READ_SELREG(ES, es);
2040 VMX_READ_SELREG(SS, ss);
2041 VMX_READ_SELREG(CS, cs);
2042 VMX_READ_SELREG(DS, ds);
2043 VMX_READ_SELREG(FS, fs);
2044 VMX_READ_SELREG(GS, gs);
2045
2046 /*
2047 * System MSRs
2048 */
2049 VMXReadCachedVMCS(VMX_VMCS32_GUEST_SYSENTER_CS, &val);
2050 pCtx->SysEnter.cs = val;
2051 VMXReadCachedVMCS(VMX_VMCS64_GUEST_SYSENTER_EIP, &val);
2052 pCtx->SysEnter.eip = val;
2053 VMXReadCachedVMCS(VMX_VMCS64_GUEST_SYSENTER_ESP, &val);
2054 pCtx->SysEnter.esp = val;
2055
2056 /* Misc. registers; must sync everything otherwise we can get out of sync when jumping to ring 3. */
2057 VMX_READ_SELREG(LDTR, ldtr);
2058
2059 VMXReadCachedVMCS(VMX_VMCS32_GUEST_GDTR_LIMIT, &val);
2060 pCtx->gdtr.cbGdt = val;
2061 VMXReadCachedVMCS(VMX_VMCS64_GUEST_GDTR_BASE, &val);
2062 pCtx->gdtr.pGdt = val;
2063
2064 VMXReadCachedVMCS(VMX_VMCS32_GUEST_IDTR_LIMIT, &val);
2065 pCtx->idtr.cbIdt = val;
2066 VMXReadCachedVMCS(VMX_VMCS64_GUEST_IDTR_BASE, &val);
2067 pCtx->idtr.pIdt = val;
2068
2069 /* Real mode emulation using v86 mode. */
2070 if ( CPUMIsGuestInRealModeEx(pCtx)
2071 && pVM->hwaccm.s.vmx.pRealModeTSS)
2072 {
2073 /* Hide our emulation flags */
2074 pCtx->eflags.Bits.u1VM = 0;
2075
2076 /* Restore original IOPL setting as we always use 0. */
2077 pCtx->eflags.Bits.u2IOPL = pVCpu->hwaccm.s.vmx.RealMode.eflags.Bits.u2IOPL;
2078
2079 /* Force a TR resync every time in case we switch modes. */
2080 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_GUEST_TR;
2081 }
2082 else
2083 {
2084 /* In real mode we have a fake TSS, so only sync it back when it's supposed to be valid. */
2085 VMX_READ_SELREG(TR, tr);
2086 }
2087
2088#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
2089 /* Save the possibly changed MSRs that we automatically restore and save during a world switch. */
2090 for (unsigned i = 0; i < pVCpu->hwaccm.s.vmx.cCachedMSRs; i++)
2091 {
2092 PVMXMSR pMsr = (PVMXMSR)pVCpu->hwaccm.s.vmx.pGuestMSR;
2093 pMsr += i;
2094
2095 switch (pMsr->u32IndexMSR)
2096 {
2097 case MSR_K8_LSTAR:
2098 pCtx->msrLSTAR = pMsr->u64Value;
2099 break;
2100 case MSR_K6_STAR:
2101 pCtx->msrSTAR = pMsr->u64Value;
2102 break;
2103 case MSR_K8_SF_MASK:
2104 pCtx->msrSFMASK = pMsr->u64Value;
2105 break;
2106 case MSR_K8_KERNEL_GS_BASE:
2107 pCtx->msrKERNELGSBASE = pMsr->u64Value;
2108 break;
2109 case MSR_K6_EFER:
2110 /* EFER can't be changed without causing a VM-exit. */
2111// Assert(pCtx->msrEFER == pMsr->u64Value);
2112 break;
2113 default:
2114 AssertFailed();
2115 return VERR_INTERNAL_ERROR;
2116 }
2117 }
2118#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
2119 return VINF_SUCCESS;
2120}
2121
2122/**
2123 * Dummy placeholder
2124 *
2125 * @param pVM The VM to operate on.
2126 * @param pVCpu The VMCPU to operate on.
2127 */
2128static void vmxR0SetupTLBDummy(PVM pVM, PVMCPU pVCpu)
2129{
2130 NOREF(pVM);
2131 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
2132 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
2133 pVCpu->hwaccm.s.TlbShootdown.cPages = 0;
2134 return;
2135}
2136
2137/**
2138 * Setup the tagged TLB for EPT
2139 *
2140 * @returns VBox status code.
2141 * @param pVM The VM to operate on.
2142 * @param pVCpu The VMCPU to operate on.
2143 */
2144static void vmxR0SetupTLBEPT(PVM pVM, PVMCPU pVCpu)
2145{
2146 PHMGLOBLCPUINFO pCpu;
2147
2148 Assert(pVM->hwaccm.s.fNestedPaging);
2149 Assert(!pVM->hwaccm.s.vmx.fVPID);
2150
2151 /* Deal with tagged TLBs if VPID or EPT is supported. */
2152 pCpu = HWACCMR0GetCurrentCpu();
2153 /* Force a TLB flush for the first world switch if the current cpu differs from the one we ran on last. */
2154 /* Note that this can happen both for start and resume due to long jumps back to ring 3. */
2155 if ( pVCpu->hwaccm.s.idLastCpu != pCpu->idCpu
2156 /* if the tlb flush count has changed, another VM has flushed the TLB of this cpu, so we can't use our current ASID anymore. */
2157 || pVCpu->hwaccm.s.cTLBFlushes != pCpu->cTLBFlushes)
2158 {
2159 /* Force a TLB flush on VM entry. */
2160 pVCpu->hwaccm.s.fForceTLBFlush = true;
2161 }
2162 /* Disabled because this has triggered every time I have suspended my
2163 * laptop with a VM running for the past three months or more. */
2164 // else
2165 // Assert(!pCpu->fFlushTLB);
2166
2167 /* Check for tlb shootdown flushes. */
2168 if (VMCPU_FF_TESTANDCLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2169 pVCpu->hwaccm.s.fForceTLBFlush = true;
2170
2171 pVCpu->hwaccm.s.idLastCpu = pCpu->idCpu;
2172 pCpu->fFlushTLB = false;
2173
2174 if (pVCpu->hwaccm.s.fForceTLBFlush)
2175 {
2176 vmxR0FlushEPT(pVM, pVCpu, pVM->hwaccm.s.vmx.enmFlushContext, 0);
2177 }
2178 else
2179 if (VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
2180 {
2181 /* Deal with pending TLB shootdown actions which were queued when we were not executing code. */
2182 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatTlbShootdown);
2183
2184 for (unsigned i=0;i<pVCpu->hwaccm.s.TlbShootdown.cPages;i++)
2185 {
2186 /* aTlbShootdownPages contains physical addresses in this case. */
2187 vmxR0FlushEPT(pVM, pVCpu, pVM->hwaccm.s.vmx.enmFlushPage, pVCpu->hwaccm.s.TlbShootdown.aPages[i]);
2188 }
2189 }
2190 pVCpu->hwaccm.s.TlbShootdown.cPages= 0;
2191 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
2192
2193#ifdef VBOX_WITH_STATISTICS
2194 if (pVCpu->hwaccm.s.fForceTLBFlush)
2195 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatFlushTLBWorldSwitch);
2196 else
2197 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatNoFlushTLBWorldSwitch);
2198#endif
2199}
2200
2201#ifdef HWACCM_VTX_WITH_VPID
2202/**
2203 * Setup the tagged TLB for VPID
2204 *
2205 * @returns VBox status code.
2206 * @param pVM The VM to operate on.
2207 * @param pVCpu The VMCPU to operate on.
2208 */
2209static void vmxR0SetupTLBVPID(PVM pVM, PVMCPU pVCpu)
2210{
2211 PHMGLOBLCPUINFO pCpu;
2212
2213 Assert(pVM->hwaccm.s.vmx.fVPID);
2214 Assert(!pVM->hwaccm.s.fNestedPaging);
2215
2216 /* Deal with tagged TLBs if VPID or EPT is supported. */
2217 pCpu = HWACCMR0GetCurrentCpu();
2218 /* Force a TLB flush for the first world switch if the current cpu differs from the one we ran on last. */
2219 /* Note that this can happen both for start and resume due to long jumps back to ring 3. */
2220 if ( pVCpu->hwaccm.s.idLastCpu != pCpu->idCpu
2221 /* if the tlb flush count has changed, another VM has flushed the TLB of this cpu, so we can't use our current ASID anymore. */
2222 || pVCpu->hwaccm.s.cTLBFlushes != pCpu->cTLBFlushes)
2223 {
2224 /* Force a TLB flush on VM entry. */
2225 pVCpu->hwaccm.s.fForceTLBFlush = true;
2226 }
2227 else
2228 Assert(!pCpu->fFlushTLB);
2229
2230 pVCpu->hwaccm.s.idLastCpu = pCpu->idCpu;
2231
2232 /* Check for tlb shootdown flushes. */
2233 if (VMCPU_FF_TESTANDCLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2234 pVCpu->hwaccm.s.fForceTLBFlush = true;
2235
2236 /* Make sure we flush the TLB when required. Switch ASID to achieve the same thing, but without actually flushing the whole TLB (which is expensive). */
2237 if (pVCpu->hwaccm.s.fForceTLBFlush)
2238 {
2239 if ( ++pCpu->uCurrentASID >= pVM->hwaccm.s.uMaxASID
2240 || pCpu->fFlushTLB)
2241 {
2242 pCpu->fFlushTLB = false;
2243 pCpu->uCurrentASID = 1; /* start at 1; host uses 0 */
2244 pCpu->cTLBFlushes++;
2245 vmxR0FlushVPID(pVM, pVCpu, VMX_FLUSH_ALL_CONTEXTS, 0);
2246 }
2247 else
2248 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatFlushASID);
2249
2250 pVCpu->hwaccm.s.fForceTLBFlush = false;
2251 pVCpu->hwaccm.s.cTLBFlushes = pCpu->cTLBFlushes;
2252 pVCpu->hwaccm.s.uCurrentASID = pCpu->uCurrentASID;
2253 }
2254 else
2255 {
2256 Assert(!pCpu->fFlushTLB);
2257 Assert(pVCpu->hwaccm.s.uCurrentASID && pCpu->uCurrentASID);
2258
2259 if (VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
2260 {
2261 /* Deal with pending TLB shootdown actions which were queued when we were not executing code. */
2262 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatTlbShootdown);
2263 for (unsigned i = 0; i < pVCpu->hwaccm.s.TlbShootdown.cPages; i++)
2264 vmxR0FlushVPID(pVM, pVCpu, pVM->hwaccm.s.vmx.enmFlushPage, pVCpu->hwaccm.s.TlbShootdown.aPages[i]);
2265 }
2266 }
2267 pVCpu->hwaccm.s.TlbShootdown.cPages = 0;
2268 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
2269
2270 AssertMsg(pVCpu->hwaccm.s.cTLBFlushes == pCpu->cTLBFlushes, ("Flush count mismatch for cpu %d (%x vs %x)\n", pCpu->idCpu, pVCpu->hwaccm.s.cTLBFlushes, pCpu->cTLBFlushes));
2271 AssertMsg(pCpu->uCurrentASID >= 1 && pCpu->uCurrentASID < pVM->hwaccm.s.uMaxASID, ("cpu%d uCurrentASID = %x\n", pCpu->idCpu, pCpu->uCurrentASID));
2272 AssertMsg(pVCpu->hwaccm.s.uCurrentASID >= 1 && pVCpu->hwaccm.s.uCurrentASID < pVM->hwaccm.s.uMaxASID, ("cpu%d VM uCurrentASID = %x\n", pCpu->idCpu, pVCpu->hwaccm.s.uCurrentASID));
2273
2274 int rc = VMXWriteVMCS(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hwaccm.s.uCurrentASID);
2275 AssertRC(rc);
2276
2277 if (pVCpu->hwaccm.s.fForceTLBFlush)
2278 vmxR0FlushVPID(pVM, pVCpu, pVM->hwaccm.s.vmx.enmFlushContext, 0);
2279
2280# ifdef VBOX_WITH_STATISTICS
2281 if (pVCpu->hwaccm.s.fForceTLBFlush)
2282 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatFlushTLBWorldSwitch);
2283 else
2284 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatNoFlushTLBWorldSwitch);
2285# endif
2286}
2287#endif /* HWACCM_VTX_WITH_VPID */
2288
2289/**
2290 * Runs guest code in a VT-x VM.
2291 *
2292 * @returns VBox status code.
2293 * @param pVM The VM to operate on.
2294 * @param pVCpu The VMCPU to operate on.
2295 * @param pCtx Guest context
2296 */
2297VMMR0DECL(int) VMXR0RunGuestCode(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
2298{
2299 STAM_PROFILE_ADV_START(&pVCpu->hwaccm.s.StatEntry, x);
2300 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hwaccm.s.StatExit1);
2301 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hwaccm.s.StatExit2);
2302
2303 VBOXSTRICTRC rc = VINF_SUCCESS;
2304 int rc2;
2305 RTGCUINTREG val;
2306 RTGCUINTREG exitReason = (RTGCUINTREG)VMX_EXIT_INVALID;
2307 RTGCUINTREG instrError, cbInstr;
2308 RTGCUINTPTR exitQualification = 0;
2309 RTGCUINTPTR intInfo = 0; /* shut up buggy gcc 4 */
2310 RTGCUINTPTR errCode, instrInfo;
2311 bool fSetupTPRCaching = false;
2312 uint64_t u64OldLSTAR = 0;
2313 uint8_t u8LastTPR = 0;
2314 RTCCUINTREG uOldEFlags = ~(RTCCUINTREG)0;
2315 unsigned cResume = 0;
2316#ifdef VBOX_STRICT
2317 RTCPUID idCpuCheck;
2318 bool fWasInLongMode = false;
2319#endif
2320#ifdef VBOX_HIGH_RES_TIMERS_HACK_IN_RING0
2321 uint64_t u64LastTime = RTTimeMilliTS();
2322#endif
2323
2324 Assert(!(pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC) || (pVCpu->hwaccm.s.vmx.pbVAPIC && pVM->hwaccm.s.vmx.pAPIC));
2325
2326 /* Check if we need to use TPR shadowing. */
2327 if ( CPUMIsGuestInLongModeEx(pCtx)
2328 || ( ((pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC) || pVM->hwaccm.s.fTRPPatchingAllowed)
2329 && pVM->hwaccm.s.fHasIoApic)
2330 )
2331 {
2332 fSetupTPRCaching = true;
2333 }
2334
2335 Log2(("\nE"));
2336
2337#ifdef VBOX_STRICT
2338 {
2339 RTCCUINTREG val2;
2340
2341 rc2 = VMXReadVMCS(VMX_VMCS_CTRL_PIN_EXEC_CONTROLS, &val2);
2342 AssertRC(rc2);
2343 Log2(("VMX_VMCS_CTRL_PIN_EXEC_CONTROLS = %08x\n", val2));
2344
2345 /* allowed zero */
2346 if ((val2 & pVM->hwaccm.s.vmx.msr.vmx_pin_ctls.n.disallowed0) != pVM->hwaccm.s.vmx.msr.vmx_pin_ctls.n.disallowed0)
2347 Log(("Invalid VMX_VMCS_CTRL_PIN_EXEC_CONTROLS: zero\n"));
2348
2349 /* allowed one */
2350 if ((val2 & ~pVM->hwaccm.s.vmx.msr.vmx_pin_ctls.n.allowed1) != 0)
2351 Log(("Invalid VMX_VMCS_CTRL_PIN_EXEC_CONTROLS: one\n"));
2352
2353 rc2 = VMXReadVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, &val2);
2354 AssertRC(rc2);
2355 Log2(("VMX_VMCS_CTRL_PROC_EXEC_CONTROLS = %08x\n", val2));
2356
2357 /* Must be set according to the MSR, but can be cleared in case of EPT. */
2358 if (pVM->hwaccm.s.fNestedPaging)
2359 val2 |= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INVLPG_EXIT
2360 | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
2361 | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT;
2362
2363 /* allowed zero */
2364 if ((val2 & pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.disallowed0) != pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.disallowed0)
2365 Log(("Invalid VMX_VMCS_CTRL_PROC_EXEC_CONTROLS: zero\n"));
2366
2367 /* allowed one */
2368 if ((val2 & ~pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1) != 0)
2369 Log(("Invalid VMX_VMCS_CTRL_PROC_EXEC_CONTROLS: one\n"));
2370
2371 rc2 = VMXReadVMCS(VMX_VMCS_CTRL_ENTRY_CONTROLS, &val2);
2372 AssertRC(rc2);
2373 Log2(("VMX_VMCS_CTRL_ENTRY_CONTROLS = %08x\n", val2));
2374
2375 /* allowed zero */
2376 if ((val2 & pVM->hwaccm.s.vmx.msr.vmx_entry.n.disallowed0) != pVM->hwaccm.s.vmx.msr.vmx_entry.n.disallowed0)
2377 Log(("Invalid VMX_VMCS_CTRL_ENTRY_CONTROLS: zero\n"));
2378
2379 /* allowed one */
2380 if ((val2 & ~pVM->hwaccm.s.vmx.msr.vmx_entry.n.allowed1) != 0)
2381 Log(("Invalid VMX_VMCS_CTRL_ENTRY_CONTROLS: one\n"));
2382
2383 rc2 = VMXReadVMCS(VMX_VMCS_CTRL_EXIT_CONTROLS, &val2);
2384 AssertRC(rc2);
2385 Log2(("VMX_VMCS_CTRL_EXIT_CONTROLS = %08x\n", val2));
2386
2387 /* allowed zero */
2388 if ((val2 & pVM->hwaccm.s.vmx.msr.vmx_exit.n.disallowed0) != pVM->hwaccm.s.vmx.msr.vmx_exit.n.disallowed0)
2389 Log(("Invalid VMX_VMCS_CTRL_EXIT_CONTROLS: zero\n"));
2390
2391 /* allowed one */
2392 if ((val2 & ~pVM->hwaccm.s.vmx.msr.vmx_exit.n.allowed1) != 0)
2393 Log(("Invalid VMX_VMCS_CTRL_EXIT_CONTROLS: one\n"));
2394 }
2395 fWasInLongMode = CPUMIsGuestInLongModeEx(pCtx);
2396#endif /* VBOX_STRICT */
2397
2398#ifdef VBOX_WITH_CRASHDUMP_MAGIC
2399 pVCpu->hwaccm.s.vmx.VMCSCache.u64TimeEntry = RTTimeNanoTS();
2400#endif
2401
2402 /* We can jump to this point to resume execution after determining that a VM-exit is innocent.
2403 */
2404ResumeExecution:
2405 if (!STAM_REL_PROFILE_ADV_IS_RUNNING(&pVCpu->hwaccm.s.StatEntry))
2406 STAM_REL_PROFILE_ADV_STOP_START(&pVCpu->hwaccm.s.StatExit2, &pVCpu->hwaccm.s.StatEntry, x);
2407 AssertMsg(pVCpu->hwaccm.s.idEnteredCpu == RTMpCpuId(),
2408 ("Expected %d, I'm %d; cResume=%d exitReason=%RGv exitQualification=%RGv\n",
2409 (int)pVCpu->hwaccm.s.idEnteredCpu, (int)RTMpCpuId(), cResume, exitReason, exitQualification));
2410 Assert(!HWACCMR0SuspendPending());
2411 /* Not allowed to switch modes without reloading the host state (32->64 switcher)!! */
2412 Assert(fWasInLongMode == CPUMIsGuestInLongModeEx(pCtx));
2413
2414 /* Safety precaution; looping for too long here can have a very bad effect on the host */
2415 if (RT_UNLIKELY(++cResume > pVM->hwaccm.s.cMaxResumeLoops))
2416 {
2417 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitMaxResume);
2418 rc = VINF_EM_RAW_INTERRUPT;
2419 goto end;
2420 }
2421
2422 /* Check for irq inhibition due to instruction fusing (sti, mov ss). */
2423 if (VMCPU_FF_ISSET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
2424 {
2425 Log(("VM_FF_INHIBIT_INTERRUPTS at %RGv successor %RGv\n", (RTGCPTR)pCtx->rip, EMGetInhibitInterruptsPC(pVCpu)));
2426 if (pCtx->rip != EMGetInhibitInterruptsPC(pVCpu))
2427 {
2428 /* Note: we intentionally don't clear VM_FF_INHIBIT_INTERRUPTS here.
2429 * Before we are able to execute this instruction in raw mode (iret to guest code) an external interrupt might
2430 * force a world switch again. Possibly allowing a guest interrupt to be dispatched in the process. This could
2431 * break the guest. Sounds very unlikely, but such timing sensitive problems are not as rare as you might think.
2432 */
2433 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
2434 /* Irq inhibition is no longer active; clear the corresponding VMX state. */
2435 rc2 = VMXWriteVMCS(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, 0);
2436 AssertRC(rc2);
2437 }
2438 }
2439 else
2440 {
2441 /* Irq inhibition is no longer active; clear the corresponding VMX state. */
2442 rc2 = VMXWriteVMCS(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, 0);
2443 AssertRC(rc2);
2444 }
2445
2446#ifdef VBOX_HIGH_RES_TIMERS_HACK_IN_RING0
2447 if (RT_UNLIKELY((cResume & 0xf) == 0))
2448 {
2449 uint64_t u64CurTime = RTTimeMilliTS();
2450
2451 if (RT_UNLIKELY(u64CurTime > u64LastTime))
2452 {
2453 u64LastTime = u64CurTime;
2454 TMTimerPollVoid(pVM, pVCpu);
2455 }
2456 }
2457#endif
2458
2459 /* Check for pending actions that force us to go back to ring 3. */
2460 if ( VM_FF_ISPENDING(pVM, VM_FF_HWACCM_TO_R3_MASK | VM_FF_REQUEST | VM_FF_PGM_POOL_FLUSH_PENDING | VM_FF_PDM_DMA)
2461 || VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_HWACCM_TO_R3_MASK | VMCPU_FF_PGM_SYNC_CR3 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL | VMCPU_FF_REQUEST))
2462 {
2463 /* Check if a sync operation is pending. */
2464 if (VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
2465 {
2466 rc = PGMSyncCR3(pVCpu, pCtx->cr0, pCtx->cr3, pCtx->cr4, VMCPU_FF_ISSET(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
2467 if (rc != VINF_SUCCESS)
2468 {
2469 AssertRC(VBOXSTRICTRC_VAL(rc));
2470 Log(("Pending pool sync is forcing us back to ring 3; rc=%d\n", VBOXSTRICTRC_VAL(rc)));
2471 goto end;
2472 }
2473 }
2474
2475#ifdef DEBUG
2476 /* Intercept X86_XCPT_DB if stepping is enabled */
2477 if (!DBGFIsStepping(pVCpu))
2478#endif
2479 {
2480 if ( VM_FF_ISPENDING(pVM, VM_FF_HWACCM_TO_R3_MASK)
2481 || VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_HWACCM_TO_R3_MASK))
2482 {
2483 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatSwitchToR3);
2484 rc = RT_UNLIKELY(VM_FF_ISPENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
2485 goto end;
2486 }
2487 }
2488
2489 /* Pending request packets might contain actions that need immediate attention, such as pending hardware interrupts. */
2490 if ( VM_FF_ISPENDING(pVM, VM_FF_REQUEST)
2491 || VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_REQUEST))
2492 {
2493 rc = VINF_EM_PENDING_REQUEST;
2494 goto end;
2495 }
2496
2497 /* Check if a pgm pool flush is in progress. */
2498 if (VM_FF_ISPENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
2499 {
2500 rc = VINF_PGM_POOL_FLUSH_PENDING;
2501 goto end;
2502 }
2503
2504 /* Check if DMA work is pending (2nd+ run). */
2505 if (VM_FF_ISPENDING(pVM, VM_FF_PDM_DMA) && cResume > 1)
2506 {
2507 rc = VINF_EM_RAW_TO_R3;
2508 goto end;
2509 }
2510 }
2511
2512#ifdef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
2513 /*
2514 * Exit to ring-3 preemption/work is pending.
2515 *
2516 * Interrupts are disabled before the call to make sure we don't miss any interrupt
2517 * that would flag preemption (IPI, timer tick, ++). (Would've been nice to do this
2518 * further down, but VMXR0CheckPendingInterrupt makes that impossible.)
2519 *
2520 * Note! Interrupts must be disabled done *before* we check for TLB flushes; TLB
2521 * shootdowns rely on this.
2522 */
2523 uOldEFlags = ASMIntDisableFlags();
2524 if (RTThreadPreemptIsPending(NIL_RTTHREAD))
2525 {
2526 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitPreemptPending);
2527 rc = VINF_EM_RAW_INTERRUPT;
2528 goto end;
2529 }
2530 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
2531#endif
2532
2533 /* When external interrupts are pending, we should exit the VM when IF is set. */
2534 /* Note! *After* VM_FF_INHIBIT_INTERRUPTS check!!! */
2535 rc = VMXR0CheckPendingInterrupt(pVM, pVCpu, pCtx);
2536 if (RT_FAILURE(rc))
2537 goto end;
2538
2539 /** @todo check timers?? */
2540
2541 /* TPR caching using CR8 is only available in 64 bits mode */
2542 /* Note the 32 bits exception for AMD (X86_CPUID_AMD_FEATURE_ECX_CR8L), but that appears missing in Intel CPUs */
2543 /* Note: we can't do this in LoadGuestState as PDMApicGetTPR can jump back to ring 3 (lock)!!!!! (no longer true) */
2544 /**
2545 * @todo query and update the TPR only when it could have been changed (mmio access & wrmsr (x2apic))
2546 */
2547 if (fSetupTPRCaching)
2548 {
2549 /* TPR caching in CR8 */
2550 bool fPending;
2551
2552 rc2 = PDMApicGetTPR(pVCpu, &u8LastTPR, &fPending);
2553 AssertRC(rc2);
2554 /* The TPR can be found at offset 0x80 in the APIC mmio page. */
2555 pVCpu->hwaccm.s.vmx.pbVAPIC[0x80] = u8LastTPR;
2556
2557 /* Two options here:
2558 * - external interrupt pending, but masked by the TPR value.
2559 * -> a CR8 update that lower the current TPR value should cause an exit
2560 * - no pending interrupts
2561 * -> We don't need to be explicitely notified. There are enough world switches for detecting pending interrupts.
2562 */
2563 rc = VMXWriteVMCS(VMX_VMCS_CTRL_TPR_THRESHOLD, (fPending) ? (u8LastTPR >> 4) : 0); /* cr8 bits 3-0 correspond to bits 7-4 of the task priority mmio register. */
2564 AssertRC(VBOXSTRICTRC_VAL(rc));
2565
2566 if (pVM->hwaccm.s.fTPRPatchingActive)
2567 {
2568 Assert(!CPUMIsGuestInLongModeEx(pCtx));
2569 /* Our patch code uses LSTAR for TPR caching. */
2570 pCtx->msrLSTAR = u8LastTPR;
2571
2572 if (fPending)
2573 {
2574 /* A TPR change could activate a pending interrupt, so catch lstar writes. */
2575 vmxR0SetMSRPermission(pVCpu, MSR_K8_LSTAR, true, false);
2576 }
2577 else
2578 {
2579 /* No interrupts are pending, so we don't need to be explicitely notified.
2580 * There are enough world switches for detecting pending interrupts.
2581 */
2582 vmxR0SetMSRPermission(pVCpu, MSR_K8_LSTAR, true, true);
2583 }
2584 }
2585 }
2586
2587#if defined(HWACCM_VTX_WITH_EPT) && defined(LOG_ENABLED)
2588 if ( pVM->hwaccm.s.fNestedPaging
2589# ifdef HWACCM_VTX_WITH_VPID
2590 || pVM->hwaccm.s.vmx.fVPID
2591# endif /* HWACCM_VTX_WITH_VPID */
2592 )
2593 {
2594 PHMGLOBLCPUINFO pCpu;
2595
2596 pCpu = HWACCMR0GetCurrentCpu();
2597 if ( pVCpu->hwaccm.s.idLastCpu != pCpu->idCpu
2598 || pVCpu->hwaccm.s.cTLBFlushes != pCpu->cTLBFlushes)
2599 {
2600 if (pVCpu->hwaccm.s.idLastCpu != pCpu->idCpu)
2601 LogFlow(("Force TLB flush due to rescheduling to a different cpu (%d vs %d)\n", pVCpu->hwaccm.s.idLastCpu, pCpu->idCpu));
2602 else
2603 LogFlow(("Force TLB flush due to changed TLB flush count (%x vs %x)\n", pVCpu->hwaccm.s.cTLBFlushes, pCpu->cTLBFlushes));
2604 }
2605 if (pCpu->fFlushTLB)
2606 LogFlow(("Force TLB flush: first time cpu %d is used -> flush\n", pCpu->idCpu));
2607 else
2608 if (pVCpu->hwaccm.s.fForceTLBFlush)
2609 LogFlow(("Manual TLB flush\n"));
2610 }
2611#endif
2612#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
2613 PGMRZDynMapFlushAutoSet(pVCpu);
2614#endif
2615
2616 /*
2617 * NOTE: DO NOT DO ANYTHING AFTER THIS POINT THAT MIGHT JUMP BACK TO RING 3!
2618 * (until the actual world switch)
2619 */
2620#ifdef VBOX_STRICT
2621 idCpuCheck = RTMpCpuId();
2622#endif
2623#ifdef LOG_ENABLED
2624 VMMR0LogFlushDisable(pVCpu);
2625#endif
2626 /* Save the host state first. */
2627 if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_HOST_CONTEXT)
2628 {
2629 rc = VMXR0SaveHostState(pVM, pVCpu);
2630 if (RT_UNLIKELY(rc != VINF_SUCCESS))
2631 {
2632 VMMR0LogFlushEnable(pVCpu);
2633 goto end;
2634 }
2635 }
2636
2637 /* Load the guest state */
2638 if (!pVCpu->hwaccm.s.fContextUseFlags)
2639 {
2640 VMXR0LoadMinimalGuestState(pVM, pVCpu, pCtx);
2641 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatLoadMinimal);
2642 }
2643 else
2644 {
2645 rc = VMXR0LoadGuestState(pVM, pVCpu, pCtx);
2646 if (RT_UNLIKELY(rc != VINF_SUCCESS))
2647 {
2648 VMMR0LogFlushEnable(pVCpu);
2649 goto end;
2650 }
2651 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatLoadFull);
2652 }
2653
2654#ifndef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
2655 /* Disable interrupts to make sure a poke will interrupt execution.
2656 * This must be done *before* we check for TLB flushes; TLB shootdowns rely on this.
2657 */
2658 uOldEFlags = ASMIntDisableFlags();
2659 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
2660#endif
2661
2662 /* Non-register state Guest Context */
2663 /** @todo change me according to cpu state */
2664 rc2 = VMXWriteVMCS(VMX_VMCS32_GUEST_ACTIVITY_STATE, VMX_CMS_GUEST_ACTIVITY_ACTIVE);
2665 AssertRC(rc2);
2666
2667 /* Set TLB flush state as checked until we return from the world switch. */
2668 ASMAtomicWriteBool(&pVCpu->hwaccm.s.fCheckedTLBFlush, true);
2669 /* Deal with tagged TLB setup and invalidation. */
2670 pVM->hwaccm.s.vmx.pfnSetupTaggedTLB(pVM, pVCpu);
2671
2672 /* Manual save and restore:
2673 * - General purpose registers except RIP, RSP
2674 *
2675 * Trashed:
2676 * - CR2 (we don't care)
2677 * - LDTR (reset to 0)
2678 * - DRx (presumably not changed at all)
2679 * - DR7 (reset to 0x400)
2680 * - EFLAGS (reset to RT_BIT(1); not relevant)
2681 *
2682 */
2683
2684 /* All done! Let's start VM execution. */
2685 STAM_PROFILE_ADV_STOP_START(&pVCpu->hwaccm.s.StatEntry, &pVCpu->hwaccm.s.StatInGC, x);
2686 Assert(idCpuCheck == RTMpCpuId());
2687
2688#ifdef VBOX_WITH_CRASHDUMP_MAGIC
2689 pVCpu->hwaccm.s.vmx.VMCSCache.cResume = cResume;
2690 pVCpu->hwaccm.s.vmx.VMCSCache.u64TimeSwitch = RTTimeNanoTS();
2691#endif
2692
2693 /* Save the current TPR value in the LSTAR msr so our patches can access it. */
2694 if (pVM->hwaccm.s.fTPRPatchingActive)
2695 {
2696 Assert(pVM->hwaccm.s.fTPRPatchingActive);
2697 u64OldLSTAR = ASMRdMsr(MSR_K8_LSTAR);
2698 ASMWrMsr(MSR_K8_LSTAR, u8LastTPR);
2699 }
2700
2701 TMNotifyStartOfExecution(pVCpu);
2702#ifdef VBOX_WITH_KERNEL_USING_XMM
2703 rc = hwaccmR0VMXStartVMWrapXMM(pVCpu->hwaccm.s.fResumeVM, pCtx, &pVCpu->hwaccm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hwaccm.s.vmx.pfnStartVM);
2704#else
2705 rc = pVCpu->hwaccm.s.vmx.pfnStartVM(pVCpu->hwaccm.s.fResumeVM, pCtx, &pVCpu->hwaccm.s.vmx.VMCSCache, pVM, pVCpu);
2706#endif
2707 ASMAtomicWriteBool(&pVCpu->hwaccm.s.fCheckedTLBFlush, false);
2708 ASMAtomicIncU32(&pVCpu->hwaccm.s.cWorldSwitchExits);
2709 /* Possibly the last TSC value seen by the guest (too high) (only when we're in tsc offset mode). */
2710 if (!(pVCpu->hwaccm.s.vmx.proc_ctls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT))
2711 TMCpuTickSetLastSeen(pVCpu, ASMReadTSC() + pVCpu->hwaccm.s.vmx.u64TSCOffset - 0x400 /* guestimate of world switch overhead in clock ticks */);
2712
2713 TMNotifyEndOfExecution(pVCpu);
2714 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED);
2715 Assert(!(ASMGetFlags() & X86_EFL_IF));
2716
2717 /* Restore the host LSTAR msr if the guest could have changed it. */
2718 if (pVM->hwaccm.s.fTPRPatchingActive)
2719 {
2720 Assert(pVM->hwaccm.s.fTPRPatchingActive);
2721 pVCpu->hwaccm.s.vmx.pbVAPIC[0x80] = pCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR);
2722 ASMWrMsr(MSR_K8_LSTAR, u64OldLSTAR);
2723 }
2724
2725 STAM_PROFILE_ADV_STOP_START(&pVCpu->hwaccm.s.StatInGC, &pVCpu->hwaccm.s.StatExit1, x);
2726 ASMSetFlags(uOldEFlags);
2727#ifdef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
2728 uOldEFlags = ~(RTCCUINTREG)0;
2729#endif
2730
2731 AssertMsg(!pVCpu->hwaccm.s.vmx.VMCSCache.Write.cValidEntries, ("pVCpu->hwaccm.s.vmx.VMCSCache.Write.cValidEntries=%d\n", pVCpu->hwaccm.s.vmx.VMCSCache.Write.cValidEntries));
2732
2733 /* In case we execute a goto ResumeExecution later on. */
2734 pVCpu->hwaccm.s.fResumeVM = true;
2735 pVCpu->hwaccm.s.fForceTLBFlush = false;
2736
2737 /*
2738 * !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
2739 * IMPORTANT: WE CAN'T DO ANY LOGGING OR OPERATIONS THAT CAN DO A LONGJMP BACK TO RING 3 *BEFORE* WE'VE SYNCED BACK (MOST OF) THE GUEST STATE
2740 * !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
2741 */
2742
2743 if (RT_UNLIKELY(rc != VINF_SUCCESS))
2744 {
2745 VMXR0ReportWorldSwitchError(pVM, pVCpu, rc, pCtx);
2746 VMMR0LogFlushEnable(pVCpu);
2747 goto end;
2748 }
2749
2750 /* Success. Query the guest state and figure out what has happened. */
2751
2752 /* Investigate why there was a VM-exit. */
2753 rc2 = VMXReadCachedVMCS(VMX_VMCS32_RO_EXIT_REASON, &exitReason);
2754 STAM_COUNTER_INC(&pVCpu->hwaccm.s.paStatExitReasonR0[exitReason & MASK_EXITREASON_STAT]);
2755
2756 exitReason &= 0xffff; /* bit 0-15 contain the exit code. */
2757 rc2 |= VMXReadCachedVMCS(VMX_VMCS32_RO_VM_INSTR_ERROR, &instrError);
2758 rc2 |= VMXReadCachedVMCS(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &cbInstr);
2759 rc2 |= VMXReadCachedVMCS(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &intInfo);
2760 /* might not be valid; depends on VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID. */
2761 rc2 |= VMXReadCachedVMCS(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERRCODE, &errCode);
2762 rc2 |= VMXReadCachedVMCS(VMX_VMCS32_RO_EXIT_INSTR_INFO, &instrInfo);
2763 rc2 |= VMXReadCachedVMCS(VMX_VMCS_RO_EXIT_QUALIFICATION, &exitQualification);
2764 AssertRC(rc2);
2765
2766 /* Sync back the guest state */
2767 rc2 = VMXR0SaveGuestState(pVM, pVCpu, pCtx);
2768 AssertRC(rc2);
2769
2770 /* Note! NOW IT'S SAFE FOR LOGGING! */
2771 VMMR0LogFlushEnable(pVCpu);
2772 Log2(("Raw exit reason %08x\n", exitReason));
2773
2774 /* Check if an injected event was interrupted prematurely. */
2775 rc2 = VMXReadCachedVMCS(VMX_VMCS32_RO_IDT_INFO, &val);
2776 AssertRC(rc2);
2777 pVCpu->hwaccm.s.Event.intInfo = VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(val);
2778 if ( VMX_EXIT_INTERRUPTION_INFO_VALID(pVCpu->hwaccm.s.Event.intInfo)
2779 /* Ignore 'int xx' as they'll be restarted anyway. */
2780 && VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hwaccm.s.Event.intInfo) != VMX_EXIT_INTERRUPTION_INFO_TYPE_SW
2781 /* Ignore software exceptions (such as int3) as they'll reoccur when we restart the instruction anyway. */
2782 && VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hwaccm.s.Event.intInfo) != VMX_EXIT_INTERRUPTION_INFO_TYPE_SWEXCPT)
2783 {
2784 Assert(!pVCpu->hwaccm.s.Event.fPending);
2785 pVCpu->hwaccm.s.Event.fPending = true;
2786 /* Error code present? */
2787 if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(pVCpu->hwaccm.s.Event.intInfo))
2788 {
2789 rc2 = VMXReadCachedVMCS(VMX_VMCS32_RO_IDT_ERRCODE, &val);
2790 AssertRC(rc2);
2791 pVCpu->hwaccm.s.Event.errCode = val;
2792 Log(("Pending inject %RX64 at %RGv exit=%08x intInfo=%08x exitQualification=%RGv pending error=%RX64\n", pVCpu->hwaccm.s.Event.intInfo, (RTGCPTR)pCtx->rip, exitReason, intInfo, exitQualification, val));
2793 }
2794 else
2795 {
2796 Log(("Pending inject %RX64 at %RGv exit=%08x intInfo=%08x exitQualification=%RGv\n", pVCpu->hwaccm.s.Event.intInfo, (RTGCPTR)pCtx->rip, exitReason, intInfo, exitQualification));
2797 pVCpu->hwaccm.s.Event.errCode = 0;
2798 }
2799 }
2800#ifdef VBOX_STRICT
2801 else
2802 if ( VMX_EXIT_INTERRUPTION_INFO_VALID(pVCpu->hwaccm.s.Event.intInfo)
2803 /* Ignore software exceptions (such as int3) as they're reoccur when we restart the instruction anyway. */
2804 && VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hwaccm.s.Event.intInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_SWEXCPT)
2805 {
2806 Log(("Ignore pending inject %RX64 at %RGv exit=%08x intInfo=%08x exitQualification=%RGv\n", pVCpu->hwaccm.s.Event.intInfo, (RTGCPTR)pCtx->rip, exitReason, intInfo, exitQualification));
2807 }
2808
2809 if (exitReason == VMX_EXIT_ERR_INVALID_GUEST_STATE)
2810 HWACCMDumpRegs(pVM, pVCpu, pCtx);
2811#endif
2812
2813 Log2(("E%d: New EIP=%x:%RGv\n", (uint32_t)exitReason, pCtx->cs, (RTGCPTR)pCtx->rip));
2814 Log2(("Exit reason %d, exitQualification %RGv\n", (uint32_t)exitReason, exitQualification));
2815 Log2(("instrInfo=%d instrError=%d instr length=%d\n", (uint32_t)instrInfo, (uint32_t)instrError, (uint32_t)cbInstr));
2816 Log2(("Interruption error code %d\n", (uint32_t)errCode));
2817 Log2(("IntInfo = %08x\n", (uint32_t)intInfo));
2818
2819 /* Sync back the TPR if it was changed. */
2820 if ( fSetupTPRCaching
2821 && u8LastTPR != pVCpu->hwaccm.s.vmx.pbVAPIC[0x80])
2822 {
2823 rc2 = PDMApicSetTPR(pVCpu, pVCpu->hwaccm.s.vmx.pbVAPIC[0x80]);
2824 AssertRC(rc2);
2825 }
2826
2827 STAM_PROFILE_ADV_STOP_START(&pVCpu->hwaccm.s.StatExit1, &pVCpu->hwaccm.s.StatExit2, x);
2828
2829 /* Some cases don't need a complete resync of the guest CPU state; handle them here. */
2830 Assert(rc == VINF_SUCCESS); /* might consider VERR_IPE_UNINITIALIZED_STATUS here later... */
2831 switch (exitReason)
2832 {
2833 case VMX_EXIT_EXCEPTION: /* 0 Exception or non-maskable interrupt (NMI). */
2834 case VMX_EXIT_EXTERNAL_IRQ: /* 1 External interrupt. */
2835 {
2836 uint32_t vector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(intInfo);
2837
2838 if (!VMX_EXIT_INTERRUPTION_INFO_VALID(intInfo))
2839 {
2840 Assert(exitReason == VMX_EXIT_EXTERNAL_IRQ);
2841#if 0 //def VBOX_WITH_VMMR0_DISABLE_PREEMPTION
2842 if ( RTThreadPreemptIsPendingTrusty()
2843 && !RTThreadPreemptIsPending(NIL_RTTHREAD))
2844 goto ResumeExecution;
2845#endif
2846 /* External interrupt; leave to allow it to be dispatched again. */
2847 rc = VINF_EM_RAW_INTERRUPT;
2848 break;
2849 }
2850 STAM_PROFILE_ADV_START(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
2851 switch (VMX_EXIT_INTERRUPTION_INFO_TYPE(intInfo))
2852 {
2853 case VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI: /* Non-maskable interrupt. */
2854 /* External interrupt; leave to allow it to be dispatched again. */
2855 rc = VINF_EM_RAW_INTERRUPT;
2856 break;
2857
2858 case VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT: /* External hardware interrupt. */
2859 AssertFailed(); /* can't come here; fails the first check. */
2860 break;
2861
2862 case VMX_EXIT_INTERRUPTION_INFO_TYPE_DBEXCPT: /* Unknown why we get this type for #DB */
2863 case VMX_EXIT_INTERRUPTION_INFO_TYPE_SWEXCPT: /* Software exception. (#BP or #OF) */
2864 Assert(vector == 1 || vector == 3 || vector == 4);
2865 /* no break */
2866 case VMX_EXIT_INTERRUPTION_INFO_TYPE_HWEXCPT: /* Hardware exception. */
2867 Log2(("Hardware/software interrupt %d\n", vector));
2868 switch (vector)
2869 {
2870 case X86_XCPT_NM:
2871 {
2872 Log(("#NM fault at %RGv error code %x\n", (RTGCPTR)pCtx->rip, errCode));
2873
2874 /** @todo don't intercept #NM exceptions anymore when we've activated the guest FPU state. */
2875 /* If we sync the FPU/XMM state on-demand, then we can continue execution as if nothing has happened. */
2876 rc = CPUMR0LoadGuestFPU(pVM, pVCpu, pCtx);
2877 if (rc == VINF_SUCCESS)
2878 {
2879 Assert(CPUMIsGuestFPUStateActive(pVCpu));
2880
2881 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitShadowNM);
2882
2883 /* Continue execution. */
2884 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_GUEST_CR0;
2885
2886 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
2887 goto ResumeExecution;
2888 }
2889
2890 Log(("Forward #NM fault to the guest\n"));
2891 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestNM);
2892 rc2 = VMXR0InjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), cbInstr, 0);
2893 AssertRC(rc2);
2894 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
2895 goto ResumeExecution;
2896 }
2897
2898 case X86_XCPT_PF: /* Page fault */
2899 {
2900#ifdef DEBUG
2901 if (pVM->hwaccm.s.fNestedPaging)
2902 { /* A genuine pagefault.
2903 * Forward the trap to the guest by injecting the exception and resuming execution.
2904 */
2905 Log(("Guest page fault at %RGv cr2=%RGv error code %RGv rsp=%RGv\n", (RTGCPTR)pCtx->rip, exitQualification, errCode, (RTGCPTR)pCtx->rsp));
2906
2907 Assert(CPUMIsGuestInPagedProtectedModeEx(pCtx));
2908
2909 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestPF);
2910
2911 /* Now we must update CR2. */
2912 pCtx->cr2 = exitQualification;
2913 rc2 = VMXR0InjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), cbInstr, errCode);
2914 AssertRC(rc2);
2915
2916 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
2917 goto ResumeExecution;
2918 }
2919#endif
2920 Assert(!pVM->hwaccm.s.fNestedPaging);
2921
2922#ifdef VBOX_HWACCM_WITH_GUEST_PATCHING
2923 /* Shortcut for APIC TPR reads and writes; 32 bits guests only */
2924 if ( pVM->hwaccm.s.fTRPPatchingAllowed
2925 && pVM->hwaccm.s.pGuestPatchMem
2926 && (exitQualification & 0xfff) == 0x080
2927 && !(errCode & X86_TRAP_PF_P) /* not present */
2928 && CPUMGetGuestCPL(pVCpu, CPUMCTX2CORE(pCtx)) == 0
2929 && !CPUMIsGuestInLongModeEx(pCtx)
2930 && pVM->hwaccm.s.cPatches < RT_ELEMENTS(pVM->hwaccm.s.aPatches))
2931 {
2932 RTGCPHYS GCPhysApicBase, GCPhys;
2933 PDMApicGetBase(pVM, &GCPhysApicBase); /* @todo cache this */
2934 GCPhysApicBase &= PAGE_BASE_GC_MASK;
2935
2936 rc = PGMGstGetPage(pVCpu, (RTGCPTR)exitQualification, NULL, &GCPhys);
2937 if ( rc == VINF_SUCCESS
2938 && GCPhys == GCPhysApicBase)
2939 {
2940 /* Only attempt to patch the instruction once. */
2941 PHWACCMTPRPATCH pPatch = (PHWACCMTPRPATCH)RTAvloU32Get(&pVM->hwaccm.s.PatchTree, (AVLOU32KEY)pCtx->eip);
2942 if (!pPatch)
2943 {
2944 rc = VINF_EM_HWACCM_PATCH_TPR_INSTR;
2945 break;
2946 }
2947 }
2948 }
2949#endif
2950
2951 Log2(("Page fault at %RGv error code %x\n", exitQualification, errCode));
2952 /* Exit qualification contains the linear address of the page fault. */
2953 TRPMAssertTrap(pVCpu, X86_XCPT_PF, TRPM_TRAP);
2954 TRPMSetErrorCode(pVCpu, errCode);
2955 TRPMSetFaultAddress(pVCpu, exitQualification);
2956
2957 /* Shortcut for APIC TPR reads and writes. */
2958 if ( (exitQualification & 0xfff) == 0x080
2959 && !(errCode & X86_TRAP_PF_P) /* not present */
2960 && fSetupTPRCaching
2961 && (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC))
2962 {
2963 RTGCPHYS GCPhysApicBase, GCPhys;
2964 PDMApicGetBase(pVM, &GCPhysApicBase); /* @todo cache this */
2965 GCPhysApicBase &= PAGE_BASE_GC_MASK;
2966
2967 rc = PGMGstGetPage(pVCpu, (RTGCPTR)exitQualification, NULL, &GCPhys);
2968 if ( rc == VINF_SUCCESS
2969 && GCPhys == GCPhysApicBase)
2970 {
2971 Log(("Enable VT-x virtual APIC access filtering\n"));
2972 rc2 = IOMMMIOMapMMIOHCPage(pVM, GCPhysApicBase, pVM->hwaccm.s.vmx.pAPICPhys, X86_PTE_RW | X86_PTE_P);
2973 AssertRC(rc2);
2974 }
2975 }
2976
2977 /* Forward it to our trap handler first, in case our shadow pages are out of sync. */
2978 rc = PGMTrap0eHandler(pVCpu, errCode, CPUMCTX2CORE(pCtx), (RTGCPTR)exitQualification);
2979 Log2(("PGMTrap0eHandler %RGv returned %Rrc\n", (RTGCPTR)pCtx->rip, VBOXSTRICTRC_VAL(rc)));
2980
2981 if (rc == VINF_SUCCESS)
2982 { /* We've successfully synced our shadow pages, so let's just continue execution. */
2983 Log2(("Shadow page fault at %RGv cr2=%RGv error code %x\n", (RTGCPTR)pCtx->rip, exitQualification ,errCode));
2984 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitShadowPF);
2985
2986 TRPMResetTrap(pVCpu);
2987 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
2988 goto ResumeExecution;
2989 }
2990 else
2991 if (rc == VINF_EM_RAW_GUEST_TRAP)
2992 { /* A genuine pagefault.
2993 * Forward the trap to the guest by injecting the exception and resuming execution.
2994 */
2995 Log2(("Forward page fault to the guest\n"));
2996
2997 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestPF);
2998 /* The error code might have been changed. */
2999 errCode = TRPMGetErrorCode(pVCpu);
3000
3001 TRPMResetTrap(pVCpu);
3002
3003 /* Now we must update CR2. */
3004 pCtx->cr2 = exitQualification;
3005 rc2 = VMXR0InjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), cbInstr, errCode);
3006 AssertRC(rc2);
3007
3008 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3009 goto ResumeExecution;
3010 }
3011#ifdef VBOX_STRICT
3012 if (rc != VINF_EM_RAW_EMULATE_INSTR && rc != VINF_EM_RAW_EMULATE_IO_BLOCK)
3013 Log2(("PGMTrap0eHandler failed with %d\n", VBOXSTRICTRC_VAL(rc)));
3014#endif
3015 /* Need to go back to the recompiler to emulate the instruction. */
3016 TRPMResetTrap(pVCpu);
3017 break;
3018 }
3019
3020 case X86_XCPT_MF: /* Floating point exception. */
3021 {
3022 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestMF);
3023 if (!(pCtx->cr0 & X86_CR0_NE))
3024 {
3025 /* old style FPU error reporting needs some extra work. */
3026 /** @todo don't fall back to the recompiler, but do it manually. */
3027 rc = VINF_EM_RAW_EMULATE_INSTR;
3028 break;
3029 }
3030 Log(("Trap %x at %04X:%RGv\n", vector, pCtx->cs, (RTGCPTR)pCtx->rip));
3031 rc2 = VMXR0InjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), cbInstr, errCode);
3032 AssertRC(rc2);
3033
3034 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3035 goto ResumeExecution;
3036 }
3037
3038 case X86_XCPT_DB: /* Debug exception. */
3039 {
3040 uint64_t uDR6;
3041
3042 /* DR6, DR7.GD and IA32_DEBUGCTL.LBR are not updated yet.
3043 *
3044 * Exit qualification bits:
3045 * 3:0 B0-B3 which breakpoint condition was met
3046 * 12:4 Reserved (0)
3047 * 13 BD - debug register access detected
3048 * 14 BS - single step execution or branch taken
3049 * 63:15 Reserved (0)
3050 */
3051 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestDB);
3052
3053 /* Note that we don't support guest and host-initiated debugging at the same time. */
3054
3055 uDR6 = X86_DR6_INIT_VAL;
3056 uDR6 |= (exitQualification & (X86_DR6_B0|X86_DR6_B1|X86_DR6_B2|X86_DR6_B3|X86_DR6_BD|X86_DR6_BS));
3057 rc = DBGFRZTrap01Handler(pVM, pVCpu, CPUMCTX2CORE(pCtx), uDR6);
3058 if (rc == VINF_EM_RAW_GUEST_TRAP)
3059 {
3060 /* Update DR6 here. */
3061 pCtx->dr[6] = uDR6;
3062
3063 /* Resync DR6 if the debug state is active. */
3064 if (CPUMIsGuestDebugStateActive(pVCpu))
3065 ASMSetDR6(pCtx->dr[6]);
3066
3067 /* X86_DR7_GD will be cleared if drx accesses should be trapped inside the guest. */
3068 pCtx->dr[7] &= ~X86_DR7_GD;
3069
3070 /* Paranoia. */
3071 pCtx->dr[7] &= 0xffffffff; /* upper 32 bits reserved */
3072 pCtx->dr[7] &= ~(RT_BIT(11) | RT_BIT(12) | RT_BIT(14) | RT_BIT(15)); /* must be zero */
3073 pCtx->dr[7] |= 0x400; /* must be one */
3074
3075 /* Resync DR7 */
3076 rc2 = VMXWriteVMCS64(VMX_VMCS64_GUEST_DR7, pCtx->dr[7]);
3077 AssertRC(rc2);
3078
3079 Log(("Trap %x (debug) at %RGv exit qualification %RX64 dr6=%x dr7=%x\n", vector, (RTGCPTR)pCtx->rip, exitQualification, (uint32_t)pCtx->dr[6], (uint32_t)pCtx->dr[7]));
3080 rc2 = VMXR0InjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), cbInstr, errCode);
3081 AssertRC(rc2);
3082
3083 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3084 goto ResumeExecution;
3085 }
3086 /* Return to ring 3 to deal with the debug exit code. */
3087 Log(("Debugger hardware BP at %04x:%RGv (rc=%Rrc)\n", pCtx->cs, pCtx->rip, VBOXSTRICTRC_VAL(rc)));
3088 break;
3089 }
3090
3091 case X86_XCPT_BP: /* Breakpoint. */
3092 {
3093 rc = DBGFRZTrap03Handler(pVM, pVCpu, CPUMCTX2CORE(pCtx));
3094 if (rc == VINF_EM_RAW_GUEST_TRAP)
3095 {
3096 Log(("Guest #BP at %04x:%RGv\n", pCtx->cs, pCtx->rip));
3097 rc2 = VMXR0InjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), cbInstr, errCode);
3098 AssertRC(rc2);
3099 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3100 goto ResumeExecution;
3101 }
3102 if (rc == VINF_SUCCESS)
3103 {
3104 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3105 goto ResumeExecution;
3106 }
3107 Log(("Debugger BP at %04x:%RGv (rc=%Rrc)\n", pCtx->cs, pCtx->rip, VBOXSTRICTRC_VAL(rc)));
3108 break;
3109 }
3110
3111 case X86_XCPT_GP: /* General protection failure exception.*/
3112 {
3113 uint32_t cbOp;
3114 uint32_t cbSize;
3115 PDISCPUSTATE pDis = &pVCpu->hwaccm.s.DisState;
3116
3117 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestGP);
3118#ifdef VBOX_STRICT
3119 if ( !CPUMIsGuestInRealModeEx(pCtx)
3120 || !pVM->hwaccm.s.vmx.pRealModeTSS)
3121 {
3122 Log(("Trap %x at %04X:%RGv errorCode=%RGv\n", vector, pCtx->cs, (RTGCPTR)pCtx->rip, errCode));
3123 rc2 = VMXR0InjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), cbInstr, errCode);
3124 AssertRC(rc2);
3125 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3126 goto ResumeExecution;
3127 }
3128#endif
3129 Assert(CPUMIsGuestInRealModeEx(pCtx));
3130
3131 LogFlow(("Real mode X86_XCPT_GP instruction emulation at %x:%RGv\n", pCtx->cs, (RTGCPTR)pCtx->rip));
3132
3133 rc2 = EMInterpretDisasOne(pVM, pVCpu, CPUMCTX2CORE(pCtx), pDis, &cbOp);
3134 if (RT_SUCCESS(rc2))
3135 {
3136 bool fUpdateRIP = true;
3137
3138 rc = VINF_SUCCESS;
3139 Assert(cbOp == pDis->opsize);
3140 switch (pDis->pCurInstr->opcode)
3141 {
3142 case OP_CLI:
3143 pCtx->eflags.Bits.u1IF = 0;
3144 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitCli);
3145 break;
3146
3147 case OP_STI:
3148 pCtx->eflags.Bits.u1IF = 1;
3149 EMSetInhibitInterruptsPC(pVCpu, pCtx->rip + pDis->opsize);
3150 Assert(VMCPU_FF_ISSET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
3151 rc2 = VMXWriteVMCS(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
3152 AssertRC(rc2);
3153 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitSti);
3154 break;
3155
3156 case OP_HLT:
3157 fUpdateRIP = false;
3158 rc = VINF_EM_HALT;
3159 pCtx->rip += pDis->opsize;
3160 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitHlt);
3161 break;
3162
3163 case OP_POPF:
3164 {
3165 RTGCPTR GCPtrStack;
3166 uint32_t cbParm;
3167 uint32_t uMask;
3168 X86EFLAGS eflags;
3169
3170 if (pDis->prefix & PREFIX_OPSIZE)
3171 {
3172 cbParm = 4;
3173 uMask = 0xffffffff;
3174 }
3175 else
3176 {
3177 cbParm = 2;
3178 uMask = 0xffff;
3179 }
3180
3181 rc2 = SELMToFlatEx(pVM, DIS_SELREG_SS, CPUMCTX2CORE(pCtx), pCtx->esp & uMask, 0, &GCPtrStack);
3182 if (RT_FAILURE(rc2))
3183 {
3184 rc = VERR_EM_INTERPRETER;
3185 break;
3186 }
3187 eflags.u = 0;
3188 rc2 = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &eflags.u, cbParm);
3189 if (RT_FAILURE(rc2))
3190 {
3191 rc = VERR_EM_INTERPRETER;
3192 break;
3193 }
3194 LogFlow(("POPF %x -> %RGv mask=%x\n", eflags.u, pCtx->rsp, uMask));
3195 pCtx->eflags.u = (pCtx->eflags.u & ~(X86_EFL_POPF_BITS & uMask)) | (eflags.u & X86_EFL_POPF_BITS & uMask);
3196 /* RF cleared when popped in real mode; see pushf description in AMD manual. */
3197 pCtx->eflags.Bits.u1RF = 0;
3198 pCtx->esp += cbParm;
3199 pCtx->esp &= uMask;
3200
3201 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitPopf);
3202 break;
3203 }
3204
3205 case OP_PUSHF:
3206 {
3207 RTGCPTR GCPtrStack;
3208 uint32_t cbParm;
3209 uint32_t uMask;
3210 X86EFLAGS eflags;
3211
3212 if (pDis->prefix & PREFIX_OPSIZE)
3213 {
3214 cbParm = 4;
3215 uMask = 0xffffffff;
3216 }
3217 else
3218 {
3219 cbParm = 2;
3220 uMask = 0xffff;
3221 }
3222
3223 rc2 = SELMToFlatEx(pVM, DIS_SELREG_SS, CPUMCTX2CORE(pCtx), (pCtx->esp - cbParm) & uMask, 0, &GCPtrStack);
3224 if (RT_FAILURE(rc2))
3225 {
3226 rc = VERR_EM_INTERPRETER;
3227 break;
3228 }
3229 eflags = pCtx->eflags;
3230 /* RF & VM cleared when pushed in real mode; see pushf description in AMD manual. */
3231 eflags.Bits.u1RF = 0;
3232 eflags.Bits.u1VM = 0;
3233
3234 rc2 = PGMPhysWrite(pVM, (RTGCPHYS)GCPtrStack, &eflags.u, cbParm);
3235 if (RT_FAILURE(rc2))
3236 {
3237 rc = VERR_EM_INTERPRETER;
3238 break;
3239 }
3240 LogFlow(("PUSHF %x -> %RGv\n", eflags.u, GCPtrStack));
3241 pCtx->esp -= cbParm;
3242 pCtx->esp &= uMask;
3243 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitPushf);
3244 break;
3245 }
3246
3247 case OP_IRET:
3248 {
3249 RTGCPTR GCPtrStack;
3250 uint32_t uMask = 0xffff;
3251 uint16_t aIretFrame[3];
3252
3253 if (pDis->prefix & (PREFIX_OPSIZE | PREFIX_ADDRSIZE))
3254 {
3255 rc = VERR_EM_INTERPRETER;
3256 break;
3257 }
3258
3259 rc2 = SELMToFlatEx(pVM, DIS_SELREG_SS, CPUMCTX2CORE(pCtx), pCtx->esp & uMask, 0, &GCPtrStack);
3260 if (RT_FAILURE(rc2))
3261 {
3262 rc = VERR_EM_INTERPRETER;
3263 break;
3264 }
3265 rc2 = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &aIretFrame[0], sizeof(aIretFrame));
3266 if (RT_FAILURE(rc2))
3267 {
3268 rc = VERR_EM_INTERPRETER;
3269 break;
3270 }
3271 pCtx->ip = aIretFrame[0];
3272 pCtx->cs = aIretFrame[1];
3273 pCtx->csHid.u64Base = pCtx->cs << 4;
3274 pCtx->eflags.u = (pCtx->eflags.u & ~(X86_EFL_POPF_BITS & uMask)) | (aIretFrame[2] & X86_EFL_POPF_BITS & uMask);
3275 pCtx->sp += sizeof(aIretFrame);
3276
3277 LogFlow(("iret to %04x:%x\n", pCtx->cs, pCtx->ip));
3278 fUpdateRIP = false;
3279 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitIret);
3280 break;
3281 }
3282
3283 case OP_INT:
3284 {
3285 uint32_t intInfo2;
3286
3287 LogFlow(("Realmode: INT %x\n", pDis->param1.parval & 0xff));
3288 intInfo2 = pDis->param1.parval & 0xff;
3289 intInfo2 |= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
3290 intInfo2 |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
3291
3292 rc = VMXR0InjectEvent(pVM, pVCpu, pCtx, intInfo2, cbOp, 0);
3293 AssertRC(VBOXSTRICTRC_VAL(rc));
3294 fUpdateRIP = false;
3295 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitInt);
3296 break;
3297 }
3298
3299 case OP_INTO:
3300 {
3301 if (pCtx->eflags.Bits.u1OF)
3302 {
3303 uint32_t intInfo2;
3304
3305 LogFlow(("Realmode: INTO\n"));
3306 intInfo2 = X86_XCPT_OF;
3307 intInfo2 |= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
3308 intInfo2 |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
3309
3310 rc = VMXR0InjectEvent(pVM, pVCpu, pCtx, intInfo2, cbOp, 0);
3311 AssertRC(VBOXSTRICTRC_VAL(rc));
3312 fUpdateRIP = false;
3313 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitInt);
3314 }
3315 break;
3316 }
3317
3318 case OP_INT3:
3319 {
3320 uint32_t intInfo2;
3321
3322 LogFlow(("Realmode: INT 3\n"));
3323 intInfo2 = 3;
3324 intInfo2 |= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
3325 intInfo2 |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
3326
3327 rc = VMXR0InjectEvent(pVM, pVCpu, pCtx, intInfo2, cbOp, 0);
3328 AssertRC(VBOXSTRICTRC_VAL(rc));
3329 fUpdateRIP = false;
3330 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitInt);
3331 break;
3332 }
3333
3334 default:
3335 rc = EMInterpretInstructionCPU(pVM, pVCpu, pDis, CPUMCTX2CORE(pCtx), 0, EMCODETYPE_SUPERVISOR, &cbSize);
3336 break;
3337 }
3338
3339 if (rc == VINF_SUCCESS)
3340 {
3341 if (fUpdateRIP)
3342 pCtx->rip += cbOp; /* Move on to the next instruction. */
3343
3344 /* lidt, lgdt can end up here. In the future crx changes as well. Just reload the whole context to be done with it. */
3345 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_ALL;
3346
3347 /* Only resume if successful. */
3348 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3349 goto ResumeExecution;
3350 }
3351 }
3352 else
3353 rc = VERR_EM_INTERPRETER;
3354
3355 AssertMsg(rc == VERR_EM_INTERPRETER || rc == VINF_PGM_CHANGE_MODE || rc == VINF_EM_HALT, ("Unexpected rc=%Rrc\n", VBOXSTRICTRC_VAL(rc)));
3356 break;
3357 }
3358
3359#ifdef VBOX_STRICT
3360 case X86_XCPT_XF: /* SIMD exception. */
3361 case X86_XCPT_DE: /* Divide error. */
3362 case X86_XCPT_UD: /* Unknown opcode exception. */
3363 case X86_XCPT_SS: /* Stack segment exception. */
3364 case X86_XCPT_NP: /* Segment not present exception. */
3365 {
3366 switch(vector)
3367 {
3368 case X86_XCPT_DE:
3369 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestDE);
3370 break;
3371 case X86_XCPT_UD:
3372 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestUD);
3373 break;
3374 case X86_XCPT_SS:
3375 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestSS);
3376 break;
3377 case X86_XCPT_NP:
3378 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestNP);
3379 break;
3380 }
3381
3382 Log(("Trap %x at %04X:%RGv\n", vector, pCtx->cs, (RTGCPTR)pCtx->rip));
3383 rc2 = VMXR0InjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), cbInstr, errCode);
3384 AssertRC(rc2);
3385
3386 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3387 goto ResumeExecution;
3388 }
3389#endif
3390 default:
3391 if ( CPUMIsGuestInRealModeEx(pCtx)
3392 && pVM->hwaccm.s.vmx.pRealModeTSS)
3393 {
3394 Log(("Real Mode Trap %x at %04x:%04X error code %x\n", vector, pCtx->cs, pCtx->eip, errCode));
3395 rc = VMXR0InjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), cbInstr, errCode);
3396 AssertRC(VBOXSTRICTRC_VAL(rc)); /* Strict RC check below. */
3397
3398 /* Go back to ring 3 in case of a triple fault. */
3399 if ( vector == X86_XCPT_DF
3400 && rc == VINF_EM_RESET)
3401 break;
3402
3403 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3404 goto ResumeExecution;
3405 }
3406 AssertMsgFailed(("Unexpected vm-exit caused by exception %x\n", vector));
3407 rc = VERR_VMX_UNEXPECTED_EXCEPTION;
3408 break;
3409 } /* switch (vector) */
3410
3411 break;
3412
3413 default:
3414 rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_CODE;
3415 AssertMsgFailed(("Unexpected interruption code %x\n", intInfo));
3416 break;
3417 }
3418
3419 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3420 break;
3421 }
3422
3423 case VMX_EXIT_EPT_VIOLATION: /* 48 EPT violation. An attempt to access memory with a guest-physical address was disallowed by the configuration of the EPT paging structures. */
3424 {
3425 RTGCPHYS GCPhys;
3426
3427 Assert(pVM->hwaccm.s.fNestedPaging);
3428
3429 rc2 = VMXReadVMCS64(VMX_VMCS_EXIT_PHYS_ADDR_FULL, &GCPhys);
3430 AssertRC(rc2);
3431 Assert(((exitQualification >> 7) & 3) != 2);
3432
3433 /* Determine the kind of violation. */
3434 errCode = 0;
3435 if (exitQualification & VMX_EXIT_QUALIFICATION_EPT_INSTR_FETCH)
3436 errCode |= X86_TRAP_PF_ID;
3437
3438 if (exitQualification & VMX_EXIT_QUALIFICATION_EPT_DATA_WRITE)
3439 errCode |= X86_TRAP_PF_RW;
3440
3441 /* If the page is present, then it's a page level protection fault. */
3442 if (exitQualification & VMX_EXIT_QUALIFICATION_EPT_ENTRY_PRESENT)
3443 {
3444 errCode |= X86_TRAP_PF_P;
3445 }
3446 else
3447 {
3448 /* Shortcut for APIC TPR reads and writes. */
3449 if ( (GCPhys & 0xfff) == 0x080
3450 && GCPhys > 0x1000000 /* to skip VGA frame buffer accesses */
3451 && fSetupTPRCaching
3452 && (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC))
3453 {
3454 RTGCPHYS GCPhysApicBase;
3455 PDMApicGetBase(pVM, &GCPhysApicBase); /* @todo cache this */
3456 GCPhysApicBase &= PAGE_BASE_GC_MASK;
3457 if (GCPhys == GCPhysApicBase + 0x80)
3458 {
3459 Log(("Enable VT-x virtual APIC access filtering\n"));
3460 rc2 = IOMMMIOMapMMIOHCPage(pVM, GCPhysApicBase, pVM->hwaccm.s.vmx.pAPICPhys, X86_PTE_RW | X86_PTE_P);
3461 AssertRC(rc2);
3462 }
3463 }
3464 }
3465 Log(("EPT Page fault %x at %RGp error code %x\n", (uint32_t)exitQualification, GCPhys, errCode));
3466
3467 /* GCPhys contains the guest physical address of the page fault. */
3468 TRPMAssertTrap(pVCpu, X86_XCPT_PF, TRPM_TRAP);
3469 TRPMSetErrorCode(pVCpu, errCode);
3470 TRPMSetFaultAddress(pVCpu, GCPhys);
3471
3472 /* Handle the pagefault trap for the nested shadow table. */
3473 rc = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, errCode, CPUMCTX2CORE(pCtx), GCPhys);
3474 Log2(("PGMR0Trap0eHandlerNestedPaging %RGv returned %Rrc\n", (RTGCPTR)pCtx->rip, VBOXSTRICTRC_VAL(rc)));
3475 if (rc == VINF_SUCCESS)
3476 { /* We've successfully synced our shadow pages, so let's just continue execution. */
3477 Log2(("Shadow page fault at %RGv cr2=%RGp error code %x\n", (RTGCPTR)pCtx->rip, exitQualification , errCode));
3478 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitReasonNPF);
3479
3480 TRPMResetTrap(pVCpu);
3481 goto ResumeExecution;
3482 }
3483
3484#ifdef VBOX_STRICT
3485 if (rc != VINF_EM_RAW_EMULATE_INSTR)
3486 LogFlow(("PGMTrap0eHandlerNestedPaging failed with %d\n", VBOXSTRICTRC_VAL(rc)));
3487#endif
3488 /* Need to go back to the recompiler to emulate the instruction. */
3489 TRPMResetTrap(pVCpu);
3490 break;
3491 }
3492
3493 case VMX_EXIT_EPT_MISCONFIG:
3494 {
3495 RTGCPHYS GCPhys;
3496
3497 Assert(pVM->hwaccm.s.fNestedPaging);
3498
3499 rc2 = VMXReadVMCS64(VMX_VMCS_EXIT_PHYS_ADDR_FULL, &GCPhys);
3500 AssertRC(rc2);
3501 Log(("VMX_EXIT_EPT_MISCONFIG for %RGp\n", GCPhys));
3502
3503 /* Shortcut for APIC TPR reads and writes. */
3504 if ( (GCPhys & 0xfff) == 0x080
3505 && GCPhys > 0x1000000 /* to skip VGA frame buffer accesses */
3506 && fSetupTPRCaching
3507 && (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC))
3508 {
3509 RTGCPHYS GCPhysApicBase;
3510 PDMApicGetBase(pVM, &GCPhysApicBase); /* @todo cache this */
3511 GCPhysApicBase &= PAGE_BASE_GC_MASK;
3512 if (GCPhys == GCPhysApicBase + 0x80)
3513 {
3514 Log(("Enable VT-x virtual APIC access filtering\n"));
3515 rc2 = IOMMMIOMapMMIOHCPage(pVM, GCPhysApicBase, pVM->hwaccm.s.vmx.pAPICPhys, X86_PTE_RW | X86_PTE_P);
3516 AssertRC(rc2);
3517 }
3518 }
3519
3520 rc = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pCtx), GCPhys, UINT32_MAX);
3521 if (rc == VINF_SUCCESS)
3522 {
3523 Log2(("PGMR0Trap0eHandlerNPMisconfig(,,,%RGp) at %RGv -> resume\n", GCPhys, (RTGCPTR)pCtx->rip));
3524 goto ResumeExecution;
3525 }
3526
3527 Log2(("PGMR0Trap0eHandlerNPMisconfig(,,,%RGp) at %RGv -> %Rrc\n", GCPhys, (RTGCPTR)pCtx->rip, VBOXSTRICTRC_VAL(rc)));
3528 break;
3529 }
3530
3531 case VMX_EXIT_IRQ_WINDOW: /* 7 Interrupt window. */
3532 /* Clear VM-exit on IF=1 change. */
3533 LogFlow(("VMX_EXIT_IRQ_WINDOW %RGv pending=%d IF=%d\n", (RTGCPTR)pCtx->rip, VMCPU_FF_ISPENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC|VMCPU_FF_INTERRUPT_PIC)), pCtx->eflags.Bits.u1IF));
3534 pVCpu->hwaccm.s.vmx.proc_ctls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_IRQ_WINDOW_EXIT;
3535 rc2 = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
3536 AssertRC(rc2);
3537 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitIrqWindow);
3538 goto ResumeExecution; /* we check for pending guest interrupts there */
3539
3540 case VMX_EXIT_WBINVD: /* 54 Guest software attempted to execute WBINVD. (conditional) */
3541 case VMX_EXIT_INVD: /* 13 Guest software attempted to execute INVD. (unconditional) */
3542 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitInvd);
3543 /* Skip instruction and continue directly. */
3544 pCtx->rip += cbInstr;
3545 /* Continue execution.*/
3546 goto ResumeExecution;
3547
3548 case VMX_EXIT_CPUID: /* 10 Guest software attempted to execute CPUID. */
3549 {
3550 Log2(("VMX: Cpuid %x\n", pCtx->eax));
3551 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitCpuid);
3552 rc = EMInterpretCpuId(pVM, pVCpu, CPUMCTX2CORE(pCtx));
3553 if (rc == VINF_SUCCESS)
3554 {
3555 /* Update EIP and continue execution. */
3556 Assert(cbInstr == 2);
3557 pCtx->rip += cbInstr;
3558 goto ResumeExecution;
3559 }
3560 AssertMsgFailed(("EMU: cpuid failed with %Rrc\n", VBOXSTRICTRC_VAL(rc)));
3561 rc = VINF_EM_RAW_EMULATE_INSTR;
3562 break;
3563 }
3564
3565 case VMX_EXIT_RDPMC: /* 15 Guest software attempted to execute RDPMC. */
3566 {
3567 Log2(("VMX: Rdpmc %x\n", pCtx->ecx));
3568 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitRdpmc);
3569 rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pCtx));
3570 if (rc == VINF_SUCCESS)
3571 {
3572 /* Update EIP and continue execution. */
3573 Assert(cbInstr == 2);
3574 pCtx->rip += cbInstr;
3575 goto ResumeExecution;
3576 }
3577 rc = VINF_EM_RAW_EMULATE_INSTR;
3578 break;
3579 }
3580
3581 case VMX_EXIT_RDTSC: /* 16 Guest software attempted to execute RDTSC. */
3582 {
3583 Log2(("VMX: Rdtsc\n"));
3584 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitRdtsc);
3585 rc = EMInterpretRdtsc(pVM, pVCpu, CPUMCTX2CORE(pCtx));
3586 if (rc == VINF_SUCCESS)
3587 {
3588 /* Update EIP and continue execution. */
3589 Assert(cbInstr == 2);
3590 pCtx->rip += cbInstr;
3591 goto ResumeExecution;
3592 }
3593 rc = VINF_EM_RAW_EMULATE_INSTR;
3594 break;
3595 }
3596
3597 case VMX_EXIT_INVPG: /* 14 Guest software attempted to execute INVPG. */
3598 {
3599 Log2(("VMX: invlpg\n"));
3600 Assert(!pVM->hwaccm.s.fNestedPaging);
3601
3602 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitInvpg);
3603 rc = EMInterpretInvlpg(pVM, pVCpu, CPUMCTX2CORE(pCtx), exitQualification);
3604 if (rc == VINF_SUCCESS)
3605 {
3606 /* Update EIP and continue execution. */
3607 pCtx->rip += cbInstr;
3608 goto ResumeExecution;
3609 }
3610 AssertMsg(rc == VERR_EM_INTERPRETER, ("EMU: invlpg %RGv failed with %Rrc\n", exitQualification, VBOXSTRICTRC_VAL(rc)));
3611 break;
3612 }
3613
3614 case VMX_EXIT_MONITOR: /* 39 Guest software attempted to execute MONITOR. */
3615 {
3616 Log2(("VMX: monitor\n"));
3617
3618 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitMonitor);
3619 rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pCtx));
3620 if (rc == VINF_SUCCESS)
3621 {
3622 /* Update EIP and continue execution. */
3623 pCtx->rip += cbInstr;
3624 goto ResumeExecution;
3625 }
3626 AssertMsg(rc == VERR_EM_INTERPRETER, ("EMU: monitor failed with %Rrc\n", VBOXSTRICTRC_VAL(rc)));
3627 break;
3628 }
3629
3630 case VMX_EXIT_WRMSR: /* 32 WRMSR. Guest software attempted to execute WRMSR. */
3631 /* When an interrupt is pending, we'll let MSR_K8_LSTAR writes fault in our TPR patch code. */
3632 if ( pVM->hwaccm.s.fTPRPatchingActive
3633 && pCtx->ecx == MSR_K8_LSTAR)
3634 {
3635 Assert(!CPUMIsGuestInLongModeEx(pCtx));
3636 if ((pCtx->eax & 0xff) != u8LastTPR)
3637 {
3638 Log(("VMX: Faulting MSR_K8_LSTAR write with new TPR value %x\n", pCtx->eax & 0xff));
3639
3640 /* Our patch code uses LSTAR for TPR caching. */
3641 rc2 = PDMApicSetTPR(pVCpu, pCtx->eax & 0xff);
3642 AssertRC(rc2);
3643 }
3644
3645 /* Skip the instruction and continue. */
3646 pCtx->rip += cbInstr; /* wrmsr = [0F 30] */
3647
3648 /* Only resume if successful. */
3649 goto ResumeExecution;
3650 }
3651 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_GUEST_MSR;
3652 /* no break */
3653 case VMX_EXIT_RDMSR: /* 31 RDMSR. Guest software attempted to execute RDMSR. */
3654 {
3655 uint32_t cbSize;
3656
3657 STAM_COUNTER_INC((exitReason == VMX_EXIT_RDMSR) ? &pVCpu->hwaccm.s.StatExitRdmsr : &pVCpu->hwaccm.s.StatExitWrmsr);
3658
3659 /* Note: the intel manual claims there's a REX version of RDMSR that's slightly different, so we play safe by completely disassembling the instruction. */
3660 Log2(("VMX: %s\n", (exitReason == VMX_EXIT_RDMSR) ? "rdmsr" : "wrmsr"));
3661 rc = EMInterpretInstruction(pVM, pVCpu, CPUMCTX2CORE(pCtx), 0, &cbSize);
3662 if (rc == VINF_SUCCESS)
3663 {
3664 /* EIP has been updated already. */
3665
3666 /* Only resume if successful. */
3667 goto ResumeExecution;
3668 }
3669 AssertMsg(rc == VERR_EM_INTERPRETER, ("EMU: %s failed with %Rrc\n", (exitReason == VMX_EXIT_RDMSR) ? "rdmsr" : "wrmsr", VBOXSTRICTRC_VAL(rc)));
3670 break;
3671 }
3672
3673 case VMX_EXIT_CRX_MOVE: /* 28 Control-register accesses. */
3674 {
3675 STAM_PROFILE_ADV_START(&pVCpu->hwaccm.s.StatExit2Sub2, y2);
3676
3677 switch (VMX_EXIT_QUALIFICATION_CRX_ACCESS(exitQualification))
3678 {
3679 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_WRITE:
3680 Log2(("VMX: %RGv mov cr%d, x\n", (RTGCPTR)pCtx->rip, VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification)));
3681 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitCRxWrite[VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification)]);
3682 rc = EMInterpretCRxWrite(pVM, pVCpu, CPUMCTX2CORE(pCtx),
3683 VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification),
3684 VMX_EXIT_QUALIFICATION_CRX_GENREG(exitQualification));
3685
3686 switch (VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification))
3687 {
3688 case 0:
3689 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_GUEST_CR0 | HWACCM_CHANGED_GUEST_CR3;
3690 break;
3691 case 2:
3692 break;
3693 case 3:
3694 Assert(!pVM->hwaccm.s.fNestedPaging || !CPUMIsGuestInPagedProtectedModeEx(pCtx));
3695 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_GUEST_CR3;
3696 break;
3697 case 4:
3698 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_GUEST_CR4;
3699 break;
3700 case 8:
3701 /* CR8 contains the APIC TPR */
3702 Assert(!(pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW));
3703 break;
3704
3705 default:
3706 AssertFailed();
3707 break;
3708 }
3709 break;
3710
3711 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ:
3712 Log2(("VMX: mov x, crx\n"));
3713 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitCRxRead[VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification)]);
3714
3715 Assert(!pVM->hwaccm.s.fNestedPaging || !CPUMIsGuestInPagedProtectedModeEx(pCtx) || VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification) != USE_REG_CR3);
3716
3717 /* CR8 reads only cause an exit when the TPR shadow feature isn't present. */
3718 Assert(VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification) != 8 || !(pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW));
3719
3720 rc = EMInterpretCRxRead(pVM, pVCpu, CPUMCTX2CORE(pCtx),
3721 VMX_EXIT_QUALIFICATION_CRX_GENREG(exitQualification),
3722 VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification));
3723 break;
3724
3725 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_CLTS:
3726 Log2(("VMX: clts\n"));
3727 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitCLTS);
3728 rc = EMInterpretCLTS(pVM, pVCpu);
3729 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_GUEST_CR0;
3730 break;
3731
3732 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_LMSW:
3733 Log2(("VMX: lmsw %x\n", VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(exitQualification)));
3734 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitLMSW);
3735 rc = EMInterpretLMSW(pVM, pVCpu, CPUMCTX2CORE(pCtx), VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(exitQualification));
3736 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_GUEST_CR0;
3737 break;
3738 }
3739
3740 /* Update EIP if no error occurred. */
3741 if (RT_SUCCESS(rc))
3742 pCtx->rip += cbInstr;
3743
3744 if (rc == VINF_SUCCESS)
3745 {
3746 /* Only resume if successful. */
3747 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub2, y2);
3748 goto ResumeExecution;
3749 }
3750 Assert(rc == VERR_EM_INTERPRETER || rc == VINF_PGM_CHANGE_MODE || rc == VINF_PGM_SYNC_CR3);
3751 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub2, y2);
3752 break;
3753 }
3754
3755 case VMX_EXIT_DRX_MOVE: /* 29 Debug-register accesses. */
3756 {
3757 if ( !DBGFIsStepping(pVCpu)
3758 && !CPUMIsHyperDebugStateActive(pVCpu))
3759 {
3760 /* Disable drx move intercepts. */
3761 pVCpu->hwaccm.s.vmx.proc_ctls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT;
3762 rc2 = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
3763 AssertRC(rc2);
3764
3765 /* Save the host and load the guest debug state. */
3766 rc2 = CPUMR0LoadGuestDebugState(pVM, pVCpu, pCtx, true /* include DR6 */);
3767 AssertRC(rc2);
3768
3769#ifdef LOG_ENABLED
3770 if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(exitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
3771 Log(("VMX_EXIT_DRX_MOVE: write DR%d genreg %d\n", VMX_EXIT_QUALIFICATION_DRX_REGISTER(exitQualification), VMX_EXIT_QUALIFICATION_DRX_GENREG(exitQualification)));
3772 else
3773 Log(("VMX_EXIT_DRX_MOVE: read DR%d\n", VMX_EXIT_QUALIFICATION_DRX_REGISTER(exitQualification)));
3774#endif
3775
3776#ifdef VBOX_WITH_STATISTICS
3777 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatDRxContextSwitch);
3778 if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(exitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
3779 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitDRxWrite);
3780 else
3781 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitDRxRead);
3782#endif
3783
3784 goto ResumeExecution;
3785 }
3786
3787 /** @todo clear VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT after the first time and restore drx registers afterwards */
3788 if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(exitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
3789 {
3790 Log2(("VMX: mov drx%d, genreg%d\n", VMX_EXIT_QUALIFICATION_DRX_REGISTER(exitQualification), VMX_EXIT_QUALIFICATION_DRX_GENREG(exitQualification)));
3791 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitDRxWrite);
3792 rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pCtx),
3793 VMX_EXIT_QUALIFICATION_DRX_REGISTER(exitQualification),
3794 VMX_EXIT_QUALIFICATION_DRX_GENREG(exitQualification));
3795 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_GUEST_DEBUG;
3796 Log2(("DR7=%08x\n", pCtx->dr[7]));
3797 }
3798 else
3799 {
3800 Log2(("VMX: mov x, drx\n"));
3801 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitDRxRead);
3802 rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pCtx),
3803 VMX_EXIT_QUALIFICATION_DRX_GENREG(exitQualification),
3804 VMX_EXIT_QUALIFICATION_DRX_REGISTER(exitQualification));
3805 }
3806 /* Update EIP if no error occurred. */
3807 if (RT_SUCCESS(rc))
3808 pCtx->rip += cbInstr;
3809
3810 if (rc == VINF_SUCCESS)
3811 {
3812 /* Only resume if successful. */
3813 goto ResumeExecution;
3814 }
3815 Assert(rc == VERR_EM_INTERPRETER);
3816 break;
3817 }
3818
3819 /* Note: We'll get a #GP if the IO instruction isn't allowed (IOPL or TSS bitmap); no need to double check. */
3820 case VMX_EXIT_PORT_IO: /* 30 I/O instruction. */
3821 {
3822 STAM_PROFILE_ADV_START(&pVCpu->hwaccm.s.StatExit2Sub1, y1);
3823 uint32_t uIOWidth = VMX_EXIT_QUALIFICATION_IO_WIDTH(exitQualification);
3824 uint32_t uPort;
3825 bool fIOWrite = (VMX_EXIT_QUALIFICATION_IO_DIRECTION(exitQualification) == VMX_EXIT_QUALIFICATION_IO_DIRECTION_OUT);
3826
3827 /** @todo necessary to make the distinction? */
3828 if (VMX_EXIT_QUALIFICATION_IO_ENCODING(exitQualification) == VMX_EXIT_QUALIFICATION_IO_ENCODING_DX)
3829 {
3830 uPort = pCtx->edx & 0xffff;
3831 }
3832 else
3833 uPort = VMX_EXIT_QUALIFICATION_IO_PORT(exitQualification); /* Immediate encoding. */
3834
3835 /* paranoia */
3836 if (RT_UNLIKELY(uIOWidth == 2 || uIOWidth >= 4))
3837 {
3838 rc = fIOWrite ? VINF_IOM_HC_IOPORT_WRITE : VINF_IOM_HC_IOPORT_READ;
3839 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub1, y1);
3840 break;
3841 }
3842
3843 uint32_t cbSize = g_aIOSize[uIOWidth];
3844
3845 if (VMX_EXIT_QUALIFICATION_IO_STRING(exitQualification))
3846 {
3847 /* ins/outs */
3848 PDISCPUSTATE pDis = &pVCpu->hwaccm.s.DisState;
3849
3850 /* Disassemble manually to deal with segment prefixes. */
3851 /** @todo VMX_VMCS_EXIT_GUEST_LINEAR_ADDR contains the flat pointer operand of the instruction. */
3852 /** @todo VMX_VMCS32_RO_EXIT_INSTR_INFO also contains segment prefix info. */
3853 rc2 = EMInterpretDisasOne(pVM, pVCpu, CPUMCTX2CORE(pCtx), pDis, NULL);
3854 if (RT_SUCCESS(rc))
3855 {
3856 if (fIOWrite)
3857 {
3858 Log2(("IOMInterpretOUTSEx %RGv %x size=%d\n", (RTGCPTR)pCtx->rip, uPort, cbSize));
3859 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitIOStringWrite);
3860 rc = IOMInterpretOUTSEx(pVM, CPUMCTX2CORE(pCtx), uPort, pDis->prefix, pDis->addrmode, cbSize);
3861 }
3862 else
3863 {
3864 Log2(("IOMInterpretINSEx %RGv %x size=%d\n", (RTGCPTR)pCtx->rip, uPort, cbSize));
3865 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitIOStringRead);
3866 rc = IOMInterpretINSEx(pVM, CPUMCTX2CORE(pCtx), uPort, pDis->prefix, pDis->addrmode, cbSize);
3867 }
3868 }
3869 else
3870 rc = VINF_EM_RAW_EMULATE_INSTR;
3871 }
3872 else
3873 {
3874 /* normal in/out */
3875 uint32_t uAndVal = g_aIOOpAnd[uIOWidth];
3876
3877 Assert(!VMX_EXIT_QUALIFICATION_IO_REP(exitQualification));
3878
3879 if (fIOWrite)
3880 {
3881 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitIOWrite);
3882 rc = IOMIOPortWrite(pVM, uPort, pCtx->eax & uAndVal, cbSize);
3883 if (rc == VINF_IOM_HC_IOPORT_WRITE)
3884 HWACCMR0SavePendingIOPortWrite(pVCpu, pCtx->rip, pCtx->rip + cbInstr, uPort, uAndVal, cbSize);
3885 }
3886 else
3887 {
3888 uint32_t u32Val = 0;
3889
3890 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitIORead);
3891 rc = IOMIOPortRead(pVM, uPort, &u32Val, cbSize);
3892 if (IOM_SUCCESS(rc))
3893 {
3894 /* Write back to the EAX register. */
3895 pCtx->eax = (pCtx->eax & ~uAndVal) | (u32Val & uAndVal);
3896 }
3897 else
3898 if (rc == VINF_IOM_HC_IOPORT_READ)
3899 HWACCMR0SavePendingIOPortRead(pVCpu, pCtx->rip, pCtx->rip + cbInstr, uPort, uAndVal, cbSize);
3900 }
3901 }
3902 /*
3903 * Handled the I/O return codes.
3904 * (The unhandled cases end up with rc == VINF_EM_RAW_EMULATE_INSTR.)
3905 */
3906 if (IOM_SUCCESS(rc))
3907 {
3908 /* Update EIP and continue execution. */
3909 pCtx->rip += cbInstr;
3910 if (RT_LIKELY(rc == VINF_SUCCESS))
3911 {
3912 /* If any IO breakpoints are armed, then we should check if a debug trap needs to be generated. */
3913 if (pCtx->dr[7] & X86_DR7_ENABLED_MASK)
3914 {
3915 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatDRxIOCheck);
3916 for (unsigned i=0;i<4;i++)
3917 {
3918 unsigned uBPLen = g_aIOSize[X86_DR7_GET_LEN(pCtx->dr[7], i)];
3919
3920 if ( (uPort >= pCtx->dr[i] && uPort < pCtx->dr[i] + uBPLen)
3921 && (pCtx->dr[7] & (X86_DR7_L(i) | X86_DR7_G(i)))
3922 && (pCtx->dr[7] & X86_DR7_RW(i, X86_DR7_RW_IO)) == X86_DR7_RW(i, X86_DR7_RW_IO))
3923 {
3924 uint64_t uDR6;
3925
3926 Assert(CPUMIsGuestDebugStateActive(pVCpu));
3927
3928 uDR6 = ASMGetDR6();
3929
3930 /* Clear all breakpoint status flags and set the one we just hit. */
3931 uDR6 &= ~(X86_DR6_B0|X86_DR6_B1|X86_DR6_B2|X86_DR6_B3);
3932 uDR6 |= (uint64_t)RT_BIT(i);
3933
3934 /* Note: AMD64 Architecture Programmer's Manual 13.1:
3935 * Bits 15:13 of the DR6 register is never cleared by the processor and must be cleared by software after
3936 * the contents have been read.
3937 */
3938 ASMSetDR6(uDR6);
3939
3940 /* X86_DR7_GD will be cleared if drx accesses should be trapped inside the guest. */
3941 pCtx->dr[7] &= ~X86_DR7_GD;
3942
3943 /* Paranoia. */
3944 pCtx->dr[7] &= 0xffffffff; /* upper 32 bits reserved */
3945 pCtx->dr[7] &= ~(RT_BIT(11) | RT_BIT(12) | RT_BIT(14) | RT_BIT(15)); /* must be zero */
3946 pCtx->dr[7] |= 0x400; /* must be one */
3947
3948 /* Resync DR7 */
3949 rc2 = VMXWriteVMCS64(VMX_VMCS64_GUEST_DR7, pCtx->dr[7]);
3950 AssertRC(rc2);
3951
3952 /* Construct inject info. */
3953 intInfo = X86_XCPT_DB;
3954 intInfo |= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
3955 intInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HWEXCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
3956
3957 Log(("Inject IO debug trap at %RGv\n", (RTGCPTR)pCtx->rip));
3958 rc2 = VMXR0InjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), 0, 0);
3959 AssertRC(rc2);
3960
3961 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub1, y1);
3962 goto ResumeExecution;
3963 }
3964 }
3965 }
3966 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub1, y1);
3967 goto ResumeExecution;
3968 }
3969 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub1, y1);
3970 break;
3971 }
3972
3973#ifdef VBOX_STRICT
3974 if (rc == VINF_IOM_HC_IOPORT_READ)
3975 Assert(!fIOWrite);
3976 else if (rc == VINF_IOM_HC_IOPORT_WRITE)
3977 Assert(fIOWrite);
3978 else
3979 AssertMsg(RT_FAILURE(rc) || rc == VINF_EM_RAW_EMULATE_INSTR || rc == VINF_EM_RAW_GUEST_TRAP || rc == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", VBOXSTRICTRC_VAL(rc)));
3980#endif
3981 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub1, y1);
3982 break;
3983 }
3984
3985 case VMX_EXIT_TPR: /* 43 TPR below threshold. Guest software executed MOV to CR8. */
3986 LogFlow(("VMX_EXIT_TPR\n"));
3987 /* RIP is already set to the next instruction and the TPR has been synced back. Just resume. */
3988 goto ResumeExecution;
3989
3990 case VMX_EXIT_APIC_ACCESS: /* 44 APIC access. Guest software attempted to access memory at a physical address on the APIC-access page. */
3991 {
3992 LogFlow(("VMX_EXIT_APIC_ACCESS\n"));
3993 unsigned uAccessType = VMX_EXIT_QUALIFICATION_APIC_ACCESS_TYPE(exitQualification);
3994
3995 switch(uAccessType)
3996 {
3997 case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
3998 case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
3999 {
4000 RTGCPHYS GCPhys;
4001 PDMApicGetBase(pVM, &GCPhys);
4002 GCPhys &= PAGE_BASE_GC_MASK;
4003 GCPhys += VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(exitQualification);
4004
4005 LogFlow(("Apic access at %RGp\n", GCPhys));
4006 rc = IOMMMIOPhysHandler(pVM, (uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ) ? 0 : X86_TRAP_PF_RW, CPUMCTX2CORE(pCtx), GCPhys);
4007 if (rc == VINF_SUCCESS)
4008 goto ResumeExecution; /* rip already updated */
4009 break;
4010 }
4011
4012 default:
4013 rc = VINF_EM_RAW_EMULATE_INSTR;
4014 break;
4015 }
4016 break;
4017 }
4018
4019 case VMX_EXIT_PREEMPTION_TIMER: /* 52 VMX-preemption timer expired. The preemption timer counted down to zero. */
4020 if (!TMTimerPollBool(pVM, pVCpu))
4021 goto ResumeExecution;
4022 rc = VINF_EM_RAW_TIMER_PENDING;
4023 break;
4024
4025 default:
4026 /* The rest is handled after syncing the entire CPU state. */
4027 break;
4028 }
4029
4030 /* Note: the guest state isn't entirely synced back at this stage. */
4031
4032 /* Investigate why there was a VM-exit. (part 2) */
4033 switch (exitReason)
4034 {
4035 case VMX_EXIT_EXCEPTION: /* 0 Exception or non-maskable interrupt (NMI). */
4036 case VMX_EXIT_EXTERNAL_IRQ: /* 1 External interrupt. */
4037 case VMX_EXIT_EPT_VIOLATION:
4038 case VMX_EXIT_EPT_MISCONFIG: /* 49 EPT misconfig is used by the PGM/MMIO optimizations. */
4039 case VMX_EXIT_PREEMPTION_TIMER: /* 52 VMX-preemption timer expired. The preemption timer counted down to zero. */
4040 /* Already handled above. */
4041 break;
4042
4043 case VMX_EXIT_TRIPLE_FAULT: /* 2 Triple fault. */
4044 rc = VINF_EM_RESET; /* Triple fault equals a reset. */
4045 break;
4046
4047 case VMX_EXIT_INIT_SIGNAL: /* 3 INIT signal. */
4048 case VMX_EXIT_SIPI: /* 4 Start-up IPI (SIPI). */
4049 rc = VINF_EM_RAW_INTERRUPT;
4050 AssertFailed(); /* Can't happen. Yet. */
4051 break;
4052
4053 case VMX_EXIT_IO_SMI_IRQ: /* 5 I/O system-management interrupt (SMI). */
4054 case VMX_EXIT_SMI_IRQ: /* 6 Other SMI. */
4055 rc = VINF_EM_RAW_INTERRUPT;
4056 AssertFailed(); /* Can't happen afaik. */
4057 break;
4058
4059 case VMX_EXIT_TASK_SWITCH: /* 9 Task switch: too complicated to emulate, so fall back to the recompiler */
4060 Log(("VMX_EXIT_TASK_SWITCH: exit=%RX64\n", exitQualification));
4061 if ( (VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE(exitQualification) == VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE_IDT)
4062 && pVCpu->hwaccm.s.Event.fPending)
4063 {
4064 /* Caused by an injected interrupt. */
4065 pVCpu->hwaccm.s.Event.fPending = false;
4066
4067 Log(("VMX_EXIT_TASK_SWITCH: reassert trap %d\n", VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVCpu->hwaccm.s.Event.intInfo)));
4068 Assert(!VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(pVCpu->hwaccm.s.Event.intInfo));
4069 rc2 = TRPMAssertTrap(pVCpu, VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVCpu->hwaccm.s.Event.intInfo), TRPM_HARDWARE_INT);
4070 AssertRC(rc2);
4071 }
4072 /* else Exceptions and software interrupts can just be restarted. */
4073 rc = VERR_EM_INTERPRETER;
4074 break;
4075
4076 case VMX_EXIT_HLT: /* 12 Guest software attempted to execute HLT. */
4077 /** Check if external interrupts are pending; if so, don't switch back. */
4078 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitHlt);
4079 pCtx->rip++; /* skip hlt */
4080 if (EMShouldContinueAfterHalt(pVCpu, pCtx))
4081 goto ResumeExecution;
4082
4083 rc = VINF_EM_HALT;
4084 break;
4085
4086 case VMX_EXIT_MWAIT: /* 36 Guest software executed MWAIT. */
4087 Log2(("VMX: mwait\n"));
4088 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitMwait);
4089 rc = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pCtx));
4090 if ( rc == VINF_EM_HALT
4091 || rc == VINF_SUCCESS)
4092 {
4093 /* Update EIP and continue execution. */
4094 pCtx->rip += cbInstr;
4095
4096 /** Check if external interrupts are pending; if so, don't switch back. */
4097 if ( rc == VINF_SUCCESS
4098 || ( rc == VINF_EM_HALT
4099 && EMShouldContinueAfterHalt(pVCpu, pCtx))
4100 )
4101 goto ResumeExecution;
4102 }
4103 AssertMsg(rc == VERR_EM_INTERPRETER || rc == VINF_EM_HALT, ("EMU: mwait failed with %Rrc\n", VBOXSTRICTRC_VAL(rc)));
4104 break;
4105
4106 case VMX_EXIT_RSM: /* 17 Guest software attempted to execute RSM in SMM. */
4107 AssertFailed(); /* can't happen. */
4108 rc = VERR_EM_INTERPRETER;
4109 break;
4110
4111 case VMX_EXIT_VMCALL: /* 18 Guest software executed VMCALL. */
4112 case VMX_EXIT_VMCLEAR: /* 19 Guest software executed VMCLEAR. */
4113 case VMX_EXIT_VMLAUNCH: /* 20 Guest software executed VMLAUNCH. */
4114 case VMX_EXIT_VMPTRLD: /* 21 Guest software executed VMPTRLD. */
4115 case VMX_EXIT_VMPTRST: /* 22 Guest software executed VMPTRST. */
4116 case VMX_EXIT_VMREAD: /* 23 Guest software executed VMREAD. */
4117 case VMX_EXIT_VMRESUME: /* 24 Guest software executed VMRESUME. */
4118 case VMX_EXIT_VMWRITE: /* 25 Guest software executed VMWRITE. */
4119 case VMX_EXIT_VMXOFF: /* 26 Guest software executed VMXOFF. */
4120 case VMX_EXIT_VMXON: /* 27 Guest software executed VMXON. */
4121 /** @todo inject #UD immediately */
4122 rc = VERR_EM_INTERPRETER;
4123 break;
4124
4125 case VMX_EXIT_CPUID: /* 10 Guest software attempted to execute CPUID. */
4126 case VMX_EXIT_RDTSC: /* 16 Guest software attempted to execute RDTSC. */
4127 case VMX_EXIT_INVPG: /* 14 Guest software attempted to execute INVPG. */
4128 case VMX_EXIT_CRX_MOVE: /* 28 Control-register accesses. */
4129 case VMX_EXIT_DRX_MOVE: /* 29 Debug-register accesses. */
4130 case VMX_EXIT_PORT_IO: /* 30 I/O instruction. */
4131 case VMX_EXIT_RDPMC: /* 15 Guest software attempted to execute RDPMC. */
4132 /* already handled above */
4133 AssertMsg( rc == VINF_PGM_CHANGE_MODE
4134 || rc == VINF_EM_RAW_INTERRUPT
4135 || rc == VERR_EM_INTERPRETER
4136 || rc == VINF_EM_RAW_EMULATE_INSTR
4137 || rc == VINF_PGM_SYNC_CR3
4138 || rc == VINF_IOM_HC_IOPORT_READ
4139 || rc == VINF_IOM_HC_IOPORT_WRITE
4140 || rc == VINF_EM_RAW_GUEST_TRAP
4141 || rc == VINF_TRPM_XCPT_DISPATCHED
4142 || rc == VINF_EM_RESCHEDULE_REM,
4143 ("rc = %d\n", VBOXSTRICTRC_VAL(rc)));
4144 break;
4145
4146 case VMX_EXIT_TPR: /* 43 TPR below threshold. Guest software executed MOV to CR8. */
4147 case VMX_EXIT_APIC_ACCESS: /* 44 APIC access. Guest software attempted to access memory at a physical address on the APIC-access page. */
4148 case VMX_EXIT_RDMSR: /* 31 RDMSR. Guest software attempted to execute RDMSR. */
4149 case VMX_EXIT_WRMSR: /* 32 WRMSR. Guest software attempted to execute WRMSR. */
4150 case VMX_EXIT_PAUSE: /* 40 Guest software attempted to execute PAUSE. */
4151 case VMX_EXIT_MONITOR: /* 39 Guest software attempted to execute MONITOR. */
4152 /* Note: If we decide to emulate them here, then we must sync the MSRs that could have been changed (sysenter, fs/gs base)!!! */
4153 rc = VERR_EM_INTERPRETER;
4154 break;
4155
4156 case VMX_EXIT_IRQ_WINDOW: /* 7 Interrupt window. */
4157 Assert(rc == VINF_EM_RAW_INTERRUPT);
4158 break;
4159
4160 case VMX_EXIT_ERR_INVALID_GUEST_STATE: /* 33 VM-entry failure due to invalid guest state. */
4161 {
4162#ifdef VBOX_STRICT
4163 RTCCUINTREG val2 = 0;
4164
4165 Log(("VMX_EXIT_ERR_INVALID_GUEST_STATE\n"));
4166
4167 VMXReadVMCS(VMX_VMCS64_GUEST_RIP, &val2);
4168 Log(("Old eip %RGv new %RGv\n", (RTGCPTR)pCtx->rip, (RTGCPTR)val2));
4169
4170 VMXReadVMCS(VMX_VMCS64_GUEST_CR0, &val2);
4171 Log(("VMX_VMCS_GUEST_CR0 %RX64\n", (uint64_t)val2));
4172
4173 VMXReadVMCS(VMX_VMCS64_GUEST_CR3, &val2);
4174 Log(("VMX_VMCS_GUEST_CR3 %RX64\n", (uint64_t)val2));
4175
4176 VMXReadVMCS(VMX_VMCS64_GUEST_CR4, &val2);
4177 Log(("VMX_VMCS_GUEST_CR4 %RX64\n", (uint64_t)val2));
4178
4179 VMXReadVMCS(VMX_VMCS_GUEST_RFLAGS, &val2);
4180 Log(("VMX_VMCS_GUEST_RFLAGS %08x\n", val2));
4181
4182 VMX_LOG_SELREG(CS, "CS", val2);
4183 VMX_LOG_SELREG(DS, "DS", val2);
4184 VMX_LOG_SELREG(ES, "ES", val2);
4185 VMX_LOG_SELREG(FS, "FS", val2);
4186 VMX_LOG_SELREG(GS, "GS", val2);
4187 VMX_LOG_SELREG(SS, "SS", val2);
4188 VMX_LOG_SELREG(TR, "TR", val2);
4189 VMX_LOG_SELREG(LDTR, "LDTR", val2);
4190
4191 VMXReadVMCS(VMX_VMCS64_GUEST_GDTR_BASE, &val2);
4192 Log(("VMX_VMCS_GUEST_GDTR_BASE %RX64\n", (uint64_t)val2));
4193 VMXReadVMCS(VMX_VMCS64_GUEST_IDTR_BASE, &val2);
4194 Log(("VMX_VMCS_GUEST_IDTR_BASE %RX64\n", (uint64_t)val2));
4195#endif /* VBOX_STRICT */
4196 rc = VERR_VMX_INVALID_GUEST_STATE;
4197 break;
4198 }
4199
4200 case VMX_EXIT_ERR_MSR_LOAD: /* 34 VM-entry failure due to MSR loading. */
4201 case VMX_EXIT_ERR_MACHINE_CHECK: /* 41 VM-entry failure due to machine-check. */
4202 default:
4203 rc = VERR_VMX_UNEXPECTED_EXIT_CODE;
4204 AssertMsgFailed(("Unexpected exit code %d\n", exitReason)); /* Can't happen. */
4205 break;
4206
4207 }
4208end:
4209
4210 /* We now going back to ring-3, so clear the action flag. */
4211 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
4212
4213 /* Signal changes for the recompiler. */
4214 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR | CPUM_CHANGED_LDTR | CPUM_CHANGED_GDTR | CPUM_CHANGED_IDTR | CPUM_CHANGED_TR | CPUM_CHANGED_HIDDEN_SEL_REGS);
4215
4216 /* If we executed vmlaunch/vmresume and an external irq was pending, then we don't have to do a full sync the next time. */
4217 if ( exitReason == VMX_EXIT_EXTERNAL_IRQ
4218 && !VMX_EXIT_INTERRUPTION_INFO_VALID(intInfo))
4219 {
4220 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatPendingHostIrq);
4221 /* On the next entry we'll only sync the host context. */
4222 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_HOST_CONTEXT;
4223 }
4224 else
4225 {
4226 /* On the next entry we'll sync everything. */
4227 /** @todo we can do better than this */
4228 /* Not in the VINF_PGM_CHANGE_MODE though! */
4229 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_ALL;
4230 }
4231
4232 /* translate into a less severe return code */
4233 if (rc == VERR_EM_INTERPRETER)
4234 rc = VINF_EM_RAW_EMULATE_INSTR;
4235 else
4236 /* Try to extract more information about what might have gone wrong here. */
4237 if (rc == VERR_VMX_INVALID_VMCS_PTR)
4238 {
4239 VMXGetActivateVMCS(&pVCpu->hwaccm.s.vmx.lasterror.u64VMCSPhys);
4240 pVCpu->hwaccm.s.vmx.lasterror.ulVMCSRevision = *(uint32_t *)pVCpu->hwaccm.s.vmx.pvVMCS;
4241 pVCpu->hwaccm.s.vmx.lasterror.idEnteredCpu = pVCpu->hwaccm.s.idEnteredCpu;
4242 pVCpu->hwaccm.s.vmx.lasterror.idCurrentCpu = RTMpCpuId();
4243 }
4244
4245 /* Just set the correct state here instead of trying to catch every goto above. */
4246 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED, VMCPUSTATE_STARTED_EXEC);
4247
4248#ifdef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
4249 /* Restore interrupts if we exitted after disabling them. */
4250 if (uOldEFlags != ~(RTCCUINTREG)0)
4251 ASMSetFlags(uOldEFlags);
4252#endif
4253
4254 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2, x);
4255 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit1, x);
4256 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatEntry, x);
4257 Log2(("X"));
4258 return VBOXSTRICTRC_TODO(rc);
4259}
4260
4261
4262/**
4263 * Enters the VT-x session
4264 *
4265 * @returns VBox status code.
4266 * @param pVM The VM to operate on.
4267 * @param pVCpu The VMCPU to operate on.
4268 * @param pCpu CPU info struct
4269 */
4270VMMR0DECL(int) VMXR0Enter(PVM pVM, PVMCPU pVCpu, PHMGLOBLCPUINFO pCpu)
4271{
4272 Assert(pVM->hwaccm.s.vmx.fSupported);
4273
4274 unsigned cr4 = ASMGetCR4();
4275 if (!(cr4 & X86_CR4_VMXE))
4276 {
4277 AssertMsgFailed(("X86_CR4_VMXE should be set!\n"));
4278 return VERR_VMX_X86_CR4_VMXE_CLEARED;
4279 }
4280
4281 /* Activate the VM Control Structure. */
4282 int rc = VMXActivateVMCS(pVCpu->hwaccm.s.vmx.HCPhysVMCS);
4283 if (RT_FAILURE(rc))
4284 return rc;
4285
4286 pVCpu->hwaccm.s.fResumeVM = false;
4287 return VINF_SUCCESS;
4288}
4289
4290
4291/**
4292 * Leaves the VT-x session
4293 *
4294 * @returns VBox status code.
4295 * @param pVM The VM to operate on.
4296 * @param pVCpu The VMCPU to operate on.
4297 * @param pCtx CPU context
4298 */
4299VMMR0DECL(int) VMXR0Leave(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
4300{
4301 Assert(pVM->hwaccm.s.vmx.fSupported);
4302
4303#ifdef DEBUG
4304 if (CPUMIsHyperDebugStateActive(pVCpu))
4305 {
4306 CPUMR0LoadHostDebugState(pVM, pVCpu);
4307 Assert(pVCpu->hwaccm.s.vmx.proc_ctls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT);
4308 }
4309 else
4310#endif
4311 /* Save the guest debug state if necessary. */
4312 if (CPUMIsGuestDebugStateActive(pVCpu))
4313 {
4314 CPUMR0SaveGuestDebugState(pVM, pVCpu, pCtx, true /* save DR6 */);
4315
4316 /* Enable drx move intercepts again. */
4317 pVCpu->hwaccm.s.vmx.proc_ctls |= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT;
4318 int rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
4319 AssertRC(rc);
4320
4321 /* Resync the debug registers the next time. */
4322 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_GUEST_DEBUG;
4323 }
4324 else
4325 Assert(pVCpu->hwaccm.s.vmx.proc_ctls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT);
4326
4327 /* Clear VM Control Structure. Marking it inactive, clearing implementation specific data and writing back VMCS data to memory. */
4328 int rc = VMXClearVMCS(pVCpu->hwaccm.s.vmx.HCPhysVMCS);
4329 AssertRC(rc);
4330
4331 return VINF_SUCCESS;
4332}
4333
4334/**
4335 * Flush the TLB (EPT)
4336 *
4337 * @returns VBox status code.
4338 * @param pVM The VM to operate on.
4339 * @param pVCpu The VM CPU to operate on.
4340 * @param enmFlush Type of flush
4341 * @param GCPhys Physical address of the page to flush
4342 */
4343static void vmxR0FlushEPT(PVM pVM, PVMCPU pVCpu, VMX_FLUSH enmFlush, RTGCPHYS GCPhys)
4344{
4345 uint64_t descriptor[2];
4346
4347 LogFlow(("vmxR0FlushEPT %d %RGv\n", enmFlush, GCPhys));
4348 Assert(pVM->hwaccm.s.fNestedPaging);
4349 descriptor[0] = pVCpu->hwaccm.s.vmx.GCPhysEPTP;
4350 descriptor[1] = GCPhys;
4351 int rc = VMXR0InvEPT(enmFlush, &descriptor[0]);
4352 AssertRC(rc);
4353}
4354
4355#ifdef HWACCM_VTX_WITH_VPID
4356/**
4357 * Flush the TLB (EPT)
4358 *
4359 * @returns VBox status code.
4360 * @param pVM The VM to operate on.
4361 * @param pVCpu The VM CPU to operate on.
4362 * @param enmFlush Type of flush
4363 * @param GCPtr Virtual address of the page to flush
4364 */
4365static void vmxR0FlushVPID(PVM pVM, PVMCPU pVCpu, VMX_FLUSH enmFlush, RTGCPTR GCPtr)
4366{
4367#if HC_ARCH_BITS == 32
4368 /* If we get a flush in 64 bits guest mode, then force a full TLB flush. Invvpid probably takes only 32 bits addresses. (@todo) */
4369 if ( CPUMIsGuestInLongMode(pVCpu)
4370 && !VMX_IS_64BIT_HOST_MODE())
4371 {
4372 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
4373 }
4374 else
4375#endif
4376 {
4377 uint64_t descriptor[2];
4378
4379 Assert(pVM->hwaccm.s.vmx.fVPID);
4380 descriptor[0] = pVCpu->hwaccm.s.uCurrentASID;
4381 descriptor[1] = GCPtr;
4382 int rc = VMXR0InvVPID(enmFlush, &descriptor[0]);
4383 AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvVPID %x %x %RGv failed with %d\n", enmFlush, pVCpu->hwaccm.s.uCurrentASID, GCPtr, rc));
4384 }
4385}
4386#endif /* HWACCM_VTX_WITH_VPID */
4387
4388/**
4389 * Invalidates a guest page
4390 *
4391 * @returns VBox status code.
4392 * @param pVM The VM to operate on.
4393 * @param pVCpu The VM CPU to operate on.
4394 * @param GCVirt Page to invalidate
4395 */
4396VMMR0DECL(int) VMXR0InvalidatePage(PVM pVM, PVMCPU pVCpu, RTGCPTR GCVirt)
4397{
4398 bool fFlushPending = VMCPU_FF_ISSET(pVCpu, VMCPU_FF_TLB_FLUSH);
4399
4400 Log2(("VMXR0InvalidatePage %RGv\n", GCVirt));
4401
4402 /* Only relevant if we want to use VPID.
4403 * In the nested paging case we still see such calls, but
4404 * can safely ignore them. (e.g. after cr3 updates)
4405 */
4406#ifdef HWACCM_VTX_WITH_VPID
4407 /* Skip it if a TLB flush is already pending. */
4408 if ( !fFlushPending
4409 && pVM->hwaccm.s.vmx.fVPID)
4410 vmxR0FlushVPID(pVM, pVCpu, pVM->hwaccm.s.vmx.enmFlushPage, GCVirt);
4411#endif /* HWACCM_VTX_WITH_VPID */
4412
4413 return VINF_SUCCESS;
4414}
4415
4416/**
4417 * Invalidates a guest page by physical address
4418 *
4419 * NOTE: Assumes the current instruction references this physical page though a virtual address!!
4420 *
4421 * @returns VBox status code.
4422 * @param pVM The VM to operate on.
4423 * @param pVCpu The VM CPU to operate on.
4424 * @param GCPhys Page to invalidate
4425 */
4426VMMR0DECL(int) VMXR0InvalidatePhysPage(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys)
4427{
4428 bool fFlushPending = VMCPU_FF_ISSET(pVCpu, VMCPU_FF_TLB_FLUSH);
4429
4430 Assert(pVM->hwaccm.s.fNestedPaging);
4431
4432 LogFlow(("VMXR0InvalidatePhysPage %RGp\n", GCPhys));
4433
4434 /* Skip it if a TLB flush is already pending. */
4435 if (!fFlushPending)
4436 vmxR0FlushEPT(pVM, pVCpu, pVM->hwaccm.s.vmx.enmFlushPage, GCPhys);
4437
4438 return VINF_SUCCESS;
4439}
4440
4441/**
4442 * Report world switch error and dump some useful debug info
4443 *
4444 * @param pVM The VM to operate on.
4445 * @param pVCpu The VMCPU to operate on.
4446 * @param rc Return code
4447 * @param pCtx Current CPU context (not updated)
4448 */
4449static void VMXR0ReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, VBOXSTRICTRC rc, PCPUMCTX pCtx)
4450{
4451 switch (VBOXSTRICTRC_VAL(rc))
4452 {
4453 case VERR_VMX_INVALID_VMXON_PTR:
4454 AssertFailed();
4455 break;
4456
4457 case VERR_VMX_UNABLE_TO_START_VM:
4458 case VERR_VMX_UNABLE_TO_RESUME_VM:
4459 {
4460 int rc2;
4461 RTCCUINTREG exitReason, instrError;
4462
4463 rc2 = VMXReadVMCS(VMX_VMCS32_RO_EXIT_REASON, &exitReason);
4464 rc2 |= VMXReadVMCS(VMX_VMCS32_RO_VM_INSTR_ERROR, &instrError);
4465 AssertRC(rc2);
4466 if (rc2 == VINF_SUCCESS)
4467 {
4468 Log(("Unable to start/resume VM for reason: %x. Instruction error %x\n", (uint32_t)exitReason, (uint32_t)instrError));
4469 Log(("Current stack %08x\n", &rc2));
4470
4471 pVCpu->hwaccm.s.vmx.lasterror.ulInstrError = instrError;
4472 pVCpu->hwaccm.s.vmx.lasterror.ulExitReason = exitReason;
4473
4474#ifdef VBOX_STRICT
4475 RTGDTR gdtr;
4476 PCX86DESCHC pDesc;
4477 RTCCUINTREG val;
4478
4479 ASMGetGDTR(&gdtr);
4480
4481 VMXReadVMCS(VMX_VMCS64_GUEST_RIP, &val);
4482 Log(("Old eip %RGv new %RGv\n", (RTGCPTR)pCtx->rip, (RTGCPTR)val));
4483 VMXReadVMCS(VMX_VMCS_CTRL_PIN_EXEC_CONTROLS, &val);
4484 Log(("VMX_VMCS_CTRL_PIN_EXEC_CONTROLS %08x\n", val));
4485 VMXReadVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, &val);
4486 Log(("VMX_VMCS_CTRL_PROC_EXEC_CONTROLS %08x\n", val));
4487 VMXReadVMCS(VMX_VMCS_CTRL_ENTRY_CONTROLS, &val);
4488 Log(("VMX_VMCS_CTRL_ENTRY_CONTROLS %08x\n", val));
4489 VMXReadVMCS(VMX_VMCS_CTRL_EXIT_CONTROLS, &val);
4490 Log(("VMX_VMCS_CTRL_EXIT_CONTROLS %08x\n", val));
4491
4492 VMXReadVMCS(VMX_VMCS_HOST_CR0, &val);
4493 Log(("VMX_VMCS_HOST_CR0 %08x\n", val));
4494
4495 VMXReadVMCS(VMX_VMCS_HOST_CR3, &val);
4496 Log(("VMX_VMCS_HOST_CR3 %08x\n", val));
4497
4498 VMXReadVMCS(VMX_VMCS_HOST_CR4, &val);
4499 Log(("VMX_VMCS_HOST_CR4 %08x\n", val));
4500
4501 VMXReadVMCS(VMX_VMCS16_HOST_FIELD_CS, &val);
4502 Log(("VMX_VMCS_HOST_FIELD_CS %08x\n", val));
4503
4504 VMXReadVMCS(VMX_VMCS_GUEST_RFLAGS, &val);
4505 Log(("VMX_VMCS_GUEST_RFLAGS %08x\n", val));
4506
4507 if (val < gdtr.cbGdt)
4508 {
4509 pDesc = (PCX86DESCHC)(gdtr.pGdt + (val & X86_SEL_MASK));
4510 HWACCMR0DumpDescriptor(pDesc, val, "CS: ");
4511 }
4512
4513 VMXReadVMCS(VMX_VMCS16_HOST_FIELD_DS, &val);
4514 Log(("VMX_VMCS_HOST_FIELD_DS %08x\n", val));
4515 if (val < gdtr.cbGdt)
4516 {
4517 pDesc = (PCX86DESCHC)(gdtr.pGdt + (val & X86_SEL_MASK));
4518 HWACCMR0DumpDescriptor(pDesc, val, "DS: ");
4519 }
4520
4521 VMXReadVMCS(VMX_VMCS16_HOST_FIELD_ES, &val);
4522 Log(("VMX_VMCS_HOST_FIELD_ES %08x\n", val));
4523 if (val < gdtr.cbGdt)
4524 {
4525 pDesc = (PCX86DESCHC)(gdtr.pGdt + (val & X86_SEL_MASK));
4526 HWACCMR0DumpDescriptor(pDesc, val, "ES: ");
4527 }
4528
4529 VMXReadVMCS(VMX_VMCS16_HOST_FIELD_FS, &val);
4530 Log(("VMX_VMCS16_HOST_FIELD_FS %08x\n", val));
4531 if (val < gdtr.cbGdt)
4532 {
4533 pDesc = (PCX86DESCHC)(gdtr.pGdt + (val & X86_SEL_MASK));
4534 HWACCMR0DumpDescriptor(pDesc, val, "FS: ");
4535 }
4536
4537 VMXReadVMCS(VMX_VMCS16_HOST_FIELD_GS, &val);
4538 Log(("VMX_VMCS16_HOST_FIELD_GS %08x\n", val));
4539 if (val < gdtr.cbGdt)
4540 {
4541 pDesc = (PCX86DESCHC)(gdtr.pGdt + (val & X86_SEL_MASK));
4542 HWACCMR0DumpDescriptor(pDesc, val, "GS: ");
4543 }
4544
4545 VMXReadVMCS(VMX_VMCS16_HOST_FIELD_SS, &val);
4546 Log(("VMX_VMCS16_HOST_FIELD_SS %08x\n", val));
4547 if (val < gdtr.cbGdt)
4548 {
4549 pDesc = (PCX86DESCHC)(gdtr.pGdt + (val & X86_SEL_MASK));
4550 HWACCMR0DumpDescriptor(pDesc, val, "SS: ");
4551 }
4552
4553 VMXReadVMCS(VMX_VMCS16_HOST_FIELD_TR, &val);
4554 Log(("VMX_VMCS16_HOST_FIELD_TR %08x\n", val));
4555 if (val < gdtr.cbGdt)
4556 {
4557 pDesc = (PCX86DESCHC)(gdtr.pGdt + (val & X86_SEL_MASK));
4558 HWACCMR0DumpDescriptor(pDesc, val, "TR: ");
4559 }
4560
4561 VMXReadVMCS(VMX_VMCS_HOST_TR_BASE, &val);
4562 Log(("VMX_VMCS_HOST_TR_BASE %RHv\n", val));
4563
4564 VMXReadVMCS(VMX_VMCS_HOST_GDTR_BASE, &val);
4565 Log(("VMX_VMCS_HOST_GDTR_BASE %RHv\n", val));
4566 VMXReadVMCS(VMX_VMCS_HOST_IDTR_BASE, &val);
4567 Log(("VMX_VMCS_HOST_IDTR_BASE %RHv\n", val));
4568
4569 VMXReadVMCS(VMX_VMCS32_HOST_SYSENTER_CS, &val);
4570 Log(("VMX_VMCS_HOST_SYSENTER_CS %08x\n", val));
4571
4572 VMXReadVMCS(VMX_VMCS_HOST_SYSENTER_EIP, &val);
4573 Log(("VMX_VMCS_HOST_SYSENTER_EIP %RHv\n", val));
4574
4575 VMXReadVMCS(VMX_VMCS_HOST_SYSENTER_ESP, &val);
4576 Log(("VMX_VMCS_HOST_SYSENTER_ESP %RHv\n", val));
4577
4578 VMXReadVMCS(VMX_VMCS_HOST_RSP, &val);
4579 Log(("VMX_VMCS_HOST_RSP %RHv\n", val));
4580 VMXReadVMCS(VMX_VMCS_HOST_RIP, &val);
4581 Log(("VMX_VMCS_HOST_RIP %RHv\n", val));
4582
4583# if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4584 if (VMX_IS_64BIT_HOST_MODE())
4585 {
4586 Log(("MSR_K6_EFER = %RX64\n", ASMRdMsr(MSR_K6_EFER)));
4587 Log(("MSR_K6_STAR = %RX64\n", ASMRdMsr(MSR_K6_STAR)));
4588 Log(("MSR_K8_LSTAR = %RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
4589 Log(("MSR_K8_CSTAR = %RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
4590 Log(("MSR_K8_SF_MASK = %RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
4591 }
4592# endif
4593#endif /* VBOX_STRICT */
4594 }
4595 break;
4596 }
4597
4598 default:
4599 /* impossible */
4600 AssertMsgFailed(("%Rrc (%#x)\n", VBOXSTRICTRC_VAL(rc), VBOXSTRICTRC_VAL(rc)));
4601 break;
4602 }
4603}
4604
4605#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4606/**
4607 * Prepares for and executes VMLAUNCH (64 bits guest mode)
4608 *
4609 * @returns VBox status code
4610 * @param fResume vmlauch/vmresume
4611 * @param pCtx Guest context
4612 * @param pCache VMCS cache
4613 * @param pVM The VM to operate on.
4614 * @param pVCpu The VMCPU to operate on.
4615 */
4616DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
4617{
4618 uint32_t aParam[6];
4619 PHMGLOBLCPUINFO pCpu;
4620 RTHCPHYS HCPhysCpuPage;
4621 int rc;
4622
4623 pCpu = HWACCMR0GetCurrentCpu();
4624 HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
4625
4626#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4627 pCache->uPos = 1;
4628 pCache->interPD = PGMGetInterPaeCR3(pVM);
4629 pCache->pSwitcher = (uint64_t)pVM->hwaccm.s.pfnHost32ToGuest64R0;
4630#endif
4631
4632#ifdef DEBUG
4633 pCache->TestIn.HCPhysCpuPage= 0;
4634 pCache->TestIn.HCPhysVMCS = 0;
4635 pCache->TestIn.pCache = 0;
4636 pCache->TestOut.HCPhysVMCS = 0;
4637 pCache->TestOut.pCache = 0;
4638 pCache->TestOut.pCtx = 0;
4639 pCache->TestOut.eflags = 0;
4640#endif
4641
4642 aParam[0] = (uint32_t)(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
4643 aParam[1] = (uint32_t)(HCPhysCpuPage >> 32); /* Param 1: VMXON physical address - Hi. */
4644 aParam[2] = (uint32_t)(pVCpu->hwaccm.s.vmx.HCPhysVMCS); /* Param 2: VMCS physical address - Lo. */
4645 aParam[3] = (uint32_t)(pVCpu->hwaccm.s.vmx.HCPhysVMCS >> 32); /* Param 2: VMCS physical address - Hi. */
4646 aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hwaccm.s.vmx.VMCSCache);
4647 aParam[5] = 0;
4648
4649#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4650 pCtx->dr[4] = pVM->hwaccm.s.vmx.pScratchPhys + 16 + 8;
4651 *(uint32_t *)(pVM->hwaccm.s.vmx.pScratch + 16 + 8) = 1;
4652#endif
4653 rc = VMXR0Execute64BitsHandler(pVM, pVCpu, pCtx, pVM->hwaccm.s.pfnVMXGCStartVM64, 6, &aParam[0]);
4654
4655#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4656 Assert(*(uint32_t *)(pVM->hwaccm.s.vmx.pScratch + 16 + 8) == 5);
4657 Assert(pCtx->dr[4] == 10);
4658 *(uint32_t *)(pVM->hwaccm.s.vmx.pScratch + 16 + 8) = 0xff;
4659#endif
4660
4661#ifdef DEBUG
4662 AssertMsg(pCache->TestIn.HCPhysCpuPage== HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
4663 AssertMsg(pCache->TestIn.HCPhysVMCS == pVCpu->hwaccm.s.vmx.HCPhysVMCS, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVMCS, pVCpu->hwaccm.s.vmx.HCPhysVMCS));
4664 AssertMsg(pCache->TestIn.HCPhysVMCS == pCache->TestOut.HCPhysVMCS, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVMCS, pCache->TestOut.HCPhysVMCS));
4665 AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache, pCache->TestOut.pCache));
4666 AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hwaccm.s.vmx.VMCSCache), ("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hwaccm.s.vmx.VMCSCache)));
4667 AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx, pCache->TestOut.pCtx));
4668 Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
4669#endif
4670 return rc;
4671}
4672
4673/**
4674 * Executes the specified handler in 64 mode
4675 *
4676 * @returns VBox status code.
4677 * @param pVM The VM to operate on.
4678 * @param pVCpu The VMCPU to operate on.
4679 * @param pCtx Guest context
4680 * @param pfnHandler RC handler
4681 * @param cbParam Number of parameters
4682 * @param paParam Array of 32 bits parameters
4683 */
4684VMMR0DECL(int) VMXR0Execute64BitsHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, RTRCPTR pfnHandler, uint32_t cbParam, uint32_t *paParam)
4685{
4686 int rc, rc2;
4687 PHMGLOBLCPUINFO pCpu;
4688 RTHCPHYS HCPhysCpuPage;
4689 RTHCUINTREG uOldEFlags;
4690
4691 AssertReturn(pVM->hwaccm.s.pfnHost32ToGuest64R0, VERR_INTERNAL_ERROR);
4692 Assert(pfnHandler);
4693 Assert(pVCpu->hwaccm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hwaccm.s.vmx.VMCSCache.Write.aField));
4694 Assert(pVCpu->hwaccm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hwaccm.s.vmx.VMCSCache.Read.aField));
4695
4696#ifdef VBOX_STRICT
4697 for (unsigned i=0;i<pVCpu->hwaccm.s.vmx.VMCSCache.Write.cValidEntries;i++)
4698 Assert(vmxR0IsValidWriteField(pVCpu->hwaccm.s.vmx.VMCSCache.Write.aField[i]));
4699
4700 for (unsigned i=0;i<pVCpu->hwaccm.s.vmx.VMCSCache.Read.cValidEntries;i++)
4701 Assert(vmxR0IsValidReadField(pVCpu->hwaccm.s.vmx.VMCSCache.Read.aField[i]));
4702#endif
4703
4704 /* Disable interrupts. */
4705 uOldEFlags = ASMIntDisableFlags();
4706
4707 pCpu = HWACCMR0GetCurrentCpu();
4708 HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
4709
4710 /* Clear VM Control Structure. Marking it inactive, clearing implementation specific data and writing back VMCS data to memory. */
4711 VMXClearVMCS(pVCpu->hwaccm.s.vmx.HCPhysVMCS);
4712
4713 /* Leave VMX Root Mode. */
4714 VMXDisable();
4715
4716 ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
4717
4718 CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
4719 CPUMSetHyperEIP(pVCpu, pfnHandler);
4720 for (int i=(int)cbParam-1;i>=0;i--)
4721 CPUMPushHyper(pVCpu, paParam[i]);
4722
4723 STAM_PROFILE_ADV_START(&pVCpu->hwaccm.s.StatWorldSwitch3264, z);
4724 /* Call switcher. */
4725 rc = pVM->hwaccm.s.pfnHost32ToGuest64R0(pVM, RT_OFFSETOF(VM, aCpus[pVCpu->idCpu].cpum) - RT_OFFSETOF(VM, cpum));
4726 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatWorldSwitch3264, z);
4727
4728 /* Make sure the VMX instructions don't cause #UD faults. */
4729 ASMSetCR4(ASMGetCR4() | X86_CR4_VMXE);
4730
4731 /* Enter VMX Root Mode */
4732 rc2 = VMXEnable(HCPhysCpuPage);
4733 if (RT_FAILURE(rc2))
4734 {
4735 ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
4736 ASMSetFlags(uOldEFlags);
4737 return VERR_VMX_VMXON_FAILED;
4738 }
4739
4740 rc2 = VMXActivateVMCS(pVCpu->hwaccm.s.vmx.HCPhysVMCS);
4741 AssertRC(rc2);
4742 Assert(!(ASMGetFlags() & X86_EFL_IF));
4743 ASMSetFlags(uOldEFlags);
4744 return rc;
4745}
4746
4747#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL) */
4748
4749
4750#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
4751/**
4752 * Executes VMWRITE
4753 *
4754 * @returns VBox status code
4755 * @param pVCpu The VMCPU to operate on.
4756 * @param idxField VMCS index
4757 * @param u64Val 16, 32 or 64 bits value
4758 */
4759VMMR0DECL(int) VMXWriteVMCS64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
4760{
4761 int rc;
4762
4763 switch (idxField)
4764 {
4765 case VMX_VMCS_CTRL_TSC_OFFSET_FULL:
4766 case VMX_VMCS_CTRL_IO_BITMAP_A_FULL:
4767 case VMX_VMCS_CTRL_IO_BITMAP_B_FULL:
4768 case VMX_VMCS_CTRL_MSR_BITMAP_FULL:
4769 case VMX_VMCS_CTRL_VMEXIT_MSR_STORE_FULL:
4770 case VMX_VMCS_CTRL_VMEXIT_MSR_LOAD_FULL:
4771 case VMX_VMCS_CTRL_VMENTRY_MSR_LOAD_FULL:
4772 case VMX_VMCS_CTRL_VAPIC_PAGEADDR_FULL:
4773 case VMX_VMCS_CTRL_APIC_ACCESSADDR_FULL:
4774 case VMX_VMCS_GUEST_LINK_PTR_FULL:
4775 case VMX_VMCS_GUEST_PDPTR0_FULL:
4776 case VMX_VMCS_GUEST_PDPTR1_FULL:
4777 case VMX_VMCS_GUEST_PDPTR2_FULL:
4778 case VMX_VMCS_GUEST_PDPTR3_FULL:
4779 case VMX_VMCS_GUEST_DEBUGCTL_FULL:
4780 case VMX_VMCS_GUEST_EFER_FULL:
4781 case VMX_VMCS_CTRL_EPTP_FULL:
4782 /* These fields consist of two parts, which are both writable in 32 bits mode. */
4783 rc = VMXWriteVMCS32(idxField, u64Val);
4784 rc |= VMXWriteVMCS32(idxField + 1, (uint32_t)(u64Val >> 32ULL));
4785 AssertRC(rc);
4786 return rc;
4787
4788 case VMX_VMCS64_GUEST_LDTR_BASE:
4789 case VMX_VMCS64_GUEST_TR_BASE:
4790 case VMX_VMCS64_GUEST_GDTR_BASE:
4791 case VMX_VMCS64_GUEST_IDTR_BASE:
4792 case VMX_VMCS64_GUEST_SYSENTER_EIP:
4793 case VMX_VMCS64_GUEST_SYSENTER_ESP:
4794 case VMX_VMCS64_GUEST_CR0:
4795 case VMX_VMCS64_GUEST_CR4:
4796 case VMX_VMCS64_GUEST_CR3:
4797 case VMX_VMCS64_GUEST_DR7:
4798 case VMX_VMCS64_GUEST_RIP:
4799 case VMX_VMCS64_GUEST_RSP:
4800 case VMX_VMCS64_GUEST_CS_BASE:
4801 case VMX_VMCS64_GUEST_DS_BASE:
4802 case VMX_VMCS64_GUEST_ES_BASE:
4803 case VMX_VMCS64_GUEST_FS_BASE:
4804 case VMX_VMCS64_GUEST_GS_BASE:
4805 case VMX_VMCS64_GUEST_SS_BASE:
4806 /* Queue a 64 bits value as we can't set it in 32 bits host mode. */
4807 if (u64Val >> 32ULL)
4808 rc = VMXWriteCachedVMCSEx(pVCpu, idxField, u64Val);
4809 else
4810 rc = VMXWriteVMCS32(idxField, (uint32_t)u64Val);
4811
4812 return rc;
4813
4814 default:
4815 AssertMsgFailed(("Unexpected field %x\n", idxField));
4816 return VERR_INVALID_PARAMETER;
4817 }
4818}
4819
4820/**
4821 * Cache VMCS writes for performance reasons (Darwin) and for running 64 bits guests on 32 bits hosts.
4822 *
4823 * @param pVCpu The VMCPU to operate on.
4824 * @param idxField VMCS field
4825 * @param u64Val Value
4826 */
4827VMMR0DECL(int) VMXWriteCachedVMCSEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
4828{
4829 PVMCSCACHE pCache = &pVCpu->hwaccm.s.vmx.VMCSCache;
4830
4831 AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1, ("entries=%x\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
4832
4833 /* Make sure there are no duplicates. */
4834 for (unsigned i=0;i<pCache->Write.cValidEntries;i++)
4835 {
4836 if (pCache->Write.aField[i] == idxField)
4837 {
4838 pCache->Write.aFieldVal[i] = u64Val;
4839 return VINF_SUCCESS;
4840 }
4841 }
4842
4843 pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
4844 pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
4845 pCache->Write.cValidEntries++;
4846 return VINF_SUCCESS;
4847}
4848
4849#endif /* HC_ARCH_BITS == 32 && !VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0 */
4850
4851#ifdef VBOX_STRICT
4852static bool vmxR0IsValidReadField(uint32_t idxField)
4853{
4854 switch(idxField)
4855 {
4856 case VMX_VMCS64_GUEST_RIP:
4857 case VMX_VMCS64_GUEST_RSP:
4858 case VMX_VMCS_GUEST_RFLAGS:
4859 case VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE:
4860 case VMX_VMCS_CTRL_CR0_READ_SHADOW:
4861 case VMX_VMCS64_GUEST_CR0:
4862 case VMX_VMCS_CTRL_CR4_READ_SHADOW:
4863 case VMX_VMCS64_GUEST_CR4:
4864 case VMX_VMCS64_GUEST_DR7:
4865 case VMX_VMCS32_GUEST_SYSENTER_CS:
4866 case VMX_VMCS64_GUEST_SYSENTER_EIP:
4867 case VMX_VMCS64_GUEST_SYSENTER_ESP:
4868 case VMX_VMCS32_GUEST_GDTR_LIMIT:
4869 case VMX_VMCS64_GUEST_GDTR_BASE:
4870 case VMX_VMCS32_GUEST_IDTR_LIMIT:
4871 case VMX_VMCS64_GUEST_IDTR_BASE:
4872 case VMX_VMCS16_GUEST_FIELD_CS:
4873 case VMX_VMCS32_GUEST_CS_LIMIT:
4874 case VMX_VMCS64_GUEST_CS_BASE:
4875 case VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS:
4876 case VMX_VMCS16_GUEST_FIELD_DS:
4877 case VMX_VMCS32_GUEST_DS_LIMIT:
4878 case VMX_VMCS64_GUEST_DS_BASE:
4879 case VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS:
4880 case VMX_VMCS16_GUEST_FIELD_ES:
4881 case VMX_VMCS32_GUEST_ES_LIMIT:
4882 case VMX_VMCS64_GUEST_ES_BASE:
4883 case VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS:
4884 case VMX_VMCS16_GUEST_FIELD_FS:
4885 case VMX_VMCS32_GUEST_FS_LIMIT:
4886 case VMX_VMCS64_GUEST_FS_BASE:
4887 case VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS:
4888 case VMX_VMCS16_GUEST_FIELD_GS:
4889 case VMX_VMCS32_GUEST_GS_LIMIT:
4890 case VMX_VMCS64_GUEST_GS_BASE:
4891 case VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS:
4892 case VMX_VMCS16_GUEST_FIELD_SS:
4893 case VMX_VMCS32_GUEST_SS_LIMIT:
4894 case VMX_VMCS64_GUEST_SS_BASE:
4895 case VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS:
4896 case VMX_VMCS16_GUEST_FIELD_LDTR:
4897 case VMX_VMCS32_GUEST_LDTR_LIMIT:
4898 case VMX_VMCS64_GUEST_LDTR_BASE:
4899 case VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS:
4900 case VMX_VMCS16_GUEST_FIELD_TR:
4901 case VMX_VMCS32_GUEST_TR_LIMIT:
4902 case VMX_VMCS64_GUEST_TR_BASE:
4903 case VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS:
4904 case VMX_VMCS32_RO_EXIT_REASON:
4905 case VMX_VMCS32_RO_VM_INSTR_ERROR:
4906 case VMX_VMCS32_RO_EXIT_INSTR_LENGTH:
4907 case VMX_VMCS32_RO_EXIT_INTERRUPTION_ERRCODE:
4908 case VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO:
4909 case VMX_VMCS32_RO_EXIT_INSTR_INFO:
4910 case VMX_VMCS_RO_EXIT_QUALIFICATION:
4911 case VMX_VMCS32_RO_IDT_INFO:
4912 case VMX_VMCS32_RO_IDT_ERRCODE:
4913 case VMX_VMCS64_GUEST_CR3:
4914 case VMX_VMCS_EXIT_PHYS_ADDR_FULL:
4915 return true;
4916 }
4917 return false;
4918}
4919
4920static bool vmxR0IsValidWriteField(uint32_t idxField)
4921{
4922 switch(idxField)
4923 {
4924 case VMX_VMCS64_GUEST_LDTR_BASE:
4925 case VMX_VMCS64_GUEST_TR_BASE:
4926 case VMX_VMCS64_GUEST_GDTR_BASE:
4927 case VMX_VMCS64_GUEST_IDTR_BASE:
4928 case VMX_VMCS64_GUEST_SYSENTER_EIP:
4929 case VMX_VMCS64_GUEST_SYSENTER_ESP:
4930 case VMX_VMCS64_GUEST_CR0:
4931 case VMX_VMCS64_GUEST_CR4:
4932 case VMX_VMCS64_GUEST_CR3:
4933 case VMX_VMCS64_GUEST_DR7:
4934 case VMX_VMCS64_GUEST_RIP:
4935 case VMX_VMCS64_GUEST_RSP:
4936 case VMX_VMCS64_GUEST_CS_BASE:
4937 case VMX_VMCS64_GUEST_DS_BASE:
4938 case VMX_VMCS64_GUEST_ES_BASE:
4939 case VMX_VMCS64_GUEST_FS_BASE:
4940 case VMX_VMCS64_GUEST_GS_BASE:
4941 case VMX_VMCS64_GUEST_SS_BASE:
4942 return true;
4943 }
4944 return false;
4945}
4946
4947#endif
Note: See TracBrowser for help on using the repository browser.

© 2024 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette