VirtualBox

source: vbox/trunk/src/VBox/VMM/VMMR0/HWVMXR0.cpp@ 40228

Last change on this file since 40228 was 39812, checked in by vboxsync, 13 years ago

Trace VM exits.
  • Property svn:eol-style set to native
  • Property svn:keywords set to Id
File size: 200.6 KB
Line 
1/* $Id: HWVMXR0.cpp 39812 2012-01-19 16:56:12Z vboxsync $ */
2/** @file
3 * HM VMX (VT-x) - Host Context Ring 0.
4 */
5
6/*
7 * Copyright (C) 2006-2011 Oracle Corporation
8 *
9 * This file is part of VirtualBox Open Source Edition (OSE), as
10 * available from http://www.virtualbox.org. This file is free software;
11 * you can redistribute it and/or modify it under the terms of the GNU
12 * General Public License (GPL) as published by the Free Software
13 * Foundation, in version 2 as it comes in the "COPYING" file of the
14 * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15 * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16 */
17
18
19/*******************************************************************************
20* Header Files *
21*******************************************************************************/
22#define LOG_GROUP LOG_GROUP_HWACCM
23#include <iprt/asm-amd64-x86.h>
24#include <VBox/vmm/hwaccm.h>
25#include <VBox/vmm/pgm.h>
26#include <VBox/vmm/dbgf.h>
27#include <VBox/vmm/dbgftrace.h>
28#include <VBox/vmm/selm.h>
29#include <VBox/vmm/iom.h>
30#include <VBox/vmm/rem.h>
31#include <VBox/vmm/tm.h>
32#include "HWACCMInternal.h"
33#include <VBox/vmm/vm.h>
34#include <VBox/vmm/pdmapi.h>
35#include <VBox/err.h>
36#include <VBox/log.h>
37#include <iprt/assert.h>
38#include <iprt/param.h>
39#include <iprt/string.h>
40#include <iprt/time.h>
41#ifdef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
42# include <iprt/thread.h>
43#endif
44#include <iprt/x86.h>
45#include "HWVMXR0.h"
46
47/*******************************************************************************
48* Defined Constants And Macros *
49*******************************************************************************/
50#if defined(RT_ARCH_AMD64)
51# define VMX_IS_64BIT_HOST_MODE() (true)
52#elif defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
53# define VMX_IS_64BIT_HOST_MODE() (g_fVMXIs64bitHost != 0)
54#else
55# define VMX_IS_64BIT_HOST_MODE() (false)
56#endif
57
58/*******************************************************************************
59* Global Variables *
60*******************************************************************************/
61/* IO operation lookup arrays. */
62static uint32_t const g_aIOSize[4] = {1, 2, 0, 4};
63static uint32_t const g_aIOOpAnd[4] = {0xff, 0xffff, 0, 0xffffffff};
64
65#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
66/** See HWACCMR0A.asm. */
67extern "C" uint32_t g_fVMXIs64bitHost;
68#endif
69
70/*******************************************************************************
71* Local Functions *
72*******************************************************************************/
73static void hmR0VmxReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, VBOXSTRICTRC rc, PCPUMCTX pCtx);
74static void hmR0VmxSetupTLBEPT(PVM pVM, PVMCPU pVCpu);
75static void hmR0VmxSetupTLBVPID(PVM pVM, PVMCPU pVCpu);
76static void hmR0VmxSetupTLBDummy(PVM pVM, PVMCPU pVCpu);
77static void hmR0VmxFlushEPT(PVM pVM, PVMCPU pVCpu, VMX_FLUSH enmFlush, RTGCPHYS GCPhys);
78static void hmR0VmxFlushVPID(PVM pVM, PVMCPU pVCpu, VMX_FLUSH enmFlush, RTGCPTR GCPtr);
79static void hmR0VmxUpdateExceptionBitmap(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx);
80static void hmR0VmxSetMSRPermission(PVMCPU pVCpu, unsigned ulMSR, bool fRead, bool fWrite);
81
82
83static void hmR0VmxCheckError(PVM pVM, PVMCPU pVCpu, int rc)
84{
85 if (rc == VERR_VMX_GENERIC)
86 {
87 RTCCUINTREG instrError;
88
89 VMXReadVMCS(VMX_VMCS32_RO_VM_INSTR_ERROR, &instrError);
90 pVCpu->hwaccm.s.vmx.lasterror.ulInstrError = instrError;
91 }
92 pVM->hwaccm.s.lLastError = rc;
93}
94
95/**
96 * Sets up and activates VT-x on the current CPU
97 *
98 * @returns VBox status code.
99 * @param pCpu CPU info struct
100 * @param pVM The VM to operate on. (can be NULL after a resume!!)
101 * @param pvCpuPage Pointer to the global cpu page.
102 * @param HCPhysCpuPage Physical address of the global cpu page.
103 */
104VMMR0DECL(int) VMXR0EnableCpu(PHMGLOBLCPUINFO pCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
105{
106 AssertReturn(HCPhysCpuPage != 0 && HCPhysCpuPage != NIL_RTHCPHYS, VERR_INVALID_PARAMETER);
107 AssertReturn(pvCpuPage, VERR_INVALID_PARAMETER);
108 NOREF(pCpu);
109
110 if (pVM)
111 {
112 /* Set revision dword at the beginning of the VMXON structure. */
113 *(uint32_t *)pvCpuPage = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hwaccm.s.vmx.msr.vmx_basic_info);
114 }
115
116 /** @todo we should unmap the two pages from the virtual address space in order to prevent accidental corruption.
117 * (which can have very bad consequences!!!)
118 */
119
120 if (ASMGetCR4() & X86_CR4_VMXE)
121 return VERR_VMX_IN_VMX_ROOT_MODE;
122
123 /* Make sure the VMX instructions don't cause #UD faults. */
124 ASMSetCR4(ASMGetCR4() | X86_CR4_VMXE);
125
126 /* Enter VMX Root Mode. */
127 int rc = VMXEnable(HCPhysCpuPage);
128 if (RT_FAILURE(rc))
129 {
130 ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
131 return VERR_VMX_VMXON_FAILED;
132 }
133 return VINF_SUCCESS;
134}
135
136/**
137 * Deactivates VT-x on the current CPU
138 *
139 * @returns VBox status code.
140 * @param pCpu CPU info struct
141 * @param pvCpuPage Pointer to the global cpu page.
142 * @param HCPhysCpuPage Physical address of the global cpu page.
143 */
144VMMR0DECL(int) VMXR0DisableCpu(PHMGLOBLCPUINFO pCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
145{
146 AssertReturn(HCPhysCpuPage != 0 && HCPhysCpuPage != NIL_RTHCPHYS, VERR_INVALID_PARAMETER);
147 AssertReturn(pvCpuPage, VERR_INVALID_PARAMETER);
148 NOREF(pCpu);
149
150 /* If we're somehow not in VMX root mode, then we shouldn't dare leaving it. */
151 if (!(ASMGetCR4() & X86_CR4_VMXE))
152 return VERR_VMX_NOT_IN_VMX_ROOT_MODE;
153
154 /* Leave VMX Root Mode. */
155 VMXDisable();
156
157 /* And clear the X86_CR4_VMXE bit. */
158 ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
159 return VINF_SUCCESS;
160}
161
162/**
163 * Does Ring-0 per VM VT-x init.
164 *
165 * @returns VBox status code.
166 * @param pVM The VM to operate on.
167 */
168VMMR0DECL(int) VMXR0InitVM(PVM pVM)
169{
170 int rc;
171
172#ifdef LOG_ENABLED
173 SUPR0Printf("VMXR0InitVM %x\n", pVM);
174#endif
175
176 pVM->hwaccm.s.vmx.pMemObjAPIC = NIL_RTR0MEMOBJ;
177
178 if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
179 {
180 /* Allocate one page for the APIC physical page (serves for filtering accesses). */
181 rc = RTR0MemObjAllocCont(&pVM->hwaccm.s.vmx.pMemObjAPIC, PAGE_SIZE, true /* executable R0 mapping */);
182 AssertRC(rc);
183 if (RT_FAILURE(rc))
184 return rc;
185
186 pVM->hwaccm.s.vmx.pAPIC = (uint8_t *)RTR0MemObjAddress(pVM->hwaccm.s.vmx.pMemObjAPIC);
187 pVM->hwaccm.s.vmx.pAPICPhys = RTR0MemObjGetPagePhysAddr(pVM->hwaccm.s.vmx.pMemObjAPIC, 0);
188 ASMMemZero32(pVM->hwaccm.s.vmx.pAPIC, PAGE_SIZE);
189 }
190 else
191 {
192 pVM->hwaccm.s.vmx.pMemObjAPIC = 0;
193 pVM->hwaccm.s.vmx.pAPIC = 0;
194 pVM->hwaccm.s.vmx.pAPICPhys = 0;
195 }
196
197#ifdef VBOX_WITH_CRASHDUMP_MAGIC
198 {
199 rc = RTR0MemObjAllocCont(&pVM->hwaccm.s.vmx.pMemObjScratch, PAGE_SIZE, true /* executable R0 mapping */);
200 AssertRC(rc);
201 if (RT_FAILURE(rc))
202 return rc;
203
204 pVM->hwaccm.s.vmx.pScratch = (uint8_t *)RTR0MemObjAddress(pVM->hwaccm.s.vmx.pMemObjScratch);
205 pVM->hwaccm.s.vmx.pScratchPhys = RTR0MemObjGetPagePhysAddr(pVM->hwaccm.s.vmx.pMemObjScratch, 0);
206
207 ASMMemZero32(pVM->hwaccm.s.vmx.pScratch, PAGE_SIZE);
208 strcpy((char *)pVM->hwaccm.s.vmx.pScratch, "SCRATCH Magic");
209 *(uint64_t *)(pVM->hwaccm.s.vmx.pScratch + 16) = UINT64_C(0xDEADBEEFDEADBEEF);
210 }
211#endif
212
213 /* Allocate VMCBs for all guest CPUs. */
214 for (VMCPUID i = 0; i < pVM->cCpus; i++)
215 {
216 PVMCPU pVCpu = &pVM->aCpus[i];
217
218 pVCpu->hwaccm.s.vmx.hMemObjVMCS = NIL_RTR0MEMOBJ;
219
220 /* Allocate one page for the VM control structure (VMCS). */
221 rc = RTR0MemObjAllocCont(&pVCpu->hwaccm.s.vmx.hMemObjVMCS, PAGE_SIZE, true /* executable R0 mapping */);
222 AssertRC(rc);
223 if (RT_FAILURE(rc))
224 return rc;
225
226 pVCpu->hwaccm.s.vmx.pvVMCS = RTR0MemObjAddress(pVCpu->hwaccm.s.vmx.hMemObjVMCS);
227 pVCpu->hwaccm.s.vmx.HCPhysVMCS = RTR0MemObjGetPagePhysAddr(pVCpu->hwaccm.s.vmx.hMemObjVMCS, 0);
228 ASMMemZeroPage(pVCpu->hwaccm.s.vmx.pvVMCS);
229
230 pVCpu->hwaccm.s.vmx.cr0_mask = 0;
231 pVCpu->hwaccm.s.vmx.cr4_mask = 0;
232
233 /* Allocate one page for the virtual APIC page for TPR caching. */
234 rc = RTR0MemObjAllocCont(&pVCpu->hwaccm.s.vmx.hMemObjVAPIC, PAGE_SIZE, true /* executable R0 mapping */);
235 AssertRC(rc);
236 if (RT_FAILURE(rc))
237 return rc;
238
239 pVCpu->hwaccm.s.vmx.pbVAPIC = (uint8_t *)RTR0MemObjAddress(pVCpu->hwaccm.s.vmx.hMemObjVAPIC);
240 pVCpu->hwaccm.s.vmx.HCPhysVAPIC = RTR0MemObjGetPagePhysAddr(pVCpu->hwaccm.s.vmx.hMemObjVAPIC, 0);
241 ASMMemZeroPage(pVCpu->hwaccm.s.vmx.pbVAPIC);
242
243 /* Allocate the MSR bitmap if this feature is supported. */
244 if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
245 {
246 rc = RTR0MemObjAllocCont(&pVCpu->hwaccm.s.vmx.pMemObjMSRBitmap, PAGE_SIZE, true /* executable R0 mapping */);
247 AssertRC(rc);
248 if (RT_FAILURE(rc))
249 return rc;
250
251 pVCpu->hwaccm.s.vmx.pMSRBitmap = (uint8_t *)RTR0MemObjAddress(pVCpu->hwaccm.s.vmx.pMemObjMSRBitmap);
252 pVCpu->hwaccm.s.vmx.pMSRBitmapPhys = RTR0MemObjGetPagePhysAddr(pVCpu->hwaccm.s.vmx.pMemObjMSRBitmap, 0);
253 memset(pVCpu->hwaccm.s.vmx.pMSRBitmap, 0xff, PAGE_SIZE);
254 }
255
256#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
257 /* Allocate one page for the guest MSR load area (for preloading guest MSRs during the world switch). */
258 rc = RTR0MemObjAllocCont(&pVCpu->hwaccm.s.vmx.pMemObjGuestMSR, PAGE_SIZE, true /* executable R0 mapping */);
259 AssertRC(rc);
260 if (RT_FAILURE(rc))
261 return rc;
262
263 pVCpu->hwaccm.s.vmx.pGuestMSR = (uint8_t *)RTR0MemObjAddress(pVCpu->hwaccm.s.vmx.pMemObjGuestMSR);
264 pVCpu->hwaccm.s.vmx.pGuestMSRPhys = RTR0MemObjGetPagePhysAddr(pVCpu->hwaccm.s.vmx.pMemObjGuestMSR, 0);
265 memset(pVCpu->hwaccm.s.vmx.pGuestMSR, 0, PAGE_SIZE);
266
267 /* Allocate one page for the host MSR load area (for restoring host MSRs after the world switch back). */
268 rc = RTR0MemObjAllocCont(&pVCpu->hwaccm.s.vmx.pMemObjHostMSR, PAGE_SIZE, true /* executable R0 mapping */);
269 AssertRC(rc);
270 if (RT_FAILURE(rc))
271 return rc;
272
273 pVCpu->hwaccm.s.vmx.pHostMSR = (uint8_t *)RTR0MemObjAddress(pVCpu->hwaccm.s.vmx.pMemObjHostMSR);
274 pVCpu->hwaccm.s.vmx.pHostMSRPhys = RTR0MemObjGetPagePhysAddr(pVCpu->hwaccm.s.vmx.pMemObjHostMSR, 0);
275 memset(pVCpu->hwaccm.s.vmx.pHostMSR, 0, PAGE_SIZE);
276#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
277
278 /* Current guest paging mode. */
279 pVCpu->hwaccm.s.vmx.enmLastSeenGuestMode = PGMMODE_REAL;
280
281#ifdef LOG_ENABLED
282 SUPR0Printf("VMXR0InitVM %x VMCS=%x (%x)\n", pVM, pVCpu->hwaccm.s.vmx.pvVMCS, (uint32_t)pVCpu->hwaccm.s.vmx.HCPhysVMCS);
283#endif
284 }
285
286 return VINF_SUCCESS;
287}
288
289/**
290 * Does Ring-0 per VM VT-x termination.
291 *
292 * @returns VBox status code.
293 * @param pVM The VM to operate on.
294 */
295VMMR0DECL(int) VMXR0TermVM(PVM pVM)
296{
297 for (VMCPUID i = 0; i < pVM->cCpus; i++)
298 {
299 PVMCPU pVCpu = &pVM->aCpus[i];
300
301 if (pVCpu->hwaccm.s.vmx.hMemObjVMCS != NIL_RTR0MEMOBJ)
302 {
303 RTR0MemObjFree(pVCpu->hwaccm.s.vmx.hMemObjVMCS, false);
304 pVCpu->hwaccm.s.vmx.hMemObjVMCS = NIL_RTR0MEMOBJ;
305 pVCpu->hwaccm.s.vmx.pvVMCS = 0;
306 pVCpu->hwaccm.s.vmx.HCPhysVMCS = 0;
307 }
308 if (pVCpu->hwaccm.s.vmx.hMemObjVAPIC != NIL_RTR0MEMOBJ)
309 {
310 RTR0MemObjFree(pVCpu->hwaccm.s.vmx.hMemObjVAPIC, false);
311 pVCpu->hwaccm.s.vmx.hMemObjVAPIC = NIL_RTR0MEMOBJ;
312 pVCpu->hwaccm.s.vmx.pbVAPIC = 0;
313 pVCpu->hwaccm.s.vmx.HCPhysVAPIC = 0;
314 }
315 if (pVCpu->hwaccm.s.vmx.pMemObjMSRBitmap != NIL_RTR0MEMOBJ)
316 {
317 RTR0MemObjFree(pVCpu->hwaccm.s.vmx.pMemObjMSRBitmap, false);
318 pVCpu->hwaccm.s.vmx.pMemObjMSRBitmap = NIL_RTR0MEMOBJ;
319 pVCpu->hwaccm.s.vmx.pMSRBitmap = 0;
320 pVCpu->hwaccm.s.vmx.pMSRBitmapPhys = 0;
321 }
322#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
323 if (pVCpu->hwaccm.s.vmx.pMemObjHostMSR != NIL_RTR0MEMOBJ)
324 {
325 RTR0MemObjFree(pVCpu->hwaccm.s.vmx.pMemObjHostMSR, false);
326 pVCpu->hwaccm.s.vmx.pMemObjHostMSR = NIL_RTR0MEMOBJ;
327 pVCpu->hwaccm.s.vmx.pHostMSR = 0;
328 pVCpu->hwaccm.s.vmx.pHostMSRPhys = 0;
329 }
330 if (pVCpu->hwaccm.s.vmx.pMemObjGuestMSR != NIL_RTR0MEMOBJ)
331 {
332 RTR0MemObjFree(pVCpu->hwaccm.s.vmx.pMemObjGuestMSR, false);
333 pVCpu->hwaccm.s.vmx.pMemObjGuestMSR = NIL_RTR0MEMOBJ;
334 pVCpu->hwaccm.s.vmx.pGuestMSR = 0;
335 pVCpu->hwaccm.s.vmx.pGuestMSRPhys = 0;
336 }
337#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
338 }
339 if (pVM->hwaccm.s.vmx.pMemObjAPIC != NIL_RTR0MEMOBJ)
340 {
341 RTR0MemObjFree(pVM->hwaccm.s.vmx.pMemObjAPIC, false);
342 pVM->hwaccm.s.vmx.pMemObjAPIC = NIL_RTR0MEMOBJ;
343 pVM->hwaccm.s.vmx.pAPIC = 0;
344 pVM->hwaccm.s.vmx.pAPICPhys = 0;
345 }
346#ifdef VBOX_WITH_CRASHDUMP_MAGIC
347 if (pVM->hwaccm.s.vmx.pMemObjScratch != NIL_RTR0MEMOBJ)
348 {
349 ASMMemZero32(pVM->hwaccm.s.vmx.pScratch, PAGE_SIZE);
350 RTR0MemObjFree(pVM->hwaccm.s.vmx.pMemObjScratch, false);
351 pVM->hwaccm.s.vmx.pMemObjScratch = NIL_RTR0MEMOBJ;
352 pVM->hwaccm.s.vmx.pScratch = 0;
353 pVM->hwaccm.s.vmx.pScratchPhys = 0;
354 }
355#endif
356 return VINF_SUCCESS;
357}
358
359/**
360 * Sets up VT-x for the specified VM
361 *
362 * @returns VBox status code.
363 * @param pVM The VM to operate on.
364 */
365VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
366{
367 int rc = VINF_SUCCESS;
368 uint32_t val;
369
370 AssertReturn(pVM, VERR_INVALID_PARAMETER);
371
372 for (VMCPUID i = 0; i < pVM->cCpus; i++)
373 {
374 PVMCPU pVCpu = &pVM->aCpus[i];
375
376 AssertPtr(pVCpu->hwaccm.s.vmx.pvVMCS);
377
378 /* Set revision dword at the beginning of the VMCS structure. */
379 *(uint32_t *)pVCpu->hwaccm.s.vmx.pvVMCS = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hwaccm.s.vmx.msr.vmx_basic_info);
380
381 /* Clear VM Control Structure. */
382 Log(("HCPhysVMCS = %RHp\n", pVCpu->hwaccm.s.vmx.HCPhysVMCS));
383 rc = VMXClearVMCS(pVCpu->hwaccm.s.vmx.HCPhysVMCS);
384 if (RT_FAILURE(rc))
385 goto vmx_end;
386
387 /* Activate the VM Control Structure. */
388 rc = VMXActivateVMCS(pVCpu->hwaccm.s.vmx.HCPhysVMCS);
389 if (RT_FAILURE(rc))
390 goto vmx_end;
391
392 /* VMX_VMCS_CTRL_PIN_EXEC_CONTROLS
393 * Set required bits to one and zero according to the MSR capabilities.
394 */
395 val = pVM->hwaccm.s.vmx.msr.vmx_pin_ctls.n.disallowed0;
396 /* External and non-maskable interrupts cause VM-exits. */
397 val |= VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_EXT_INT_EXIT | VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_NMI_EXIT;
398 /* enable the preemption timer. */
399 if (pVM->hwaccm.s.vmx.fUsePreemptTimer)
400 val |= VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_PREEMPT_TIMER;
401 val &= pVM->hwaccm.s.vmx.msr.vmx_pin_ctls.n.allowed1;
402
403 rc = VMXWriteVMCS(VMX_VMCS_CTRL_PIN_EXEC_CONTROLS, val);
404 AssertRC(rc);
405
406 /* VMX_VMCS_CTRL_PROC_EXEC_CONTROLS
407 * Set required bits to one and zero according to the MSR capabilities.
408 */
409 val = pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.disallowed0;
410 /* Program which event cause VM-exits and which features we want to use. */
411 val = val | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_HLT_EXIT
412 | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_TSC_OFFSET
413 | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT
414 | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_UNCOND_IO_EXIT
415 | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDPMC_EXIT
416 | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_EXIT
417 | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MWAIT_EXIT; /* don't execute mwait or else we'll idle inside the guest (host thinks the cpu load is high) */
418
419 /* Without nested paging we should intercept invlpg and cr3 mov instructions. */
420 if (!pVM->hwaccm.s.fNestedPaging)
421 val |= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INVLPG_EXIT
422 | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
423 | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT;
424
425 /* Note: VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MWAIT_EXIT might cause a vmlaunch failure with an invalid control fields error. (combined with some other exit reasons) */
426 if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
427 {
428 /* CR8 reads from the APIC shadow page; writes cause an exit is they lower the TPR below the threshold */
429 val |= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW;
430 Assert(pVM->hwaccm.s.vmx.pAPIC);
431 }
432 else
433 /* Exit on CR8 reads & writes in case the TPR shadow feature isn't present. */
434 val |= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR8_STORE_EXIT | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR8_LOAD_EXIT;
435
436 if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
437 {
438 Assert(pVCpu->hwaccm.s.vmx.pMSRBitmapPhys);
439 val |= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS;
440 }
441
442 /* We will use the secondary control if it's present. */
443 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
444
445 /* Mask away the bits that the CPU doesn't support */
446 /** @todo make sure they don't conflict with the above requirements. */
447 val &= pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1;
448 pVCpu->hwaccm.s.vmx.proc_ctls = val;
449
450 rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, val);
451 AssertRC(rc);
452
453 if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
454 {
455 /* VMX_VMCS_CTRL_PROC_EXEC_CONTROLS2
456 * Set required bits to one and zero according to the MSR capabilities.
457 */
458 val = pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2.n.disallowed0;
459 val |= VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT;
460
461#ifdef HWACCM_VTX_WITH_EPT
462 if (pVM->hwaccm.s.fNestedPaging)
463 val |= VMX_VMCS_CTRL_PROC_EXEC2_EPT;
464#endif /* HWACCM_VTX_WITH_EPT */
465#ifdef HWACCM_VTX_WITH_VPID
466 else
467 if (pVM->hwaccm.s.vmx.fVPID)
468 val |= VMX_VMCS_CTRL_PROC_EXEC2_VPID;
469#endif /* HWACCM_VTX_WITH_VPID */
470
471 if (pVM->hwaccm.s.fHasIoApic)
472 val |= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC;
473
474 if (pVM->hwaccm.s.vmx.fUnrestrictedGuest)
475 val |= VMX_VMCS_CTRL_PROC_EXEC2_REAL_MODE;
476
477 /* Mask away the bits that the CPU doesn't support */
478 /** @todo make sure they don't conflict with the above requirements. */
479 val &= pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2.n.allowed1;
480 pVCpu->hwaccm.s.vmx.proc_ctls2 = val;
481 rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS2, val);
482 AssertRC(rc);
483 }
484
485 /* VMX_VMCS_CTRL_CR3_TARGET_COUNT
486 * Set required bits to one and zero according to the MSR capabilities.
487 */
488 rc = VMXWriteVMCS(VMX_VMCS_CTRL_CR3_TARGET_COUNT, 0);
489 AssertRC(rc);
490
491 /* Forward all exception except #NM & #PF to the guest.
492 * We always need to check pagefaults since our shadow page table can be out of sync.
493 * And we always lazily sync the FPU & XMM state.
494 */
495
496 /** @todo Possible optimization:
497 * Keep the FPU and XMM state current in the EM thread. That way there's no need to
498 * lazily sync anything, but the downside is that we can't use the FPU stack or XMM
499 * registers ourselves of course.
500 *
501 * Note: only possible if the current state is actually ours (X86_CR0_TS flag)
502 */
503
504 /* Don't filter page faults; all of them should cause a switch. */
505 rc = VMXWriteVMCS(VMX_VMCS_CTRL_PAGEFAULT_ERROR_MASK, 0);
506 rc |= VMXWriteVMCS(VMX_VMCS_CTRL_PAGEFAULT_ERROR_MATCH, 0);
507 AssertRC(rc);
508
509 /* Init TSC offset to zero. */
510 rc = VMXWriteVMCS64(VMX_VMCS_CTRL_TSC_OFFSET_FULL, 0);
511 AssertRC(rc);
512
513 rc = VMXWriteVMCS64(VMX_VMCS_CTRL_IO_BITMAP_A_FULL, 0);
514 AssertRC(rc);
515
516 rc = VMXWriteVMCS64(VMX_VMCS_CTRL_IO_BITMAP_B_FULL, 0);
517 AssertRC(rc);
518
519 /* Set the MSR bitmap address. */
520 if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
521 {
522 Assert(pVCpu->hwaccm.s.vmx.pMSRBitmapPhys);
523
524 rc = VMXWriteVMCS64(VMX_VMCS_CTRL_MSR_BITMAP_FULL, pVCpu->hwaccm.s.vmx.pMSRBitmapPhys);
525 AssertRC(rc);
526
527 /* Allow the guest to directly modify these MSRs; they are restored and saved automatically. */
528 hmR0VmxSetMSRPermission(pVCpu, MSR_IA32_SYSENTER_CS, true, true);
529 hmR0VmxSetMSRPermission(pVCpu, MSR_IA32_SYSENTER_ESP, true, true);
530 hmR0VmxSetMSRPermission(pVCpu, MSR_IA32_SYSENTER_EIP, true, true);
531 hmR0VmxSetMSRPermission(pVCpu, MSR_K8_LSTAR, true, true);
532 hmR0VmxSetMSRPermission(pVCpu, MSR_K6_STAR, true, true);
533 hmR0VmxSetMSRPermission(pVCpu, MSR_K8_SF_MASK, true, true);
534 hmR0VmxSetMSRPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, true, true);
535 hmR0VmxSetMSRPermission(pVCpu, MSR_K8_GS_BASE, true, true);
536 hmR0VmxSetMSRPermission(pVCpu, MSR_K8_FS_BASE, true, true);
537 }
538
539#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
540 /* Set the guest & host MSR load/store physical addresses. */
541 Assert(pVCpu->hwaccm.s.vmx.pGuestMSRPhys);
542 rc = VMXWriteVMCS64(VMX_VMCS_CTRL_VMENTRY_MSR_LOAD_FULL, pVCpu->hwaccm.s.vmx.pGuestMSRPhys);
543 AssertRC(rc);
544 rc = VMXWriteVMCS64(VMX_VMCS_CTRL_VMEXIT_MSR_STORE_FULL, pVCpu->hwaccm.s.vmx.pGuestMSRPhys);
545 AssertRC(rc);
546
547 Assert(pVCpu->hwaccm.s.vmx.pHostMSRPhys);
548 rc = VMXWriteVMCS64(VMX_VMCS_CTRL_VMEXIT_MSR_LOAD_FULL, pVCpu->hwaccm.s.vmx.pHostMSRPhys);
549 AssertRC(rc);
550#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
551
552 rc = VMXWriteVMCS(VMX_VMCS_CTRL_ENTRY_MSR_LOAD_COUNT, 0);
553 AssertRC(rc);
554
555 rc = VMXWriteVMCS(VMX_VMCS_CTRL_EXIT_MSR_STORE_COUNT, 0);
556 AssertRC(rc);
557
558 if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
559 {
560 Assert(pVM->hwaccm.s.vmx.pMemObjAPIC);
561 /* Optional */
562 rc = VMXWriteVMCS(VMX_VMCS_CTRL_TPR_THRESHOLD, 0);
563 rc |= VMXWriteVMCS64(VMX_VMCS_CTRL_VAPIC_PAGEADDR_FULL, pVCpu->hwaccm.s.vmx.HCPhysVAPIC);
564
565 if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
566 rc |= VMXWriteVMCS64(VMX_VMCS_CTRL_APIC_ACCESSADDR_FULL, pVM->hwaccm.s.vmx.pAPICPhys);
567
568 AssertRC(rc);
569 }
570
571 /* Set link pointer to -1. Not currently used. */
572 rc = VMXWriteVMCS64(VMX_VMCS_GUEST_LINK_PTR_FULL, 0xFFFFFFFFFFFFFFFFULL);
573 AssertRC(rc);
574
575 /* Clear VM Control Structure. Marking it inactive, clearing implementation specific data and writing back VMCS data to memory. */
576 rc = VMXClearVMCS(pVCpu->hwaccm.s.vmx.HCPhysVMCS);
577 AssertRC(rc);
578
579 /* Configure the VMCS read cache. */
580 PVMCSCACHE pCache = &pVCpu->hwaccm.s.vmx.VMCSCache;
581
582 VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_RIP);
583 VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_RSP);
584 VMXSetupCachedReadVMCS(pCache, VMX_VMCS_GUEST_RFLAGS);
585 VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE);
586 VMXSetupCachedReadVMCS(pCache, VMX_VMCS_CTRL_CR0_READ_SHADOW);
587 VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_CR0);
588 VMXSetupCachedReadVMCS(pCache, VMX_VMCS_CTRL_CR4_READ_SHADOW);
589 VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_CR4);
590 VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_DR7);
591 VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_GUEST_SYSENTER_CS);
592 VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_SYSENTER_EIP);
593 VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_SYSENTER_ESP);
594 VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_GUEST_GDTR_LIMIT);
595 VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_GDTR_BASE);
596 VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_GUEST_IDTR_LIMIT);
597 VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_IDTR_BASE);
598
599 VMX_SETUP_SELREG(ES, pCache);
600 VMX_SETUP_SELREG(SS, pCache);
601 VMX_SETUP_SELREG(CS, pCache);
602 VMX_SETUP_SELREG(DS, pCache);
603 VMX_SETUP_SELREG(FS, pCache);
604 VMX_SETUP_SELREG(GS, pCache);
605 VMX_SETUP_SELREG(LDTR, pCache);
606 VMX_SETUP_SELREG(TR, pCache);
607
608 /* Status code VMCS reads. */
609 VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_EXIT_REASON);
610 VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_VM_INSTR_ERROR);
611 VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_EXIT_INSTR_LENGTH);
612 VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_EXIT_INTERRUPTION_ERRCODE);
613 VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO);
614 VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_EXIT_INSTR_INFO);
615 VMXSetupCachedReadVMCS(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
616 VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_IDT_INFO);
617 VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_IDT_ERRCODE);
618
619 if (pVM->hwaccm.s.fNestedPaging)
620 {
621 VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_CR3);
622 VMXSetupCachedReadVMCS(pCache, VMX_VMCS_EXIT_PHYS_ADDR_FULL);
623 pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
624 }
625 else
626 pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
627 } /* for each VMCPU */
628
629 /* Choose the right TLB setup function. */
630 if (pVM->hwaccm.s.fNestedPaging)
631 {
632 pVM->hwaccm.s.vmx.pfnSetupTaggedTLB = hmR0VmxSetupTLBEPT;
633
634 /* Default values for flushing. */
635 pVM->hwaccm.s.vmx.enmFlushPage = VMX_FLUSH_ALL_CONTEXTS;
636 pVM->hwaccm.s.vmx.enmFlushContext = VMX_FLUSH_ALL_CONTEXTS;
637
638 /* If the capabilities specify we can do more, then make use of it. */
639 if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVEPT_CAPS_INDIV)
640 pVM->hwaccm.s.vmx.enmFlushPage = VMX_FLUSH_PAGE;
641 else
642 if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVEPT_CAPS_CONTEXT)
643 pVM->hwaccm.s.vmx.enmFlushPage = VMX_FLUSH_SINGLE_CONTEXT;
644
645 if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVEPT_CAPS_CONTEXT)
646 pVM->hwaccm.s.vmx.enmFlushContext = VMX_FLUSH_SINGLE_CONTEXT;
647 }
648#ifdef HWACCM_VTX_WITH_VPID
649 else
650 if (pVM->hwaccm.s.vmx.fVPID)
651 {
652 pVM->hwaccm.s.vmx.pfnSetupTaggedTLB = hmR0VmxSetupTLBVPID;
653
654 /* Default values for flushing. */
655 pVM->hwaccm.s.vmx.enmFlushPage = VMX_FLUSH_ALL_CONTEXTS;
656 pVM->hwaccm.s.vmx.enmFlushContext = VMX_FLUSH_ALL_CONTEXTS;
657
658 /* If the capabilities specify we can do more, then make use of it. */
659 if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVVPID_CAPS_INDIV)
660 pVM->hwaccm.s.vmx.enmFlushPage = VMX_FLUSH_PAGE;
661 else
662 if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVVPID_CAPS_CONTEXT)
663 pVM->hwaccm.s.vmx.enmFlushPage = VMX_FLUSH_SINGLE_CONTEXT;
664
665 if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVVPID_CAPS_CONTEXT)
666 pVM->hwaccm.s.vmx.enmFlushContext = VMX_FLUSH_SINGLE_CONTEXT;
667 }
668#endif /* HWACCM_VTX_WITH_VPID */
669 else
670 pVM->hwaccm.s.vmx.pfnSetupTaggedTLB = hmR0VmxSetupTLBDummy;
671
672vmx_end:
673 hmR0VmxCheckError(pVM, &pVM->aCpus[0], rc);
674 return rc;
675}
676
677/**
678 * Sets the permission bits for the specified MSR
679 *
680 * @param pVCpu The VMCPU to operate on.
681 * @param ulMSR MSR value
682 * @param fRead Reading allowed/disallowed
683 * @param fWrite Writing allowed/disallowed
684 */
685static void hmR0VmxSetMSRPermission(PVMCPU pVCpu, unsigned ulMSR, bool fRead, bool fWrite)
686{
687 unsigned ulBit;
688 uint8_t *pMSRBitmap = (uint8_t *)pVCpu->hwaccm.s.vmx.pMSRBitmap;
689
690 /* Layout:
691 * 0x000 - 0x3ff - Low MSR read bits
692 * 0x400 - 0x7ff - High MSR read bits
693 * 0x800 - 0xbff - Low MSR write bits
694 * 0xc00 - 0xfff - High MSR write bits
695 */
696 if (ulMSR <= 0x00001FFF)
697 {
698 /* Pentium-compatible MSRs */
699 ulBit = ulMSR;
700 }
701 else
702 if ( ulMSR >= 0xC0000000
703 && ulMSR <= 0xC0001FFF)
704 {
705 /* AMD Sixth Generation x86 Processor MSRs */
706 ulBit = (ulMSR - 0xC0000000);
707 pMSRBitmap += 0x400;
708 }
709 else
710 {
711 AssertFailed();
712 return;
713 }
714
715 Assert(ulBit <= 0x1fff);
716 if (fRead)
717 ASMBitClear(pMSRBitmap, ulBit);
718 else
719 ASMBitSet(pMSRBitmap, ulBit);
720
721 if (fWrite)
722 ASMBitClear(pMSRBitmap + 0x800, ulBit);
723 else
724 ASMBitSet(pMSRBitmap + 0x800, ulBit);
725}
726
727
728/**
729 * Injects an event (trap or external interrupt)
730 *
731 * @returns VBox status code. Note that it may return VINF_EM_RESET to
732 * indicate a triple fault when injecting X86_XCPT_DF.
733 *
734 * @param pVM The VM to operate on.
735 * @param pVCpu The VMCPU to operate on.
736 * @param pCtx CPU Context
737 * @param intInfo VMX interrupt info
738 * @param cbInstr Opcode length of faulting instruction
739 * @param errCode Error code (optional)
740 */
741static int hmR0VmxInjectEvent(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, uint32_t intInfo, uint32_t cbInstr, uint32_t errCode)
742{
743 int rc;
744 uint32_t iGate = VMX_EXIT_INTERRUPTION_INFO_VECTOR(intInfo);
745
746#ifdef VBOX_WITH_STATISTICS
747 STAM_COUNTER_INC(&pVCpu->hwaccm.s.paStatInjectedIrqsR0[iGate & MASK_INJECT_IRQ_STAT]);
748#endif
749
750#ifdef VBOX_STRICT
751 if (iGate == 0xE)
752 LogFlow(("hmR0VmxInjectEvent: Injecting interrupt %d at %RGv error code=%08x CR2=%RGv intInfo=%08x\n", iGate, (RTGCPTR)pCtx->rip, errCode, pCtx->cr2, intInfo));
753 else
754 if (iGate < 0x20)
755 LogFlow(("hmR0VmxInjectEvent: Injecting interrupt %d at %RGv error code=%08x\n", iGate, (RTGCPTR)pCtx->rip, errCode));
756 else
757 {
758 LogFlow(("INJ-EI: %x at %RGv\n", iGate, (RTGCPTR)pCtx->rip));
759 Assert( VMX_EXIT_INTERRUPTION_INFO_TYPE(intInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW
760 || !VMCPU_FF_ISSET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
761 Assert( VMX_EXIT_INTERRUPTION_INFO_TYPE(intInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW
762 || pCtx->eflags.u32 & X86_EFL_IF);
763 }
764#endif
765
766 if ( CPUMIsGuestInRealModeEx(pCtx)
767 && pVM->hwaccm.s.vmx.pRealModeTSS)
768 {
769 RTGCPHYS GCPhysHandler;
770 uint16_t offset, ip;
771 RTSEL sel;
772
773 /* Injecting events doesn't work right with real mode emulation.
774 * (#GP if we try to inject external hardware interrupts)
775 * Inject the interrupt or trap directly instead.
776 *
777 * ASSUMES no access handlers for the bits we read or write below (should be safe).
778 */
779 Log(("Manual interrupt/trap '%x' inject (real mode)\n", iGate));
780
781 /* Check if the interrupt handler is present. */
782 if (iGate * 4 + 3 > pCtx->idtr.cbIdt)
783 {
784 Log(("IDT cbIdt violation\n"));
785 if (iGate != X86_XCPT_DF)
786 {
787 uint32_t intInfo2;
788
789 intInfo2 = (iGate == X86_XCPT_GP) ? (uint32_t)X86_XCPT_DF : iGate;
790 intInfo2 |= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
791 intInfo2 |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
792 intInfo2 |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HWEXCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
793
794 return hmR0VmxInjectEvent(pVM, pVCpu, pCtx, intInfo2, 0, 0 /* no error code according to the Intel docs */);
795 }
796 Log(("Triple fault -> reset the VM!\n"));
797 return VINF_EM_RESET;
798 }
799 if ( VMX_EXIT_INTERRUPTION_INFO_TYPE(intInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW
800 || iGate == 3 /* Both #BP and #OF point to the instruction after. */
801 || iGate == 4)
802 {
803 ip = pCtx->ip + cbInstr;
804 }
805 else
806 ip = pCtx->ip;
807
808 /* Read the selector:offset pair of the interrupt handler. */
809 GCPhysHandler = (RTGCPHYS)pCtx->idtr.pIdt + iGate * 4;
810 rc = PGMPhysSimpleReadGCPhys(pVM, &offset, GCPhysHandler, sizeof(offset)); AssertRC(rc);
811 rc = PGMPhysSimpleReadGCPhys(pVM, &sel, GCPhysHandler + 2, sizeof(sel)); AssertRC(rc);
812
813 LogFlow(("IDT handler %04X:%04X\n", sel, offset));
814
815 /* Construct the stack frame. */
816 /** @todo should check stack limit. */
817 pCtx->sp -= 2;
818 LogFlow(("ss:sp %04X:%04X eflags=%x\n", pCtx->ss, pCtx->sp, pCtx->eflags.u));
819 rc = PGMPhysSimpleWriteGCPhys(pVM, pCtx->ssHid.u64Base + pCtx->sp, &pCtx->eflags, sizeof(uint16_t)); AssertRC(rc);
820 pCtx->sp -= 2;
821 LogFlow(("ss:sp %04X:%04X cs=%x\n", pCtx->ss, pCtx->sp, pCtx->cs));
822 rc = PGMPhysSimpleWriteGCPhys(pVM, pCtx->ssHid.u64Base + pCtx->sp, &pCtx->cs, sizeof(uint16_t)); AssertRC(rc);
823 pCtx->sp -= 2;
824 LogFlow(("ss:sp %04X:%04X ip=%x\n", pCtx->ss, pCtx->sp, ip));
825 rc = PGMPhysSimpleWriteGCPhys(pVM, pCtx->ssHid.u64Base + pCtx->sp, &ip, sizeof(ip)); AssertRC(rc);
826
827 /* Update the CPU state for executing the handler. */
828 pCtx->rip = offset;
829 pCtx->cs = sel;
830 pCtx->csHid.u64Base = sel << 4;
831 pCtx->eflags.u &= ~(X86_EFL_IF|X86_EFL_TF|X86_EFL_RF|X86_EFL_AC);
832
833 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_GUEST_SEGMENT_REGS;
834 return VINF_SUCCESS;
835 }
836
837 /* Set event injection state. */
838 rc = VMXWriteVMCS(VMX_VMCS_CTRL_ENTRY_IRQ_INFO, intInfo | (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT));
839
840 rc |= VMXWriteVMCS(VMX_VMCS_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
841 rc |= VMXWriteVMCS(VMX_VMCS_CTRL_ENTRY_EXCEPTION_ERRCODE, errCode);
842
843 AssertRC(rc);
844 return rc;
845}
846
847
848/**
849 * Checks for pending guest interrupts and injects them
850 *
851 * @returns VBox status code.
852 * @param pVM The VM to operate on.
853 * @param pVCpu The VMCPU to operate on.
854 * @param pCtx CPU Context
855 */
856static int hmR0VmxCheckPendingInterrupt(PVM pVM, PVMCPU pVCpu, CPUMCTX *pCtx)
857{
858 int rc;
859
860 /* Dispatch any pending interrupts. (injected before, but a VM exit occurred prematurely) */
861 if (pVCpu->hwaccm.s.Event.fPending)
862 {
863 Log(("CPU%d: Reinjecting event %RX64 %08x at %RGv cr2=%RX64\n", pVCpu->idCpu, pVCpu->hwaccm.s.Event.intInfo, pVCpu->hwaccm.s.Event.errCode, (RTGCPTR)pCtx->rip, pCtx->cr2));
864 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatIntReinject);
865 rc = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, pVCpu->hwaccm.s.Event.intInfo, 0, pVCpu->hwaccm.s.Event.errCode);
866 AssertRC(rc);
867
868 pVCpu->hwaccm.s.Event.fPending = false;
869 return VINF_SUCCESS;
870 }
871
872 /* If an active trap is already pending, then we must forward it first! */
873 if (!TRPMHasTrap(pVCpu))
874 {
875 if (VMCPU_FF_TESTANDCLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI))
876 {
877 RTGCUINTPTR intInfo;
878
879 Log(("CPU%d: injecting #NMI\n", pVCpu->idCpu));
880
881 intInfo = X86_XCPT_NMI;
882 intInfo |= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
883 intInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
884
885 rc = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, intInfo, 0, 0);
886 AssertRC(rc);
887
888 return VINF_SUCCESS;
889 }
890
891 /* @todo SMI interrupts. */
892
893 /* When external interrupts are pending, we should exit the VM when IF is set. */
894 if (VMCPU_FF_ISPENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC|VMCPU_FF_INTERRUPT_PIC)))
895 {
896 if (!(pCtx->eflags.u32 & X86_EFL_IF))
897 {
898 if (!(pVCpu->hwaccm.s.vmx.proc_ctls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_IRQ_WINDOW_EXIT))
899 {
900 LogFlow(("Enable irq window exit!\n"));
901 pVCpu->hwaccm.s.vmx.proc_ctls |= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_IRQ_WINDOW_EXIT;
902 rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
903 AssertRC(rc);
904 }
905 /* else nothing to do but wait */
906 }
907 else
908 if (!VMCPU_FF_ISSET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
909 {
910 uint8_t u8Interrupt;
911
912 rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
913 Log(("CPU%d: Dispatch interrupt: u8Interrupt=%x (%d) rc=%Rrc cs:rip=%04X:%RGv\n", pVCpu->idCpu, u8Interrupt, u8Interrupt, rc, pCtx->cs, (RTGCPTR)pCtx->rip));
914 if (RT_SUCCESS(rc))
915 {
916 rc = TRPMAssertTrap(pVCpu, u8Interrupt, TRPM_HARDWARE_INT);
917 AssertRC(rc);
918 }
919 else
920 {
921 /* Can only happen in rare cases where a pending interrupt is cleared behind our back */
922 Assert(!VMCPU_FF_ISPENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC|VMCPU_FF_INTERRUPT_PIC)));
923 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatSwitchGuestIrq);
924 /* Just continue */
925 }
926 }
927 else
928 Log(("Pending interrupt blocked at %RGv by VM_FF_INHIBIT_INTERRUPTS!!\n", (RTGCPTR)pCtx->rip));
929 }
930 }
931
932#ifdef VBOX_STRICT
933 if (TRPMHasTrap(pVCpu))
934 {
935 uint8_t u8Vector;
936 rc = TRPMQueryTrapAll(pVCpu, &u8Vector, 0, 0, 0);
937 AssertRC(rc);
938 }
939#endif
940
941 if ( (pCtx->eflags.u32 & X86_EFL_IF)
942 && (!VMCPU_FF_ISSET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
943 && TRPMHasTrap(pVCpu)
944 )
945 {
946 uint8_t u8Vector;
947 TRPMEVENT enmType;
948 RTGCUINTPTR intInfo;
949 RTGCUINT errCode;
950
951 /* If a new event is pending, then dispatch it now. */
952 rc = TRPMQueryTrapAll(pVCpu, &u8Vector, &enmType, &errCode, 0);
953 AssertRC(rc);
954 Assert(pCtx->eflags.Bits.u1IF == 1 || enmType == TRPM_TRAP);
955 Assert(enmType != TRPM_SOFTWARE_INT);
956
957 /* Clear the pending trap. */
958 rc = TRPMResetTrap(pVCpu);
959 AssertRC(rc);
960
961 intInfo = u8Vector;
962 intInfo |= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
963
964 if (enmType == TRPM_TRAP)
965 {
966 switch (u8Vector) {
967 case 8:
968 case 10:
969 case 11:
970 case 12:
971 case 13:
972 case 14:
973 case 17:
974 /* Valid error codes. */
975 intInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
976 break;
977 default:
978 break;
979 }
980 if (u8Vector == X86_XCPT_BP || u8Vector == X86_XCPT_OF)
981 intInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SWEXCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
982 else
983 intInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HWEXCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
984 }
985 else
986 intInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
987
988 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatIntInject);
989 rc = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, intInfo, 0, errCode);
990 AssertRC(rc);
991 } /* if (interrupts can be dispatched) */
992
993 return VINF_SUCCESS;
994}
995
996/**
997 * Save the host state
998 *
999 * @returns VBox status code.
1000 * @param pVM The VM to operate on.
1001 * @param pVCpu The VMCPU to operate on.
1002 */
1003VMMR0DECL(int) VMXR0SaveHostState(PVM pVM, PVMCPU pVCpu)
1004{
1005 int rc = VINF_SUCCESS;
1006 NOREF(pVM);
1007
1008 /*
1009 * Host CPU Context
1010 */
1011 if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_HOST_CONTEXT)
1012 {
1013 RTIDTR idtr;
1014 RTGDTR gdtr;
1015 RTSEL SelTR;
1016 PCX86DESCHC pDesc;
1017 uintptr_t trBase;
1018 RTSEL cs;
1019 RTSEL ss;
1020 uint64_t cr3;
1021
1022 /* Control registers */
1023 rc = VMXWriteVMCS(VMX_VMCS_HOST_CR0, ASMGetCR0());
1024#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
1025 if (VMX_IS_64BIT_HOST_MODE())
1026 {
1027 cr3 = hwaccmR0Get64bitCR3();
1028 rc |= VMXWriteVMCS64(VMX_VMCS_HOST_CR3, cr3);
1029 }
1030 else
1031#endif
1032 {
1033 cr3 = ASMGetCR3();
1034 rc |= VMXWriteVMCS(VMX_VMCS_HOST_CR3, cr3);
1035 }
1036 rc |= VMXWriteVMCS(VMX_VMCS_HOST_CR4, ASMGetCR4());
1037 AssertRC(rc);
1038 Log2(("VMX_VMCS_HOST_CR0 %08x\n", ASMGetCR0()));
1039 Log2(("VMX_VMCS_HOST_CR3 %08RX64\n", cr3));
1040 Log2(("VMX_VMCS_HOST_CR4 %08x\n", ASMGetCR4()));
1041
1042 /* Selector registers. */
1043#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
1044 if (VMX_IS_64BIT_HOST_MODE())
1045 {
1046 cs = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelCS;
1047 ss = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelSS;
1048 }
1049 else
1050 {
1051 /* sysenter loads LDT cs & ss, VMX doesn't like this. Load the GDT ones (safe). */
1052 cs = (RTSEL)(uintptr_t)&SUPR0AbsKernelCS;
1053 ss = (RTSEL)(uintptr_t)&SUPR0AbsKernelSS;
1054 }
1055#else
1056 cs = ASMGetCS();
1057 ss = ASMGetSS();
1058#endif
1059 Assert(!(cs & X86_SEL_LDT)); Assert((cs & X86_SEL_RPL) == 0);
1060 Assert(!(ss & X86_SEL_LDT)); Assert((ss & X86_SEL_RPL) == 0);
1061 rc = VMXWriteVMCS(VMX_VMCS16_HOST_FIELD_CS, cs);
1062 /* Note: VMX is (again) very picky about the RPL of the selectors here; we'll restore them manually. */
1063 rc |= VMXWriteVMCS(VMX_VMCS16_HOST_FIELD_DS, 0);
1064 rc |= VMXWriteVMCS(VMX_VMCS16_HOST_FIELD_ES, 0);
1065#if HC_ARCH_BITS == 32
1066 if (!VMX_IS_64BIT_HOST_MODE())
1067 {
1068 rc |= VMXWriteVMCS(VMX_VMCS16_HOST_FIELD_FS, 0);
1069 rc |= VMXWriteVMCS(VMX_VMCS16_HOST_FIELD_GS, 0);
1070 }
1071#endif
1072 rc |= VMXWriteVMCS(VMX_VMCS16_HOST_FIELD_SS, ss);
1073 SelTR = ASMGetTR();
1074 rc |= VMXWriteVMCS(VMX_VMCS16_HOST_FIELD_TR, SelTR);
1075 AssertRC(rc);
1076 Log2(("VMX_VMCS_HOST_FIELD_CS %08x (%08x)\n", cs, ASMGetSS()));
1077 Log2(("VMX_VMCS_HOST_FIELD_DS 00000000 (%08x)\n", ASMGetDS()));
1078 Log2(("VMX_VMCS_HOST_FIELD_ES 00000000 (%08x)\n", ASMGetES()));
1079 Log2(("VMX_VMCS_HOST_FIELD_FS 00000000 (%08x)\n", ASMGetFS()));
1080 Log2(("VMX_VMCS_HOST_FIELD_GS 00000000 (%08x)\n", ASMGetGS()));
1081 Log2(("VMX_VMCS_HOST_FIELD_SS %08x (%08x)\n", ss, ASMGetSS()));
1082 Log2(("VMX_VMCS_HOST_FIELD_TR %08x\n", ASMGetTR()));
1083
1084 /* GDTR & IDTR */
1085#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
1086 if (VMX_IS_64BIT_HOST_MODE())
1087 {
1088 X86XDTR64 gdtr64, idtr64;
1089 hwaccmR0Get64bitGDTRandIDTR(&gdtr64, &idtr64);
1090 rc = VMXWriteVMCS64(VMX_VMCS_HOST_GDTR_BASE, gdtr64.uAddr);
1091 rc |= VMXWriteVMCS64(VMX_VMCS_HOST_IDTR_BASE, gdtr64.uAddr);
1092 AssertRC(rc);
1093 Log2(("VMX_VMCS_HOST_GDTR_BASE %RX64\n", gdtr64.uAddr));
1094 Log2(("VMX_VMCS_HOST_IDTR_BASE %RX64\n", idtr64.uAddr));
1095 gdtr.cbGdt = gdtr64.cb;
1096 gdtr.pGdt = (uintptr_t)gdtr64.uAddr;
1097 }
1098 else
1099#endif
1100 {
1101 ASMGetGDTR(&gdtr);
1102 rc = VMXWriteVMCS(VMX_VMCS_HOST_GDTR_BASE, gdtr.pGdt);
1103 ASMGetIDTR(&idtr);
1104 rc |= VMXWriteVMCS(VMX_VMCS_HOST_IDTR_BASE, idtr.pIdt);
1105 AssertRC(rc);
1106 Log2(("VMX_VMCS_HOST_GDTR_BASE %RHv\n", gdtr.pGdt));
1107 Log2(("VMX_VMCS_HOST_IDTR_BASE %RHv\n", idtr.pIdt));
1108 }
1109
1110 /* Save the base address of the TR selector. */
1111 if (SelTR > gdtr.cbGdt)
1112 {
1113 AssertMsgFailed(("Invalid TR selector %x. GDTR.cbGdt=%x\n", SelTR, gdtr.cbGdt));
1114 return VERR_VMX_INVALID_HOST_STATE;
1115 }
1116
1117 pDesc = (PCX86DESCHC)(gdtr.pGdt + (SelTR & X86_SEL_MASK));
1118#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
1119 if (VMX_IS_64BIT_HOST_MODE())
1120 {
1121 uint64_t trBase64 = X86DESC64_BASE(*(PX86DESC64)pDesc);
1122 rc = VMXWriteVMCS64(VMX_VMCS_HOST_TR_BASE, trBase64);
1123 Log2(("VMX_VMCS_HOST_TR_BASE %RX64\n", trBase64));
1124 AssertRC(rc);
1125 }
1126 else
1127#endif
1128 {
1129#if HC_ARCH_BITS == 64
1130 trBase = X86DESC64_BASE(*pDesc);
1131#else
1132 trBase = X86DESC_BASE(*pDesc);
1133#endif
1134 rc = VMXWriteVMCS(VMX_VMCS_HOST_TR_BASE, trBase);
1135 AssertRC(rc);
1136 Log2(("VMX_VMCS_HOST_TR_BASE %RHv\n", trBase));
1137 }
1138
1139 /* FS and GS base. */
1140#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
1141 if (VMX_IS_64BIT_HOST_MODE())
1142 {
1143 Log2(("MSR_K8_FS_BASE = %RX64\n", ASMRdMsr(MSR_K8_FS_BASE)));
1144 Log2(("MSR_K8_GS_BASE = %RX64\n", ASMRdMsr(MSR_K8_GS_BASE)));
1145 rc = VMXWriteVMCS64(VMX_VMCS_HOST_FS_BASE, ASMRdMsr(MSR_K8_FS_BASE));
1146 rc |= VMXWriteVMCS64(VMX_VMCS_HOST_GS_BASE, ASMRdMsr(MSR_K8_GS_BASE));
1147 }
1148#endif
1149 AssertRC(rc);
1150
1151 /* Sysenter MSRs. */
1152 /** @todo expensive!! */
1153 rc = VMXWriteVMCS(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
1154 Log2(("VMX_VMCS_HOST_SYSENTER_CS %08x\n", ASMRdMsr_Low(MSR_IA32_SYSENTER_CS)));
1155#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
1156 if (VMX_IS_64BIT_HOST_MODE())
1157 {
1158 Log2(("VMX_VMCS_HOST_SYSENTER_EIP %RX64\n", ASMRdMsr(MSR_IA32_SYSENTER_EIP)));
1159 Log2(("VMX_VMCS_HOST_SYSENTER_ESP %RX64\n", ASMRdMsr(MSR_IA32_SYSENTER_ESP)));
1160 rc |= VMXWriteVMCS64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
1161 rc |= VMXWriteVMCS64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
1162 }
1163 else
1164 {
1165 rc |= VMXWriteVMCS(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
1166 rc |= VMXWriteVMCS(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
1167 Log2(("VMX_VMCS_HOST_SYSENTER_EIP %RX32\n", ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP)));
1168 Log2(("VMX_VMCS_HOST_SYSENTER_ESP %RX32\n", ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP)));
1169 }
1170#elif HC_ARCH_BITS == 32
1171 rc |= VMXWriteVMCS(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
1172 rc |= VMXWriteVMCS(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
1173 Log2(("VMX_VMCS_HOST_SYSENTER_EIP %RX32\n", ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP)));
1174 Log2(("VMX_VMCS_HOST_SYSENTER_ESP %RX32\n", ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP)));
1175#else
1176 Log2(("VMX_VMCS_HOST_SYSENTER_EIP %RX64\n", ASMRdMsr(MSR_IA32_SYSENTER_EIP)));
1177 Log2(("VMX_VMCS_HOST_SYSENTER_ESP %RX64\n", ASMRdMsr(MSR_IA32_SYSENTER_ESP)));
1178 rc |= VMXWriteVMCS64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
1179 rc |= VMXWriteVMCS64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
1180#endif
1181 AssertRC(rc);
1182
1183#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
1184 /* Store all host MSRs in the VM-Exit load area, so they will be reloaded after the world switch back to the host. */
1185 PVMXMSR pMsr = (PVMXMSR)pVCpu->hwaccm.s.vmx.pHostMSR;
1186 unsigned idxMsr = 0;
1187
1188 /* EFER MSR present? */
1189 if (ASMCpuId_EDX(0x80000001) & (X86_CPUID_AMD_FEATURE_EDX_NX|X86_CPUID_AMD_FEATURE_EDX_LONG_MODE))
1190 {
1191 if (ASMCpuId_EDX(0x80000001) & X86_CPUID_AMD_FEATURE_EDX_SEP)
1192 {
1193 pMsr->u32IndexMSR = MSR_K6_STAR;
1194 pMsr->u32Reserved = 0;
1195 pMsr->u64Value = ASMRdMsr(MSR_K6_STAR); /* legacy syscall eip, cs & ss */
1196 pMsr++; idxMsr++;
1197 }
1198
1199 pMsr->u32IndexMSR = MSR_K6_EFER;
1200 pMsr->u32Reserved = 0;
1201# if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
1202 if (CPUMIsGuestInLongMode(pVCpu))
1203 {
1204 /* Must match the efer value in our 64 bits switcher. */
1205 pMsr->u64Value = ASMRdMsr(MSR_K6_EFER) | MSR_K6_EFER_LME | MSR_K6_EFER_SCE | MSR_K6_EFER_NXE;
1206 }
1207 else
1208# endif
1209 pMsr->u64Value = ASMRdMsr(MSR_K6_EFER);
1210 pMsr++; idxMsr++;
1211 }
1212
1213# if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
1214 if (VMX_IS_64BIT_HOST_MODE())
1215 {
1216 pMsr->u32IndexMSR = MSR_K8_LSTAR;
1217 pMsr->u32Reserved = 0;
1218 pMsr->u64Value = ASMRdMsr(MSR_K8_LSTAR); /* 64 bits mode syscall rip */
1219 pMsr++; idxMsr++;
1220 pMsr->u32IndexMSR = MSR_K8_SF_MASK;
1221 pMsr->u32Reserved = 0;
1222 pMsr->u64Value = ASMRdMsr(MSR_K8_SF_MASK); /* syscall flag mask */
1223 pMsr++; idxMsr++;
1224 pMsr->u32IndexMSR = MSR_K8_KERNEL_GS_BASE;
1225 pMsr->u32Reserved = 0;
1226 pMsr->u64Value = ASMRdMsr(MSR_K8_KERNEL_GS_BASE); /* swapgs exchange value */
1227 pMsr++; idxMsr++;
1228 }
1229# endif
1230 rc = VMXWriteVMCS(VMX_VMCS_CTRL_EXIT_MSR_LOAD_COUNT, idxMsr);
1231 AssertRC(rc);
1232#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
1233
1234 pVCpu->hwaccm.s.fContextUseFlags &= ~HWACCM_CHANGED_HOST_CONTEXT;
1235 }
1236 return rc;
1237}
1238
1239/**
1240 * Loads the 4 PDPEs into the guest state when nested paging is used and the
1241 * guest operates in PAE mode.
1242 *
1243 * @returns VINF_SUCCESS or fatal error.
1244 * @param pVCpu The VMCPU to operate on.
1245 * @param pCtx Guest context
1246 */
1247static int hmR0VmxLoadPaePdpes(PVMCPU pVCpu, PCPUMCTX pCtx)
1248{
1249 if (CPUMIsGuestInPAEModeEx(pCtx))
1250 {
1251 X86PDPE aPdpes[4];
1252 int rc = PGMGstGetPaePdpes(pVCpu, &aPdpes[0]);
1253 AssertRCReturn(rc, rc);
1254
1255 rc = VMXWriteVMCS64(VMX_VMCS_GUEST_PDPTR0_FULL, aPdpes[0].u); AssertRCReturn(rc, rc);
1256 rc = VMXWriteVMCS64(VMX_VMCS_GUEST_PDPTR1_FULL, aPdpes[1].u); AssertRCReturn(rc, rc);
1257 rc = VMXWriteVMCS64(VMX_VMCS_GUEST_PDPTR2_FULL, aPdpes[2].u); AssertRCReturn(rc, rc);
1258 rc = VMXWriteVMCS64(VMX_VMCS_GUEST_PDPTR3_FULL, aPdpes[3].u); AssertRCReturn(rc, rc);
1259 }
1260 return VINF_SUCCESS;
1261}
1262
1263/**
1264 * Saves the 4 PDPEs into the guest state when nested paging is used and the
1265 * guest operates in PAE mode.
1266 *
1267 * @returns VINF_SUCCESS or fatal error.
1268 * @param pVCpu The VMCPU to operate on.
1269 * @param pCtx Guest context
1270 *
1271 * @remarks Tell PGM about CR3 changes before calling this helper.
1272 */
1273static int hmR0VmxSavePaePdpes(PVMCPU pVCpu, PCPUMCTX pCtx)
1274{
1275 if (CPUMIsGuestInPAEModeEx(pCtx))
1276 {
1277 int rc;
1278 X86PDPE aPdpes[4];
1279 rc = VMXReadVMCS64(VMX_VMCS_GUEST_PDPTR0_FULL, &aPdpes[0].u); AssertRCReturn(rc, rc);
1280 rc = VMXReadVMCS64(VMX_VMCS_GUEST_PDPTR1_FULL, &aPdpes[1].u); AssertRCReturn(rc, rc);
1281 rc = VMXReadVMCS64(VMX_VMCS_GUEST_PDPTR2_FULL, &aPdpes[2].u); AssertRCReturn(rc, rc);
1282 rc = VMXReadVMCS64(VMX_VMCS_GUEST_PDPTR3_FULL, &aPdpes[3].u); AssertRCReturn(rc, rc);
1283
1284 rc = PGMGstUpdatePaePdpes(pVCpu, &aPdpes[0]);
1285 AssertRCReturn(rc, rc);
1286 }
1287 return VINF_SUCCESS;
1288}
1289
1290
1291/**
1292 * Update the exception bitmap according to the current CPU state
1293 *
1294 * @param pVM The VM to operate on.
1295 * @param pVCpu The VMCPU to operate on.
1296 * @param pCtx Guest context
1297 */
1298static void hmR0VmxUpdateExceptionBitmap(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
1299{
1300 uint32_t u32TrapMask;
1301 Assert(pCtx);
1302
1303 u32TrapMask = HWACCM_VMX_TRAP_MASK;
1304#ifndef DEBUG
1305 if (pVM->hwaccm.s.fNestedPaging)
1306 u32TrapMask &= ~RT_BIT(X86_XCPT_PF); /* no longer need to intercept #PF. */
1307#endif
1308
1309 /* Also catch floating point exceptions as we need to report them to the guest in a different way. */
1310 if ( CPUMIsGuestFPUStateActive(pVCpu) == true
1311 && !(pCtx->cr0 & X86_CR0_NE)
1312 && !pVCpu->hwaccm.s.fFPUOldStyleOverride)
1313 {
1314 u32TrapMask |= RT_BIT(X86_XCPT_MF);
1315 pVCpu->hwaccm.s.fFPUOldStyleOverride = true;
1316 }
1317
1318#ifdef VBOX_STRICT
1319 Assert(u32TrapMask & RT_BIT(X86_XCPT_GP));
1320#endif
1321
1322 /* Intercept all exceptions in real mode as none of them can be injected directly (#GP otherwise). */
1323 if ( CPUMIsGuestInRealModeEx(pCtx)
1324 && pVM->hwaccm.s.vmx.pRealModeTSS)
1325 u32TrapMask |= HWACCM_VMX_TRAP_MASK_REALMODE;
1326
1327 int rc = VMXWriteVMCS(VMX_VMCS_CTRL_EXCEPTION_BITMAP, u32TrapMask);
1328 AssertRC(rc);
1329}
1330
1331/**
1332 * Loads a minimal guest state
1333 *
1334 * NOTE: Don't do anything here that can cause a jump back to ring 3!!!!!
1335 *
1336 * @param pVM The VM to operate on.
1337 * @param pVCpu The VMCPU to operate on.
1338 * @param pCtx Guest context
1339 */
1340VMMR0DECL(void) VMXR0LoadMinimalGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
1341{
1342 int rc;
1343 X86EFLAGS eflags;
1344
1345 Assert(!(pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_ALL_GUEST));
1346
1347 /* EIP, ESP and EFLAGS */
1348 rc = VMXWriteVMCS64(VMX_VMCS64_GUEST_RIP, pCtx->rip);
1349 rc |= VMXWriteVMCS64(VMX_VMCS64_GUEST_RSP, pCtx->rsp);
1350 AssertRC(rc);
1351
1352 /* Bits 22-31, 15, 5 & 3 must be zero. Bit 1 must be 1. */
1353 eflags = pCtx->eflags;
1354 eflags.u32 &= VMX_EFLAGS_RESERVED_0;
1355 eflags.u32 |= VMX_EFLAGS_RESERVED_1;
1356
1357 /* Real mode emulation using v86 mode. */
1358 if ( CPUMIsGuestInRealModeEx(pCtx)
1359 && pVM->hwaccm.s.vmx.pRealModeTSS)
1360 {
1361 pVCpu->hwaccm.s.vmx.RealMode.eflags = eflags;
1362
1363 eflags.Bits.u1VM = 1;
1364 eflags.Bits.u2IOPL = 0; /* must always be 0 or else certain instructions won't cause faults. */
1365 }
1366 rc = VMXWriteVMCS(VMX_VMCS_GUEST_RFLAGS, eflags.u32);
1367 AssertRC(rc);
1368}
1369
1370/**
1371 * Loads the guest state
1372 *
1373 * NOTE: Don't do anything here that can cause a jump back to ring 3!!!!!
1374 *
1375 * @returns VBox status code.
1376 * @param pVM The VM to operate on.
1377 * @param pVCpu The VMCPU to operate on.
1378 * @param pCtx Guest context
1379 */
1380VMMR0DECL(int) VMXR0LoadGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
1381{
1382 int rc = VINF_SUCCESS;
1383 RTGCUINTPTR val;
1384
1385 /* VMX_VMCS_CTRL_ENTRY_CONTROLS
1386 * Set required bits to one and zero according to the MSR capabilities.
1387 */
1388 val = pVM->hwaccm.s.vmx.msr.vmx_entry.n.disallowed0;
1389 /* Load guest debug controls (dr7 & IA32_DEBUGCTL_MSR) (forced to 1 on the 'first' VT-x capable CPUs; this actually includes the newest Nehalem CPUs) */
1390 val |= VMX_VMCS_CTRL_ENTRY_CONTROLS_LOAD_DEBUG;
1391 /* 64 bits guest mode? */
1392 if (CPUMIsGuestInLongModeEx(pCtx))
1393 val |= VMX_VMCS_CTRL_ENTRY_CONTROLS_IA64_MODE;
1394 /* else Must be zero when AMD64 is not available. */
1395
1396 /* Mask away the bits that the CPU doesn't support */
1397 val &= pVM->hwaccm.s.vmx.msr.vmx_entry.n.allowed1;
1398 rc = VMXWriteVMCS(VMX_VMCS_CTRL_ENTRY_CONTROLS, val);
1399 AssertRC(rc);
1400
1401 /* VMX_VMCS_CTRL_EXIT_CONTROLS
1402 * Set required bits to one and zero according to the MSR capabilities.
1403 */
1404 val = pVM->hwaccm.s.vmx.msr.vmx_exit.n.disallowed0;
1405
1406 /* Save debug controls (dr7 & IA32_DEBUGCTL_MSR) (forced to 1 on the 'first' VT-x capable CPUs; this actually includes the newest Nehalem CPUs) */
1407 val |= VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_DEBUG;
1408
1409#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
1410 if (VMX_IS_64BIT_HOST_MODE())
1411 val |= VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_AMD64;
1412 /* else: Must be zero when AMD64 is not available. */
1413#elif HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
1414 if (CPUMIsGuestInLongModeEx(pCtx))
1415 val |= VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_AMD64; /* our switcher goes to long mode */
1416 else
1417 Assert(!(val & VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_AMD64));
1418#endif
1419 val &= pVM->hwaccm.s.vmx.msr.vmx_exit.n.allowed1;
1420 /* Don't acknowledge external interrupts on VM-exit. */
1421 rc = VMXWriteVMCS(VMX_VMCS_CTRL_EXIT_CONTROLS, val);
1422 AssertRC(rc);
1423
1424 /* Guest CPU context: ES, CS, SS, DS, FS, GS. */
1425 if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_SEGMENT_REGS)
1426 {
1427 if (pVM->hwaccm.s.vmx.pRealModeTSS)
1428 {
1429 PGMMODE enmGuestMode = PGMGetGuestMode(pVCpu);
1430 if (pVCpu->hwaccm.s.vmx.enmLastSeenGuestMode != enmGuestMode)
1431 {
1432 /* Correct weird requirements for switching to protected mode. */
1433 if ( pVCpu->hwaccm.s.vmx.enmLastSeenGuestMode == PGMMODE_REAL
1434 && enmGuestMode >= PGMMODE_PROTECTED)
1435 {
1436 /* Flush the recompiler code cache as it's not unlikely
1437 * the guest will rewrite code it will later execute in real
1438 * mode (OpenBSD 4.0 is one such example)
1439 */
1440 REMFlushTBs(pVM);
1441
1442 /* DPL of all hidden selector registers must match the current CPL (0). */
1443 pCtx->csHid.Attr.n.u2Dpl = 0;
1444 pCtx->csHid.Attr.n.u4Type = X86_SEL_TYPE_CODE | X86_SEL_TYPE_RW_ACC;
1445
1446 pCtx->dsHid.Attr.n.u2Dpl = 0;
1447 pCtx->esHid.Attr.n.u2Dpl = 0;
1448 pCtx->fsHid.Attr.n.u2Dpl = 0;
1449 pCtx->gsHid.Attr.n.u2Dpl = 0;
1450 pCtx->ssHid.Attr.n.u2Dpl = 0;
1451
1452 /* The limit must correspond to the 32 bits setting. */
1453 if (!pCtx->csHid.Attr.n.u1DefBig)
1454 pCtx->csHid.u32Limit &= 0xffff;
1455 if (!pCtx->dsHid.Attr.n.u1DefBig)
1456 pCtx->dsHid.u32Limit &= 0xffff;
1457 if (!pCtx->esHid.Attr.n.u1DefBig)
1458 pCtx->esHid.u32Limit &= 0xffff;
1459 if (!pCtx->fsHid.Attr.n.u1DefBig)
1460 pCtx->fsHid.u32Limit &= 0xffff;
1461 if (!pCtx->gsHid.Attr.n.u1DefBig)
1462 pCtx->gsHid.u32Limit &= 0xffff;
1463 if (!pCtx->ssHid.Attr.n.u1DefBig)
1464 pCtx->ssHid.u32Limit &= 0xffff;
1465 }
1466 else
1467 /* Switching from protected mode to real mode. */
1468 if ( pVCpu->hwaccm.s.vmx.enmLastSeenGuestMode >= PGMMODE_PROTECTED
1469 && enmGuestMode == PGMMODE_REAL)
1470 {
1471 /* The limit must also be set to 0xffff. */
1472 pCtx->csHid.u32Limit = 0xffff;
1473 pCtx->dsHid.u32Limit = 0xffff;
1474 pCtx->esHid.u32Limit = 0xffff;
1475 pCtx->fsHid.u32Limit = 0xffff;
1476 pCtx->gsHid.u32Limit = 0xffff;
1477 pCtx->ssHid.u32Limit = 0xffff;
1478
1479 Assert(pCtx->csHid.u64Base <= 0xfffff);
1480 Assert(pCtx->dsHid.u64Base <= 0xfffff);
1481 Assert(pCtx->esHid.u64Base <= 0xfffff);
1482 Assert(pCtx->fsHid.u64Base <= 0xfffff);
1483 Assert(pCtx->gsHid.u64Base <= 0xfffff);
1484 }
1485 pVCpu->hwaccm.s.vmx.enmLastSeenGuestMode = enmGuestMode;
1486 }
1487 else
1488 /* VT-x will fail with a guest invalid state otherwise... (CPU state after a reset) */
1489 if ( CPUMIsGuestInRealModeEx(pCtx)
1490 && pCtx->csHid.u64Base == 0xffff0000)
1491 {
1492 pCtx->csHid.u64Base = 0xf0000;
1493 pCtx->cs = 0xf000;
1494 }
1495 }
1496
1497 VMX_WRITE_SELREG(ES, es);
1498 AssertRC(rc);
1499
1500 VMX_WRITE_SELREG(CS, cs);
1501 AssertRC(rc);
1502
1503 VMX_WRITE_SELREG(SS, ss);
1504 AssertRC(rc);
1505
1506 VMX_WRITE_SELREG(DS, ds);
1507 AssertRC(rc);
1508
1509 VMX_WRITE_SELREG(FS, fs);
1510 AssertRC(rc);
1511
1512 VMX_WRITE_SELREG(GS, gs);
1513 AssertRC(rc);
1514 }
1515
1516 /* Guest CPU context: LDTR. */
1517 if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_LDTR)
1518 {
1519 if (pCtx->ldtr == 0)
1520 {
1521 rc = VMXWriteVMCS(VMX_VMCS16_GUEST_FIELD_LDTR, 0);
1522 rc |= VMXWriteVMCS(VMX_VMCS32_GUEST_LDTR_LIMIT, 0);
1523 rc |= VMXWriteVMCS64(VMX_VMCS64_GUEST_LDTR_BASE, 0);
1524 /* Note: vmlaunch will fail with 0 or just 0x02. No idea why. */
1525 rc |= VMXWriteVMCS(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, 0x82 /* present, LDT */);
1526 }
1527 else
1528 {
1529 rc = VMXWriteVMCS(VMX_VMCS16_GUEST_FIELD_LDTR, pCtx->ldtr);
1530 rc |= VMXWriteVMCS(VMX_VMCS32_GUEST_LDTR_LIMIT, pCtx->ldtrHid.u32Limit);
1531 rc |= VMXWriteVMCS64(VMX_VMCS64_GUEST_LDTR_BASE, pCtx->ldtrHid.u64Base);
1532 rc |= VMXWriteVMCS(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, pCtx->ldtrHid.Attr.u);
1533 }
1534 AssertRC(rc);
1535 }
1536 /* Guest CPU context: TR. */
1537 if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_TR)
1538 {
1539 /* Real mode emulation using v86 mode with CR4.VME (interrupt redirection using the int bitmap in the TSS) */
1540 if ( CPUMIsGuestInRealModeEx(pCtx)
1541 && pVM->hwaccm.s.vmx.pRealModeTSS)
1542 {
1543 RTGCPHYS GCPhys;
1544
1545 /* We convert it here every time as pci regions could be reconfigured. */
1546 rc = PDMVMMDevHeapR3ToGCPhys(pVM, pVM->hwaccm.s.vmx.pRealModeTSS, &GCPhys);
1547 AssertRC(rc);
1548
1549 rc = VMXWriteVMCS(VMX_VMCS16_GUEST_FIELD_TR, 0);
1550 rc |= VMXWriteVMCS(VMX_VMCS32_GUEST_TR_LIMIT, HWACCM_VTX_TSS_SIZE);
1551 rc |= VMXWriteVMCS64(VMX_VMCS64_GUEST_TR_BASE, GCPhys /* phys = virt in this mode */);
1552
1553 X86DESCATTR attr;
1554
1555 attr.u = 0;
1556 attr.n.u1Present = 1;
1557 attr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
1558 val = attr.u;
1559 }
1560 else
1561 {
1562 rc = VMXWriteVMCS(VMX_VMCS16_GUEST_FIELD_TR, pCtx->tr);
1563 rc |= VMXWriteVMCS(VMX_VMCS32_GUEST_TR_LIMIT, pCtx->trHid.u32Limit);
1564 rc |= VMXWriteVMCS64(VMX_VMCS64_GUEST_TR_BASE, pCtx->trHid.u64Base);
1565
1566 val = pCtx->trHid.Attr.u;
1567
1568 /* The TSS selector must be busy (REM bugs? see defect #XXXX). */
1569 if (!(val & X86_SEL_TYPE_SYS_TSS_BUSY_MASK))
1570 {
1571 if (val & 0xf)
1572 val |= X86_SEL_TYPE_SYS_TSS_BUSY_MASK;
1573 else
1574 /* Default if no TR selector has been set (otherwise vmlaunch will fail!) */
1575 val = (val & ~0xF) | X86_SEL_TYPE_SYS_386_TSS_BUSY;
1576 }
1577 AssertMsg((val & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY || (val & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("%#x\n", val));
1578 }
1579 rc |= VMXWriteVMCS(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, val);
1580 AssertRC(rc);
1581 }
1582 /* Guest CPU context: GDTR. */
1583 if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_GDTR)
1584 {
1585 rc = VMXWriteVMCS(VMX_VMCS32_GUEST_GDTR_LIMIT, pCtx->gdtr.cbGdt);
1586 rc |= VMXWriteVMCS64(VMX_VMCS64_GUEST_GDTR_BASE, pCtx->gdtr.pGdt);
1587 AssertRC(rc);
1588 }
1589 /* Guest CPU context: IDTR. */
1590 if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_IDTR)
1591 {
1592 rc = VMXWriteVMCS(VMX_VMCS32_GUEST_IDTR_LIMIT, pCtx->idtr.cbIdt);
1593 rc |= VMXWriteVMCS64(VMX_VMCS64_GUEST_IDTR_BASE, pCtx->idtr.pIdt);
1594 AssertRC(rc);
1595 }
1596
1597 /*
1598 * Sysenter MSRs
1599 */
1600 if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_MSR)
1601 {
1602 rc = VMXWriteVMCS(VMX_VMCS32_GUEST_SYSENTER_CS, pCtx->SysEnter.cs);
1603 rc |= VMXWriteVMCS64(VMX_VMCS64_GUEST_SYSENTER_EIP, pCtx->SysEnter.eip);
1604 rc |= VMXWriteVMCS64(VMX_VMCS64_GUEST_SYSENTER_ESP, pCtx->SysEnter.esp);
1605 AssertRC(rc);
1606 }
1607
1608 /* Control registers */
1609 if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_CR0)
1610 {
1611 val = pCtx->cr0;
1612 rc = VMXWriteVMCS(VMX_VMCS_CTRL_CR0_READ_SHADOW, val);
1613 Log2(("Guest CR0-shadow %08x\n", val));
1614 if (CPUMIsGuestFPUStateActive(pVCpu) == false)
1615 {
1616 /* Always use #NM exceptions to load the FPU/XMM state on demand. */
1617 val |= X86_CR0_TS | X86_CR0_ET | X86_CR0_NE | X86_CR0_MP;
1618 }
1619 else
1620 {
1621 /** @todo check if we support the old style mess correctly. */
1622 if (!(val & X86_CR0_NE))
1623 Log(("Forcing X86_CR0_NE!!!\n"));
1624
1625 val |= X86_CR0_NE; /* always turn on the native mechanism to report FPU errors (old style uses interrupts) */
1626 }
1627 /* Note: protected mode & paging are always enabled; we use them for emulating real and protected mode without paging too. */
1628 if (!pVM->hwaccm.s.vmx.fUnrestrictedGuest)
1629 val |= X86_CR0_PE | X86_CR0_PG;
1630
1631 if (pVM->hwaccm.s.fNestedPaging)
1632 {
1633 if (CPUMIsGuestInPagedProtectedModeEx(pCtx))
1634 {
1635 /* Disable cr3 read/write monitoring as we don't need it for EPT. */
1636 pVCpu->hwaccm.s.vmx.proc_ctls &= ~( VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
1637 | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT);
1638 }
1639 else
1640 {
1641 /* Reenable cr3 read/write monitoring as our identity mapped page table is active. */
1642 pVCpu->hwaccm.s.vmx.proc_ctls |= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
1643 | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT;
1644 }
1645 rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
1646 AssertRC(rc);
1647 }
1648 else
1649 {
1650 /* Note: We must also set this as we rely on protecting various pages for which supervisor writes must be caught. */
1651 val |= X86_CR0_WP;
1652 }
1653
1654 /* Always enable caching. */
1655 val &= ~(X86_CR0_CD|X86_CR0_NW);
1656
1657 rc |= VMXWriteVMCS64(VMX_VMCS64_GUEST_CR0, val);
1658 Log2(("Guest CR0 %08x\n", val));
1659 /* CR0 flags owned by the host; if the guests attempts to change them, then
1660 * the VM will exit.
1661 */
1662 val = X86_CR0_PE /* Must monitor this bit (assumptions are made for real mode emulation) */
1663 | X86_CR0_WP /* Must monitor this bit (it must always be enabled). */
1664 | X86_CR0_PG /* Must monitor this bit (assumptions are made for real mode & protected mode without paging emulation) */
1665 | X86_CR0_CD /* Bit not restored during VM-exit! */
1666 | X86_CR0_NW /* Bit not restored during VM-exit! */
1667 | X86_CR0_NE;
1668
1669 /* When the guest's FPU state is active, then we no longer care about
1670 * the FPU related bits.
1671 */
1672 if (CPUMIsGuestFPUStateActive(pVCpu) == false)
1673 val |= X86_CR0_TS | X86_CR0_ET | X86_CR0_MP;
1674
1675 pVCpu->hwaccm.s.vmx.cr0_mask = val;
1676
1677 rc |= VMXWriteVMCS(VMX_VMCS_CTRL_CR0_MASK, val);
1678 Log2(("Guest CR0-mask %08x\n", val));
1679 AssertRC(rc);
1680 }
1681 if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_CR4)
1682 {
1683 /* CR4 */
1684 rc = VMXWriteVMCS(VMX_VMCS_CTRL_CR4_READ_SHADOW, pCtx->cr4);
1685 Log2(("Guest CR4-shadow %08x\n", pCtx->cr4));
1686 /* Set the required bits in cr4 too (currently X86_CR4_VMXE). */
1687 val = pCtx->cr4 | (uint32_t)pVM->hwaccm.s.vmx.msr.vmx_cr4_fixed0;
1688
1689 if (!pVM->hwaccm.s.fNestedPaging)
1690 {
1691 switch(pVCpu->hwaccm.s.enmShadowMode)
1692 {
1693 case PGMMODE_REAL: /* Real mode -> emulated using v86 mode */
1694 case PGMMODE_PROTECTED: /* Protected mode, no paging -> emulated using identity mapping. */
1695 case PGMMODE_32_BIT: /* 32-bit paging. */
1696 val &= ~X86_CR4_PAE;
1697 break;
1698
1699 case PGMMODE_PAE: /* PAE paging. */
1700 case PGMMODE_PAE_NX: /* PAE paging with NX enabled. */
1701 /** Must use PAE paging as we could use physical memory > 4 GB */
1702 val |= X86_CR4_PAE;
1703 break;
1704
1705 case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
1706 case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
1707#ifdef VBOX_ENABLE_64_BITS_GUESTS
1708 break;
1709#else
1710 AssertFailed();
1711 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
1712#endif
1713 default: /* shut up gcc */
1714 AssertFailed();
1715 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
1716 }
1717 }
1718 else
1719 if ( !CPUMIsGuestInPagedProtectedModeEx(pCtx)
1720 && !pVM->hwaccm.s.vmx.fUnrestrictedGuest)
1721 {
1722 /* We use 4 MB pages in our identity mapping page table for real and protected mode without paging. */
1723 val |= X86_CR4_PSE;
1724 /* Our identity mapping is a 32 bits page directory. */
1725 val &= ~X86_CR4_PAE;
1726 }
1727
1728 /* Turn off VME if we're in emulated real mode. */
1729 if ( CPUMIsGuestInRealModeEx(pCtx)
1730 && pVM->hwaccm.s.vmx.pRealModeTSS)
1731 val &= ~X86_CR4_VME;
1732
1733 rc |= VMXWriteVMCS64(VMX_VMCS64_GUEST_CR4, val);
1734 Log2(("Guest CR4 %08x\n", val));
1735 /* CR4 flags owned by the host; if the guests attempts to change them, then
1736 * the VM will exit.
1737 */
1738 val = 0
1739 | X86_CR4_VME
1740 | X86_CR4_PAE
1741 | X86_CR4_PGE
1742 | X86_CR4_PSE
1743 | X86_CR4_VMXE;
1744 pVCpu->hwaccm.s.vmx.cr4_mask = val;
1745
1746 rc |= VMXWriteVMCS(VMX_VMCS_CTRL_CR4_MASK, val);
1747 Log2(("Guest CR4-mask %08x\n", val));
1748 AssertRC(rc);
1749 }
1750
1751 if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_CR3)
1752 {
1753 if (pVM->hwaccm.s.fNestedPaging)
1754 {
1755 Assert(PGMGetHyperCR3(pVCpu));
1756 pVCpu->hwaccm.s.vmx.GCPhysEPTP = PGMGetHyperCR3(pVCpu);
1757
1758 Assert(!(pVCpu->hwaccm.s.vmx.GCPhysEPTP & 0xfff));
1759 /** @todo Check the IA32_VMX_EPT_VPID_CAP MSR for other supported memory types. */
1760 pVCpu->hwaccm.s.vmx.GCPhysEPTP |= VMX_EPT_MEMTYPE_WB
1761 | (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
1762
1763 rc = VMXWriteVMCS64(VMX_VMCS_CTRL_EPTP_FULL, pVCpu->hwaccm.s.vmx.GCPhysEPTP);
1764 AssertRC(rc);
1765
1766 if ( !CPUMIsGuestInPagedProtectedModeEx(pCtx)
1767 && !pVM->hwaccm.s.vmx.fUnrestrictedGuest)
1768 {
1769 RTGCPHYS GCPhys;
1770
1771 /* We convert it here every time as pci regions could be reconfigured. */
1772 rc = PDMVMMDevHeapR3ToGCPhys(pVM, pVM->hwaccm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
1773 AssertMsgRC(rc, ("pNonPagingModeEPTPageTable = %RGv\n", pVM->hwaccm.s.vmx.pNonPagingModeEPTPageTable));
1774
1775 /* We use our identity mapping page table here as we need to map guest virtual to guest physical addresses; EPT will
1776 * take care of the translation to host physical addresses.
1777 */
1778 val = GCPhys;
1779 }
1780 else
1781 {
1782 /* Save the real guest CR3 in VMX_VMCS_GUEST_CR3 */
1783 val = pCtx->cr3;
1784 rc = hmR0VmxLoadPaePdpes(pVCpu, pCtx);
1785 AssertRCReturn(rc, rc);
1786 }
1787 }
1788 else
1789 {
1790 val = PGMGetHyperCR3(pVCpu);
1791 Assert(val || VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL));
1792 }
1793
1794 /* Save our shadow CR3 register. */
1795 rc = VMXWriteVMCS64(VMX_VMCS64_GUEST_CR3, val);
1796 AssertRC(rc);
1797 }
1798
1799 /* Debug registers. */
1800 if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_DEBUG)
1801 {
1802 pCtx->dr[6] |= X86_DR6_INIT_VAL; /* set all reserved bits to 1. */
1803 pCtx->dr[6] &= ~RT_BIT(12); /* must be zero. */
1804
1805 pCtx->dr[7] &= 0xffffffff; /* upper 32 bits reserved */
1806 pCtx->dr[7] &= ~(RT_BIT(11) | RT_BIT(12) | RT_BIT(14) | RT_BIT(15)); /* must be zero */
1807 pCtx->dr[7] |= 0x400; /* must be one */
1808
1809 /* Resync DR7 */
1810 rc = VMXWriteVMCS64(VMX_VMCS64_GUEST_DR7, pCtx->dr[7]);
1811 AssertRC(rc);
1812
1813#ifdef DEBUG
1814 /* Sync the hypervisor debug state now if any breakpoint is armed. */
1815 if ( CPUMGetHyperDR7(pVCpu) & (X86_DR7_ENABLED_MASK|X86_DR7_GD)
1816 && !CPUMIsHyperDebugStateActive(pVCpu)
1817 && !DBGFIsStepping(pVCpu))
1818 {
1819 /* Save the host and load the hypervisor debug state. */
1820 rc = CPUMR0LoadHyperDebugState(pVM, pVCpu, pCtx, true /* include DR6 */);
1821 AssertRC(rc);
1822
1823 /* DRx intercepts remain enabled. */
1824
1825 /* Override dr7 with the hypervisor value. */
1826 rc = VMXWriteVMCS64(VMX_VMCS64_GUEST_DR7, CPUMGetHyperDR7(pVCpu));
1827 AssertRC(rc);
1828 }
1829 else
1830#endif
1831 /* Sync the debug state now if any breakpoint is armed. */
1832 if ( (pCtx->dr[7] & (X86_DR7_ENABLED_MASK|X86_DR7_GD))
1833 && !CPUMIsGuestDebugStateActive(pVCpu)
1834 && !DBGFIsStepping(pVCpu))
1835 {
1836 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatDRxArmed);
1837
1838 /* Disable drx move intercepts. */
1839 pVCpu->hwaccm.s.vmx.proc_ctls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT;
1840 rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
1841 AssertRC(rc);
1842
1843 /* Save the host and load the guest debug state. */
1844 rc = CPUMR0LoadGuestDebugState(pVM, pVCpu, pCtx, true /* include DR6 */);
1845 AssertRC(rc);
1846 }
1847
1848 /* IA32_DEBUGCTL MSR. */
1849 rc = VMXWriteVMCS64(VMX_VMCS_GUEST_DEBUGCTL_FULL, 0);
1850 AssertRC(rc);
1851
1852 /** @todo do we really ever need this? */
1853 rc |= VMXWriteVMCS(VMX_VMCS_GUEST_DEBUG_EXCEPTIONS, 0);
1854 AssertRC(rc);
1855 }
1856
1857 /* 64 bits guest mode? */
1858 if (CPUMIsGuestInLongModeEx(pCtx))
1859 {
1860#if !defined(VBOX_ENABLE_64_BITS_GUESTS)
1861 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
1862#elif HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
1863 pVCpu->hwaccm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
1864#else
1865# ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
1866 if (!pVM->hwaccm.s.fAllow64BitGuests)
1867 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
1868# endif
1869 pVCpu->hwaccm.s.vmx.pfnStartVM = VMXR0StartVM64;
1870#endif
1871 if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_MSR)
1872 {
1873 /* Update these as wrmsr might have changed them. */
1874 rc = VMXWriteVMCS64(VMX_VMCS64_GUEST_FS_BASE, pCtx->fsHid.u64Base);
1875 AssertRC(rc);
1876 rc = VMXWriteVMCS64(VMX_VMCS64_GUEST_GS_BASE, pCtx->gsHid.u64Base);
1877 AssertRC(rc);
1878 }
1879 }
1880 else
1881 {
1882 pVCpu->hwaccm.s.vmx.pfnStartVM = VMXR0StartVM32;
1883 }
1884
1885 hmR0VmxUpdateExceptionBitmap(pVM, pVCpu, pCtx);
1886
1887#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
1888 /* Store all guest MSRs in the VM-Entry load area, so they will be loaded during the world switch. */
1889 PVMXMSR pMsr = (PVMXMSR)pVCpu->hwaccm.s.vmx.pGuestMSR;
1890 unsigned idxMsr = 0;
1891
1892 uint32_t ulEdx;
1893 uint32_t ulTemp;
1894 CPUMGetGuestCpuId(pVCpu, 0x80000001, &ulTemp, &ulTemp, &ulTemp, &ulEdx);
1895 /* EFER MSR present? */
1896 if (ulEdx & (X86_CPUID_AMD_FEATURE_EDX_NX|X86_CPUID_AMD_FEATURE_EDX_LONG_MODE))
1897 {
1898 pMsr->u32IndexMSR = MSR_K6_EFER;
1899 pMsr->u32Reserved = 0;
1900 pMsr->u64Value = pCtx->msrEFER;
1901 /* VT-x will complain if only MSR_K6_EFER_LME is set. */
1902 if (!CPUMIsGuestInLongModeEx(pCtx))
1903 pMsr->u64Value &= ~(MSR_K6_EFER_LMA|MSR_K6_EFER_LME);
1904 pMsr++; idxMsr++;
1905
1906 if (ulEdx & X86_CPUID_AMD_FEATURE_EDX_LONG_MODE)
1907 {
1908 pMsr->u32IndexMSR = MSR_K8_LSTAR;
1909 pMsr->u32Reserved = 0;
1910 pMsr->u64Value = pCtx->msrLSTAR; /* 64 bits mode syscall rip */
1911 pMsr++; idxMsr++;
1912 pMsr->u32IndexMSR = MSR_K6_STAR;
1913 pMsr->u32Reserved = 0;
1914 pMsr->u64Value = pCtx->msrSTAR; /* legacy syscall eip, cs & ss */
1915 pMsr++; idxMsr++;
1916 pMsr->u32IndexMSR = MSR_K8_SF_MASK;
1917 pMsr->u32Reserved = 0;
1918 pMsr->u64Value = pCtx->msrSFMASK; /* syscall flag mask */
1919 pMsr++; idxMsr++;
1920 pMsr->u32IndexMSR = MSR_K8_KERNEL_GS_BASE;
1921 pMsr->u32Reserved = 0;
1922 pMsr->u64Value = pCtx->msrKERNELGSBASE; /* swapgs exchange value */
1923 pMsr++; idxMsr++;
1924 }
1925 }
1926 pVCpu->hwaccm.s.vmx.cCachedMSRs = idxMsr;
1927
1928 rc = VMXWriteVMCS(VMX_VMCS_CTRL_ENTRY_MSR_LOAD_COUNT, idxMsr);
1929 AssertRC(rc);
1930
1931 rc = VMXWriteVMCS(VMX_VMCS_CTRL_EXIT_MSR_STORE_COUNT, idxMsr);
1932 AssertRC(rc);
1933#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
1934
1935 bool fOffsettedTsc;
1936 if (pVM->hwaccm.s.vmx.fUsePreemptTimer)
1937 {
1938 uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVCpu, &fOffsettedTsc, &pVCpu->hwaccm.s.vmx.u64TSCOffset);
1939
1940 /* Make sure the returned values have sane upper and lower boundaries. */
1941 uint64_t u64CpuHz = SUPGetCpuHzFromGIP(g_pSUPGlobalInfoPage);
1942
1943 cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64 of a second */
1944 cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
1945
1946 cTicksToDeadline >>= pVM->hwaccm.s.vmx.cPreemptTimerShift;
1947 uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
1948 rc = VMXWriteVMCS(VMX_VMCS32_GUEST_PREEMPTION_TIMER_VALUE, cPreemptionTickCount);
1949 AssertRC(rc);
1950 }
1951 else
1952 fOffsettedTsc = TMCpuTickCanUseRealTSC(pVCpu, &pVCpu->hwaccm.s.vmx.u64TSCOffset);
1953 if (fOffsettedTsc)
1954 {
1955 uint64_t u64CurTSC = ASMReadTSC();
1956 if (u64CurTSC + pVCpu->hwaccm.s.vmx.u64TSCOffset >= TMCpuTickGetLastSeen(pVCpu))
1957 {
1958 /* Note: VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT takes precedence over TSC_OFFSET */
1959 rc = VMXWriteVMCS64(VMX_VMCS_CTRL_TSC_OFFSET_FULL, pVCpu->hwaccm.s.vmx.u64TSCOffset);
1960 AssertRC(rc);
1961
1962 pVCpu->hwaccm.s.vmx.proc_ctls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT;
1963 rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
1964 AssertRC(rc);
1965 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatTSCOffset);
1966 }
1967 else
1968 {
1969 /* Fall back to rdtsc emulation as we would otherwise pass decreasing tsc values to the guest. */
1970 LogFlow(("TSC %RX64 offset %RX64 time=%RX64 last=%RX64 (diff=%RX64, virt_tsc=%RX64)\n", u64CurTSC, pVCpu->hwaccm.s.vmx.u64TSCOffset, u64CurTSC + pVCpu->hwaccm.s.vmx.u64TSCOffset, TMCpuTickGetLastSeen(pVCpu), TMCpuTickGetLastSeen(pVCpu) - u64CurTSC - pVCpu->hwaccm.s.vmx.u64TSCOffset, TMCpuTickGet(pVCpu)));
1971 pVCpu->hwaccm.s.vmx.proc_ctls |= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT;
1972 rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
1973 AssertRC(rc);
1974 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatTSCInterceptOverFlow);
1975 }
1976 }
1977 else
1978 {
1979 pVCpu->hwaccm.s.vmx.proc_ctls |= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT;
1980 rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
1981 AssertRC(rc);
1982 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatTSCIntercept);
1983 }
1984
1985 /* Done with the major changes */
1986 pVCpu->hwaccm.s.fContextUseFlags &= ~HWACCM_CHANGED_ALL_GUEST;
1987
1988 /* Minimal guest state update (esp, eip, eflags mostly) */
1989 VMXR0LoadMinimalGuestState(pVM, pVCpu, pCtx);
1990 return rc;
1991}
1992
1993/**
1994 * Syncs back the guest state
1995 *
1996 * @returns VBox status code.
1997 * @param pVM The VM to operate on.
1998 * @param pVCpu The VMCPU to operate on.
1999 * @param pCtx Guest context
2000 */
2001DECLINLINE(int) VMXR0SaveGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
2002{
2003 RTGCUINTREG val, valShadow;
2004 RTGCUINTPTR uInterruptState;
2005 int rc;
2006
2007 /* Let's first sync back eip, esp, and eflags. */
2008 rc = VMXReadCachedVMCS(VMX_VMCS64_GUEST_RIP, &val);
2009 AssertRC(rc);
2010 pCtx->rip = val;
2011 rc = VMXReadCachedVMCS(VMX_VMCS64_GUEST_RSP, &val);
2012 AssertRC(rc);
2013 pCtx->rsp = val;
2014 rc = VMXReadCachedVMCS(VMX_VMCS_GUEST_RFLAGS, &val);
2015 AssertRC(rc);
2016 pCtx->eflags.u32 = val;
2017
2018 /* Take care of instruction fusing (sti, mov ss) */
2019 rc |= VMXReadCachedVMCS(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &val);
2020 uInterruptState = val;
2021 if (uInterruptState != 0)
2022 {
2023 Assert(uInterruptState <= 2); /* only sti & mov ss */
2024 Log(("uInterruptState %x eip=%RGv\n", (uint32_t)uInterruptState, pCtx->rip));
2025 EMSetInhibitInterruptsPC(pVCpu, pCtx->rip);
2026 }
2027 else
2028 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
2029
2030 /* Control registers. */
2031 VMXReadCachedVMCS(VMX_VMCS_CTRL_CR0_READ_SHADOW, &valShadow);
2032 VMXReadCachedVMCS(VMX_VMCS64_GUEST_CR0, &val);
2033 val = (valShadow & pVCpu->hwaccm.s.vmx.cr0_mask) | (val & ~pVCpu->hwaccm.s.vmx.cr0_mask);
2034 CPUMSetGuestCR0(pVCpu, val);
2035
2036 VMXReadCachedVMCS(VMX_VMCS_CTRL_CR4_READ_SHADOW, &valShadow);
2037 VMXReadCachedVMCS(VMX_VMCS64_GUEST_CR4, &val);
2038 val = (valShadow & pVCpu->hwaccm.s.vmx.cr4_mask) | (val & ~pVCpu->hwaccm.s.vmx.cr4_mask);
2039 CPUMSetGuestCR4(pVCpu, val);
2040
2041 /* Note: no reason to sync back the CRx registers. They can't be changed by the guest. */
2042 /* Note: only in the nested paging case can CR3 & CR4 be changed by the guest. */
2043 if ( pVM->hwaccm.s.fNestedPaging
2044 && CPUMIsGuestInPagedProtectedModeEx(pCtx)) /** @todo check if we will always catch mode switches and such... */
2045 {
2046 PVMCSCACHE pCache = &pVCpu->hwaccm.s.vmx.VMCSCache;
2047
2048 /* Can be updated behind our back in the nested paging case. */
2049 CPUMSetGuestCR2(pVCpu, pCache->cr2);
2050
2051 VMXReadCachedVMCS(VMX_VMCS64_GUEST_CR3, &val);
2052
2053 if (val != pCtx->cr3)
2054 {
2055 CPUMSetGuestCR3(pVCpu, val);
2056 PGMUpdateCR3(pVCpu, val);
2057 }
2058 rc = hmR0VmxSavePaePdpes(pVCpu, pCtx);
2059 AssertRCReturn(rc, rc);
2060 }
2061
2062 /* Sync back DR7 here. */
2063 VMXReadCachedVMCS(VMX_VMCS64_GUEST_DR7, &val);
2064 pCtx->dr[7] = val;
2065
2066 /* Guest CPU context: ES, CS, SS, DS, FS, GS. */
2067 VMX_READ_SELREG(ES, es);
2068 VMX_READ_SELREG(SS, ss);
2069 VMX_READ_SELREG(CS, cs);
2070 VMX_READ_SELREG(DS, ds);
2071 VMX_READ_SELREG(FS, fs);
2072 VMX_READ_SELREG(GS, gs);
2073
2074 /*
2075 * System MSRs
2076 */
2077 VMXReadCachedVMCS(VMX_VMCS32_GUEST_SYSENTER_CS, &val);
2078 pCtx->SysEnter.cs = val;
2079 VMXReadCachedVMCS(VMX_VMCS64_GUEST_SYSENTER_EIP, &val);
2080 pCtx->SysEnter.eip = val;
2081 VMXReadCachedVMCS(VMX_VMCS64_GUEST_SYSENTER_ESP, &val);
2082 pCtx->SysEnter.esp = val;
2083
2084 /* Misc. registers; must sync everything otherwise we can get out of sync when jumping to ring 3. */
2085 VMX_READ_SELREG(LDTR, ldtr);
2086
2087 VMXReadCachedVMCS(VMX_VMCS32_GUEST_GDTR_LIMIT, &val);
2088 pCtx->gdtr.cbGdt = val;
2089 VMXReadCachedVMCS(VMX_VMCS64_GUEST_GDTR_BASE, &val);
2090 pCtx->gdtr.pGdt = val;
2091
2092 VMXReadCachedVMCS(VMX_VMCS32_GUEST_IDTR_LIMIT, &val);
2093 pCtx->idtr.cbIdt = val;
2094 VMXReadCachedVMCS(VMX_VMCS64_GUEST_IDTR_BASE, &val);
2095 pCtx->idtr.pIdt = val;
2096
2097 /* Real mode emulation using v86 mode. */
2098 if ( CPUMIsGuestInRealModeEx(pCtx)
2099 && pVM->hwaccm.s.vmx.pRealModeTSS)
2100 {
2101 /* Hide our emulation flags */
2102 pCtx->eflags.Bits.u1VM = 0;
2103
2104 /* Restore original IOPL setting as we always use 0. */
2105 pCtx->eflags.Bits.u2IOPL = pVCpu->hwaccm.s.vmx.RealMode.eflags.Bits.u2IOPL;
2106
2107 /* Force a TR resync every time in case we switch modes. */
2108 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_GUEST_TR;
2109 }
2110 else
2111 {
2112 /* In real mode we have a fake TSS, so only sync it back when it's supposed to be valid. */
2113 VMX_READ_SELREG(TR, tr);
2114 }
2115
2116#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
2117 /* Save the possibly changed MSRs that we automatically restore and save during a world switch. */
2118 for (unsigned i = 0; i < pVCpu->hwaccm.s.vmx.cCachedMSRs; i++)
2119 {
2120 PVMXMSR pMsr = (PVMXMSR)pVCpu->hwaccm.s.vmx.pGuestMSR;
2121 pMsr += i;
2122
2123 switch (pMsr->u32IndexMSR)
2124 {
2125 case MSR_K8_LSTAR:
2126 pCtx->msrLSTAR = pMsr->u64Value;
2127 break;
2128 case MSR_K6_STAR:
2129 pCtx->msrSTAR = pMsr->u64Value;
2130 break;
2131 case MSR_K8_SF_MASK:
2132 pCtx->msrSFMASK = pMsr->u64Value;
2133 break;
2134 case MSR_K8_KERNEL_GS_BASE:
2135 pCtx->msrKERNELGSBASE = pMsr->u64Value;
2136 break;
2137 case MSR_K6_EFER:
2138 /* EFER can't be changed without causing a VM-exit. */
2139// Assert(pCtx->msrEFER == pMsr->u64Value);
2140 break;
2141 default:
2142 AssertFailed();
2143 return VERR_HM_UNEXPECTED_LD_ST_MSR;
2144 }
2145 }
2146#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
2147 return VINF_SUCCESS;
2148}
2149
2150/**
2151 * Dummy placeholder
2152 *
2153 * @param pVM The VM to operate on.
2154 * @param pVCpu The VMCPU to operate on.
2155 */
2156static void hmR0VmxSetupTLBDummy(PVM pVM, PVMCPU pVCpu)
2157{
2158 NOREF(pVM);
2159 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
2160 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
2161 pVCpu->hwaccm.s.TlbShootdown.cPages = 0;
2162 return;
2163}
2164
2165/**
2166 * Setup the tagged TLB for EPT
2167 *
2168 * @returns VBox status code.
2169 * @param pVM The VM to operate on.
2170 * @param pVCpu The VMCPU to operate on.
2171 */
2172static void hmR0VmxSetupTLBEPT(PVM pVM, PVMCPU pVCpu)
2173{
2174 PHMGLOBLCPUINFO pCpu;
2175
2176 Assert(pVM->hwaccm.s.fNestedPaging);
2177 Assert(!pVM->hwaccm.s.vmx.fVPID);
2178
2179 /* Deal with tagged TLBs if VPID or EPT is supported. */
2180 pCpu = HWACCMR0GetCurrentCpu();
2181 /* Force a TLB flush for the first world switch if the current cpu differs from the one we ran on last. */
2182 /* Note that this can happen both for start and resume due to long jumps back to ring 3. */
2183 if ( pVCpu->hwaccm.s.idLastCpu != pCpu->idCpu
2184 /* if the tlb flush count has changed, another VM has flushed the TLB of this cpu, so we can't use our current ASID anymore. */
2185 || pVCpu->hwaccm.s.cTLBFlushes != pCpu->cTLBFlushes)
2186 {
2187 /* Force a TLB flush on VM entry. */
2188 pVCpu->hwaccm.s.fForceTLBFlush = true;
2189 }
2190 /* Disabled because this has triggered every time I have suspended my
2191 * laptop with a VM running for the past three months or more. */
2192 // else
2193 // Assert(!pCpu->fFlushTLB);
2194
2195 /* Check for tlb shootdown flushes. */
2196 if (VMCPU_FF_TESTANDCLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2197 pVCpu->hwaccm.s.fForceTLBFlush = true;
2198
2199 pVCpu->hwaccm.s.idLastCpu = pCpu->idCpu;
2200 pCpu->fFlushTLB = false;
2201
2202 if (pVCpu->hwaccm.s.fForceTLBFlush)
2203 {
2204 hmR0VmxFlushEPT(pVM, pVCpu, pVM->hwaccm.s.vmx.enmFlushContext, 0);
2205 }
2206 else
2207 if (VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
2208 {
2209 /* Deal with pending TLB shootdown actions which were queued when we were not executing code. */
2210 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatTlbShootdown);
2211
2212 for (unsigned i=0;i<pVCpu->hwaccm.s.TlbShootdown.cPages;i++)
2213 {
2214 /* aTlbShootdownPages contains physical addresses in this case. */
2215 hmR0VmxFlushEPT(pVM, pVCpu, pVM->hwaccm.s.vmx.enmFlushPage, pVCpu->hwaccm.s.TlbShootdown.aPages[i]);
2216 }
2217 }
2218 pVCpu->hwaccm.s.TlbShootdown.cPages= 0;
2219 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
2220
2221#ifdef VBOX_WITH_STATISTICS
2222 if (pVCpu->hwaccm.s.fForceTLBFlush)
2223 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatFlushTLBWorldSwitch);
2224 else
2225 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatNoFlushTLBWorldSwitch);
2226#endif
2227}
2228
2229#ifdef HWACCM_VTX_WITH_VPID
2230/**
2231 * Setup the tagged TLB for VPID
2232 *
2233 * @returns VBox status code.
2234 * @param pVM The VM to operate on.
2235 * @param pVCpu The VMCPU to operate on.
2236 */
2237static void hmR0VmxSetupTLBVPID(PVM pVM, PVMCPU pVCpu)
2238{
2239 PHMGLOBLCPUINFO pCpu;
2240
2241 Assert(pVM->hwaccm.s.vmx.fVPID);
2242 Assert(!pVM->hwaccm.s.fNestedPaging);
2243
2244 /* Deal with tagged TLBs if VPID or EPT is supported. */
2245 pCpu = HWACCMR0GetCurrentCpu();
2246 /* Force a TLB flush for the first world switch if the current cpu differs from the one we ran on last. */
2247 /* Note that this can happen both for start and resume due to long jumps back to ring 3. */
2248 if ( pVCpu->hwaccm.s.idLastCpu != pCpu->idCpu
2249 /* if the tlb flush count has changed, another VM has flushed the TLB of this cpu, so we can't use our current ASID anymore. */
2250 || pVCpu->hwaccm.s.cTLBFlushes != pCpu->cTLBFlushes)
2251 {
2252 /* Force a TLB flush on VM entry. */
2253 pVCpu->hwaccm.s.fForceTLBFlush = true;
2254 }
2255 else
2256 Assert(!pCpu->fFlushTLB);
2257
2258 pVCpu->hwaccm.s.idLastCpu = pCpu->idCpu;
2259
2260 /* Check for tlb shootdown flushes. */
2261 if (VMCPU_FF_TESTANDCLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2262 pVCpu->hwaccm.s.fForceTLBFlush = true;
2263
2264 /* Make sure we flush the TLB when required. Switch ASID to achieve the same thing, but without actually flushing the whole TLB (which is expensive). */
2265 if (pVCpu->hwaccm.s.fForceTLBFlush)
2266 {
2267 if ( ++pCpu->uCurrentASID >= pVM->hwaccm.s.uMaxASID
2268 || pCpu->fFlushTLB)
2269 {
2270 pCpu->fFlushTLB = false;
2271 pCpu->uCurrentASID = 1; /* start at 1; host uses 0 */
2272 pCpu->cTLBFlushes++;
2273 hmR0VmxFlushVPID(pVM, pVCpu, VMX_FLUSH_ALL_CONTEXTS, 0);
2274 }
2275 else
2276 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatFlushASID);
2277
2278 pVCpu->hwaccm.s.fForceTLBFlush = false;
2279 pVCpu->hwaccm.s.cTLBFlushes = pCpu->cTLBFlushes;
2280 pVCpu->hwaccm.s.uCurrentASID = pCpu->uCurrentASID;
2281 }
2282 else
2283 {
2284 Assert(!pCpu->fFlushTLB);
2285 Assert(pVCpu->hwaccm.s.uCurrentASID && pCpu->uCurrentASID);
2286
2287 if (VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
2288 {
2289 /* Deal with pending TLB shootdown actions which were queued when we were not executing code. */
2290 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatTlbShootdown);
2291 for (unsigned i = 0; i < pVCpu->hwaccm.s.TlbShootdown.cPages; i++)
2292 hmR0VmxFlushVPID(pVM, pVCpu, pVM->hwaccm.s.vmx.enmFlushPage, pVCpu->hwaccm.s.TlbShootdown.aPages[i]);
2293 }
2294 }
2295 pVCpu->hwaccm.s.TlbShootdown.cPages = 0;
2296 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
2297
2298 AssertMsg(pVCpu->hwaccm.s.cTLBFlushes == pCpu->cTLBFlushes, ("Flush count mismatch for cpu %d (%x vs %x)\n", pCpu->idCpu, pVCpu->hwaccm.s.cTLBFlushes, pCpu->cTLBFlushes));
2299 AssertMsg(pCpu->uCurrentASID >= 1 && pCpu->uCurrentASID < pVM->hwaccm.s.uMaxASID, ("cpu%d uCurrentASID = %x\n", pCpu->idCpu, pCpu->uCurrentASID));
2300 AssertMsg(pVCpu->hwaccm.s.uCurrentASID >= 1 && pVCpu->hwaccm.s.uCurrentASID < pVM->hwaccm.s.uMaxASID, ("cpu%d VM uCurrentASID = %x\n", pCpu->idCpu, pVCpu->hwaccm.s.uCurrentASID));
2301
2302 int rc = VMXWriteVMCS(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hwaccm.s.uCurrentASID);
2303 AssertRC(rc);
2304
2305 if (pVCpu->hwaccm.s.fForceTLBFlush)
2306 hmR0VmxFlushVPID(pVM, pVCpu, pVM->hwaccm.s.vmx.enmFlushContext, 0);
2307
2308# ifdef VBOX_WITH_STATISTICS
2309 if (pVCpu->hwaccm.s.fForceTLBFlush)
2310 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatFlushTLBWorldSwitch);
2311 else
2312 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatNoFlushTLBWorldSwitch);
2313# endif
2314}
2315#endif /* HWACCM_VTX_WITH_VPID */
2316
2317/**
2318 * Runs guest code in a VT-x VM.
2319 *
2320 * @returns VBox status code.
2321 * @param pVM The VM to operate on.
2322 * @param pVCpu The VMCPU to operate on.
2323 * @param pCtx Guest context
2324 */
2325VMMR0DECL(int) VMXR0RunGuestCode(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
2326{
2327 STAM_PROFILE_ADV_START(&pVCpu->hwaccm.s.StatEntry, x);
2328 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hwaccm.s.StatExit1);
2329 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hwaccm.s.StatExit2);
2330
2331 VBOXSTRICTRC rc = VINF_SUCCESS;
2332 int rc2;
2333 RTGCUINTREG val;
2334 RTGCUINTREG exitReason = (RTGCUINTREG)VMX_EXIT_INVALID;
2335 RTGCUINTREG instrError, cbInstr;
2336 RTGCUINTPTR exitQualification = 0;
2337 RTGCUINTPTR intInfo = 0; /* shut up buggy gcc 4 */
2338 RTGCUINTPTR errCode, instrInfo;
2339 bool fSetupTPRCaching = false;
2340 uint64_t u64OldLSTAR = 0;
2341 uint8_t u8LastTPR = 0;
2342 RTCCUINTREG uOldEFlags = ~(RTCCUINTREG)0;
2343 unsigned cResume = 0;
2344#ifdef VBOX_STRICT
2345 RTCPUID idCpuCheck;
2346 bool fWasInLongMode = false;
2347#endif
2348#ifdef VBOX_HIGH_RES_TIMERS_HACK_IN_RING0
2349 uint64_t u64LastTime = RTTimeMilliTS();
2350#endif
2351
2352 Assert(!(pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC) || (pVCpu->hwaccm.s.vmx.pbVAPIC && pVM->hwaccm.s.vmx.pAPIC));
2353
2354 /* Check if we need to use TPR shadowing. */
2355 if ( CPUMIsGuestInLongModeEx(pCtx)
2356 || ( ((pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC) || pVM->hwaccm.s.fTRPPatchingAllowed)
2357 && pVM->hwaccm.s.fHasIoApic)
2358 )
2359 {
2360 fSetupTPRCaching = true;
2361 }
2362
2363 Log2(("\nE"));
2364
2365#ifdef VBOX_STRICT
2366 {
2367 RTCCUINTREG val2;
2368
2369 rc2 = VMXReadVMCS(VMX_VMCS_CTRL_PIN_EXEC_CONTROLS, &val2);
2370 AssertRC(rc2);
2371 Log2(("VMX_VMCS_CTRL_PIN_EXEC_CONTROLS = %08x\n", val2));
2372
2373 /* allowed zero */
2374 if ((val2 & pVM->hwaccm.s.vmx.msr.vmx_pin_ctls.n.disallowed0) != pVM->hwaccm.s.vmx.msr.vmx_pin_ctls.n.disallowed0)
2375 Log(("Invalid VMX_VMCS_CTRL_PIN_EXEC_CONTROLS: zero\n"));
2376
2377 /* allowed one */
2378 if ((val2 & ~pVM->hwaccm.s.vmx.msr.vmx_pin_ctls.n.allowed1) != 0)
2379 Log(("Invalid VMX_VMCS_CTRL_PIN_EXEC_CONTROLS: one\n"));
2380
2381 rc2 = VMXReadVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, &val2);
2382 AssertRC(rc2);
2383 Log2(("VMX_VMCS_CTRL_PROC_EXEC_CONTROLS = %08x\n", val2));
2384
2385 /* Must be set according to the MSR, but can be cleared in case of EPT. */
2386 if (pVM->hwaccm.s.fNestedPaging)
2387 val2 |= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INVLPG_EXIT
2388 | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
2389 | VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT;
2390
2391 /* allowed zero */
2392 if ((val2 & pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.disallowed0) != pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.disallowed0)
2393 Log(("Invalid VMX_VMCS_CTRL_PROC_EXEC_CONTROLS: zero\n"));
2394
2395 /* allowed one */
2396 if ((val2 & ~pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1) != 0)
2397 Log(("Invalid VMX_VMCS_CTRL_PROC_EXEC_CONTROLS: one\n"));
2398
2399 rc2 = VMXReadVMCS(VMX_VMCS_CTRL_ENTRY_CONTROLS, &val2);
2400 AssertRC(rc2);
2401 Log2(("VMX_VMCS_CTRL_ENTRY_CONTROLS = %08x\n", val2));
2402
2403 /* allowed zero */
2404 if ((val2 & pVM->hwaccm.s.vmx.msr.vmx_entry.n.disallowed0) != pVM->hwaccm.s.vmx.msr.vmx_entry.n.disallowed0)
2405 Log(("Invalid VMX_VMCS_CTRL_ENTRY_CONTROLS: zero\n"));
2406
2407 /* allowed one */
2408 if ((val2 & ~pVM->hwaccm.s.vmx.msr.vmx_entry.n.allowed1) != 0)
2409 Log(("Invalid VMX_VMCS_CTRL_ENTRY_CONTROLS: one\n"));
2410
2411 rc2 = VMXReadVMCS(VMX_VMCS_CTRL_EXIT_CONTROLS, &val2);
2412 AssertRC(rc2);
2413 Log2(("VMX_VMCS_CTRL_EXIT_CONTROLS = %08x\n", val2));
2414
2415 /* allowed zero */
2416 if ((val2 & pVM->hwaccm.s.vmx.msr.vmx_exit.n.disallowed0) != pVM->hwaccm.s.vmx.msr.vmx_exit.n.disallowed0)
2417 Log(("Invalid VMX_VMCS_CTRL_EXIT_CONTROLS: zero\n"));
2418
2419 /* allowed one */
2420 if ((val2 & ~pVM->hwaccm.s.vmx.msr.vmx_exit.n.allowed1) != 0)
2421 Log(("Invalid VMX_VMCS_CTRL_EXIT_CONTROLS: one\n"));
2422 }
2423 fWasInLongMode = CPUMIsGuestInLongModeEx(pCtx);
2424#endif /* VBOX_STRICT */
2425
2426#ifdef VBOX_WITH_CRASHDUMP_MAGIC
2427 pVCpu->hwaccm.s.vmx.VMCSCache.u64TimeEntry = RTTimeNanoTS();
2428#endif
2429
2430 /* We can jump to this point to resume execution after determining that a VM-exit is innocent.
2431 */
2432ResumeExecution:
2433 if (!STAM_REL_PROFILE_ADV_IS_RUNNING(&pVCpu->hwaccm.s.StatEntry))
2434 STAM_REL_PROFILE_ADV_STOP_START(&pVCpu->hwaccm.s.StatExit2, &pVCpu->hwaccm.s.StatEntry, x);
2435 AssertMsg(pVCpu->hwaccm.s.idEnteredCpu == RTMpCpuId(),
2436 ("Expected %d, I'm %d; cResume=%d exitReason=%RGv exitQualification=%RGv\n",
2437 (int)pVCpu->hwaccm.s.idEnteredCpu, (int)RTMpCpuId(), cResume, exitReason, exitQualification));
2438 Assert(!HWACCMR0SuspendPending());
2439 /* Not allowed to switch modes without reloading the host state (32->64 switcher)!! */
2440 Assert(fWasInLongMode == CPUMIsGuestInLongModeEx(pCtx));
2441
2442 /* Safety precaution; looping for too long here can have a very bad effect on the host */
2443 if (RT_UNLIKELY(++cResume > pVM->hwaccm.s.cMaxResumeLoops))
2444 {
2445 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitMaxResume);
2446 rc = VINF_EM_RAW_INTERRUPT;
2447 goto end;
2448 }
2449
2450 /* Check for irq inhibition due to instruction fusing (sti, mov ss). */
2451 if (VMCPU_FF_ISSET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
2452 {
2453 Log(("VM_FF_INHIBIT_INTERRUPTS at %RGv successor %RGv\n", (RTGCPTR)pCtx->rip, EMGetInhibitInterruptsPC(pVCpu)));
2454 if (pCtx->rip != EMGetInhibitInterruptsPC(pVCpu))
2455 {
2456 /* Note: we intentionally don't clear VM_FF_INHIBIT_INTERRUPTS here.
2457 * Before we are able to execute this instruction in raw mode (iret to guest code) an external interrupt might
2458 * force a world switch again. Possibly allowing a guest interrupt to be dispatched in the process. This could
2459 * break the guest. Sounds very unlikely, but such timing sensitive problems are not as rare as you might think.
2460 */
2461 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
2462 /* Irq inhibition is no longer active; clear the corresponding VMX state. */
2463 rc2 = VMXWriteVMCS(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, 0);
2464 AssertRC(rc2);
2465 }
2466 }
2467 else
2468 {
2469 /* Irq inhibition is no longer active; clear the corresponding VMX state. */
2470 rc2 = VMXWriteVMCS(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, 0);
2471 AssertRC(rc2);
2472 }
2473
2474#ifdef VBOX_HIGH_RES_TIMERS_HACK_IN_RING0
2475 if (RT_UNLIKELY((cResume & 0xf) == 0))
2476 {
2477 uint64_t u64CurTime = RTTimeMilliTS();
2478
2479 if (RT_UNLIKELY(u64CurTime > u64LastTime))
2480 {
2481 u64LastTime = u64CurTime;
2482 TMTimerPollVoid(pVM, pVCpu);
2483 }
2484 }
2485#endif
2486
2487 /* Check for pending actions that force us to go back to ring 3. */
2488 if ( VM_FF_ISPENDING(pVM, VM_FF_HWACCM_TO_R3_MASK | VM_FF_REQUEST | VM_FF_PGM_POOL_FLUSH_PENDING | VM_FF_PDM_DMA)
2489 || VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_HWACCM_TO_R3_MASK | VMCPU_FF_PGM_SYNC_CR3 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL | VMCPU_FF_REQUEST))
2490 {
2491 /* Check if a sync operation is pending. */
2492 if (VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
2493 {
2494 rc = PGMSyncCR3(pVCpu, pCtx->cr0, pCtx->cr3, pCtx->cr4, VMCPU_FF_ISSET(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
2495 if (rc != VINF_SUCCESS)
2496 {
2497 AssertRC(VBOXSTRICTRC_VAL(rc));
2498 Log(("Pending pool sync is forcing us back to ring 3; rc=%d\n", VBOXSTRICTRC_VAL(rc)));
2499 goto end;
2500 }
2501 }
2502
2503#ifdef DEBUG
2504 /* Intercept X86_XCPT_DB if stepping is enabled */
2505 if (!DBGFIsStepping(pVCpu))
2506#endif
2507 {
2508 if ( VM_FF_ISPENDING(pVM, VM_FF_HWACCM_TO_R3_MASK)
2509 || VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_HWACCM_TO_R3_MASK))
2510 {
2511 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatSwitchToR3);
2512 rc = RT_UNLIKELY(VM_FF_ISPENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
2513 goto end;
2514 }
2515 }
2516
2517 /* Pending request packets might contain actions that need immediate attention, such as pending hardware interrupts. */
2518 if ( VM_FF_ISPENDING(pVM, VM_FF_REQUEST)
2519 || VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_REQUEST))
2520 {
2521 rc = VINF_EM_PENDING_REQUEST;
2522 goto end;
2523 }
2524
2525 /* Check if a pgm pool flush is in progress. */
2526 if (VM_FF_ISPENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
2527 {
2528 rc = VINF_PGM_POOL_FLUSH_PENDING;
2529 goto end;
2530 }
2531
2532 /* Check if DMA work is pending (2nd+ run). */
2533 if (VM_FF_ISPENDING(pVM, VM_FF_PDM_DMA) && cResume > 1)
2534 {
2535 rc = VINF_EM_RAW_TO_R3;
2536 goto end;
2537 }
2538 }
2539
2540#ifdef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
2541 /*
2542 * Exit to ring-3 preemption/work is pending.
2543 *
2544 * Interrupts are disabled before the call to make sure we don't miss any interrupt
2545 * that would flag preemption (IPI, timer tick, ++). (Would've been nice to do this
2546 * further down, but hmR0VmxCheckPendingInterrupt makes that impossible.)
2547 *
2548 * Note! Interrupts must be disabled done *before* we check for TLB flushes; TLB
2549 * shootdowns rely on this.
2550 */
2551 uOldEFlags = ASMIntDisableFlags();
2552 if (RTThreadPreemptIsPending(NIL_RTTHREAD))
2553 {
2554 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitPreemptPending);
2555 rc = VINF_EM_RAW_INTERRUPT;
2556 goto end;
2557 }
2558 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
2559#endif
2560
2561 /* When external interrupts are pending, we should exit the VM when IF is set. */
2562 /* Note! *After* VM_FF_INHIBIT_INTERRUPTS check!!! */
2563 rc = hmR0VmxCheckPendingInterrupt(pVM, pVCpu, pCtx);
2564 if (RT_FAILURE(rc))
2565 goto end;
2566
2567 /** @todo check timers?? */
2568
2569 /* TPR caching using CR8 is only available in 64 bits mode */
2570 /* Note the 32 bits exception for AMD (X86_CPUID_AMD_FEATURE_ECX_CR8L), but that appears missing in Intel CPUs */
2571 /* Note: we can't do this in LoadGuestState as PDMApicGetTPR can jump back to ring 3 (lock)!!!!! (no longer true) */
2572 /**
2573 * @todo query and update the TPR only when it could have been changed (mmio access & wrmsr (x2apic))
2574 */
2575 if (fSetupTPRCaching)
2576 {
2577 /* TPR caching in CR8 */
2578 bool fPending;
2579
2580 rc2 = PDMApicGetTPR(pVCpu, &u8LastTPR, &fPending);
2581 AssertRC(rc2);
2582 /* The TPR can be found at offset 0x80 in the APIC mmio page. */
2583 pVCpu->hwaccm.s.vmx.pbVAPIC[0x80] = u8LastTPR;
2584
2585 /* Two options here:
2586 * - external interrupt pending, but masked by the TPR value.
2587 * -> a CR8 update that lower the current TPR value should cause an exit
2588 * - no pending interrupts
2589 * -> We don't need to be explicitely notified. There are enough world switches for detecting pending interrupts.
2590 */
2591 rc = VMXWriteVMCS(VMX_VMCS_CTRL_TPR_THRESHOLD, (fPending) ? (u8LastTPR >> 4) : 0); /* cr8 bits 3-0 correspond to bits 7-4 of the task priority mmio register. */
2592 AssertRC(VBOXSTRICTRC_VAL(rc));
2593
2594 if (pVM->hwaccm.s.fTPRPatchingActive)
2595 {
2596 Assert(!CPUMIsGuestInLongModeEx(pCtx));
2597 /* Our patch code uses LSTAR for TPR caching. */
2598 pCtx->msrLSTAR = u8LastTPR;
2599
2600 if (fPending)
2601 {
2602 /* A TPR change could activate a pending interrupt, so catch lstar writes. */
2603 hmR0VmxSetMSRPermission(pVCpu, MSR_K8_LSTAR, true, false);
2604 }
2605 else
2606 {
2607 /* No interrupts are pending, so we don't need to be explicitely notified.
2608 * There are enough world switches for detecting pending interrupts.
2609 */
2610 hmR0VmxSetMSRPermission(pVCpu, MSR_K8_LSTAR, true, true);
2611 }
2612 }
2613 }
2614
2615#if defined(HWACCM_VTX_WITH_EPT) && defined(LOG_ENABLED)
2616 if ( pVM->hwaccm.s.fNestedPaging
2617# ifdef HWACCM_VTX_WITH_VPID
2618 || pVM->hwaccm.s.vmx.fVPID
2619# endif /* HWACCM_VTX_WITH_VPID */
2620 )
2621 {
2622 PHMGLOBLCPUINFO pCpu;
2623
2624 pCpu = HWACCMR0GetCurrentCpu();
2625 if ( pVCpu->hwaccm.s.idLastCpu != pCpu->idCpu
2626 || pVCpu->hwaccm.s.cTLBFlushes != pCpu->cTLBFlushes)
2627 {
2628 if (pVCpu->hwaccm.s.idLastCpu != pCpu->idCpu)
2629 LogFlow(("Force TLB flush due to rescheduling to a different cpu (%d vs %d)\n", pVCpu->hwaccm.s.idLastCpu, pCpu->idCpu));
2630 else
2631 LogFlow(("Force TLB flush due to changed TLB flush count (%x vs %x)\n", pVCpu->hwaccm.s.cTLBFlushes, pCpu->cTLBFlushes));
2632 }
2633 if (pCpu->fFlushTLB)
2634 LogFlow(("Force TLB flush: first time cpu %d is used -> flush\n", pCpu->idCpu));
2635 else
2636 if (pVCpu->hwaccm.s.fForceTLBFlush)
2637 LogFlow(("Manual TLB flush\n"));
2638 }
2639#endif
2640#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
2641 PGMRZDynMapFlushAutoSet(pVCpu);
2642#endif
2643
2644 /*
2645 * NOTE: DO NOT DO ANYTHING AFTER THIS POINT THAT MIGHT JUMP BACK TO RING 3!
2646 * (until the actual world switch)
2647 */
2648#ifdef VBOX_STRICT
2649 idCpuCheck = RTMpCpuId();
2650#endif
2651#ifdef LOG_ENABLED
2652 VMMR0LogFlushDisable(pVCpu);
2653#endif
2654 /* Save the host state first. */
2655 if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_HOST_CONTEXT)
2656 {
2657 rc = VMXR0SaveHostState(pVM, pVCpu);
2658 if (RT_UNLIKELY(rc != VINF_SUCCESS))
2659 {
2660 VMMR0LogFlushEnable(pVCpu);
2661 goto end;
2662 }
2663 }
2664
2665 /* Load the guest state */
2666 if (!pVCpu->hwaccm.s.fContextUseFlags)
2667 {
2668 VMXR0LoadMinimalGuestState(pVM, pVCpu, pCtx);
2669 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatLoadMinimal);
2670 }
2671 else
2672 {
2673 rc = VMXR0LoadGuestState(pVM, pVCpu, pCtx);
2674 if (RT_UNLIKELY(rc != VINF_SUCCESS))
2675 {
2676 VMMR0LogFlushEnable(pVCpu);
2677 goto end;
2678 }
2679 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatLoadFull);
2680 }
2681
2682#ifndef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
2683 /* Disable interrupts to make sure a poke will interrupt execution.
2684 * This must be done *before* we check for TLB flushes; TLB shootdowns rely on this.
2685 */
2686 uOldEFlags = ASMIntDisableFlags();
2687 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
2688#endif
2689
2690 /* Non-register state Guest Context */
2691 /** @todo change me according to cpu state */
2692 rc2 = VMXWriteVMCS(VMX_VMCS32_GUEST_ACTIVITY_STATE, VMX_CMS_GUEST_ACTIVITY_ACTIVE);
2693 AssertRC(rc2);
2694
2695 /* Set TLB flush state as checked until we return from the world switch. */
2696 ASMAtomicWriteBool(&pVCpu->hwaccm.s.fCheckedTLBFlush, true);
2697 /* Deal with tagged TLB setup and invalidation. */
2698 pVM->hwaccm.s.vmx.pfnSetupTaggedTLB(pVM, pVCpu);
2699
2700 /* Manual save and restore:
2701 * - General purpose registers except RIP, RSP
2702 *
2703 * Trashed:
2704 * - CR2 (we don't care)
2705 * - LDTR (reset to 0)
2706 * - DRx (presumably not changed at all)
2707 * - DR7 (reset to 0x400)
2708 * - EFLAGS (reset to RT_BIT(1); not relevant)
2709 *
2710 */
2711
2712 /* All done! Let's start VM execution. */
2713 STAM_PROFILE_ADV_STOP_START(&pVCpu->hwaccm.s.StatEntry, &pVCpu->hwaccm.s.StatInGC, x);
2714 Assert(idCpuCheck == RTMpCpuId());
2715
2716#ifdef VBOX_WITH_CRASHDUMP_MAGIC
2717 pVCpu->hwaccm.s.vmx.VMCSCache.cResume = cResume;
2718 pVCpu->hwaccm.s.vmx.VMCSCache.u64TimeSwitch = RTTimeNanoTS();
2719#endif
2720
2721 /* Save the current TPR value in the LSTAR msr so our patches can access it. */
2722 if (pVM->hwaccm.s.fTPRPatchingActive)
2723 {
2724 Assert(pVM->hwaccm.s.fTPRPatchingActive);
2725 u64OldLSTAR = ASMRdMsr(MSR_K8_LSTAR);
2726 ASMWrMsr(MSR_K8_LSTAR, u8LastTPR);
2727 }
2728
2729 TMNotifyStartOfExecution(pVCpu);
2730#ifdef VBOX_WITH_KERNEL_USING_XMM
2731 rc = hwaccmR0VMXStartVMWrapXMM(pVCpu->hwaccm.s.fResumeVM, pCtx, &pVCpu->hwaccm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hwaccm.s.vmx.pfnStartVM);
2732#else
2733 rc = pVCpu->hwaccm.s.vmx.pfnStartVM(pVCpu->hwaccm.s.fResumeVM, pCtx, &pVCpu->hwaccm.s.vmx.VMCSCache, pVM, pVCpu);
2734#endif
2735 ASMAtomicWriteBool(&pVCpu->hwaccm.s.fCheckedTLBFlush, false);
2736 ASMAtomicIncU32(&pVCpu->hwaccm.s.cWorldSwitchExits);
2737 /* Possibly the last TSC value seen by the guest (too high) (only when we're in tsc offset mode). */
2738 if (!(pVCpu->hwaccm.s.vmx.proc_ctls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT))
2739 TMCpuTickSetLastSeen(pVCpu, ASMReadTSC() + pVCpu->hwaccm.s.vmx.u64TSCOffset - 0x400 /* guestimate of world switch overhead in clock ticks */);
2740
2741 TMNotifyEndOfExecution(pVCpu);
2742 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED);
2743 Assert(!(ASMGetFlags() & X86_EFL_IF));
2744
2745 /* Restore the host LSTAR msr if the guest could have changed it. */
2746 if (pVM->hwaccm.s.fTPRPatchingActive)
2747 {
2748 Assert(pVM->hwaccm.s.fTPRPatchingActive);
2749 pVCpu->hwaccm.s.vmx.pbVAPIC[0x80] = pCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR);
2750 ASMWrMsr(MSR_K8_LSTAR, u64OldLSTAR);
2751 }
2752
2753 STAM_PROFILE_ADV_STOP_START(&pVCpu->hwaccm.s.StatInGC, &pVCpu->hwaccm.s.StatExit1, x);
2754 ASMSetFlags(uOldEFlags);
2755#ifdef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
2756 uOldEFlags = ~(RTCCUINTREG)0;
2757#endif
2758
2759 AssertMsg(!pVCpu->hwaccm.s.vmx.VMCSCache.Write.cValidEntries, ("pVCpu->hwaccm.s.vmx.VMCSCache.Write.cValidEntries=%d\n", pVCpu->hwaccm.s.vmx.VMCSCache.Write.cValidEntries));
2760
2761 /* In case we execute a goto ResumeExecution later on. */
2762 pVCpu->hwaccm.s.fResumeVM = true;
2763 pVCpu->hwaccm.s.fForceTLBFlush = false;
2764
2765 /*
2766 * !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
2767 * IMPORTANT: WE CAN'T DO ANY LOGGING OR OPERATIONS THAT CAN DO A LONGJMP BACK TO RING 3 *BEFORE* WE'VE SYNCED BACK (MOST OF) THE GUEST STATE
2768 * !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
2769 */
2770
2771 if (RT_UNLIKELY(rc != VINF_SUCCESS))
2772 {
2773 hmR0VmxReportWorldSwitchError(pVM, pVCpu, rc, pCtx);
2774 VMMR0LogFlushEnable(pVCpu);
2775 goto end;
2776 }
2777
2778 /* Success. Query the guest state and figure out what has happened. */
2779
2780 /* Investigate why there was a VM-exit. */
2781 rc2 = VMXReadCachedVMCS(VMX_VMCS32_RO_EXIT_REASON, &exitReason);
2782 STAM_COUNTER_INC(&pVCpu->hwaccm.s.paStatExitReasonR0[exitReason & MASK_EXITREASON_STAT]);
2783
2784 exitReason &= 0xffff; /* bit 0-15 contain the exit code. */
2785 rc2 |= VMXReadCachedVMCS(VMX_VMCS32_RO_VM_INSTR_ERROR, &instrError);
2786 rc2 |= VMXReadCachedVMCS(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &cbInstr);
2787 rc2 |= VMXReadCachedVMCS(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &intInfo);
2788 /* might not be valid; depends on VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID. */
2789 rc2 |= VMXReadCachedVMCS(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERRCODE, &errCode);
2790 rc2 |= VMXReadCachedVMCS(VMX_VMCS32_RO_EXIT_INSTR_INFO, &instrInfo);
2791 rc2 |= VMXReadCachedVMCS(VMX_VMCS_RO_EXIT_QUALIFICATION, &exitQualification);
2792 AssertRC(rc2);
2793
2794 /* Sync back the guest state */
2795 rc2 = VMXR0SaveGuestState(pVM, pVCpu, pCtx);
2796 AssertRC(rc2);
2797
2798 /* Note! NOW IT'S SAFE FOR LOGGING! */
2799 VMMR0LogFlushEnable(pVCpu);
2800 Log2(("Raw exit reason %08x\n", exitReason));
2801
2802 /* Check if an injected event was interrupted prematurely. */
2803 rc2 = VMXReadCachedVMCS(VMX_VMCS32_RO_IDT_INFO, &val);
2804 AssertRC(rc2);
2805 pVCpu->hwaccm.s.Event.intInfo = VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(val);
2806 if ( VMX_EXIT_INTERRUPTION_INFO_VALID(pVCpu->hwaccm.s.Event.intInfo)
2807 /* Ignore 'int xx' as they'll be restarted anyway. */
2808 && VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hwaccm.s.Event.intInfo) != VMX_EXIT_INTERRUPTION_INFO_TYPE_SW
2809 /* Ignore software exceptions (such as int3) as they'll reoccur when we restart the instruction anyway. */
2810 && VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hwaccm.s.Event.intInfo) != VMX_EXIT_INTERRUPTION_INFO_TYPE_SWEXCPT)
2811 {
2812 Assert(!pVCpu->hwaccm.s.Event.fPending);
2813 pVCpu->hwaccm.s.Event.fPending = true;
2814 /* Error code present? */
2815 if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(pVCpu->hwaccm.s.Event.intInfo))
2816 {
2817 rc2 = VMXReadCachedVMCS(VMX_VMCS32_RO_IDT_ERRCODE, &val);
2818 AssertRC(rc2);
2819 pVCpu->hwaccm.s.Event.errCode = val;
2820 Log(("Pending inject %RX64 at %RGv exit=%08x intInfo=%08x exitQualification=%RGv pending error=%RX64\n", pVCpu->hwaccm.s.Event.intInfo, (RTGCPTR)pCtx->rip, exitReason, intInfo, exitQualification, val));
2821 }
2822 else
2823 {
2824 Log(("Pending inject %RX64 at %RGv exit=%08x intInfo=%08x exitQualification=%RGv\n", pVCpu->hwaccm.s.Event.intInfo, (RTGCPTR)pCtx->rip, exitReason, intInfo, exitQualification));
2825 pVCpu->hwaccm.s.Event.errCode = 0;
2826 }
2827 }
2828#ifdef VBOX_STRICT
2829 else
2830 if ( VMX_EXIT_INTERRUPTION_INFO_VALID(pVCpu->hwaccm.s.Event.intInfo)
2831 /* Ignore software exceptions (such as int3) as they're reoccur when we restart the instruction anyway. */
2832 && VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hwaccm.s.Event.intInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_SWEXCPT)
2833 {
2834 Log(("Ignore pending inject %RX64 at %RGv exit=%08x intInfo=%08x exitQualification=%RGv\n", pVCpu->hwaccm.s.Event.intInfo, (RTGCPTR)pCtx->rip, exitReason, intInfo, exitQualification));
2835 }
2836
2837 if (exitReason == VMX_EXIT_ERR_INVALID_GUEST_STATE)
2838 HWACCMDumpRegs(pVM, pVCpu, pCtx);
2839#endif
2840
2841 Log2(("E%d: New EIP=%x:%RGv\n", (uint32_t)exitReason, pCtx->cs, (RTGCPTR)pCtx->rip));
2842 Log2(("Exit reason %d, exitQualification %RGv\n", (uint32_t)exitReason, exitQualification));
2843 Log2(("instrInfo=%d instrError=%d instr length=%d\n", (uint32_t)instrInfo, (uint32_t)instrError, (uint32_t)cbInstr));
2844 Log2(("Interruption error code %d\n", (uint32_t)errCode));
2845 Log2(("IntInfo = %08x\n", (uint32_t)intInfo));
2846
2847 /* Sync back the TPR if it was changed. */
2848 if ( fSetupTPRCaching
2849 && u8LastTPR != pVCpu->hwaccm.s.vmx.pbVAPIC[0x80])
2850 {
2851 rc2 = PDMApicSetTPR(pVCpu, pVCpu->hwaccm.s.vmx.pbVAPIC[0x80]);
2852 AssertRC(rc2);
2853 }
2854
2855#ifdef DBGFTRACE_ENABLED /** @todo DTrace later. */
2856 RTTraceBufAddMsgF(pVM->CTX_SUFF(hTraceBuf), "vmexit %08x %016RX64 at %04:%08RX64 %RX64",
2857 exitReason, (uint64_t)exitQualification, pCtx->cs, pCtx->rip, (uint64_t)intInfo);
2858#endif
2859 STAM_PROFILE_ADV_STOP_START(&pVCpu->hwaccm.s.StatExit1, &pVCpu->hwaccm.s.StatExit2, x);
2860
2861 /* Some cases don't need a complete resync of the guest CPU state; handle them here. */
2862 Assert(rc == VINF_SUCCESS); /* might consider VERR_IPE_UNINITIALIZED_STATUS here later... */
2863 switch (exitReason)
2864 {
2865 case VMX_EXIT_EXCEPTION: /* 0 Exception or non-maskable interrupt (NMI). */
2866 case VMX_EXIT_EXTERNAL_IRQ: /* 1 External interrupt. */
2867 {
2868 uint32_t vector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(intInfo);
2869
2870 if (!VMX_EXIT_INTERRUPTION_INFO_VALID(intInfo))
2871 {
2872 Assert(exitReason == VMX_EXIT_EXTERNAL_IRQ);
2873#if 0 //def VBOX_WITH_VMMR0_DISABLE_PREEMPTION
2874 if ( RTThreadPreemptIsPendingTrusty()
2875 && !RTThreadPreemptIsPending(NIL_RTTHREAD))
2876 goto ResumeExecution;
2877#endif
2878 /* External interrupt; leave to allow it to be dispatched again. */
2879 rc = VINF_EM_RAW_INTERRUPT;
2880 break;
2881 }
2882 STAM_PROFILE_ADV_START(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
2883 switch (VMX_EXIT_INTERRUPTION_INFO_TYPE(intInfo))
2884 {
2885 case VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI: /* Non-maskable interrupt. */
2886 /* External interrupt; leave to allow it to be dispatched again. */
2887 rc = VINF_EM_RAW_INTERRUPT;
2888 break;
2889
2890 case VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT: /* External hardware interrupt. */
2891 AssertFailed(); /* can't come here; fails the first check. */
2892 break;
2893
2894 case VMX_EXIT_INTERRUPTION_INFO_TYPE_DBEXCPT: /* Unknown why we get this type for #DB */
2895 case VMX_EXIT_INTERRUPTION_INFO_TYPE_SWEXCPT: /* Software exception. (#BP or #OF) */
2896 Assert(vector == 1 || vector == 3 || vector == 4);
2897 /* no break */
2898 case VMX_EXIT_INTERRUPTION_INFO_TYPE_HWEXCPT: /* Hardware exception. */
2899 Log2(("Hardware/software interrupt %d\n", vector));
2900 switch (vector)
2901 {
2902 case X86_XCPT_NM:
2903 {
2904 Log(("#NM fault at %RGv error code %x\n", (RTGCPTR)pCtx->rip, errCode));
2905
2906 /** @todo don't intercept #NM exceptions anymore when we've activated the guest FPU state. */
2907 /* If we sync the FPU/XMM state on-demand, then we can continue execution as if nothing has happened. */
2908 rc = CPUMR0LoadGuestFPU(pVM, pVCpu, pCtx);
2909 if (rc == VINF_SUCCESS)
2910 {
2911 Assert(CPUMIsGuestFPUStateActive(pVCpu));
2912
2913 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitShadowNM);
2914
2915 /* Continue execution. */
2916 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_GUEST_CR0;
2917
2918 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
2919 goto ResumeExecution;
2920 }
2921
2922 Log(("Forward #NM fault to the guest\n"));
2923 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestNM);
2924 rc2 = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), cbInstr, 0);
2925 AssertRC(rc2);
2926 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
2927 goto ResumeExecution;
2928 }
2929
2930 case X86_XCPT_PF: /* Page fault */
2931 {
2932#ifdef DEBUG
2933 if (pVM->hwaccm.s.fNestedPaging)
2934 { /* A genuine pagefault.
2935 * Forward the trap to the guest by injecting the exception and resuming execution.
2936 */
2937 Log(("Guest page fault at %RGv cr2=%RGv error code %RGv rsp=%RGv\n", (RTGCPTR)pCtx->rip, exitQualification, errCode, (RTGCPTR)pCtx->rsp));
2938
2939 Assert(CPUMIsGuestInPagedProtectedModeEx(pCtx));
2940
2941 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestPF);
2942
2943 /* Now we must update CR2. */
2944 pCtx->cr2 = exitQualification;
2945 rc2 = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), cbInstr, errCode);
2946 AssertRC(rc2);
2947
2948 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
2949 goto ResumeExecution;
2950 }
2951#endif
2952 Assert(!pVM->hwaccm.s.fNestedPaging);
2953
2954#ifdef VBOX_HWACCM_WITH_GUEST_PATCHING
2955 /* Shortcut for APIC TPR reads and writes; 32 bits guests only */
2956 if ( pVM->hwaccm.s.fTRPPatchingAllowed
2957 && pVM->hwaccm.s.pGuestPatchMem
2958 && (exitQualification & 0xfff) == 0x080
2959 && !(errCode & X86_TRAP_PF_P) /* not present */
2960 && CPUMGetGuestCPL(pVCpu, CPUMCTX2CORE(pCtx)) == 0
2961 && !CPUMIsGuestInLongModeEx(pCtx)
2962 && pVM->hwaccm.s.cPatches < RT_ELEMENTS(pVM->hwaccm.s.aPatches))
2963 {
2964 RTGCPHYS GCPhysApicBase, GCPhys;
2965 PDMApicGetBase(pVM, &GCPhysApicBase); /* @todo cache this */
2966 GCPhysApicBase &= PAGE_BASE_GC_MASK;
2967
2968 rc = PGMGstGetPage(pVCpu, (RTGCPTR)exitQualification, NULL, &GCPhys);
2969 if ( rc == VINF_SUCCESS
2970 && GCPhys == GCPhysApicBase)
2971 {
2972 /* Only attempt to patch the instruction once. */
2973 PHWACCMTPRPATCH pPatch = (PHWACCMTPRPATCH)RTAvloU32Get(&pVM->hwaccm.s.PatchTree, (AVLOU32KEY)pCtx->eip);
2974 if (!pPatch)
2975 {
2976 rc = VINF_EM_HWACCM_PATCH_TPR_INSTR;
2977 break;
2978 }
2979 }
2980 }
2981#endif
2982
2983 Log2(("Page fault at %RGv error code %x\n", exitQualification, errCode));
2984 /* Exit qualification contains the linear address of the page fault. */
2985 TRPMAssertTrap(pVCpu, X86_XCPT_PF, TRPM_TRAP);
2986 TRPMSetErrorCode(pVCpu, errCode);
2987 TRPMSetFaultAddress(pVCpu, exitQualification);
2988
2989 /* Shortcut for APIC TPR reads and writes. */
2990 if ( (exitQualification & 0xfff) == 0x080
2991 && !(errCode & X86_TRAP_PF_P) /* not present */
2992 && fSetupTPRCaching
2993 && (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC))
2994 {
2995 RTGCPHYS GCPhysApicBase, GCPhys;
2996 PDMApicGetBase(pVM, &GCPhysApicBase); /* @todo cache this */
2997 GCPhysApicBase &= PAGE_BASE_GC_MASK;
2998
2999 rc = PGMGstGetPage(pVCpu, (RTGCPTR)exitQualification, NULL, &GCPhys);
3000 if ( rc == VINF_SUCCESS
3001 && GCPhys == GCPhysApicBase)
3002 {
3003 Log(("Enable VT-x virtual APIC access filtering\n"));
3004 rc2 = IOMMMIOMapMMIOHCPage(pVM, GCPhysApicBase, pVM->hwaccm.s.vmx.pAPICPhys, X86_PTE_RW | X86_PTE_P);
3005 AssertRC(rc2);
3006 }
3007 }
3008
3009 /* Forward it to our trap handler first, in case our shadow pages are out of sync. */
3010 rc = PGMTrap0eHandler(pVCpu, errCode, CPUMCTX2CORE(pCtx), (RTGCPTR)exitQualification);
3011 Log2(("PGMTrap0eHandler %RGv returned %Rrc\n", (RTGCPTR)pCtx->rip, VBOXSTRICTRC_VAL(rc)));
3012
3013 if (rc == VINF_SUCCESS)
3014 { /* We've successfully synced our shadow pages, so let's just continue execution. */
3015 Log2(("Shadow page fault at %RGv cr2=%RGv error code %x\n", (RTGCPTR)pCtx->rip, exitQualification ,errCode));
3016 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitShadowPF);
3017
3018 TRPMResetTrap(pVCpu);
3019 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3020 goto ResumeExecution;
3021 }
3022 else
3023 if (rc == VINF_EM_RAW_GUEST_TRAP)
3024 { /* A genuine pagefault.
3025 * Forward the trap to the guest by injecting the exception and resuming execution.
3026 */
3027 Log2(("Forward page fault to the guest\n"));
3028
3029 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestPF);
3030 /* The error code might have been changed. */
3031 errCode = TRPMGetErrorCode(pVCpu);
3032
3033 TRPMResetTrap(pVCpu);
3034
3035 /* Now we must update CR2. */
3036 pCtx->cr2 = exitQualification;
3037 rc2 = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), cbInstr, errCode);
3038 AssertRC(rc2);
3039
3040 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3041 goto ResumeExecution;
3042 }
3043#ifdef VBOX_STRICT
3044 if (rc != VINF_EM_RAW_EMULATE_INSTR && rc != VINF_EM_RAW_EMULATE_IO_BLOCK)
3045 Log2(("PGMTrap0eHandler failed with %d\n", VBOXSTRICTRC_VAL(rc)));
3046#endif
3047 /* Need to go back to the recompiler to emulate the instruction. */
3048 TRPMResetTrap(pVCpu);
3049 break;
3050 }
3051
3052 case X86_XCPT_MF: /* Floating point exception. */
3053 {
3054 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestMF);
3055 if (!(pCtx->cr0 & X86_CR0_NE))
3056 {
3057 /* old style FPU error reporting needs some extra work. */
3058 /** @todo don't fall back to the recompiler, but do it manually. */
3059 rc = VINF_EM_RAW_EMULATE_INSTR;
3060 break;
3061 }
3062 Log(("Trap %x at %04X:%RGv\n", vector, pCtx->cs, (RTGCPTR)pCtx->rip));
3063 rc2 = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), cbInstr, errCode);
3064 AssertRC(rc2);
3065
3066 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3067 goto ResumeExecution;
3068 }
3069
3070 case X86_XCPT_DB: /* Debug exception. */
3071 {
3072 uint64_t uDR6;
3073
3074 /* DR6, DR7.GD and IA32_DEBUGCTL.LBR are not updated yet.
3075 *
3076 * Exit qualification bits:
3077 * 3:0 B0-B3 which breakpoint condition was met
3078 * 12:4 Reserved (0)
3079 * 13 BD - debug register access detected
3080 * 14 BS - single step execution or branch taken
3081 * 63:15 Reserved (0)
3082 */
3083 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestDB);
3084
3085 /* Note that we don't support guest and host-initiated debugging at the same time. */
3086
3087 uDR6 = X86_DR6_INIT_VAL;
3088 uDR6 |= (exitQualification & (X86_DR6_B0|X86_DR6_B1|X86_DR6_B2|X86_DR6_B3|X86_DR6_BD|X86_DR6_BS));
3089 rc = DBGFRZTrap01Handler(pVM, pVCpu, CPUMCTX2CORE(pCtx), uDR6);
3090 if (rc == VINF_EM_RAW_GUEST_TRAP)
3091 {
3092 /* Update DR6 here. */
3093 pCtx->dr[6] = uDR6;
3094
3095 /* Resync DR6 if the debug state is active. */
3096 if (CPUMIsGuestDebugStateActive(pVCpu))
3097 ASMSetDR6(pCtx->dr[6]);
3098
3099 /* X86_DR7_GD will be cleared if drx accesses should be trapped inside the guest. */
3100 pCtx->dr[7] &= ~X86_DR7_GD;
3101
3102 /* Paranoia. */
3103 pCtx->dr[7] &= 0xffffffff; /* upper 32 bits reserved */
3104 pCtx->dr[7] &= ~(RT_BIT(11) | RT_BIT(12) | RT_BIT(14) | RT_BIT(15)); /* must be zero */
3105 pCtx->dr[7] |= 0x400; /* must be one */
3106
3107 /* Resync DR7 */
3108 rc2 = VMXWriteVMCS64(VMX_VMCS64_GUEST_DR7, pCtx->dr[7]);
3109 AssertRC(rc2);
3110
3111 Log(("Trap %x (debug) at %RGv exit qualification %RX64 dr6=%x dr7=%x\n", vector, (RTGCPTR)pCtx->rip, exitQualification, (uint32_t)pCtx->dr[6], (uint32_t)pCtx->dr[7]));
3112 rc2 = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), cbInstr, errCode);
3113 AssertRC(rc2);
3114
3115 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3116 goto ResumeExecution;
3117 }
3118 /* Return to ring 3 to deal with the debug exit code. */
3119 Log(("Debugger hardware BP at %04x:%RGv (rc=%Rrc)\n", pCtx->cs, pCtx->rip, VBOXSTRICTRC_VAL(rc)));
3120 break;
3121 }
3122
3123 case X86_XCPT_BP: /* Breakpoint. */
3124 {
3125 rc = DBGFRZTrap03Handler(pVM, pVCpu, CPUMCTX2CORE(pCtx));
3126 if (rc == VINF_EM_RAW_GUEST_TRAP)
3127 {
3128 Log(("Guest #BP at %04x:%RGv\n", pCtx->cs, pCtx->rip));
3129 rc2 = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), cbInstr, errCode);
3130 AssertRC(rc2);
3131 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3132 goto ResumeExecution;
3133 }
3134 if (rc == VINF_SUCCESS)
3135 {
3136 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3137 goto ResumeExecution;
3138 }
3139 Log(("Debugger BP at %04x:%RGv (rc=%Rrc)\n", pCtx->cs, pCtx->rip, VBOXSTRICTRC_VAL(rc)));
3140 break;
3141 }
3142
3143 case X86_XCPT_GP: /* General protection failure exception.*/
3144 {
3145 uint32_t cbOp;
3146 uint32_t cbSize;
3147 PDISCPUSTATE pDis = &pVCpu->hwaccm.s.DisState;
3148
3149 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestGP);
3150#ifdef VBOX_STRICT
3151 if ( !CPUMIsGuestInRealModeEx(pCtx)
3152 || !pVM->hwaccm.s.vmx.pRealModeTSS)
3153 {
3154 Log(("Trap %x at %04X:%RGv errorCode=%RGv\n", vector, pCtx->cs, (RTGCPTR)pCtx->rip, errCode));
3155 rc2 = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), cbInstr, errCode);
3156 AssertRC(rc2);
3157 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3158 goto ResumeExecution;
3159 }
3160#endif
3161 Assert(CPUMIsGuestInRealModeEx(pCtx));
3162
3163 LogFlow(("Real mode X86_XCPT_GP instruction emulation at %x:%RGv\n", pCtx->cs, (RTGCPTR)pCtx->rip));
3164
3165 rc2 = EMInterpretDisasOne(pVM, pVCpu, CPUMCTX2CORE(pCtx), pDis, &cbOp);
3166 if (RT_SUCCESS(rc2))
3167 {
3168 bool fUpdateRIP = true;
3169
3170 rc = VINF_SUCCESS;
3171 Assert(cbOp == pDis->opsize);
3172 switch (pDis->pCurInstr->opcode)
3173 {
3174 case OP_CLI:
3175 pCtx->eflags.Bits.u1IF = 0;
3176 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitCli);
3177 break;
3178
3179 case OP_STI:
3180 pCtx->eflags.Bits.u1IF = 1;
3181 EMSetInhibitInterruptsPC(pVCpu, pCtx->rip + pDis->opsize);
3182 Assert(VMCPU_FF_ISSET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
3183 rc2 = VMXWriteVMCS(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
3184 AssertRC(rc2);
3185 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitSti);
3186 break;
3187
3188 case OP_HLT:
3189 fUpdateRIP = false;
3190 rc = VINF_EM_HALT;
3191 pCtx->rip += pDis->opsize;
3192 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitHlt);
3193 break;
3194
3195 case OP_POPF:
3196 {
3197 RTGCPTR GCPtrStack;
3198 uint32_t cbParm;
3199 uint32_t uMask;
3200 X86EFLAGS eflags;
3201
3202 if (pDis->prefix & PREFIX_OPSIZE)
3203 {
3204 cbParm = 4;
3205 uMask = 0xffffffff;
3206 }
3207 else
3208 {
3209 cbParm = 2;
3210 uMask = 0xffff;
3211 }
3212
3213 rc2 = SELMToFlatEx(pVM, DIS_SELREG_SS, CPUMCTX2CORE(pCtx), pCtx->esp & uMask, 0, &GCPtrStack);
3214 if (RT_FAILURE(rc2))
3215 {
3216 rc = VERR_EM_INTERPRETER;
3217 break;
3218 }
3219 eflags.u = 0;
3220 rc2 = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &eflags.u, cbParm);
3221 if (RT_FAILURE(rc2))
3222 {
3223 rc = VERR_EM_INTERPRETER;
3224 break;
3225 }
3226 LogFlow(("POPF %x -> %RGv mask=%x\n", eflags.u, pCtx->rsp, uMask));
3227 pCtx->eflags.u = (pCtx->eflags.u & ~(X86_EFL_POPF_BITS & uMask)) | (eflags.u & X86_EFL_POPF_BITS & uMask);
3228 /* RF cleared when popped in real mode; see pushf description in AMD manual. */
3229 pCtx->eflags.Bits.u1RF = 0;
3230 pCtx->esp += cbParm;
3231 pCtx->esp &= uMask;
3232
3233 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitPopf);
3234 break;
3235 }
3236
3237 case OP_PUSHF:
3238 {
3239 RTGCPTR GCPtrStack;
3240 uint32_t cbParm;
3241 uint32_t uMask;
3242 X86EFLAGS eflags;
3243
3244 if (pDis->prefix & PREFIX_OPSIZE)
3245 {
3246 cbParm = 4;
3247 uMask = 0xffffffff;
3248 }
3249 else
3250 {
3251 cbParm = 2;
3252 uMask = 0xffff;
3253 }
3254
3255 rc2 = SELMToFlatEx(pVM, DIS_SELREG_SS, CPUMCTX2CORE(pCtx), (pCtx->esp - cbParm) & uMask, 0, &GCPtrStack);
3256 if (RT_FAILURE(rc2))
3257 {
3258 rc = VERR_EM_INTERPRETER;
3259 break;
3260 }
3261 eflags = pCtx->eflags;
3262 /* RF & VM cleared when pushed in real mode; see pushf description in AMD manual. */
3263 eflags.Bits.u1RF = 0;
3264 eflags.Bits.u1VM = 0;
3265
3266 rc2 = PGMPhysWrite(pVM, (RTGCPHYS)GCPtrStack, &eflags.u, cbParm);
3267 if (RT_FAILURE(rc2))
3268 {
3269 rc = VERR_EM_INTERPRETER;
3270 break;
3271 }
3272 LogFlow(("PUSHF %x -> %RGv\n", eflags.u, GCPtrStack));
3273 pCtx->esp -= cbParm;
3274 pCtx->esp &= uMask;
3275 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitPushf);
3276 break;
3277 }
3278
3279 case OP_IRET:
3280 {
3281 RTGCPTR GCPtrStack;
3282 uint32_t uMask = 0xffff;
3283 uint16_t aIretFrame[3];
3284
3285 if (pDis->prefix & (PREFIX_OPSIZE | PREFIX_ADDRSIZE))
3286 {
3287 rc = VERR_EM_INTERPRETER;
3288 break;
3289 }
3290
3291 rc2 = SELMToFlatEx(pVM, DIS_SELREG_SS, CPUMCTX2CORE(pCtx), pCtx->esp & uMask, 0, &GCPtrStack);
3292 if (RT_FAILURE(rc2))
3293 {
3294 rc = VERR_EM_INTERPRETER;
3295 break;
3296 }
3297 rc2 = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &aIretFrame[0], sizeof(aIretFrame));
3298 if (RT_FAILURE(rc2))
3299 {
3300 rc = VERR_EM_INTERPRETER;
3301 break;
3302 }
3303 pCtx->ip = aIretFrame[0];
3304 pCtx->cs = aIretFrame[1];
3305 pCtx->csHid.u64Base = pCtx->cs << 4;
3306 pCtx->eflags.u = (pCtx->eflags.u & ~(X86_EFL_POPF_BITS & uMask)) | (aIretFrame[2] & X86_EFL_POPF_BITS & uMask);
3307 pCtx->sp += sizeof(aIretFrame);
3308
3309 LogFlow(("iret to %04x:%x\n", pCtx->cs, pCtx->ip));
3310 fUpdateRIP = false;
3311 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitIret);
3312 break;
3313 }
3314
3315 case OP_INT:
3316 {
3317 uint32_t intInfo2;
3318
3319 LogFlow(("Realmode: INT %x\n", pDis->param1.parval & 0xff));
3320 intInfo2 = pDis->param1.parval & 0xff;
3321 intInfo2 |= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
3322 intInfo2 |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
3323
3324 rc = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, intInfo2, cbOp, 0);
3325 AssertRC(VBOXSTRICTRC_VAL(rc));
3326 fUpdateRIP = false;
3327 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitInt);
3328 break;
3329 }
3330
3331 case OP_INTO:
3332 {
3333 if (pCtx->eflags.Bits.u1OF)
3334 {
3335 uint32_t intInfo2;
3336
3337 LogFlow(("Realmode: INTO\n"));
3338 intInfo2 = X86_XCPT_OF;
3339 intInfo2 |= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
3340 intInfo2 |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
3341
3342 rc = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, intInfo2, cbOp, 0);
3343 AssertRC(VBOXSTRICTRC_VAL(rc));
3344 fUpdateRIP = false;
3345 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitInt);
3346 }
3347 break;
3348 }
3349
3350 case OP_INT3:
3351 {
3352 uint32_t intInfo2;
3353
3354 LogFlow(("Realmode: INT 3\n"));
3355 intInfo2 = 3;
3356 intInfo2 |= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
3357 intInfo2 |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
3358
3359 rc = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, intInfo2, cbOp, 0);
3360 AssertRC(VBOXSTRICTRC_VAL(rc));
3361 fUpdateRIP = false;
3362 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitInt);
3363 break;
3364 }
3365
3366 default:
3367 rc = EMInterpretInstructionCPU(pVM, pVCpu, pDis, CPUMCTX2CORE(pCtx), 0, EMCODETYPE_SUPERVISOR, &cbSize);
3368 break;
3369 }
3370
3371 if (rc == VINF_SUCCESS)
3372 {
3373 if (fUpdateRIP)
3374 pCtx->rip += cbOp; /* Move on to the next instruction. */
3375
3376 /* lidt, lgdt can end up here. In the future crx changes as well. Just reload the whole context to be done with it. */
3377 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_ALL;
3378
3379 /* Only resume if successful. */
3380 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3381 goto ResumeExecution;
3382 }
3383 }
3384 else
3385 rc = VERR_EM_INTERPRETER;
3386
3387 AssertMsg(rc == VERR_EM_INTERPRETER || rc == VINF_PGM_CHANGE_MODE || rc == VINF_EM_HALT, ("Unexpected rc=%Rrc\n", VBOXSTRICTRC_VAL(rc)));
3388 break;
3389 }
3390
3391#ifdef VBOX_STRICT
3392 case X86_XCPT_XF: /* SIMD exception. */
3393 case X86_XCPT_DE: /* Divide error. */
3394 case X86_XCPT_UD: /* Unknown opcode exception. */
3395 case X86_XCPT_SS: /* Stack segment exception. */
3396 case X86_XCPT_NP: /* Segment not present exception. */
3397 {
3398 switch(vector)
3399 {
3400 case X86_XCPT_DE:
3401 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestDE);
3402 break;
3403 case X86_XCPT_UD:
3404 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestUD);
3405 break;
3406 case X86_XCPT_SS:
3407 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestSS);
3408 break;
3409 case X86_XCPT_NP:
3410 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestNP);
3411 break;
3412 }
3413
3414 Log(("Trap %x at %04X:%RGv\n", vector, pCtx->cs, (RTGCPTR)pCtx->rip));
3415 rc2 = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), cbInstr, errCode);
3416 AssertRC(rc2);
3417
3418 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3419 goto ResumeExecution;
3420 }
3421#endif
3422 default:
3423 if ( CPUMIsGuestInRealModeEx(pCtx)
3424 && pVM->hwaccm.s.vmx.pRealModeTSS)
3425 {
3426 Log(("Real Mode Trap %x at %04x:%04X error code %x\n", vector, pCtx->cs, pCtx->eip, errCode));
3427 rc = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), cbInstr, errCode);
3428 AssertRC(VBOXSTRICTRC_VAL(rc)); /* Strict RC check below. */
3429
3430 /* Go back to ring 3 in case of a triple fault. */
3431 if ( vector == X86_XCPT_DF
3432 && rc == VINF_EM_RESET)
3433 break;
3434
3435 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3436 goto ResumeExecution;
3437 }
3438 AssertMsgFailed(("Unexpected vm-exit caused by exception %x\n", vector));
3439 rc = VERR_VMX_UNEXPECTED_EXCEPTION;
3440 break;
3441 } /* switch (vector) */
3442
3443 break;
3444
3445 default:
3446 rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_CODE;
3447 AssertMsgFailed(("Unexpected interruption code %x\n", intInfo));
3448 break;
3449 }
3450
3451 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3452 break;
3453 }
3454
3455 case VMX_EXIT_EPT_VIOLATION: /* 48 EPT violation. An attempt to access memory with a guest-physical address was disallowed by the configuration of the EPT paging structures. */
3456 {
3457 RTGCPHYS GCPhys;
3458
3459 Assert(pVM->hwaccm.s.fNestedPaging);
3460
3461 rc2 = VMXReadVMCS64(VMX_VMCS_EXIT_PHYS_ADDR_FULL, &GCPhys);
3462 AssertRC(rc2);
3463 Assert(((exitQualification >> 7) & 3) != 2);
3464
3465 /* Determine the kind of violation. */
3466 errCode = 0;
3467 if (exitQualification & VMX_EXIT_QUALIFICATION_EPT_INSTR_FETCH)
3468 errCode |= X86_TRAP_PF_ID;
3469
3470 if (exitQualification & VMX_EXIT_QUALIFICATION_EPT_DATA_WRITE)
3471 errCode |= X86_TRAP_PF_RW;
3472
3473 /* If the page is present, then it's a page level protection fault. */
3474 if (exitQualification & VMX_EXIT_QUALIFICATION_EPT_ENTRY_PRESENT)
3475 {
3476 errCode |= X86_TRAP_PF_P;
3477 }
3478 else
3479 {
3480 /* Shortcut for APIC TPR reads and writes. */
3481 if ( (GCPhys & 0xfff) == 0x080
3482 && GCPhys > 0x1000000 /* to skip VGA frame buffer accesses */
3483 && fSetupTPRCaching
3484 && (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC))
3485 {
3486 RTGCPHYS GCPhysApicBase;
3487 PDMApicGetBase(pVM, &GCPhysApicBase); /* @todo cache this */
3488 GCPhysApicBase &= PAGE_BASE_GC_MASK;
3489 if (GCPhys == GCPhysApicBase + 0x80)
3490 {
3491 Log(("Enable VT-x virtual APIC access filtering\n"));
3492 rc2 = IOMMMIOMapMMIOHCPage(pVM, GCPhysApicBase, pVM->hwaccm.s.vmx.pAPICPhys, X86_PTE_RW | X86_PTE_P);
3493 AssertRC(rc2);
3494 }
3495 }
3496 }
3497 Log(("EPT Page fault %x at %RGp error code %x\n", (uint32_t)exitQualification, GCPhys, errCode));
3498
3499 /* GCPhys contains the guest physical address of the page fault. */
3500 TRPMAssertTrap(pVCpu, X86_XCPT_PF, TRPM_TRAP);
3501 TRPMSetErrorCode(pVCpu, errCode);
3502 TRPMSetFaultAddress(pVCpu, GCPhys);
3503
3504 /* Handle the pagefault trap for the nested shadow table. */
3505 rc = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, errCode, CPUMCTX2CORE(pCtx), GCPhys);
3506 Log2(("PGMR0Trap0eHandlerNestedPaging %RGv returned %Rrc\n", (RTGCPTR)pCtx->rip, VBOXSTRICTRC_VAL(rc)));
3507 if (rc == VINF_SUCCESS)
3508 { /* We've successfully synced our shadow pages, so let's just continue execution. */
3509 Log2(("Shadow page fault at %RGv cr2=%RGp error code %x\n", (RTGCPTR)pCtx->rip, exitQualification , errCode));
3510 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitReasonNPF);
3511
3512 TRPMResetTrap(pVCpu);
3513 goto ResumeExecution;
3514 }
3515
3516#ifdef VBOX_STRICT
3517 if (rc != VINF_EM_RAW_EMULATE_INSTR)
3518 LogFlow(("PGMTrap0eHandlerNestedPaging failed with %d\n", VBOXSTRICTRC_VAL(rc)));
3519#endif
3520 /* Need to go back to the recompiler to emulate the instruction. */
3521 TRPMResetTrap(pVCpu);
3522 break;
3523 }
3524
3525 case VMX_EXIT_EPT_MISCONFIG:
3526 {
3527 RTGCPHYS GCPhys;
3528
3529 Assert(pVM->hwaccm.s.fNestedPaging);
3530
3531 rc2 = VMXReadVMCS64(VMX_VMCS_EXIT_PHYS_ADDR_FULL, &GCPhys);
3532 AssertRC(rc2);
3533 Log(("VMX_EXIT_EPT_MISCONFIG for %RGp\n", GCPhys));
3534
3535 /* Shortcut for APIC TPR reads and writes. */
3536 if ( (GCPhys & 0xfff) == 0x080
3537 && GCPhys > 0x1000000 /* to skip VGA frame buffer accesses */
3538 && fSetupTPRCaching
3539 && (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC))
3540 {
3541 RTGCPHYS GCPhysApicBase;
3542 PDMApicGetBase(pVM, &GCPhysApicBase); /* @todo cache this */
3543 GCPhysApicBase &= PAGE_BASE_GC_MASK;
3544 if (GCPhys == GCPhysApicBase + 0x80)
3545 {
3546 Log(("Enable VT-x virtual APIC access filtering\n"));
3547 rc2 = IOMMMIOMapMMIOHCPage(pVM, GCPhysApicBase, pVM->hwaccm.s.vmx.pAPICPhys, X86_PTE_RW | X86_PTE_P);
3548 AssertRC(rc2);
3549 }
3550 }
3551
3552 rc = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pCtx), GCPhys, UINT32_MAX);
3553 if (rc == VINF_SUCCESS)
3554 {
3555 Log2(("PGMR0Trap0eHandlerNPMisconfig(,,,%RGp) at %RGv -> resume\n", GCPhys, (RTGCPTR)pCtx->rip));
3556 goto ResumeExecution;
3557 }
3558
3559 Log2(("PGMR0Trap0eHandlerNPMisconfig(,,,%RGp) at %RGv -> %Rrc\n", GCPhys, (RTGCPTR)pCtx->rip, VBOXSTRICTRC_VAL(rc)));
3560 break;
3561 }
3562
3563 case VMX_EXIT_IRQ_WINDOW: /* 7 Interrupt window. */
3564 /* Clear VM-exit on IF=1 change. */
3565 LogFlow(("VMX_EXIT_IRQ_WINDOW %RGv pending=%d IF=%d\n", (RTGCPTR)pCtx->rip, VMCPU_FF_ISPENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC|VMCPU_FF_INTERRUPT_PIC)), pCtx->eflags.Bits.u1IF));
3566 pVCpu->hwaccm.s.vmx.proc_ctls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_IRQ_WINDOW_EXIT;
3567 rc2 = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
3568 AssertRC(rc2);
3569 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitIrqWindow);
3570 goto ResumeExecution; /* we check for pending guest interrupts there */
3571
3572 case VMX_EXIT_WBINVD: /* 54 Guest software attempted to execute WBINVD. (conditional) */
3573 case VMX_EXIT_INVD: /* 13 Guest software attempted to execute INVD. (unconditional) */
3574 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitInvd);
3575 /* Skip instruction and continue directly. */
3576 pCtx->rip += cbInstr;
3577 /* Continue execution.*/
3578 goto ResumeExecution;
3579
3580 case VMX_EXIT_CPUID: /* 10 Guest software attempted to execute CPUID. */
3581 {
3582 Log2(("VMX: Cpuid %x\n", pCtx->eax));
3583 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitCpuid);
3584 rc = EMInterpretCpuId(pVM, pVCpu, CPUMCTX2CORE(pCtx));
3585 if (rc == VINF_SUCCESS)
3586 {
3587 /* Update EIP and continue execution. */
3588 Assert(cbInstr == 2);
3589 pCtx->rip += cbInstr;
3590 goto ResumeExecution;
3591 }
3592 AssertMsgFailed(("EMU: cpuid failed with %Rrc\n", VBOXSTRICTRC_VAL(rc)));
3593 rc = VINF_EM_RAW_EMULATE_INSTR;
3594 break;
3595 }
3596
3597 case VMX_EXIT_RDPMC: /* 15 Guest software attempted to execute RDPMC. */
3598 {
3599 Log2(("VMX: Rdpmc %x\n", pCtx->ecx));
3600 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitRdpmc);
3601 rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pCtx));
3602 if (rc == VINF_SUCCESS)
3603 {
3604 /* Update EIP and continue execution. */
3605 Assert(cbInstr == 2);
3606 pCtx->rip += cbInstr;
3607 goto ResumeExecution;
3608 }
3609 rc = VINF_EM_RAW_EMULATE_INSTR;
3610 break;
3611 }
3612
3613 case VMX_EXIT_RDTSC: /* 16 Guest software attempted to execute RDTSC. */
3614 {
3615 Log2(("VMX: Rdtsc\n"));
3616 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitRdtsc);
3617 rc = EMInterpretRdtsc(pVM, pVCpu, CPUMCTX2CORE(pCtx));
3618 if (rc == VINF_SUCCESS)
3619 {
3620 /* Update EIP and continue execution. */
3621 Assert(cbInstr == 2);
3622 pCtx->rip += cbInstr;
3623 goto ResumeExecution;
3624 }
3625 rc = VINF_EM_RAW_EMULATE_INSTR;
3626 break;
3627 }
3628
3629 case VMX_EXIT_INVPG: /* 14 Guest software attempted to execute INVPG. */
3630 {
3631 Log2(("VMX: invlpg\n"));
3632 Assert(!pVM->hwaccm.s.fNestedPaging);
3633
3634 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitInvpg);
3635 rc = EMInterpretInvlpg(pVM, pVCpu, CPUMCTX2CORE(pCtx), exitQualification);
3636 if (rc == VINF_SUCCESS)
3637 {
3638 /* Update EIP and continue execution. */
3639 pCtx->rip += cbInstr;
3640 goto ResumeExecution;
3641 }
3642 AssertMsg(rc == VERR_EM_INTERPRETER, ("EMU: invlpg %RGv failed with %Rrc\n", exitQualification, VBOXSTRICTRC_VAL(rc)));
3643 break;
3644 }
3645
3646 case VMX_EXIT_MONITOR: /* 39 Guest software attempted to execute MONITOR. */
3647 {
3648 Log2(("VMX: monitor\n"));
3649
3650 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitMonitor);
3651 rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pCtx));
3652 if (rc == VINF_SUCCESS)
3653 {
3654 /* Update EIP and continue execution. */
3655 pCtx->rip += cbInstr;
3656 goto ResumeExecution;
3657 }
3658 AssertMsg(rc == VERR_EM_INTERPRETER, ("EMU: monitor failed with %Rrc\n", VBOXSTRICTRC_VAL(rc)));
3659 break;
3660 }
3661
3662 case VMX_EXIT_WRMSR: /* 32 WRMSR. Guest software attempted to execute WRMSR. */
3663 /* When an interrupt is pending, we'll let MSR_K8_LSTAR writes fault in our TPR patch code. */
3664 if ( pVM->hwaccm.s.fTPRPatchingActive
3665 && pCtx->ecx == MSR_K8_LSTAR)
3666 {
3667 Assert(!CPUMIsGuestInLongModeEx(pCtx));
3668 if ((pCtx->eax & 0xff) != u8LastTPR)
3669 {
3670 Log(("VMX: Faulting MSR_K8_LSTAR write with new TPR value %x\n", pCtx->eax & 0xff));
3671
3672 /* Our patch code uses LSTAR for TPR caching. */
3673 rc2 = PDMApicSetTPR(pVCpu, pCtx->eax & 0xff);
3674 AssertRC(rc2);
3675 }
3676
3677 /* Skip the instruction and continue. */
3678 pCtx->rip += cbInstr; /* wrmsr = [0F 30] */
3679
3680 /* Only resume if successful. */
3681 goto ResumeExecution;
3682 }
3683 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_GUEST_MSR;
3684 /* no break */
3685 case VMX_EXIT_RDMSR: /* 31 RDMSR. Guest software attempted to execute RDMSR. */
3686 {
3687 uint32_t cbSize;
3688
3689 STAM_COUNTER_INC((exitReason == VMX_EXIT_RDMSR) ? &pVCpu->hwaccm.s.StatExitRdmsr : &pVCpu->hwaccm.s.StatExitWrmsr);
3690
3691 /* Note: the intel manual claims there's a REX version of RDMSR that's slightly different, so we play safe by completely disassembling the instruction. */
3692 Log2(("VMX: %s\n", (exitReason == VMX_EXIT_RDMSR) ? "rdmsr" : "wrmsr"));
3693 rc = EMInterpretInstruction(pVM, pVCpu, CPUMCTX2CORE(pCtx), 0, &cbSize);
3694 if (rc == VINF_SUCCESS)
3695 {
3696 /* EIP has been updated already. */
3697
3698 /* Only resume if successful. */
3699 goto ResumeExecution;
3700 }
3701 AssertMsg(rc == VERR_EM_INTERPRETER, ("EMU: %s failed with %Rrc\n", (exitReason == VMX_EXIT_RDMSR) ? "rdmsr" : "wrmsr", VBOXSTRICTRC_VAL(rc)));
3702 break;
3703 }
3704
3705 case VMX_EXIT_CRX_MOVE: /* 28 Control-register accesses. */
3706 {
3707 STAM_PROFILE_ADV_START(&pVCpu->hwaccm.s.StatExit2Sub2, y2);
3708
3709 switch (VMX_EXIT_QUALIFICATION_CRX_ACCESS(exitQualification))
3710 {
3711 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_WRITE:
3712 Log2(("VMX: %RGv mov cr%d, x\n", (RTGCPTR)pCtx->rip, VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification)));
3713 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitCRxWrite[VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification)]);
3714 rc = EMInterpretCRxWrite(pVM, pVCpu, CPUMCTX2CORE(pCtx),
3715 VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification),
3716 VMX_EXIT_QUALIFICATION_CRX_GENREG(exitQualification));
3717
3718 switch (VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification))
3719 {
3720 case 0:
3721 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_GUEST_CR0 | HWACCM_CHANGED_GUEST_CR3;
3722 break;
3723 case 2:
3724 break;
3725 case 3:
3726 Assert(!pVM->hwaccm.s.fNestedPaging || !CPUMIsGuestInPagedProtectedModeEx(pCtx));
3727 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_GUEST_CR3;
3728 break;
3729 case 4:
3730 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_GUEST_CR4;
3731 break;
3732 case 8:
3733 /* CR8 contains the APIC TPR */
3734 Assert(!(pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW));
3735 break;
3736
3737 default:
3738 AssertFailed();
3739 break;
3740 }
3741 break;
3742
3743 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ:
3744 Log2(("VMX: mov x, crx\n"));
3745 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitCRxRead[VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification)]);
3746
3747 Assert(!pVM->hwaccm.s.fNestedPaging || !CPUMIsGuestInPagedProtectedModeEx(pCtx) || VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification) != USE_REG_CR3);
3748
3749 /* CR8 reads only cause an exit when the TPR shadow feature isn't present. */
3750 Assert(VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification) != 8 || !(pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW));
3751
3752 rc = EMInterpretCRxRead(pVM, pVCpu, CPUMCTX2CORE(pCtx),
3753 VMX_EXIT_QUALIFICATION_CRX_GENREG(exitQualification),
3754 VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification));
3755 break;
3756
3757 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_CLTS:
3758 Log2(("VMX: clts\n"));
3759 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitCLTS);
3760 rc = EMInterpretCLTS(pVM, pVCpu);
3761 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_GUEST_CR0;
3762 break;
3763
3764 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_LMSW:
3765 Log2(("VMX: lmsw %x\n", VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(exitQualification)));
3766 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitLMSW);
3767 rc = EMInterpretLMSW(pVM, pVCpu, CPUMCTX2CORE(pCtx), VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(exitQualification));
3768 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_GUEST_CR0;
3769 break;
3770 }
3771
3772 /* Update EIP if no error occurred. */
3773 if (RT_SUCCESS(rc))
3774 pCtx->rip += cbInstr;
3775
3776 if (rc == VINF_SUCCESS)
3777 {
3778 /* Only resume if successful. */
3779 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub2, y2);
3780 goto ResumeExecution;
3781 }
3782 Assert(rc == VERR_EM_INTERPRETER || rc == VINF_PGM_CHANGE_MODE || rc == VINF_PGM_SYNC_CR3);
3783 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub2, y2);
3784 break;
3785 }
3786
3787 case VMX_EXIT_DRX_MOVE: /* 29 Debug-register accesses. */
3788 {
3789 if ( !DBGFIsStepping(pVCpu)
3790 && !CPUMIsHyperDebugStateActive(pVCpu))
3791 {
3792 /* Disable drx move intercepts. */
3793 pVCpu->hwaccm.s.vmx.proc_ctls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT;
3794 rc2 = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
3795 AssertRC(rc2);
3796
3797 /* Save the host and load the guest debug state. */
3798 rc2 = CPUMR0LoadGuestDebugState(pVM, pVCpu, pCtx, true /* include DR6 */);
3799 AssertRC(rc2);
3800
3801#ifdef LOG_ENABLED
3802 if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(exitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
3803 Log(("VMX_EXIT_DRX_MOVE: write DR%d genreg %d\n", VMX_EXIT_QUALIFICATION_DRX_REGISTER(exitQualification), VMX_EXIT_QUALIFICATION_DRX_GENREG(exitQualification)));
3804 else
3805 Log(("VMX_EXIT_DRX_MOVE: read DR%d\n", VMX_EXIT_QUALIFICATION_DRX_REGISTER(exitQualification)));
3806#endif
3807
3808#ifdef VBOX_WITH_STATISTICS
3809 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatDRxContextSwitch);
3810 if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(exitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
3811 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitDRxWrite);
3812 else
3813 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitDRxRead);
3814#endif
3815
3816 goto ResumeExecution;
3817 }
3818
3819 /** @todo clear VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT after the first time and restore drx registers afterwards */
3820 if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(exitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
3821 {
3822 Log2(("VMX: mov drx%d, genreg%d\n", VMX_EXIT_QUALIFICATION_DRX_REGISTER(exitQualification), VMX_EXIT_QUALIFICATION_DRX_GENREG(exitQualification)));
3823 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitDRxWrite);
3824 rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pCtx),
3825 VMX_EXIT_QUALIFICATION_DRX_REGISTER(exitQualification),
3826 VMX_EXIT_QUALIFICATION_DRX_GENREG(exitQualification));
3827 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_GUEST_DEBUG;
3828 Log2(("DR7=%08x\n", pCtx->dr[7]));
3829 }
3830 else
3831 {
3832 Log2(("VMX: mov x, drx\n"));
3833 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitDRxRead);
3834 rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pCtx),
3835 VMX_EXIT_QUALIFICATION_DRX_GENREG(exitQualification),
3836 VMX_EXIT_QUALIFICATION_DRX_REGISTER(exitQualification));
3837 }
3838 /* Update EIP if no error occurred. */
3839 if (RT_SUCCESS(rc))
3840 pCtx->rip += cbInstr;
3841
3842 if (rc == VINF_SUCCESS)
3843 {
3844 /* Only resume if successful. */
3845 goto ResumeExecution;
3846 }
3847 Assert(rc == VERR_EM_INTERPRETER);
3848 break;
3849 }
3850
3851 /* Note: We'll get a #GP if the IO instruction isn't allowed (IOPL or TSS bitmap); no need to double check. */
3852 case VMX_EXIT_PORT_IO: /* 30 I/O instruction. */
3853 {
3854 STAM_PROFILE_ADV_START(&pVCpu->hwaccm.s.StatExit2Sub1, y1);
3855 uint32_t uIOWidth = VMX_EXIT_QUALIFICATION_IO_WIDTH(exitQualification);
3856 uint32_t uPort;
3857 bool fIOWrite = (VMX_EXIT_QUALIFICATION_IO_DIRECTION(exitQualification) == VMX_EXIT_QUALIFICATION_IO_DIRECTION_OUT);
3858
3859 /** @todo necessary to make the distinction? */
3860 if (VMX_EXIT_QUALIFICATION_IO_ENCODING(exitQualification) == VMX_EXIT_QUALIFICATION_IO_ENCODING_DX)
3861 {
3862 uPort = pCtx->edx & 0xffff;
3863 }
3864 else
3865 uPort = VMX_EXIT_QUALIFICATION_IO_PORT(exitQualification); /* Immediate encoding. */
3866
3867 /* paranoia */
3868 if (RT_UNLIKELY(uIOWidth == 2 || uIOWidth >= 4))
3869 {
3870 rc = fIOWrite ? VINF_IOM_HC_IOPORT_WRITE : VINF_IOM_HC_IOPORT_READ;
3871 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub1, y1);
3872 break;
3873 }
3874
3875 uint32_t cbSize = g_aIOSize[uIOWidth];
3876
3877 if (VMX_EXIT_QUALIFICATION_IO_STRING(exitQualification))
3878 {
3879 /* ins/outs */
3880 PDISCPUSTATE pDis = &pVCpu->hwaccm.s.DisState;
3881
3882 /* Disassemble manually to deal with segment prefixes. */
3883 /** @todo VMX_VMCS_EXIT_GUEST_LINEAR_ADDR contains the flat pointer operand of the instruction. */
3884 /** @todo VMX_VMCS32_RO_EXIT_INSTR_INFO also contains segment prefix info. */
3885 rc2 = EMInterpretDisasOne(pVM, pVCpu, CPUMCTX2CORE(pCtx), pDis, NULL);
3886 if (RT_SUCCESS(rc))
3887 {
3888 if (fIOWrite)
3889 {
3890 Log2(("IOMInterpretOUTSEx %RGv %x size=%d\n", (RTGCPTR)pCtx->rip, uPort, cbSize));
3891 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitIOStringWrite);
3892 rc = IOMInterpretOUTSEx(pVM, CPUMCTX2CORE(pCtx), uPort, pDis->prefix, pDis->addrmode, cbSize);
3893 }
3894 else
3895 {
3896 Log2(("IOMInterpretINSEx %RGv %x size=%d\n", (RTGCPTR)pCtx->rip, uPort, cbSize));
3897 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitIOStringRead);
3898 rc = IOMInterpretINSEx(pVM, CPUMCTX2CORE(pCtx), uPort, pDis->prefix, pDis->addrmode, cbSize);
3899 }
3900 }
3901 else
3902 rc = VINF_EM_RAW_EMULATE_INSTR;
3903 }
3904 else
3905 {
3906 /* normal in/out */
3907 uint32_t uAndVal = g_aIOOpAnd[uIOWidth];
3908
3909 Assert(!VMX_EXIT_QUALIFICATION_IO_REP(exitQualification));
3910
3911 if (fIOWrite)
3912 {
3913 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitIOWrite);
3914 rc = IOMIOPortWrite(pVM, uPort, pCtx->eax & uAndVal, cbSize);
3915 if (rc == VINF_IOM_HC_IOPORT_WRITE)
3916 HWACCMR0SavePendingIOPortWrite(pVCpu, pCtx->rip, pCtx->rip + cbInstr, uPort, uAndVal, cbSize);
3917 }
3918 else
3919 {
3920 uint32_t u32Val = 0;
3921
3922 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitIORead);
3923 rc = IOMIOPortRead(pVM, uPort, &u32Val, cbSize);
3924 if (IOM_SUCCESS(rc))
3925 {
3926 /* Write back to the EAX register. */
3927 pCtx->eax = (pCtx->eax & ~uAndVal) | (u32Val & uAndVal);
3928 }
3929 else
3930 if (rc == VINF_IOM_HC_IOPORT_READ)
3931 HWACCMR0SavePendingIOPortRead(pVCpu, pCtx->rip, pCtx->rip + cbInstr, uPort, uAndVal, cbSize);
3932 }
3933 }
3934 /*
3935 * Handled the I/O return codes.
3936 * (The unhandled cases end up with rc == VINF_EM_RAW_EMULATE_INSTR.)
3937 */
3938 if (IOM_SUCCESS(rc))
3939 {
3940 /* Update EIP and continue execution. */
3941 pCtx->rip += cbInstr;
3942 if (RT_LIKELY(rc == VINF_SUCCESS))
3943 {
3944 /* If any IO breakpoints are armed, then we should check if a debug trap needs to be generated. */
3945 if (pCtx->dr[7] & X86_DR7_ENABLED_MASK)
3946 {
3947 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatDRxIOCheck);
3948 for (unsigned i=0;i<4;i++)
3949 {
3950 unsigned uBPLen = g_aIOSize[X86_DR7_GET_LEN(pCtx->dr[7], i)];
3951
3952 if ( (uPort >= pCtx->dr[i] && uPort < pCtx->dr[i] + uBPLen)
3953 && (pCtx->dr[7] & (X86_DR7_L(i) | X86_DR7_G(i)))
3954 && (pCtx->dr[7] & X86_DR7_RW(i, X86_DR7_RW_IO)) == X86_DR7_RW(i, X86_DR7_RW_IO))
3955 {
3956 uint64_t uDR6;
3957
3958 Assert(CPUMIsGuestDebugStateActive(pVCpu));
3959
3960 uDR6 = ASMGetDR6();
3961
3962 /* Clear all breakpoint status flags and set the one we just hit. */
3963 uDR6 &= ~(X86_DR6_B0|X86_DR6_B1|X86_DR6_B2|X86_DR6_B3);
3964 uDR6 |= (uint64_t)RT_BIT(i);
3965
3966 /* Note: AMD64 Architecture Programmer's Manual 13.1:
3967 * Bits 15:13 of the DR6 register is never cleared by the processor and must be cleared by software after
3968 * the contents have been read.
3969 */
3970 ASMSetDR6(uDR6);
3971
3972 /* X86_DR7_GD will be cleared if drx accesses should be trapped inside the guest. */
3973 pCtx->dr[7] &= ~X86_DR7_GD;
3974
3975 /* Paranoia. */
3976 pCtx->dr[7] &= 0xffffffff; /* upper 32 bits reserved */
3977 pCtx->dr[7] &= ~(RT_BIT(11) | RT_BIT(12) | RT_BIT(14) | RT_BIT(15)); /* must be zero */
3978 pCtx->dr[7] |= 0x400; /* must be one */
3979
3980 /* Resync DR7 */
3981 rc2 = VMXWriteVMCS64(VMX_VMCS64_GUEST_DR7, pCtx->dr[7]);
3982 AssertRC(rc2);
3983
3984 /* Construct inject info. */
3985 intInfo = X86_XCPT_DB;
3986 intInfo |= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
3987 intInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HWEXCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
3988
3989 Log(("Inject IO debug trap at %RGv\n", (RTGCPTR)pCtx->rip));
3990 rc2 = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), 0, 0);
3991 AssertRC(rc2);
3992
3993 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub1, y1);
3994 goto ResumeExecution;
3995 }
3996 }
3997 }
3998 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub1, y1);
3999 goto ResumeExecution;
4000 }
4001 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub1, y1);
4002 break;
4003 }
4004
4005#ifdef VBOX_STRICT
4006 if (rc == VINF_IOM_HC_IOPORT_READ)
4007 Assert(!fIOWrite);
4008 else if (rc == VINF_IOM_HC_IOPORT_WRITE)
4009 Assert(fIOWrite);
4010 else
4011 AssertMsg(RT_FAILURE(rc) || rc == VINF_EM_RAW_EMULATE_INSTR || rc == VINF_EM_RAW_GUEST_TRAP || rc == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", VBOXSTRICTRC_VAL(rc)));
4012#endif
4013 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub1, y1);
4014 break;
4015 }
4016
4017 case VMX_EXIT_TPR: /* 43 TPR below threshold. Guest software executed MOV to CR8. */
4018 LogFlow(("VMX_EXIT_TPR\n"));
4019 /* RIP is already set to the next instruction and the TPR has been synced back. Just resume. */
4020 goto ResumeExecution;
4021
4022 case VMX_EXIT_APIC_ACCESS: /* 44 APIC access. Guest software attempted to access memory at a physical address on the APIC-access page. */
4023 {
4024 LogFlow(("VMX_EXIT_APIC_ACCESS\n"));
4025 unsigned uAccessType = VMX_EXIT_QUALIFICATION_APIC_ACCESS_TYPE(exitQualification);
4026
4027 switch(uAccessType)
4028 {
4029 case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
4030 case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
4031 {
4032 RTGCPHYS GCPhys;
4033 PDMApicGetBase(pVM, &GCPhys);
4034 GCPhys &= PAGE_BASE_GC_MASK;
4035 GCPhys += VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(exitQualification);
4036
4037 LogFlow(("Apic access at %RGp\n", GCPhys));
4038 rc = IOMMMIOPhysHandler(pVM, (uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ) ? 0 : X86_TRAP_PF_RW, CPUMCTX2CORE(pCtx), GCPhys);
4039 if (rc == VINF_SUCCESS)
4040 goto ResumeExecution; /* rip already updated */
4041 break;
4042 }
4043
4044 default:
4045 rc = VINF_EM_RAW_EMULATE_INSTR;
4046 break;
4047 }
4048 break;
4049 }
4050
4051 case VMX_EXIT_PREEMPTION_TIMER: /* 52 VMX-preemption timer expired. The preemption timer counted down to zero. */
4052 if (!TMTimerPollBool(pVM, pVCpu))
4053 goto ResumeExecution;
4054 rc = VINF_EM_RAW_TIMER_PENDING;
4055 break;
4056
4057 default:
4058 /* The rest is handled after syncing the entire CPU state. */
4059 break;
4060 }
4061
4062 /* Note: the guest state isn't entirely synced back at this stage. */
4063
4064 /* Investigate why there was a VM-exit. (part 2) */
4065 switch (exitReason)
4066 {
4067 case VMX_EXIT_EXCEPTION: /* 0 Exception or non-maskable interrupt (NMI). */
4068 case VMX_EXIT_EXTERNAL_IRQ: /* 1 External interrupt. */
4069 case VMX_EXIT_EPT_VIOLATION:
4070 case VMX_EXIT_EPT_MISCONFIG: /* 49 EPT misconfig is used by the PGM/MMIO optimizations. */
4071 case VMX_EXIT_PREEMPTION_TIMER: /* 52 VMX-preemption timer expired. The preemption timer counted down to zero. */
4072 /* Already handled above. */
4073 break;
4074
4075 case VMX_EXIT_TRIPLE_FAULT: /* 2 Triple fault. */
4076 rc = VINF_EM_RESET; /* Triple fault equals a reset. */
4077 break;
4078
4079 case VMX_EXIT_INIT_SIGNAL: /* 3 INIT signal. */
4080 case VMX_EXIT_SIPI: /* 4 Start-up IPI (SIPI). */
4081 rc = VINF_EM_RAW_INTERRUPT;
4082 AssertFailed(); /* Can't happen. Yet. */
4083 break;
4084
4085 case VMX_EXIT_IO_SMI_IRQ: /* 5 I/O system-management interrupt (SMI). */
4086 case VMX_EXIT_SMI_IRQ: /* 6 Other SMI. */
4087 rc = VINF_EM_RAW_INTERRUPT;
4088 AssertFailed(); /* Can't happen afaik. */
4089 break;
4090
4091 case VMX_EXIT_TASK_SWITCH: /* 9 Task switch: too complicated to emulate, so fall back to the recompiler */
4092 Log(("VMX_EXIT_TASK_SWITCH: exit=%RX64\n", exitQualification));
4093 if ( (VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE(exitQualification) == VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE_IDT)
4094 && pVCpu->hwaccm.s.Event.fPending)
4095 {
4096 /* Caused by an injected interrupt. */
4097 pVCpu->hwaccm.s.Event.fPending = false;
4098
4099 Log(("VMX_EXIT_TASK_SWITCH: reassert trap %d\n", VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVCpu->hwaccm.s.Event.intInfo)));
4100 Assert(!VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(pVCpu->hwaccm.s.Event.intInfo));
4101 rc2 = TRPMAssertTrap(pVCpu, VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVCpu->hwaccm.s.Event.intInfo), TRPM_HARDWARE_INT);
4102 AssertRC(rc2);
4103 }
4104 /* else Exceptions and software interrupts can just be restarted. */
4105 rc = VERR_EM_INTERPRETER;
4106 break;
4107
4108 case VMX_EXIT_HLT: /* 12 Guest software attempted to execute HLT. */
4109 /** Check if external interrupts are pending; if so, don't switch back. */
4110 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitHlt);
4111 pCtx->rip++; /* skip hlt */
4112 if (EMShouldContinueAfterHalt(pVCpu, pCtx))
4113 goto ResumeExecution;
4114
4115 rc = VINF_EM_HALT;
4116 break;
4117
4118 case VMX_EXIT_MWAIT: /* 36 Guest software executed MWAIT. */
4119 Log2(("VMX: mwait\n"));
4120 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitMwait);
4121 rc = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pCtx));
4122 if ( rc == VINF_EM_HALT
4123 || rc == VINF_SUCCESS)
4124 {
4125 /* Update EIP and continue execution. */
4126 pCtx->rip += cbInstr;
4127
4128 /** Check if external interrupts are pending; if so, don't switch back. */
4129 if ( rc == VINF_SUCCESS
4130 || ( rc == VINF_EM_HALT
4131 && EMShouldContinueAfterHalt(pVCpu, pCtx))
4132 )
4133 goto ResumeExecution;
4134 }
4135 AssertMsg(rc == VERR_EM_INTERPRETER || rc == VINF_EM_HALT, ("EMU: mwait failed with %Rrc\n", VBOXSTRICTRC_VAL(rc)));
4136 break;
4137
4138 case VMX_EXIT_RSM: /* 17 Guest software attempted to execute RSM in SMM. */
4139 AssertFailed(); /* can't happen. */
4140 rc = VERR_EM_INTERPRETER;
4141 break;
4142
4143 case VMX_EXIT_VMCALL: /* 18 Guest software executed VMCALL. */
4144 case VMX_EXIT_VMCLEAR: /* 19 Guest software executed VMCLEAR. */
4145 case VMX_EXIT_VMLAUNCH: /* 20 Guest software executed VMLAUNCH. */
4146 case VMX_EXIT_VMPTRLD: /* 21 Guest software executed VMPTRLD. */
4147 case VMX_EXIT_VMPTRST: /* 22 Guest software executed VMPTRST. */
4148 case VMX_EXIT_VMREAD: /* 23 Guest software executed VMREAD. */
4149 case VMX_EXIT_VMRESUME: /* 24 Guest software executed VMRESUME. */
4150 case VMX_EXIT_VMWRITE: /* 25 Guest software executed VMWRITE. */
4151 case VMX_EXIT_VMXOFF: /* 26 Guest software executed VMXOFF. */
4152 case VMX_EXIT_VMXON: /* 27 Guest software executed VMXON. */
4153 /** @todo inject #UD immediately */
4154 rc = VERR_EM_INTERPRETER;
4155 break;
4156
4157 case VMX_EXIT_CPUID: /* 10 Guest software attempted to execute CPUID. */
4158 case VMX_EXIT_RDTSC: /* 16 Guest software attempted to execute RDTSC. */
4159 case VMX_EXIT_INVPG: /* 14 Guest software attempted to execute INVPG. */
4160 case VMX_EXIT_CRX_MOVE: /* 28 Control-register accesses. */
4161 case VMX_EXIT_DRX_MOVE: /* 29 Debug-register accesses. */
4162 case VMX_EXIT_PORT_IO: /* 30 I/O instruction. */
4163 case VMX_EXIT_RDPMC: /* 15 Guest software attempted to execute RDPMC. */
4164 /* already handled above */
4165 AssertMsg( rc == VINF_PGM_CHANGE_MODE
4166 || rc == VINF_EM_RAW_INTERRUPT
4167 || rc == VERR_EM_INTERPRETER
4168 || rc == VINF_EM_RAW_EMULATE_INSTR
4169 || rc == VINF_PGM_SYNC_CR3
4170 || rc == VINF_IOM_HC_IOPORT_READ
4171 || rc == VINF_IOM_HC_IOPORT_WRITE
4172 || rc == VINF_EM_RAW_GUEST_TRAP
4173 || rc == VINF_TRPM_XCPT_DISPATCHED
4174 || rc == VINF_EM_RESCHEDULE_REM,
4175 ("rc = %d\n", VBOXSTRICTRC_VAL(rc)));
4176 break;
4177
4178 case VMX_EXIT_TPR: /* 43 TPR below threshold. Guest software executed MOV to CR8. */
4179 case VMX_EXIT_APIC_ACCESS: /* 44 APIC access. Guest software attempted to access memory at a physical address on the APIC-access page. */
4180 case VMX_EXIT_RDMSR: /* 31 RDMSR. Guest software attempted to execute RDMSR. */
4181 case VMX_EXIT_WRMSR: /* 32 WRMSR. Guest software attempted to execute WRMSR. */
4182 case VMX_EXIT_PAUSE: /* 40 Guest software attempted to execute PAUSE. */
4183 case VMX_EXIT_MONITOR: /* 39 Guest software attempted to execute MONITOR. */
4184 /* Note: If we decide to emulate them here, then we must sync the MSRs that could have been changed (sysenter, fs/gs base)!!! */
4185 rc = VERR_EM_INTERPRETER;
4186 break;
4187
4188 case VMX_EXIT_IRQ_WINDOW: /* 7 Interrupt window. */
4189 Assert(rc == VINF_EM_RAW_INTERRUPT);
4190 break;
4191
4192 case VMX_EXIT_ERR_INVALID_GUEST_STATE: /* 33 VM-entry failure due to invalid guest state. */
4193 {
4194#ifdef VBOX_STRICT
4195 RTCCUINTREG val2 = 0;
4196
4197 Log(("VMX_EXIT_ERR_INVALID_GUEST_STATE\n"));
4198
4199 VMXReadVMCS(VMX_VMCS64_GUEST_RIP, &val2);
4200 Log(("Old eip %RGv new %RGv\n", (RTGCPTR)pCtx->rip, (RTGCPTR)val2));
4201
4202 VMXReadVMCS(VMX_VMCS64_GUEST_CR0, &val2);
4203 Log(("VMX_VMCS_GUEST_CR0 %RX64\n", (uint64_t)val2));
4204
4205 VMXReadVMCS(VMX_VMCS64_GUEST_CR3, &val2);
4206 Log(("VMX_VMCS_GUEST_CR3 %RX64\n", (uint64_t)val2));
4207
4208 VMXReadVMCS(VMX_VMCS64_GUEST_CR4, &val2);
4209 Log(("VMX_VMCS_GUEST_CR4 %RX64\n", (uint64_t)val2));
4210
4211 VMXReadVMCS(VMX_VMCS_GUEST_RFLAGS, &val2);
4212 Log(("VMX_VMCS_GUEST_RFLAGS %08x\n", val2));
4213
4214 VMX_LOG_SELREG(CS, "CS", val2);
4215 VMX_LOG_SELREG(DS, "DS", val2);
4216 VMX_LOG_SELREG(ES, "ES", val2);
4217 VMX_LOG_SELREG(FS, "FS", val2);
4218 VMX_LOG_SELREG(GS, "GS", val2);
4219 VMX_LOG_SELREG(SS, "SS", val2);
4220 VMX_LOG_SELREG(TR, "TR", val2);
4221 VMX_LOG_SELREG(LDTR, "LDTR", val2);
4222
4223 VMXReadVMCS(VMX_VMCS64_GUEST_GDTR_BASE, &val2);
4224 Log(("VMX_VMCS_GUEST_GDTR_BASE %RX64\n", (uint64_t)val2));
4225 VMXReadVMCS(VMX_VMCS64_GUEST_IDTR_BASE, &val2);
4226 Log(("VMX_VMCS_GUEST_IDTR_BASE %RX64\n", (uint64_t)val2));
4227#endif /* VBOX_STRICT */
4228 rc = VERR_VMX_INVALID_GUEST_STATE;
4229 break;
4230 }
4231
4232 case VMX_EXIT_ERR_MSR_LOAD: /* 34 VM-entry failure due to MSR loading. */
4233 case VMX_EXIT_ERR_MACHINE_CHECK: /* 41 VM-entry failure due to machine-check. */
4234 default:
4235 rc = VERR_VMX_UNEXPECTED_EXIT_CODE;
4236 AssertMsgFailed(("Unexpected exit code %d\n", exitReason)); /* Can't happen. */
4237 break;
4238
4239 }
4240end:
4241
4242 /* We now going back to ring-3, so clear the action flag. */
4243 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
4244
4245 /* Signal changes for the recompiler. */
4246 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR | CPUM_CHANGED_LDTR | CPUM_CHANGED_GDTR | CPUM_CHANGED_IDTR | CPUM_CHANGED_TR | CPUM_CHANGED_HIDDEN_SEL_REGS);
4247
4248 /* If we executed vmlaunch/vmresume and an external irq was pending, then we don't have to do a full sync the next time. */
4249 if ( exitReason == VMX_EXIT_EXTERNAL_IRQ
4250 && !VMX_EXIT_INTERRUPTION_INFO_VALID(intInfo))
4251 {
4252 STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatPendingHostIrq);
4253 /* On the next entry we'll only sync the host context. */
4254 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_HOST_CONTEXT;
4255 }
4256 else
4257 {
4258 /* On the next entry we'll sync everything. */
4259 /** @todo we can do better than this */
4260 /* Not in the VINF_PGM_CHANGE_MODE though! */
4261 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_ALL;
4262 }
4263
4264 /* translate into a less severe return code */
4265 if (rc == VERR_EM_INTERPRETER)
4266 rc = VINF_EM_RAW_EMULATE_INSTR;
4267 else
4268 /* Try to extract more information about what might have gone wrong here. */
4269 if (rc == VERR_VMX_INVALID_VMCS_PTR)
4270 {
4271 VMXGetActivateVMCS(&pVCpu->hwaccm.s.vmx.lasterror.u64VMCSPhys);
4272 pVCpu->hwaccm.s.vmx.lasterror.ulVMCSRevision = *(uint32_t *)pVCpu->hwaccm.s.vmx.pvVMCS;
4273 pVCpu->hwaccm.s.vmx.lasterror.idEnteredCpu = pVCpu->hwaccm.s.idEnteredCpu;
4274 pVCpu->hwaccm.s.vmx.lasterror.idCurrentCpu = RTMpCpuId();
4275 }
4276
4277 /* Just set the correct state here instead of trying to catch every goto above. */
4278 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED, VMCPUSTATE_STARTED_EXEC);
4279
4280#ifdef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
4281 /* Restore interrupts if we exitted after disabling them. */
4282 if (uOldEFlags != ~(RTCCUINTREG)0)
4283 ASMSetFlags(uOldEFlags);
4284#endif
4285
4286 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2, x);
4287 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit1, x);
4288 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatEntry, x);
4289 Log2(("X"));
4290 return VBOXSTRICTRC_TODO(rc);
4291}
4292
4293
4294/**
4295 * Enters the VT-x session
4296 *
4297 * @returns VBox status code.
4298 * @param pVM The VM to operate on.
4299 * @param pVCpu The VMCPU to operate on.
4300 * @param pCpu CPU info struct
4301 */
4302VMMR0DECL(int) VMXR0Enter(PVM pVM, PVMCPU pVCpu, PHMGLOBLCPUINFO pCpu)
4303{
4304 Assert(pVM->hwaccm.s.vmx.fSupported);
4305 NOREF(pCpu);
4306
4307 unsigned cr4 = ASMGetCR4();
4308 if (!(cr4 & X86_CR4_VMXE))
4309 {
4310 AssertMsgFailed(("X86_CR4_VMXE should be set!\n"));
4311 return VERR_VMX_X86_CR4_VMXE_CLEARED;
4312 }
4313
4314 /* Activate the VM Control Structure. */
4315 int rc = VMXActivateVMCS(pVCpu->hwaccm.s.vmx.HCPhysVMCS);
4316 if (RT_FAILURE(rc))
4317 return rc;
4318
4319 pVCpu->hwaccm.s.fResumeVM = false;
4320 return VINF_SUCCESS;
4321}
4322
4323
4324/**
4325 * Leaves the VT-x session
4326 *
4327 * @returns VBox status code.
4328 * @param pVM The VM to operate on.
4329 * @param pVCpu The VMCPU to operate on.
4330 * @param pCtx CPU context
4331 */
4332VMMR0DECL(int) VMXR0Leave(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
4333{
4334 Assert(pVM->hwaccm.s.vmx.fSupported);
4335
4336#ifdef DEBUG
4337 if (CPUMIsHyperDebugStateActive(pVCpu))
4338 {
4339 CPUMR0LoadHostDebugState(pVM, pVCpu);
4340 Assert(pVCpu->hwaccm.s.vmx.proc_ctls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT);
4341 }
4342 else
4343#endif
4344 /* Save the guest debug state if necessary. */
4345 if (CPUMIsGuestDebugStateActive(pVCpu))
4346 {
4347 CPUMR0SaveGuestDebugState(pVM, pVCpu, pCtx, true /* save DR6 */);
4348
4349 /* Enable drx move intercepts again. */
4350 pVCpu->hwaccm.s.vmx.proc_ctls |= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT;
4351 int rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
4352 AssertRC(rc);
4353
4354 /* Resync the debug registers the next time. */
4355 pVCpu->hwaccm.s.fContextUseFlags |= HWACCM_CHANGED_GUEST_DEBUG;
4356 }
4357 else
4358 Assert(pVCpu->hwaccm.s.vmx.proc_ctls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT);
4359
4360 /* Clear VM Control Structure. Marking it inactive, clearing implementation specific data and writing back VMCS data to memory. */
4361 int rc = VMXClearVMCS(pVCpu->hwaccm.s.vmx.HCPhysVMCS);
4362 AssertRC(rc);
4363
4364 return VINF_SUCCESS;
4365}
4366
4367/**
4368 * Flush the TLB (EPT)
4369 *
4370 * @returns VBox status code.
4371 * @param pVM The VM to operate on.
4372 * @param pVCpu The VM CPU to operate on.
4373 * @param enmFlush Type of flush
4374 * @param GCPhys Physical address of the page to flush
4375 */
4376static void hmR0VmxFlushEPT(PVM pVM, PVMCPU pVCpu, VMX_FLUSH enmFlush, RTGCPHYS GCPhys)
4377{
4378 uint64_t descriptor[2];
4379
4380 LogFlow(("hmR0VmxFlushEPT %d %RGv\n", enmFlush, GCPhys));
4381 Assert(pVM->hwaccm.s.fNestedPaging);
4382 descriptor[0] = pVCpu->hwaccm.s.vmx.GCPhysEPTP;
4383 descriptor[1] = GCPhys;
4384 int rc = VMXR0InvEPT(enmFlush, &descriptor[0]);
4385 AssertRC(rc);
4386}
4387
4388#ifdef HWACCM_VTX_WITH_VPID
4389/**
4390 * Flush the TLB (EPT)
4391 *
4392 * @returns VBox status code.
4393 * @param pVM The VM to operate on.
4394 * @param pVCpu The VM CPU to operate on.
4395 * @param enmFlush Type of flush
4396 * @param GCPtr Virtual address of the page to flush
4397 */
4398static void hmR0VmxFlushVPID(PVM pVM, PVMCPU pVCpu, VMX_FLUSH enmFlush, RTGCPTR GCPtr)
4399{
4400#if HC_ARCH_BITS == 32
4401 /* If we get a flush in 64 bits guest mode, then force a full TLB flush. Invvpid probably takes only 32 bits addresses. (@todo) */
4402 if ( CPUMIsGuestInLongMode(pVCpu)
4403 && !VMX_IS_64BIT_HOST_MODE())
4404 {
4405 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
4406 }
4407 else
4408#endif
4409 {
4410 uint64_t descriptor[2];
4411
4412 Assert(pVM->hwaccm.s.vmx.fVPID);
4413 descriptor[0] = pVCpu->hwaccm.s.uCurrentASID;
4414 descriptor[1] = GCPtr;
4415 int rc = VMXR0InvVPID(enmFlush, &descriptor[0]); NOREF(rc);
4416 AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvVPID %x %x %RGv failed with %d\n", enmFlush, pVCpu->hwaccm.s.uCurrentASID, GCPtr, rc));
4417 }
4418}
4419#endif /* HWACCM_VTX_WITH_VPID */
4420
4421/**
4422 * Invalidates a guest page
4423 *
4424 * @returns VBox status code.
4425 * @param pVM The VM to operate on.
4426 * @param pVCpu The VM CPU to operate on.
4427 * @param GCVirt Page to invalidate
4428 */
4429VMMR0DECL(int) VMXR0InvalidatePage(PVM pVM, PVMCPU pVCpu, RTGCPTR GCVirt)
4430{
4431 bool fFlushPending = VMCPU_FF_ISSET(pVCpu, VMCPU_FF_TLB_FLUSH);
4432
4433 Log2(("VMXR0InvalidatePage %RGv\n", GCVirt));
4434
4435 /* Only relevant if we want to use VPID.
4436 * In the nested paging case we still see such calls, but
4437 * can safely ignore them. (e.g. after cr3 updates)
4438 */
4439#ifdef HWACCM_VTX_WITH_VPID
4440 /* Skip it if a TLB flush is already pending. */
4441 if ( !fFlushPending
4442 && pVM->hwaccm.s.vmx.fVPID)
4443 hmR0VmxFlushVPID(pVM, pVCpu, pVM->hwaccm.s.vmx.enmFlushPage, GCVirt);
4444#endif /* HWACCM_VTX_WITH_VPID */
4445
4446 return VINF_SUCCESS;
4447}
4448
4449/**
4450 * Invalidates a guest page by physical address
4451 *
4452 * NOTE: Assumes the current instruction references this physical page though a virtual address!!
4453 *
4454 * @returns VBox status code.
4455 * @param pVM The VM to operate on.
4456 * @param pVCpu The VM CPU to operate on.
4457 * @param GCPhys Page to invalidate
4458 */
4459VMMR0DECL(int) VMXR0InvalidatePhysPage(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys)
4460{
4461 bool fFlushPending = VMCPU_FF_ISSET(pVCpu, VMCPU_FF_TLB_FLUSH);
4462
4463 Assert(pVM->hwaccm.s.fNestedPaging);
4464
4465 LogFlow(("VMXR0InvalidatePhysPage %RGp\n", GCPhys));
4466
4467 /* Skip it if a TLB flush is already pending. */
4468 if (!fFlushPending)
4469 hmR0VmxFlushEPT(pVM, pVCpu, pVM->hwaccm.s.vmx.enmFlushPage, GCPhys);
4470
4471 return VINF_SUCCESS;
4472}
4473
4474/**
4475 * Report world switch error and dump some useful debug info
4476 *
4477 * @param pVM The VM to operate on.
4478 * @param pVCpu The VMCPU to operate on.
4479 * @param rc Return code
4480 * @param pCtx Current CPU context (not updated)
4481 */
4482static void hmR0VmxReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, VBOXSTRICTRC rc, PCPUMCTX pCtx)
4483{
4484 NOREF(pVM);
4485
4486 switch (VBOXSTRICTRC_VAL(rc))
4487 {
4488 case VERR_VMX_INVALID_VMXON_PTR:
4489 AssertFailed();
4490 break;
4491
4492 case VERR_VMX_UNABLE_TO_START_VM:
4493 case VERR_VMX_UNABLE_TO_RESUME_VM:
4494 {
4495 int rc2;
4496 RTCCUINTREG exitReason, instrError;
4497
4498 rc2 = VMXReadVMCS(VMX_VMCS32_RO_EXIT_REASON, &exitReason);
4499 rc2 |= VMXReadVMCS(VMX_VMCS32_RO_VM_INSTR_ERROR, &instrError);
4500 AssertRC(rc2);
4501 if (rc2 == VINF_SUCCESS)
4502 {
4503 Log(("Unable to start/resume VM for reason: %x. Instruction error %x\n", (uint32_t)exitReason, (uint32_t)instrError));
4504 Log(("Current stack %08x\n", &rc2));
4505
4506 pVCpu->hwaccm.s.vmx.lasterror.ulInstrError = instrError;
4507 pVCpu->hwaccm.s.vmx.lasterror.ulExitReason = exitReason;
4508
4509#ifdef VBOX_STRICT
4510 RTGDTR gdtr;
4511 PCX86DESCHC pDesc;
4512 RTCCUINTREG val;
4513
4514 ASMGetGDTR(&gdtr);
4515
4516 VMXReadVMCS(VMX_VMCS64_GUEST_RIP, &val);
4517 Log(("Old eip %RGv new %RGv\n", (RTGCPTR)pCtx->rip, (RTGCPTR)val));
4518 VMXReadVMCS(VMX_VMCS_CTRL_PIN_EXEC_CONTROLS, &val);
4519 Log(("VMX_VMCS_CTRL_PIN_EXEC_CONTROLS %08x\n", val));
4520 VMXReadVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, &val);
4521 Log(("VMX_VMCS_CTRL_PROC_EXEC_CONTROLS %08x\n", val));
4522 VMXReadVMCS(VMX_VMCS_CTRL_ENTRY_CONTROLS, &val);
4523 Log(("VMX_VMCS_CTRL_ENTRY_CONTROLS %08x\n", val));
4524 VMXReadVMCS(VMX_VMCS_CTRL_EXIT_CONTROLS, &val);
4525 Log(("VMX_VMCS_CTRL_EXIT_CONTROLS %08x\n", val));
4526
4527 VMXReadVMCS(VMX_VMCS_HOST_CR0, &val);
4528 Log(("VMX_VMCS_HOST_CR0 %08x\n", val));
4529
4530 VMXReadVMCS(VMX_VMCS_HOST_CR3, &val);
4531 Log(("VMX_VMCS_HOST_CR3 %08x\n", val));
4532
4533 VMXReadVMCS(VMX_VMCS_HOST_CR4, &val);
4534 Log(("VMX_VMCS_HOST_CR4 %08x\n", val));
4535
4536 VMXReadVMCS(VMX_VMCS16_HOST_FIELD_CS, &val);
4537 Log(("VMX_VMCS_HOST_FIELD_CS %08x\n", val));
4538
4539 VMXReadVMCS(VMX_VMCS_GUEST_RFLAGS, &val);
4540 Log(("VMX_VMCS_GUEST_RFLAGS %08x\n", val));
4541
4542 if (val < gdtr.cbGdt)
4543 {
4544 pDesc = (PCX86DESCHC)(gdtr.pGdt + (val & X86_SEL_MASK));
4545 HWACCMR0DumpDescriptor(pDesc, val, "CS: ");
4546 }
4547
4548 VMXReadVMCS(VMX_VMCS16_HOST_FIELD_DS, &val);
4549 Log(("VMX_VMCS_HOST_FIELD_DS %08x\n", val));
4550 if (val < gdtr.cbGdt)
4551 {
4552 pDesc = (PCX86DESCHC)(gdtr.pGdt + (val & X86_SEL_MASK));
4553 HWACCMR0DumpDescriptor(pDesc, val, "DS: ");
4554 }
4555
4556 VMXReadVMCS(VMX_VMCS16_HOST_FIELD_ES, &val);
4557 Log(("VMX_VMCS_HOST_FIELD_ES %08x\n", val));
4558 if (val < gdtr.cbGdt)
4559 {
4560 pDesc = (PCX86DESCHC)(gdtr.pGdt + (val & X86_SEL_MASK));
4561 HWACCMR0DumpDescriptor(pDesc, val, "ES: ");
4562 }
4563
4564 VMXReadVMCS(VMX_VMCS16_HOST_FIELD_FS, &val);
4565 Log(("VMX_VMCS16_HOST_FIELD_FS %08x\n", val));
4566 if (val < gdtr.cbGdt)
4567 {
4568 pDesc = (PCX86DESCHC)(gdtr.pGdt + (val & X86_SEL_MASK));
4569 HWACCMR0DumpDescriptor(pDesc, val, "FS: ");
4570 }
4571
4572 VMXReadVMCS(VMX_VMCS16_HOST_FIELD_GS, &val);
4573 Log(("VMX_VMCS16_HOST_FIELD_GS %08x\n", val));
4574 if (val < gdtr.cbGdt)
4575 {
4576 pDesc = (PCX86DESCHC)(gdtr.pGdt + (val & X86_SEL_MASK));
4577 HWACCMR0DumpDescriptor(pDesc, val, "GS: ");
4578 }
4579
4580 VMXReadVMCS(VMX_VMCS16_HOST_FIELD_SS, &val);
4581 Log(("VMX_VMCS16_HOST_FIELD_SS %08x\n", val));
4582 if (val < gdtr.cbGdt)
4583 {
4584 pDesc = (PCX86DESCHC)(gdtr.pGdt + (val & X86_SEL_MASK));
4585 HWACCMR0DumpDescriptor(pDesc, val, "SS: ");
4586 }
4587
4588 VMXReadVMCS(VMX_VMCS16_HOST_FIELD_TR, &val);
4589 Log(("VMX_VMCS16_HOST_FIELD_TR %08x\n", val));
4590 if (val < gdtr.cbGdt)
4591 {
4592 pDesc = (PCX86DESCHC)(gdtr.pGdt + (val & X86_SEL_MASK));
4593 HWACCMR0DumpDescriptor(pDesc, val, "TR: ");
4594 }
4595
4596 VMXReadVMCS(VMX_VMCS_HOST_TR_BASE, &val);
4597 Log(("VMX_VMCS_HOST_TR_BASE %RHv\n", val));
4598
4599 VMXReadVMCS(VMX_VMCS_HOST_GDTR_BASE, &val);
4600 Log(("VMX_VMCS_HOST_GDTR_BASE %RHv\n", val));
4601 VMXReadVMCS(VMX_VMCS_HOST_IDTR_BASE, &val);
4602 Log(("VMX_VMCS_HOST_IDTR_BASE %RHv\n", val));
4603
4604 VMXReadVMCS(VMX_VMCS32_HOST_SYSENTER_CS, &val);
4605 Log(("VMX_VMCS_HOST_SYSENTER_CS %08x\n", val));
4606
4607 VMXReadVMCS(VMX_VMCS_HOST_SYSENTER_EIP, &val);
4608 Log(("VMX_VMCS_HOST_SYSENTER_EIP %RHv\n", val));
4609
4610 VMXReadVMCS(VMX_VMCS_HOST_SYSENTER_ESP, &val);
4611 Log(("VMX_VMCS_HOST_SYSENTER_ESP %RHv\n", val));
4612
4613 VMXReadVMCS(VMX_VMCS_HOST_RSP, &val);
4614 Log(("VMX_VMCS_HOST_RSP %RHv\n", val));
4615 VMXReadVMCS(VMX_VMCS_HOST_RIP, &val);
4616 Log(("VMX_VMCS_HOST_RIP %RHv\n", val));
4617
4618# if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4619 if (VMX_IS_64BIT_HOST_MODE())
4620 {
4621 Log(("MSR_K6_EFER = %RX64\n", ASMRdMsr(MSR_K6_EFER)));
4622 Log(("MSR_K6_STAR = %RX64\n", ASMRdMsr(MSR_K6_STAR)));
4623 Log(("MSR_K8_LSTAR = %RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
4624 Log(("MSR_K8_CSTAR = %RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
4625 Log(("MSR_K8_SF_MASK = %RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
4626 }
4627# endif
4628#endif /* VBOX_STRICT */
4629 }
4630 break;
4631 }
4632
4633 default:
4634 /* impossible */
4635 AssertMsgFailed(("%Rrc (%#x)\n", VBOXSTRICTRC_VAL(rc), VBOXSTRICTRC_VAL(rc)));
4636 break;
4637 }
4638}
4639
4640#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4641
4642/**
4643 * Prepares for and executes VMLAUNCH (64 bits guest mode)
4644 *
4645 * @returns VBox status code
4646 * @param fResume vmlauch/vmresume
4647 * @param pCtx Guest context
4648 * @param pCache VMCS cache
4649 * @param pVM The VM to operate on.
4650 * @param pVCpu The VMCPU to operate on.
4651 */
4652DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
4653{
4654 uint32_t aParam[6];
4655 PHMGLOBLCPUINFO pCpu;
4656 RTHCPHYS HCPhysCpuPage;
4657 int rc;
4658
4659 pCpu = HWACCMR0GetCurrentCpu();
4660 HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
4661
4662#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4663 pCache->uPos = 1;
4664 pCache->interPD = PGMGetInterPaeCR3(pVM);
4665 pCache->pSwitcher = (uint64_t)pVM->hwaccm.s.pfnHost32ToGuest64R0;
4666#endif
4667
4668#ifdef DEBUG
4669 pCache->TestIn.HCPhysCpuPage= 0;
4670 pCache->TestIn.HCPhysVMCS = 0;
4671 pCache->TestIn.pCache = 0;
4672 pCache->TestOut.HCPhysVMCS = 0;
4673 pCache->TestOut.pCache = 0;
4674 pCache->TestOut.pCtx = 0;
4675 pCache->TestOut.eflags = 0;
4676#endif
4677
4678 aParam[0] = (uint32_t)(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
4679 aParam[1] = (uint32_t)(HCPhysCpuPage >> 32); /* Param 1: VMXON physical address - Hi. */
4680 aParam[2] = (uint32_t)(pVCpu->hwaccm.s.vmx.HCPhysVMCS); /* Param 2: VMCS physical address - Lo. */
4681 aParam[3] = (uint32_t)(pVCpu->hwaccm.s.vmx.HCPhysVMCS >> 32); /* Param 2: VMCS physical address - Hi. */
4682 aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hwaccm.s.vmx.VMCSCache);
4683 aParam[5] = 0;
4684
4685#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4686 pCtx->dr[4] = pVM->hwaccm.s.vmx.pScratchPhys + 16 + 8;
4687 *(uint32_t *)(pVM->hwaccm.s.vmx.pScratch + 16 + 8) = 1;
4688#endif
4689 rc = VMXR0Execute64BitsHandler(pVM, pVCpu, pCtx, pVM->hwaccm.s.pfnVMXGCStartVM64, 6, &aParam[0]);
4690
4691#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4692 Assert(*(uint32_t *)(pVM->hwaccm.s.vmx.pScratch + 16 + 8) == 5);
4693 Assert(pCtx->dr[4] == 10);
4694 *(uint32_t *)(pVM->hwaccm.s.vmx.pScratch + 16 + 8) = 0xff;
4695#endif
4696
4697#ifdef DEBUG
4698 AssertMsg(pCache->TestIn.HCPhysCpuPage== HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
4699 AssertMsg(pCache->TestIn.HCPhysVMCS == pVCpu->hwaccm.s.vmx.HCPhysVMCS, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVMCS, pVCpu->hwaccm.s.vmx.HCPhysVMCS));
4700 AssertMsg(pCache->TestIn.HCPhysVMCS == pCache->TestOut.HCPhysVMCS, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVMCS, pCache->TestOut.HCPhysVMCS));
4701 AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache, pCache->TestOut.pCache));
4702 AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hwaccm.s.vmx.VMCSCache), ("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hwaccm.s.vmx.VMCSCache)));
4703 AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx, pCache->TestOut.pCtx));
4704 Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
4705#endif
4706 return rc;
4707}
4708
4709# ifdef VBOX_STRICT
4710
4711static bool hmR0VmxIsValidReadField(uint32_t idxField)
4712{
4713 switch(idxField)
4714 {
4715 case VMX_VMCS64_GUEST_RIP:
4716 case VMX_VMCS64_GUEST_RSP:
4717 case VMX_VMCS_GUEST_RFLAGS:
4718 case VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE:
4719 case VMX_VMCS_CTRL_CR0_READ_SHADOW:
4720 case VMX_VMCS64_GUEST_CR0:
4721 case VMX_VMCS_CTRL_CR4_READ_SHADOW:
4722 case VMX_VMCS64_GUEST_CR4:
4723 case VMX_VMCS64_GUEST_DR7:
4724 case VMX_VMCS32_GUEST_SYSENTER_CS:
4725 case VMX_VMCS64_GUEST_SYSENTER_EIP:
4726 case VMX_VMCS64_GUEST_SYSENTER_ESP:
4727 case VMX_VMCS32_GUEST_GDTR_LIMIT:
4728 case VMX_VMCS64_GUEST_GDTR_BASE:
4729 case VMX_VMCS32_GUEST_IDTR_LIMIT:
4730 case VMX_VMCS64_GUEST_IDTR_BASE:
4731 case VMX_VMCS16_GUEST_FIELD_CS:
4732 case VMX_VMCS32_GUEST_CS_LIMIT:
4733 case VMX_VMCS64_GUEST_CS_BASE:
4734 case VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS:
4735 case VMX_VMCS16_GUEST_FIELD_DS:
4736 case VMX_VMCS32_GUEST_DS_LIMIT:
4737 case VMX_VMCS64_GUEST_DS_BASE:
4738 case VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS:
4739 case VMX_VMCS16_GUEST_FIELD_ES:
4740 case VMX_VMCS32_GUEST_ES_LIMIT:
4741 case VMX_VMCS64_GUEST_ES_BASE:
4742 case VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS:
4743 case VMX_VMCS16_GUEST_FIELD_FS:
4744 case VMX_VMCS32_GUEST_FS_LIMIT:
4745 case VMX_VMCS64_GUEST_FS_BASE:
4746 case VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS:
4747 case VMX_VMCS16_GUEST_FIELD_GS:
4748 case VMX_VMCS32_GUEST_GS_LIMIT:
4749 case VMX_VMCS64_GUEST_GS_BASE:
4750 case VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS:
4751 case VMX_VMCS16_GUEST_FIELD_SS:
4752 case VMX_VMCS32_GUEST_SS_LIMIT:
4753 case VMX_VMCS64_GUEST_SS_BASE:
4754 case VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS:
4755 case VMX_VMCS16_GUEST_FIELD_LDTR:
4756 case VMX_VMCS32_GUEST_LDTR_LIMIT:
4757 case VMX_VMCS64_GUEST_LDTR_BASE:
4758 case VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS:
4759 case VMX_VMCS16_GUEST_FIELD_TR:
4760 case VMX_VMCS32_GUEST_TR_LIMIT:
4761 case VMX_VMCS64_GUEST_TR_BASE:
4762 case VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS:
4763 case VMX_VMCS32_RO_EXIT_REASON:
4764 case VMX_VMCS32_RO_VM_INSTR_ERROR:
4765 case VMX_VMCS32_RO_EXIT_INSTR_LENGTH:
4766 case VMX_VMCS32_RO_EXIT_INTERRUPTION_ERRCODE:
4767 case VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO:
4768 case VMX_VMCS32_RO_EXIT_INSTR_INFO:
4769 case VMX_VMCS_RO_EXIT_QUALIFICATION:
4770 case VMX_VMCS32_RO_IDT_INFO:
4771 case VMX_VMCS32_RO_IDT_ERRCODE:
4772 case VMX_VMCS64_GUEST_CR3:
4773 case VMX_VMCS_EXIT_PHYS_ADDR_FULL:
4774 return true;
4775 }
4776 return false;
4777}
4778
4779static bool hmR0VmxIsValidWriteField(uint32_t idxField)
4780{
4781 switch(idxField)
4782 {
4783 case VMX_VMCS64_GUEST_LDTR_BASE:
4784 case VMX_VMCS64_GUEST_TR_BASE:
4785 case VMX_VMCS64_GUEST_GDTR_BASE:
4786 case VMX_VMCS64_GUEST_IDTR_BASE:
4787 case VMX_VMCS64_GUEST_SYSENTER_EIP:
4788 case VMX_VMCS64_GUEST_SYSENTER_ESP:
4789 case VMX_VMCS64_GUEST_CR0:
4790 case VMX_VMCS64_GUEST_CR4:
4791 case VMX_VMCS64_GUEST_CR3:
4792 case VMX_VMCS64_GUEST_DR7:
4793 case VMX_VMCS64_GUEST_RIP:
4794 case VMX_VMCS64_GUEST_RSP:
4795 case VMX_VMCS64_GUEST_CS_BASE:
4796 case VMX_VMCS64_GUEST_DS_BASE:
4797 case VMX_VMCS64_GUEST_ES_BASE:
4798 case VMX_VMCS64_GUEST_FS_BASE:
4799 case VMX_VMCS64_GUEST_GS_BASE:
4800 case VMX_VMCS64_GUEST_SS_BASE:
4801 return true;
4802 }
4803 return false;
4804}
4805
4806# endif /* VBOX_STRICT */
4807
4808/**
4809 * Executes the specified handler in 64 mode
4810 *
4811 * @returns VBox status code.
4812 * @param pVM The VM to operate on.
4813 * @param pVCpu The VMCPU to operate on.
4814 * @param pCtx Guest context
4815 * @param pfnHandler RC handler
4816 * @param cbParam Number of parameters
4817 * @param paParam Array of 32 bits parameters
4818 */
4819VMMR0DECL(int) VMXR0Execute64BitsHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, RTRCPTR pfnHandler, uint32_t cbParam, uint32_t *paParam)
4820{
4821 int rc, rc2;
4822 PHMGLOBLCPUINFO pCpu;
4823 RTHCPHYS HCPhysCpuPage;
4824 RTHCUINTREG uOldEFlags;
4825
4826 AssertReturn(pVM->hwaccm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
4827 Assert(pfnHandler);
4828 Assert(pVCpu->hwaccm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hwaccm.s.vmx.VMCSCache.Write.aField));
4829 Assert(pVCpu->hwaccm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hwaccm.s.vmx.VMCSCache.Read.aField));
4830
4831#ifdef VBOX_STRICT
4832 for (unsigned i=0;i<pVCpu->hwaccm.s.vmx.VMCSCache.Write.cValidEntries;i++)
4833 Assert(hmR0VmxIsValidWriteField(pVCpu->hwaccm.s.vmx.VMCSCache.Write.aField[i]));
4834
4835 for (unsigned i=0;i<pVCpu->hwaccm.s.vmx.VMCSCache.Read.cValidEntries;i++)
4836 Assert(hmR0VmxIsValidReadField(pVCpu->hwaccm.s.vmx.VMCSCache.Read.aField[i]));
4837#endif
4838
4839 /* Disable interrupts. */
4840 uOldEFlags = ASMIntDisableFlags();
4841
4842 pCpu = HWACCMR0GetCurrentCpu();
4843 HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
4844
4845 /* Clear VM Control Structure. Marking it inactive, clearing implementation specific data and writing back VMCS data to memory. */
4846 VMXClearVMCS(pVCpu->hwaccm.s.vmx.HCPhysVMCS);
4847
4848 /* Leave VMX Root Mode. */
4849 VMXDisable();
4850
4851 ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
4852
4853 CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
4854 CPUMSetHyperEIP(pVCpu, pfnHandler);
4855 for (int i=(int)cbParam-1;i>=0;i--)
4856 CPUMPushHyper(pVCpu, paParam[i]);
4857
4858 STAM_PROFILE_ADV_START(&pVCpu->hwaccm.s.StatWorldSwitch3264, z);
4859 /* Call switcher. */
4860 rc = pVM->hwaccm.s.pfnHost32ToGuest64R0(pVM, RT_OFFSETOF(VM, aCpus[pVCpu->idCpu].cpum) - RT_OFFSETOF(VM, cpum));
4861 STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatWorldSwitch3264, z);
4862
4863 /* Make sure the VMX instructions don't cause #UD faults. */
4864 ASMSetCR4(ASMGetCR4() | X86_CR4_VMXE);
4865
4866 /* Enter VMX Root Mode */
4867 rc2 = VMXEnable(HCPhysCpuPage);
4868 if (RT_FAILURE(rc2))
4869 {
4870 ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
4871 ASMSetFlags(uOldEFlags);
4872 return VERR_VMX_VMXON_FAILED;
4873 }
4874
4875 rc2 = VMXActivateVMCS(pVCpu->hwaccm.s.vmx.HCPhysVMCS);
4876 AssertRC(rc2);
4877 Assert(!(ASMGetFlags() & X86_EFL_IF));
4878 ASMSetFlags(uOldEFlags);
4879 return rc;
4880}
4881
4882#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL) */
4883
4884
4885#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
4886/**
4887 * Executes VMWRITE
4888 *
4889 * @returns VBox status code
4890 * @param pVCpu The VMCPU to operate on.
4891 * @param idxField VMCS index
4892 * @param u64Val 16, 32 or 64 bits value
4893 */
4894VMMR0DECL(int) VMXWriteVMCS64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
4895{
4896 int rc;
4897
4898 switch (idxField)
4899 {
4900 case VMX_VMCS_CTRL_TSC_OFFSET_FULL:
4901 case VMX_VMCS_CTRL_IO_BITMAP_A_FULL:
4902 case VMX_VMCS_CTRL_IO_BITMAP_B_FULL:
4903 case VMX_VMCS_CTRL_MSR_BITMAP_FULL:
4904 case VMX_VMCS_CTRL_VMEXIT_MSR_STORE_FULL:
4905 case VMX_VMCS_CTRL_VMEXIT_MSR_LOAD_FULL:
4906 case VMX_VMCS_CTRL_VMENTRY_MSR_LOAD_FULL:
4907 case VMX_VMCS_CTRL_VAPIC_PAGEADDR_FULL:
4908 case VMX_VMCS_CTRL_APIC_ACCESSADDR_FULL:
4909 case VMX_VMCS_GUEST_LINK_PTR_FULL:
4910 case VMX_VMCS_GUEST_PDPTR0_FULL:
4911 case VMX_VMCS_GUEST_PDPTR1_FULL:
4912 case VMX_VMCS_GUEST_PDPTR2_FULL:
4913 case VMX_VMCS_GUEST_PDPTR3_FULL:
4914 case VMX_VMCS_GUEST_DEBUGCTL_FULL:
4915 case VMX_VMCS_GUEST_EFER_FULL:
4916 case VMX_VMCS_CTRL_EPTP_FULL:
4917 /* These fields consist of two parts, which are both writable in 32 bits mode. */
4918 rc = VMXWriteVMCS32(idxField, u64Val);
4919 rc |= VMXWriteVMCS32(idxField + 1, (uint32_t)(u64Val >> 32ULL));
4920 AssertRC(rc);
4921 return rc;
4922
4923 case VMX_VMCS64_GUEST_LDTR_BASE:
4924 case VMX_VMCS64_GUEST_TR_BASE:
4925 case VMX_VMCS64_GUEST_GDTR_BASE:
4926 case VMX_VMCS64_GUEST_IDTR_BASE:
4927 case VMX_VMCS64_GUEST_SYSENTER_EIP:
4928 case VMX_VMCS64_GUEST_SYSENTER_ESP:
4929 case VMX_VMCS64_GUEST_CR0:
4930 case VMX_VMCS64_GUEST_CR4:
4931 case VMX_VMCS64_GUEST_CR3:
4932 case VMX_VMCS64_GUEST_DR7:
4933 case VMX_VMCS64_GUEST_RIP:
4934 case VMX_VMCS64_GUEST_RSP:
4935 case VMX_VMCS64_GUEST_CS_BASE:
4936 case VMX_VMCS64_GUEST_DS_BASE:
4937 case VMX_VMCS64_GUEST_ES_BASE:
4938 case VMX_VMCS64_GUEST_FS_BASE:
4939 case VMX_VMCS64_GUEST_GS_BASE:
4940 case VMX_VMCS64_GUEST_SS_BASE:
4941 /* Queue a 64 bits value as we can't set it in 32 bits host mode. */
4942 if (u64Val >> 32ULL)
4943 rc = VMXWriteCachedVMCSEx(pVCpu, idxField, u64Val);
4944 else
4945 rc = VMXWriteVMCS32(idxField, (uint32_t)u64Val);
4946
4947 return rc;
4948
4949 default:
4950 AssertMsgFailed(("Unexpected field %x\n", idxField));
4951 return VERR_INVALID_PARAMETER;
4952 }
4953}
4954
4955/**
4956 * Cache VMCS writes for performance reasons (Darwin) and for running 64 bits guests on 32 bits hosts.
4957 *
4958 * @param pVCpu The VMCPU to operate on.
4959 * @param idxField VMCS field
4960 * @param u64Val Value
4961 */
4962VMMR0DECL(int) VMXWriteCachedVMCSEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
4963{
4964 PVMCSCACHE pCache = &pVCpu->hwaccm.s.vmx.VMCSCache;
4965
4966 AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1, ("entries=%x\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
4967
4968 /* Make sure there are no duplicates. */
4969 for (unsigned i=0;i<pCache->Write.cValidEntries;i++)
4970 {
4971 if (pCache->Write.aField[i] == idxField)
4972 {
4973 pCache->Write.aFieldVal[i] = u64Val;
4974 return VINF_SUCCESS;
4975 }
4976 }
4977
4978 pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
4979 pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
4980 pCache->Write.cValidEntries++;
4981 return VINF_SUCCESS;
4982}
4983
4984#endif /* HC_ARCH_BITS == 32 && !VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0 */
4985
Note: See TracBrowser for help on using the repository browser.

© 2024 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette