HWVMXR0.cpp@ 47062

Last change on this file since 47062 was 46871, checked in by vboxsync, 12 years ago
VMM/HM: Cleanup event injection statistics.
Property svn:eol-style set to `native` Property svn:keywords set to `Id Revision`
File size: 228.4 KB

Line
1	/* $Id: HWVMXR0.cpp 46871 2013-06-28 16:25:06Z vboxsync $ */
2	/** @file
3	* HM VMX (VT-x) - Host Context Ring-0.
4	*/
5
6	/*
7	* Copyright (C) 2006-2013 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18
19	/*******************************************************************************
20	* Header Files *
21	*******************************************************************************/
22	#define LOG_GROUP LOG_GROUP_HM
23	#include <iprt/asm-amd64-x86.h>
24	#include <VBox/vmm/hm.h>
25	#include <VBox/vmm/pgm.h>
26	#include <VBox/vmm/dbgf.h>
27	#include <VBox/vmm/dbgftrace.h>
28	#include <VBox/vmm/selm.h>
29	#include <VBox/vmm/iom.h>
30	#ifdef VBOX_WITH_REM
31	# include <VBox/vmm/rem.h>
32	#endif
33	#include <VBox/vmm/tm.h>
34	#include "HMInternal.h"
35	#include <VBox/vmm/vm.h>
36	#include <VBox/vmm/pdmapi.h>
37	#include <VBox/err.h>
38	#include <VBox/log.h>
39	#include <iprt/assert.h>
40	#include <iprt/param.h>
41	#include <iprt/string.h>
42	#include <iprt/time.h>
43	#ifdef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
44	# include <iprt/thread.h>
45	#endif
46	#include <iprt/x86.h>
47	#include "HWVMXR0.h"
48
49	#include "dtrace/VBoxVMM.h"
50
51
52	/*******************************************************************************
53	* Defined Constants And Macros *
54	*******************************************************************************/
55	#if defined(RT_ARCH_AMD64)
56	# define VMX_IS_64BIT_HOST_MODE() (true)
57	#elif defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
58	# define VMX_IS_64BIT_HOST_MODE() (g_fVMXIs64bitHost != 0)
59	#else
60	# define VMX_IS_64BIT_HOST_MODE() (false)
61	#endif
62
63	# define VMX_WRITE_SELREG(REG, reg) \
64	do \
65	{ \
66	rc = VMXWriteVmcs(VMX_VMCS16_GUEST_FIELD_##REG, pCtx->reg.Sel); \
67	rc \|= VMXWriteVmcs(VMX_VMCS32_GUEST_##REG##_LIMIT, pCtx->reg.u32Limit); \
68	rc \|= VMXWriteVmcs64(VMX_VMCS_GUEST_##REG##_BASE, pCtx->reg.u64Base); \
69	if ((pCtx->eflags.u32 & X86_EFL_VM)) \
70	{ \
71	/* Must override this or else VT-x will fail with invalid guest state errors. */ \
72	/* DPL=3, present, code/data, r/w/accessed. */ \
73	/** @todo we shouldn't have to do this, if it is not 0xf3 it means we screwed up elsewhere (recompiler). */ \
74	/** @todo VT-x docs explicitly mentions 0xF3. Why not just val = 0xf3 ??. */ \
75	val = (pCtx->reg.Attr.u & ~0xFF) \| 0xF3; \
76	} \
77	else \
78	if ( CPUMIsGuestInRealModeEx(pCtx) \
79	&& !pVM->hm.s.vmx.fUnrestrictedGuest) \
80	{ \
81	/** @todo shouldn't the 'if' condition above check for 'pRealModeTSS' ? */ \
82	/* Must override this or else VT-x will fail with invalid guest state errors. */ \
83	/* DPL=3, present, code/data, r/w/accessed. */ \
84	val = 0xf3; \
85	} \
86	else \
87	if ( ( pCtx->reg.Sel \
88	\|\| !CPUMIsGuestInPagedProtectedModeEx(pCtx) \
89	\|\| (!pCtx->cs.Attr.n.u1DefBig && !CPUMIsGuestIn64BitCodeEx(pCtx)) \
90	) \
91	&& pCtx->reg.Attr.n.u1Present == 1) \
92	{ \
93	val = pCtx->reg.Attr.u \| X86_SEL_TYPE_ACCESSED; \
94	} \
95	else \
96	val = 0x10000; /* Invalid guest state error otherwise. (BIT(16) = Unusable) */ \
97	\
98	rc \|= VMXWriteVmcs(VMX_VMCS32_GUEST_##REG##_ACCESS_RIGHTS, val); \
99	} while (0)
100
101	# define VMX_READ_SELREG(REG, reg) \
102	do \
103	{ \
104	VMXReadCachedVmcs(VMX_VMCS16_GUEST_FIELD_##REG, &val); \
105	pCtx->reg.Sel = val; \
106	pCtx->reg.ValidSel = val; \
107	pCtx->reg.fFlags = CPUMSELREG_FLAGS_VALID; \
108	VMXReadCachedVmcs(VMX_VMCS32_GUEST_##REG##_LIMIT, &val); \
109	pCtx->reg.u32Limit = val; \
110	VMXReadCachedVmcs(VMX_VMCS_GUEST_##REG##_BASE, &val); \
111	pCtx->reg.u64Base = val; \
112	VMXReadCachedVmcs(VMX_VMCS32_GUEST_##REG##_ACCESS_RIGHTS, &val); \
113	pCtx->reg.Attr.u = val; \
114	} while (0)
115
116	/* Don't read from the cache in this macro; used only in case of failure where the cache is out of sync. */
117	# define VMX_LOG_SELREG(REG, szSelReg, val) \
118	do \
119	{ \
120	VMXReadVmcs(VMX_VMCS16_GUEST_FIELD_##REG, &(val)); \
121	Log(("%s Selector %x\n", szSelReg, (val))); \
122	VMXReadVmcs(VMX_VMCS32_GUEST_##REG##_LIMIT, &(val)); \
123	Log(("%s Limit %x\n", szSelReg, (val))); \
124	VMXReadVmcs(VMX_VMCS_GUEST_##REG##_BASE, &(val)); \
125	Log(("%s Base %RX64\n", szSelReg, (uint64_t)(val))); \
126	VMXReadVmcs(VMX_VMCS32_GUEST_##REG##_ACCESS_RIGHTS, &(val)); \
127	Log(("%s Attributes %x\n", szSelReg, (val))); \
128	} while (0)
129
130	#define VMXSetupCachedReadVmcs(pCache, idxField) \
131	{ \
132	Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
133	pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
134	pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
135	}
136	#define VMX_SETUP_SELREG(REG, pCache) \
137	{ \
138	VMXSetupCachedReadVmcs(pCache, VMX_VMCS16_GUEST_FIELD_##REG); \
139	VMXSetupCachedReadVmcs(pCache, VMX_VMCS32_GUEST_##REG##_LIMIT); \
140	VMXSetupCachedReadVmcs(pCache, VMX_VMCS_GUEST_##REG##_BASE); \
141	VMXSetupCachedReadVmcs(pCache, VMX_VMCS32_GUEST_##REG##_ACCESS_RIGHTS); \
142	}
143
144
145	/*******************************************************************************
146	* Global Variables *
147	*******************************************************************************/
148	/* IO operation lookup arrays. */
149	static uint32_t const g_aIOSize[4] = {1, 2, 0, 4};
150	static uint32_t const g_aIOOpAnd[4] = {0xff, 0xffff, 0, 0xffffffff};
151
152	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
153	/** See HMR0A.asm. */
154	extern "C" uint32_t g_fVMXIs64bitHost;
155	#endif
156
157
158	/*******************************************************************************
159	* Local Functions *
160	*******************************************************************************/
161	static DECLCALLBACK(void) hmR0VmxSetupTLBEPT(PVM pVM, PVMCPU pVCpu);
162	static DECLCALLBACK(void) hmR0VmxSetupTLBVPID(PVM pVM, PVMCPU pVCpu);
163	static DECLCALLBACK(void) hmR0VmxSetupTLBBoth(PVM pVM, PVMCPU pVCpu);
164	static DECLCALLBACK(void) hmR0VmxSetupTLBDummy(PVM pVM, PVMCPU pVCpu);
165	static void hmR0VmxFlushEPT(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_EPT enmFlush);
166	static void hmR0VmxFlushVPID(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_VPID enmFlush, RTGCPTR GCPtr);
167	static void hmR0VmxUpdateExceptionBitmap(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx);
168	static void hmR0VmxSetMSRPermission(PVMCPU pVCpu, unsigned ulMSR, bool fRead, bool fWrite);
169	static void hmR0VmxReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, VBOXSTRICTRC rc, PCPUMCTX pCtx);
170
171
172	/**
173	* Updates error from VMCS to HMCPU's lasterror record.
174	*
175	* @param pVM Pointer to the VM.
176	* @param pVCpu Pointer to the VMCPU.
177	* @param rc The error code.
178	*/
179	static void hmR0VmxCheckError(PVM pVM, PVMCPU pVCpu, int rc)
180	{
181	if ( rc == VERR_VMX_UNABLE_TO_START_VM
182	\|\| rc == VERR_VMX_INVALID_VMCS_FIELD)
183	{
184	RTCCUINTREG instrError;
185
186	VMXReadVmcs(VMX_VMCS32_RO_VM_INSTR_ERROR, &instrError);
187	pVCpu->hm.s.vmx.lasterror.u32InstrError = instrError;
188	}
189	pVM->hm.s.lLastError = rc;
190	}
191
192
193	/**
194	* Sets up and activates VT-x on the current CPU.
195	*
196	* @returns VBox status code.
197	* @param pCpu Pointer to the CPU info struct.
198	* @param pVM Pointer to the VM. (can be NULL after a resume!!)
199	* @param pvCpuPage Pointer to the global CPU page.
200	* @param HCPhysCpuPage Physical address of the global CPU page.
201	* @param fEnabledByHost Set if SUPR0EnableVTx or similar was used to enable
202	* VT-x/AMD-V on the host.
203	*/
204	VMMR0DECL(int) VMXR0EnableCpu(PHMGLOBLCPUINFO pCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost)
205	{
206	if (!fEnabledByHost)
207	{
208	AssertReturn(HCPhysCpuPage != 0 && HCPhysCpuPage != NIL_RTHCPHYS, VERR_INVALID_PARAMETER);
209	AssertReturn(pvCpuPage, VERR_INVALID_PARAMETER);
210
211	if (pVM)
212	{
213	/* Set revision dword at the beginning of the VMXON structure. */
214	(uint32_t )pvCpuPage = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.msr.vmx_basic_info);
215	}
216
217	/** @todo we should unmap the two pages from the virtual address space in order to prevent accidental corruption.
218	* (which can have very bad consequences!!!)
219	*/
220
221	/** @todo r=bird: Why is this code different than the probing code earlier
222	* on? It just sets VMXE if needed and doesn't check that it isn't
223	* set. Mac OS X host_vmxoff may leave this set and we'll fail here
224	* and debug-assert in the calling code. This is what caused the
225	* "regression" after backing out the SUPR0EnableVTx code hours before
226	* 4.2.0GA (reboot fixed the issue). I've changed here to do the same
227	* as the init code. */
228	uint64_t uCr4 = ASMGetCR4();
229	if (!(uCr4 & X86_CR4_VMXE))
230	ASMSetCR4(ASMGetCR4() \| X86_CR4_VMXE); /* Make sure the VMX instructions don't cause #UD faults. */
231
232	/*
233	* Enter VM root mode.
234	*/
235	int rc = VMXEnable(HCPhysCpuPage);
236	if (RT_FAILURE(rc))
237	{
238	ASMSetCR4(uCr4);
239	return VERR_VMX_VMXON_FAILED;
240	}
241	}
242
243	/*
244	* Flush all VPIDs (in case we or any other hypervisor have been using VPIDs) so that
245	* we can avoid an explicit flush while using new VPIDs. We would still need to flush
246	* each time while reusing a VPID after hitting the MaxASID limit once.
247	*/
248	if ( pVM
249	&& pVM->hm.s.vmx.fVpid
250	&& (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS))
251	{
252	hmR0VmxFlushVPID(pVM, NULL /* pvCpu /, VMX_FLUSH_VPID_ALL_CONTEXTS, 0 / GCPtr */);
253	pCpu->fFlushAsidBeforeUse = false;
254	}
255	else
256	pCpu->fFlushAsidBeforeUse = true;
257
258	/*
259	* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}.
260	*/
261	++pCpu->cTlbFlushes;
262
263	return VINF_SUCCESS;
264	}
265
266
267	/**
268	* Deactivates VT-x on the current CPU.
269	*
270	* @returns VBox status code.
271	* @param pCpu Pointer to the CPU info struct.
272	* @param pvCpuPage Pointer to the global CPU page.
273	* @param HCPhysCpuPage Physical address of the global CPU page.
274	*/
275	VMMR0DECL(int) VMXR0DisableCpu(PHMGLOBLCPUINFO pCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
276	{
277	AssertReturn(HCPhysCpuPage != 0 && HCPhysCpuPage != NIL_RTHCPHYS, VERR_INVALID_PARAMETER);
278	AssertReturn(pvCpuPage, VERR_INVALID_PARAMETER);
279	NOREF(pCpu);
280
281	/* If we're somehow not in VMX root mode, then we shouldn't dare leaving it. */
282	if (!(ASMGetCR4() & X86_CR4_VMXE))
283	return VERR_VMX_NOT_IN_VMX_ROOT_MODE;
284
285	/* Leave VMX Root Mode. */
286	VMXDisable();
287
288	/* And clear the X86_CR4_VMXE bit. */
289	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
290	return VINF_SUCCESS;
291	}
292
293	VMMR0DECL(int) VMXR0GlobalInit(void)
294	{
295	/* Nothing to do. */
296	return VINF_SUCCESS;
297	}
298
299	VMMR0DECL(void) VMXR0GlobalTerm(void)
300	{
301	/* Nothing to do. */
302	}
303
304	/**
305	* Does Ring-0 per VM VT-x initialization.
306	*
307	* @returns VBox status code.
308	* @param pVM Pointer to the VM.
309	*/
310	VMMR0DECL(int) VMXR0InitVM(PVM pVM)
311	{
312	int rc;
313
314	#ifdef LOG_ENABLED
315	SUPR0Printf("VMXR0InitVM %p\n", pVM);
316	#endif
317
318	pVM->hm.s.vmx.hMemObjApicAccess = NIL_RTR0MEMOBJ;
319
320	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
321	{
322	/* Allocate one page for the APIC physical page (serves for filtering accesses). */
323	rc = RTR0MemObjAllocCont(&pVM->hm.s.vmx.hMemObjApicAccess, PAGE_SIZE, false /* fExecutable */);
324	AssertRC(rc);
325	if (RT_FAILURE(rc))
326	return rc;
327
328	pVM->hm.s.vmx.pbApicAccess = (uint8_t *)RTR0MemObjAddress(pVM->hm.s.vmx.hMemObjApicAccess);
329	pVM->hm.s.vmx.HCPhysApicAccess = RTR0MemObjGetPagePhysAddr(pVM->hm.s.vmx.hMemObjApicAccess, 0);
330	ASMMemZero32(pVM->hm.s.vmx.pbApicAccess, PAGE_SIZE);
331	}
332	else
333	{
334	pVM->hm.s.vmx.hMemObjApicAccess = 0;
335	pVM->hm.s.vmx.pbApicAccess = 0;
336	pVM->hm.s.vmx.HCPhysApicAccess = 0;
337	}
338
339	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
340	{
341	rc = RTR0MemObjAllocCont(&pVM->hm.s.vmx.hMemObjScratch, PAGE_SIZE, false /* fExecutable */);
342	AssertRC(rc);
343	if (RT_FAILURE(rc))
344	return rc;
345
346	pVM->hm.s.vmx.pScratch = (uint8_t *)RTR0MemObjAddress(pVM->hm.s.vmx.hMemObjScratch);
347	pVM->hm.s.vmx.pScratchPhys = RTR0MemObjGetPagePhysAddr(pVM->hm.s.vmx.hMemObjScratch, 0);
348
349	ASMMemZero32(pVM->hm.s.vmx.pbScratch, PAGE_SIZE);
350	strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
351	(uint64_t )(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xDEADBEEFDEADBEEF);
352	}
353	#endif
354
355	/* Allocate VMCSs for all guest CPUs. */
356	for (VMCPUID i = 0; i < pVM->cCpus; i++)
357	{
358	PVMCPU pVCpu = &pVM->aCpus[i];
359
360	pVCpu->hm.s.vmx.hMemObjVmcs = NIL_RTR0MEMOBJ;
361
362	/* Allocate one page for the VM control structure (VMCS). */
363	rc = RTR0MemObjAllocCont(&pVCpu->hm.s.vmx.hMemObjVmcs, PAGE_SIZE, false /* fExecutable */);
364	AssertRC(rc);
365	if (RT_FAILURE(rc))
366	return rc;
367
368	pVCpu->hm.s.vmx.pvVmcs = RTR0MemObjAddress(pVCpu->hm.s.vmx.hMemObjVmcs);
369	pVCpu->hm.s.vmx.HCPhysVmcs = RTR0MemObjGetPagePhysAddr(pVCpu->hm.s.vmx.hMemObjVmcs, 0);
370	ASMMemZeroPage(pVCpu->hm.s.vmx.pvVmcs);
371
372	pVCpu->hm.s.vmx.u32CR0Mask = 0;
373	pVCpu->hm.s.vmx.u32CR4Mask = 0;
374
375	/* Allocate one page for the virtual APIC page for TPR caching. */
376	rc = RTR0MemObjAllocCont(&pVCpu->hm.s.vmx.hMemObjVirtApic, PAGE_SIZE, false /* fExecutable */);
377	AssertRC(rc);
378	if (RT_FAILURE(rc))
379	return rc;
380
381	pVCpu->hm.s.vmx.pbVirtApic = (uint8_t *)RTR0MemObjAddress(pVCpu->hm.s.vmx.hMemObjVirtApic);
382	pVCpu->hm.s.vmx.HCPhysVirtApic = RTR0MemObjGetPagePhysAddr(pVCpu->hm.s.vmx.hMemObjVirtApic, 0);
383	ASMMemZeroPage(pVCpu->hm.s.vmx.pbVirtApic);
384
385	/* Allocate the MSR bitmap if this feature is supported. */
386	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
387	{
388	rc = RTR0MemObjAllocCont(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, PAGE_SIZE, false /* fExecutable */);
389	AssertRC(rc);
390	if (RT_FAILURE(rc))
391	return rc;
392
393	pVCpu->hm.s.vmx.pvMsrBitmap = (uint8_t *)RTR0MemObjAddress(pVCpu->hm.s.vmx.hMemObjMsrBitmap);
394	pVCpu->hm.s.vmx.HCPhysMsrBitmap = RTR0MemObjGetPagePhysAddr(pVCpu->hm.s.vmx.hMemObjMsrBitmap, 0);
395	memset(pVCpu->hm.s.vmx.pvMsrBitmap, 0xff, PAGE_SIZE);
396	}
397
398	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
399	/* Allocate one page for the guest MSR load area (for preloading guest MSRs during the world switch). */
400	rc = RTR0MemObjAllocCont(&pVCpu->hm.s.vmx.hMemObjGuestMsr, PAGE_SIZE, false /* fExecutable */);
401	AssertRC(rc);
402	if (RT_FAILURE(rc))
403	return rc;
404
405	pVCpu->hm.s.vmx.pvGuestMsr = (uint8_t *)RTR0MemObjAddress(pVCpu->hm.s.vmx.hMemObjGuestMsr);
406	pVCpu->hm.s.vmx.HCPhysGuestMsr = RTR0MemObjGetPagePhysAddr(pVCpu->hm.s.vmx.hMemObjGuestMsr, 0);
407	Assert(!(pVCpu->hm.s.vmx.HCPhysGuestMsr & 0xf));
408	memset(pVCpu->hm.s.vmx.pvGuestMsr, 0, PAGE_SIZE);
409
410	/* Allocate one page for the host MSR load area (for restoring host MSRs after the world switch back). */
411	rc = RTR0MemObjAllocCont(&pVCpu->hm.s.vmx.hMemObjHostMsr, PAGE_SIZE, false /* fExecutable */);
412	AssertRC(rc);
413	if (RT_FAILURE(rc))
414	return rc;
415
416	pVCpu->hm.s.vmx.pvHostMsr = (uint8_t *)RTR0MemObjAddress(pVCpu->hm.s.vmx.hMemObjHostMsr);
417	pVCpu->hm.s.vmx.HCPhysHostMsr = RTR0MemObjGetPagePhysAddr(pVCpu->hm.s.vmx.hMemObjHostMsr, 0);
418	Assert(!(pVCpu->hm.s.vmx.HCPhysHostMsr & 0xf));
419	memset(pVCpu->hm.s.vmx.pvHostMsr, 0, PAGE_SIZE);
420	#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
421
422	/* Current guest paging mode. */
423	pVCpu->hm.s.vmx.enmLastSeenGuestMode = PGMMODE_REAL;
424
425	#ifdef LOG_ENABLED
426	SUPR0Printf("VMXR0InitVM %x VMCS=%x (%x)\n", pVM, pVCpu->hm.s.vmx.pvVmcs, (uint32_t)pVCpu->hm.s.vmx.HCPhysVmcs);
427	#endif
428	}
429
430	return VINF_SUCCESS;
431	}
432
433
434	/**
435	* Does Ring-0 per VM VT-x termination.
436	*
437	* @returns VBox status code.
438	* @param pVM Pointer to the VM.
439	*/
440	VMMR0DECL(int) VMXR0TermVM(PVM pVM)
441	{
442	for (VMCPUID i = 0; i < pVM->cCpus; i++)
443	{
444	PVMCPU pVCpu = &pVM->aCpus[i];
445
446	if (pVCpu->hm.s.vmx.hMemObjVmcs != NIL_RTR0MEMOBJ)
447	{
448	RTR0MemObjFree(pVCpu->hm.s.vmx.hMemObjVmcs, false);
449	pVCpu->hm.s.vmx.hMemObjVmcs = NIL_RTR0MEMOBJ;
450	pVCpu->hm.s.vmx.pvVmcs = 0;
451	pVCpu->hm.s.vmx.HCPhysVmcs = 0;
452	}
453	if (pVCpu->hm.s.vmx.hMemObjVirtApic != NIL_RTR0MEMOBJ)
454	{
455	RTR0MemObjFree(pVCpu->hm.s.vmx.hMemObjVirtApic, false);
456	pVCpu->hm.s.vmx.hMemObjVirtApic = NIL_RTR0MEMOBJ;
457	pVCpu->hm.s.vmx.pbVirtApic = 0;
458	pVCpu->hm.s.vmx.HCPhysVirtApic = 0;
459	}
460	if (pVCpu->hm.s.vmx.hMemObjMsrBitmap != NIL_RTR0MEMOBJ)
461	{
462	RTR0MemObjFree(pVCpu->hm.s.vmx.hMemObjMsrBitmap, false);
463	pVCpu->hm.s.vmx.hMemObjMsrBitmap = NIL_RTR0MEMOBJ;
464	pVCpu->hm.s.vmx.pvMsrBitmap = 0;
465	pVCpu->hm.s.vmx.HCPhysMsrBitmap = 0;
466	}
467	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
468	if (pVCpu->hm.s.vmx.hMemObjHostMsr != NIL_RTR0MEMOBJ)
469	{
470	RTR0MemObjFree(pVCpu->hm.s.vmx.hMemObjHostMsr, false);
471	pVCpu->hm.s.vmx.hMemObjHostMsr = NIL_RTR0MEMOBJ;
472	pVCpu->hm.s.vmx.pvHostMsr = 0;
473	pVCpu->hm.s.vmx.HCPhysHostMsr = 0;
474	}
475	if (pVCpu->hm.s.vmx.hMemObjGuestMsr != NIL_RTR0MEMOBJ)
476	{
477	RTR0MemObjFree(pVCpu->hm.s.vmx.hMemObjGuestMsr, false);
478	pVCpu->hm.s.vmx.hMemObjGuestMsr = NIL_RTR0MEMOBJ;
479	pVCpu->hm.s.vmx.pvGuestMsr = 0;
480	pVCpu->hm.s.vmx.HCPhysGuestMsr = 0;
481	}
482	#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
483	}
484	if (pVM->hm.s.vmx.hMemObjApicAccess != NIL_RTR0MEMOBJ)
485	{
486	RTR0MemObjFree(pVM->hm.s.vmx.hMemObjApicAccess, false);
487	pVM->hm.s.vmx.hMemObjApicAccess = NIL_RTR0MEMOBJ;
488	pVM->hm.s.vmx.pbApicAccess = 0;
489	pVM->hm.s.vmx.HCPhysApicAccess = 0;
490	}
491	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
492	if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
493	{
494	ASMMemZero32(pVM->hm.s.vmx.pScratch, PAGE_SIZE);
495	RTR0MemObjFree(pVM->hm.s.vmx.hMemObjScratch, false);
496	pVM->hm.s.vmx.hMemObjScratch = NIL_RTR0MEMOBJ;
497	pVM->hm.s.vmx.pScratch = 0;
498	pVM->hm.s.vmx.pScratchPhys = 0;
499	}
500	#endif
501	return VINF_SUCCESS;
502	}
503
504
505	/**
506	* Sets up VT-x for the specified VM.
507	*
508	* @returns VBox status code.
509	* @param pVM Pointer to the VM.
510	*/
511	VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
512	{
513	int rc = VINF_SUCCESS;
514	uint32_t val;
515
516	AssertReturn(pVM, VERR_INVALID_PARAMETER);
517
518	/* Initialize these always, see hmR3InitFinalizeR0().*/
519	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NONE;
520	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NONE;
521
522	/* Determine optimal flush type for EPT. */
523	if (pVM->hm.s.fNestedPaging)
524	{
525	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
526	{
527	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
528	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_SINGLE_CONTEXT;
529	else if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
530	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_ALL_CONTEXTS;
531	else
532	{
533	/*
534	* Should never really happen. EPT is supported but no suitable flush types supported.
535	* We cannot ignore EPT at this point as we've already setup Unrestricted Guest execution.
536	*/
537	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
538	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
539	}
540	}
541	else
542	{
543	/*
544	* Should never really happen. EPT is supported but INVEPT instruction is not supported.
545	*/
546	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
547	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
548	}
549	}
550
551	/* Determine optimal flush type for VPID. */
552	if (pVM->hm.s.vmx.fVpid)
553	{
554	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
555	{
556	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
557	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_SINGLE_CONTEXT;
558	else if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
559	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_ALL_CONTEXTS;
560	else
561	{
562	/*
563	* Neither SINGLE nor ALL context flush types for VPID supported by the CPU.
564	* We do not handle other flush type combinations, ignore VPID capabilities.
565	*/
566	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
567	Log(("VMXR0SetupVM: Only VMX_FLUSH_VPID_INDIV_ADDR supported. Ignoring VPID.\n"));
568	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
569	Log(("VMXR0SetupVM: Only VMX_FLUSH_VPID_SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
570	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NOT_SUPPORTED;
571	pVM->hm.s.vmx.fVpid = false;
572	}
573	}
574	else
575	{
576	/*
577	* Should not really happen. EPT is supported but INVEPT is not supported.
578	* Ignore VPID capabilities as our code relies on using INVEPT for selective flushing.
579	*/
580	Log(("VMXR0SetupVM: VPID supported without INVEPT support. Ignoring VPID.\n"));
581	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NOT_SUPPORTED;
582	pVM->hm.s.vmx.fVpid = false;
583	}
584	}
585
586	for (VMCPUID i = 0; i < pVM->cCpus; i++)
587	{
588	PVMCPU pVCpu = &pVM->aCpus[i];
589
590	AssertPtr(pVCpu->hm.s.vmx.pvVmcs);
591
592	/* Set revision dword at the beginning of the VMCS structure. */
593	(uint32_t )pVCpu->hm.s.vmx.pvVmcs = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.msr.vmx_basic_info);
594
595	/*
596	* Clear and activate the VMCS.
597	*/
598	Log(("HCPhysVmcs = %RHp\n", pVCpu->hm.s.vmx.HCPhysVmcs));
599	rc = VMXClearVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
600	if (RT_FAILURE(rc))
601	goto vmx_end;
602
603	rc = VMXActivateVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
604	if (RT_FAILURE(rc))
605	goto vmx_end;
606
607	/*
608	* VMX_VMCS_CTRL_PIN_EXEC
609	* Set required bits to one and zero according to the MSR capabilities.
610	*/
611	val = pVM->hm.s.vmx.msr.vmx_pin_ctls.n.disallowed0;
612	val \|= VMX_VMCS_CTRL_PIN_EXEC_EXT_INT_EXIT /* External interrupts */
613	\| VMX_VMCS_CTRL_PIN_EXEC_NMI_EXIT; /* Non-maskable interrupts */
614
615	/*
616	* Enable the VMX preemption timer.
617	*/
618	if (pVM->hm.s.vmx.fUsePreemptTimer)
619	val \|= VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER;
620	val &= pVM->hm.s.vmx.msr.vmx_pin_ctls.n.allowed1;
621
622	rc = VMXWriteVmcs(VMX_VMCS32_CTRL_PIN_EXEC, val);
623	AssertRC(rc);
624	pVCpu->hm.s.vmx.u32PinCtls = val;
625
626	/*
627	* VMX_VMCS_CTRL_PROC_EXEC
628	* Set required bits to one and zero according to the MSR capabilities.
629	*/
630	val = pVM->hm.s.vmx.msr.vmx_proc_ctls.n.disallowed0;
631	/* Program which event cause VM-exits and which features we want to use. */
632	val \|= VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT
633	\| VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING
634	\| VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT
635	\| VMX_VMCS_CTRL_PROC_EXEC_UNCOND_IO_EXIT
636	\| VMX_VMCS_CTRL_PROC_EXEC_RDPMC_EXIT
637	\| VMX_VMCS_CTRL_PROC_EXEC_MONITOR_EXIT
638	\| VMX_VMCS_CTRL_PROC_EXEC_MWAIT_EXIT; /* don't execute mwait or else we'll idle inside
639	the guest (host thinks the cpu load is high) */
640
641	/* Without nested paging we should intercept invlpg and cr3 mov instructions. */
642	if (!pVM->hm.s.fNestedPaging)
643	{
644	val \|= VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT
645	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
646	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
647	}
648
649	/*
650	* VMX_VMCS_CTRL_PROC_EXEC_MWAIT_EXIT might cause a vmlaunch
651	* failure with an invalid control fields error. (combined with some other exit reasons)
652	*/
653	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
654	{
655	/* CR8 reads from the APIC shadow page; writes cause an exit is they lower the TPR below the threshold */
656	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW;
657	Assert(pVM->hm.s.vmx.pbApicAccess);
658	}
659	else
660	/* Exit on CR8 reads & writes in case the TPR shadow feature isn't present. */
661	val \|= VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT \| VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT;
662
663	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
664	{
665	Assert(pVCpu->hm.s.vmx.HCPhysMsrBitmap);
666	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS;
667	}
668
669	/* We will use the secondary control if it's present. */
670	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
671
672	/* Mask away the bits that the CPU doesn't support */
673	/** @todo make sure they don't conflict with the above requirements. */
674	val &= pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1;
675	pVCpu->hm.s.vmx.u32ProcCtls = val;
676
677	rc = VMXWriteVmcs(VMX_VMCS32_CTRL_PROC_EXEC, val);
678	AssertRC(rc);
679
680	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
681	{
682	/*
683	* VMX_VMCS_CTRL_PROC_EXEC2
684	* Set required bits to one and zero according to the MSR capabilities.
685	*/
686	val = pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.disallowed0;
687	val \|= VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT;
688
689	if (pVM->hm.s.fNestedPaging)
690	val \|= VMX_VMCS_CTRL_PROC_EXEC2_EPT;
691
692	if (pVM->hm.s.vmx.fVpid)
693	val \|= VMX_VMCS_CTRL_PROC_EXEC2_VPID;
694
695	if (pVM->hm.s.fHasIoApic)
696	val \|= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC;
697
698	if (pVM->hm.s.vmx.fUnrestrictedGuest)
699	val \|= VMX_VMCS_CTRL_PROC_EXEC2_UNRESTRICTED_GUEST;
700
701	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
702	val \|= VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP;
703
704	/* Mask away the bits that the CPU doesn't support */
705	/** @todo make sure they don't conflict with the above requirements. */
706	val &= pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1;
707	pVCpu->hm.s.vmx.u32ProcCtls2 = val;
708	rc = VMXWriteVmcs(VMX_VMCS32_CTRL_PROC_EXEC2, val);
709	AssertRC(rc);
710	}
711
712	/*
713	* VMX_VMCS_CTRL_CR3_TARGET_COUNT
714	* Set required bits to one and zero according to the MSR capabilities.
715	*/
716	rc = VMXWriteVmcs(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, 0);
717	AssertRC(rc);
718
719	/*
720	* Forward all exception except #NM & #PF to the guest.
721	* We always need to check pagefaults since our shadow page table can be out of sync.
722	* And we always lazily sync the FPU & XMM state. .
723	*/
724
725	/** @todo Possible optimization:
726	* Keep the FPU and XMM state current in the EM thread. That way there's no need to
727	* lazily sync anything, but the downside is that we can't use the FPU stack or XMM
728	* registers ourselves of course.
729	*
730	* Note: only possible if the current state is actually ours (X86_CR0_TS flag)
731	*/
732
733	/*
734	* Don't filter page faults, all of them should cause a world switch.
735	*/
736	rc = VMXWriteVmcs(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, 0);
737	AssertRC(rc);
738	rc = VMXWriteVmcs(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, 0);
739	AssertRC(rc);
740
741	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, 0);
742	AssertRC(rc);
743	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_A_FULL, 0);
744	AssertRC(rc);
745	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_B_FULL, 0);
746	AssertRC(rc);
747
748	/*
749	* Set the MSR bitmap address.
750	*/
751	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
752	{
753	Assert(pVCpu->hm.s.vmx.HCPhysMsrBitmap);
754
755	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, pVCpu->hm.s.vmx.HCPhysMsrBitmap);
756	AssertRC(rc);
757
758	/*
759	* Allow the guest to directly modify these MSRs; they are loaded/stored automatically
760	* using MSR-load/store areas in the VMCS.
761	*/
762	hmR0VmxSetMSRPermission(pVCpu, MSR_IA32_SYSENTER_CS, true, true);
763	hmR0VmxSetMSRPermission(pVCpu, MSR_IA32_SYSENTER_ESP, true, true);
764	hmR0VmxSetMSRPermission(pVCpu, MSR_IA32_SYSENTER_EIP, true, true);
765	hmR0VmxSetMSRPermission(pVCpu, MSR_K8_LSTAR, true, true);
766	hmR0VmxSetMSRPermission(pVCpu, MSR_K6_STAR, true, true);
767	hmR0VmxSetMSRPermission(pVCpu, MSR_K8_SF_MASK, true, true);
768	hmR0VmxSetMSRPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, true, true);
769	hmR0VmxSetMSRPermission(pVCpu, MSR_K8_GS_BASE, true, true);
770	hmR0VmxSetMSRPermission(pVCpu, MSR_K8_FS_BASE, true, true);
771	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
772	hmR0VmxSetMSRPermission(pVCpu, MSR_K8_TSC_AUX, true, true);
773	}
774
775	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
776	/*
777	* Set the guest & host MSR load/store physical addresses.
778	*/
779	Assert(pVCpu->hm.s.vmx.HCPhysGuestMsr);
780	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
781	AssertRC(rc);
782	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
783	AssertRC(rc);
784	Assert(pVCpu->hm.s.vmx.HCPhysHostMsr);
785	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysHostMsr);
786	AssertRC(rc);
787	#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
788
789	rc = VMXWriteVmcs(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, 0);
790	AssertRC(rc);
791	rc = VMXWriteVmcs(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, 0);
792	AssertRC(rc);
793	rc = VMXWriteVmcs(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, 0);
794	AssertRC(rc);
795
796	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
797	{
798	Assert(pVM->hm.s.vmx.hMemObjApicAccess);
799	/* Optional */
800	rc = VMXWriteVmcs(VMX_VMCS32_CTRL_TPR_THRESHOLD, 0);
801	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL, pVCpu->hm.s.vmx.HCPhysVirtApic);
802
803	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
804	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, pVM->hm.s.vmx.HCPhysApicAccess);
805
806	AssertRC(rc);
807	}
808
809	/* Set link pointer to -1. Not currently used. */
810	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, 0xFFFFFFFFFFFFFFFFULL);
811	AssertRC(rc);
812
813	/*
814	* Clear VMCS, marking it inactive. Clear implementation specific data and writing back
815	* VMCS data back to memory.
816	*/
817	rc = VMXClearVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
818	AssertRC(rc);
819
820	/*
821	* Configure the VMCS read cache.
822	*/
823	PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
824
825	VMXSetupCachedReadVmcs(pCache, VMX_VMCS_GUEST_RIP);
826	VMXSetupCachedReadVmcs(pCache, VMX_VMCS_GUEST_RSP);
827	VMXSetupCachedReadVmcs(pCache, VMX_VMCS_GUEST_RFLAGS);
828	VMXSetupCachedReadVmcs(pCache, VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE);
829	VMXSetupCachedReadVmcs(pCache, VMX_VMCS_CTRL_CR0_READ_SHADOW);
830	VMXSetupCachedReadVmcs(pCache, VMX_VMCS_GUEST_CR0);
831	VMXSetupCachedReadVmcs(pCache, VMX_VMCS_CTRL_CR4_READ_SHADOW);
832	VMXSetupCachedReadVmcs(pCache, VMX_VMCS_GUEST_CR4);
833	VMXSetupCachedReadVmcs(pCache, VMX_VMCS_GUEST_DR7);
834	VMXSetupCachedReadVmcs(pCache, VMX_VMCS32_GUEST_SYSENTER_CS);
835	VMXSetupCachedReadVmcs(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
836	VMXSetupCachedReadVmcs(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
837	VMXSetupCachedReadVmcs(pCache, VMX_VMCS32_GUEST_GDTR_LIMIT);
838	VMXSetupCachedReadVmcs(pCache, VMX_VMCS_GUEST_GDTR_BASE);
839	VMXSetupCachedReadVmcs(pCache, VMX_VMCS32_GUEST_IDTR_LIMIT);
840	VMXSetupCachedReadVmcs(pCache, VMX_VMCS_GUEST_IDTR_BASE);
841
842	VMX_SETUP_SELREG(ES, pCache);
843	VMX_SETUP_SELREG(SS, pCache);
844	VMX_SETUP_SELREG(CS, pCache);
845	VMX_SETUP_SELREG(DS, pCache);
846	VMX_SETUP_SELREG(FS, pCache);
847	VMX_SETUP_SELREG(GS, pCache);
848	VMX_SETUP_SELREG(LDTR, pCache);
849	VMX_SETUP_SELREG(TR, pCache);
850
851	/*
852	* Status code VMCS reads.
853	*/
854	VMXSetupCachedReadVmcs(pCache, VMX_VMCS32_RO_EXIT_REASON);
855	VMXSetupCachedReadVmcs(pCache, VMX_VMCS32_RO_VM_INSTR_ERROR);
856	VMXSetupCachedReadVmcs(pCache, VMX_VMCS32_RO_EXIT_INSTR_LENGTH);
857	VMXSetupCachedReadVmcs(pCache, VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE);
858	VMXSetupCachedReadVmcs(pCache, VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO);
859	VMXSetupCachedReadVmcs(pCache, VMX_VMCS32_RO_EXIT_INSTR_INFO);
860	VMXSetupCachedReadVmcs(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
861	VMXSetupCachedReadVmcs(pCache, VMX_VMCS32_RO_IDT_INFO);
862	VMXSetupCachedReadVmcs(pCache, VMX_VMCS32_RO_IDT_ERROR_CODE);
863
864	if (pVM->hm.s.fNestedPaging)
865	{
866	VMXSetupCachedReadVmcs(pCache, VMX_VMCS_GUEST_CR3);
867	VMXSetupCachedReadVmcs(pCache, VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL);
868	pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
869	}
870	else
871	pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
872	} /* for each VMCPU */
873
874	/*
875	* Setup the right TLB function based on CPU capabilities.
876	*/
877	if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
878	pVM->hm.s.vmx.pfnFlushTaggedTlb = hmR0VmxSetupTLBBoth;
879	else if (pVM->hm.s.fNestedPaging)
880	pVM->hm.s.vmx.pfnFlushTaggedTlb = hmR0VmxSetupTLBEPT;
881	else if (pVM->hm.s.vmx.fVpid)
882	pVM->hm.s.vmx.pfnFlushTaggedTlb = hmR0VmxSetupTLBVPID;
883	else
884	pVM->hm.s.vmx.pfnFlushTaggedTlb = hmR0VmxSetupTLBDummy;
885
886	vmx_end:
887	hmR0VmxCheckError(pVM, &pVM->aCpus[0], rc);
888	return rc;
889	}
890
891
892	/**
893	* Sets the permission bits for the specified MSR.
894	*
895	* @param pVCpu Pointer to the VMCPU.
896	* @param ulMSR The MSR value.
897	* @param fRead Whether reading is allowed.
898	* @param fWrite Whether writing is allowed.
899	*/
900	static void hmR0VmxSetMSRPermission(PVMCPU pVCpu, unsigned ulMSR, bool fRead, bool fWrite)
901	{
902	unsigned ulBit;
903	uint8_t pvMsrBitmap = (uint8_t )pVCpu->hm.s.vmx.pvMsrBitmap;
904
905	/*
906	* Layout:
907	* 0x000 - 0x3ff - Low MSR read bits
908	* 0x400 - 0x7ff - High MSR read bits
909	* 0x800 - 0xbff - Low MSR write bits
910	* 0xc00 - 0xfff - High MSR write bits
911	*/
912	if (ulMSR <= 0x00001FFF)
913	{
914	/* Pentium-compatible MSRs */
915	ulBit = ulMSR;
916	}
917	else if ( ulMSR >= 0xC0000000
918	&& ulMSR <= 0xC0001FFF)
919	{
920	/* AMD Sixth Generation x86 Processor MSRs */
921	ulBit = (ulMSR - 0xC0000000);
922	pvMsrBitmap += 0x400;
923	}
924	else
925	{
926	AssertFailed();
927	return;
928	}
929
930	Assert(ulBit <= 0x1fff);
931	if (fRead)
932	ASMBitClear(pvMsrBitmap, ulBit);
933	else
934	ASMBitSet(pvMsrBitmap, ulBit);
935
936	if (fWrite)
937	ASMBitClear(pvMsrBitmap + 0x800, ulBit);
938	else
939	ASMBitSet(pvMsrBitmap + 0x800, ulBit);
940	}
941
942
943	/**
944	* Injects an event (trap or external interrupt).
945	*
946	* @returns VBox status code. Note that it may return VINF_EM_RESET to
947	* indicate a triple fault when injecting X86_XCPT_DF.
948	*
949	* @param pVM Pointer to the VM.
950	* @param pVCpu Pointer to the VMCPU.
951	* @param pCtx Pointer to the guest CPU Context.
952	* @param intInfo VMX interrupt info.
953	* @param cbInstr Opcode length of faulting instruction.
954	* @param errCode Error code (optional).
955	*/
956	static int hmR0VmxInjectEvent(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, uint32_t intInfo, uint32_t cbInstr, uint32_t errCode)
957	{
958	int rc;
959	uint32_t iGate = VMX_EXIT_INTERRUPTION_INFO_VECTOR(intInfo);
960
961	#ifdef VBOX_WITH_STATISTICS
962	STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[iGate & MASK_INJECT_IRQ_STAT]);
963	#endif
964
965	#ifdef VBOX_STRICT
966	if (iGate == 0xE)
967	{
968	LogFlow(("hmR0VmxInjectEvent: Injecting interrupt %d at %RGv error code=%08x CR2=%RGv intInfo=%08x\n", iGate,
969	(RTGCPTR)pCtx->rip, errCode, pCtx->cr2, intInfo));
970	}
971	else if (iGate < 0x20)
972	{
973	LogFlow(("hmR0VmxInjectEvent: Injecting interrupt %d at %RGv error code=%08x\n", iGate, (RTGCPTR)pCtx->rip,
974	errCode));
975	}
976	else
977	{
978	LogFlow(("INJ-EI: %x at %RGv\n", iGate, (RTGCPTR)pCtx->rip));
979	Assert( VMX_EXIT_INTERRUPTION_INFO_TYPE(intInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT
980	\|\| !VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
981	Assert( VMX_EXIT_INTERRUPTION_INFO_TYPE(intInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT
982	\|\| pCtx->eflags.u32 & X86_EFL_IF);
983	}
984	#endif
985
986	if ( CPUMIsGuestInRealModeEx(pCtx)
987	&& pVM->hm.s.vmx.pRealModeTSS)
988	{
989	RTGCPHYS GCPhysHandler;
990	uint16_t offset, ip;
991	RTSEL sel;
992
993	/*
994	* Injecting events doesn't work right with real mode emulation.
995	* (#GP if we try to inject external hardware interrupts)
996	* Inject the interrupt or trap directly instead.
997	*
998	* ASSUMES no access handlers for the bits we read or write below (should be safe).
999	*/
1000	Log(("Manual interrupt/trap '%x' inject (real mode)\n", iGate));
1001
1002	/*
1003	* Check if the interrupt handler is present.
1004	*/
1005	if (iGate * 4 + 3 > pCtx->idtr.cbIdt)
1006	{
1007	Log(("IDT cbIdt violation\n"));
1008	if (iGate != X86_XCPT_DF)
1009	{
1010	uint32_t intInfo2;
1011
1012	intInfo2 = (iGate == X86_XCPT_GP) ? (uint32_t)X86_XCPT_DF : (uint32_t)X86_XCPT_GP;
1013	intInfo2 \|= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
1014	intInfo2 \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
1015	intInfo2 \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
1016
1017	return hmR0VmxInjectEvent(pVM, pVCpu, pCtx, intInfo2, 0, 0 /* no error code according to the Intel docs */);
1018	}
1019	Log(("Triple fault -> reset the VM!\n"));
1020	return VINF_EM_RESET;
1021	}
1022	if ( VMX_EXIT_INTERRUPTION_INFO_TYPE(intInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT
1023	\|\| iGate == 3 /* Both #BP and #OF point to the instruction after. */
1024	\|\| iGate == 4)
1025	{
1026	ip = pCtx->ip + cbInstr;
1027	}
1028	else
1029	ip = pCtx->ip;
1030
1031	/*
1032	* Read the selector:offset pair of the interrupt handler.
1033	*/
1034	GCPhysHandler = (RTGCPHYS)pCtx->idtr.pIdt + iGate * 4;
1035	rc = PGMPhysSimpleReadGCPhys(pVM, &offset, GCPhysHandler, sizeof(offset)); AssertRC(rc);
1036	rc = PGMPhysSimpleReadGCPhys(pVM, &sel, GCPhysHandler + 2, sizeof(sel)); AssertRC(rc);
1037
1038	LogFlow(("IDT handler %04X:%04X\n", sel, offset));
1039
1040	/*
1041	* Construct the stack frame.
1042	*/
1043	/** @todo Check stack limit. */
1044	pCtx->sp -= 2;
1045	LogFlow(("ss:sp %04X:%04X eflags=%x\n", pCtx->ss.Sel, pCtx->sp, pCtx->eflags.u));
1046	rc = PGMPhysSimpleWriteGCPhys(pVM, pCtx->ss.u64Base + pCtx->sp, &pCtx->eflags, sizeof(uint16_t)); AssertRC(rc);
1047	pCtx->sp -= 2;
1048	LogFlow(("ss:sp %04X:%04X cs=%x\n", pCtx->ss.Sel, pCtx->sp, pCtx->cs.Sel));
1049	rc = PGMPhysSimpleWriteGCPhys(pVM, pCtx->ss.u64Base + pCtx->sp, &pCtx->cs, sizeof(uint16_t)); AssertRC(rc);
1050	pCtx->sp -= 2;
1051	LogFlow(("ss:sp %04X:%04X ip=%x\n", pCtx->ss.Sel, pCtx->sp, ip));
1052	rc = PGMPhysSimpleWriteGCPhys(pVM, pCtx->ss.u64Base + pCtx->sp, &ip, sizeof(ip)); AssertRC(rc);
1053
1054	/*
1055	* Update the CPU state for executing the handler.
1056	*/
1057	pCtx->rip = offset;
1058	pCtx->cs.Sel = sel;
1059	pCtx->cs.u64Base = sel << 4;
1060	pCtx->eflags.u &= ~(X86_EFL_IF \| X86_EFL_TF \| X86_EFL_RF \| X86_EFL_AC);
1061
1062	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SEGMENT_REGS;
1063	return VINF_SUCCESS;
1064	}
1065
1066	/*
1067	* Set event injection state.
1068	*/
1069	rc = VMXWriteVmcs(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, intInfo \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT));
1070	rc \|= VMXWriteVmcs(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
1071	rc \|= VMXWriteVmcs(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, errCode);
1072
1073	AssertRC(rc);
1074	return rc;
1075	}
1076
1077
1078	/**
1079	* Checks for pending guest interrupts and injects them.
1080	*
1081	* @returns VBox status code.
1082	* @param pVM Pointer to the VM.
1083	* @param pVCpu Pointer to the VMCPU.
1084	* @param pCtx Pointer to the guest CPU context.
1085	*/
1086	static int hmR0VmxCheckPendingInterrupt(PVM pVM, PVMCPU pVCpu, CPUMCTX *pCtx)
1087	{
1088	int rc;
1089
1090	/*
1091	* Dispatch any pending interrupts (injected before, but a VM exit occurred prematurely).
1092	*/
1093	if (pVCpu->hm.s.Event.fPending)
1094	{
1095	Log(("CPU%d: Reinjecting event %RX64 %08x at %RGv cr2=%RX64\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntrInfo,
1096	pVCpu->hm.s.Event.u32ErrCode, (RTGCPTR)pCtx->rip, pCtx->cr2));
1097	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
1098	rc = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, pVCpu->hm.s.Event.u64IntrInfo, 0, pVCpu->hm.s.Event.u32ErrCode);
1099	AssertRC(rc);
1100
1101	pVCpu->hm.s.Event.fPending = false;
1102	return VINF_SUCCESS;
1103	}
1104
1105	/*
1106	* If an active trap is already pending, we must forward it first!
1107	*/
1108	if (!TRPMHasTrap(pVCpu))
1109	{
1110	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI))
1111	{
1112	RTGCUINTPTR intInfo;
1113
1114	Log(("CPU%d: injecting #NMI\n", pVCpu->idCpu));
1115
1116	intInfo = X86_XCPT_NMI;
1117	intInfo \|= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
1118	intInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
1119
1120	rc = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, intInfo, 0, 0);
1121	AssertRC(rc);
1122
1123	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
1124	return VINF_SUCCESS;
1125	}
1126
1127	/** @todo SMI interrupts. */
1128
1129	/*
1130	* When external interrupts are pending, we should exit the VM when IF is set.
1131	*/
1132	if (VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC\|VMCPU_FF_INTERRUPT_PIC)))
1133	{
1134	if (!(pCtx->eflags.u32 & X86_EFL_IF))
1135	{
1136	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
1137	{
1138	LogFlow(("Enable irq window exit!\n"));
1139	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
1140	rc = VMXWriteVmcs(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
1141	AssertRC(rc);
1142	}
1143	/* else nothing to do but wait */
1144	}
1145	else if (!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
1146	{
1147	uint8_t u8Interrupt;
1148
1149	rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
1150	Log(("CPU%d: Dispatch interrupt: u8Interrupt=%x (%d) rc=%Rrc cs:rip=%04X:%RGv\n", pVCpu->idCpu,
1151	u8Interrupt, u8Interrupt, rc, pCtx->cs.Sel, (RTGCPTR)pCtx->rip));
1152	if (RT_SUCCESS(rc))
1153	{
1154	rc = TRPMAssertTrap(pVCpu, u8Interrupt, TRPM_HARDWARE_INT);
1155	AssertRC(rc);
1156	}
1157	else
1158	{
1159	/* Can only happen in rare cases where a pending interrupt is cleared behind our back */
1160	Assert(!VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC\|VMCPU_FF_INTERRUPT_PIC)));
1161	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
1162	/* Just continue */
1163	}
1164	}
1165	else
1166	Log(("Pending interrupt blocked at %RGv by VM_FF_INHIBIT_INTERRUPTS!!\n", (RTGCPTR)pCtx->rip));
1167	}
1168	}
1169
1170	#ifdef VBOX_STRICT
1171	if (TRPMHasTrap(pVCpu))
1172	{
1173	uint8_t u8Vector;
1174	rc = TRPMQueryTrapAll(pVCpu, &u8Vector, 0, NULL, NULL, NULL);
1175	AssertRC(rc);
1176	}
1177	#endif
1178
1179	if ( (pCtx->eflags.u32 & X86_EFL_IF)
1180	&& (!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
1181	&& TRPMHasTrap(pVCpu)
1182	)
1183	{
1184	uint8_t u8Vector;
1185	TRPMEVENT enmType;
1186	RTGCUINTPTR intInfo;
1187	RTGCUINT errCode;
1188
1189	/*
1190	* If a new event is pending, dispatch it now.
1191	*/
1192	rc = TRPMQueryTrapAll(pVCpu, &u8Vector, &enmType, &errCode, NULL, NULL);
1193	AssertRC(rc);
1194	Assert(pCtx->eflags.Bits.u1IF == 1 \|\| enmType == TRPM_TRAP);
1195	Assert(enmType != TRPM_SOFTWARE_INT);
1196
1197	/*
1198	* Clear the pending trap.
1199	*/
1200	rc = TRPMResetTrap(pVCpu);
1201	AssertRC(rc);
1202
1203	intInfo = u8Vector;
1204	intInfo \|= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
1205
1206	if (enmType == TRPM_TRAP)
1207	{
1208	switch (u8Vector)
1209	{
1210	case X86_XCPT_DF:
1211	case X86_XCPT_TS:
1212	case X86_XCPT_NP:
1213	case X86_XCPT_SS:
1214	case X86_XCPT_GP:
1215	case X86_XCPT_PF:
1216	case X86_XCPT_AC:
1217	{
1218	/** @todo r=ramshankar: setting this bit would blow up for real-mode guests with
1219	* unrestricted guest execution. */
1220	/* Valid error codes. */
1221	intInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
1222	break;
1223	}
1224
1225	default:
1226	break;
1227	}
1228
1229	if ( u8Vector == X86_XCPT_BP
1230	\|\| u8Vector == X86_XCPT_OF)
1231	{
1232	intInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
1233	}
1234	else
1235	intInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
1236
1237	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
1238	}
1239	else
1240	{
1241	intInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
1242	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
1243	}
1244
1245	rc = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, intInfo, 0, errCode);
1246	AssertRC(rc);
1247	} /* if (interrupts can be dispatched) */
1248
1249	return VINF_SUCCESS;
1250	}
1251
1252	/**
1253	* Checks for pending VMX events and converts them to TRPM. Before we execute any instruction
1254	* outside of VMX, any pending VMX event must be converted so that it can be delivered properly.
1255	*
1256	* @returns VBox status code.
1257	* @param pVCpu Pointer to the VMCPU.
1258	*/
1259	static int hmR0VmxCheckPendingEvent(PVMCPU pVCpu)
1260	{
1261	if (pVCpu->hm.s.Event.fPending)
1262	{
1263	TRPMEVENT enmTrapType;
1264
1265	/* If a trap was already pending, we did something wrong! */
1266	Assert((TRPMQueryTrap(pVCpu, NULL, NULL) == VERR_TRPM_NO_ACTIVE_TRAP));
1267
1268	/*
1269	* Clear the pending event and move it over to TRPM for the rest
1270	* of the world to see.
1271	*/
1272	pVCpu->hm.s.Event.fPending = false;
1273	switch (VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hm.s.Event.u64IntrInfo))
1274	{
1275	case VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT:
1276	case VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI:
1277	enmTrapType = TRPM_HARDWARE_INT;
1278	break;
1279	case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT:
1280	case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT: /** @todo Is classifying #BP, #OF as TRPM_SOFTWARE_INT correct? */
1281	case VMX_EXIT_INTERRUPTION_INFO_TYPE_DB_XCPT:
1282	enmTrapType = TRPM_SOFTWARE_INT;
1283	break;
1284	case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
1285	enmTrapType = TRPM_TRAP;
1286	break;
1287	default:
1288	enmTrapType = TRPM_32BIT_HACK; /* Can't get here. */
1289	AssertFailed();
1290	}
1291	TRPMAssertTrap(pVCpu, VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVCpu->hm.s.Event.u64IntrInfo), enmTrapType);
1292	if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntrInfo))
1293	TRPMSetErrorCode(pVCpu, pVCpu->hm.s.Event.u32ErrCode);
1294	//@todo: Is there any situation where we need to call TRPMSetFaultAddress()?
1295	}
1296	return VINF_SUCCESS;
1297	}
1298
1299	/**
1300	* Save the host state into the VMCS.
1301	*
1302	* @returns VBox status code.
1303	* @param pVM Pointer to the VM.
1304	* @param pVCpu Pointer to the VMCPU.
1305	*/
1306	VMMR0DECL(int) VMXR0SaveHostState(PVM pVM, PVMCPU pVCpu)
1307	{
1308	int rc = VINF_SUCCESS;
1309	NOREF(pVM);
1310
1311	/*
1312	* Host CPU Context.
1313	*/
1314	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_HOST_CONTEXT)
1315	{
1316	RTIDTR idtr;
1317	RTGDTR gdtr;
1318	RTSEL SelTR;
1319	PCX86DESCHC pDesc;
1320	uintptr_t trBase;
1321	RTSEL cs;
1322	RTSEL ss;
1323	uint64_t cr3;
1324
1325	/*
1326	* Control registers.
1327	*/
1328	rc = VMXWriteVmcs(VMX_VMCS_HOST_CR0, ASMGetCR0());
1329	Log2(("VMX_VMCS_HOST_CR0 %08x\n", ASMGetCR0()));
1330	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
1331	if (VMX_IS_64BIT_HOST_MODE())
1332	{
1333	cr3 = HMR0Get64bitCR3();
1334	rc \|= VMXWriteVmcs64(VMX_VMCS_HOST_CR3, cr3);
1335	}
1336	else
1337	#endif
1338	{
1339	cr3 = ASMGetCR3();
1340	rc \|= VMXWriteVmcs(VMX_VMCS_HOST_CR3, cr3);
1341	}
1342	Log2(("VMX_VMCS_HOST_CR3 %08RX64\n", cr3));
1343	rc \|= VMXWriteVmcs(VMX_VMCS_HOST_CR4, ASMGetCR4());
1344	Log2(("VMX_VMCS_HOST_CR4 %08x\n", ASMGetCR4()));
1345	AssertRC(rc);
1346
1347	/*
1348	* Selector registers.
1349	*/
1350	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
1351	if (VMX_IS_64BIT_HOST_MODE())
1352	{
1353	cs = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelCS;
1354	ss = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelSS;
1355	}
1356	else
1357	{
1358	/* sysenter loads LDT cs & ss, VMX doesn't like this. Load the GDT ones (safe). */
1359	cs = (RTSEL)(uintptr_t)&SUPR0AbsKernelCS;
1360	ss = (RTSEL)(uintptr_t)&SUPR0AbsKernelSS;
1361	}
1362	#else
1363	cs = ASMGetCS();
1364	ss = ASMGetSS();
1365	#endif
1366	Assert(!(cs & X86_SEL_LDT)); Assert((cs & X86_SEL_RPL) == 0);
1367	Assert(!(ss & X86_SEL_LDT)); Assert((ss & X86_SEL_RPL) == 0);
1368	rc = VMXWriteVmcs(VMX_VMCS16_HOST_FIELD_CS, cs);
1369	/* Note: VMX is (again) very picky about the RPL of the selectors here; we'll restore them manually. */
1370	rc \|= VMXWriteVmcs(VMX_VMCS16_HOST_FIELD_DS, 0);
1371	rc \|= VMXWriteVmcs(VMX_VMCS16_HOST_FIELD_ES, 0);
1372	#if HC_ARCH_BITS == 32
1373	if (!VMX_IS_64BIT_HOST_MODE())
1374	{
1375	rc \|= VMXWriteVmcs(VMX_VMCS16_HOST_FIELD_FS, 0);
1376	rc \|= VMXWriteVmcs(VMX_VMCS16_HOST_FIELD_GS, 0);
1377	}
1378	#endif
1379	rc \|= VMXWriteVmcs(VMX_VMCS16_HOST_FIELD_SS, ss);
1380	SelTR = ASMGetTR();
1381	rc \|= VMXWriteVmcs(VMX_VMCS16_HOST_FIELD_TR, SelTR);
1382	AssertRC(rc);
1383	Log2(("VMX_VMCS_HOST_FIELD_CS %08x (%08x)\n", cs, ASMGetSS()));
1384	Log2(("VMX_VMCS_HOST_FIELD_DS 00000000 (%08x)\n", ASMGetDS()));
1385	Log2(("VMX_VMCS_HOST_FIELD_ES 00000000 (%08x)\n", ASMGetES()));
1386	Log2(("VMX_VMCS_HOST_FIELD_FS 00000000 (%08x)\n", ASMGetFS()));
1387	Log2(("VMX_VMCS_HOST_FIELD_GS 00000000 (%08x)\n", ASMGetGS()));
1388	Log2(("VMX_VMCS_HOST_FIELD_SS %08x (%08x)\n", ss, ASMGetSS()));
1389	Log2(("VMX_VMCS_HOST_FIELD_TR %08x\n", ASMGetTR()));
1390
1391	/*
1392	* GDTR & IDTR.
1393	*/
1394	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
1395	if (VMX_IS_64BIT_HOST_MODE())
1396	{
1397	X86XDTR64 gdtr64, idtr64;
1398	HMR0Get64bitGdtrAndIdtr(&gdtr64, &idtr64);
1399	rc = VMXWriteVmcs64(VMX_VMCS_HOST_GDTR_BASE, gdtr64.uAddr);
1400	rc \|= VMXWriteVmcs64(VMX_VMCS_HOST_IDTR_BASE, idtr64.uAddr);
1401	AssertRC(rc);
1402	Log2(("VMX_VMCS_HOST_GDTR_BASE %RX64\n", gdtr64.uAddr));
1403	Log2(("VMX_VMCS_HOST_IDTR_BASE %RX64\n", idtr64.uAddr));
1404	gdtr.cbGdt = gdtr64.cb;
1405	gdtr.pGdt = (uintptr_t)gdtr64.uAddr;
1406	}
1407	else
1408	#endif
1409	{
1410	ASMGetGDTR(&gdtr);
1411	rc = VMXWriteVmcs(VMX_VMCS_HOST_GDTR_BASE, gdtr.pGdt);
1412	ASMGetIDTR(&idtr);
1413	rc \|= VMXWriteVmcs(VMX_VMCS_HOST_IDTR_BASE, idtr.pIdt);
1414	AssertRC(rc);
1415	Log2(("VMX_VMCS_HOST_GDTR_BASE %RHv\n", gdtr.pGdt));
1416	Log2(("VMX_VMCS_HOST_IDTR_BASE %RHv\n", idtr.pIdt));
1417	}
1418
1419	/*
1420	* Save the base address of the TR selector.
1421	*/
1422	if (SelTR > gdtr.cbGdt)
1423	{
1424	AssertMsgFailed(("Invalid TR selector %x. GDTR.cbGdt=%x\n", SelTR, gdtr.cbGdt));
1425	return VERR_VMX_INVALID_HOST_STATE;
1426	}
1427
1428	pDesc = (PCX86DESCHC)(gdtr.pGdt + (SelTR & X86_SEL_MASK));
1429	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
1430	if (VMX_IS_64BIT_HOST_MODE())
1431	{
1432	uint64_t trBase64 = X86DESC64_BASE((PX86DESC64)pDesc);
1433	rc = VMXWriteVmcs64(VMX_VMCS_HOST_TR_BASE, trBase64);
1434	Log2(("VMX_VMCS_HOST_TR_BASE %RX64\n", trBase64));
1435	AssertRC(rc);
1436	}
1437	else
1438	#endif
1439	{
1440	#if HC_ARCH_BITS == 64
1441	trBase = X86DESC64_BASE(pDesc);
1442	#else
1443	trBase = X86DESC_BASE(pDesc);
1444	#endif
1445	rc = VMXWriteVmcs(VMX_VMCS_HOST_TR_BASE, trBase);
1446	AssertRC(rc);
1447	Log2(("VMX_VMCS_HOST_TR_BASE %RHv\n", trBase));
1448	}
1449
1450	/*
1451	* FS base and GS base.
1452	*/
1453	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
1454	if (VMX_IS_64BIT_HOST_MODE())
1455	{
1456	Log2(("MSR_K8_FS_BASE = %RX64\n", ASMRdMsr(MSR_K8_FS_BASE)));
1457	Log2(("MSR_K8_GS_BASE = %RX64\n", ASMRdMsr(MSR_K8_GS_BASE)));
1458	rc = VMXWriteVmcs64(VMX_VMCS_HOST_FS_BASE, ASMRdMsr(MSR_K8_FS_BASE));
1459	rc \|= VMXWriteVmcs64(VMX_VMCS_HOST_GS_BASE, ASMRdMsr(MSR_K8_GS_BASE));
1460	}
1461	#endif
1462	AssertRC(rc);
1463
1464	/*
1465	* Sysenter MSRs.
1466	*/
1467	/** @todo expensive!! */
1468	rc = VMXWriteVmcs(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
1469	Log2(("VMX_VMCS_HOST_SYSENTER_CS %08x\n", ASMRdMsr_Low(MSR_IA32_SYSENTER_CS)));
1470	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
1471	if (VMX_IS_64BIT_HOST_MODE())
1472	{
1473	Log2(("VMX_VMCS_HOST_SYSENTER_EIP %RX64\n", ASMRdMsr(MSR_IA32_SYSENTER_EIP)));
1474	Log2(("VMX_VMCS_HOST_SYSENTER_ESP %RX64\n", ASMRdMsr(MSR_IA32_SYSENTER_ESP)));
1475	rc \|= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
1476	rc \|= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
1477	}
1478	else
1479	{
1480	rc \|= VMXWriteVmcs(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
1481	rc \|= VMXWriteVmcs(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
1482	Log2(("VMX_VMCS_HOST_SYSENTER_EIP %RX32\n", ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP)));
1483	Log2(("VMX_VMCS_HOST_SYSENTER_ESP %RX32\n", ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP)));
1484	}
1485	#elif HC_ARCH_BITS == 32
1486	rc \|= VMXWriteVmcs(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
1487	rc \|= VMXWriteVmcs(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
1488	Log2(("VMX_VMCS_HOST_SYSENTER_EIP %RX32\n", ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP)));
1489	Log2(("VMX_VMCS_HOST_SYSENTER_ESP %RX32\n", ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP)));
1490	#else
1491	Log2(("VMX_VMCS_HOST_SYSENTER_EIP %RX64\n", ASMRdMsr(MSR_IA32_SYSENTER_EIP)));
1492	Log2(("VMX_VMCS_HOST_SYSENTER_ESP %RX64\n", ASMRdMsr(MSR_IA32_SYSENTER_ESP)));
1493	rc \|= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
1494	rc \|= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
1495	#endif
1496	AssertRC(rc);
1497
1498
1499	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
1500	/*
1501	* Store all host MSRs in the VM-Exit load area, so they will be reloaded after
1502	* the world switch back to the host.
1503	*/
1504	PVMXMSR pMsr = (PVMXMSR)pVCpu->hm.s.vmx.pvHostMsr;
1505	unsigned idxMsr = 0;
1506
1507	uint32_t u32HostExtFeatures = ASMCpuId_EDX(0x80000001);
1508	if (u32HostExtFeatures & (X86_CPUID_EXT_FEATURE_EDX_NX \| X86_CPUID_EXT_FEATURE_EDX_LONG_MODE))
1509	{
1510	pMsr->u32IndexMSR = MSR_K6_EFER;
1511	pMsr->u32Reserved = 0;
1512	# if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
1513	if (CPUMIsGuestInLongMode(pVCpu))
1514	{
1515	/* Must match the EFER value in our 64 bits switcher. */
1516	pMsr->u64Value = ASMRdMsr(MSR_K6_EFER) \| MSR_K6_EFER_LME \| MSR_K6_EFER_SCE \| MSR_K6_EFER_NXE;
1517	}
1518	else
1519	# endif
1520	pMsr->u64Value = ASMRdMsr(MSR_K6_EFER);
1521	pMsr++; idxMsr++;
1522	}
1523
1524	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
1525	if (VMX_IS_64BIT_HOST_MODE())
1526	{
1527	pMsr->u32IndexMSR = MSR_K6_STAR;
1528	pMsr->u32Reserved = 0;
1529	pMsr->u64Value = ASMRdMsr(MSR_K6_STAR); /* legacy syscall eip, cs & ss */
1530	pMsr++; idxMsr++;
1531	pMsr->u32IndexMSR = MSR_K8_LSTAR;
1532	pMsr->u32Reserved = 0;
1533	pMsr->u64Value = ASMRdMsr(MSR_K8_LSTAR); /* 64 bits mode syscall rip */
1534	pMsr++; idxMsr++;
1535	pMsr->u32IndexMSR = MSR_K8_SF_MASK;
1536	pMsr->u32Reserved = 0;
1537	pMsr->u64Value = ASMRdMsr(MSR_K8_SF_MASK); /* syscall flag mask */
1538	pMsr++; idxMsr++;
1539	# if 0
1540	/* The KERNEL_GS_BASE MSR does not work reliably with auto load/store. See @bugref{6208} */
1541	pMsr->u32IndexMSR = MSR_K8_KERNEL_GS_BASE;
1542	pMsr->u32Reserved = 0;
1543	pMsr->u64Value = ASMRdMsr(MSR_K8_KERNEL_GS_BASE); /* swapgs exchange value */
1544	pMsr++; idxMsr++;
1545	# endif
1546	}
1547	# endif
1548
1549	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
1550	{
1551	pMsr->u32IndexMSR = MSR_K8_TSC_AUX;
1552	pMsr->u32Reserved = 0;
1553	pMsr->u64Value = ASMRdMsr(MSR_K8_TSC_AUX);
1554	pMsr++; idxMsr++;
1555	}
1556
1557	/** @todo r=ramshankar: check IA32_VMX_MISC bits 27:25 for valid idxMsr
1558	* range. */
1559	rc = VMXWriteVmcs(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, idxMsr);
1560	AssertRC(rc);
1561	#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
1562
1563	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_HOST_CONTEXT;
1564	}
1565	return rc;
1566	}
1567
1568
1569	/**
1570	* Loads the 4 PDPEs into the guest state when nested paging is used and the
1571	* guest operates in PAE mode.
1572	*
1573	* @returns VBox status code.
1574	* @param pVCpu Pointer to the VMCPU.
1575	* @param pCtx Pointer to the guest CPU context.
1576	*/
1577	static int hmR0VmxLoadPaePdpes(PVMCPU pVCpu, PCPUMCTX pCtx)
1578	{
1579	if (CPUMIsGuestInPAEModeEx(pCtx))
1580	{
1581	X86PDPE aPdpes[4];
1582	int rc = PGMGstGetPaePdpes(pVCpu, &aPdpes[0]);
1583	AssertRCReturn(rc, rc);
1584
1585	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, aPdpes[0].u); AssertRCReturn(rc, rc);
1586	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, aPdpes[1].u); AssertRCReturn(rc, rc);
1587	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, aPdpes[2].u); AssertRCReturn(rc, rc);
1588	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, aPdpes[3].u); AssertRCReturn(rc, rc);
1589	}
1590	return VINF_SUCCESS;
1591	}
1592
1593
1594	/**
1595	* Saves the 4 PDPEs into the guest state when nested paging is used and the
1596	* guest operates in PAE mode.
1597	*
1598	* @returns VBox status code.
1599	* @param pVCpu Pointer to the VM CPU.
1600	* @param pCtx Pointer to the guest CPU context.
1601	*
1602	* @remarks Tell PGM about CR3 changes before calling this helper.
1603	*/
1604	static int hmR0VmxSavePaePdpes(PVMCPU pVCpu, PCPUMCTX pCtx)
1605	{
1606	if (CPUMIsGuestInPAEModeEx(pCtx))
1607	{
1608	int rc;
1609	X86PDPE aPdpes[4];
1610	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &aPdpes[0].u); AssertRCReturn(rc, rc);
1611	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &aPdpes[1].u); AssertRCReturn(rc, rc);
1612	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &aPdpes[2].u); AssertRCReturn(rc, rc);
1613	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &aPdpes[3].u); AssertRCReturn(rc, rc);
1614
1615	rc = PGMGstUpdatePaePdpes(pVCpu, &aPdpes[0]);
1616	AssertRCReturn(rc, rc);
1617	}
1618	return VINF_SUCCESS;
1619	}
1620
1621
1622	/**
1623	* Update the exception bitmap according to the current CPU state.
1624	*
1625	* @param pVM Pointer to the VM.
1626	* @param pVCpu Pointer to the VMCPU.
1627	* @param pCtx Pointer to the guest CPU context.
1628	*/
1629	static void hmR0VmxUpdateExceptionBitmap(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
1630	{
1631	uint32_t u32TrapMask;
1632	Assert(pCtx);
1633
1634	/*
1635	* Set up a mask for intercepting traps.
1636	*/
1637	/** @todo Do we really need to always intercept #DB? */
1638	u32TrapMask = RT_BIT(X86_XCPT_DB)
1639	\| RT_BIT(X86_XCPT_NM)
1640	#ifdef VBOX_ALWAYS_TRAP_PF
1641	\| RT_BIT(X86_XCPT_PF)
1642	#endif
1643	#ifdef VBOX_STRICT
1644	\| RT_BIT(X86_XCPT_BP)
1645	\| RT_BIT(X86_XCPT_DB)
1646	\| RT_BIT(X86_XCPT_DE)
1647	\| RT_BIT(X86_XCPT_NM)
1648	\| RT_BIT(X86_XCPT_UD)
1649	\| RT_BIT(X86_XCPT_NP)
1650	\| RT_BIT(X86_XCPT_SS)
1651	\| RT_BIT(X86_XCPT_GP)
1652	\| RT_BIT(X86_XCPT_MF)
1653	#endif
1654	;
1655
1656	/*
1657	* Without nested paging, #PF must be intercepted to implement shadow paging.
1658	*/
1659	/** @todo NP state won't change so maybe we should build the initial trap mask up front? */
1660	if (!pVM->hm.s.fNestedPaging)
1661	u32TrapMask \|= RT_BIT(X86_XCPT_PF);
1662
1663	/* Catch floating point exceptions if we need to report them to the guest in a different way. */
1664	if (!(pCtx->cr0 & X86_CR0_NE))
1665	u32TrapMask \|= RT_BIT(X86_XCPT_MF);
1666
1667	#ifdef VBOX_STRICT
1668	Assert(u32TrapMask & RT_BIT(X86_XCPT_GP));
1669	#endif
1670
1671	/*
1672	* Intercept all exceptions in real mode as none of them can be injected directly (#GP otherwise).
1673	*/
1674	/** @todo Despite the claim to intercept everything, with NP we do not intercept #PF. Should we? */
1675	if ( CPUMIsGuestInRealModeEx(pCtx)
1676	&& pVM->hm.s.vmx.pRealModeTSS)
1677	{
1678	u32TrapMask \|= RT_BIT(X86_XCPT_DE)
1679	\| RT_BIT(X86_XCPT_DB)
1680	\| RT_BIT(X86_XCPT_NMI)
1681	\| RT_BIT(X86_XCPT_BP)
1682	\| RT_BIT(X86_XCPT_OF)
1683	\| RT_BIT(X86_XCPT_BR)
1684	\| RT_BIT(X86_XCPT_UD)
1685	\| RT_BIT(X86_XCPT_DF)
1686	\| RT_BIT(X86_XCPT_CO_SEG_OVERRUN)
1687	\| RT_BIT(X86_XCPT_TS)
1688	\| RT_BIT(X86_XCPT_NP)
1689	\| RT_BIT(X86_XCPT_SS)
1690	\| RT_BIT(X86_XCPT_GP)
1691	\| RT_BIT(X86_XCPT_MF)
1692	\| RT_BIT(X86_XCPT_AC)
1693	\| RT_BIT(X86_XCPT_MC)
1694	\| RT_BIT(X86_XCPT_XF)
1695	;
1696	}
1697
1698	int rc = VMXWriteVmcs(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32TrapMask);
1699	AssertRC(rc);
1700	}
1701
1702
1703	/**
1704	* Loads a minimal guest state.
1705	*
1706	* NOTE: Don't do anything here that can cause a jump back to ring 3!!!!!
1707	*
1708	* @param pVM Pointer to the VM.
1709	* @param pVCpu Pointer to the VMCPU.
1710	* @param pCtx Pointer to the guest CPU context.
1711	*/
1712	VMMR0DECL(void) VMXR0LoadMinimalGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
1713	{
1714	int rc;
1715	X86EFLAGS eflags;
1716
1717	Assert(!(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_ALL_GUEST));
1718
1719	/*
1720	* Load EIP, ESP and EFLAGS.
1721	*/
1722	rc = VMXWriteVmcs64(VMX_VMCS_GUEST_RIP, pCtx->rip);
1723	rc \|= VMXWriteVmcs64(VMX_VMCS_GUEST_RSP, pCtx->rsp);
1724	AssertRC(rc);
1725
1726	/*
1727	* Bits 22-31, 15, 5 & 3 must be zero. Bit 1 must be 1.
1728	*/
1729	eflags = pCtx->eflags;
1730	eflags.u32 &= VMX_EFLAGS_RESERVED_0;
1731	eflags.u32 \|= VMX_EFLAGS_RESERVED_1;
1732
1733	/*
1734	* Check if real mode emulation using v86 mode.
1735	*/
1736	if ( CPUMIsGuestInRealModeEx(pCtx)
1737	&& pVM->hm.s.vmx.pRealModeTSS)
1738	{
1739	pVCpu->hm.s.vmx.RealMode.eflags = eflags;
1740
1741	eflags.Bits.u1VM = 1;
1742	eflags.Bits.u2IOPL = 0; /* must always be 0 or else certain instructions won't cause faults. */
1743	}
1744	rc = VMXWriteVmcs(VMX_VMCS_GUEST_RFLAGS, eflags.u32);
1745	AssertRC(rc);
1746	}
1747
1748
1749	/**
1750	* Sets up TSC offsetting and VMX preemption, if supported/configured/available.
1751	*
1752	* NOTE: This function reads the host TSC value. Therefore it must be executed very
1753	* shortly before a VM entry and execution MUST NOT be rescheduled between a call to
1754	* this function and a VM entry without calling this function again.
1755	*
1756	* @returns VBox status code.
1757	* @param pVM Pointer to the VM.
1758	* @param pVCpu Pointer to the VMCPU.
1759	*/
1760	VMMR0DECL(int) VMXR0SetupTscOffsetAndPreemption(PVM pVM, PVMCPU pVCpu)
1761	{
1762	int rc = VINF_SUCCESS;
1763	bool fOffsettedTsc;
1764
1765	if (pVM->hm.s.vmx.fUsePreemptTimer)
1766	{
1767	uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVCpu, &fOffsettedTsc, &pVCpu->hm.s.vmx.u64TSCOffset);
1768
1769	/* Make sure the returned values have sane upper and lower boundaries. */
1770	uint64_t u64CpuHz = SUPGetCpuHzFromGIP(g_pSUPGlobalInfoPage);
1771
1772	cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64 of a second */
1773	cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
1774
1775	cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
1776	uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
1777	rc = VMXWriteVmcs(VMX_VMCS32_GUEST_PREEMPT_TIMER_VALUE, cPreemptionTickCount);
1778	AssertRC(rc);
1779	}
1780	else
1781	fOffsettedTsc = TMCpuTickCanUseRealTSC(pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset);
1782
1783	if (fOffsettedTsc)
1784	{
1785	uint64_t u64CurTSC = ASMReadTSC();
1786	if (u64CurTSC + pVCpu->hm.s.vmx.u64TSCOffset > TMCpuTickGetLastSeen(pVCpu))
1787	{
1788	/* Note: VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT takes precedence over TSC_OFFSET, applies to RDTSCP too. */
1789	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, pVCpu->hm.s.vmx.u64TSCOffset);
1790	AssertRC(rc);
1791
1792	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
1793	rc = VMXWriteVmcs(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
1794	AssertRC(rc);
1795	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
1796	}
1797	else
1798	{
1799	/* Fall back to rdtsc, rdtscp emulation as we would otherwise pass decreasing tsc values to the guest. */
1800	LogFlow(("TSC %RX64 offset %RX64 time=%RX64 last=%RX64 (diff=%RX64, virt_tsc=%RX64)\n", u64CurTSC,
1801	pVCpu->hm.s.vmx.u64TSCOffset, u64CurTSC + pVCpu->hm.s.vmx.u64TSCOffset,
1802	TMCpuTickGetLastSeen(pVCpu), TMCpuTickGetLastSeen(pVCpu) - u64CurTSC - pVCpu->hm.s.vmx.u64TSCOffset,
1803	TMCpuTickGet(pVCpu)));
1804	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
1805	rc = VMXWriteVmcs(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
1806	AssertRC(rc);
1807	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscInterceptOverFlow);
1808	}
1809	}
1810	else
1811	{
1812	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
1813	rc = VMXWriteVmcs(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
1814	AssertRC(rc);
1815	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
1816	}
1817	return rc;
1818	}
1819
1820	/**
1821	* Loads the guest state.
1822	*
1823	* NOTE: Don't do anything here that can cause a jump back to ring 3!!!!!
1824	*
1825	* @returns VBox status code.
1826	* @param pVM Pointer to the VM.
1827	* @param pVCpu Pointer to the VMCPU.
1828	* @param pCtx Pointer to the guest CPU context.
1829	*/
1830	VMMR0DECL(int) VMXR0LoadGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
1831	{
1832	int rc = VINF_SUCCESS;
1833	RTGCUINTPTR val;
1834
1835	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestState, x);
1836
1837	/*
1838	* VMX_VMCS_CTRL_ENTRY
1839	* Set required bits to one and zero according to the MSR capabilities.
1840	*/
1841	val = pVM->hm.s.vmx.msr.vmx_entry.n.disallowed0;
1842
1843	/*
1844	* Load guest debug controls (DR7 & IA32_DEBUGCTL_MSR).
1845	* Forced to 1 on the 'first' VT-x capable CPUs; this actually includes the newest Nehalem CPUs
1846	*/
1847	val \|= VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG;
1848
1849	if (CPUMIsGuestInLongModeEx(pCtx))
1850	val \|= VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST;
1851	/* else Must be zero when AMD64 is not available. */
1852
1853	/*
1854	* Mask away the bits that the CPU doesn't support.
1855	*/
1856	val &= pVM->hm.s.vmx.msr.vmx_entry.n.allowed1;
1857	rc = VMXWriteVmcs(VMX_VMCS32_CTRL_ENTRY, val);
1858	AssertRC(rc);
1859
1860	/*
1861	* VMX_VMCS_CTRL_EXIT
1862	* Set required bits to one and zero according to the MSR capabilities.
1863	*/
1864	val = pVM->hm.s.vmx.msr.vmx_exit.n.disallowed0;
1865
1866	/*
1867	* Save debug controls (DR7 & IA32_DEBUGCTL_MSR)
1868	* Forced to 1 on the 'first' VT-x capable CPUs; this actually includes the newest Nehalem CPUs
1869	*/
1870	val \|= VMX_VMCS_CTRL_EXIT_SAVE_DEBUG;
1871
1872	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
1873	if (VMX_IS_64BIT_HOST_MODE())
1874	val \|= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
1875	/* else Must be zero when AMD64 is not available. */
1876	#elif HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
1877	if (CPUMIsGuestInLongModeEx(pCtx))
1878	val \|= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE; /* our switcher goes to long mode */
1879	else
1880	Assert(!(val & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
1881	#endif
1882	val &= pVM->hm.s.vmx.msr.vmx_exit.n.allowed1;
1883
1884	/*
1885	* Don't acknowledge external interrupts on VM-exit.
1886	*/
1887	rc = VMXWriteVmcs(VMX_VMCS32_CTRL_EXIT, val);
1888	AssertRC(rc);
1889
1890	/*
1891	* Guest CPU context: ES, CS, SS, DS, FS, GS.
1892	*/
1893	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SEGMENT_REGS)
1894	{
1895	if (pVM->hm.s.vmx.pRealModeTSS)
1896	{
1897	PGMMODE enmGuestMode = PGMGetGuestMode(pVCpu);
1898	if (pVCpu->hm.s.vmx.enmLastSeenGuestMode != enmGuestMode)
1899	{
1900	/*
1901	* Correct weird requirements for switching to protected mode.
1902	*/
1903	if ( pVCpu->hm.s.vmx.enmLastSeenGuestMode == PGMMODE_REAL
1904	&& enmGuestMode >= PGMMODE_PROTECTED)
1905	{
1906	#ifdef VBOX_WITH_REM
1907	/*
1908	* Flush the recompiler code cache as it's not unlikely the guest will rewrite code
1909	* it will later execute in real mode (OpenBSD 4.0 is one such example)
1910	*/
1911	REMFlushTBs(pVM);
1912	#endif
1913
1914	/*
1915	* DPL of all hidden selector registers must match the current CPL (0).
1916	*/
1917	pCtx->cs.Attr.n.u2Dpl = 0;
1918	pCtx->cs.Attr.n.u4Type = X86_SEL_TYPE_CODE \| X86_SEL_TYPE_RW_ACC;
1919
1920	pCtx->ds.Attr.n.u2Dpl = 0;
1921	pCtx->es.Attr.n.u2Dpl = 0;
1922	pCtx->fs.Attr.n.u2Dpl = 0;
1923	pCtx->gs.Attr.n.u2Dpl = 0;
1924	pCtx->ss.Attr.n.u2Dpl = 0;
1925	}
1926	pVCpu->hm.s.vmx.enmLastSeenGuestMode = enmGuestMode;
1927	}
1928	}
1929
1930	VMX_WRITE_SELREG(ES, es);
1931	AssertRC(rc);
1932
1933	VMX_WRITE_SELREG(CS, cs);
1934	AssertRC(rc);
1935
1936	VMX_WRITE_SELREG(SS, ss);
1937	AssertRC(rc);
1938
1939	VMX_WRITE_SELREG(DS, ds);
1940	AssertRC(rc);
1941
1942	VMX_WRITE_SELREG(FS, fs);
1943	AssertRC(rc);
1944
1945	VMX_WRITE_SELREG(GS, gs);
1946	AssertRC(rc);
1947	}
1948
1949	/*
1950	* Guest CPU context: LDTR.
1951	*/
1952	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_LDTR)
1953	{
1954	if (pCtx->ldtr.Sel == 0)
1955	{
1956	rc = VMXWriteVmcs(VMX_VMCS16_GUEST_FIELD_LDTR, 0);
1957	rc \|= VMXWriteVmcs(VMX_VMCS32_GUEST_LDTR_LIMIT, 0);
1958	rc \|= VMXWriteVmcs64(VMX_VMCS_GUEST_LDTR_BASE, 0); /* @todo removing "64" in the function should be the same. */
1959	/* Note: vmlaunch will fail with 0 or just 0x02. No idea why. */
1960	rc \|= VMXWriteVmcs(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, 0x82 /* present, LDT */);
1961	}
1962	else
1963	{
1964	rc = VMXWriteVmcs(VMX_VMCS16_GUEST_FIELD_LDTR, pCtx->ldtr.Sel);
1965	rc \|= VMXWriteVmcs(VMX_VMCS32_GUEST_LDTR_LIMIT, pCtx->ldtr.u32Limit);
1966	rc \|= VMXWriteVmcs64(VMX_VMCS_GUEST_LDTR_BASE, pCtx->ldtr.u64Base); /* @todo removing "64" and it should be the same */
1967	rc \|= VMXWriteVmcs(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, pCtx->ldtr.Attr.u);
1968	}
1969	AssertRC(rc);
1970	}
1971
1972	/*
1973	* Guest CPU context: TR.
1974	*/
1975	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_TR)
1976	{
1977	/*
1978	* Real mode emulation using v86 mode with CR4.VME (interrupt redirection
1979	* using the int bitmap in the TSS).
1980	*/
1981	if ( CPUMIsGuestInRealModeEx(pCtx)
1982	&& pVM->hm.s.vmx.pRealModeTSS)
1983	{
1984	RTGCPHYS GCPhys;
1985
1986	/* We convert it here every time as PCI regions could be reconfigured. */
1987	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
1988	AssertRC(rc);
1989
1990	rc = VMXWriteVmcs(VMX_VMCS16_GUEST_FIELD_TR, 0);
1991	rc \|= VMXWriteVmcs(VMX_VMCS32_GUEST_TR_LIMIT, HM_VTX_TSS_SIZE);
1992	rc \|= VMXWriteVmcs64(VMX_VMCS_GUEST_TR_BASE, GCPhys /* phys = virt in this mode */);
1993
1994	X86DESCATTR attr;
1995
1996	attr.u = 0;
1997	attr.n.u1Present = 1;
1998	attr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
1999	val = attr.u;
2000	}
2001	else
2002	{
2003	rc = VMXWriteVmcs(VMX_VMCS16_GUEST_FIELD_TR, pCtx->tr.Sel);
2004	rc \|= VMXWriteVmcs(VMX_VMCS32_GUEST_TR_LIMIT, pCtx->tr.u32Limit);
2005	rc \|= VMXWriteVmcs64(VMX_VMCS_GUEST_TR_BASE, pCtx->tr.u64Base);
2006
2007	val = pCtx->tr.Attr.u;
2008
2009	/* The TSS selector must be busy (REM bugs? see defect #XXXX). */
2010	if (!(val & X86_SEL_TYPE_SYS_TSS_BUSY_MASK))
2011	{
2012	if (val & 0xf)
2013	val \|= X86_SEL_TYPE_SYS_TSS_BUSY_MASK;
2014	else
2015	/* Default if no TR selector has been set (otherwise vmlaunch will fail!) */
2016	val = (val & ~0xF) \| X86_SEL_TYPE_SYS_386_TSS_BUSY;
2017	}
2018	AssertMsg((val & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY \|\| (val & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY,
2019	("%#x\n", val));
2020	}
2021	rc \|= VMXWriteVmcs(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, val);
2022	AssertRC(rc);
2023	}
2024
2025	/*
2026	* Guest CPU context: GDTR.
2027	*/
2028	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_GDTR)
2029	{
2030	rc = VMXWriteVmcs(VMX_VMCS32_GUEST_GDTR_LIMIT, pCtx->gdtr.cbGdt);
2031	rc \|= VMXWriteVmcs64(VMX_VMCS_GUEST_GDTR_BASE, pCtx->gdtr.pGdt);
2032	AssertRC(rc);
2033	}
2034
2035	/*
2036	* Guest CPU context: IDTR.
2037	*/
2038	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_IDTR)
2039	{
2040	rc = VMXWriteVmcs(VMX_VMCS32_GUEST_IDTR_LIMIT, pCtx->idtr.cbIdt);
2041	rc \|= VMXWriteVmcs64(VMX_VMCS_GUEST_IDTR_BASE, pCtx->idtr.pIdt);
2042	AssertRC(rc);
2043	}
2044
2045	/*
2046	* Sysenter MSRs.
2047	*/
2048	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_MSR)
2049	{
2050	rc = VMXWriteVmcs(VMX_VMCS32_GUEST_SYSENTER_CS, pCtx->SysEnter.cs);
2051	rc \|= VMXWriteVmcs64(VMX_VMCS_GUEST_SYSENTER_EIP, pCtx->SysEnter.eip);
2052	rc \|= VMXWriteVmcs64(VMX_VMCS_GUEST_SYSENTER_ESP, pCtx->SysEnter.esp);
2053	AssertRC(rc);
2054	}
2055
2056	/*
2057	* Guest CPU context: Control registers.
2058	*/
2059	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR0)
2060	{
2061	val = pCtx->cr0;
2062	rc = VMXWriteVmcs(VMX_VMCS_CTRL_CR0_READ_SHADOW, val);
2063	Log2(("Guest CR0-shadow %08x\n", val));
2064	if (CPUMIsGuestFPUStateActive(pVCpu) == false)
2065	{
2066	/* Always use #NM exceptions to load the FPU/XMM state on demand. */
2067	val \|= X86_CR0_TS \| X86_CR0_ET \| X86_CR0_NE \| X86_CR0_MP;
2068	}
2069	else
2070	{
2071	/** @todo check if we support the old style mess correctly. */
2072	if (!(val & X86_CR0_NE))
2073	Log(("Forcing X86_CR0_NE!!!\n"));
2074
2075	val \|= X86_CR0_NE; /* always turn on the native mechanism to report FPU errors (old style uses interrupts) */
2076	}
2077	/* Protected mode & paging are always enabled; we use them for emulating real and protected mode without paging too. */
2078	if (!pVM->hm.s.vmx.fUnrestrictedGuest)
2079	val \|= X86_CR0_PE \| X86_CR0_PG;
2080
2081	if (pVM->hm.s.fNestedPaging)
2082	{
2083	if (CPUMIsGuestInPagedProtectedModeEx(pCtx))
2084	{
2085	/* Disable CR3 read/write monitoring as we don't need it for EPT. */
2086	pVCpu->hm.s.vmx.u32ProcCtls &= ~( VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
2087	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT);
2088	}
2089	else
2090	{
2091	/* Reenable CR3 read/write monitoring as our identity mapped page table is active. */
2092	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
2093	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
2094	}
2095	rc = VMXWriteVmcs(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
2096	AssertRC(rc);
2097	}
2098	else
2099	{
2100	/* Note: We must also set this as we rely on protecting various pages for which supervisor writes must be caught. */
2101	val \|= X86_CR0_WP;
2102	}
2103
2104	/* Always enable caching. */
2105	val &= ~(X86_CR0_CD\|X86_CR0_NW);
2106
2107	rc \|= VMXWriteVmcs64(VMX_VMCS_GUEST_CR0, val);
2108	Log2(("Guest CR0 %08x\n", val));
2109
2110	/*
2111	* CR0 flags owned by the host; if the guests attempts to change them, then the VM will exit.
2112	*/
2113	val = X86_CR0_PE /* Must monitor this bit (assumptions are made for real mode emulation) */
2114	\| X86_CR0_WP /* Must monitor this bit (it must always be enabled). */
2115	\| X86_CR0_PG /* Must monitor this bit (assumptions are made for real mode & protected mode without paging emulation) */
2116	\| X86_CR0_CD /* Bit not restored during VM-exit! */
2117	\| X86_CR0_NW /* Bit not restored during VM-exit! */
2118	\| X86_CR0_NE;
2119
2120	/*
2121	* When the guest's FPU state is active, then we no longer care about the FPU related bits.
2122	*/
2123	if (CPUMIsGuestFPUStateActive(pVCpu) == false)
2124	val \|= X86_CR0_TS \| X86_CR0_ET \| X86_CR0_MP;
2125
2126	pVCpu->hm.s.vmx.u32CR0Mask = val;
2127
2128	rc \|= VMXWriteVmcs(VMX_VMCS_CTRL_CR0_MASK, val);
2129	Log2(("Guest CR0-mask %08x\n", val));
2130	AssertRC(rc);
2131	}
2132
2133	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR4)
2134	{
2135	rc = VMXWriteVmcs(VMX_VMCS_CTRL_CR4_READ_SHADOW, pCtx->cr4);
2136	Log2(("Guest CR4-shadow %08x\n", pCtx->cr4));
2137	/* Set the required bits in cr4 too (currently X86_CR4_VMXE). */
2138	val = pCtx->cr4 \| (uint32_t)pVM->hm.s.vmx.msr.vmx_cr4_fixed0;
2139
2140	if (!pVM->hm.s.fNestedPaging)
2141	{
2142	switch (pVCpu->hm.s.enmShadowMode)
2143	{
2144	case PGMMODE_REAL: /* Real mode -> emulated using v86 mode */
2145	case PGMMODE_PROTECTED: /* Protected mode, no paging -> emulated using identity mapping. */
2146	case PGMMODE_32_BIT: /* 32-bit paging. */
2147	val &= ~X86_CR4_PAE;
2148	break;
2149
2150	case PGMMODE_PAE: /* PAE paging. */
2151	case PGMMODE_PAE_NX: /* PAE paging with NX enabled. */
2152	/** Must use PAE paging as we could use physical memory > 4 GB */
2153	val \|= X86_CR4_PAE;
2154	break;
2155
2156	case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
2157	case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
2158	#ifdef VBOX_ENABLE_64_BITS_GUESTS
2159	break;
2160	#else
2161	AssertFailed();
2162	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
2163	#endif
2164	default: /* shut up gcc */
2165	AssertFailed();
2166	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
2167	}
2168	}
2169	else if ( !CPUMIsGuestInPagedProtectedModeEx(pCtx)
2170	&& !pVM->hm.s.vmx.fUnrestrictedGuest)
2171	{
2172	/* We use 4 MB pages in our identity mapping page table for real and protected mode without paging. */
2173	val \|= X86_CR4_PSE;
2174	/* Our identity mapping is a 32 bits page directory. */
2175	val &= ~X86_CR4_PAE;
2176	}
2177
2178	/*
2179	* Turn off VME if we're in emulated real mode.
2180	*/
2181	if ( CPUMIsGuestInRealModeEx(pCtx)
2182	&& pVM->hm.s.vmx.pRealModeTSS)
2183	{
2184	val &= ~X86_CR4_VME;
2185	}
2186
2187	rc \|= VMXWriteVmcs64(VMX_VMCS_GUEST_CR4, val);
2188	Log2(("Guest CR4 %08x\n", val));
2189
2190	/*
2191	* CR4 flags owned by the host; if the guests attempts to change them, then the VM will exit.
2192	*/
2193	val = 0
2194	\| X86_CR4_VME
2195	\| X86_CR4_PAE
2196	\| X86_CR4_PGE
2197	\| X86_CR4_PSE
2198	\| X86_CR4_VMXE;
2199	pVCpu->hm.s.vmx.u32CR4Mask = val;
2200
2201	rc \|= VMXWriteVmcs(VMX_VMCS_CTRL_CR4_MASK, val);
2202	Log2(("Guest CR4-mask %08x\n", val));
2203	AssertRC(rc);
2204	}
2205
2206	#if 0
2207	/* Enable single stepping if requested and CPU supports it. */
2208	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG)
2209	if (DBGFIsStepping(pVCpu))
2210	{
2211	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
2212	rc = VMXWriteVmcs(VMX_VMCS_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
2213	AssertRC(rc);
2214	}
2215	#endif
2216
2217	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR3)
2218	{
2219	if (pVM->hm.s.fNestedPaging)
2220	{
2221	Assert(PGMGetHyperCR3(pVCpu));
2222	pVCpu->hm.s.vmx.HCPhysEPTP = PGMGetHyperCR3(pVCpu);
2223
2224	Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & 0xfff));
2225	/** @todo Check the IA32_VMX_EPT_VPID_CAP MSR for other supported memory types. */
2226	pVCpu->hm.s.vmx.HCPhysEPTP \|= VMX_EPT_MEMTYPE_WB
2227	\| (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
2228
2229	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVCpu->hm.s.vmx.HCPhysEPTP);
2230	AssertRC(rc);
2231
2232	if ( !CPUMIsGuestInPagedProtectedModeEx(pCtx)
2233	&& !pVM->hm.s.vmx.fUnrestrictedGuest)
2234	{
2235	RTGCPHYS GCPhys;
2236
2237	/* We convert it here every time as PCI regions could be reconfigured. */
2238	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
2239	AssertMsgRC(rc, ("pNonPagingModeEPTPageTable = %RGv\n", pVM->hm.s.vmx.pNonPagingModeEPTPageTable));
2240
2241	/*
2242	* We use our identity mapping page table here as we need to map guest virtual to
2243	* guest physical addresses; EPT will take care of the translation to host physical addresses.
2244	*/
2245	val = GCPhys;
2246	}
2247	else
2248	{
2249	/* Save the real guest CR3 in VMX_VMCS_GUEST_CR3 */
2250	val = pCtx->cr3;
2251	rc = hmR0VmxLoadPaePdpes(pVCpu, pCtx);
2252	AssertRCReturn(rc, rc);
2253	}
2254	}
2255	else
2256	{
2257	val = PGMGetHyperCR3(pVCpu);
2258	Assert(val \|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL));
2259	}
2260
2261	/* Save our shadow CR3 register. */
2262	rc = VMXWriteVmcs64(VMX_VMCS_GUEST_CR3, val);
2263	AssertRC(rc);
2264	}
2265
2266	/*
2267	* Guest CPU context: Debug registers.
2268	*/
2269	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_DEBUG)
2270	{
2271	pCtx->dr[6] \|= X86_DR6_INIT_VAL; /* set all reserved bits to 1. */
2272	pCtx->dr[6] &= ~RT_BIT(12); /* must be zero. */
2273
2274	pCtx->dr[7] &= 0xffffffff; /* upper 32 bits reserved */
2275	pCtx->dr[7] &= ~(RT_BIT(11) \| RT_BIT(12) \| RT_BIT(14) \| RT_BIT(15)); /* must be zero */
2276	pCtx->dr[7] \|= 0x400; /* must be one */
2277
2278	/* Resync DR7 */
2279	rc = VMXWriteVmcs64(VMX_VMCS_GUEST_DR7, pCtx->dr[7]);
2280	AssertRC(rc);
2281
2282	#ifdef DEBUG
2283	/* Sync the hypervisor debug state now if any breakpoint is armed. */
2284	if ( CPUMGetHyperDR7(pVCpu) & (X86_DR7_ENABLED_MASK\|X86_DR7_GD)
2285	&& !CPUMIsHyperDebugStateActive(pVCpu)
2286	&& !DBGFIsStepping(pVCpu))
2287	{
2288	/* Save the host and load the hypervisor debug state. */
2289	rc = CPUMR0LoadHyperDebugState(pVM, pVCpu, pCtx, true /* include DR6 */);
2290	AssertRC(rc);
2291
2292	/* DRx intercepts remain enabled. */
2293
2294	/* Override dr7 with the hypervisor value. */
2295	rc = VMXWriteVmcs64(VMX_VMCS_GUEST_DR7, CPUMGetHyperDR7(pVCpu));
2296	AssertRC(rc);
2297	}
2298	else
2299	#endif
2300	/* Sync the debug state now if any breakpoint is armed. */
2301	if ( (pCtx->dr[7] & (X86_DR7_ENABLED_MASK\|X86_DR7_GD))
2302	&& !CPUMIsGuestDebugStateActive(pVCpu)
2303	&& !DBGFIsStepping(pVCpu))
2304	{
2305	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
2306
2307	/* Disable DRx move intercepts. */
2308	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
2309	rc = VMXWriteVmcs(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
2310	AssertRC(rc);
2311
2312	/* Save the host and load the guest debug state. */
2313	rc = CPUMR0LoadGuestDebugState(pVM, pVCpu, pCtx, true /* include DR6 */);
2314	AssertRC(rc);
2315	}
2316
2317	/* IA32_DEBUGCTL MSR. */
2318	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, 0);
2319	AssertRC(rc);
2320
2321	/** @todo do we really ever need this? */
2322	rc \|= VMXWriteVmcs(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
2323	AssertRC(rc);
2324	}
2325
2326	/*
2327	* 64-bit guest mode.
2328	*/
2329	if (CPUMIsGuestInLongModeEx(pCtx))
2330	{
2331	#if !defined(VBOX_ENABLE_64_BITS_GUESTS)
2332	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
2333	#elif HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2334	pVCpu->hm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
2335	#else
2336	# ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2337	if (!pVM->hm.s.fAllow64BitGuests)
2338	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
2339	# endif
2340	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM64;
2341	#endif
2342	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_MSR)
2343	{
2344	/* Update these as wrmsr might have changed them. */
2345	rc = VMXWriteVmcs64(VMX_VMCS_GUEST_FS_BASE, pCtx->fs.u64Base);
2346	AssertRC(rc);
2347	rc = VMXWriteVmcs64(VMX_VMCS_GUEST_GS_BASE, pCtx->gs.u64Base);
2348	AssertRC(rc);
2349	}
2350	}
2351	else
2352	{
2353	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
2354	}
2355
2356	hmR0VmxUpdateExceptionBitmap(pVM, pVCpu, pCtx);
2357
2358	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
2359	/*
2360	* Store all guest MSRs in the VM-entry load area, so they will be loaded
2361	* during VM-entry and restored into the VM-exit store area during VM-exit.
2362	*/
2363	PVMXMSR pMsr = (PVMXMSR)pVCpu->hm.s.vmx.pvGuestMsr;
2364	unsigned idxMsr = 0;
2365
2366	uint32_t u32GstExtFeatures;
2367	uint32_t u32Temp;
2368	CPUMGetGuestCpuId(pVCpu, 0x80000001, &u32Temp, &u32Temp, &u32Temp, &u32GstExtFeatures);
2369
2370	if (u32GstExtFeatures & (X86_CPUID_EXT_FEATURE_EDX_NX \| X86_CPUID_EXT_FEATURE_EDX_LONG_MODE))
2371	{
2372	pMsr->u32IndexMSR = MSR_K6_EFER;
2373	pMsr->u32Reserved = 0;
2374	pMsr->u64Value = pCtx->msrEFER;
2375	/* VT-x will complain if only MSR_K6_EFER_LME is set. */
2376	if (!CPUMIsGuestInLongModeEx(pCtx))
2377	pMsr->u64Value &= ~(MSR_K6_EFER_LMA \| MSR_K6_EFER_LME);
2378	pMsr++; idxMsr++;
2379
2380	if (u32GstExtFeatures & X86_CPUID_EXT_FEATURE_EDX_LONG_MODE)
2381	{
2382	pMsr->u32IndexMSR = MSR_K8_LSTAR;
2383	pMsr->u32Reserved = 0;
2384	pMsr->u64Value = pCtx->msrLSTAR; /* 64 bits mode syscall rip */
2385	pMsr++; idxMsr++;
2386	pMsr->u32IndexMSR = MSR_K6_STAR;
2387	pMsr->u32Reserved = 0;
2388	pMsr->u64Value = pCtx->msrSTAR; /* legacy syscall eip, cs & ss */
2389	pMsr++; idxMsr++;
2390	pMsr->u32IndexMSR = MSR_K8_SF_MASK;
2391	pMsr->u32Reserved = 0;
2392	pMsr->u64Value = pCtx->msrSFMASK; /* syscall flag mask */
2393	pMsr++; idxMsr++;
2394	#if 0
2395	/* The KERNEL_GS_BASE MSR does not work reliably with auto load/store. See @bugref{6208} */
2396	pMsr->u32IndexMSR = MSR_K8_KERNEL_GS_BASE;
2397	pMsr->u32Reserved = 0;
2398	pMsr->u64Value = pCtx->msrKERNELGSBASE; /* swapgs exchange value */
2399	pMsr++; idxMsr++;
2400	#endif
2401	}
2402	}
2403
2404	if ( pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP
2405	&& (u32GstExtFeatures & X86_CPUID_EXT_FEATURE_EDX_RDTSCP))
2406	{
2407	pMsr->u32IndexMSR = MSR_K8_TSC_AUX;
2408	pMsr->u32Reserved = 0;
2409	rc = CPUMQueryGuestMsr(pVCpu, MSR_K8_TSC_AUX, &pMsr->u64Value);
2410	AssertRC(rc);
2411	pMsr++; idxMsr++;
2412	}
2413
2414	pVCpu->hm.s.vmx.cGuestMsrs = idxMsr;
2415
2416	rc = VMXWriteVmcs(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, idxMsr);
2417	AssertRC(rc);
2418
2419	rc = VMXWriteVmcs(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, idxMsr);
2420	AssertRC(rc);
2421	#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
2422
2423	/* Done with the major changes */
2424	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_ALL_GUEST;
2425
2426	/* Minimal guest state update (ESP, EIP, EFLAGS mostly) */
2427	VMXR0LoadMinimalGuestState(pVM, pVCpu, pCtx);
2428
2429	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestState, x);
2430	return rc;
2431	}
2432
2433
2434	/**
2435	* Syncs back the guest state from VMCS.
2436	*
2437	* @returns VBox status code.
2438	* @param pVM Pointer to the VM.
2439	* @param pVCpu Pointer to the VMCPU.
2440	* @param pCtx Pointer to the guest CPU context.
2441	*/
2442	DECLINLINE(int) VMXR0SaveGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
2443	{
2444	RTGCUINTREG val, valShadow;
2445	RTGCUINTPTR uInterruptState;
2446	int rc;
2447
2448	/* First sync back EIP, ESP, and EFLAGS. */
2449	rc = VMXReadCachedVmcs(VMX_VMCS_GUEST_RIP, &val);
2450	AssertRC(rc);
2451	pCtx->rip = val;
2452	rc = VMXReadCachedVmcs(VMX_VMCS_GUEST_RSP, &val);
2453	AssertRC(rc);
2454	pCtx->rsp = val;
2455	rc = VMXReadCachedVmcs(VMX_VMCS_GUEST_RFLAGS, &val);
2456	AssertRC(rc);
2457	pCtx->eflags.u32 = val;
2458
2459	/* Take care of instruction fusing (sti, mov ss) */
2460	rc \|= VMXReadCachedVmcs(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &val);
2461	uInterruptState = val;
2462	if (uInterruptState != 0)
2463	{
2464	Assert(uInterruptState <= 2); /* only sti & mov ss */
2465	Log(("uInterruptState %x eip=%RGv\n", (uint32_t)uInterruptState, pCtx->rip));
2466	EMSetInhibitInterruptsPC(pVCpu, pCtx->rip);
2467	}
2468	else
2469	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
2470
2471	/* Control registers. */
2472	VMXReadCachedVmcs(VMX_VMCS_CTRL_CR0_READ_SHADOW, &valShadow);
2473	VMXReadCachedVmcs(VMX_VMCS_GUEST_CR0, &val);
2474	val = (valShadow & pVCpu->hm.s.vmx.u32CR0Mask) \| (val & ~pVCpu->hm.s.vmx.u32CR0Mask);
2475	CPUMSetGuestCR0(pVCpu, val);
2476
2477	VMXReadCachedVmcs(VMX_VMCS_CTRL_CR4_READ_SHADOW, &valShadow);
2478	VMXReadCachedVmcs(VMX_VMCS_GUEST_CR4, &val);
2479	val = (valShadow & pVCpu->hm.s.vmx.u32CR4Mask) \| (val & ~pVCpu->hm.s.vmx.u32CR4Mask);
2480	CPUMSetGuestCR4(pVCpu, val);
2481
2482	/*
2483	* No reason to sync back the CRx registers. They can't be changed by the guest unless in
2484	* the nested paging case where CR3 & CR4 can be changed by the guest.
2485	*/
2486	if ( pVM->hm.s.fNestedPaging
2487	&& CPUMIsGuestInPagedProtectedModeEx(pCtx)) /** @todo check if we will always catch mode switches and such... */
2488	{
2489	PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
2490
2491	/* Can be updated behind our back in the nested paging case. */
2492	CPUMSetGuestCR2(pVCpu, pCache->cr2);
2493
2494	VMXReadCachedVmcs(VMX_VMCS_GUEST_CR3, &val);
2495
2496	if (val != pCtx->cr3)
2497	{
2498	CPUMSetGuestCR3(pVCpu, val);
2499	PGMUpdateCR3(pVCpu, val);
2500	}
2501	rc = hmR0VmxSavePaePdpes(pVCpu, pCtx);
2502	AssertRCReturn(rc, rc);
2503	}
2504
2505	/* Sync back DR7. */
2506	VMXReadCachedVmcs(VMX_VMCS_GUEST_DR7, &val);
2507	pCtx->dr[7] = val;
2508
2509	/* Guest CPU context: ES, CS, SS, DS, FS, GS. */
2510	VMX_READ_SELREG(ES, es);
2511	VMX_READ_SELREG(SS, ss);
2512	VMX_READ_SELREG(CS, cs);
2513	VMX_READ_SELREG(DS, ds);
2514	VMX_READ_SELREG(FS, fs);
2515	VMX_READ_SELREG(GS, gs);
2516
2517	/* System MSRs */
2518	VMXReadCachedVmcs(VMX_VMCS32_GUEST_SYSENTER_CS, &val);
2519	pCtx->SysEnter.cs = val;
2520	VMXReadCachedVmcs(VMX_VMCS_GUEST_SYSENTER_EIP, &val);
2521	pCtx->SysEnter.eip = val;
2522	VMXReadCachedVmcs(VMX_VMCS_GUEST_SYSENTER_ESP, &val);
2523	pCtx->SysEnter.esp = val;
2524
2525	/* Misc. registers; must sync everything otherwise we can get out of sync when jumping to ring 3. */
2526	VMX_READ_SELREG(LDTR, ldtr);
2527
2528	VMXReadCachedVmcs(VMX_VMCS32_GUEST_GDTR_LIMIT, &val);
2529	pCtx->gdtr.cbGdt = val;
2530	VMXReadCachedVmcs(VMX_VMCS_GUEST_GDTR_BASE, &val);
2531	pCtx->gdtr.pGdt = val;
2532
2533	VMXReadCachedVmcs(VMX_VMCS32_GUEST_IDTR_LIMIT, &val);
2534	pCtx->idtr.cbIdt = val;
2535	VMXReadCachedVmcs(VMX_VMCS_GUEST_IDTR_BASE, &val);
2536	pCtx->idtr.pIdt = val;
2537
2538	/* Real mode emulation using v86 mode. */
2539	if ( CPUMIsGuestInRealModeEx(pCtx)
2540	&& pVM->hm.s.vmx.pRealModeTSS)
2541	{
2542	/* Hide our emulation flags */
2543	pCtx->eflags.Bits.u1VM = 0;
2544
2545	/* Restore original IOPL setting as we always use 0. */
2546	pCtx->eflags.Bits.u2IOPL = pVCpu->hm.s.vmx.RealMode.eflags.Bits.u2IOPL;
2547
2548	/* Force a TR resync every time in case we switch modes. */
2549	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_TR;
2550	}
2551	else
2552	{
2553	/* In real mode we have a fake TSS, so only sync it back when it's supposed to be valid. */
2554	VMX_READ_SELREG(TR, tr);
2555	}
2556
2557	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
2558	/*
2559	* Save the possibly changed MSRs that we automatically restore and save during a world switch.
2560	*/
2561	for (unsigned i = 0; i < pVCpu->hm.s.vmx.cGuestMsrs; i++)
2562	{
2563	PVMXMSR pMsr = (PVMXMSR)pVCpu->hm.s.vmx.pvGuestMsr;
2564	pMsr += i;
2565
2566	switch (pMsr->u32IndexMSR)
2567	{
2568	case MSR_K8_LSTAR:
2569	pCtx->msrLSTAR = pMsr->u64Value;
2570	break;
2571	case MSR_K6_STAR:
2572	pCtx->msrSTAR = pMsr->u64Value;
2573	break;
2574	case MSR_K8_SF_MASK:
2575	pCtx->msrSFMASK = pMsr->u64Value;
2576	break;
2577	#if 0
2578	/* The KERNEL_GS_BASE MSR does not work reliably with auto load/store. See @bugref{6208} */
2579	case MSR_K8_KERNEL_GS_BASE:
2580	pCtx->msrKERNELGSBASE = pMsr->u64Value;
2581	break;
2582	#endif
2583	case MSR_K8_TSC_AUX:
2584	CPUMSetGuestMsr(pVCpu, MSR_K8_TSC_AUX, pMsr->u64Value);
2585	break;
2586
2587	case MSR_K6_EFER:
2588	/* EFER can't be changed without causing a VM-exit. */
2589	/* Assert(pCtx->msrEFER == pMsr->u64Value); */
2590	break;
2591
2592	default:
2593	AssertFailed();
2594	return VERR_HM_UNEXPECTED_LD_ST_MSR;
2595	}
2596	}
2597	#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
2598	return VINF_SUCCESS;
2599	}
2600
2601
2602	/**
2603	* Dummy placeholder for TLB flush handling before VM-entry. Used in the case
2604	* where neither EPT nor VPID is supported by the CPU.
2605	*
2606	* @param pVM Pointer to the VM.
2607	* @param pVCpu Pointer to the VMCPU.
2608	*/
2609	static DECLCALLBACK(void) hmR0VmxSetupTLBDummy(PVM pVM, PVMCPU pVCpu)
2610	{
2611	NOREF(pVM);
2612	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
2613	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
2614	pVCpu->hm.s.TlbShootdown.cPages = 0;
2615	}
2616
2617
2618	/**
2619	* Setup the tagged TLB for EPT+VPID.
2620	*
2621	* @param pVM Pointer to the VM.
2622	* @param pVCpu Pointer to the VMCPU.
2623	*/
2624	static DECLCALLBACK(void) hmR0VmxSetupTLBBoth(PVM pVM, PVMCPU pVCpu)
2625	{
2626	PHMGLOBLCPUINFO pCpu;
2627
2628	Assert(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid);
2629
2630	pCpu = HMR0GetCurrentCpu();
2631
2632	/*
2633	* Force a TLB flush for the first world switch if the current CPU differs from the one we ran on last
2634	* This can happen both for start & resume due to long jumps back to ring-3.
2635	* If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
2636	* or the host Cpu is online after a suspend/resume, so we cannot reuse the current ASID anymore.
2637	*/
2638	bool fNewAsid = false;
2639	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2640	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2641	{
2642	pVCpu->hm.s.fForceTLBFlush = true;
2643	fNewAsid = true;
2644	}
2645
2646	/*
2647	* Check for explicit TLB shootdowns.
2648	*/
2649	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2650	pVCpu->hm.s.fForceTLBFlush = true;
2651
2652	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2653
2654	if (pVCpu->hm.s.fForceTLBFlush)
2655	{
2656	if (fNewAsid)
2657	{
2658	++pCpu->uCurrentAsid;
2659	if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
2660	{
2661	pCpu->uCurrentAsid = 1; /* start at 1; host uses 0 */
2662	pCpu->cTlbFlushes++;
2663	pCpu->fFlushAsidBeforeUse = true;
2664	}
2665
2666	pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
2667	if (pCpu->fFlushAsidBeforeUse)
2668	hmR0VmxFlushVPID(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
2669	}
2670	else
2671	{
2672	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
2673	hmR0VmxFlushVPID(pVM, pVCpu, VMX_FLUSH_VPID_SINGLE_CONTEXT, 0 /* GCPtr */);
2674	else
2675	hmR0VmxFlushEPT(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
2676	}
2677
2678	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2679	pVCpu->hm.s.fForceTLBFlush = false;
2680	}
2681	else
2682	{
2683	AssertMsg(pVCpu->hm.s.uCurrentAsid && pCpu->uCurrentAsid,
2684	("hm->uCurrentAsid=%lu hm->cTlbFlushes=%lu cpu->uCurrentAsid=%lu cpu->cTlbFlushes=%lu\n",
2685	pVCpu->hm.s.uCurrentAsid, pVCpu->hm.s.cTlbFlushes,
2686	pCpu->uCurrentAsid, pCpu->cTlbFlushes));
2687
2688	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
2689	* not be executed. See hmQueueInvlPage() where it is commented
2690	* out. Support individual entry flushing someday. */
2691	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
2692	{
2693	STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
2694
2695	/*
2696	* Flush individual guest entries using VPID from the TLB or as little as possible with EPT
2697	* as supported by the CPU.
2698	*/
2699	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
2700	{
2701	for (uint32_t i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
2702	hmR0VmxFlushVPID(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
2703	}
2704	else
2705	hmR0VmxFlushEPT(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
2706	}
2707	else
2708	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
2709	}
2710
2711	pVCpu->hm.s.TlbShootdown.cPages = 0;
2712	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
2713
2714	AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
2715	("Flush count mismatch for cpu %d (%x vs %x)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
2716	AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2717	("cpu%d uCurrentAsid = %x\n", pCpu->idCpu, pCpu->uCurrentAsid));
2718	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2719	("cpu%d VM uCurrentAsid = %x\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2720
2721	/* Update VMCS with the VPID. */
2722	int rc = VMXWriteVmcs(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
2723	AssertRC(rc);
2724	}
2725
2726
2727	/**
2728	* Setup the tagged TLB for EPT only.
2729	*
2730	* @returns VBox status code.
2731	* @param pVM Pointer to the VM.
2732	* @param pVCpu Pointer to the VMCPU.
2733	*/
2734	static DECLCALLBACK(void) hmR0VmxSetupTLBEPT(PVM pVM, PVMCPU pVCpu)
2735	{
2736	PHMGLOBLCPUINFO pCpu;
2737
2738	Assert(pVM->hm.s.fNestedPaging);
2739	Assert(!pVM->hm.s.vmx.fVpid);
2740
2741	pCpu = HMR0GetCurrentCpu();
2742
2743	/*
2744	* Force a TLB flush for the first world switch if the current CPU differs from the one we ran on last
2745	* This can happen both for start & resume due to long jumps back to ring-3.
2746	* A change in the TLB flush count implies the host Cpu is online after a suspend/resume.
2747	*/
2748	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2749	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2750	{
2751	pVCpu->hm.s.fForceTLBFlush = true;
2752	}
2753
2754	/*
2755	* Check for explicit TLB shootdown flushes.
2756	*/
2757	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2758	pVCpu->hm.s.fForceTLBFlush = true;
2759
2760	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2761	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2762
2763	if (pVCpu->hm.s.fForceTLBFlush)
2764	hmR0VmxFlushEPT(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
2765	else
2766	{
2767	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
2768	* not be executed. See hmQueueInvlPage() where it is commented
2769	* out. Support individual entry flushing someday. */
2770	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
2771	{
2772	/*
2773	* We cannot flush individual entries without VPID support. Flush using EPT.
2774	*/
2775	STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
2776	hmR0VmxFlushEPT(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
2777	}
2778	}
2779	pVCpu->hm.s.TlbShootdown.cPages = 0;
2780	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
2781
2782	#ifdef VBOX_WITH_STATISTICS
2783	/** @todo r=ramshankar: this is not accurate anymore with the VPID+EPT
2784	* handling. Should be fixed later. */
2785	if (pVCpu->hm.s.fForceTLBFlush)
2786	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2787	else
2788	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
2789	#endif
2790	}
2791
2792
2793	/**
2794	* Setup the tagged TLB for VPID.
2795	*
2796	* @returns VBox status code.
2797	* @param pVM Pointer to the VM.
2798	* @param pVCpu Pointer to the VMCPU.
2799	*/
2800	static DECLCALLBACK(void) hmR0VmxSetupTLBVPID(PVM pVM, PVMCPU pVCpu)
2801	{
2802	PHMGLOBLCPUINFO pCpu;
2803
2804	Assert(pVM->hm.s.vmx.fVpid);
2805	Assert(!pVM->hm.s.fNestedPaging);
2806
2807	pCpu = HMR0GetCurrentCpu();
2808
2809	/*
2810	* Force a TLB flush for the first world switch if the current CPU differs from the one we ran on last
2811	* This can happen both for start & resume due to long jumps back to ring-3.
2812	* If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
2813	* or the host Cpu is online after a suspend/resume, so we cannot reuse the current ASID anymore.
2814	*/
2815	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2816	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2817	{
2818	/* Force a TLB flush on VM entry. */
2819	pVCpu->hm.s.fForceTLBFlush = true;
2820	}
2821
2822	/*
2823	* Check for explicit TLB shootdown flushes.
2824	*/
2825	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2826	pVCpu->hm.s.fForceTLBFlush = true;
2827
2828	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2829
2830	if (pVCpu->hm.s.fForceTLBFlush)
2831	{
2832	++pCpu->uCurrentAsid;
2833	if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
2834	{
2835	pCpu->uCurrentAsid = 1; /* start at 1; host uses 0 */
2836	pCpu->cTlbFlushes++;
2837	pCpu->fFlushAsidBeforeUse = true;
2838	}
2839
2840	pVCpu->hm.s.fForceTLBFlush = false;
2841	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2842	pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
2843	if (pCpu->fFlushAsidBeforeUse)
2844	hmR0VmxFlushVPID(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
2845	}
2846	else
2847	{
2848	AssertMsg(pVCpu->hm.s.uCurrentAsid && pCpu->uCurrentAsid,
2849	("hm->uCurrentAsid=%lu hm->cTlbFlushes=%lu cpu->uCurrentAsid=%lu cpu->cTlbFlushes=%lu\n",
2850	pVCpu->hm.s.uCurrentAsid, pVCpu->hm.s.cTlbFlushes,
2851	pCpu->uCurrentAsid, pCpu->cTlbFlushes));
2852
2853	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
2854	* not be executed. See hmQueueInvlPage() where it is commented
2855	* out. Support individual entry flushing someday. */
2856	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
2857	{
2858	/*
2859	* Flush individual guest entries using VPID from the TLB or as little as possible with EPT
2860	* as supported by the CPU.
2861	*/
2862	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
2863	{
2864	for (uint32_t i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
2865	hmR0VmxFlushVPID(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
2866	}
2867	else
2868	hmR0VmxFlushVPID(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
2869	}
2870	}
2871	pVCpu->hm.s.TlbShootdown.cPages = 0;
2872	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
2873
2874	AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
2875	("Flush count mismatch for cpu %d (%x vs %x)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
2876	AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2877	("cpu%d uCurrentAsid = %x\n", pCpu->idCpu, pCpu->uCurrentAsid));
2878	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2879	("cpu%d VM uCurrentAsid = %x\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2880
2881	int rc = VMXWriteVmcs(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
2882	AssertRC(rc);
2883
2884	# ifdef VBOX_WITH_STATISTICS
2885	/** @todo r=ramshankar: this is not accurate anymore with EPT+VPID handling.
2886	* Should be fixed later. */
2887	if (pVCpu->hm.s.fForceTLBFlush)
2888	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2889	else
2890	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
2891	# endif
2892	}
2893
2894
2895	/**
2896	* Runs guest code in a VT-x VM.
2897	*
2898	* @returns VBox status code.
2899	* @param pVM Pointer to the VM.
2900	* @param pVCpu Pointer to the VMCPU.
2901	* @param pCtx Pointer to the guest CPU context.
2902	*/
2903	VMMR0DECL(int) VMXR0RunGuestCode(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
2904	{
2905	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
2906	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit1);
2907	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit2);
2908
2909	VBOXSTRICTRC rc = VINF_SUCCESS;
2910	int rc2;
2911	RTGCUINTREG val;
2912	RTGCUINTREG exitReason = (RTGCUINTREG)VMX_EXIT_INVALID;
2913	RTGCUINTREG instrError, cbInstr;
2914	RTGCUINTPTR exitQualification = 0;
2915	RTGCUINTPTR intInfo = 0; /* shut up buggy gcc 4 */
2916	RTGCUINTPTR errCode, instrInfo;
2917	bool fSetupTPRCaching = false;
2918	bool fNeedTscSetup = true;
2919	uint64_t u64OldLSTAR = 0;
2920	uint8_t u8LastTPR = 0;
2921	RTCCUINTREG uOldEFlags = ~(RTCCUINTREG)0;
2922	unsigned cResume = 0;
2923	#ifdef VBOX_STRICT
2924	RTCPUID idCpuCheck;
2925	bool fWasInLongMode = false;
2926	#endif
2927	#ifdef VBOX_HIGH_RES_TIMERS_HACK_IN_RING0
2928	uint64_t u64LastTime = RTTimeMilliTS();
2929	#endif
2930
2931	Assert(!(pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
2932	\|\| (pVCpu->hm.s.vmx.pbVirtApic && pVM->hm.s.vmx.pbApicAccess));
2933
2934	/*
2935	* Check if we need to use TPR shadowing.
2936	*/
2937	if ( CPUMIsGuestInLongModeEx(pCtx)
2938	\|\| ( (( pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
2939	\|\| pVM->hm.s.fTRPPatchingAllowed)
2940	&& pVM->hm.s.fHasIoApic)
2941	)
2942	{
2943	fSetupTPRCaching = true;
2944	}
2945
2946	Log2(("\n"));
2947
2948	/* This is not ideal, but if we don't clear the event injection in the VMCS right here,
2949	* we may end up injecting some stale event into a VM, including injecting an event that
2950	* originated before a VM reset after the VM has been reset. See @bugref{6220}.
2951	*/
2952	VMXWriteVmcs(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, 0);
2953
2954	#ifdef VBOX_STRICT
2955	{
2956	RTCCUINTREG val2;
2957
2958	rc2 = VMXReadVmcs(VMX_VMCS32_CTRL_PIN_EXEC, &val2);
2959	AssertRC(rc2);
2960	Log2(("VMX_VMCS_CTRL_PIN_EXEC = %08x\n", val2));
2961
2962	/* allowed zero */
2963	if ((val2 & pVM->hm.s.vmx.msr.vmx_pin_ctls.n.disallowed0) != pVM->hm.s.vmx.msr.vmx_pin_ctls.n.disallowed0)
2964	Log(("Invalid VMX_VMCS_CTRL_PIN_EXEC: zero\n"));
2965
2966	/* allowed one */
2967	if ((val2 & ~pVM->hm.s.vmx.msr.vmx_pin_ctls.n.allowed1) != 0)
2968	Log(("Invalid VMX_VMCS_CTRL_PIN_EXEC: one\n"));
2969
2970	rc2 = VMXReadVmcs(VMX_VMCS32_CTRL_PROC_EXEC, &val2);
2971	AssertRC(rc2);
2972	Log2(("VMX_VMCS_CTRL_PROC_EXEC = %08x\n", val2));
2973
2974	/*
2975	* Must be set according to the MSR, but can be cleared if nested paging is used.
2976	*/
2977	if (pVM->hm.s.fNestedPaging)
2978	{
2979	val2 \|= VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT
2980	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
2981	\| VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
2982	}
2983
2984	/* allowed zero */
2985	if ((val2 & pVM->hm.s.vmx.msr.vmx_proc_ctls.n.disallowed0) != pVM->hm.s.vmx.msr.vmx_proc_ctls.n.disallowed0)
2986	Log(("Invalid VMX_VMCS_CTRL_PROC_EXEC: zero\n"));
2987
2988	/* allowed one */
2989	if ((val2 & ~pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1) != 0)
2990	Log(("Invalid VMX_VMCS_CTRL_PROC_EXEC: one\n"));
2991
2992	rc2 = VMXReadVmcs(VMX_VMCS32_CTRL_ENTRY, &val2);
2993	AssertRC(rc2);
2994	Log2(("VMX_VMCS_CTRL_ENTRY = %08x\n", val2));
2995
2996	/* allowed zero */
2997	if ((val2 & pVM->hm.s.vmx.msr.vmx_entry.n.disallowed0) != pVM->hm.s.vmx.msr.vmx_entry.n.disallowed0)
2998	Log(("Invalid VMX_VMCS_CTRL_ENTRY: zero\n"));
2999
3000	/* allowed one */
3001	if ((val2 & ~pVM->hm.s.vmx.msr.vmx_entry.n.allowed1) != 0)
3002	Log(("Invalid VMX_VMCS_CTRL_ENTRY: one\n"));
3003
3004	rc2 = VMXReadVmcs(VMX_VMCS32_CTRL_EXIT, &val2);
3005	AssertRC(rc2);
3006	Log2(("VMX_VMCS_CTRL_EXIT = %08x\n", val2));
3007
3008	/* allowed zero */
3009	if ((val2 & pVM->hm.s.vmx.msr.vmx_exit.n.disallowed0) != pVM->hm.s.vmx.msr.vmx_exit.n.disallowed0)
3010	Log(("Invalid VMX_VMCS_CTRL_EXIT: zero\n"));
3011
3012	/* allowed one */
3013	if ((val2 & ~pVM->hm.s.vmx.msr.vmx_exit.n.allowed1) != 0)
3014	Log(("Invalid VMX_VMCS_CTRL_EXIT: one\n"));
3015	}
3016	fWasInLongMode = CPUMIsGuestInLongModeEx(pCtx);
3017	#endif /* VBOX_STRICT */
3018
3019	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
3020	pVCpu->hm.s.vmx.VMCSCache.u64TimeEntry = RTTimeNanoTS();
3021	#endif
3022
3023	/*
3024	* We can jump to this point to resume execution after determining that a VM-exit is innocent.
3025	*/
3026	ResumeExecution:
3027	if (!STAM_REL_PROFILE_ADV_IS_RUNNING(&pVCpu->hm.s.StatEntry))
3028	STAM_REL_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit2, &pVCpu->hm.s.StatEntry, x);
3029	AssertMsg(pVCpu->hm.s.idEnteredCpu == RTMpCpuId(),
3030	("Expected %d, I'm %d; cResume=%d exitReason=%RGv exitQualification=%RGv\n",
3031	(int)pVCpu->hm.s.idEnteredCpu, (int)RTMpCpuId(), cResume, exitReason, exitQualification));
3032	Assert(!HMR0SuspendPending());
3033	/* Not allowed to switch modes without reloading the host state (32->64 switcher)!! */
3034	Assert(fWasInLongMode == CPUMIsGuestInLongModeEx(pCtx));
3035
3036	/*
3037	* Safety precaution; looping for too long here can have a very bad effect on the host.
3038	*/
3039	if (RT_UNLIKELY(++cResume > pVM->hm.s.cMaxResumeLoops))
3040	{
3041	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMaxResume);
3042	rc = VINF_EM_RAW_INTERRUPT;
3043	goto end;
3044	}
3045
3046	/*
3047	* Check for IRQ inhibition due to instruction fusing (sti, mov ss).
3048	*/
3049	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3050	{
3051	Log(("VM_FF_INHIBIT_INTERRUPTS at %RGv successor %RGv\n", (RTGCPTR)pCtx->rip, EMGetInhibitInterruptsPC(pVCpu)));
3052	if (pCtx->rip != EMGetInhibitInterruptsPC(pVCpu))
3053	{
3054	/*
3055	* Note: we intentionally don't clear VM_FF_INHIBIT_INTERRUPTS here.
3056	* Before we are able to execute this instruction in raw mode (iret to guest code) an external interrupt might
3057	* force a world switch again. Possibly allowing a guest interrupt to be dispatched in the process. This could
3058	* break the guest. Sounds very unlikely, but such timing sensitive problems are not as rare as you might think.
3059	*/
3060	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
3061	/* Irq inhibition is no longer active; clear the corresponding VMX state. */
3062	rc2 = VMXWriteVmcs(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, 0);
3063	AssertRC(rc2);
3064	}
3065	}
3066	else
3067	{
3068	/* Irq inhibition is no longer active; clear the corresponding VMX state. */
3069	rc2 = VMXWriteVmcs(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, 0);
3070	AssertRC(rc2);
3071	}
3072
3073	#ifdef VBOX_HIGH_RES_TIMERS_HACK_IN_RING0
3074	if (RT_UNLIKELY((cResume & 0xf) == 0))
3075	{
3076	uint64_t u64CurTime = RTTimeMilliTS();
3077
3078	if (RT_UNLIKELY(u64CurTime > u64LastTime))
3079	{
3080	u64LastTime = u64CurTime;
3081	TMTimerPollVoid(pVM, pVCpu);
3082	}
3083	}
3084	#endif
3085
3086	/*
3087	* Check for pending actions that force us to go back to ring-3.
3088	*/
3089	if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK \| VM_FF_REQUEST \| VM_FF_PGM_POOL_FLUSH_PENDING \| VM_FF_PDM_DMA)
3090	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK \| VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL \| VMCPU_FF_REQUEST))
3091	{
3092	/* Check if a sync operation is pending. */
3093	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
3094	{
3095	rc = PGMSyncCR3(pVCpu, pCtx->cr0, pCtx->cr3, pCtx->cr4, VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
3096	if (rc != VINF_SUCCESS)
3097	{
3098	AssertRC(VBOXSTRICTRC_VAL(rc));
3099	Log(("Pending pool sync is forcing us back to ring 3; rc=%d\n", VBOXSTRICTRC_VAL(rc)));
3100	goto end;
3101	}
3102	}
3103
3104	#ifdef DEBUG
3105	/* Intercept X86_XCPT_DB if stepping is enabled */
3106	if (!DBGFIsStepping(pVCpu))
3107	#endif
3108	{
3109	if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK)
3110	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
3111	{
3112	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
3113	rc = RT_UNLIKELY(VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
3114	goto end;
3115	}
3116	}
3117
3118	/* Pending request packets might contain actions that need immediate attention, such as pending hardware interrupts. */
3119	if ( VM_FF_IS_PENDING(pVM, VM_FF_REQUEST)
3120	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_REQUEST))
3121	{
3122	rc = VINF_EM_PENDING_REQUEST;
3123	goto end;
3124	}
3125
3126	/* Check if a pgm pool flush is in progress. */
3127	if (VM_FF_IS_PENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
3128	{
3129	rc = VINF_PGM_POOL_FLUSH_PENDING;
3130	goto end;
3131	}
3132
3133	/* Check if DMA work is pending (2nd+ run). */
3134	if (VM_FF_IS_PENDING(pVM, VM_FF_PDM_DMA) && cResume > 1)
3135	{
3136	rc = VINF_EM_RAW_TO_R3;
3137	goto end;
3138	}
3139	}
3140
3141	#ifdef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
3142	/*
3143	* Exit to ring-3 preemption/work is pending.
3144	*
3145	* Interrupts are disabled before the call to make sure we don't miss any interrupt
3146	* that would flag preemption (IPI, timer tick, ++). (Would've been nice to do this
3147	* further down, but hmR0VmxCheckPendingInterrupt makes that impossible.)
3148	*
3149	* Note! Interrupts must be disabled done before we check for TLB flushes; TLB
3150	* shootdowns rely on this.
3151	*/
3152	uOldEFlags = ASMIntDisableFlags();
3153	if (RTThreadPreemptIsPending(NIL_RTTHREAD))
3154	{
3155	STAM_COUNTER_INC(&pVCpu->hm.s.StatPendingHostIrq);
3156	rc = VINF_EM_RAW_INTERRUPT;
3157	goto end;
3158	}
3159	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
3160	#endif
3161
3162	/*
3163	* When external interrupts are pending, we should exit the VM when IF is set.
3164	* Note: After VM_FF_INHIBIT_INTERRUPTS check!
3165	*/
3166	rc = hmR0VmxCheckPendingInterrupt(pVM, pVCpu, pCtx);
3167	if (RT_FAILURE(rc))
3168	goto end;
3169
3170	/** @todo check timers?? */
3171
3172	/*
3173	* TPR caching using CR8 is only available in 64-bit mode.
3174	* Note: The 32-bit exception for AMD (X86_CPUID_AMD_FEATURE_ECX_CR8L), but this appears missing in Intel CPUs.
3175	* Note: We can't do this in LoadGuestState() as PDMApicGetTPR can jump back to ring-3 (lock)!! (no longer true) .
3176	*/
3177	/** @todo query and update the TPR only when it could have been changed (mmio
3178	* access & wrsmr (x2apic) */
3179	if (fSetupTPRCaching)
3180	{
3181	/* TPR caching in CR8 */
3182	bool fPending;
3183
3184	rc2 = PDMApicGetTPR(pVCpu, &u8LastTPR, &fPending, NULL /* pu8PendingIrq */);
3185	AssertRC(rc2);
3186	/* The TPR can be found at offset 0x80 in the APIC mmio page. */
3187	pVCpu->hm.s.vmx.pbVirtApic[0x80] = u8LastTPR;
3188
3189	/*
3190	* Two options here:
3191	* - external interrupt pending, but masked by the TPR value.
3192	* -> a CR8 update that lower the current TPR value should cause an exit
3193	* - no pending interrupts
3194	* -> We don't need to be explicitely notified. There are enough world switches for detecting pending interrupts.
3195	*/
3196
3197	/* cr8 bits 3-0 correspond to bits 7-4 of the task priority mmio register. */
3198	rc = VMXWriteVmcs(VMX_VMCS32_CTRL_TPR_THRESHOLD, (fPending) ? (u8LastTPR >> 4) : 0);
3199	AssertRC(VBOXSTRICTRC_VAL(rc));
3200
3201	if (pVM->hm.s.fTPRPatchingActive)
3202	{
3203	Assert(!CPUMIsGuestInLongModeEx(pCtx));
3204	/* Our patch code uses LSTAR for TPR caching. */
3205	pCtx->msrLSTAR = u8LastTPR;
3206
3207	/** @todo r=ramshankar: we should check for MSR-bitmap support here. */
3208	if (fPending)
3209	{
3210	/* A TPR change could activate a pending interrupt, so catch lstar writes. */
3211	hmR0VmxSetMSRPermission(pVCpu, MSR_K8_LSTAR, true, false);
3212	}
3213	else
3214	{
3215	/*
3216	* No interrupts are pending, so we don't need to be explicitely notified.
3217	* There are enough world switches for detecting pending interrupts.
3218	*/
3219	hmR0VmxSetMSRPermission(pVCpu, MSR_K8_LSTAR, true, true);
3220	}
3221	}
3222	}
3223
3224	#ifdef LOG_ENABLED
3225	if ( pVM->hm.s.fNestedPaging
3226	\|\| pVM->hm.s.vmx.fVpid)
3227	{
3228	PHMGLOBLCPUINFO pCpu = HMR0GetCurrentCpu();
3229	if (pVCpu->hm.s.idLastCpu != pCpu->idCpu)
3230	{
3231	LogFlow(("Force TLB flush due to rescheduling to a different cpu (%d vs %d)\n", pVCpu->hm.s.idLastCpu,
3232	pCpu->idCpu));
3233	}
3234	else if (pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
3235	{
3236	LogFlow(("Force TLB flush due to changed TLB flush count (%x vs %x)\n", pVCpu->hm.s.cTlbFlushes,
3237	pCpu->cTlbFlushes));
3238	}
3239	else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_TLB_FLUSH))
3240	LogFlow(("Manual TLB flush\n"));
3241	}
3242	#endif
3243	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
3244	PGMRZDynMapFlushAutoSet(pVCpu);
3245	#endif
3246
3247	/*
3248	* NOTE: DO NOT DO ANYTHING AFTER THIS POINT THAT MIGHT JUMP BACK TO RING-3!
3249	* (until the actual world switch)
3250	*/
3251	#ifdef VBOX_STRICT
3252	idCpuCheck = RTMpCpuId();
3253	#endif
3254	#ifdef LOG_ENABLED
3255	VMMR0LogFlushDisable(pVCpu);
3256	#endif
3257
3258	/*
3259	* Save the host state first.
3260	*/
3261	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_HOST_CONTEXT)
3262	{
3263	rc = VMXR0SaveHostState(pVM, pVCpu);
3264	if (RT_UNLIKELY(rc != VINF_SUCCESS))
3265	{
3266	VMMR0LogFlushEnable(pVCpu);
3267	goto end;
3268	}
3269	}
3270
3271	/*
3272	* Load the guest state.
3273	*/
3274	if (!pVCpu->hm.s.fContextUseFlags)
3275	{
3276	VMXR0LoadMinimalGuestState(pVM, pVCpu, pCtx);
3277	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadMinimal);
3278	if (fNeedTscSetup)
3279	{
3280	VMXR0SetupTscOffsetAndPreemption(pVM, pVCpu);
3281	fNeedTscSetup = false;
3282	}
3283	}
3284	else
3285	{
3286	rc = VMXR0LoadGuestState(pVM, pVCpu, pCtx);
3287	if (RT_UNLIKELY(rc != VINF_SUCCESS))
3288	{
3289	VMMR0LogFlushEnable(pVCpu);
3290	goto end;
3291	}
3292	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadFull);
3293	VMXR0SetupTscOffsetAndPreemption(pVM, pVCpu);
3294	}
3295
3296	#ifndef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
3297	/*
3298	* Disable interrupts to make sure a poke will interrupt execution.
3299	* This must be done before we check for TLB flushes; TLB shootdowns rely on this.
3300	*/
3301	uOldEFlags = ASMIntDisableFlags();
3302	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
3303	#endif
3304
3305	/* Non-register state Guest Context */
3306	/** @todo change me according to cpu state */
3307	rc2 = VMXWriteVmcs(VMX_VMCS32_GUEST_ACTIVITY_STATE, VMX_VMCS_GUEST_ACTIVITY_ACTIVE);
3308	AssertRC(rc2);
3309
3310	/* Set TLB flush state as checked until we return from the world switch. */
3311	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true);
3312	/* Deal with tagged TLB setup and invalidation. */
3313	pVM->hm.s.vmx.pfnFlushTaggedTlb(pVM, pVCpu);
3314
3315	/*
3316	* Manual save and restore:
3317	* - General purpose registers except RIP, RSP
3318	*
3319	* Trashed:
3320	* - CR2 (we don't care)
3321	* - LDTR (reset to 0)
3322	* - DRx (presumably not changed at all)
3323	* - DR7 (reset to 0x400)
3324	* - EFLAGS (reset to RT_BIT(1); not relevant)
3325	*/
3326
3327	/* All done! Let's start VM execution. */
3328	Assert(idCpuCheck == RTMpCpuId());
3329
3330	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
3331	pVCpu->hm.s.vmx.VMCSCache.cResume = cResume;
3332	pVCpu->hm.s.vmx.VMCSCache.u64TimeSwitch = RTTimeNanoTS();
3333	#endif
3334
3335	/*
3336	* Save the current TPR value in the LSTAR MSR so our patches can access it.
3337	*/
3338	if (pVM->hm.s.fTPRPatchingActive)
3339	{
3340	Assert(pVM->hm.s.fTPRPatchingActive);
3341	u64OldLSTAR = ASMRdMsr(MSR_K8_LSTAR);
3342	ASMWrMsr(MSR_K8_LSTAR, u8LastTPR);
3343	}
3344
3345	TMNotifyStartOfExecution(pVCpu);
3346
3347	#ifndef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
3348	/*
3349	* Save the current Host TSC_AUX and write the guest TSC_AUX to the host, so that
3350	* RDTSCPs (that don't cause exits) reads the guest MSR. See @bugref{3324}.
3351	*/
3352	if ( (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
3353	&& !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
3354	{
3355	pVCpu->hm.s.u64HostTscAux = ASMRdMsr(MSR_K8_TSC_AUX);
3356	uint64_t u64HostTscAux = 0;
3357	rc2 = CPUMQueryGuestMsr(pVCpu, MSR_K8_TSC_AUX, &u64HostTscAux);
3358	AssertRC(rc2);
3359	ASMWrMsr(MSR_K8_TSC_AUX, u64HostTscAux);
3360	}
3361	#endif
3362
3363	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
3364	#ifdef VBOX_WITH_KERNEL_USING_XMM
3365	rc = HMR0VMXStartVMWrapXMM(pVCpu->hm.s.fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hm.s.vmx.pfnStartVM);
3366	#else
3367	rc = pVCpu->hm.s.vmx.pfnStartVM(pVCpu->hm.s.fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu);
3368	#endif
3369	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatExit1, x);
3370	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false);
3371	ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits);
3372
3373	/* Possibly the last TSC value seen by the guest (too high) (only when we're in TSC offset mode). */
3374	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
3375	{
3376	#ifndef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
3377	/* Restore host's TSC_AUX. */
3378	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
3379	ASMWrMsr(MSR_K8_TSC_AUX, pVCpu->hm.s.u64HostTscAux);
3380	#endif
3381
3382	TMCpuTickSetLastSeen(pVCpu,
3383	ASMReadTSC() + pVCpu->hm.s.vmx.u64TSCOffset - 0x400 /* guestimate of world switch overhead in clock ticks */);
3384	}
3385
3386	TMNotifyEndOfExecution(pVCpu);
3387	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
3388	Assert(!(ASMGetFlags() & X86_EFL_IF));
3389
3390	/*
3391	* Restore the host LSTAR MSR if the guest could have changed it.
3392	*/
3393	if (pVM->hm.s.fTPRPatchingActive)
3394	{
3395	Assert(pVM->hm.s.fTPRPatchingActive);
3396	pVCpu->hm.s.vmx.pbVirtApic[0x80] = pCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR);
3397	ASMWrMsr(MSR_K8_LSTAR, u64OldLSTAR);
3398	}
3399
3400	ASMSetFlags(uOldEFlags);
3401	#ifdef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
3402	uOldEFlags = ~(RTCCUINTREG)0;
3403	#endif
3404
3405	AssertMsg(!pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries, ("pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries=%d\n",
3406	pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries));
3407
3408	/* In case we execute a goto ResumeExecution later on. */
3409	pVCpu->hm.s.fResumeVM = true;
3410	pVCpu->hm.s.fForceTLBFlush = false;
3411
3412	/*
3413	* !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
3414	* IMPORTANT: WE CAN'T DO ANY LOGGING OR OPERATIONS THAT CAN DO A LONGJMP BACK TO RING 3 BEFORE WE'VE SYNCED BACK (MOST OF) THE GUEST STATE
3415	* !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
3416	*/
3417
3418	if (RT_UNLIKELY(rc != VINF_SUCCESS))
3419	{
3420	hmR0VmxReportWorldSwitchError(pVM, pVCpu, rc, pCtx);
3421	VMMR0LogFlushEnable(pVCpu);
3422	goto end;
3423	}
3424
3425	/* Success. Query the guest state and figure out what has happened. */
3426
3427	/* Investigate why there was a VM-exit. */
3428	rc2 = VMXReadCachedVmcs(VMX_VMCS32_RO_EXIT_REASON, &exitReason);
3429	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[exitReason & MASK_EXITREASON_STAT]);
3430
3431	exitReason &= 0xffff; /* bit 0-15 contain the exit code. */
3432	rc2 \|= VMXReadCachedVmcs(VMX_VMCS32_RO_VM_INSTR_ERROR, &instrError);
3433	rc2 \|= VMXReadCachedVmcs(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &cbInstr);
3434	rc2 \|= VMXReadCachedVmcs(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &intInfo);
3435	/* might not be valid; depends on VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID. */
3436	rc2 \|= VMXReadCachedVmcs(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &errCode);
3437	rc2 \|= VMXReadCachedVmcs(VMX_VMCS32_RO_EXIT_INSTR_INFO, &instrInfo);
3438	rc2 \|= VMXReadCachedVmcs(VMX_VMCS_RO_EXIT_QUALIFICATION, &exitQualification);
3439	AssertRC(rc2);
3440
3441	/*
3442	* Sync back the guest state.
3443	*/
3444	rc2 = VMXR0SaveGuestState(pVM, pVCpu, pCtx);
3445	AssertRC(rc2);
3446
3447	/* Note! NOW IT'S SAFE FOR LOGGING! */
3448	VMMR0LogFlushEnable(pVCpu);
3449	Log2(("Raw exit reason %08x\n", exitReason));
3450	#if ARCH_BITS == 64 /* for the time being */
3451	VBOXVMM_R0_HMVMX_VMEXIT(pVCpu, pCtx, exitReason);
3452	#endif
3453
3454	/*
3455	* Check if an injected event was interrupted prematurely.
3456	*/
3457	rc2 = VMXReadCachedVmcs(VMX_VMCS32_RO_IDT_INFO, &val);
3458	AssertRC(rc2);
3459	pVCpu->hm.s.Event.u64IntrInfo = VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(val);
3460	if ( VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVCpu->hm.s.Event.u64IntrInfo)
3461	/* Ignore 'int xx' as they'll be restarted anyway. */
3462	&& VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hm.s.Event.u64IntrInfo) != VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT
3463	/* Ignore software exceptions (such as int3) as they'll reoccur when we restart the instruction anyway. */
3464	&& VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hm.s.Event.u64IntrInfo) != VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT)
3465	{
3466	Assert(!pVCpu->hm.s.Event.fPending);
3467	pVCpu->hm.s.Event.fPending = true;
3468	/* Error code present? */
3469	if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntrInfo))
3470	{
3471	rc2 = VMXReadCachedVmcs(VMX_VMCS32_RO_IDT_ERROR_CODE, &val);
3472	AssertRC(rc2);
3473	pVCpu->hm.s.Event.u32ErrCode = val;
3474	Log(("Pending inject %RX64 at %RGv exit=%08x intInfo=%08x exitQualification=%RGv pending error=%RX64\n",
3475	pVCpu->hm.s.Event.u64IntrInfo, (RTGCPTR)pCtx->rip, exitReason, intInfo, exitQualification, val));
3476	}
3477	else
3478	{
3479	Log(("Pending inject %RX64 at %RGv exit=%08x intInfo=%08x exitQualification=%RGv\n", pVCpu->hm.s.Event.u64IntrInfo,
3480	(RTGCPTR)pCtx->rip, exitReason, intInfo, exitQualification));
3481	pVCpu->hm.s.Event.u32ErrCode = 0;
3482	}
3483	}
3484	#ifdef VBOX_STRICT
3485	else if ( VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVCpu->hm.s.Event.u64IntrInfo)
3486	/* Ignore software exceptions (such as int3) as they're reoccur when we restart the instruction anyway. */
3487	&& VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hm.s.Event.u64IntrInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT)
3488	{
3489	Log(("Ignore pending inject %RX64 at %RGv exit=%08x intInfo=%08x exitQualification=%RGv\n",
3490	pVCpu->hm.s.Event.u64IntrInfo, (RTGCPTR)pCtx->rip, exitReason, intInfo, exitQualification));
3491	}
3492
3493	if (exitReason == VMX_EXIT_ERR_INVALID_GUEST_STATE)
3494	HMDumpRegs(pVM, pVCpu, pCtx);
3495	#endif
3496
3497	Log2(("E%d: New EIP=%x:%RGv\n", (uint32_t)exitReason, pCtx->cs.Sel, (RTGCPTR)pCtx->rip));
3498	Log2(("Exit reason %d, exitQualification %RGv\n", (uint32_t)exitReason, exitQualification));
3499	Log2(("instrInfo=%d instrError=%d instr length=%d\n", (uint32_t)instrInfo, (uint32_t)instrError, (uint32_t)cbInstr));
3500	Log2(("Interruption error code %d\n", (uint32_t)errCode));
3501	Log2(("IntInfo = %08x\n", (uint32_t)intInfo));
3502
3503	/*
3504	* Sync back the TPR if it was changed.
3505	*/
3506	if ( fSetupTPRCaching
3507	&& u8LastTPR != pVCpu->hm.s.vmx.pbVirtApic[0x80])
3508	{
3509	rc2 = PDMApicSetTPR(pVCpu, pVCpu->hm.s.vmx.pbVirtApic[0x80]);
3510	AssertRC(rc2);
3511	}
3512
3513	#ifdef DBGFTRACE_ENABLED /** @todo DTrace later. */
3514	RTTraceBufAddMsgF(pVM->CTX_SUFF(hTraceBuf), "vmexit %08x %016RX64 at %04:%08RX64 %RX64",
3515	exitReason, (uint64_t)exitQualification, pCtx->cs.Sel, pCtx->rip, (uint64_t)intInfo);
3516	#endif
3517	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
3518
3519	/* Some cases don't need a complete resync of the guest CPU state; handle them here. */
3520	Assert(rc == VINF_SUCCESS); /* might consider VERR_IPE_UNINITIALIZED_STATUS here later... */
3521	switch (exitReason)
3522	{
3523	case VMX_EXIT_XCPT_NMI: /* 0 Exception or non-maskable interrupt (NMI). */
3524	case VMX_EXIT_EXT_INT: /* 1 External interrupt. */
3525	{
3526	uint32_t vector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(intInfo);
3527
3528	if (!VMX_EXIT_INTERRUPTION_INFO_IS_VALID(intInfo))
3529	{
3530	Assert(exitReason == VMX_EXIT_EXT_INT);
3531	/* External interrupt; leave to allow it to be dispatched again. */
3532	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
3533	rc = VINF_EM_RAW_INTERRUPT;
3534	break;
3535	}
3536	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
3537	switch (VMX_EXIT_INTERRUPTION_INFO_TYPE(intInfo))
3538	{
3539	case VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI: /* Non-maskable interrupt. */
3540	/* External interrupt; leave to allow it to be dispatched again. */
3541	rc = VINF_EM_RAW_INTERRUPT;
3542	break;
3543
3544	case VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT: /* External hardware interrupt. */
3545	AssertFailed(); /* can't come here; fails the first check. */
3546	break;
3547
3548	case VMX_EXIT_INTERRUPTION_INFO_TYPE_DB_XCPT: /* Unknown why we get this type for #DB */
3549	case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
3550	Assert(vector == 1 \|\| vector == 3 \|\| vector == 4);
3551	/* no break */
3552	case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT: /* Hardware exception. */
3553	Log2(("Hardware/software interrupt %d\n", vector));
3554	switch (vector)
3555	{
3556	case X86_XCPT_NM:
3557	{
3558	Log(("#NM fault at %RGv error code %x\n", (RTGCPTR)pCtx->rip, errCode));
3559
3560	/** @todo don't intercept #NM exceptions anymore when we've activated the guest FPU state. */
3561	/* If we sync the FPU/XMM state on-demand, then we can continue execution as if nothing has happened. */
3562	rc = CPUMR0LoadGuestFPU(pVM, pVCpu, pCtx);
3563	if (rc == VINF_SUCCESS)
3564	{
3565	Assert(CPUMIsGuestFPUStateActive(pVCpu));
3566
3567	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowNM);
3568
3569	/* Continue execution. */
3570	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
3571
3572	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
3573	goto ResumeExecution;
3574	}
3575
3576	Log(("Forward #NM fault to the guest\n"));
3577	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
3578	rc2 = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo),
3579	cbInstr, 0);
3580	AssertRC(rc2);
3581	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
3582	goto ResumeExecution;
3583	}
3584
3585	case X86_XCPT_PF: /* Page fault */
3586	{
3587	#ifdef VBOX_ALWAYS_TRAP_PF
3588	if (pVM->hm.s.fNestedPaging)
3589	{
3590	/*
3591	* A genuine pagefault. Forward the trap to the guest by injecting the exception and resuming execution.
3592	*/
3593	Log(("Guest page fault at %RGv cr2=%RGv error code %RGv rsp=%RGv\n", (RTGCPTR)pCtx->rip, exitQualification,
3594	errCode, (RTGCPTR)pCtx->rsp));
3595
3596	Assert(CPUMIsGuestInPagedProtectedModeEx(pCtx));
3597
3598	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
3599
3600	/* Now we must update CR2. */
3601	pCtx->cr2 = exitQualification;
3602	rc2 = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo),
3603	cbInstr, errCode);
3604	AssertRC(rc2);
3605
3606	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
3607	goto ResumeExecution;
3608	}
3609	#else
3610	Assert(!pVM->hm.s.fNestedPaging);
3611	#endif
3612
3613	#ifdef VBOX_HM_WITH_GUEST_PATCHING
3614	/* Shortcut for APIC TPR reads and writes; 32 bits guests only */
3615	if ( pVM->hm.s.fTRPPatchingAllowed
3616	&& pVM->hm.s.pGuestPatchMem
3617	&& (exitQualification & 0xfff) == 0x080
3618	&& !(errCode & X86_TRAP_PF_P) /* not present */
3619	&& CPUMGetGuestCPL(pVCpu) == 0
3620	&& !CPUMIsGuestInLongModeEx(pCtx)
3621	&& pVM->hm.s.cPatches < RT_ELEMENTS(pVM->hm.s.aPatches))
3622	{
3623	RTGCPHYS GCPhysApicBase, GCPhys;
3624	GCPhysApicBase = pCtx->msrApicBase;
3625	GCPhysApicBase &= PAGE_BASE_GC_MASK;
3626
3627	rc = PGMGstGetPage(pVCpu, (RTGCPTR)exitQualification, NULL, &GCPhys);
3628	if ( rc == VINF_SUCCESS
3629	&& GCPhys == GCPhysApicBase)
3630	{
3631	/* Only attempt to patch the instruction once. */
3632	PHMTPRPATCH pPatch = (PHMTPRPATCH)RTAvloU32Get(&pVM->hm.s.PatchTree, (AVLOU32KEY)pCtx->eip);
3633	if (!pPatch)
3634	{
3635	rc = VINF_EM_HM_PATCH_TPR_INSTR;
3636	break;
3637	}
3638	}
3639	}
3640	#endif
3641
3642	Log2(("Page fault at %RGv error code %x\n", exitQualification, errCode));
3643	/* Exit qualification contains the linear address of the page fault. */
3644	TRPMAssertTrap(pVCpu, X86_XCPT_PF, TRPM_TRAP);
3645	TRPMSetErrorCode(pVCpu, errCode);
3646	TRPMSetFaultAddress(pVCpu, exitQualification);
3647
3648	/* Shortcut for APIC TPR reads and writes. */
3649	if ( (exitQualification & 0xfff) == 0x080
3650	&& !(errCode & X86_TRAP_PF_P) /* not present */
3651	&& fSetupTPRCaching
3652	&& (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC))
3653	{
3654	RTGCPHYS GCPhysApicBase, GCPhys;
3655	GCPhysApicBase = pCtx->msrApicBase;
3656	GCPhysApicBase &= PAGE_BASE_GC_MASK;
3657
3658	rc = PGMGstGetPage(pVCpu, (RTGCPTR)exitQualification, NULL, &GCPhys);
3659	if ( rc == VINF_SUCCESS
3660	&& GCPhys == GCPhysApicBase)
3661	{
3662	Log(("Enable VT-x virtual APIC access filtering\n"));
3663	rc2 = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess,
3664	X86_PTE_RW \| X86_PTE_P);
3665	AssertRC(rc2);
3666	}
3667	}
3668
3669	/* Forward it to our trap handler first, in case our shadow pages are out of sync. */
3670	rc = PGMTrap0eHandler(pVCpu, errCode, CPUMCTX2CORE(pCtx), (RTGCPTR)exitQualification);
3671	Log2(("PGMTrap0eHandler %RGv returned %Rrc\n", (RTGCPTR)pCtx->rip, VBOXSTRICTRC_VAL(rc)));
3672
3673	if (rc == VINF_SUCCESS)
3674	{ /* We've successfully synced our shadow pages, so let's just continue execution. */
3675	Log2(("Shadow page fault at %RGv cr2=%RGv error code %x\n", (RTGCPTR)pCtx->rip, exitQualification ,errCode));
3676	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
3677
3678	TRPMResetTrap(pVCpu);
3679	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
3680	goto ResumeExecution;
3681	}
3682	else if (rc == VINF_EM_RAW_GUEST_TRAP)
3683	{
3684	/*
3685	* A genuine pagefault. Forward the trap to the guest by injecting the exception and resuming execution.
3686	*/
3687	Log2(("Forward page fault to the guest\n"));
3688
3689	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
3690	/* The error code might have been changed. */
3691	errCode = TRPMGetErrorCode(pVCpu);
3692
3693	TRPMResetTrap(pVCpu);
3694
3695	/* Now we must update CR2. */
3696	pCtx->cr2 = exitQualification;
3697	rc2 = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo),
3698	cbInstr, errCode);
3699	AssertRC(rc2);
3700
3701	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
3702	goto ResumeExecution;
3703	}
3704	#ifdef VBOX_STRICT
3705	if (rc != VINF_EM_RAW_EMULATE_INSTR && rc != VINF_EM_RAW_EMULATE_IO_BLOCK)
3706	Log2(("PGMTrap0eHandler failed with %d\n", VBOXSTRICTRC_VAL(rc)));
3707	#endif
3708	/* Need to go back to the recompiler to emulate the instruction. */
3709	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
3710	TRPMResetTrap(pVCpu);
3711
3712	/* If event delivery caused the #PF (shadow or not), tell TRPM. */
3713	hmR0VmxCheckPendingEvent(pVCpu);
3714	break;
3715	}
3716
3717	case X86_XCPT_MF: /* Floating point exception. */
3718	{
3719	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
3720	if (!(pCtx->cr0 & X86_CR0_NE))
3721	{
3722	/* old style FPU error reporting needs some extra work. */
3723	/** @todo don't fall back to the recompiler, but do it manually. */
3724	rc = VINF_EM_RAW_EMULATE_INSTR;
3725	break;
3726	}
3727	Log(("Trap %x at %04X:%RGv\n", vector, pCtx->cs.Sel, (RTGCPTR)pCtx->rip));
3728	rc2 = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo),
3729	cbInstr, errCode);
3730	AssertRC(rc2);
3731
3732	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
3733	goto ResumeExecution;
3734	}
3735
3736	case X86_XCPT_DB: /* Debug exception. */
3737	{
3738	uint64_t uDR6;
3739
3740	/*
3741	* DR6, DR7.GD and IA32_DEBUGCTL.LBR are not updated yet.
3742	*
3743	* Exit qualification bits:
3744	* 3:0 B0-B3 which breakpoint condition was met
3745	* 12:4 Reserved (0)
3746	* 13 BD - debug register access detected
3747	* 14 BS - single step execution or branch taken
3748	* 63:15 Reserved (0)
3749	*/
3750	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
3751
3752	/* Note that we don't support guest and host-initiated debugging at the same time. */
3753
3754	uDR6 = X86_DR6_INIT_VAL;
3755	uDR6 \|= (exitQualification & (X86_DR6_B0\|X86_DR6_B1\|X86_DR6_B2\|X86_DR6_B3\|X86_DR6_BD\|X86_DR6_BS));
3756	rc = DBGFRZTrap01Handler(pVM, pVCpu, CPUMCTX2CORE(pCtx), uDR6);
3757	if (rc == VINF_EM_RAW_GUEST_TRAP)
3758	{
3759	/* Update DR6 here. */
3760	pCtx->dr[6] = uDR6;
3761
3762	/* Resync DR6 if the debug state is active. */
3763	if (CPUMIsGuestDebugStateActive(pVCpu))
3764	ASMSetDR6(pCtx->dr[6]);
3765
3766	/* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
3767	pCtx->dr[7] &= ~X86_DR7_GD;
3768
3769	/* Paranoia. */
3770	pCtx->dr[7] &= 0xffffffff; /* upper 32 bits reserved */
3771	pCtx->dr[7] &= ~(RT_BIT(11) \| RT_BIT(12) \| RT_BIT(14) \| RT_BIT(15)); /* must be zero */
3772	pCtx->dr[7] \|= 0x400; /* must be one */
3773
3774	/* Resync DR7 */
3775	rc2 = VMXWriteVmcs64(VMX_VMCS_GUEST_DR7, pCtx->dr[7]);
3776	AssertRC(rc2);
3777
3778	Log(("Trap %x (debug) at %RGv exit qualification %RX64 dr6=%x dr7=%x\n", vector, (RTGCPTR)pCtx->rip,
3779	exitQualification, (uint32_t)pCtx->dr[6], (uint32_t)pCtx->dr[7]));
3780	rc2 = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo),
3781	cbInstr, errCode);
3782	AssertRC(rc2);
3783
3784	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
3785	goto ResumeExecution;
3786	}
3787	/* Return to ring 3 to deal with the debug exit code. */
3788	Log(("Debugger hardware BP at %04x:%RGv (rc=%Rrc)\n", pCtx->cs.Sel, pCtx->rip, VBOXSTRICTRC_VAL(rc)));
3789	break;
3790	}
3791
3792	case X86_XCPT_BP: /* Breakpoint. */
3793	{
3794	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
3795	rc = DBGFRZTrap03Handler(pVM, pVCpu, CPUMCTX2CORE(pCtx));
3796	if (rc == VINF_EM_RAW_GUEST_TRAP)
3797	{
3798	Log(("Guest #BP at %04x:%RGv\n", pCtx->cs.Sel, pCtx->rip));
3799	rc2 = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo),
3800	cbInstr, errCode);
3801	AssertRC(rc2);
3802	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
3803	goto ResumeExecution;
3804	}
3805	if (rc == VINF_SUCCESS)
3806	{
3807	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
3808	goto ResumeExecution;
3809	}
3810	Log(("Debugger BP at %04x:%RGv (rc=%Rrc)\n", pCtx->cs.Sel, pCtx->rip, VBOXSTRICTRC_VAL(rc)));
3811	break;
3812	}
3813
3814	case X86_XCPT_GP: /* General protection failure exception. */
3815	{
3816	uint32_t cbOp;
3817	PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
3818
3819	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
3820	#ifdef VBOX_STRICT
3821	if ( !CPUMIsGuestInRealModeEx(pCtx)
3822	\|\| !pVM->hm.s.vmx.pRealModeTSS)
3823	{
3824	Log(("Trap %x at %04X:%RGv errorCode=%RGv\n", vector, pCtx->cs.Sel, (RTGCPTR)pCtx->rip, errCode));
3825	rc2 = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo),
3826	cbInstr, errCode);
3827	AssertRC(rc2);
3828	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
3829	goto ResumeExecution;
3830	}
3831	#endif
3832	Assert(CPUMIsGuestInRealModeEx(pCtx));
3833
3834	LogFlow(("Real mode X86_XCPT_GP instruction emulation at %x:%RGv\n", pCtx->cs.Sel, (RTGCPTR)pCtx->rip));
3835
3836	rc2 = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbOp);
3837	if (RT_SUCCESS(rc2))
3838	{
3839	bool fUpdateRIP = true;
3840
3841	rc = VINF_SUCCESS;
3842	Assert(cbOp == pDis->cbInstr);
3843	switch (pDis->pCurInstr->uOpcode)
3844	{
3845	case OP_CLI:
3846	pCtx->eflags.Bits.u1IF = 0;
3847	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCli);
3848	break;
3849
3850	case OP_STI:
3851	pCtx->eflags.Bits.u1IF = 1;
3852	EMSetInhibitInterruptsPC(pVCpu, pCtx->rip + pDis->cbInstr);
3853	Assert(VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
3854	rc2 = VMXWriteVmcs(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE,
3855	VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
3856	AssertRC(rc2);
3857	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitSti);
3858	break;
3859
3860	case OP_HLT:
3861	fUpdateRIP = false;
3862	rc = VINF_EM_HALT;
3863	pCtx->rip += pDis->cbInstr;
3864	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
3865	break;
3866
3867	case OP_POPF:
3868	{
3869	RTGCPTR GCPtrStack;
3870	uint32_t cbParm;
3871	uint32_t uMask;
3872	X86EFLAGS eflags;
3873
3874	if (pDis->fPrefix & DISPREFIX_OPSIZE)
3875	{
3876	cbParm = 4;
3877	uMask = 0xffffffff;
3878	}
3879	else
3880	{
3881	cbParm = 2;
3882	uMask = 0xffff;
3883	}
3884
3885	rc2 = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pCtx), pCtx->esp & uMask, 0, &GCPtrStack);
3886	if (RT_FAILURE(rc2))
3887	{
3888	rc = VERR_EM_INTERPRETER;
3889	break;
3890	}
3891	eflags.u = 0;
3892	rc2 = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &eflags.u, cbParm);
3893	if (RT_FAILURE(rc2))
3894	{
3895	rc = VERR_EM_INTERPRETER;
3896	break;
3897	}
3898	LogFlow(("POPF %x -> %RGv mask=%x\n", eflags.u, pCtx->rsp, uMask));
3899	pCtx->eflags.u = (pCtx->eflags.u & ~(X86_EFL_POPF_BITS & uMask))
3900	\| (eflags.u & X86_EFL_POPF_BITS & uMask);
3901	/* RF cleared when popped in real mode; see pushf description in AMD manual. */
3902	pCtx->eflags.Bits.u1RF = 0;
3903	pCtx->esp += cbParm;
3904	pCtx->esp &= uMask;
3905
3906	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPopf);
3907	break;
3908	}
3909
3910	case OP_PUSHF:
3911	{
3912	RTGCPTR GCPtrStack;
3913	uint32_t cbParm;
3914	uint32_t uMask;
3915	X86EFLAGS eflags;
3916
3917	if (pDis->fPrefix & DISPREFIX_OPSIZE)
3918	{
3919	cbParm = 4;
3920	uMask = 0xffffffff;
3921	}
3922	else
3923	{
3924	cbParm = 2;
3925	uMask = 0xffff;
3926	}
3927
3928	rc2 = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pCtx), (pCtx->esp - cbParm) & uMask, 0,
3929	&GCPtrStack);
3930	if (RT_FAILURE(rc2))
3931	{
3932	rc = VERR_EM_INTERPRETER;
3933	break;
3934	}
3935	eflags = pCtx->eflags;
3936	/* RF & VM cleared when pushed in real mode; see pushf description in AMD manual. */
3937	eflags.Bits.u1RF = 0;
3938	eflags.Bits.u1VM = 0;
3939
3940	rc2 = PGMPhysWrite(pVM, (RTGCPHYS)GCPtrStack, &eflags.u, cbParm);
3941	if (RT_FAILURE(rc2))
3942	{
3943	rc = VERR_EM_INTERPRETER;
3944	break;
3945	}
3946	LogFlow(("PUSHF %x -> %RGv\n", eflags.u, GCPtrStack));
3947	pCtx->esp -= cbParm;
3948	pCtx->esp &= uMask;
3949	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPushf);
3950	break;
3951	}
3952
3953	case OP_IRET:
3954	{
3955	RTGCPTR GCPtrStack;
3956	uint32_t uMask = 0xffff;
3957	uint16_t aIretFrame[3];
3958
3959	if (pDis->fPrefix & (DISPREFIX_OPSIZE \| DISPREFIX_ADDRSIZE))
3960	{
3961	rc = VERR_EM_INTERPRETER;
3962	break;
3963	}
3964
3965	rc2 = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pCtx), pCtx->esp & uMask, 0, &GCPtrStack);
3966	if (RT_FAILURE(rc2))
3967	{
3968	rc = VERR_EM_INTERPRETER;
3969	break;
3970	}
3971	rc2 = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &aIretFrame[0], sizeof(aIretFrame));
3972	if (RT_FAILURE(rc2))
3973	{
3974	rc = VERR_EM_INTERPRETER;
3975	break;
3976	}
3977	pCtx->ip = aIretFrame[0];
3978	pCtx->cs.Sel = aIretFrame[1];
3979	pCtx->cs.ValidSel = aIretFrame[1];
3980	pCtx->cs.u64Base = (uint32_t)pCtx->cs.Sel << 4;
3981	pCtx->eflags.u = (pCtx->eflags.u & ~(X86_EFL_POPF_BITS & uMask))
3982	\| (aIretFrame[2] & X86_EFL_POPF_BITS & uMask);
3983	pCtx->sp += sizeof(aIretFrame);
3984
3985	LogFlow(("iret to %04x:%x\n", pCtx->cs.Sel, pCtx->ip));
3986	fUpdateRIP = false;
3987	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIret);
3988	break;
3989	}
3990
3991	case OP_INT:
3992	{
3993	uint32_t intInfo2;
3994
3995	LogFlow(("Realmode: INT %x\n", pDis->Param1.uValue & 0xff));
3996	intInfo2 = pDis->Param1.uValue & 0xff;
3997	intInfo2 \|= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
3998	intInfo2 \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
3999
4000	rc = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, intInfo2, cbOp, 0);
4001	AssertRC(VBOXSTRICTRC_VAL(rc));
4002	fUpdateRIP = false;
4003	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
4004	break;
4005	}
4006
4007	case OP_INTO:
4008	{
4009	if (pCtx->eflags.Bits.u1OF)
4010	{
4011	uint32_t intInfo2;
4012
4013	LogFlow(("Realmode: INTO\n"));
4014	intInfo2 = X86_XCPT_OF;
4015	intInfo2 \|= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
4016	intInfo2 \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
4017
4018	rc = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, intInfo2, cbOp, 0);
4019	AssertRC(VBOXSTRICTRC_VAL(rc));
4020	fUpdateRIP = false;
4021	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
4022	}
4023	break;
4024	}
4025
4026	case OP_INT3:
4027	{
4028	uint32_t intInfo2;
4029
4030	LogFlow(("Realmode: INT 3\n"));
4031	intInfo2 = 3;
4032	intInfo2 \|= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
4033	intInfo2 \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
4034
4035	rc = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, intInfo2, cbOp, 0);
4036	AssertRC(VBOXSTRICTRC_VAL(rc));
4037	fUpdateRIP = false;
4038	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
4039	break;
4040	}
4041
4042	default:
4043	rc = EMInterpretInstructionDisasState(pVCpu, pDis, CPUMCTX2CORE(pCtx), 0, EMCODETYPE_SUPERVISOR);
4044	fUpdateRIP = false;
4045	break;
4046	}
4047
4048	if (rc == VINF_SUCCESS)
4049	{
4050	if (fUpdateRIP)
4051	pCtx->rip += cbOp; /* Move on to the next instruction. */
4052
4053	/*
4054	* LIDT, LGDT can end up here. In the future CRx changes as well. Just reload the
4055	* whole context to be done with it.
4056	*/
4057	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_ALL_GUEST;
4058
4059	/* Only resume if successful. */
4060	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
4061	goto ResumeExecution;
4062	}
4063	}
4064	else
4065	rc = VERR_EM_INTERPRETER;
4066
4067	AssertMsg(rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_EM_HALT,
4068	("Unexpected rc=%Rrc\n", VBOXSTRICTRC_VAL(rc)));
4069	break;
4070	}
4071
4072	#ifdef VBOX_STRICT
4073	case X86_XCPT_XF: /* SIMD exception. */
4074	case X86_XCPT_DE: /* Divide error. */
4075	case X86_XCPT_UD: /* Unknown opcode exception. */
4076	case X86_XCPT_SS: /* Stack segment exception. */
4077	case X86_XCPT_NP: /* Segment not present exception. */
4078	{
4079	switch (vector)
4080	{
4081	case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE); break;
4082	case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD); break;
4083	case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS); break;
4084	case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP); break;
4085	case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF); break;
4086	}
4087
4088	Log(("Trap %x at %04X:%RGv\n", vector, pCtx->cs.Sel, (RTGCPTR)pCtx->rip));
4089	rc2 = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo),
4090	cbInstr, errCode);
4091	AssertRC(rc2);
4092
4093	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
4094	goto ResumeExecution;
4095	}
4096	#endif
4097	default:
4098	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
4099	if ( CPUMIsGuestInRealModeEx(pCtx)
4100	&& pVM->hm.s.vmx.pRealModeTSS)
4101	{
4102	Log(("Real Mode Trap %x at %04x:%04X error code %x\n", vector, pCtx->cs.Sel, pCtx->eip, errCode));
4103	rc = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo),
4104	cbInstr, errCode);
4105	AssertRC(VBOXSTRICTRC_VAL(rc)); /* Strict RC check below. */
4106
4107	/* Go back to ring-3 in case of a triple fault. */
4108	if ( vector == X86_XCPT_DF
4109	&& rc == VINF_EM_RESET)
4110	{
4111	break;
4112	}
4113
4114	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
4115	goto ResumeExecution;
4116	}
4117	AssertMsgFailed(("Unexpected vm-exit caused by exception %x\n", vector));
4118	rc = VERR_VMX_UNEXPECTED_EXCEPTION;
4119	break;
4120	} /* switch (vector) */
4121
4122	break;
4123
4124	default:
4125	rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_CODE;
4126	AssertMsgFailed(("Unexpected interruption code %x\n", intInfo));
4127	break;
4128	}
4129
4130	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
4131	break;
4132	}
4133
4134	/*
4135	* 48 EPT violation. An attempt to access memory with a guest-physical address was disallowed
4136	* by the configuration of the EPT paging structures.
4137	*/
4138	case VMX_EXIT_EPT_VIOLATION:
4139	{
4140	RTGCPHYS GCPhys;
4141
4142	Assert(pVM->hm.s.fNestedPaging);
4143
4144	rc2 = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
4145	AssertRC(rc2);
4146	Assert(((exitQualification >> 7) & 3) != 2);
4147
4148	/* Determine the kind of violation. */
4149	errCode = 0;
4150	if (exitQualification & VMX_EXIT_QUALIFICATION_EPT_INSTR_FETCH)
4151	errCode \|= X86_TRAP_PF_ID;
4152
4153	if (exitQualification & VMX_EXIT_QUALIFICATION_EPT_DATA_WRITE)
4154	errCode \|= X86_TRAP_PF_RW;
4155
4156	/* If the page is present, then it's a page level protection fault. */
4157	if (exitQualification & VMX_EXIT_QUALIFICATION_EPT_ENTRY_PRESENT)
4158	errCode \|= X86_TRAP_PF_P;
4159	else
4160	{
4161	/* Shortcut for APIC TPR reads and writes. */
4162	if ( (GCPhys & 0xfff) == 0x080
4163	&& GCPhys > 0x1000000 /* to skip VGA frame buffer accesses */
4164	&& fSetupTPRCaching
4165	&& (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC))
4166	{
4167	RTGCPHYS GCPhysApicBase;
4168	GCPhysApicBase = pCtx->msrApicBase;
4169	GCPhysApicBase &= PAGE_BASE_GC_MASK;
4170	if (GCPhys == GCPhysApicBase + 0x80)
4171	{
4172	Log(("Enable VT-x virtual APIC access filtering\n"));
4173	rc2 = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess,
4174	X86_PTE_RW \| X86_PTE_P);
4175	AssertRC(rc2);
4176	}
4177	}
4178	}
4179	Log(("EPT Page fault %x at %RGp error code %x\n", (uint32_t)exitQualification, GCPhys, errCode));
4180
4181	/* GCPhys contains the guest physical address of the page fault. */
4182	TRPMAssertTrap(pVCpu, X86_XCPT_PF, TRPM_TRAP);
4183	TRPMSetErrorCode(pVCpu, errCode);
4184	TRPMSetFaultAddress(pVCpu, GCPhys);
4185
4186	/* Handle the pagefault trap for the nested shadow table. */
4187	rc = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, errCode, CPUMCTX2CORE(pCtx), GCPhys);
4188
4189	/*
4190	* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment below, @bugref{6043}.
4191	*/
4192	if ( rc == VINF_SUCCESS
4193	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
4194	\|\| rc == VERR_PAGE_NOT_PRESENT)
4195	{
4196	/* We've successfully synced our shadow pages, so let's just continue execution. */
4197	Log2(("Shadow page fault at %RGv cr2=%RGp error code %x\n", (RTGCPTR)pCtx->rip, exitQualification , errCode));
4198	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
4199
4200	TRPMResetTrap(pVCpu);
4201	goto ResumeExecution;
4202	}
4203
4204	#ifdef VBOX_STRICT
4205	if (rc != VINF_EM_RAW_EMULATE_INSTR)
4206	LogFlow(("PGMTrap0eHandlerNestedPaging at %RGv failed with %Rrc\n", (RTGCPTR)pCtx->rip, VBOXSTRICTRC_VAL(rc)));
4207	#endif
4208	/* Need to go back to the recompiler to emulate the instruction. */
4209	TRPMResetTrap(pVCpu);
4210	break;
4211	}
4212
4213	case VMX_EXIT_EPT_MISCONFIG:
4214	{
4215	RTGCPHYS GCPhys;
4216
4217	Assert(pVM->hm.s.fNestedPaging);
4218
4219	rc2 = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
4220	AssertRC(rc2);
4221	Log(("VMX_EXIT_EPT_MISCONFIG for %RGp\n", GCPhys));
4222
4223	/* Shortcut for APIC TPR reads and writes. */
4224	if ( (GCPhys & 0xfff) == 0x080
4225	&& GCPhys > 0x1000000 /* to skip VGA frame buffer accesses */
4226	&& fSetupTPRCaching
4227	&& (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC))
4228	{
4229	RTGCPHYS GCPhysApicBase = pCtx->msrApicBase;
4230	GCPhysApicBase &= PAGE_BASE_GC_MASK;
4231	if (GCPhys == GCPhysApicBase + 0x80)
4232	{
4233	Log(("Enable VT-x virtual APIC access filtering\n"));
4234	rc2 = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess,
4235	X86_PTE_RW \| X86_PTE_P);
4236	AssertRC(rc2);
4237	}
4238	}
4239
4240	rc = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pCtx), GCPhys, UINT32_MAX);
4241
4242	/*
4243	* If we succeed, resume execution.
4244	* Or, if fail in interpreting the instruction because we couldn't get the guest physical address
4245	* of the page containing the instruction via the guest's page tables (we would invalidate the guest page
4246	* in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
4247	* weird case. See @bugref{6043}.
4248	*/
4249	if ( rc == VINF_SUCCESS
4250	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
4251	\|\| rc == VERR_PAGE_NOT_PRESENT)
4252	{
4253	Log2(("PGMR0Trap0eHandlerNPMisconfig(,,,%RGp) at %RGv -> resume\n", GCPhys, (RTGCPTR)pCtx->rip));
4254	goto ResumeExecution;
4255	}
4256
4257	Log2(("PGMR0Trap0eHandlerNPMisconfig(,,,%RGp) at %RGv -> %Rrc\n", GCPhys, (RTGCPTR)pCtx->rip, VBOXSTRICTRC_VAL(rc)));
4258	break;
4259	}
4260
4261	case VMX_EXIT_INT_WINDOW: /* 7 Interrupt window exiting. */
4262	/* Clear VM-exit on IF=1 change. */
4263	LogFlow(("VMX_EXIT_INT_WINDOW %RGv pending=%d IF=%d\n", (RTGCPTR)pCtx->rip,
4264	VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC\|VMCPU_FF_INTERRUPT_PIC)), pCtx->eflags.Bits.u1IF));
4265	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
4266	rc2 = VMXWriteVmcs(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
4267	AssertRC(rc2);
4268	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
4269	goto ResumeExecution; /* we check for pending guest interrupts there */
4270
4271	case VMX_EXIT_WBINVD: /* 54 Guest software attempted to execute WBINVD. (conditional) */
4272	case VMX_EXIT_INVD: /* 13 Guest software attempted to execute INVD. (unconditional) */
4273	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvd);
4274	/* Skip instruction and continue directly. */
4275	pCtx->rip += cbInstr;
4276	/* Continue execution.*/
4277	goto ResumeExecution;
4278
4279	case VMX_EXIT_CPUID: /* 10 Guest software attempted to execute CPUID. */
4280	{
4281	Log2(("VMX: Cpuid %x\n", pCtx->eax));
4282	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCpuid);
4283	rc = EMInterpretCpuId(pVM, pVCpu, CPUMCTX2CORE(pCtx));
4284	if (rc == VINF_SUCCESS)
4285	{
4286	/* Update EIP and continue execution. */
4287	Assert(cbInstr == 2);
4288	pCtx->rip += cbInstr;
4289	goto ResumeExecution;
4290	}
4291	AssertMsgFailed(("EMU: cpuid failed with %Rrc\n", VBOXSTRICTRC_VAL(rc)));
4292	rc = VINF_EM_RAW_EMULATE_INSTR;
4293	break;
4294	}
4295
4296	case VMX_EXIT_RDPMC: /* 15 Guest software attempted to execute RDPMC. */
4297	{
4298	Log2(("VMX: Rdpmc %x\n", pCtx->ecx));
4299	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdpmc);
4300	rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pCtx));
4301	if (rc == VINF_SUCCESS)
4302	{
4303	/* Update EIP and continue execution. */
4304	Assert(cbInstr == 2);
4305	pCtx->rip += cbInstr;
4306	goto ResumeExecution;
4307	}
4308	rc = VINF_EM_RAW_EMULATE_INSTR;
4309	break;
4310	}
4311
4312	case VMX_EXIT_RDTSC: /* 16 Guest software attempted to execute RDTSC. */
4313	{
4314	Log2(("VMX: Rdtsc\n"));
4315	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
4316	rc = EMInterpretRdtsc(pVM, pVCpu, CPUMCTX2CORE(pCtx));
4317	if (rc == VINF_SUCCESS)
4318	{
4319	/* Update EIP and continue execution. */
4320	Assert(cbInstr == 2);
4321	pCtx->rip += cbInstr;
4322	fNeedTscSetup = true; /* See @bugref{6634}. */
4323	goto ResumeExecution;
4324	}
4325	rc = VINF_EM_RAW_EMULATE_INSTR;
4326	break;
4327	}
4328
4329	case VMX_EXIT_RDTSCP: /* 51 Guest software attempted to execute RDTSCP. */
4330	{
4331	Log2(("VMX: Rdtscp\n"));
4332	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtscp);
4333	rc = EMInterpretRdtscp(pVM, pVCpu, pCtx);
4334	if (rc == VINF_SUCCESS)
4335	{
4336	/* Update EIP and continue execution. */
4337	Assert(cbInstr == 3);
4338	pCtx->rip += cbInstr;
4339	fNeedTscSetup = true; /* See @bugref{6634}. */
4340	goto ResumeExecution;
4341	}
4342	rc = VINF_EM_RAW_EMULATE_INSTR;
4343	break;
4344	}
4345
4346	case VMX_EXIT_INVLPG: /* 14 Guest software attempted to execute INVLPG. */
4347	{
4348	Log2(("VMX: invlpg\n"));
4349	Assert(!pVM->hm.s.fNestedPaging);
4350
4351	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvlpg);
4352	rc = EMInterpretInvlpg(pVM, pVCpu, CPUMCTX2CORE(pCtx), exitQualification);
4353	if (rc == VINF_SUCCESS)
4354	{
4355	/* Update EIP and continue execution. */
4356	pCtx->rip += cbInstr;
4357	goto ResumeExecution;
4358	}
4359	AssertMsg(rc == VERR_EM_INTERPRETER, ("EMU: invlpg %RGv failed with %Rrc\n", exitQualification, VBOXSTRICTRC_VAL(rc)));
4360	break;
4361	}
4362
4363	case VMX_EXIT_MONITOR: /* 39 Guest software attempted to execute MONITOR. */
4364	{
4365	Log2(("VMX: monitor\n"));
4366
4367	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMonitor);
4368	rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pCtx));
4369	if (rc == VINF_SUCCESS)
4370	{
4371	/* Update EIP and continue execution. */
4372	pCtx->rip += cbInstr;
4373	goto ResumeExecution;
4374	}
4375	AssertMsg(rc == VERR_EM_INTERPRETER, ("EMU: monitor failed with %Rrc\n", VBOXSTRICTRC_VAL(rc)));
4376	break;
4377	}
4378
4379	case VMX_EXIT_WRMSR: /* 32 WRMSR. Guest software attempted to execute WRMSR. */
4380	/* When an interrupt is pending, we'll let MSR_K8_LSTAR writes fault in our TPR patch code. */
4381	if ( pVM->hm.s.fTPRPatchingActive
4382	&& pCtx->ecx == MSR_K8_LSTAR)
4383	{
4384	Assert(!CPUMIsGuestInLongModeEx(pCtx));
4385	if ((pCtx->eax & 0xff) != u8LastTPR)
4386	{
4387	Log(("VMX: Faulting MSR_K8_LSTAR write with new TPR value %x\n", pCtx->eax & 0xff));
4388
4389	/* Our patch code uses LSTAR for TPR caching. */
4390	rc2 = PDMApicSetTPR(pVCpu, pCtx->eax & 0xff);
4391	AssertRC(rc2);
4392	}
4393
4394	/* Skip the instruction and continue. */
4395	pCtx->rip += cbInstr; /* wrmsr = [0F 30] */
4396
4397	/* Only resume if successful. */
4398	goto ResumeExecution;
4399	}
4400	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_MSR;
4401	/* no break */
4402	case VMX_EXIT_RDMSR: /* 31 RDMSR. Guest software attempted to execute RDMSR. */
4403	{
4404	STAM_COUNTER_INC((exitReason == VMX_EXIT_RDMSR) ? &pVCpu->hm.s.StatExitRdmsr : &pVCpu->hm.s.StatExitWrmsr);
4405
4406	Log2(("VMX: %s\n", (exitReason == VMX_EXIT_RDMSR) ? "rdmsr" : "wrmsr"));
4407	rc = EMInterpretInstruction(pVCpu, CPUMCTX2CORE(pCtx), 0);
4408	if (rc == VINF_SUCCESS)
4409	{
4410	/* EIP has been updated already. */
4411	/* Only resume if successful. */
4412	goto ResumeExecution;
4413	}
4414	AssertMsg(rc == VERR_EM_INTERPRETER, ("EMU: %s failed with %Rrc\n",
4415	(exitReason == VMX_EXIT_RDMSR) ? "rdmsr" : "wrmsr", VBOXSTRICTRC_VAL(rc)));
4416	break;
4417	}
4418
4419	case VMX_EXIT_MOV_CRX: /* 28 Control-register accesses. */
4420	{
4421	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
4422
4423	switch (VMX_EXIT_QUALIFICATION_CRX_ACCESS(exitQualification))
4424	{
4425	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_WRITE:
4426	{
4427	Log2(("VMX: %RGv mov cr%d, x\n", (RTGCPTR)pCtx->rip, VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification)));
4428	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxWrite[VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification)]);
4429	rc = EMInterpretCRxWrite(pVM, pVCpu, CPUMCTX2CORE(pCtx),
4430	VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification),
4431	VMX_EXIT_QUALIFICATION_CRX_GENREG(exitQualification));
4432	switch (VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification))
4433	{
4434	case 0:
4435	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0 \| HM_CHANGED_GUEST_CR3;
4436	break;
4437	case 2:
4438	break;
4439	case 3:
4440	Assert(!pVM->hm.s.fNestedPaging \|\| !CPUMIsGuestInPagedProtectedModeEx(pCtx));
4441	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR3;
4442	break;
4443	case 4:
4444	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR4;
4445	break;
4446	case 8:
4447	/* CR8 contains the APIC TPR */
4448	Assert(!(pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1
4449	& VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
4450	break;
4451
4452	default:
4453	AssertFailed();
4454	break;
4455	}
4456	break;
4457	}
4458
4459	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ:
4460	{
4461	Log2(("VMX: mov x, crx\n"));
4462	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxRead[VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification)]);
4463
4464	Assert( !pVM->hm.s.fNestedPaging
4465	\|\| !CPUMIsGuestInPagedProtectedModeEx(pCtx)
4466	\|\| VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification) != DISCREG_CR3);
4467
4468	/* CR8 reads only cause an exit when the TPR shadow feature isn't present. */
4469	Assert( VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification) != 8
4470	\|\| !(pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
4471
4472	rc = EMInterpretCRxRead(pVM, pVCpu, CPUMCTX2CORE(pCtx),
4473	VMX_EXIT_QUALIFICATION_CRX_GENREG(exitQualification),
4474	VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification));
4475	break;
4476	}
4477
4478	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_CLTS:
4479	{
4480	Log2(("VMX: clts\n"));
4481	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
4482	rc = EMInterpretCLTS(pVM, pVCpu);
4483	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
4484	break;
4485	}
4486
4487	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_LMSW:
4488	{
4489	Log2(("VMX: lmsw %x\n", VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(exitQualification)));
4490	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
4491	rc = EMInterpretLMSW(pVM, pVCpu, CPUMCTX2CORE(pCtx), VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(exitQualification));
4492	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
4493	break;
4494	}
4495	}
4496
4497	/* Update EIP if no error occurred. */
4498	if (RT_SUCCESS(rc))
4499	pCtx->rip += cbInstr;
4500
4501	if (rc == VINF_SUCCESS)
4502	{
4503	/* Only resume if successful. */
4504	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
4505	goto ResumeExecution;
4506	}
4507	Assert(rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_PGM_SYNC_CR3);
4508	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
4509	break;
4510	}
4511
4512	case VMX_EXIT_MOV_DRX: /* 29 Debug-register accesses. */
4513	{
4514	if ( !DBGFIsStepping(pVCpu)
4515	&& !CPUMIsHyperDebugStateActive(pVCpu))
4516	{
4517	/* Disable DRx move intercepts. */
4518	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
4519	rc2 = VMXWriteVmcs(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
4520	AssertRC(rc2);
4521
4522	/* Save the host and load the guest debug state. */
4523	rc2 = CPUMR0LoadGuestDebugState(pVM, pVCpu, pCtx, true /* include DR6 */);
4524	AssertRC(rc2);
4525
4526	#ifdef LOG_ENABLED
4527	if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(exitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
4528	{
4529	Log(("VMX_EXIT_MOV_DRX: write DR%d genreg %d\n", VMX_EXIT_QUALIFICATION_DRX_REGISTER(exitQualification),
4530	VMX_EXIT_QUALIFICATION_DRX_GENREG(exitQualification)));
4531	}
4532	else
4533	Log(("VMX_EXIT_MOV_DRX: read DR%d\n", VMX_EXIT_QUALIFICATION_DRX_REGISTER(exitQualification)));
4534	#endif
4535
4536	#ifdef VBOX_WITH_STATISTICS
4537	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
4538	if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(exitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
4539	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
4540	else
4541	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
4542	#endif
4543
4544	goto ResumeExecution;
4545	}
4546
4547	/** @todo clear VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT after the first
4548	* time and restore DRx registers afterwards */
4549	if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(exitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
4550	{
4551	Log2(("VMX: mov DRx%d, genreg%d\n", VMX_EXIT_QUALIFICATION_DRX_REGISTER(exitQualification),
4552	VMX_EXIT_QUALIFICATION_DRX_GENREG(exitQualification)));
4553	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
4554	rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pCtx),
4555	VMX_EXIT_QUALIFICATION_DRX_REGISTER(exitQualification),
4556	VMX_EXIT_QUALIFICATION_DRX_GENREG(exitQualification));
4557	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_DEBUG;
4558	Log2(("DR7=%08x\n", pCtx->dr[7]));
4559	}
4560	else
4561	{
4562	Log2(("VMX: mov x, DRx\n"));
4563	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
4564	rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pCtx),
4565	VMX_EXIT_QUALIFICATION_DRX_GENREG(exitQualification),
4566	VMX_EXIT_QUALIFICATION_DRX_REGISTER(exitQualification));
4567	}
4568	/* Update EIP if no error occurred. */
4569	if (RT_SUCCESS(rc))
4570	pCtx->rip += cbInstr;
4571
4572	if (rc == VINF_SUCCESS)
4573	{
4574	/* Only resume if successful. */
4575	goto ResumeExecution;
4576	}
4577	Assert(rc == VERR_EM_INTERPRETER);
4578	break;
4579	}
4580
4581	/* Note: We'll get a #GP if the IO instruction isn't allowed (IOPL or TSS bitmap); no need to double check. */
4582	case VMX_EXIT_IO_INSTR: /* 30 I/O instruction. */
4583	{
4584	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
4585	uint32_t uPort;
4586	uint32_t uIOWidth = VMX_EXIT_QUALIFICATION_IO_WIDTH(exitQualification);
4587	bool fIOWrite = (VMX_EXIT_QUALIFICATION_IO_DIRECTION(exitQualification) == VMX_EXIT_QUALIFICATION_IO_DIRECTION_OUT);
4588
4589	/** @todo necessary to make the distinction? */
4590	if (VMX_EXIT_QUALIFICATION_IO_ENCODING(exitQualification) == VMX_EXIT_QUALIFICATION_IO_ENCODING_DX)
4591	uPort = pCtx->edx & 0xffff;
4592	else
4593	uPort = VMX_EXIT_QUALIFICATION_IO_PORT(exitQualification); /* Immediate encoding. */
4594
4595	if (RT_UNLIKELY(uIOWidth == 2 \|\| uIOWidth >= 4)) /* paranoia */
4596	{
4597	rc = fIOWrite ? VINF_IOM_R3_IOPORT_WRITE : VINF_IOM_R3_IOPORT_READ;
4598	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
4599	break;
4600	}
4601
4602	uint32_t cbSize = g_aIOSize[uIOWidth];
4603	if (VMX_EXIT_QUALIFICATION_IO_STRING(exitQualification))
4604	{
4605	/* ins/outs */
4606	PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
4607
4608	/* Disassemble manually to deal with segment prefixes. */
4609	/** @todo VMX_VMCS_RO_EXIT_GUEST_LINEAR_ADDR contains the flat pointer
4610	* operand of the instruction. */
4611	/** @todo VMX_VMCS32_RO_EXIT_INSTR_INFO also contains segment prefix info. */
4612	rc2 = EMInterpretDisasCurrent(pVM, pVCpu, pDis, NULL);
4613	if (RT_SUCCESS(rc))
4614	{
4615	if (fIOWrite)
4616	{
4617	Log2(("IOMInterpretOUTSEx %RGv %x size=%d\n", (RTGCPTR)pCtx->rip, uPort, cbSize));
4618	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringWrite);
4619	rc = IOMInterpretOUTSEx(pVM, pVCpu, CPUMCTX2CORE(pCtx), uPort, pDis->fPrefix,
4620	(DISCPUMODE)pDis->uAddrMode, cbSize);
4621	}
4622	else
4623	{
4624	Log2(("IOMInterpretINSEx %RGv %x size=%d\n", (RTGCPTR)pCtx->rip, uPort, cbSize));
4625	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringRead);
4626	rc = IOMInterpretINSEx(pVM, pVCpu, CPUMCTX2CORE(pCtx), uPort, pDis->fPrefix,
4627	(DISCPUMODE)pDis->uAddrMode, cbSize);
4628	}
4629	}
4630	else
4631	rc = VINF_EM_RAW_EMULATE_INSTR;
4632	}
4633	else
4634	{
4635	/* Normal in/out */
4636	uint32_t uAndVal = g_aIOOpAnd[uIOWidth];
4637
4638	Assert(!VMX_EXIT_QUALIFICATION_IO_REP(exitQualification));
4639
4640	if (fIOWrite)
4641	{
4642	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
4643	rc = IOMIOPortWrite(pVM, pVCpu, uPort, pCtx->eax & uAndVal, cbSize);
4644	if (rc == VINF_IOM_R3_IOPORT_WRITE)
4645	HMR0SavePendingIOPortWrite(pVCpu, pCtx->rip, pCtx->rip + cbInstr, uPort, uAndVal, cbSize);
4646	}
4647	else
4648	{
4649	uint32_t u32Val = 0;
4650
4651	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
4652	rc = IOMIOPortRead(pVM, pVCpu, uPort, &u32Val, cbSize);
4653	if (IOM_SUCCESS(rc))
4654	{
4655	/* Write back to the EAX register. */
4656	pCtx->eax = (pCtx->eax & ~uAndVal) \| (u32Val & uAndVal);
4657	}
4658	else
4659	if (rc == VINF_IOM_R3_IOPORT_READ)
4660	HMR0SavePendingIOPortRead(pVCpu, pCtx->rip, pCtx->rip + cbInstr, uPort, uAndVal, cbSize);
4661	}
4662	}
4663
4664	/*
4665	* Handled the I/O return codes.
4666	* (The unhandled cases end up with rc == VINF_EM_RAW_EMULATE_INSTR.)
4667	*/
4668	if (IOM_SUCCESS(rc))
4669	{
4670	/* Update EIP and continue execution. */
4671	pCtx->rip += cbInstr;
4672	if (RT_LIKELY(rc == VINF_SUCCESS))
4673	{
4674	/* If any IO breakpoints are armed, then we should check if a debug trap needs to be generated. */
4675	if (pCtx->dr[7] & X86_DR7_ENABLED_MASK)
4676	{
4677	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
4678	for (unsigned i = 0; i < 4; i++)
4679	{
4680	unsigned uBPLen = g_aIOSize[X86_DR7_GET_LEN(pCtx->dr[7], i)];
4681
4682	if ( (uPort >= pCtx->dr[i] && uPort < pCtx->dr[i] + uBPLen)
4683	&& (pCtx->dr[7] & (X86_DR7_L(i) \| X86_DR7_G(i)))
4684	&& (pCtx->dr[7] & X86_DR7_RW(i, X86_DR7_RW_IO)) == X86_DR7_RW(i, X86_DR7_RW_IO))
4685	{
4686	uint64_t uDR6;
4687
4688	Assert(CPUMIsGuestDebugStateActive(pVCpu));
4689
4690	uDR6 = ASMGetDR6();
4691
4692	/* Clear all breakpoint status flags and set the one we just hit. */
4693	uDR6 &= ~(X86_DR6_B0\|X86_DR6_B1\|X86_DR6_B2\|X86_DR6_B3);
4694	uDR6 \|= (uint64_t)RT_BIT(i);
4695
4696	/*
4697	* Note: AMD64 Architecture Programmer's Manual 13.1:
4698	* Bits 15:13 of the DR6 register is never cleared by the processor and must
4699	* be cleared by software after the contents have been read.
4700	*/
4701	ASMSetDR6(uDR6);
4702
4703	/* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
4704	pCtx->dr[7] &= ~X86_DR7_GD;
4705
4706	/* Paranoia. */
4707	pCtx->dr[7] &= 0xffffffff; /* upper 32 bits reserved */
4708	pCtx->dr[7] &= ~(RT_BIT(11) \| RT_BIT(12) \| RT_BIT(14) \| RT_BIT(15)); /* must be zero */
4709	pCtx->dr[7] \|= 0x400; /* must be one */
4710
4711	/* Resync DR7 */
4712	rc2 = VMXWriteVmcs64(VMX_VMCS_GUEST_DR7, pCtx->dr[7]);
4713	AssertRC(rc2);
4714
4715	/* Construct inject info. */
4716	intInfo = X86_XCPT_DB;
4717	intInfo \|= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
4718	intInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
4719
4720	Log(("Inject IO debug trap at %RGv\n", (RTGCPTR)pCtx->rip));
4721	rc2 = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo),
4722	0 /* cbInstr /, 0 / errCode */);
4723	AssertRC(rc2);
4724
4725	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
4726	goto ResumeExecution;
4727	}
4728	}
4729	}
4730	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
4731	goto ResumeExecution;
4732	}
4733	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
4734	break;
4735	}
4736
4737	#ifdef VBOX_STRICT
4738	if (rc == VINF_IOM_R3_IOPORT_READ)
4739	Assert(!fIOWrite);
4740	else if (rc == VINF_IOM_R3_IOPORT_WRITE)
4741	Assert(fIOWrite);
4742	else
4743	{
4744	AssertMsg( RT_FAILURE(rc)
4745	\|\| rc == VINF_EM_RAW_EMULATE_INSTR
4746	\|\| rc == VINF_EM_RAW_GUEST_TRAP
4747	\|\| rc == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", VBOXSTRICTRC_VAL(rc)));
4748	}
4749	#endif
4750	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
4751	break;
4752	}
4753
4754	case VMX_EXIT_TPR_BELOW_THRESHOLD: /* 43 TPR below threshold. Guest software executed MOV to CR8. */
4755	LogFlow(("VMX_EXIT_TPR_BELOW_THRESHOLD\n"));
4756	/* RIP is already set to the next instruction and the TPR has been synced back. Just resume. */
4757	goto ResumeExecution;
4758
4759	case VMX_EXIT_APIC_ACCESS: /* 44 APIC access. Guest software attempted to access memory at a physical address
4760	on the APIC-access page. */
4761	{
4762	LogFlow(("VMX_EXIT_APIC_ACCESS\n"));
4763	unsigned uAccessType = VMX_EXIT_QUALIFICATION_APIC_ACCESS_TYPE(exitQualification);
4764
4765	switch (uAccessType)
4766	{
4767	case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
4768	case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
4769	{
4770	RTGCPHYS GCPhys = pCtx->msrApicBase;
4771	GCPhys &= PAGE_BASE_GC_MASK;
4772	GCPhys += VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(exitQualification);
4773
4774	LogFlow(("Apic access at %RGp\n", GCPhys));
4775	rc = IOMMMIOPhysHandler(pVM, pVCpu, (uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ) ? 0 : X86_TRAP_PF_RW,
4776	CPUMCTX2CORE(pCtx), GCPhys);
4777	if (rc == VINF_SUCCESS)
4778	goto ResumeExecution; /* rip already updated */
4779	break;
4780	}
4781
4782	default:
4783	rc = VINF_EM_RAW_EMULATE_INSTR;
4784	break;
4785	}
4786	break;
4787	}
4788
4789	case VMX_EXIT_PREEMPT_TIMER: /* 52 VMX-preemption timer expired. The preemption timer counted down to zero. */
4790	if (!TMTimerPollBool(pVM, pVCpu))
4791	goto ResumeExecution;
4792	rc = VINF_EM_RAW_TIMER_PENDING;
4793	break;
4794
4795	default:
4796	/* The rest is handled after syncing the entire CPU state. */
4797	break;
4798	}
4799
4800
4801	/*
4802	* Note: The guest state is not entirely synced back at this stage!
4803	*/
4804
4805	/* Investigate why there was a VM-exit. (part 2) */
4806	switch (exitReason)
4807	{
4808	case VMX_EXIT_XCPT_NMI: /* 0 Exception or non-maskable interrupt (NMI). */
4809	case VMX_EXIT_EXT_INT: /* 1 External interrupt. */
4810	case VMX_EXIT_EPT_VIOLATION:
4811	case VMX_EXIT_EPT_MISCONFIG: /* 49 EPT misconfig is used by the PGM/MMIO optimizations. */
4812	case VMX_EXIT_PREEMPT_TIMER: /* 52 VMX-preemption timer expired. The preemption timer counted down to zero. */
4813	/* Already handled above. */
4814	break;
4815
4816	case VMX_EXIT_TRIPLE_FAULT: /* 2 Triple fault. */
4817	rc = VINF_EM_RESET; /* Triple fault equals a reset. */
4818	break;
4819
4820	case VMX_EXIT_INIT_SIGNAL: /* 3 INIT signal. */
4821	case VMX_EXIT_SIPI: /* 4 Start-up IPI (SIPI). */
4822	rc = VINF_EM_RAW_INTERRUPT;
4823	AssertFailed(); /* Can't happen. Yet. */
4824	break;
4825
4826	case VMX_EXIT_IO_SMI: /* 5 I/O system-management interrupt (SMI). */
4827	case VMX_EXIT_SMI: /* 6 Other SMI. */
4828	rc = VINF_EM_RAW_INTERRUPT;
4829	AssertFailed(); /* Can't happen afaik. */
4830	break;
4831
4832	case VMX_EXIT_TASK_SWITCH: /* 9 Task switch: too complicated to emulate, so fall back to the recompiler */
4833	Log(("VMX_EXIT_TASK_SWITCH: exit=%RX64\n", exitQualification));
4834	if ( (VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE(exitQualification) == VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE_IDT)
4835	&& pVCpu->hm.s.Event.fPending)
4836	{
4837	/* Caused by an injected interrupt. */
4838	pVCpu->hm.s.Event.fPending = false;
4839
4840	Log(("VMX_EXIT_TASK_SWITCH: reassert trap %d\n", VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVCpu->hm.s.Event.u64IntrInfo)));
4841	Assert(!VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntrInfo));
4842	//@todo: Why do we assume this had to be a hardware interrupt? What about software interrupts or exceptions?
4843	rc2 = TRPMAssertTrap(pVCpu, VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVCpu->hm.s.Event.u64IntrInfo), TRPM_HARDWARE_INT);
4844	AssertRC(rc2);
4845	}
4846	/* else Exceptions and software interrupts can just be restarted. */
4847	rc = VERR_EM_INTERPRETER;
4848	break;
4849
4850	case VMX_EXIT_HLT: /* 12 Guest software attempted to execute HLT. */
4851	/* Check if external interrupts are pending; if so, don't switch back. */
4852	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
4853	pCtx->rip++; /* skip hlt */
4854	if (EMShouldContinueAfterHalt(pVCpu, pCtx))
4855	goto ResumeExecution;
4856
4857	rc = VINF_EM_HALT;
4858	break;
4859
4860	case VMX_EXIT_MWAIT: /* 36 Guest software executed MWAIT. */
4861	Log2(("VMX: mwait\n"));
4862	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMwait);
4863	rc = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pCtx));
4864	if ( rc == VINF_EM_HALT
4865	\|\| rc == VINF_SUCCESS)
4866	{
4867	/* Update EIP and continue execution. */
4868	pCtx->rip += cbInstr;
4869
4870	/* Check if external interrupts are pending; if so, don't switch back. */
4871	if ( rc == VINF_SUCCESS
4872	\|\| ( rc == VINF_EM_HALT
4873	&& EMShouldContinueAfterHalt(pVCpu, pCtx))
4874	)
4875	goto ResumeExecution;
4876	}
4877	AssertMsg(rc == VERR_EM_INTERPRETER \|\| rc == VINF_EM_HALT, ("EMU: mwait failed with %Rrc\n", VBOXSTRICTRC_VAL(rc)));
4878	break;
4879
4880	case VMX_EXIT_RSM: /* 17 Guest software attempted to execute RSM in SMM. */
4881	AssertFailed(); /* can't happen. */
4882	rc = VERR_EM_INTERPRETER;
4883	break;
4884
4885	case VMX_EXIT_MTF: /* 37 Exit due to Monitor Trap Flag. */
4886	LogFlow(("VMX_EXIT_MTF at %RGv\n", (RTGCPTR)pCtx->rip));
4887	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
4888	rc2 = VMXWriteVmcs(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
4889	AssertRC(rc2);
4890	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMtf);
4891	#if 0
4892	DBGFDoneStepping(pVCpu);
4893	#endif
4894	rc = VINF_EM_DBG_STOP;
4895	break;
4896
4897	case VMX_EXIT_VMCALL: /* 18 Guest software executed VMCALL. */
4898	case VMX_EXIT_VMCLEAR: /* 19 Guest software executed VMCLEAR. */
4899	case VMX_EXIT_VMLAUNCH: /* 20 Guest software executed VMLAUNCH. */
4900	case VMX_EXIT_VMPTRLD: /* 21 Guest software executed VMPTRLD. */
4901	case VMX_EXIT_VMPTRST: /* 22 Guest software executed VMPTRST. */
4902	case VMX_EXIT_VMREAD: /* 23 Guest software executed VMREAD. */
4903	case VMX_EXIT_VMRESUME: /* 24 Guest software executed VMRESUME. */
4904	case VMX_EXIT_VMWRITE: /* 25 Guest software executed VMWRITE. */
4905	case VMX_EXIT_VMXOFF: /* 26 Guest software executed VMXOFF. */
4906	case VMX_EXIT_VMXON: /* 27 Guest software executed VMXON. */
4907	/** @todo inject #UD immediately */
4908	rc = VERR_EM_INTERPRETER;
4909	break;
4910
4911	case VMX_EXIT_CPUID: /* 10 Guest software attempted to execute CPUID. */
4912	case VMX_EXIT_RDTSC: /* 16 Guest software attempted to execute RDTSC. */
4913	case VMX_EXIT_INVLPG: /* 14 Guest software attempted to execute INVLPG. */
4914	case VMX_EXIT_MOV_CRX: /* 28 Control-register accesses. */
4915	case VMX_EXIT_MOV_DRX: /* 29 Debug-register accesses. */
4916	case VMX_EXIT_IO_INSTR: /* 30 I/O instruction. */
4917	case VMX_EXIT_RDPMC: /* 15 Guest software attempted to execute RDPMC. */
4918	case VMX_EXIT_RDTSCP: /* 51 Guest software attempted to execute RDTSCP. */
4919	/* already handled above */
4920	AssertMsg( rc == VINF_PGM_CHANGE_MODE
4921	\|\| rc == VINF_EM_RAW_INTERRUPT
4922	\|\| rc == VERR_EM_INTERPRETER
4923	\|\| rc == VINF_EM_RAW_EMULATE_INSTR
4924	\|\| rc == VINF_PGM_SYNC_CR3
4925	\|\| rc == VINF_IOM_R3_IOPORT_READ
4926	\|\| rc == VINF_IOM_R3_IOPORT_WRITE
4927	\|\| rc == VINF_EM_RAW_GUEST_TRAP
4928	\|\| rc == VINF_TRPM_XCPT_DISPATCHED
4929	\|\| rc == VINF_EM_RESCHEDULE_REM,
4930	("rc = %d\n", VBOXSTRICTRC_VAL(rc)));
4931	break;
4932
4933	case VMX_EXIT_TPR_BELOW_THRESHOLD: /* 43 TPR below threshold. Guest software executed MOV to CR8. */
4934	case VMX_EXIT_RDMSR: /* 31 RDMSR. Guest software attempted to execute RDMSR. */
4935	case VMX_EXIT_WRMSR: /* 32 WRMSR. Guest software attempted to execute WRMSR. */
4936	case VMX_EXIT_PAUSE: /* 40 Guest software attempted to execute PAUSE. */
4937	case VMX_EXIT_MONITOR: /* 39 Guest software attempted to execute MONITOR. */
4938	case VMX_EXIT_APIC_ACCESS: /* 44 APIC access. Guest software attempted to access memory at a physical address
4939	on the APIC-access page. */
4940	{
4941	/*
4942	* If we decided to emulate them here, then we must sync the MSRs that could have been changed (sysenter, FS/GS base)
4943	*/
4944	rc = VERR_EM_INTERPRETER;
4945	break;
4946	}
4947
4948	case VMX_EXIT_INT_WINDOW: /* 7 Interrupt window. */
4949	Assert(rc == VINF_EM_RAW_INTERRUPT);
4950	break;
4951
4952	case VMX_EXIT_ERR_INVALID_GUEST_STATE: /* 33 VM-entry failure due to invalid guest state. */
4953	{
4954	#ifdef VBOX_STRICT
4955	RTCCUINTREG val2 = 0;
4956
4957	Log(("VMX_EXIT_ERR_INVALID_GUEST_STATE\n"));
4958
4959	VMXReadVmcs(VMX_VMCS_GUEST_RIP, &val2);
4960	Log(("Old eip %RGv new %RGv\n", (RTGCPTR)pCtx->rip, (RTGCPTR)val2));
4961
4962	VMXReadVmcs(VMX_VMCS_GUEST_CR0, &val2);
4963	Log(("VMX_VMCS_GUEST_CR0 %RX64\n", (uint64_t)val2));
4964
4965	VMXReadVmcs(VMX_VMCS_GUEST_CR3, &val2);
4966	Log(("VMX_VMCS_GUEST_CR3 %RX64\n", (uint64_t)val2));
4967
4968	VMXReadVmcs(VMX_VMCS_GUEST_CR4, &val2);
4969	Log(("VMX_VMCS_GUEST_CR4 %RX64\n", (uint64_t)val2));
4970
4971	VMXReadVmcs(VMX_VMCS_GUEST_RFLAGS, &val2);
4972	Log(("VMX_VMCS_GUEST_RFLAGS %08x\n", val2));
4973
4974	VMX_LOG_SELREG(CS, "CS", val2);
4975	VMX_LOG_SELREG(DS, "DS", val2);
4976	VMX_LOG_SELREG(ES, "ES", val2);
4977	VMX_LOG_SELREG(FS, "FS", val2);
4978	VMX_LOG_SELREG(GS, "GS", val2);
4979	VMX_LOG_SELREG(SS, "SS", val2);
4980	VMX_LOG_SELREG(TR, "TR", val2);
4981	VMX_LOG_SELREG(LDTR, "LDTR", val2);
4982
4983	VMXReadVmcs(VMX_VMCS_GUEST_GDTR_BASE, &val2);
4984	Log(("VMX_VMCS_GUEST_GDTR_BASE %RX64\n", (uint64_t)val2));
4985	VMXReadVmcs(VMX_VMCS_GUEST_IDTR_BASE, &val2);
4986	Log(("VMX_VMCS_GUEST_IDTR_BASE %RX64\n", (uint64_t)val2));
4987	#endif /* VBOX_STRICT */
4988	rc = VERR_VMX_INVALID_GUEST_STATE;
4989	break;
4990	}
4991
4992	case VMX_EXIT_ERR_MSR_LOAD: /* 34 VM-entry failure due to MSR loading. */
4993	case VMX_EXIT_ERR_MACHINE_CHECK: /* 41 VM-entry failure due to machine-check. */
4994	default:
4995	rc = VERR_VMX_UNEXPECTED_EXIT_CODE;
4996	AssertMsgFailed(("Unexpected exit code %d\n", exitReason)); /* Can't happen. */
4997	break;
4998
4999	}
5000
5001	end:
5002	/* We now going back to ring-3, so clear the action flag. */
5003	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
5004
5005	/*
5006	* Signal changes for the recompiler.
5007	*/
5008	CPUMSetChangedFlags(pVCpu,
5009	CPUM_CHANGED_SYSENTER_MSR
5010	\| CPUM_CHANGED_LDTR
5011	\| CPUM_CHANGED_GDTR
5012	\| CPUM_CHANGED_IDTR
5013	\| CPUM_CHANGED_TR
5014	\| CPUM_CHANGED_HIDDEN_SEL_REGS);
5015
5016	/*
5017	* If we executed vmlaunch/vmresume and an external IRQ was pending, then we don't have to do a full sync the next time.
5018	*/
5019	if ( exitReason == VMX_EXIT_EXT_INT
5020	&& !VMX_EXIT_INTERRUPTION_INFO_IS_VALID(intInfo))
5021	{
5022	/* On the next entry we'll only sync the host context. */
5023	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_HOST_CONTEXT;
5024	}
5025	else
5026	{
5027	/* On the next entry we'll sync everything. */
5028	/** @todo we can do better than this */
5029	/* Not in the VINF_PGM_CHANGE_MODE though! */
5030	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_ALL_GUEST;
5031	}
5032
5033	/* Translate into a less severe return code */
5034	if (rc == VERR_EM_INTERPRETER)
5035	rc = VINF_EM_RAW_EMULATE_INSTR;
5036	else if (rc == VERR_VMX_INVALID_VMCS_PTR)
5037	{
5038	/* Try to extract more information about what might have gone wrong here. */
5039	VMXGetActivateVMCS(&pVCpu->hm.s.vmx.lasterror.u64VMCSPhys);
5040	pVCpu->hm.s.vmx.lasterror.u32VMCSRevision = (uint32_t )pVCpu->hm.s.vmx.pvVmcs;
5041	pVCpu->hm.s.vmx.lasterror.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
5042	pVCpu->hm.s.vmx.lasterror.idCurrentCpu = RTMpCpuId();
5043	}
5044
5045	/* Just set the correct state here instead of trying to catch every goto above. */
5046	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
5047
5048	#ifdef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
5049	/* Restore interrupts if we exited after disabling them. */
5050	if (uOldEFlags != ~(RTCCUINTREG)0)
5051	ASMSetFlags(uOldEFlags);
5052	#endif
5053
5054	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
5055	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
5056	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
5057	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
5058	Log2(("X"));
5059	return VBOXSTRICTRC_TODO(rc);
5060	}
5061
5062
5063	/**
5064	* Enters the VT-x session.
5065	*
5066	* @returns VBox status code.
5067	* @param pVM Pointer to the VM.
5068	* @param pVCpu Pointer to the VMCPU.
5069	* @param pCpu Pointer to the CPU info struct.
5070	*/
5071	VMMR0DECL(int) VMXR0Enter(PVM pVM, PVMCPU pVCpu, PHMGLOBLCPUINFO pCpu)
5072	{
5073	Assert(pVM->hm.s.vmx.fSupported);
5074	NOREF(pCpu);
5075
5076	unsigned cr4 = ASMGetCR4();
5077	if (!(cr4 & X86_CR4_VMXE))
5078	{
5079	AssertMsgFailed(("X86_CR4_VMXE should be set!\n"));
5080	return VERR_VMX_X86_CR4_VMXE_CLEARED;
5081	}
5082
5083	/* Activate the VMCS. */
5084	int rc = VMXActivateVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
5085	if (RT_FAILURE(rc))
5086	return rc;
5087
5088	pVCpu->hm.s.fResumeVM = false;
5089	return VINF_SUCCESS;
5090	}
5091
5092
5093	/**
5094	* Leaves the VT-x session.
5095	*
5096	* @returns VBox status code.
5097	* @param pVM Pointer to the VM.
5098	* @param pVCpu Pointer to the VMCPU.
5099	* @param pCtx Pointer to the guests CPU context.
5100	*/
5101	VMMR0DECL(int) VMXR0Leave(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
5102	{
5103	Assert(pVM->hm.s.vmx.fSupported);
5104
5105	#ifdef DEBUG
5106	if (CPUMIsHyperDebugStateActive(pVCpu))
5107	{
5108	CPUMR0LoadHostDebugState(pVM, pVCpu);
5109	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT);
5110	}
5111	else
5112	#endif
5113
5114	/*
5115	* Save the guest debug state if necessary.
5116	*/
5117	if (CPUMIsGuestDebugStateActive(pVCpu))
5118	{
5119	CPUMR0SaveGuestDebugState(pVM, pVCpu, pCtx, true /* save DR6 */);
5120
5121	/* Enable DRx move intercepts again. */
5122	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
5123	int rc = VMXWriteVmcs(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
5124	AssertRC(rc);
5125
5126	/* Resync the debug registers the next time. */
5127	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_DEBUG;
5128	}
5129	else
5130	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT);
5131
5132	/*
5133	* Clear VMCS, marking it inactive, clearing implementation-specific data and writing
5134	* VMCS data back to memory.
5135	*/
5136	int rc = VMXClearVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
5137	AssertRC(rc);
5138
5139	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
5140	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
5141	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
5142	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
5143	return VINF_SUCCESS;
5144	}
5145
5146
5147	/**
5148	* Flush the TLB using EPT.
5149	*
5150	* @returns VBox status code.
5151	* @param pVM Pointer to the VM.
5152	* @param pVCpu Pointer to the VMCPU.
5153	* @param enmFlush Type of flush.
5154	*/
5155	static void hmR0VmxFlushEPT(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_EPT enmFlush)
5156	{
5157	uint64_t descriptor[2];
5158
5159	LogFlow(("hmR0VmxFlushEPT %d\n", enmFlush));
5160	Assert(pVM->hm.s.fNestedPaging);
5161	descriptor[0] = pVCpu->hm.s.vmx.HCPhysEPTP;
5162	descriptor[1] = 0; /* MBZ. Intel spec. 33.3 VMX Instructions */
5163	int rc = VMXR0InvEPT(enmFlush, &descriptor[0]);
5164	AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %x %RGv failed with %d\n", enmFlush, pVCpu->hm.s.vmx.HCPhysEPTP, rc));
5165	#ifdef VBOX_WITH_STATISTICS
5166	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
5167	#endif
5168	}
5169
5170
5171	/**
5172	* Flush the TLB using VPID.
5173	*
5174	* @returns VBox status code.
5175	* @param pVM Pointer to the VM.
5176	* @param pVCpu Pointer to the VMCPU (can be NULL depending on @a
5177	* enmFlush).
5178	* @param enmFlush Type of flush.
5179	* @param GCPtr Virtual address of the page to flush (can be 0 depending
5180	* on @a enmFlush).
5181	*/
5182	static void hmR0VmxFlushVPID(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_VPID enmFlush, RTGCPTR GCPtr)
5183	{
5184	uint64_t descriptor[2];
5185
5186	Assert(pVM->hm.s.vmx.fVpid);
5187	if (enmFlush == VMX_FLUSH_VPID_ALL_CONTEXTS)
5188	{
5189	descriptor[0] = 0;
5190	descriptor[1] = 0;
5191	}
5192	else
5193	{
5194	AssertPtr(pVCpu);
5195	AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
5196	AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
5197	descriptor[0] = pVCpu->hm.s.uCurrentAsid;
5198	descriptor[1] = GCPtr;
5199	}
5200	int rc = VMXR0InvVPID(enmFlush, &descriptor[0]); NOREF(rc);
5201	AssertMsg(rc == VINF_SUCCESS,
5202	("VMXR0InvVPID %x %x %RGv failed with %d\n", enmFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
5203	#ifdef VBOX_WITH_STATISTICS
5204	if (pVCpu)
5205	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
5206	#endif
5207	}
5208
5209
5210	/**
5211	* Invalidates a guest page by guest virtual address. Only relevant for
5212	* EPT/VPID, otherwise there is nothing really to invalidate.
5213	*
5214	* @returns VBox status code.
5215	* @param pVM Pointer to the VM.
5216	* @param pVCpu Pointer to the VMCPU.
5217	* @param GCVirt Guest virtual address of the page to invalidate.
5218	*/
5219	VMMR0DECL(int) VMXR0InvalidatePage(PVM pVM, PVMCPU pVCpu, RTGCPTR GCVirt)
5220	{
5221	bool fFlushPending = VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
5222
5223	Log2(("VMXR0InvalidatePage %RGv\n", GCVirt));
5224
5225	if (!fFlushPending)
5226	{
5227	/*
5228	* We must invalidate the guest TLB entry in either case, we cannot ignore it even for the EPT case
5229	* See @bugref{6043} and @bugref{6177}
5230	*
5231	* Set the VMCPU_FF_TLB_FLUSH force flag and flush before VMENTRY in hmR0VmxSetupTLB*() as this
5232	* function maybe called in a loop with individual addresses.
5233	*/
5234	if (pVM->hm.s.vmx.fVpid)
5235	{
5236	/* If we can flush just this page do it, otherwise flush as little as possible. */
5237	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
5238	hmR0VmxFlushVPID(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, GCVirt);
5239	else
5240	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
5241	}
5242	else if (pVM->hm.s.fNestedPaging)
5243	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
5244	}
5245
5246	return VINF_SUCCESS;
5247	}
5248
5249
5250	/**
5251	* Invalidates a guest page by physical address. Only relevant for EPT/VPID,
5252	* otherwise there is nothing really to invalidate.
5253	*
5254	* NOTE: Assumes the current instruction references this physical page though a virtual address!!
5255	*
5256	* @returns VBox status code.
5257	* @param pVM Pointer to the VM.
5258	* @param pVCpu Pointer to the VMCPU.
5259	* @param GCPhys Guest physical address of the page to invalidate.
5260	*/
5261	VMMR0DECL(int) VMXR0InvalidatePhysPage(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys)
5262	{
5263	LogFlow(("VMXR0InvalidatePhysPage %RGp\n", GCPhys));
5264
5265	/*
5266	* We cannot flush a page by guest-physical address. invvpid takes only a linear address
5267	* while invept only flushes by EPT not individual addresses. We update the force flag here
5268	* and flush before VMENTRY in hmR0VmxSetupTLB*(). This function might be called in a loop.
5269	*/
5270	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
5271	return VINF_SUCCESS;
5272	}
5273
5274
5275	/**
5276	* Report world switch error and dump some useful debug info.
5277	*
5278	* @param pVM Pointer to the VM.
5279	* @param pVCpu Pointer to the VMCPU.
5280	* @param rc Return code.
5281	* @param pCtx Pointer to the current guest CPU context (not updated).
5282	*/
5283	static void hmR0VmxReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, VBOXSTRICTRC rc, PCPUMCTX pCtx)
5284	{
5285	NOREF(pVM);
5286
5287	switch (VBOXSTRICTRC_VAL(rc))
5288	{
5289	case VERR_VMX_INVALID_VMXON_PTR:
5290	AssertFailed();
5291	break;
5292
5293	case VERR_VMX_UNABLE_TO_START_VM:
5294	{
5295	int rc2;
5296	RTCCUINTREG exitReason, instrError;
5297
5298	rc2 = VMXReadVmcs(VMX_VMCS32_RO_EXIT_REASON, &exitReason);
5299	rc2 \|= VMXReadVmcs(VMX_VMCS32_RO_VM_INSTR_ERROR, &instrError);
5300	AssertRC(rc2);
5301	if (rc2 == VINF_SUCCESS)
5302	{
5303	Log(("Unable to start/resume VM for reason: %x. Instruction error %x\n", (uint32_t)exitReason,
5304	(uint32_t)instrError));
5305	Log(("Current stack %08x\n", &rc2));
5306
5307	pVCpu->hm.s.vmx.lasterror.u32InstrError = instrError;
5308	pVCpu->hm.s.vmx.lasterror.u32ExitReason = exitReason;
5309
5310	#ifdef VBOX_STRICT
5311	RTGDTR gdtr;
5312	PCX86DESCHC pDesc;
5313	RTCCUINTREG val;
5314
5315	ASMGetGDTR(&gdtr);
5316
5317	VMXReadVmcs(VMX_VMCS_GUEST_RIP, &val);
5318	Log(("Old eip %RGv new %RGv\n", (RTGCPTR)pCtx->rip, (RTGCPTR)val));
5319	VMXReadVmcs(VMX_VMCS32_CTRL_PIN_EXEC, &val);
5320	Log(("VMX_VMCS_CTRL_PIN_EXEC %08x\n", val));
5321	VMXReadVmcs(VMX_VMCS32_CTRL_PROC_EXEC, &val);
5322	Log(("VMX_VMCS_CTRL_PROC_EXEC %08x\n", val));
5323	VMXReadVmcs(VMX_VMCS32_CTRL_ENTRY, &val);
5324	Log(("VMX_VMCS_CTRL_ENTRY %08x\n", val));
5325	VMXReadVmcs(VMX_VMCS32_CTRL_EXIT, &val);
5326	Log(("VMX_VMCS_CTRL_EXIT %08x\n", val));
5327
5328	VMXReadVmcs(VMX_VMCS_HOST_CR0, &val);
5329	Log(("VMX_VMCS_HOST_CR0 %08x\n", val));
5330	VMXReadVmcs(VMX_VMCS_HOST_CR3, &val);
5331	Log(("VMX_VMCS_HOST_CR3 %08x\n", val));
5332	VMXReadVmcs(VMX_VMCS_HOST_CR4, &val);
5333	Log(("VMX_VMCS_HOST_CR4 %08x\n", val));
5334
5335	VMXReadVmcs(VMX_VMCS16_HOST_FIELD_CS, &val);
5336	Log(("VMX_VMCS_HOST_FIELD_CS %08x\n", val));
5337	VMXReadVmcs(VMX_VMCS_GUEST_RFLAGS, &val);
5338	Log(("VMX_VMCS_GUEST_RFLAGS %08x\n", val));
5339
5340	if (val < gdtr.cbGdt)
5341	{
5342	pDesc = (PCX86DESCHC)(gdtr.pGdt + (val & X86_SEL_MASK));
5343	HMR0DumpDescriptor(pDesc, val, "CS: ");
5344	}
5345
5346	VMXReadVmcs(VMX_VMCS16_HOST_FIELD_DS, &val);
5347	Log(("VMX_VMCS_HOST_FIELD_DS %08x\n", val));
5348	if (val < gdtr.cbGdt)
5349	{
5350	pDesc = (PCX86DESCHC)(gdtr.pGdt + (val & X86_SEL_MASK));
5351	HMR0DumpDescriptor(pDesc, val, "DS: ");
5352	}
5353
5354	VMXReadVmcs(VMX_VMCS16_HOST_FIELD_ES, &val);
5355	Log(("VMX_VMCS_HOST_FIELD_ES %08x\n", val));
5356	if (val < gdtr.cbGdt)
5357	{
5358	pDesc = (PCX86DESCHC)(gdtr.pGdt + (val & X86_SEL_MASK));
5359	HMR0DumpDescriptor(pDesc, val, "ES: ");
5360	}
5361
5362	VMXReadVmcs(VMX_VMCS16_HOST_FIELD_FS, &val);
5363	Log(("VMX_VMCS16_HOST_FIELD_FS %08x\n", val));
5364	if (val < gdtr.cbGdt)
5365	{
5366	pDesc = (PCX86DESCHC)(gdtr.pGdt + (val & X86_SEL_MASK));
5367	HMR0DumpDescriptor(pDesc, val, "FS: ");
5368	}
5369
5370	VMXReadVmcs(VMX_VMCS16_HOST_FIELD_GS, &val);
5371	Log(("VMX_VMCS16_HOST_FIELD_GS %08x\n", val));
5372	if (val < gdtr.cbGdt)
5373	{
5374	pDesc = (PCX86DESCHC)(gdtr.pGdt + (val & X86_SEL_MASK));
5375	HMR0DumpDescriptor(pDesc, val, "GS: ");
5376	}
5377
5378	VMXReadVmcs(VMX_VMCS16_HOST_FIELD_SS, &val);
5379	Log(("VMX_VMCS16_HOST_FIELD_SS %08x\n", val));
5380	if (val < gdtr.cbGdt)
5381	{
5382	pDesc = (PCX86DESCHC)(gdtr.pGdt + (val & X86_SEL_MASK));
5383	HMR0DumpDescriptor(pDesc, val, "SS: ");
5384	}
5385
5386	VMXReadVmcs(VMX_VMCS16_HOST_FIELD_TR, &val);
5387	Log(("VMX_VMCS16_HOST_FIELD_TR %08x\n", val));
5388	if (val < gdtr.cbGdt)
5389	{
5390	pDesc = (PCX86DESCHC)(gdtr.pGdt + (val & X86_SEL_MASK));
5391	HMR0DumpDescriptor(pDesc, val, "TR: ");
5392	}
5393
5394	VMXReadVmcs(VMX_VMCS_HOST_TR_BASE, &val);
5395	Log(("VMX_VMCS_HOST_TR_BASE %RHv\n", val));
5396	VMXReadVmcs(VMX_VMCS_HOST_GDTR_BASE, &val);
5397	Log(("VMX_VMCS_HOST_GDTR_BASE %RHv\n", val));
5398	VMXReadVmcs(VMX_VMCS_HOST_IDTR_BASE, &val);
5399	Log(("VMX_VMCS_HOST_IDTR_BASE %RHv\n", val));
5400	VMXReadVmcs(VMX_VMCS32_HOST_SYSENTER_CS, &val);
5401	Log(("VMX_VMCS_HOST_SYSENTER_CS %08x\n", val));
5402	VMXReadVmcs(VMX_VMCS_HOST_SYSENTER_EIP, &val);
5403	Log(("VMX_VMCS_HOST_SYSENTER_EIP %RHv\n", val));
5404	VMXReadVmcs(VMX_VMCS_HOST_SYSENTER_ESP, &val);
5405	Log(("VMX_VMCS_HOST_SYSENTER_ESP %RHv\n", val));
5406	VMXReadVmcs(VMX_VMCS_HOST_RSP, &val);
5407	Log(("VMX_VMCS_HOST_RSP %RHv\n", val));
5408	VMXReadVmcs(VMX_VMCS_HOST_RIP, &val);
5409	Log(("VMX_VMCS_HOST_RIP %RHv\n", val));
5410	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
5411	if (VMX_IS_64BIT_HOST_MODE())
5412	{
5413	Log(("MSR_K6_EFER = %RX64\n", ASMRdMsr(MSR_K6_EFER)));
5414	Log(("MSR_K6_STAR = %RX64\n", ASMRdMsr(MSR_K6_STAR)));
5415	Log(("MSR_K8_LSTAR = %RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
5416	Log(("MSR_K8_CSTAR = %RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
5417	Log(("MSR_K8_SF_MASK = %RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
5418	Log(("MSR_K8_KERNEL_GS_BASE = %RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
5419	}
5420	# endif
5421	#endif /* VBOX_STRICT */
5422	}
5423	break;
5424	}
5425
5426	default:
5427	/* impossible */
5428	AssertMsgFailed(("%Rrc (%#x)\n", VBOXSTRICTRC_VAL(rc), VBOXSTRICTRC_VAL(rc)));
5429	break;
5430	}
5431	}
5432
5433
5434	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
5435	/**
5436	* Prepares for and executes VMLAUNCH (64 bits guest mode).
5437	*
5438	* @returns VBox status code.
5439	* @param fResume Whether to vmlauch/vmresume.
5440	* @param pCtx Pointer to the guest CPU context.
5441	* @param pCache Pointer to the VMCS cache.
5442	* @param pVM Pointer to the VM.
5443	* @param pVCpu Pointer to the VMCPU.
5444	*/
5445	DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
5446	{
5447	uint32_t aParam[6];
5448	PHMGLOBLCPUINFO pCpu;
5449	RTHCPHYS HCPhysCpuPage;
5450	int rc;
5451
5452	pCpu = HMR0GetCurrentCpu();
5453	HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
5454
5455	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5456	pCache->uPos = 1;
5457	pCache->interPD = PGMGetInterPaeCR3(pVM);
5458	pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
5459	#endif
5460
5461	#ifdef DEBUG
5462	pCache->TestIn.HCPhysCpuPage= 0;
5463	pCache->TestIn.HCPhysVmcs = 0;
5464	pCache->TestIn.pCache = 0;
5465	pCache->TestOut.HCPhysVmcs = 0;
5466	pCache->TestOut.pCache = 0;
5467	pCache->TestOut.pCtx = 0;
5468	pCache->TestOut.eflags = 0;
5469	#endif
5470
5471	aParam[0] = (uint32_t)(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
5472	aParam[1] = (uint32_t)(HCPhysCpuPage >> 32); /* Param 1: VMXON physical address - Hi. */
5473	aParam[2] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
5474	aParam[3] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs >> 32); /* Param 2: VMCS physical address - Hi. */
5475	aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache);
5476	aParam[5] = 0;
5477
5478	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5479	pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
5480	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
5481	#endif
5482	rc = VMXR0Execute64BitsHandler(pVM, pVCpu, pCtx, HM64ON32OP_VMXRCStartVM64, 6, &aParam[0]);
5483
5484	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5485	Assert((uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
5486	Assert(pCtx->dr[4] == 10);
5487	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
5488	#endif
5489
5490	#ifdef DEBUG
5491	AssertMsg(pCache->TestIn.HCPhysCpuPage== HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
5492	AssertMsg(pCache->TestIn.HCPhysVmcs == pVCpu->hm.s.vmx.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5493	pVCpu->hm.s.vmx.HCPhysVmcs));
5494	AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5495	pCache->TestOut.HCPhysVmcs));
5496	AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
5497	pCache->TestOut.pCache));
5498	AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache),
5499	("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache)));
5500	AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
5501	pCache->TestOut.pCtx));
5502	Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
5503	#endif
5504	return rc;
5505	}
5506
5507
5508	#ifdef VBOX_STRICT
5509	static bool hmR0VmxIsValidReadField(uint32_t idxField)
5510	{
5511	switch (idxField)
5512	{
5513	case VMX_VMCS_GUEST_RIP:
5514	case VMX_VMCS_GUEST_RSP:
5515	case VMX_VMCS_GUEST_RFLAGS:
5516	case VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE:
5517	case VMX_VMCS_CTRL_CR0_READ_SHADOW:
5518	case VMX_VMCS_GUEST_CR0:
5519	case VMX_VMCS_CTRL_CR4_READ_SHADOW:
5520	case VMX_VMCS_GUEST_CR4:
5521	case VMX_VMCS_GUEST_DR7:
5522	case VMX_VMCS32_GUEST_SYSENTER_CS:
5523	case VMX_VMCS_GUEST_SYSENTER_EIP:
5524	case VMX_VMCS_GUEST_SYSENTER_ESP:
5525	case VMX_VMCS32_GUEST_GDTR_LIMIT:
5526	case VMX_VMCS_GUEST_GDTR_BASE:
5527	case VMX_VMCS32_GUEST_IDTR_LIMIT:
5528	case VMX_VMCS_GUEST_IDTR_BASE:
5529	case VMX_VMCS16_GUEST_FIELD_CS:
5530	case VMX_VMCS32_GUEST_CS_LIMIT:
5531	case VMX_VMCS_GUEST_CS_BASE:
5532	case VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS:
5533	case VMX_VMCS16_GUEST_FIELD_DS:
5534	case VMX_VMCS32_GUEST_DS_LIMIT:
5535	case VMX_VMCS_GUEST_DS_BASE:
5536	case VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS:
5537	case VMX_VMCS16_GUEST_FIELD_ES:
5538	case VMX_VMCS32_GUEST_ES_LIMIT:
5539	case VMX_VMCS_GUEST_ES_BASE:
5540	case VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS:
5541	case VMX_VMCS16_GUEST_FIELD_FS:
5542	case VMX_VMCS32_GUEST_FS_LIMIT:
5543	case VMX_VMCS_GUEST_FS_BASE:
5544	case VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS:
5545	case VMX_VMCS16_GUEST_FIELD_GS:
5546	case VMX_VMCS32_GUEST_GS_LIMIT:
5547	case VMX_VMCS_GUEST_GS_BASE:
5548	case VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS:
5549	case VMX_VMCS16_GUEST_FIELD_SS:
5550	case VMX_VMCS32_GUEST_SS_LIMIT:
5551	case VMX_VMCS_GUEST_SS_BASE:
5552	case VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS:
5553	case VMX_VMCS16_GUEST_FIELD_LDTR:
5554	case VMX_VMCS32_GUEST_LDTR_LIMIT:
5555	case VMX_VMCS_GUEST_LDTR_BASE:
5556	case VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS:
5557	case VMX_VMCS16_GUEST_FIELD_TR:
5558	case VMX_VMCS32_GUEST_TR_LIMIT:
5559	case VMX_VMCS_GUEST_TR_BASE:
5560	case VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS:
5561	case VMX_VMCS32_RO_EXIT_REASON:
5562	case VMX_VMCS32_RO_VM_INSTR_ERROR:
5563	case VMX_VMCS32_RO_EXIT_INSTR_LENGTH:
5564	case VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE:
5565	case VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO:
5566	case VMX_VMCS32_RO_EXIT_INSTR_INFO:
5567	case VMX_VMCS_RO_EXIT_QUALIFICATION:
5568	case VMX_VMCS32_RO_IDT_INFO:
5569	case VMX_VMCS32_RO_IDT_ERROR_CODE:
5570	case VMX_VMCS_GUEST_CR3:
5571	case VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL:
5572	return true;
5573	}
5574	return false;
5575	}
5576
5577
5578	static bool hmR0VmxIsValidWriteField(uint32_t idxField)
5579	{
5580	switch (idxField)
5581	{
5582	case VMX_VMCS_GUEST_LDTR_BASE:
5583	case VMX_VMCS_GUEST_TR_BASE:
5584	case VMX_VMCS_GUEST_GDTR_BASE:
5585	case VMX_VMCS_GUEST_IDTR_BASE:
5586	case VMX_VMCS_GUEST_SYSENTER_EIP:
5587	case VMX_VMCS_GUEST_SYSENTER_ESP:
5588	case VMX_VMCS_GUEST_CR0:
5589	case VMX_VMCS_GUEST_CR4:
5590	case VMX_VMCS_GUEST_CR3:
5591	case VMX_VMCS_GUEST_DR7:
5592	case VMX_VMCS_GUEST_RIP:
5593	case VMX_VMCS_GUEST_RSP:
5594	case VMX_VMCS_GUEST_CS_BASE:
5595	case VMX_VMCS_GUEST_DS_BASE:
5596	case VMX_VMCS_GUEST_ES_BASE:
5597	case VMX_VMCS_GUEST_FS_BASE:
5598	case VMX_VMCS_GUEST_GS_BASE:
5599	case VMX_VMCS_GUEST_SS_BASE:
5600	return true;
5601	}
5602	return false;
5603	}
5604	#endif /* VBOX_STRICT */
5605
5606
5607	/**
5608	* Executes the specified handler in 64-bit mode.
5609	*
5610	* @returns VBox status code.
5611	* @param pVM Pointer to the VM.
5612	* @param pVCpu Pointer to the VMCPU.
5613	* @param pCtx Pointer to the guest CPU context.
5614	* @param enmOp The operation to perform.
5615	* @param cbParam Number of parameters.
5616	* @param paParam Array of 32-bit parameters.
5617	*/
5618	VMMR0DECL(int) VMXR0Execute64BitsHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, HM64ON32OP enmOp, uint32_t cbParam,
5619	uint32_t *paParam)
5620	{
5621	int rc, rc2;
5622	PHMGLOBLCPUINFO pCpu;
5623	RTHCPHYS HCPhysCpuPage;
5624	RTHCUINTREG uOldEFlags;
5625
5626	AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
5627	Assert(enmOp > HM64ON32OP_INVALID && enmOp < HM64ON32OP_END);
5628	Assert(pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Write.aField));
5629	Assert(pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Read.aField));
5630
5631	#ifdef VBOX_STRICT
5632	for (unsigned i=0;i<pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries;i++)
5633	Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VMCSCache.Write.aField[i]));
5634
5635	for (unsigned i=0;i<pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries;i++)
5636	Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VMCSCache.Read.aField[i]));
5637	#endif
5638
5639	/* Disable interrupts. */
5640	uOldEFlags = ASMIntDisableFlags();
5641
5642	#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
5643	RTCPUID idHostCpu = RTMpCpuId();
5644	CPUMR0SetLApic(pVM, idHostCpu);
5645	#endif
5646
5647	pCpu = HMR0GetCurrentCpu();
5648	HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
5649
5650	/* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
5651	VMXClearVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
5652
5653	/* Leave VMX Root Mode. */
5654	VMXDisable();
5655
5656	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
5657
5658	CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
5659	CPUMSetHyperEIP(pVCpu, enmOp);
5660	for (int i=(int)cbParam-1;i>=0;i--)
5661	CPUMPushHyper(pVCpu, paParam[i]);
5662
5663	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
5664
5665	/* Call switcher. */
5666	rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_OFFSETOF(VM, aCpus[pVCpu->idCpu].cpum) - RT_OFFSETOF(VM, cpum));
5667	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
5668
5669	/* Make sure the VMX instructions don't cause #UD faults. */
5670	ASMSetCR4(ASMGetCR4() \| X86_CR4_VMXE);
5671
5672	/* Enter VMX Root Mode */
5673	rc2 = VMXEnable(HCPhysCpuPage);
5674	if (RT_FAILURE(rc2))
5675	{
5676	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
5677	ASMSetFlags(uOldEFlags);
5678	return VERR_VMX_VMXON_FAILED;
5679	}
5680
5681	rc2 = VMXActivateVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
5682	AssertRC(rc2);
5683	Assert(!(ASMGetFlags() & X86_EFL_IF));
5684	ASMSetFlags(uOldEFlags);
5685	return rc;
5686	}
5687	#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL) */
5688
5689
5690	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
5691	/**
5692	* Executes VMWRITE.
5693	*
5694	* @returns VBox status code
5695	* @param pVCpu Pointer to the VMCPU.
5696	* @param idxField VMCS field index.
5697	* @param u64Val 16, 32 or 64 bits value.
5698	*/
5699	VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5700	{
5701	int rc;
5702	switch (idxField)
5703	{
5704	case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
5705	case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
5706	case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
5707	case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
5708	case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
5709	case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
5710	case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
5711	case VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL:
5712	case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
5713	case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
5714	case VMX_VMCS64_GUEST_PDPTE0_FULL:
5715	case VMX_VMCS64_GUEST_PDPTE1_FULL:
5716	case VMX_VMCS64_GUEST_PDPTE2_FULL:
5717	case VMX_VMCS64_GUEST_PDPTE3_FULL:
5718	case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
5719	case VMX_VMCS64_GUEST_EFER_FULL:
5720	case VMX_VMCS64_CTRL_EPTP_FULL:
5721	/* These fields consist of two parts, which are both writable in 32 bits mode. */
5722	rc = VMXWriteVmcs32(idxField, u64Val);
5723	rc \|= VMXWriteVmcs32(idxField + 1, (uint32_t)(u64Val >> 32ULL));
5724	AssertRC(rc);
5725	return rc;
5726
5727	case VMX_VMCS_GUEST_LDTR_BASE:
5728	case VMX_VMCS_GUEST_TR_BASE:
5729	case VMX_VMCS_GUEST_GDTR_BASE:
5730	case VMX_VMCS_GUEST_IDTR_BASE:
5731	case VMX_VMCS_GUEST_SYSENTER_EIP:
5732	case VMX_VMCS_GUEST_SYSENTER_ESP:
5733	case VMX_VMCS_GUEST_CR0:
5734	case VMX_VMCS_GUEST_CR4:
5735	case VMX_VMCS_GUEST_CR3:
5736	case VMX_VMCS_GUEST_DR7:
5737	case VMX_VMCS_GUEST_RIP:
5738	case VMX_VMCS_GUEST_RSP:
5739	case VMX_VMCS_GUEST_CS_BASE:
5740	case VMX_VMCS_GUEST_DS_BASE:
5741	case VMX_VMCS_GUEST_ES_BASE:
5742	case VMX_VMCS_GUEST_FS_BASE:
5743	case VMX_VMCS_GUEST_GS_BASE:
5744	case VMX_VMCS_GUEST_SS_BASE:
5745	/* Queue a 64 bits value as we can't set it in 32 bits host mode. */
5746	if (u64Val >> 32ULL)
5747	rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
5748	else
5749	rc = VMXWriteVmcs32(idxField, (uint32_t)u64Val);
5750
5751	return rc;
5752
5753	default:
5754	AssertMsgFailed(("Unexpected field %x\n", idxField));
5755	return VERR_INVALID_PARAMETER;
5756	}
5757	}
5758
5759
5760	/**
5761	* Cache VMCS writes for running 64 bits guests on 32 bits hosts.
5762	*
5763	* @param pVCpu Pointer to the VMCPU.
5764	* @param idxField VMCS field index.
5765	* @param u64Val 16, 32 or 64 bits value.
5766	*/
5767	VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5768	{
5769	PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5770
5771	AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1,
5772	("entries=%x\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
5773
5774	/* Make sure there are no duplicates. */
5775	for (unsigned i = 0; i < pCache->Write.cValidEntries; i++)
5776	{
5777	if (pCache->Write.aField[i] == idxField)
5778	{
5779	pCache->Write.aFieldVal[i] = u64Val;
5780	return VINF_SUCCESS;
5781	}
5782	}
5783
5784	pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
5785	pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
5786	pCache->Write.cValidEntries++;
5787	return VINF_SUCCESS;
5788	}
5789
5790	#endif /* HC_ARCH_BITS == 32 && !VBOX_WITH_HYBRID_32BIT_KERNEL */
5791

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMR0/HWVMXR0.cpp@ 47062

Download in other formats: