HWVMXR0.cpp@ 45474

Last change on this file since 45474 was 45474, checked in by vboxsync, 12 years ago
VMM: Remove unused error codes, use more specific error codes. Restore CR4 on VMXON failures in the new VT-x code.
Property svn:eol-style set to `native` Property svn:keywords set to `Id Revision`
File size: 228.7 KB

Line
1	/* $Id: HWVMXR0.cpp 45474 2013-04-10 20:25:33Z vboxsync $ */
2	/** @file
3	* HM VMX (VT-x) - Host Context Ring-0.
4	*/
5
6	/*
7	* Copyright (C) 2006-2013 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18
19	/*******************************************************************************
20	* Header Files *
21	*******************************************************************************/
22	#define LOG_GROUP LOG_GROUP_HM
23	#include <iprt/asm-amd64-x86.h>
24	#include <VBox/vmm/hm.h>
25	#include <VBox/vmm/pgm.h>
26	#include <VBox/vmm/dbgf.h>
27	#include <VBox/vmm/dbgftrace.h>
28	#include <VBox/vmm/selm.h>
29	#include <VBox/vmm/iom.h>
30	#ifdef VBOX_WITH_REM
31	# include <VBox/vmm/rem.h>
32	#endif
33	#include <VBox/vmm/tm.h>
34	#include "HMInternal.h"
35	#include <VBox/vmm/vm.h>
36	#include <VBox/vmm/pdmapi.h>
37	#include <VBox/err.h>
38	#include <VBox/log.h>
39	#include <iprt/assert.h>
40	#include <iprt/param.h>
41	#include <iprt/string.h>
42	#include <iprt/time.h>
43	#ifdef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
44	# include <iprt/thread.h>
45	#endif
46	#include <iprt/x86.h>
47	#include "HWVMXR0.h"
48
49	#include "dtrace/VBoxVMM.h"
50
51
52	/*******************************************************************************
53	* Defined Constants And Macros *
54	*******************************************************************************/
55	#if defined(RT_ARCH_AMD64)
56	# define VMX_IS_64BIT_HOST_MODE() (true)
57	#elif defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
58	# define VMX_IS_64BIT_HOST_MODE() (g_fVMXIs64bitHost != 0)
59	#else
60	# define VMX_IS_64BIT_HOST_MODE() (false)
61	#endif
62
63	# define VMX_WRITE_SELREG(REG, reg) \
64	do \
65	{ \
66	rc = VMXWriteVmcs(VMX_VMCS16_GUEST_FIELD_##REG, pCtx->reg.Sel); \
67	rc \|= VMXWriteVmcs(VMX_VMCS32_GUEST_##REG##_LIMIT, pCtx->reg.u32Limit); \
68	rc \|= VMXWriteVmcs64(VMX_VMCS_GUEST_##REG##_BASE, pCtx->reg.u64Base); \
69	if ((pCtx->eflags.u32 & X86_EFL_VM)) \
70	{ \
71	/* Must override this or else VT-x will fail with invalid guest state errors. */ \
72	/* DPL=3, present, code/data, r/w/accessed. */ \
73	/** @todo we shouldn't have to do this, if it is not 0xf3 it means we screwed up elsewhere (recompiler). */ \
74	/** @todo VT-x docs explicitly mentions 0xF3. Why not just val = 0xf3 ??. */ \
75	val = (pCtx->reg.Attr.u & ~0xFF) \| 0xF3; \
76	} \
77	else \
78	if ( CPUMIsGuestInRealModeEx(pCtx) \
79	&& !pVM->hm.s.vmx.fUnrestrictedGuest) \
80	{ \
81	/** @todo shouldn't the 'if' condition above check for 'pRealModeTSS' ? */ \
82	/* Must override this or else VT-x will fail with invalid guest state errors. */ \
83	/* DPL=3, present, code/data, r/w/accessed. */ \
84	val = 0xf3; \
85	} \
86	else \
87	if ( ( pCtx->reg.Sel \
88	\|\| !CPUMIsGuestInPagedProtectedModeEx(pCtx) \
89	\|\| (!pCtx->cs.Attr.n.u1DefBig && !CPUMIsGuestIn64BitCodeEx(pCtx)) \
90	) \
91	&& pCtx->reg.Attr.n.u1Present == 1) \
92	{ \
93	val = pCtx->reg.Attr.u \| X86_SEL_TYPE_ACCESSED; \
94	} \
95	else \
96	val = 0x10000; /* Invalid guest state error otherwise. (BIT(16) = Unusable) */ \
97	\
98	rc \|= VMXWriteVmcs(VMX_VMCS32_GUEST_##REG##_ACCESS_RIGHTS, val); \
99	} while (0)
100
101	# define VMX_READ_SELREG(REG, reg) \
102	do \
103	{ \
104	VMXReadCachedVmcs(VMX_VMCS16_GUEST_FIELD_##REG, &val); \
105	pCtx->reg.Sel = val; \
106	pCtx->reg.ValidSel = val; \
107	pCtx->reg.fFlags = CPUMSELREG_FLAGS_VALID; \
108	VMXReadCachedVmcs(VMX_VMCS32_GUEST_##REG##_LIMIT, &val); \
109	pCtx->reg.u32Limit = val; \
110	VMXReadCachedVmcs(VMX_VMCS_GUEST_##REG##_BASE, &val); \
111	pCtx->reg.u64Base = val; \
112	VMXReadCachedVmcs(VMX_VMCS32_GUEST_##REG##_ACCESS_RIGHTS, &val); \
113	pCtx->reg.Attr.u = val; \
114	} while (0)
115
116	/* Don't read from the cache in this macro; used only in case of failure where the cache is out of sync. */
117	# define VMX_LOG_SELREG(REG, szSelReg, val) \
118	do \
119	{ \
120	VMXReadVmcs(VMX_VMCS16_GUEST_FIELD_##REG, &(val)); \
121	Log(("%s Selector %x\n", szSelReg, (val))); \
122	VMXReadVmcs(VMX_VMCS32_GUEST_##REG##_LIMIT, &(val)); \
123	Log(("%s Limit %x\n", szSelReg, (val))); \
124	VMXReadVmcs(VMX_VMCS_GUEST_##REG##_BASE, &(val)); \
125	Log(("%s Base %RX64\n", szSelReg, (uint64_t)(val))); \
126	VMXReadVmcs(VMX_VMCS32_GUEST_##REG##_ACCESS_RIGHTS, &(val)); \
127	Log(("%s Attributes %x\n", szSelReg, (val))); \
128	} while (0)
129
130	#define VMXSetupCachedReadVmcs(pCache, idxField) \
131	{ \
132	Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
133	pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
134	pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
135	}
136	#define VMX_SETUP_SELREG(REG, pCache) \
137	{ \
138	VMXSetupCachedReadVmcs(pCache, VMX_VMCS16_GUEST_FIELD_##REG); \
139	VMXSetupCachedReadVmcs(pCache, VMX_VMCS32_GUEST_##REG##_LIMIT); \
140	VMXSetupCachedReadVmcs(pCache, VMX_VMCS_GUEST_##REG##_BASE); \
141	VMXSetupCachedReadVmcs(pCache, VMX_VMCS32_GUEST_##REG##_ACCESS_RIGHTS); \
142	}
143
144
145	/*******************************************************************************
146	* Global Variables *
147	*******************************************************************************/
148	/* IO operation lookup arrays. */
149	static uint32_t const g_aIOSize[4] = {1, 2, 0, 4};
150	static uint32_t const g_aIOOpAnd[4] = {0xff, 0xffff, 0, 0xffffffff};
151
152	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
153	/** See HMR0A.asm. */
154	extern "C" uint32_t g_fVMXIs64bitHost;
155	#endif
156
157
158	/*******************************************************************************
159	* Local Functions *
160	*******************************************************************************/
161	static DECLCALLBACK(void) hmR0VmxSetupTLBEPT(PVM pVM, PVMCPU pVCpu);
162	static DECLCALLBACK(void) hmR0VmxSetupTLBVPID(PVM pVM, PVMCPU pVCpu);
163	static DECLCALLBACK(void) hmR0VmxSetupTLBBoth(PVM pVM, PVMCPU pVCpu);
164	static DECLCALLBACK(void) hmR0VmxSetupTLBDummy(PVM pVM, PVMCPU pVCpu);
165	static void hmR0VmxFlushEPT(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_EPT enmFlush);
166	static void hmR0VmxFlushVPID(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_VPID enmFlush, RTGCPTR GCPtr);
167	static void hmR0VmxUpdateExceptionBitmap(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx);
168	static void hmR0VmxSetMSRPermission(PVMCPU pVCpu, unsigned ulMSR, bool fRead, bool fWrite);
169	static void hmR0VmxReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, VBOXSTRICTRC rc, PCPUMCTX pCtx);
170
171
172	/**
173	* Updates error from VMCS to HMCPU's lasterror record.
174	*
175	* @param pVM Pointer to the VM.
176	* @param pVCpu Pointer to the VMCPU.
177	* @param rc The error code.
178	*/
179	static void hmR0VmxCheckError(PVM pVM, PVMCPU pVCpu, int rc)
180	{
181	if ( rc == VERR_VMX_UNABLE_TO_START_VM
182	\|\| rc == VERR_VMX_INVALID_VMCS_FIELD)
183	{
184	RTCCUINTREG instrError;
185
186	VMXReadVmcs(VMX_VMCS32_RO_VM_INSTR_ERROR, &instrError);
187	pVCpu->hm.s.vmx.lasterror.u32InstrError = instrError;
188	}
189	pVM->hm.s.lLastError = rc;
190	}
191
192
193	/**
194	* Sets up and activates VT-x on the current CPU.
195	*
196	* @returns VBox status code.
197	* @param pCpu Pointer to the CPU info struct.
198	* @param pVM Pointer to the VM. (can be NULL after a resume!!)
199	* @param pvCpuPage Pointer to the global CPU page.
200	* @param HCPhysCpuPage Physical address of the global CPU page.
201	* @param fEnabledByHost Set if SUPR0EnableVTx or similar was used to enable
202	* VT-x/AMD-V on the host.
203	*/
204	VMMR0DECL(int) VMXR0EnableCpu(PHMGLOBLCPUINFO pCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost)
205	{
206	if (!fEnabledByHost)
207	{
208	AssertReturn(HCPhysCpuPage != 0 && HCPhysCpuPage != NIL_RTHCPHYS, VERR_INVALID_PARAMETER);
209	AssertReturn(pvCpuPage, VERR_INVALID_PARAMETER);
210
211	if (pVM)
212	{
213	/* Set revision dword at the beginning of the VMXON structure. */
214	(uint32_t )pvCpuPage = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.msr.vmx_basic_info);
215	}
216
217	/** @todo we should unmap the two pages from the virtual address space in order to prevent accidental corruption.
218	* (which can have very bad consequences!!!)
219	*/
220
221	/** @todo r=bird: Why is this code different than the probing code earlier
222	* on? It just sets VMXE if needed and doesn't check that it isn't
223	* set. Mac OS X host_vmxoff may leave this set and we'll fail here
224	* and debug-assert in the calling code. This is what caused the
225	* "regression" after backing out the SUPR0EnableVTx code hours before
226	* 4.2.0GA (reboot fixed the issue). I've changed here to do the same
227	* as the init code. */
228	uint64_t uCr4 = ASMGetCR4();
229	if (!(uCr4 & X86_CR4_VMXE))
230	ASMSetCR4(ASMGetCR4() \| X86_CR4_VMXE); /* Make sure the VMX instructions don't cause #UD faults. */
231
232	/*
233	* Enter VM root mode.
234	*/
235	int rc = VMXEnable(HCPhysCpuPage);
236	if (RT_FAILURE(rc))
237	{
238	ASMSetCR4(uCr4);
239	return VERR_VMX_VMXON_FAILED;
240	}
241	}
242
243	/*
244	* Flush all VPIDs (in case we or any other hypervisor have been using VPIDs) so that
245	* we can avoid an explicit flush while using new VPIDs. We would still need to flush
246	* each time while reusing a VPID after hitting the MaxASID limit once.
247	*/
248	if ( pVM
249	&& pVM->hm.s.vmx.fVpid
250	&& (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS))
251	{
252	hmR0VmxFlushVPID(pVM, NULL /* pvCpu /, VMX_FLUSH_VPID_ALL_CONTEXTS, 0 / GCPtr */);
253	pCpu->fFlushAsidBeforeUse = false;
254	}
255	else
256	pCpu->fFlushAsidBeforeUse = true;
257
258	/*
259	* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}.
260	*/
261	++pCpu->cTlbFlushes;
262
263	return VINF_SUCCESS;
264	}
265
266
267	/**
268	* Deactivates VT-x on the current CPU.
269	*
270	* @returns VBox status code.
271	* @param pCpu Pointer to the CPU info struct.
272	* @param pvCpuPage Pointer to the global CPU page.
273	* @param HCPhysCpuPage Physical address of the global CPU page.
274	*/
275	VMMR0DECL(int) VMXR0DisableCpu(PHMGLOBLCPUINFO pCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
276	{
277	AssertReturn(HCPhysCpuPage != 0 && HCPhysCpuPage != NIL_RTHCPHYS, VERR_INVALID_PARAMETER);
278	AssertReturn(pvCpuPage, VERR_INVALID_PARAMETER);
279	NOREF(pCpu);
280
281	/* If we're somehow not in VMX root mode, then we shouldn't dare leaving it. */
282	if (!(ASMGetCR4() & X86_CR4_VMXE))
283	return VERR_VMX_NOT_IN_VMX_ROOT_MODE;
284
285	/* Leave VMX Root Mode. */
286	VMXDisable();
287
288	/* And clear the X86_CR4_VMXE bit. */
289	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
290	return VINF_SUCCESS;
291	}
292
293	VMMR0DECL(int) VMXR0GlobalInit(void)
294	{
295	/* Nothing to do. */
296	return VINF_SUCCESS;
297	}
298
299	VMMR0DECL(void) VMXR0GlobalTerm(void)
300	{
301	/* Nothing to do. */
302	}
303
304	/**
305	* Does Ring-0 per VM VT-x initialization.
306	*
307	* @returns VBox status code.
308	* @param pVM Pointer to the VM.
309	*/
310	VMMR0DECL(int) VMXR0InitVM(PVM pVM)
311	{
312	int rc;
313
314	#ifdef LOG_ENABLED
315	SUPR0Printf("VMXR0InitVM %p\n", pVM);
316	#endif
317
318	pVM->hm.s.vmx.hMemObjApicAccess = NIL_RTR0MEMOBJ;
319
320	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
321	{
322	/* Allocate one page for the APIC physical page (serves for filtering accesses). */
323	rc = RTR0MemObjAllocCont(&pVM->hm.s.vmx.hMemObjApicAccess, PAGE_SIZE, false /* fExecutable */);
324	AssertRC(rc);
325	if (RT_FAILURE(rc))
326	return rc;
327
328	pVM->hm.s.vmx.pbApicAccess = (uint8_t *)RTR0MemObjAddress(pVM->hm.s.vmx.hMemObjApicAccess);
329	pVM->hm.s.vmx.HCPhysApicAccess = RTR0MemObjGetPagePhysAddr(pVM->hm.s.vmx.hMemObjApicAccess, 0);
330	ASMMemZero32(pVM->hm.s.vmx.pbApicAccess, PAGE_SIZE);
331	}
332	else
333	{
334	pVM->hm.s.vmx.hMemObjApicAccess = 0;
335	pVM->hm.s.vmx.pbApicAccess = 0;
336	pVM->hm.s.vmx.HCPhysApicAccess = 0;
337	}
338
339	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
340	{
341	rc = RTR0MemObjAllocCont(&pVM->hm.s.vmx.hMemObjScratch, PAGE_SIZE, false /* fExecutable */);
342	AssertRC(rc);
343	if (RT_FAILURE(rc))
344	return rc;
345
346	pVM->hm.s.vmx.pScratch = (uint8_t *)RTR0MemObjAddress(pVM->hm.s.vmx.hMemObjScratch);
347	pVM->hm.s.vmx.pScratchPhys = RTR0MemObjGetPagePhysAddr(pVM->hm.s.vmx.hMemObjScratch, 0);
348
349	ASMMemZero32(pVM->hm.s.vmx.pbScratch, PAGE_SIZE);
350	strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
351	(uint64_t )(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xDEADBEEFDEADBEEF);
352	}
353	#endif
354
355	/* Allocate VMCSs for all guest CPUs. */
356	for (VMCPUID i = 0; i < pVM->cCpus; i++)
357	{
358	PVMCPU pVCpu = &pVM->aCpus[i];
359
360	pVCpu->hm.s.vmx.hMemObjVmcs = NIL_RTR0MEMOBJ;
361
362	/* Allocate one page for the VM control structure (VMCS). */
363	rc = RTR0MemObjAllocCont(&pVCpu->hm.s.vmx.hMemObjVmcs, PAGE_SIZE, false /* fExecutable */);
364	AssertRC(rc);
365	if (RT_FAILURE(rc))
366	return rc;
367
368	pVCpu->hm.s.vmx.pvVmcs = RTR0MemObjAddress(pVCpu->hm.s.vmx.hMemObjVmcs);
369	pVCpu->hm.s.vmx.HCPhysVmcs = RTR0MemObjGetPagePhysAddr(pVCpu->hm.s.vmx.hMemObjVmcs, 0);
370	ASMMemZeroPage(pVCpu->hm.s.vmx.pvVmcs);
371
372	pVCpu->hm.s.vmx.cr0_mask = 0;
373	pVCpu->hm.s.vmx.cr4_mask = 0;
374
375	/* Allocate one page for the virtual APIC page for TPR caching. */
376	rc = RTR0MemObjAllocCont(&pVCpu->hm.s.vmx.hMemObjVirtApic, PAGE_SIZE, false /* fExecutable */);
377	AssertRC(rc);
378	if (RT_FAILURE(rc))
379	return rc;
380
381	pVCpu->hm.s.vmx.pbVirtApic = (uint8_t *)RTR0MemObjAddress(pVCpu->hm.s.vmx.hMemObjVirtApic);
382	pVCpu->hm.s.vmx.HCPhysVirtApic = RTR0MemObjGetPagePhysAddr(pVCpu->hm.s.vmx.hMemObjVirtApic, 0);
383	ASMMemZeroPage(pVCpu->hm.s.vmx.pbVirtApic);
384
385	/* Allocate the MSR bitmap if this feature is supported. */
386	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
387	{
388	rc = RTR0MemObjAllocCont(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, PAGE_SIZE, false /* fExecutable */);
389	AssertRC(rc);
390	if (RT_FAILURE(rc))
391	return rc;
392
393	pVCpu->hm.s.vmx.pvMsrBitmap = (uint8_t *)RTR0MemObjAddress(pVCpu->hm.s.vmx.hMemObjMsrBitmap);
394	pVCpu->hm.s.vmx.HCPhysMsrBitmap = RTR0MemObjGetPagePhysAddr(pVCpu->hm.s.vmx.hMemObjMsrBitmap, 0);
395	memset(pVCpu->hm.s.vmx.pvMsrBitmap, 0xff, PAGE_SIZE);
396	}
397
398	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
399	/* Allocate one page for the guest MSR load area (for preloading guest MSRs during the world switch). */
400	rc = RTR0MemObjAllocCont(&pVCpu->hm.s.vmx.hMemObjGuestMsr, PAGE_SIZE, false /* fExecutable */);
401	AssertRC(rc);
402	if (RT_FAILURE(rc))
403	return rc;
404
405	pVCpu->hm.s.vmx.pvGuestMsr = (uint8_t *)RTR0MemObjAddress(pVCpu->hm.s.vmx.hMemObjGuestMsr);
406	pVCpu->hm.s.vmx.HCPhysGuestMsr = RTR0MemObjGetPagePhysAddr(pVCpu->hm.s.vmx.hMemObjGuestMsr, 0);
407	Assert(!(pVCpu->hm.s.vmx.HCPhysGuestMsr & 0xf));
408	memset(pVCpu->hm.s.vmx.pvGuestMsr, 0, PAGE_SIZE);
409
410	/* Allocate one page for the host MSR load area (for restoring host MSRs after the world switch back). */
411	rc = RTR0MemObjAllocCont(&pVCpu->hm.s.vmx.hMemObjHostMsr, PAGE_SIZE, false /* fExecutable */);
412	AssertRC(rc);
413	if (RT_FAILURE(rc))
414	return rc;
415
416	pVCpu->hm.s.vmx.pvHostMsr = (uint8_t *)RTR0MemObjAddress(pVCpu->hm.s.vmx.hMemObjHostMsr);
417	pVCpu->hm.s.vmx.HCPhysHostMsr = RTR0MemObjGetPagePhysAddr(pVCpu->hm.s.vmx.hMemObjHostMsr, 0);
418	Assert(!(pVCpu->hm.s.vmx.HCPhysHostMsr & 0xf));
419	memset(pVCpu->hm.s.vmx.pvHostMsr, 0, PAGE_SIZE);
420	#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
421
422	/* Current guest paging mode. */
423	pVCpu->hm.s.vmx.enmLastSeenGuestMode = PGMMODE_REAL;
424
425	#ifdef LOG_ENABLED
426	SUPR0Printf("VMXR0InitVM %x VMCS=%x (%x)\n", pVM, pVCpu->hm.s.vmx.pvVmcs, (uint32_t)pVCpu->hm.s.vmx.HCPhysVmcs);
427	#endif
428	}
429
430	return VINF_SUCCESS;
431	}
432
433
434	/**
435	* Does Ring-0 per VM VT-x termination.
436	*
437	* @returns VBox status code.
438	* @param pVM Pointer to the VM.
439	*/
440	VMMR0DECL(int) VMXR0TermVM(PVM pVM)
441	{
442	for (VMCPUID i = 0; i < pVM->cCpus; i++)
443	{
444	PVMCPU pVCpu = &pVM->aCpus[i];
445
446	if (pVCpu->hm.s.vmx.hMemObjVmcs != NIL_RTR0MEMOBJ)
447	{
448	RTR0MemObjFree(pVCpu->hm.s.vmx.hMemObjVmcs, false);
449	pVCpu->hm.s.vmx.hMemObjVmcs = NIL_RTR0MEMOBJ;
450	pVCpu->hm.s.vmx.pvVmcs = 0;
451	pVCpu->hm.s.vmx.HCPhysVmcs = 0;
452	}
453	if (pVCpu->hm.s.vmx.hMemObjVirtApic != NIL_RTR0MEMOBJ)
454	{
455	RTR0MemObjFree(pVCpu->hm.s.vmx.hMemObjVirtApic, false);
456	pVCpu->hm.s.vmx.hMemObjVirtApic = NIL_RTR0MEMOBJ;
457	pVCpu->hm.s.vmx.pbVirtApic = 0;
458	pVCpu->hm.s.vmx.HCPhysVirtApic = 0;
459	}
460	if (pVCpu->hm.s.vmx.hMemObjMsrBitmap != NIL_RTR0MEMOBJ)
461	{
462	RTR0MemObjFree(pVCpu->hm.s.vmx.hMemObjMsrBitmap, false);
463	pVCpu->hm.s.vmx.hMemObjMsrBitmap = NIL_RTR0MEMOBJ;
464	pVCpu->hm.s.vmx.pvMsrBitmap = 0;
465	pVCpu->hm.s.vmx.HCPhysMsrBitmap = 0;
466	}
467	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
468	if (pVCpu->hm.s.vmx.hMemObjHostMsr != NIL_RTR0MEMOBJ)
469	{
470	RTR0MemObjFree(pVCpu->hm.s.vmx.hMemObjHostMsr, false);
471	pVCpu->hm.s.vmx.hMemObjHostMsr = NIL_RTR0MEMOBJ;
472	pVCpu->hm.s.vmx.pvHostMsr = 0;
473	pVCpu->hm.s.vmx.HCPhysHostMsr = 0;
474	}
475	if (pVCpu->hm.s.vmx.hMemObjGuestMsr != NIL_RTR0MEMOBJ)
476	{
477	RTR0MemObjFree(pVCpu->hm.s.vmx.hMemObjGuestMsr, false);
478	pVCpu->hm.s.vmx.hMemObjGuestMsr = NIL_RTR0MEMOBJ;
479	pVCpu->hm.s.vmx.pvGuestMsr = 0;
480	pVCpu->hm.s.vmx.HCPhysGuestMsr = 0;
481	}
482	#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
483	}
484	if (pVM->hm.s.vmx.hMemObjApicAccess != NIL_RTR0MEMOBJ)
485	{
486	RTR0MemObjFree(pVM->hm.s.vmx.hMemObjApicAccess, false);
487	pVM->hm.s.vmx.hMemObjApicAccess = NIL_RTR0MEMOBJ;
488	pVM->hm.s.vmx.pbApicAccess = 0;
489	pVM->hm.s.vmx.HCPhysApicAccess = 0;
490	}
491	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
492	if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
493	{
494	ASMMemZero32(pVM->hm.s.vmx.pScratch, PAGE_SIZE);
495	RTR0MemObjFree(pVM->hm.s.vmx.hMemObjScratch, false);
496	pVM->hm.s.vmx.hMemObjScratch = NIL_RTR0MEMOBJ;
497	pVM->hm.s.vmx.pScratch = 0;
498	pVM->hm.s.vmx.pScratchPhys = 0;
499	}
500	#endif
501	return VINF_SUCCESS;
502	}
503
504
505	/**
506	* Sets up VT-x for the specified VM.
507	*
508	* @returns VBox status code.
509	* @param pVM Pointer to the VM.
510	*/
511	VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
512	{
513	int rc = VINF_SUCCESS;
514	uint32_t val;
515
516	AssertReturn(pVM, VERR_INVALID_PARAMETER);
517
518	/* Initialize these always, see hmR3InitFinalizeR0().*/
519	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NONE;
520	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NONE;
521
522	/* Determine optimal flush type for EPT. */
523	if (pVM->hm.s.fNestedPaging)
524	{
525	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
526	{
527	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
528	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_SINGLE_CONTEXT;
529	else if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
530	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_ALL_CONTEXTS;
531	else
532	{
533	/*
534	* Should never really happen. EPT is supported but no suitable flush types supported.
535	* We cannot ignore EPT at this point as we've already setup Unrestricted Guest execution.
536	*/
537	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
538	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
539	}
540	}
541	else
542	{
543	/*
544	* Should never really happen. EPT is supported but INVEPT instruction is not supported.
545	*/
546	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
547	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
548	}
549	}
550
551	/* Determine optimal flush type for VPID. */
552	if (pVM->hm.s.vmx.fVpid)
553	{
554	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
555	{
556	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
557	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_SINGLE_CONTEXT;
558	else if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
559	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_ALL_CONTEXTS;
560	else
561	{
562	/*
563	* Neither SINGLE nor ALL context flush types for VPID supported by the CPU.
564	* We do not handle other flush type combinations, ignore VPID capabilities.
565	*/
566	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
567	Log(("VMXR0SetupVM: Only VMX_FLUSH_VPID_INDIV_ADDR supported. Ignoring VPID.\n"));
568	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
569	Log(("VMXR0SetupVM: Only VMX_FLUSH_VPID_SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
570	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NOT_SUPPORTED;
571	pVM->hm.s.vmx.fVpid = false;
572	}
573	}
574	else
575	{
576	/*
577	* Should not really happen. EPT is supported but INVEPT is not supported.
578	* Ignore VPID capabilities as our code relies on using INVEPT for selective flushing.
579	*/
580	Log(("VMXR0SetupVM: VPID supported without INVEPT support. Ignoring VPID.\n"));
581	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NOT_SUPPORTED;
582	pVM->hm.s.vmx.fVpid = false;
583	}
584	}
585
586	for (VMCPUID i = 0; i < pVM->cCpus; i++)
587	{
588	PVMCPU pVCpu = &pVM->aCpus[i];
589
590	AssertPtr(pVCpu->hm.s.vmx.pvVmcs);
591
592	/* Set revision dword at the beginning of the VMCS structure. */
593	(uint32_t )pVCpu->hm.s.vmx.pvVmcs = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.msr.vmx_basic_info);
594
595	/*
596	* Clear and activate the VMCS.
597	*/
598	Log(("HCPhysVmcs = %RHp\n", pVCpu->hm.s.vmx.HCPhysVmcs));
599	rc = VMXClearVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
600	if (RT_FAILURE(rc))
601	goto vmx_end;
602
603	rc = VMXActivateVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
604	if (RT_FAILURE(rc))
605	goto vmx_end;
606
607	/*
608	* VMX_VMCS_CTRL_PIN_EXEC_CONTROLS
609	* Set required bits to one and zero according to the MSR capabilities.
610	*/
611	val = pVM->hm.s.vmx.msr.vmx_pin_ctls.n.disallowed0;
612	val \|= VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_EXT_INT_EXIT /* External interrupts */
613	\| VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_NMI_EXIT; /* Non-maskable interrupts */
614
615	/*
616	* Enable the VMX preemption timer.
617	*/
618	if (pVM->hm.s.vmx.fUsePreemptTimer)
619	val \|= VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_PREEMPT_TIMER;
620	val &= pVM->hm.s.vmx.msr.vmx_pin_ctls.n.allowed1;
621
622	rc = VMXWriteVmcs(VMX_VMCS32_CTRL_PIN_EXEC_CONTROLS, val);
623	AssertRC(rc);
624	pVCpu->hm.s.vmx.u32PinCtls = val;
625
626	/*
627	* VMX_VMCS_CTRL_PROC_EXEC_CONTROLS
628	* Set required bits to one and zero according to the MSR capabilities.
629	*/
630	val = pVM->hm.s.vmx.msr.vmx_proc_ctls.n.disallowed0;
631	/* Program which event cause VM-exits and which features we want to use. */
632	val \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_HLT_EXIT
633	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TSC_OFFSETTING
634	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT
635	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_UNCOND_IO_EXIT
636	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDPMC_EXIT
637	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_EXIT
638	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MWAIT_EXIT; /* don't execute mwait or else we'll idle inside
639	the guest (host thinks the cpu load is high) */
640
641	/* Without nested paging we should intercept invlpg and cr3 mov instructions. */
642	if (!pVM->hm.s.fNestedPaging)
643	{
644	val \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INVLPG_EXIT
645	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
646	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT;
647	}
648
649	/*
650	* VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MWAIT_EXIT might cause a vmlaunch
651	* failure with an invalid control fields error. (combined with some other exit reasons)
652	*/
653	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
654	{
655	/* CR8 reads from the APIC shadow page; writes cause an exit is they lower the TPR below the threshold */
656	val \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW;
657	Assert(pVM->hm.s.vmx.pbApicAccess);
658	}
659	else
660	/* Exit on CR8 reads & writes in case the TPR shadow feature isn't present. */
661	val \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR8_STORE_EXIT \| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR8_LOAD_EXIT;
662
663	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
664	{
665	Assert(pVCpu->hm.s.vmx.HCPhysMsrBitmap);
666	val \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS;
667	}
668
669	/* We will use the secondary control if it's present. */
670	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
671
672	/* Mask away the bits that the CPU doesn't support */
673	/** @todo make sure they don't conflict with the above requirements. */
674	val &= pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1;
675	pVCpu->hm.s.vmx.u32ProcCtls = val;
676
677	rc = VMXWriteVmcs(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, val);
678	AssertRC(rc);
679
680	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
681	{
682	/*
683	* VMX_VMCS_CTRL_PROC_EXEC_CONTROLS2
684	* Set required bits to one and zero according to the MSR capabilities.
685	*/
686	val = pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.disallowed0;
687	val \|= VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT;
688
689	if (pVM->hm.s.fNestedPaging)
690	val \|= VMX_VMCS_CTRL_PROC_EXEC2_EPT;
691
692	if (pVM->hm.s.vmx.fVpid)
693	val \|= VMX_VMCS_CTRL_PROC_EXEC2_VPID;
694
695	if (pVM->hm.s.fHasIoApic)
696	val \|= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC;
697
698	if (pVM->hm.s.vmx.fUnrestrictedGuest)
699	val \|= VMX_VMCS_CTRL_PROC_EXEC2_UNRESTRICTED_GUEST;
700
701	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
702	val \|= VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP;
703
704	/* Mask away the bits that the CPU doesn't support */
705	/** @todo make sure they don't conflict with the above requirements. */
706	val &= pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1;
707	pVCpu->hm.s.vmx.u32ProcCtls2 = val;
708	rc = VMXWriteVmcs(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS2, val);
709	AssertRC(rc);
710	}
711
712	/*
713	* VMX_VMCS_CTRL_CR3_TARGET_COUNT
714	* Set required bits to one and zero according to the MSR capabilities.
715	*/
716	rc = VMXWriteVmcs(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, 0);
717	AssertRC(rc);
718
719	/*
720	* Forward all exception except #NM & #PF to the guest.
721	* We always need to check pagefaults since our shadow page table can be out of sync.
722	* And we always lazily sync the FPU & XMM state. .
723	*/
724
725	/** @todo Possible optimization:
726	* Keep the FPU and XMM state current in the EM thread. That way there's no need to
727	* lazily sync anything, but the downside is that we can't use the FPU stack or XMM
728	* registers ourselves of course.
729	*
730	* Note: only possible if the current state is actually ours (X86_CR0_TS flag)
731	*/
732
733	/*
734	* Don't filter page faults, all of them should cause a world switch.
735	*/
736	rc = VMXWriteVmcs(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, 0);
737	AssertRC(rc);
738	rc = VMXWriteVmcs(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, 0);
739	AssertRC(rc);
740
741	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, 0);
742	AssertRC(rc);
743	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_A_FULL, 0);
744	AssertRC(rc);
745	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_B_FULL, 0);
746	AssertRC(rc);
747
748	/*
749	* Set the MSR bitmap address.
750	*/
751	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
752	{
753	Assert(pVCpu->hm.s.vmx.HCPhysMsrBitmap);
754
755	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, pVCpu->hm.s.vmx.HCPhysMsrBitmap);
756	AssertRC(rc);
757
758	/*
759	* Allow the guest to directly modify these MSRs; they are loaded/stored automatically
760	* using MSR-load/store areas in the VMCS.
761	*/
762	hmR0VmxSetMSRPermission(pVCpu, MSR_IA32_SYSENTER_CS, true, true);
763	hmR0VmxSetMSRPermission(pVCpu, MSR_IA32_SYSENTER_ESP, true, true);
764	hmR0VmxSetMSRPermission(pVCpu, MSR_IA32_SYSENTER_EIP, true, true);
765	hmR0VmxSetMSRPermission(pVCpu, MSR_K8_LSTAR, true, true);
766	hmR0VmxSetMSRPermission(pVCpu, MSR_K6_STAR, true, true);
767	hmR0VmxSetMSRPermission(pVCpu, MSR_K8_SF_MASK, true, true);
768	hmR0VmxSetMSRPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, true, true);
769	hmR0VmxSetMSRPermission(pVCpu, MSR_K8_GS_BASE, true, true);
770	hmR0VmxSetMSRPermission(pVCpu, MSR_K8_FS_BASE, true, true);
771	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
772	hmR0VmxSetMSRPermission(pVCpu, MSR_K8_TSC_AUX, true, true);
773	}
774
775	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
776	/*
777	* Set the guest & host MSR load/store physical addresses.
778	*/
779	Assert(pVCpu->hm.s.vmx.HCPhysGuestMsr);
780	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
781	AssertRC(rc);
782	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
783	AssertRC(rc);
784	Assert(pVCpu->hm.s.vmx.HCPhysHostMsr);
785	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysHostMsr);
786	AssertRC(rc);
787	#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
788
789	rc = VMXWriteVmcs(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, 0);
790	AssertRC(rc);
791	rc = VMXWriteVmcs(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, 0);
792	AssertRC(rc);
793	rc = VMXWriteVmcs(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, 0);
794	AssertRC(rc);
795
796	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
797	{
798	Assert(pVM->hm.s.vmx.hMemObjApicAccess);
799	/* Optional */
800	rc = VMXWriteVmcs(VMX_VMCS32_CTRL_TPR_THRESHOLD, 0);
801	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL, pVCpu->hm.s.vmx.HCPhysVirtApic);
802
803	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
804	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, pVM->hm.s.vmx.HCPhysApicAccess);
805
806	AssertRC(rc);
807	}
808
809	/* Set link pointer to -1. Not currently used. */
810	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, 0xFFFFFFFFFFFFFFFFULL);
811	AssertRC(rc);
812
813	/*
814	* Clear VMCS, marking it inactive. Clear implementation specific data and writing back
815	* VMCS data back to memory.
816	*/
817	rc = VMXClearVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
818	AssertRC(rc);
819
820	/*
821	* Configure the VMCS read cache.
822	*/
823	PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
824
825	VMXSetupCachedReadVmcs(pCache, VMX_VMCS_GUEST_RIP);
826	VMXSetupCachedReadVmcs(pCache, VMX_VMCS_GUEST_RSP);
827	VMXSetupCachedReadVmcs(pCache, VMX_VMCS_GUEST_RFLAGS);
828	VMXSetupCachedReadVmcs(pCache, VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE);
829	VMXSetupCachedReadVmcs(pCache, VMX_VMCS_CTRL_CR0_READ_SHADOW);
830	VMXSetupCachedReadVmcs(pCache, VMX_VMCS_GUEST_CR0);
831	VMXSetupCachedReadVmcs(pCache, VMX_VMCS_CTRL_CR4_READ_SHADOW);
832	VMXSetupCachedReadVmcs(pCache, VMX_VMCS_GUEST_CR4);
833	VMXSetupCachedReadVmcs(pCache, VMX_VMCS_GUEST_DR7);
834	VMXSetupCachedReadVmcs(pCache, VMX_VMCS32_GUEST_SYSENTER_CS);
835	VMXSetupCachedReadVmcs(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
836	VMXSetupCachedReadVmcs(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
837	VMXSetupCachedReadVmcs(pCache, VMX_VMCS32_GUEST_GDTR_LIMIT);
838	VMXSetupCachedReadVmcs(pCache, VMX_VMCS_GUEST_GDTR_BASE);
839	VMXSetupCachedReadVmcs(pCache, VMX_VMCS32_GUEST_IDTR_LIMIT);
840	VMXSetupCachedReadVmcs(pCache, VMX_VMCS_GUEST_IDTR_BASE);
841
842	VMX_SETUP_SELREG(ES, pCache);
843	VMX_SETUP_SELREG(SS, pCache);
844	VMX_SETUP_SELREG(CS, pCache);
845	VMX_SETUP_SELREG(DS, pCache);
846	VMX_SETUP_SELREG(FS, pCache);
847	VMX_SETUP_SELREG(GS, pCache);
848	VMX_SETUP_SELREG(LDTR, pCache);
849	VMX_SETUP_SELREG(TR, pCache);
850
851	/*
852	* Status code VMCS reads.
853	*/
854	VMXSetupCachedReadVmcs(pCache, VMX_VMCS32_RO_EXIT_REASON);
855	VMXSetupCachedReadVmcs(pCache, VMX_VMCS32_RO_VM_INSTR_ERROR);
856	VMXSetupCachedReadVmcs(pCache, VMX_VMCS32_RO_EXIT_INSTR_LENGTH);
857	VMXSetupCachedReadVmcs(pCache, VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE);
858	VMXSetupCachedReadVmcs(pCache, VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO);
859	VMXSetupCachedReadVmcs(pCache, VMX_VMCS32_RO_EXIT_INSTR_INFO);
860	VMXSetupCachedReadVmcs(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
861	VMXSetupCachedReadVmcs(pCache, VMX_VMCS32_RO_IDT_INFO);
862	VMXSetupCachedReadVmcs(pCache, VMX_VMCS32_RO_IDT_ERROR_CODE);
863
864	if (pVM->hm.s.fNestedPaging)
865	{
866	VMXSetupCachedReadVmcs(pCache, VMX_VMCS_GUEST_CR3);
867	VMXSetupCachedReadVmcs(pCache, VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL);
868	pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
869	}
870	else
871	pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
872	} /* for each VMCPU */
873
874	/*
875	* Setup the right TLB function based on CPU capabilities.
876	*/
877	if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
878	pVM->hm.s.vmx.pfnFlushTaggedTlb = hmR0VmxSetupTLBBoth;
879	else if (pVM->hm.s.fNestedPaging)
880	pVM->hm.s.vmx.pfnFlushTaggedTlb = hmR0VmxSetupTLBEPT;
881	else if (pVM->hm.s.vmx.fVpid)
882	pVM->hm.s.vmx.pfnFlushTaggedTlb = hmR0VmxSetupTLBVPID;
883	else
884	pVM->hm.s.vmx.pfnFlushTaggedTlb = hmR0VmxSetupTLBDummy;
885
886	vmx_end:
887	hmR0VmxCheckError(pVM, &pVM->aCpus[0], rc);
888	return rc;
889	}
890
891
892	/**
893	* Sets the permission bits for the specified MSR.
894	*
895	* @param pVCpu Pointer to the VMCPU.
896	* @param ulMSR The MSR value.
897	* @param fRead Whether reading is allowed.
898	* @param fWrite Whether writing is allowed.
899	*/
900	static void hmR0VmxSetMSRPermission(PVMCPU pVCpu, unsigned ulMSR, bool fRead, bool fWrite)
901	{
902	unsigned ulBit;
903	uint8_t pvMsrBitmap = (uint8_t )pVCpu->hm.s.vmx.pvMsrBitmap;
904
905	/*
906	* Layout:
907	* 0x000 - 0x3ff - Low MSR read bits
908	* 0x400 - 0x7ff - High MSR read bits
909	* 0x800 - 0xbff - Low MSR write bits
910	* 0xc00 - 0xfff - High MSR write bits
911	*/
912	if (ulMSR <= 0x00001FFF)
913	{
914	/* Pentium-compatible MSRs */
915	ulBit = ulMSR;
916	}
917	else if ( ulMSR >= 0xC0000000
918	&& ulMSR <= 0xC0001FFF)
919	{
920	/* AMD Sixth Generation x86 Processor MSRs */
921	ulBit = (ulMSR - 0xC0000000);
922	pvMsrBitmap += 0x400;
923	}
924	else
925	{
926	AssertFailed();
927	return;
928	}
929
930	Assert(ulBit <= 0x1fff);
931	if (fRead)
932	ASMBitClear(pvMsrBitmap, ulBit);
933	else
934	ASMBitSet(pvMsrBitmap, ulBit);
935
936	if (fWrite)
937	ASMBitClear(pvMsrBitmap + 0x800, ulBit);
938	else
939	ASMBitSet(pvMsrBitmap + 0x800, ulBit);
940	}
941
942
943	/**
944	* Injects an event (trap or external interrupt).
945	*
946	* @returns VBox status code. Note that it may return VINF_EM_RESET to
947	* indicate a triple fault when injecting X86_XCPT_DF.
948	*
949	* @param pVM Pointer to the VM.
950	* @param pVCpu Pointer to the VMCPU.
951	* @param pCtx Pointer to the guest CPU Context.
952	* @param intInfo VMX interrupt info.
953	* @param cbInstr Opcode length of faulting instruction.
954	* @param errCode Error code (optional).
955	*/
956	static int hmR0VmxInjectEvent(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, uint32_t intInfo, uint32_t cbInstr, uint32_t errCode)
957	{
958	int rc;
959	uint32_t iGate = VMX_EXIT_INTERRUPTION_INFO_VECTOR(intInfo);
960
961	#ifdef VBOX_WITH_STATISTICS
962	STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[iGate & MASK_INJECT_IRQ_STAT]);
963	#endif
964
965	#ifdef VBOX_STRICT
966	if (iGate == 0xE)
967	{
968	LogFlow(("hmR0VmxInjectEvent: Injecting interrupt %d at %RGv error code=%08x CR2=%RGv intInfo=%08x\n", iGate,
969	(RTGCPTR)pCtx->rip, errCode, pCtx->cr2, intInfo));
970	}
971	else if (iGate < 0x20)
972	{
973	LogFlow(("hmR0VmxInjectEvent: Injecting interrupt %d at %RGv error code=%08x\n", iGate, (RTGCPTR)pCtx->rip,
974	errCode));
975	}
976	else
977	{
978	LogFlow(("INJ-EI: %x at %RGv\n", iGate, (RTGCPTR)pCtx->rip));
979	Assert( VMX_EXIT_INTERRUPTION_INFO_TYPE(intInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT
980	\|\| !VMCPU_FF_ISSET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
981	Assert( VMX_EXIT_INTERRUPTION_INFO_TYPE(intInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT
982	\|\| pCtx->eflags.u32 & X86_EFL_IF);
983	}
984	#endif
985
986	if ( CPUMIsGuestInRealModeEx(pCtx)
987	&& pVM->hm.s.vmx.pRealModeTSS)
988	{
989	RTGCPHYS GCPhysHandler;
990	uint16_t offset, ip;
991	RTSEL sel;
992
993	/*
994	* Injecting events doesn't work right with real mode emulation.
995	* (#GP if we try to inject external hardware interrupts)
996	* Inject the interrupt or trap directly instead.
997	*
998	* ASSUMES no access handlers for the bits we read or write below (should be safe).
999	*/
1000	Log(("Manual interrupt/trap '%x' inject (real mode)\n", iGate));
1001
1002	/*
1003	* Check if the interrupt handler is present.
1004	*/
1005	if (iGate * 4 + 3 > pCtx->idtr.cbIdt)
1006	{
1007	Log(("IDT cbIdt violation\n"));
1008	if (iGate != X86_XCPT_DF)
1009	{
1010	uint32_t intInfo2;
1011
1012	intInfo2 = (iGate == X86_XCPT_GP) ? (uint32_t)X86_XCPT_DF : (uint32_t)X86_XCPT_GP;
1013	intInfo2 \|= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
1014	intInfo2 \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
1015	intInfo2 \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
1016
1017	return hmR0VmxInjectEvent(pVM, pVCpu, pCtx, intInfo2, 0, 0 /* no error code according to the Intel docs */);
1018	}
1019	Log(("Triple fault -> reset the VM!\n"));
1020	return VINF_EM_RESET;
1021	}
1022	if ( VMX_EXIT_INTERRUPTION_INFO_TYPE(intInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT
1023	\|\| iGate == 3 /* Both #BP and #OF point to the instruction after. */
1024	\|\| iGate == 4)
1025	{
1026	ip = pCtx->ip + cbInstr;
1027	}
1028	else
1029	ip = pCtx->ip;
1030
1031	/*
1032	* Read the selector:offset pair of the interrupt handler.
1033	*/
1034	GCPhysHandler = (RTGCPHYS)pCtx->idtr.pIdt + iGate * 4;
1035	rc = PGMPhysSimpleReadGCPhys(pVM, &offset, GCPhysHandler, sizeof(offset)); AssertRC(rc);
1036	rc = PGMPhysSimpleReadGCPhys(pVM, &sel, GCPhysHandler + 2, sizeof(sel)); AssertRC(rc);
1037
1038	LogFlow(("IDT handler %04X:%04X\n", sel, offset));
1039
1040	/*
1041	* Construct the stack frame.
1042	*/
1043	/** @todo Check stack limit. */
1044	pCtx->sp -= 2;
1045	LogFlow(("ss:sp %04X:%04X eflags=%x\n", pCtx->ss.Sel, pCtx->sp, pCtx->eflags.u));
1046	rc = PGMPhysSimpleWriteGCPhys(pVM, pCtx->ss.u64Base + pCtx->sp, &pCtx->eflags, sizeof(uint16_t)); AssertRC(rc);
1047	pCtx->sp -= 2;
1048	LogFlow(("ss:sp %04X:%04X cs=%x\n", pCtx->ss.Sel, pCtx->sp, pCtx->cs.Sel));
1049	rc = PGMPhysSimpleWriteGCPhys(pVM, pCtx->ss.u64Base + pCtx->sp, &pCtx->cs, sizeof(uint16_t)); AssertRC(rc);
1050	pCtx->sp -= 2;
1051	LogFlow(("ss:sp %04X:%04X ip=%x\n", pCtx->ss.Sel, pCtx->sp, ip));
1052	rc = PGMPhysSimpleWriteGCPhys(pVM, pCtx->ss.u64Base + pCtx->sp, &ip, sizeof(ip)); AssertRC(rc);
1053
1054	/*
1055	* Update the CPU state for executing the handler.
1056	*/
1057	pCtx->rip = offset;
1058	pCtx->cs.Sel = sel;
1059	pCtx->cs.u64Base = sel << 4;
1060	pCtx->eflags.u &= ~(X86_EFL_IF \| X86_EFL_TF \| X86_EFL_RF \| X86_EFL_AC);
1061
1062	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SEGMENT_REGS;
1063	return VINF_SUCCESS;
1064	}
1065
1066	/*
1067	* Set event injection state.
1068	*/
1069	rc = VMXWriteVmcs(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, intInfo \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT));
1070	rc \|= VMXWriteVmcs(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
1071	rc \|= VMXWriteVmcs(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, errCode);
1072
1073	AssertRC(rc);
1074	return rc;
1075	}
1076
1077
1078	/**
1079	* Checks for pending guest interrupts and injects them.
1080	*
1081	* @returns VBox status code.
1082	* @param pVM Pointer to the VM.
1083	* @param pVCpu Pointer to the VMCPU.
1084	* @param pCtx Pointer to the guest CPU context.
1085	*/
1086	static int hmR0VmxCheckPendingInterrupt(PVM pVM, PVMCPU pVCpu, CPUMCTX *pCtx)
1087	{
1088	int rc;
1089
1090	/*
1091	* Dispatch any pending interrupts (injected before, but a VM exit occurred prematurely).
1092	*/
1093	if (pVCpu->hm.s.Event.fPending)
1094	{
1095	Log(("CPU%d: Reinjecting event %RX64 %08x at %RGv cr2=%RX64\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntrInfo,
1096	pVCpu->hm.s.Event.u32ErrCode, (RTGCPTR)pCtx->rip, pCtx->cr2));
1097	STAM_COUNTER_INC(&pVCpu->hm.s.StatIntReinject);
1098	rc = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, pVCpu->hm.s.Event.u64IntrInfo, 0, pVCpu->hm.s.Event.u32ErrCode);
1099	AssertRC(rc);
1100
1101	pVCpu->hm.s.Event.fPending = false;
1102	return VINF_SUCCESS;
1103	}
1104
1105	/*
1106	* If an active trap is already pending, we must forward it first!
1107	*/
1108	if (!TRPMHasTrap(pVCpu))
1109	{
1110	if (VMCPU_FF_TESTANDCLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI))
1111	{
1112	RTGCUINTPTR intInfo;
1113
1114	Log(("CPU%d: injecting #NMI\n", pVCpu->idCpu));
1115
1116	intInfo = X86_XCPT_NMI;
1117	intInfo \|= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
1118	intInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
1119
1120	rc = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, intInfo, 0, 0);
1121	AssertRC(rc);
1122
1123	return VINF_SUCCESS;
1124	}
1125
1126	/** @todo SMI interrupts. */
1127
1128	/*
1129	* When external interrupts are pending, we should exit the VM when IF is set.
1130	*/
1131	if (VMCPU_FF_ISPENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC\|VMCPU_FF_INTERRUPT_PIC)))
1132	{
1133	if (!(pCtx->eflags.u32 & X86_EFL_IF))
1134	{
1135	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INT_WINDOW_EXIT))
1136	{
1137	LogFlow(("Enable irq window exit!\n"));
1138	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INT_WINDOW_EXIT;
1139	rc = VMXWriteVmcs(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls);
1140	AssertRC(rc);
1141	}
1142	/* else nothing to do but wait */
1143	}
1144	else if (!VMCPU_FF_ISSET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
1145	{
1146	uint8_t u8Interrupt;
1147
1148	rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
1149	Log(("CPU%d: Dispatch interrupt: u8Interrupt=%x (%d) rc=%Rrc cs:rip=%04X:%RGv\n", pVCpu->idCpu,
1150	u8Interrupt, u8Interrupt, rc, pCtx->cs.Sel, (RTGCPTR)pCtx->rip));
1151	if (RT_SUCCESS(rc))
1152	{
1153	rc = TRPMAssertTrap(pVCpu, u8Interrupt, TRPM_HARDWARE_INT);
1154	AssertRC(rc);
1155	}
1156	else
1157	{
1158	/* Can only happen in rare cases where a pending interrupt is cleared behind our back */
1159	Assert(!VMCPU_FF_ISPENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC\|VMCPU_FF_INTERRUPT_PIC)));
1160	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
1161	/* Just continue */
1162	}
1163	}
1164	else
1165	Log(("Pending interrupt blocked at %RGv by VM_FF_INHIBIT_INTERRUPTS!!\n", (RTGCPTR)pCtx->rip));
1166	}
1167	}
1168
1169	#ifdef VBOX_STRICT
1170	if (TRPMHasTrap(pVCpu))
1171	{
1172	uint8_t u8Vector;
1173	rc = TRPMQueryTrapAll(pVCpu, &u8Vector, 0, 0, 0);
1174	AssertRC(rc);
1175	}
1176	#endif
1177
1178	if ( (pCtx->eflags.u32 & X86_EFL_IF)
1179	&& (!VMCPU_FF_ISSET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
1180	&& TRPMHasTrap(pVCpu)
1181	)
1182	{
1183	uint8_t u8Vector;
1184	TRPMEVENT enmType;
1185	RTGCUINTPTR intInfo;
1186	RTGCUINT errCode;
1187
1188	/*
1189	* If a new event is pending, dispatch it now.
1190	*/
1191	rc = TRPMQueryTrapAll(pVCpu, &u8Vector, &enmType, &errCode, 0);
1192	AssertRC(rc);
1193	Assert(pCtx->eflags.Bits.u1IF == 1 \|\| enmType == TRPM_TRAP);
1194	Assert(enmType != TRPM_SOFTWARE_INT);
1195
1196	/*
1197	* Clear the pending trap.
1198	*/
1199	rc = TRPMResetTrap(pVCpu);
1200	AssertRC(rc);
1201
1202	intInfo = u8Vector;
1203	intInfo \|= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
1204
1205	if (enmType == TRPM_TRAP)
1206	{
1207	switch (u8Vector)
1208	{
1209	case X86_XCPT_DF:
1210	case X86_XCPT_TS:
1211	case X86_XCPT_NP:
1212	case X86_XCPT_SS:
1213	case X86_XCPT_GP:
1214	case X86_XCPT_PF:
1215	case X86_XCPT_AC:
1216	{
1217	/** @todo r=ramshankar: setting this bit would blow up for real-mode guests with
1218	* unrestricted guest execution. */
1219	/* Valid error codes. */
1220	intInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
1221	break;
1222	}
1223
1224	default:
1225	break;
1226	}
1227
1228	if ( u8Vector == X86_XCPT_BP
1229	\|\| u8Vector == X86_XCPT_OF)
1230	{
1231	intInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
1232	}
1233	else
1234	intInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
1235	}
1236	else
1237	intInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
1238
1239	STAM_COUNTER_INC(&pVCpu->hm.s.StatIntInject);
1240	rc = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, intInfo, 0, errCode);
1241	AssertRC(rc);
1242	} /* if (interrupts can be dispatched) */
1243
1244	return VINF_SUCCESS;
1245	}
1246
1247	/**
1248	* Checks for pending VMX events and converts them to TRPM. Before we execute any instruction
1249	* outside of VMX, any pending VMX event must be converted so that it can be delivered properly.
1250	*
1251	* @returns VBox status code.
1252	* @param pVCpu Pointer to the VMCPU.
1253	*/
1254	static int hmR0VmxCheckPendingEvent(PVMCPU pVCpu)
1255	{
1256	if (pVCpu->hm.s.Event.fPending)
1257	{
1258	TRPMEVENT enmTrapType;
1259
1260	/* If a trap was already pending, we did something wrong! */
1261	Assert((TRPMQueryTrap(pVCpu, NULL, NULL) == VERR_TRPM_NO_ACTIVE_TRAP));
1262
1263	/*
1264	* Clear the pending event and move it over to TRPM for the rest
1265	* of the world to see.
1266	*/
1267	pVCpu->hm.s.Event.fPending = false;
1268	switch (VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hm.s.Event.u64IntrInfo))
1269	{
1270	case VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT:
1271	case VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI:
1272	enmTrapType = TRPM_HARDWARE_INT;
1273	break;
1274	case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT:
1275	case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT: /** @todo Is classifying #BP, #OF as TRPM_SOFTWARE_INT correct? */
1276	case VMX_EXIT_INTERRUPTION_INFO_TYPE_DB_XCPT:
1277	enmTrapType = TRPM_SOFTWARE_INT;
1278	break;
1279	case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
1280	enmTrapType = TRPM_TRAP;
1281	break;
1282	default:
1283	enmTrapType = TRPM_32BIT_HACK; /* Can't get here. */
1284	AssertFailed();
1285	}
1286	TRPMAssertTrap(pVCpu, VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVCpu->hm.s.Event.u64IntrInfo), enmTrapType);
1287	if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntrInfo))
1288	TRPMSetErrorCode(pVCpu, pVCpu->hm.s.Event.u32ErrCode);
1289	//@todo: Is there any situation where we need to call TRPMSetFaultAddress()?
1290	}
1291	return VINF_SUCCESS;
1292	}
1293
1294	/**
1295	* Save the host state into the VMCS.
1296	*
1297	* @returns VBox status code.
1298	* @param pVM Pointer to the VM.
1299	* @param pVCpu Pointer to the VMCPU.
1300	*/
1301	VMMR0DECL(int) VMXR0SaveHostState(PVM pVM, PVMCPU pVCpu)
1302	{
1303	int rc = VINF_SUCCESS;
1304	NOREF(pVM);
1305
1306	/*
1307	* Host CPU Context.
1308	*/
1309	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_HOST_CONTEXT)
1310	{
1311	RTIDTR idtr;
1312	RTGDTR gdtr;
1313	RTSEL SelTR;
1314	PCX86DESCHC pDesc;
1315	uintptr_t trBase;
1316	RTSEL cs;
1317	RTSEL ss;
1318	uint64_t cr3;
1319
1320	/*
1321	* Control registers.
1322	*/
1323	rc = VMXWriteVmcs(VMX_VMCS_HOST_CR0, ASMGetCR0());
1324	Log2(("VMX_VMCS_HOST_CR0 %08x\n", ASMGetCR0()));
1325	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
1326	if (VMX_IS_64BIT_HOST_MODE())
1327	{
1328	cr3 = hmR0Get64bitCR3();
1329	rc \|= VMXWriteVmcs64(VMX_VMCS_HOST_CR3, cr3);
1330	}
1331	else
1332	#endif
1333	{
1334	cr3 = ASMGetCR3();
1335	rc \|= VMXWriteVmcs(VMX_VMCS_HOST_CR3, cr3);
1336	}
1337	Log2(("VMX_VMCS_HOST_CR3 %08RX64\n", cr3));
1338	rc \|= VMXWriteVmcs(VMX_VMCS_HOST_CR4, ASMGetCR4());
1339	Log2(("VMX_VMCS_HOST_CR4 %08x\n", ASMGetCR4()));
1340	AssertRC(rc);
1341
1342	/*
1343	* Selector registers.
1344	*/
1345	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
1346	if (VMX_IS_64BIT_HOST_MODE())
1347	{
1348	cs = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelCS;
1349	ss = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelSS;
1350	}
1351	else
1352	{
1353	/* sysenter loads LDT cs & ss, VMX doesn't like this. Load the GDT ones (safe). */
1354	cs = (RTSEL)(uintptr_t)&SUPR0AbsKernelCS;
1355	ss = (RTSEL)(uintptr_t)&SUPR0AbsKernelSS;
1356	}
1357	#else
1358	cs = ASMGetCS();
1359	ss = ASMGetSS();
1360	#endif
1361	Assert(!(cs & X86_SEL_LDT)); Assert((cs & X86_SEL_RPL) == 0);
1362	Assert(!(ss & X86_SEL_LDT)); Assert((ss & X86_SEL_RPL) == 0);
1363	rc = VMXWriteVmcs(VMX_VMCS16_HOST_FIELD_CS, cs);
1364	/* Note: VMX is (again) very picky about the RPL of the selectors here; we'll restore them manually. */
1365	rc \|= VMXWriteVmcs(VMX_VMCS16_HOST_FIELD_DS, 0);
1366	rc \|= VMXWriteVmcs(VMX_VMCS16_HOST_FIELD_ES, 0);
1367	#if HC_ARCH_BITS == 32
1368	if (!VMX_IS_64BIT_HOST_MODE())
1369	{
1370	rc \|= VMXWriteVmcs(VMX_VMCS16_HOST_FIELD_FS, 0);
1371	rc \|= VMXWriteVmcs(VMX_VMCS16_HOST_FIELD_GS, 0);
1372	}
1373	#endif
1374	rc \|= VMXWriteVmcs(VMX_VMCS16_HOST_FIELD_SS, ss);
1375	SelTR = ASMGetTR();
1376	rc \|= VMXWriteVmcs(VMX_VMCS16_HOST_FIELD_TR, SelTR);
1377	AssertRC(rc);
1378	Log2(("VMX_VMCS_HOST_FIELD_CS %08x (%08x)\n", cs, ASMGetSS()));
1379	Log2(("VMX_VMCS_HOST_FIELD_DS 00000000 (%08x)\n", ASMGetDS()));
1380	Log2(("VMX_VMCS_HOST_FIELD_ES 00000000 (%08x)\n", ASMGetES()));
1381	Log2(("VMX_VMCS_HOST_FIELD_FS 00000000 (%08x)\n", ASMGetFS()));
1382	Log2(("VMX_VMCS_HOST_FIELD_GS 00000000 (%08x)\n", ASMGetGS()));
1383	Log2(("VMX_VMCS_HOST_FIELD_SS %08x (%08x)\n", ss, ASMGetSS()));
1384	Log2(("VMX_VMCS_HOST_FIELD_TR %08x\n", ASMGetTR()));
1385
1386	/*
1387	* GDTR & IDTR.
1388	*/
1389	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
1390	if (VMX_IS_64BIT_HOST_MODE())
1391	{
1392	X86XDTR64 gdtr64, idtr64;
1393	hmR0Get64bitGdtrAndIdtr(&gdtr64, &idtr64);
1394	rc = VMXWriteVmcs64(VMX_VMCS_HOST_GDTR_BASE, gdtr64.uAddr);
1395	rc \|= VMXWriteVmcs64(VMX_VMCS_HOST_IDTR_BASE, idtr64.uAddr);
1396	AssertRC(rc);
1397	Log2(("VMX_VMCS_HOST_GDTR_BASE %RX64\n", gdtr64.uAddr));
1398	Log2(("VMX_VMCS_HOST_IDTR_BASE %RX64\n", idtr64.uAddr));
1399	gdtr.cbGdt = gdtr64.cb;
1400	gdtr.pGdt = (uintptr_t)gdtr64.uAddr;
1401	}
1402	else
1403	#endif
1404	{
1405	ASMGetGDTR(&gdtr);
1406	rc = VMXWriteVmcs(VMX_VMCS_HOST_GDTR_BASE, gdtr.pGdt);
1407	ASMGetIDTR(&idtr);
1408	rc \|= VMXWriteVmcs(VMX_VMCS_HOST_IDTR_BASE, idtr.pIdt);
1409	AssertRC(rc);
1410	Log2(("VMX_VMCS_HOST_GDTR_BASE %RHv\n", gdtr.pGdt));
1411	Log2(("VMX_VMCS_HOST_IDTR_BASE %RHv\n", idtr.pIdt));
1412	}
1413
1414	/*
1415	* Save the base address of the TR selector.
1416	*/
1417	if (SelTR > gdtr.cbGdt)
1418	{
1419	AssertMsgFailed(("Invalid TR selector %x. GDTR.cbGdt=%x\n", SelTR, gdtr.cbGdt));
1420	return VERR_VMX_INVALID_HOST_STATE;
1421	}
1422
1423	pDesc = (PCX86DESCHC)(gdtr.pGdt + (SelTR & X86_SEL_MASK));
1424	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
1425	if (VMX_IS_64BIT_HOST_MODE())
1426	{
1427	uint64_t trBase64 = X86DESC64_BASE((PX86DESC64)pDesc);
1428	rc = VMXWriteVmcs64(VMX_VMCS_HOST_TR_BASE, trBase64);
1429	Log2(("VMX_VMCS_HOST_TR_BASE %RX64\n", trBase64));
1430	AssertRC(rc);
1431	}
1432	else
1433	#endif
1434	{
1435	#if HC_ARCH_BITS == 64
1436	trBase = X86DESC64_BASE(pDesc);
1437	#else
1438	trBase = X86DESC_BASE(pDesc);
1439	#endif
1440	rc = VMXWriteVmcs(VMX_VMCS_HOST_TR_BASE, trBase);
1441	AssertRC(rc);
1442	Log2(("VMX_VMCS_HOST_TR_BASE %RHv\n", trBase));
1443	}
1444
1445	/*
1446	* FS base and GS base.
1447	*/
1448	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
1449	if (VMX_IS_64BIT_HOST_MODE())
1450	{
1451	Log2(("MSR_K8_FS_BASE = %RX64\n", ASMRdMsr(MSR_K8_FS_BASE)));
1452	Log2(("MSR_K8_GS_BASE = %RX64\n", ASMRdMsr(MSR_K8_GS_BASE)));
1453	rc = VMXWriteVmcs64(VMX_VMCS_HOST_FS_BASE, ASMRdMsr(MSR_K8_FS_BASE));
1454	rc \|= VMXWriteVmcs64(VMX_VMCS_HOST_GS_BASE, ASMRdMsr(MSR_K8_GS_BASE));
1455	}
1456	#endif
1457	AssertRC(rc);
1458
1459	/*
1460	* Sysenter MSRs.
1461	*/
1462	/** @todo expensive!! */
1463	rc = VMXWriteVmcs(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
1464	Log2(("VMX_VMCS_HOST_SYSENTER_CS %08x\n", ASMRdMsr_Low(MSR_IA32_SYSENTER_CS)));
1465	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
1466	if (VMX_IS_64BIT_HOST_MODE())
1467	{
1468	Log2(("VMX_VMCS_HOST_SYSENTER_EIP %RX64\n", ASMRdMsr(MSR_IA32_SYSENTER_EIP)));
1469	Log2(("VMX_VMCS_HOST_SYSENTER_ESP %RX64\n", ASMRdMsr(MSR_IA32_SYSENTER_ESP)));
1470	rc \|= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
1471	rc \|= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
1472	}
1473	else
1474	{
1475	rc \|= VMXWriteVmcs(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
1476	rc \|= VMXWriteVmcs(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
1477	Log2(("VMX_VMCS_HOST_SYSENTER_EIP %RX32\n", ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP)));
1478	Log2(("VMX_VMCS_HOST_SYSENTER_ESP %RX32\n", ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP)));
1479	}
1480	#elif HC_ARCH_BITS == 32
1481	rc \|= VMXWriteVmcs(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
1482	rc \|= VMXWriteVmcs(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
1483	Log2(("VMX_VMCS_HOST_SYSENTER_EIP %RX32\n", ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP)));
1484	Log2(("VMX_VMCS_HOST_SYSENTER_ESP %RX32\n", ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP)));
1485	#else
1486	Log2(("VMX_VMCS_HOST_SYSENTER_EIP %RX64\n", ASMRdMsr(MSR_IA32_SYSENTER_EIP)));
1487	Log2(("VMX_VMCS_HOST_SYSENTER_ESP %RX64\n", ASMRdMsr(MSR_IA32_SYSENTER_ESP)));
1488	rc \|= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
1489	rc \|= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
1490	#endif
1491	AssertRC(rc);
1492
1493
1494	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
1495	/*
1496	* Store all host MSRs in the VM-Exit load area, so they will be reloaded after
1497	* the world switch back to the host.
1498	*/
1499	PVMXMSR pMsr = (PVMXMSR)pVCpu->hm.s.vmx.pvHostMsr;
1500	unsigned idxMsr = 0;
1501
1502	uint32_t u32HostExtFeatures = ASMCpuId_EDX(0x80000001);
1503	if (u32HostExtFeatures & (X86_CPUID_EXT_FEATURE_EDX_NX \| X86_CPUID_EXT_FEATURE_EDX_LONG_MODE))
1504	{
1505	pMsr->u32IndexMSR = MSR_K6_EFER;
1506	pMsr->u32Reserved = 0;
1507	# if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
1508	if (CPUMIsGuestInLongMode(pVCpu))
1509	{
1510	/* Must match the EFER value in our 64 bits switcher. */
1511	pMsr->u64Value = ASMRdMsr(MSR_K6_EFER) \| MSR_K6_EFER_LME \| MSR_K6_EFER_SCE \| MSR_K6_EFER_NXE;
1512	}
1513	else
1514	# endif
1515	pMsr->u64Value = ASMRdMsr(MSR_K6_EFER);
1516	pMsr++; idxMsr++;
1517	}
1518
1519	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
1520	if (VMX_IS_64BIT_HOST_MODE())
1521	{
1522	pMsr->u32IndexMSR = MSR_K6_STAR;
1523	pMsr->u32Reserved = 0;
1524	pMsr->u64Value = ASMRdMsr(MSR_K6_STAR); /* legacy syscall eip, cs & ss */
1525	pMsr++; idxMsr++;
1526	pMsr->u32IndexMSR = MSR_K8_LSTAR;
1527	pMsr->u32Reserved = 0;
1528	pMsr->u64Value = ASMRdMsr(MSR_K8_LSTAR); /* 64 bits mode syscall rip */
1529	pMsr++; idxMsr++;
1530	pMsr->u32IndexMSR = MSR_K8_SF_MASK;
1531	pMsr->u32Reserved = 0;
1532	pMsr->u64Value = ASMRdMsr(MSR_K8_SF_MASK); /* syscall flag mask */
1533	pMsr++; idxMsr++;
1534
1535	/* The KERNEL_GS_BASE MSR doesn't work reliably with auto load/store. See @bugref{6208} */
1536	#if 0
1537	pMsr->u32IndexMSR = MSR_K8_KERNEL_GS_BASE;
1538	pMsr->u32Reserved = 0;
1539	pMsr->u64Value = ASMRdMsr(MSR_K8_KERNEL_GS_BASE); /* swapgs exchange value */
1540	pMsr++; idxMsr++;
1541	#endif
1542	}
1543	# endif
1544
1545	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
1546	{
1547	pMsr->u32IndexMSR = MSR_K8_TSC_AUX;
1548	pMsr->u32Reserved = 0;
1549	pMsr->u64Value = ASMRdMsr(MSR_K8_TSC_AUX);
1550	pMsr++; idxMsr++;
1551	}
1552
1553	/** @todo r=ramshankar: check IA32_VMX_MISC bits 27:25 for valid idxMsr
1554	* range. */
1555	rc = VMXWriteVmcs(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, idxMsr);
1556	AssertRC(rc);
1557	#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
1558
1559	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_HOST_CONTEXT;
1560	}
1561	return rc;
1562	}
1563
1564
1565	/**
1566	* Loads the 4 PDPEs into the guest state when nested paging is used and the
1567	* guest operates in PAE mode.
1568	*
1569	* @returns VBox status code.
1570	* @param pVCpu Pointer to the VMCPU.
1571	* @param pCtx Pointer to the guest CPU context.
1572	*/
1573	static int hmR0VmxLoadPaePdpes(PVMCPU pVCpu, PCPUMCTX pCtx)
1574	{
1575	if (CPUMIsGuestInPAEModeEx(pCtx))
1576	{
1577	X86PDPE aPdpes[4];
1578	int rc = PGMGstGetPaePdpes(pVCpu, &aPdpes[0]);
1579	AssertRCReturn(rc, rc);
1580
1581	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, aPdpes[0].u); AssertRCReturn(rc, rc);
1582	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, aPdpes[1].u); AssertRCReturn(rc, rc);
1583	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, aPdpes[2].u); AssertRCReturn(rc, rc);
1584	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, aPdpes[3].u); AssertRCReturn(rc, rc);
1585	}
1586	return VINF_SUCCESS;
1587	}
1588
1589
1590	/**
1591	* Saves the 4 PDPEs into the guest state when nested paging is used and the
1592	* guest operates in PAE mode.
1593	*
1594	* @returns VBox status code.
1595	* @param pVCpu Pointer to the VM CPU.
1596	* @param pCtx Pointer to the guest CPU context.
1597	*
1598	* @remarks Tell PGM about CR3 changes before calling this helper.
1599	*/
1600	static int hmR0VmxSavePaePdpes(PVMCPU pVCpu, PCPUMCTX pCtx)
1601	{
1602	if (CPUMIsGuestInPAEModeEx(pCtx))
1603	{
1604	int rc;
1605	X86PDPE aPdpes[4];
1606	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &aPdpes[0].u); AssertRCReturn(rc, rc);
1607	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &aPdpes[1].u); AssertRCReturn(rc, rc);
1608	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &aPdpes[2].u); AssertRCReturn(rc, rc);
1609	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &aPdpes[3].u); AssertRCReturn(rc, rc);
1610
1611	rc = PGMGstUpdatePaePdpes(pVCpu, &aPdpes[0]);
1612	AssertRCReturn(rc, rc);
1613	}
1614	return VINF_SUCCESS;
1615	}
1616
1617
1618	/**
1619	* Update the exception bitmap according to the current CPU state.
1620	*
1621	* @param pVM Pointer to the VM.
1622	* @param pVCpu Pointer to the VMCPU.
1623	* @param pCtx Pointer to the guest CPU context.
1624	*/
1625	static void hmR0VmxUpdateExceptionBitmap(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
1626	{
1627	uint32_t u32TrapMask;
1628	Assert(pCtx);
1629
1630	/*
1631	* Set up a mask for intercepting traps.
1632	*/
1633	/** @todo Do we really need to always intercept #DB? */
1634	u32TrapMask = RT_BIT(X86_XCPT_DB)
1635	\| RT_BIT(X86_XCPT_NM)
1636	#ifdef VBOX_ALWAYS_TRAP_PF
1637	\| RT_BIT(X86_XCPT_PF)
1638	#endif
1639	#ifdef VBOX_STRICT
1640	\| RT_BIT(X86_XCPT_BP)
1641	\| RT_BIT(X86_XCPT_DB)
1642	\| RT_BIT(X86_XCPT_DE)
1643	\| RT_BIT(X86_XCPT_NM)
1644	\| RT_BIT(X86_XCPT_UD)
1645	\| RT_BIT(X86_XCPT_NP)
1646	\| RT_BIT(X86_XCPT_SS)
1647	\| RT_BIT(X86_XCPT_GP)
1648	\| RT_BIT(X86_XCPT_MF)
1649	#endif
1650	;
1651
1652	/*
1653	* Without nested paging, #PF must be intercepted to implement shadow paging.
1654	*/
1655	/** @todo NP state won't change so maybe we should build the initial trap mask up front? */
1656	if (!pVM->hm.s.fNestedPaging)
1657	u32TrapMask \|= RT_BIT(X86_XCPT_PF);
1658
1659	/* Catch floating point exceptions if we need to report them to the guest in a different way. */
1660	if (!(pCtx->cr0 & X86_CR0_NE))
1661	u32TrapMask \|= RT_BIT(X86_XCPT_MF);
1662
1663	#ifdef VBOX_STRICT
1664	Assert(u32TrapMask & RT_BIT(X86_XCPT_GP));
1665	#endif
1666
1667	/*
1668	* Intercept all exceptions in real mode as none of them can be injected directly (#GP otherwise).
1669	*/
1670	/** @todo Despite the claim to intercept everything, with NP we do not intercept #PF. Should we? */
1671	if ( CPUMIsGuestInRealModeEx(pCtx)
1672	&& pVM->hm.s.vmx.pRealModeTSS)
1673	{
1674	u32TrapMask \|= RT_BIT(X86_XCPT_DE)
1675	\| RT_BIT(X86_XCPT_DB)
1676	\| RT_BIT(X86_XCPT_NMI)
1677	\| RT_BIT(X86_XCPT_BP)
1678	\| RT_BIT(X86_XCPT_OF)
1679	\| RT_BIT(X86_XCPT_BR)
1680	\| RT_BIT(X86_XCPT_UD)
1681	\| RT_BIT(X86_XCPT_DF)
1682	\| RT_BIT(X86_XCPT_CO_SEG_OVERRUN)
1683	\| RT_BIT(X86_XCPT_TS)
1684	\| RT_BIT(X86_XCPT_NP)
1685	\| RT_BIT(X86_XCPT_SS)
1686	\| RT_BIT(X86_XCPT_GP)
1687	\| RT_BIT(X86_XCPT_MF)
1688	\| RT_BIT(X86_XCPT_AC)
1689	\| RT_BIT(X86_XCPT_MC)
1690	\| RT_BIT(X86_XCPT_XF)
1691	;
1692	}
1693
1694	int rc = VMXWriteVmcs(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32TrapMask);
1695	AssertRC(rc);
1696	}
1697
1698
1699	/**
1700	* Loads a minimal guest state.
1701	*
1702	* NOTE: Don't do anything here that can cause a jump back to ring 3!!!!!
1703	*
1704	* @param pVM Pointer to the VM.
1705	* @param pVCpu Pointer to the VMCPU.
1706	* @param pCtx Pointer to the guest CPU context.
1707	*/
1708	VMMR0DECL(void) VMXR0LoadMinimalGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
1709	{
1710	int rc;
1711	X86EFLAGS eflags;
1712
1713	Assert(!(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_ALL_GUEST));
1714
1715	/*
1716	* Load EIP, ESP and EFLAGS.
1717	*/
1718	rc = VMXWriteVmcs64(VMX_VMCS_GUEST_RIP, pCtx->rip);
1719	rc \|= VMXWriteVmcs64(VMX_VMCS_GUEST_RSP, pCtx->rsp);
1720	AssertRC(rc);
1721
1722	/*
1723	* Bits 22-31, 15, 5 & 3 must be zero. Bit 1 must be 1.
1724	*/
1725	eflags = pCtx->eflags;
1726	eflags.u32 &= VMX_EFLAGS_RESERVED_0;
1727	eflags.u32 \|= VMX_EFLAGS_RESERVED_1;
1728
1729	/*
1730	* Check if real mode emulation using v86 mode.
1731	*/
1732	if ( CPUMIsGuestInRealModeEx(pCtx)
1733	&& pVM->hm.s.vmx.pRealModeTSS)
1734	{
1735	pVCpu->hm.s.vmx.RealMode.eflags = eflags;
1736
1737	eflags.Bits.u1VM = 1;
1738	eflags.Bits.u2IOPL = 0; /* must always be 0 or else certain instructions won't cause faults. */
1739	}
1740	rc = VMXWriteVmcs(VMX_VMCS_GUEST_RFLAGS, eflags.u32);
1741	AssertRC(rc);
1742	}
1743
1744
1745	/**
1746	* Sets up TSC offsetting and VMX preemption, if supported/configured/available.
1747	*
1748	* NOTE: This function reads the host TSC value. Therefore it must be executed very
1749	* shortly before a VM entry and execution MUST NOT be rescheduled between a call to
1750	* this function and a VM entry without calling this function again.
1751	*
1752	* @returns VBox status code.
1753	* @param pVM Pointer to the VM.
1754	* @param pVCpu Pointer to the VMCPU.
1755	*/
1756	VMMR0DECL(int) VMXR0SetupTscOffsetAndPreemption(PVM pVM, PVMCPU pVCpu)
1757	{
1758	int rc = VINF_SUCCESS;
1759	bool fOffsettedTsc;
1760
1761	if (pVM->hm.s.vmx.fUsePreemptTimer)
1762	{
1763	uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVCpu, &fOffsettedTsc, &pVCpu->hm.s.vmx.u64TSCOffset);
1764
1765	/* Make sure the returned values have sane upper and lower boundaries. */
1766	uint64_t u64CpuHz = SUPGetCpuHzFromGIP(g_pSUPGlobalInfoPage);
1767
1768	cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64 of a second */
1769	cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
1770
1771	cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
1772	uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
1773	rc = VMXWriteVmcs(VMX_VMCS32_GUEST_PREEMPTION_TIMER_VALUE, cPreemptionTickCount);
1774	AssertRC(rc);
1775	}
1776	else
1777	fOffsettedTsc = TMCpuTickCanUseRealTSC(pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset);
1778
1779	if (fOffsettedTsc)
1780	{
1781	uint64_t u64CurTSC = ASMReadTSC();
1782	if (u64CurTSC + pVCpu->hm.s.vmx.u64TSCOffset > TMCpuTickGetLastSeen(pVCpu))
1783	{
1784	/* Note: VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT takes precedence over TSC_OFFSET, applies to RDTSCP too. */
1785	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, pVCpu->hm.s.vmx.u64TSCOffset);
1786	AssertRC(rc);
1787
1788	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT;
1789	rc = VMXWriteVmcs(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls);
1790	AssertRC(rc);
1791	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
1792	}
1793	else
1794	{
1795	/* Fall back to rdtsc, rdtscp emulation as we would otherwise pass decreasing tsc values to the guest. */
1796	LogFlow(("TSC %RX64 offset %RX64 time=%RX64 last=%RX64 (diff=%RX64, virt_tsc=%RX64)\n", u64CurTSC,
1797	pVCpu->hm.s.vmx.u64TSCOffset, u64CurTSC + pVCpu->hm.s.vmx.u64TSCOffset,
1798	TMCpuTickGetLastSeen(pVCpu), TMCpuTickGetLastSeen(pVCpu) - u64CurTSC - pVCpu->hm.s.vmx.u64TSCOffset,
1799	TMCpuTickGet(pVCpu)));
1800	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT;
1801	rc = VMXWriteVmcs(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls);
1802	AssertRC(rc);
1803	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscInterceptOverFlow);
1804	}
1805	}
1806	else
1807	{
1808	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT;
1809	rc = VMXWriteVmcs(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls);
1810	AssertRC(rc);
1811	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
1812	}
1813	return rc;
1814	}
1815
1816	/**
1817	* Loads the guest state.
1818	*
1819	* NOTE: Don't do anything here that can cause a jump back to ring 3!!!!!
1820	*
1821	* @returns VBox status code.
1822	* @param pVM Pointer to the VM.
1823	* @param pVCpu Pointer to the VMCPU.
1824	* @param pCtx Pointer to the guest CPU context.
1825	*/
1826	VMMR0DECL(int) VMXR0LoadGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
1827	{
1828	int rc = VINF_SUCCESS;
1829	RTGCUINTPTR val;
1830
1831	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestState, x);
1832
1833	/*
1834	* VMX_VMCS_CTRL_ENTRY_CONTROLS
1835	* Set required bits to one and zero according to the MSR capabilities.
1836	*/
1837	val = pVM->hm.s.vmx.msr.vmx_entry.n.disallowed0;
1838
1839	/*
1840	* Load guest debug controls (DR7 & IA32_DEBUGCTL_MSR).
1841	* Forced to 1 on the 'first' VT-x capable CPUs; this actually includes the newest Nehalem CPUs
1842	*/
1843	val \|= VMX_VMCS_CTRL_ENTRY_CONTROLS_LOAD_DEBUG;
1844
1845	if (CPUMIsGuestInLongModeEx(pCtx))
1846	val \|= VMX_VMCS_CTRL_ENTRY_CONTROLS_IA32E_MODE_GUEST;
1847	/* else Must be zero when AMD64 is not available. */
1848
1849	/*
1850	* Mask away the bits that the CPU doesn't support.
1851	*/
1852	val &= pVM->hm.s.vmx.msr.vmx_entry.n.allowed1;
1853	rc = VMXWriteVmcs(VMX_VMCS32_CTRL_ENTRY_CONTROLS, val);
1854	AssertRC(rc);
1855
1856	/*
1857	* VMX_VMCS_CTRL_EXIT_CONTROLS
1858	* Set required bits to one and zero according to the MSR capabilities.
1859	*/
1860	val = pVM->hm.s.vmx.msr.vmx_exit.n.disallowed0;
1861
1862	/*
1863	* Save debug controls (DR7 & IA32_DEBUGCTL_MSR)
1864	* Forced to 1 on the 'first' VT-x capable CPUs; this actually includes the newest Nehalem CPUs
1865	*/
1866	val \|= VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_DEBUG;
1867
1868	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
1869	if (VMX_IS_64BIT_HOST_MODE())
1870	val \|= VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_ADDR_SPACE_SIZE;
1871	/* else Must be zero when AMD64 is not available. */
1872	#elif HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
1873	if (CPUMIsGuestInLongModeEx(pCtx))
1874	val \|= VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_ADDR_SPACE_SIZE; /* our switcher goes to long mode */
1875	else
1876	Assert(!(val & VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_ADDR_SPACE_SIZE));
1877	#endif
1878	val &= pVM->hm.s.vmx.msr.vmx_exit.n.allowed1;
1879
1880	/*
1881	* Don't acknowledge external interrupts on VM-exit.
1882	*/
1883	rc = VMXWriteVmcs(VMX_VMCS32_CTRL_EXIT_CONTROLS, val);
1884	AssertRC(rc);
1885
1886	/*
1887	* Guest CPU context: ES, CS, SS, DS, FS, GS.
1888	*/
1889	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SEGMENT_REGS)
1890	{
1891	if (pVM->hm.s.vmx.pRealModeTSS)
1892	{
1893	PGMMODE enmGuestMode = PGMGetGuestMode(pVCpu);
1894	if (pVCpu->hm.s.vmx.enmLastSeenGuestMode != enmGuestMode)
1895	{
1896	/*
1897	* Correct weird requirements for switching to protected mode.
1898	*/
1899	if ( pVCpu->hm.s.vmx.enmLastSeenGuestMode == PGMMODE_REAL
1900	&& enmGuestMode >= PGMMODE_PROTECTED)
1901	{
1902	#ifdef VBOX_WITH_REM
1903	/*
1904	* Flush the recompiler code cache as it's not unlikely the guest will rewrite code
1905	* it will later execute in real mode (OpenBSD 4.0 is one such example)
1906	*/
1907	REMFlushTBs(pVM);
1908	#endif
1909
1910	/*
1911	* DPL of all hidden selector registers must match the current CPL (0).
1912	*/
1913	pCtx->cs.Attr.n.u2Dpl = 0;
1914	pCtx->cs.Attr.n.u4Type = X86_SEL_TYPE_CODE \| X86_SEL_TYPE_RW_ACC;
1915
1916	pCtx->ds.Attr.n.u2Dpl = 0;
1917	pCtx->es.Attr.n.u2Dpl = 0;
1918	pCtx->fs.Attr.n.u2Dpl = 0;
1919	pCtx->gs.Attr.n.u2Dpl = 0;
1920	pCtx->ss.Attr.n.u2Dpl = 0;
1921	}
1922	pVCpu->hm.s.vmx.enmLastSeenGuestMode = enmGuestMode;
1923	}
1924	}
1925
1926	VMX_WRITE_SELREG(ES, es);
1927	AssertRC(rc);
1928
1929	VMX_WRITE_SELREG(CS, cs);
1930	AssertRC(rc);
1931
1932	VMX_WRITE_SELREG(SS, ss);
1933	AssertRC(rc);
1934
1935	VMX_WRITE_SELREG(DS, ds);
1936	AssertRC(rc);
1937
1938	VMX_WRITE_SELREG(FS, fs);
1939	AssertRC(rc);
1940
1941	VMX_WRITE_SELREG(GS, gs);
1942	AssertRC(rc);
1943	}
1944
1945	/*
1946	* Guest CPU context: LDTR.
1947	*/
1948	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_LDTR)
1949	{
1950	if (pCtx->ldtr.Sel == 0)
1951	{
1952	rc = VMXWriteVmcs(VMX_VMCS16_GUEST_FIELD_LDTR, 0);
1953	rc \|= VMXWriteVmcs(VMX_VMCS32_GUEST_LDTR_LIMIT, 0);
1954	rc \|= VMXWriteVmcs64(VMX_VMCS_GUEST_LDTR_BASE, 0); /* @todo removing "64" in the function should be the same. */
1955	/* Note: vmlaunch will fail with 0 or just 0x02. No idea why. */
1956	rc \|= VMXWriteVmcs(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, 0x82 /* present, LDT */);
1957	}
1958	else
1959	{
1960	rc = VMXWriteVmcs(VMX_VMCS16_GUEST_FIELD_LDTR, pCtx->ldtr.Sel);
1961	rc \|= VMXWriteVmcs(VMX_VMCS32_GUEST_LDTR_LIMIT, pCtx->ldtr.u32Limit);
1962	rc \|= VMXWriteVmcs64(VMX_VMCS_GUEST_LDTR_BASE, pCtx->ldtr.u64Base); /* @todo removing "64" and it should be the same */
1963	rc \|= VMXWriteVmcs(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, pCtx->ldtr.Attr.u);
1964	}
1965	AssertRC(rc);
1966	}
1967
1968	/*
1969	* Guest CPU context: TR.
1970	*/
1971	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_TR)
1972	{
1973	/*
1974	* Real mode emulation using v86 mode with CR4.VME (interrupt redirection
1975	* using the int bitmap in the TSS).
1976	*/
1977	if ( CPUMIsGuestInRealModeEx(pCtx)
1978	&& pVM->hm.s.vmx.pRealModeTSS)
1979	{
1980	RTGCPHYS GCPhys;
1981
1982	/* We convert it here every time as PCI regions could be reconfigured. */
1983	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
1984	AssertRC(rc);
1985
1986	rc = VMXWriteVmcs(VMX_VMCS16_GUEST_FIELD_TR, 0);
1987	rc \|= VMXWriteVmcs(VMX_VMCS32_GUEST_TR_LIMIT, HM_VTX_TSS_SIZE);
1988	rc \|= VMXWriteVmcs64(VMX_VMCS_GUEST_TR_BASE, GCPhys /* phys = virt in this mode */);
1989
1990	X86DESCATTR attr;
1991
1992	attr.u = 0;
1993	attr.n.u1Present = 1;
1994	attr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
1995	val = attr.u;
1996	}
1997	else
1998	{
1999	rc = VMXWriteVmcs(VMX_VMCS16_GUEST_FIELD_TR, pCtx->tr.Sel);
2000	rc \|= VMXWriteVmcs(VMX_VMCS32_GUEST_TR_LIMIT, pCtx->tr.u32Limit);
2001	rc \|= VMXWriteVmcs64(VMX_VMCS_GUEST_TR_BASE, pCtx->tr.u64Base);
2002
2003	val = pCtx->tr.Attr.u;
2004
2005	/* The TSS selector must be busy (REM bugs? see defect #XXXX). */
2006	if (!(val & X86_SEL_TYPE_SYS_TSS_BUSY_MASK))
2007	{
2008	if (val & 0xf)
2009	val \|= X86_SEL_TYPE_SYS_TSS_BUSY_MASK;
2010	else
2011	/* Default if no TR selector has been set (otherwise vmlaunch will fail!) */
2012	val = (val & ~0xF) \| X86_SEL_TYPE_SYS_386_TSS_BUSY;
2013	}
2014	AssertMsg((val & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY \|\| (val & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY,
2015	("%#x\n", val));
2016	}
2017	rc \|= VMXWriteVmcs(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, val);
2018	AssertRC(rc);
2019	}
2020
2021	/*
2022	* Guest CPU context: GDTR.
2023	*/
2024	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_GDTR)
2025	{
2026	rc = VMXWriteVmcs(VMX_VMCS32_GUEST_GDTR_LIMIT, pCtx->gdtr.cbGdt);
2027	rc \|= VMXWriteVmcs64(VMX_VMCS_GUEST_GDTR_BASE, pCtx->gdtr.pGdt);
2028	AssertRC(rc);
2029	}
2030
2031	/*
2032	* Guest CPU context: IDTR.
2033	*/
2034	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_IDTR)
2035	{
2036	rc = VMXWriteVmcs(VMX_VMCS32_GUEST_IDTR_LIMIT, pCtx->idtr.cbIdt);
2037	rc \|= VMXWriteVmcs64(VMX_VMCS_GUEST_IDTR_BASE, pCtx->idtr.pIdt);
2038	AssertRC(rc);
2039	}
2040
2041	/*
2042	* Sysenter MSRs.
2043	*/
2044	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_MSR)
2045	{
2046	rc = VMXWriteVmcs(VMX_VMCS32_GUEST_SYSENTER_CS, pCtx->SysEnter.cs);
2047	rc \|= VMXWriteVmcs64(VMX_VMCS_GUEST_SYSENTER_EIP, pCtx->SysEnter.eip);
2048	rc \|= VMXWriteVmcs64(VMX_VMCS_GUEST_SYSENTER_ESP, pCtx->SysEnter.esp);
2049	AssertRC(rc);
2050	}
2051
2052	/*
2053	* Guest CPU context: Control registers.
2054	*/
2055	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR0)
2056	{
2057	val = pCtx->cr0;
2058	rc = VMXWriteVmcs(VMX_VMCS_CTRL_CR0_READ_SHADOW, val);
2059	Log2(("Guest CR0-shadow %08x\n", val));
2060	if (CPUMIsGuestFPUStateActive(pVCpu) == false)
2061	{
2062	/* Always use #NM exceptions to load the FPU/XMM state on demand. */
2063	val \|= X86_CR0_TS \| X86_CR0_ET \| X86_CR0_NE \| X86_CR0_MP;
2064	}
2065	else
2066	{
2067	/** @todo check if we support the old style mess correctly. */
2068	if (!(val & X86_CR0_NE))
2069	Log(("Forcing X86_CR0_NE!!!\n"));
2070
2071	val \|= X86_CR0_NE; /* always turn on the native mechanism to report FPU errors (old style uses interrupts) */
2072	}
2073	/* Protected mode & paging are always enabled; we use them for emulating real and protected mode without paging too. */
2074	if (!pVM->hm.s.vmx.fUnrestrictedGuest)
2075	val \|= X86_CR0_PE \| X86_CR0_PG;
2076
2077	if (pVM->hm.s.fNestedPaging)
2078	{
2079	if (CPUMIsGuestInPagedProtectedModeEx(pCtx))
2080	{
2081	/* Disable CR3 read/write monitoring as we don't need it for EPT. */
2082	pVCpu->hm.s.vmx.u32ProcCtls &= ~( VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
2083	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT);
2084	}
2085	else
2086	{
2087	/* Reenable CR3 read/write monitoring as our identity mapped page table is active. */
2088	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
2089	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT;
2090	}
2091	rc = VMXWriteVmcs(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls);
2092	AssertRC(rc);
2093	}
2094	else
2095	{
2096	/* Note: We must also set this as we rely on protecting various pages for which supervisor writes must be caught. */
2097	val \|= X86_CR0_WP;
2098	}
2099
2100	/* Always enable caching. */
2101	val &= ~(X86_CR0_CD\|X86_CR0_NW);
2102
2103	rc \|= VMXWriteVmcs64(VMX_VMCS_GUEST_CR0, val);
2104	Log2(("Guest CR0 %08x\n", val));
2105
2106	/*
2107	* CR0 flags owned by the host; if the guests attempts to change them, then the VM will exit.
2108	*/
2109	val = X86_CR0_PE /* Must monitor this bit (assumptions are made for real mode emulation) */
2110	\| X86_CR0_WP /* Must monitor this bit (it must always be enabled). */
2111	\| X86_CR0_PG /* Must monitor this bit (assumptions are made for real mode & protected mode without paging emulation) */
2112	\| X86_CR0_CD /* Bit not restored during VM-exit! */
2113	\| X86_CR0_NW /* Bit not restored during VM-exit! */
2114	\| X86_CR0_NE;
2115
2116	/*
2117	* When the guest's FPU state is active, then we no longer care about the FPU related bits.
2118	*/
2119	if (CPUMIsGuestFPUStateActive(pVCpu) == false)
2120	val \|= X86_CR0_TS \| X86_CR0_ET \| X86_CR0_MP;
2121
2122	pVCpu->hm.s.vmx.cr0_mask = val;
2123
2124	rc \|= VMXWriteVmcs(VMX_VMCS_CTRL_CR0_MASK, val);
2125	Log2(("Guest CR0-mask %08x\n", val));
2126	AssertRC(rc);
2127	}
2128
2129	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR4)
2130	{
2131	rc = VMXWriteVmcs(VMX_VMCS_CTRL_CR4_READ_SHADOW, pCtx->cr4);
2132	Log2(("Guest CR4-shadow %08x\n", pCtx->cr4));
2133	/* Set the required bits in cr4 too (currently X86_CR4_VMXE). */
2134	val = pCtx->cr4 \| (uint32_t)pVM->hm.s.vmx.msr.vmx_cr4_fixed0;
2135
2136	if (!pVM->hm.s.fNestedPaging)
2137	{
2138	switch (pVCpu->hm.s.enmShadowMode)
2139	{
2140	case PGMMODE_REAL: /* Real mode -> emulated using v86 mode */
2141	case PGMMODE_PROTECTED: /* Protected mode, no paging -> emulated using identity mapping. */
2142	case PGMMODE_32_BIT: /* 32-bit paging. */
2143	val &= ~X86_CR4_PAE;
2144	break;
2145
2146	case PGMMODE_PAE: /* PAE paging. */
2147	case PGMMODE_PAE_NX: /* PAE paging with NX enabled. */
2148	/** Must use PAE paging as we could use physical memory > 4 GB */
2149	val \|= X86_CR4_PAE;
2150	break;
2151
2152	case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
2153	case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
2154	#ifdef VBOX_ENABLE_64_BITS_GUESTS
2155	break;
2156	#else
2157	AssertFailed();
2158	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
2159	#endif
2160	default: /* shut up gcc */
2161	AssertFailed();
2162	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
2163	}
2164	}
2165	else if ( !CPUMIsGuestInPagedProtectedModeEx(pCtx)
2166	&& !pVM->hm.s.vmx.fUnrestrictedGuest)
2167	{
2168	/* We use 4 MB pages in our identity mapping page table for real and protected mode without paging. */
2169	val \|= X86_CR4_PSE;
2170	/* Our identity mapping is a 32 bits page directory. */
2171	val &= ~X86_CR4_PAE;
2172	}
2173
2174	/*
2175	* Turn off VME if we're in emulated real mode.
2176	*/
2177	if ( CPUMIsGuestInRealModeEx(pCtx)
2178	&& pVM->hm.s.vmx.pRealModeTSS)
2179	{
2180	val &= ~X86_CR4_VME;
2181	}
2182
2183	rc \|= VMXWriteVmcs64(VMX_VMCS_GUEST_CR4, val);
2184	Log2(("Guest CR4 %08x\n", val));
2185
2186	/*
2187	* CR4 flags owned by the host; if the guests attempts to change them, then the VM will exit.
2188	*/
2189	val = 0
2190	\| X86_CR4_VME
2191	\| X86_CR4_PAE
2192	\| X86_CR4_PGE
2193	\| X86_CR4_PSE
2194	\| X86_CR4_VMXE;
2195	pVCpu->hm.s.vmx.cr4_mask = val;
2196
2197	rc \|= VMXWriteVmcs(VMX_VMCS_CTRL_CR4_MASK, val);
2198	Log2(("Guest CR4-mask %08x\n", val));
2199	AssertRC(rc);
2200	}
2201
2202	#if 0
2203	/* Enable single stepping if requested and CPU supports it. */
2204	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_TRAP_FLAG)
2205	if (DBGFIsStepping(pVCpu))
2206	{
2207	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_TRAP_FLAG;
2208	rc = VMXWriteVmcs(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls);
2209	AssertRC(rc);
2210	}
2211	#endif
2212
2213	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR3)
2214	{
2215	if (pVM->hm.s.fNestedPaging)
2216	{
2217	Assert(PGMGetHyperCR3(pVCpu));
2218	pVCpu->hm.s.vmx.GCPhysEPTP = PGMGetHyperCR3(pVCpu);
2219
2220	Assert(!(pVCpu->hm.s.vmx.GCPhysEPTP & 0xfff));
2221	/** @todo Check the IA32_VMX_EPT_VPID_CAP MSR for other supported memory types. */
2222	pVCpu->hm.s.vmx.GCPhysEPTP \|= VMX_EPT_MEMTYPE_WB
2223	\| (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
2224
2225	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVCpu->hm.s.vmx.GCPhysEPTP);
2226	AssertRC(rc);
2227
2228	if ( !CPUMIsGuestInPagedProtectedModeEx(pCtx)
2229	&& !pVM->hm.s.vmx.fUnrestrictedGuest)
2230	{
2231	RTGCPHYS GCPhys;
2232
2233	/* We convert it here every time as PCI regions could be reconfigured. */
2234	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
2235	AssertMsgRC(rc, ("pNonPagingModeEPTPageTable = %RGv\n", pVM->hm.s.vmx.pNonPagingModeEPTPageTable));
2236
2237	/*
2238	* We use our identity mapping page table here as we need to map guest virtual to
2239	* guest physical addresses; EPT will take care of the translation to host physical addresses.
2240	*/
2241	val = GCPhys;
2242	}
2243	else
2244	{
2245	/* Save the real guest CR3 in VMX_VMCS_GUEST_CR3 */
2246	val = pCtx->cr3;
2247	rc = hmR0VmxLoadPaePdpes(pVCpu, pCtx);
2248	AssertRCReturn(rc, rc);
2249	}
2250	}
2251	else
2252	{
2253	val = PGMGetHyperCR3(pVCpu);
2254	Assert(val \|\| VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL));
2255	}
2256
2257	/* Save our shadow CR3 register. */
2258	rc = VMXWriteVmcs64(VMX_VMCS_GUEST_CR3, val);
2259	AssertRC(rc);
2260	}
2261
2262	/*
2263	* Guest CPU context: Debug registers.
2264	*/
2265	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_DEBUG)
2266	{
2267	pCtx->dr[6] \|= X86_DR6_INIT_VAL; /* set all reserved bits to 1. */
2268	pCtx->dr[6] &= ~RT_BIT(12); /* must be zero. */
2269
2270	pCtx->dr[7] &= 0xffffffff; /* upper 32 bits reserved */
2271	pCtx->dr[7] &= ~(RT_BIT(11) \| RT_BIT(12) \| RT_BIT(14) \| RT_BIT(15)); /* must be zero */
2272	pCtx->dr[7] \|= 0x400; /* must be one */
2273
2274	/* Resync DR7 */
2275	rc = VMXWriteVmcs64(VMX_VMCS_GUEST_DR7, pCtx->dr[7]);
2276	AssertRC(rc);
2277
2278	#ifdef DEBUG
2279	/* Sync the hypervisor debug state now if any breakpoint is armed. */
2280	if ( CPUMGetHyperDR7(pVCpu) & (X86_DR7_ENABLED_MASK\|X86_DR7_GD)
2281	&& !CPUMIsHyperDebugStateActive(pVCpu)
2282	&& !DBGFIsStepping(pVCpu))
2283	{
2284	/* Save the host and load the hypervisor debug state. */
2285	rc = CPUMR0LoadHyperDebugState(pVM, pVCpu, pCtx, true /* include DR6 */);
2286	AssertRC(rc);
2287
2288	/* DRx intercepts remain enabled. */
2289
2290	/* Override dr7 with the hypervisor value. */
2291	rc = VMXWriteVmcs64(VMX_VMCS_GUEST_DR7, CPUMGetHyperDR7(pVCpu));
2292	AssertRC(rc);
2293	}
2294	else
2295	#endif
2296	/* Sync the debug state now if any breakpoint is armed. */
2297	if ( (pCtx->dr[7] & (X86_DR7_ENABLED_MASK\|X86_DR7_GD))
2298	&& !CPUMIsGuestDebugStateActive(pVCpu)
2299	&& !DBGFIsStepping(pVCpu))
2300	{
2301	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
2302
2303	/* Disable DRx move intercepts. */
2304	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT;
2305	rc = VMXWriteVmcs(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls);
2306	AssertRC(rc);
2307
2308	/* Save the host and load the guest debug state. */
2309	rc = CPUMR0LoadGuestDebugState(pVM, pVCpu, pCtx, true /* include DR6 */);
2310	AssertRC(rc);
2311	}
2312
2313	/* IA32_DEBUGCTL MSR. */
2314	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, 0);
2315	AssertRC(rc);
2316
2317	/** @todo do we really ever need this? */
2318	rc \|= VMXWriteVmcs(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
2319	AssertRC(rc);
2320	}
2321
2322	/*
2323	* 64-bit guest mode.
2324	*/
2325	if (CPUMIsGuestInLongModeEx(pCtx))
2326	{
2327	#if !defined(VBOX_ENABLE_64_BITS_GUESTS)
2328	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
2329	#elif HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2330	pVCpu->hm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
2331	#else
2332	# ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2333	if (!pVM->hm.s.fAllow64BitGuests)
2334	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
2335	# endif
2336	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM64;
2337	#endif
2338	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_MSR)
2339	{
2340	/* Update these as wrmsr might have changed them. */
2341	rc = VMXWriteVmcs64(VMX_VMCS_GUEST_FS_BASE, pCtx->fs.u64Base);
2342	AssertRC(rc);
2343	rc = VMXWriteVmcs64(VMX_VMCS_GUEST_GS_BASE, pCtx->gs.u64Base);
2344	AssertRC(rc);
2345	}
2346	}
2347	else
2348	{
2349	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
2350	}
2351
2352	hmR0VmxUpdateExceptionBitmap(pVM, pVCpu, pCtx);
2353
2354	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
2355	/*
2356	* Store all guest MSRs in the VM-entry load area, so they will be loaded
2357	* during VM-entry and restored into the VM-exit store area during VM-exit.
2358	*/
2359	PVMXMSR pMsr = (PVMXMSR)pVCpu->hm.s.vmx.pvGuestMsr;
2360	unsigned idxMsr = 0;
2361
2362	uint32_t u32GstExtFeatures;
2363	uint32_t u32Temp;
2364	CPUMGetGuestCpuId(pVCpu, 0x80000001, &u32Temp, &u32Temp, &u32Temp, &u32GstExtFeatures);
2365
2366	if (u32GstExtFeatures & (X86_CPUID_EXT_FEATURE_EDX_NX \| X86_CPUID_EXT_FEATURE_EDX_LONG_MODE))
2367	{
2368	pMsr->u32IndexMSR = MSR_K6_EFER;
2369	pMsr->u32Reserved = 0;
2370	pMsr->u64Value = pCtx->msrEFER;
2371	/* VT-x will complain if only MSR_K6_EFER_LME is set. */
2372	if (!CPUMIsGuestInLongModeEx(pCtx))
2373	pMsr->u64Value &= ~(MSR_K6_EFER_LMA \| MSR_K6_EFER_LME);
2374	pMsr++; idxMsr++;
2375
2376	if (u32GstExtFeatures & X86_CPUID_EXT_FEATURE_EDX_LONG_MODE)
2377	{
2378	pMsr->u32IndexMSR = MSR_K8_LSTAR;
2379	pMsr->u32Reserved = 0;
2380	pMsr->u64Value = pCtx->msrLSTAR; /* 64 bits mode syscall rip */
2381	pMsr++; idxMsr++;
2382	pMsr->u32IndexMSR = MSR_K6_STAR;
2383	pMsr->u32Reserved = 0;
2384	pMsr->u64Value = pCtx->msrSTAR; /* legacy syscall eip, cs & ss */
2385	pMsr++; idxMsr++;
2386	pMsr->u32IndexMSR = MSR_K8_SF_MASK;
2387	pMsr->u32Reserved = 0;
2388	pMsr->u64Value = pCtx->msrSFMASK; /* syscall flag mask */
2389	pMsr++; idxMsr++;
2390
2391	/* The KERNEL_GS_BASE MSR doesn't work reliably with auto load/store. See @bugref{6208} */
2392	#if 0
2393	pMsr->u32IndexMSR = MSR_K8_KERNEL_GS_BASE;
2394	pMsr->u32Reserved = 0;
2395	pMsr->u64Value = pCtx->msrKERNELGSBASE; /* swapgs exchange value */
2396	pMsr++; idxMsr++;
2397	#endif
2398	}
2399	}
2400
2401	if ( pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP
2402	&& (u32GstExtFeatures & X86_CPUID_EXT_FEATURE_EDX_RDTSCP))
2403	{
2404	pMsr->u32IndexMSR = MSR_K8_TSC_AUX;
2405	pMsr->u32Reserved = 0;
2406	rc = CPUMQueryGuestMsr(pVCpu, MSR_K8_TSC_AUX, &pMsr->u64Value);
2407	AssertRC(rc);
2408	pMsr++; idxMsr++;
2409	}
2410
2411	pVCpu->hm.s.vmx.cGuestMsrs = idxMsr;
2412
2413	rc = VMXWriteVmcs(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, idxMsr);
2414	AssertRC(rc);
2415
2416	rc = VMXWriteVmcs(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, idxMsr);
2417	AssertRC(rc);
2418	#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
2419
2420	/* Done with the major changes */
2421	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_ALL_GUEST;
2422
2423	/* Minimal guest state update (ESP, EIP, EFLAGS mostly) */
2424	VMXR0LoadMinimalGuestState(pVM, pVCpu, pCtx);
2425
2426	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestState, x);
2427	return rc;
2428	}
2429
2430
2431	/**
2432	* Syncs back the guest state from VMCS.
2433	*
2434	* @returns VBox status code.
2435	* @param pVM Pointer to the VM.
2436	* @param pVCpu Pointer to the VMCPU.
2437	* @param pCtx Pointer to the guest CPU context.
2438	*/
2439	DECLINLINE(int) VMXR0SaveGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
2440	{
2441	RTGCUINTREG val, valShadow;
2442	RTGCUINTPTR uInterruptState;
2443	int rc;
2444
2445	/* First sync back EIP, ESP, and EFLAGS. */
2446	rc = VMXReadCachedVmcs(VMX_VMCS_GUEST_RIP, &val);
2447	AssertRC(rc);
2448	pCtx->rip = val;
2449	rc = VMXReadCachedVmcs(VMX_VMCS_GUEST_RSP, &val);
2450	AssertRC(rc);
2451	pCtx->rsp = val;
2452	rc = VMXReadCachedVmcs(VMX_VMCS_GUEST_RFLAGS, &val);
2453	AssertRC(rc);
2454	pCtx->eflags.u32 = val;
2455
2456	/* Take care of instruction fusing (sti, mov ss) */
2457	rc \|= VMXReadCachedVmcs(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &val);
2458	uInterruptState = val;
2459	if (uInterruptState != 0)
2460	{
2461	Assert(uInterruptState <= 2); /* only sti & mov ss */
2462	Log(("uInterruptState %x eip=%RGv\n", (uint32_t)uInterruptState, pCtx->rip));
2463	EMSetInhibitInterruptsPC(pVCpu, pCtx->rip);
2464	}
2465	else
2466	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
2467
2468	/* Control registers. */
2469	VMXReadCachedVmcs(VMX_VMCS_CTRL_CR0_READ_SHADOW, &valShadow);
2470	VMXReadCachedVmcs(VMX_VMCS_GUEST_CR0, &val);
2471	val = (valShadow & pVCpu->hm.s.vmx.cr0_mask) \| (val & ~pVCpu->hm.s.vmx.cr0_mask);
2472	CPUMSetGuestCR0(pVCpu, val);
2473
2474	VMXReadCachedVmcs(VMX_VMCS_CTRL_CR4_READ_SHADOW, &valShadow);
2475	VMXReadCachedVmcs(VMX_VMCS_GUEST_CR4, &val);
2476	val = (valShadow & pVCpu->hm.s.vmx.cr4_mask) \| (val & ~pVCpu->hm.s.vmx.cr4_mask);
2477	CPUMSetGuestCR4(pVCpu, val);
2478
2479	/*
2480	* No reason to sync back the CRx registers. They can't be changed by the guest unless in
2481	* the nested paging case where CR3 & CR4 can be changed by the guest.
2482	*/
2483	if ( pVM->hm.s.fNestedPaging
2484	&& CPUMIsGuestInPagedProtectedModeEx(pCtx)) /** @todo check if we will always catch mode switches and such... */
2485	{
2486	PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
2487
2488	/* Can be updated behind our back in the nested paging case. */
2489	CPUMSetGuestCR2(pVCpu, pCache->cr2);
2490
2491	VMXReadCachedVmcs(VMX_VMCS_GUEST_CR3, &val);
2492
2493	if (val != pCtx->cr3)
2494	{
2495	CPUMSetGuestCR3(pVCpu, val);
2496	PGMUpdateCR3(pVCpu, val);
2497	}
2498	rc = hmR0VmxSavePaePdpes(pVCpu, pCtx);
2499	AssertRCReturn(rc, rc);
2500	}
2501
2502	/* Sync back DR7. */
2503	VMXReadCachedVmcs(VMX_VMCS_GUEST_DR7, &val);
2504	pCtx->dr[7] = val;
2505
2506	/* Guest CPU context: ES, CS, SS, DS, FS, GS. */
2507	VMX_READ_SELREG(ES, es);
2508	VMX_READ_SELREG(SS, ss);
2509	VMX_READ_SELREG(CS, cs);
2510	VMX_READ_SELREG(DS, ds);
2511	VMX_READ_SELREG(FS, fs);
2512	VMX_READ_SELREG(GS, gs);
2513
2514	/* System MSRs */
2515	VMXReadCachedVmcs(VMX_VMCS32_GUEST_SYSENTER_CS, &val);
2516	pCtx->SysEnter.cs = val;
2517	VMXReadCachedVmcs(VMX_VMCS_GUEST_SYSENTER_EIP, &val);
2518	pCtx->SysEnter.eip = val;
2519	VMXReadCachedVmcs(VMX_VMCS_GUEST_SYSENTER_ESP, &val);
2520	pCtx->SysEnter.esp = val;
2521
2522	/* Misc. registers; must sync everything otherwise we can get out of sync when jumping to ring 3. */
2523	VMX_READ_SELREG(LDTR, ldtr);
2524
2525	VMXReadCachedVmcs(VMX_VMCS32_GUEST_GDTR_LIMIT, &val);
2526	pCtx->gdtr.cbGdt = val;
2527	VMXReadCachedVmcs(VMX_VMCS_GUEST_GDTR_BASE, &val);
2528	pCtx->gdtr.pGdt = val;
2529
2530	VMXReadCachedVmcs(VMX_VMCS32_GUEST_IDTR_LIMIT, &val);
2531	pCtx->idtr.cbIdt = val;
2532	VMXReadCachedVmcs(VMX_VMCS_GUEST_IDTR_BASE, &val);
2533	pCtx->idtr.pIdt = val;
2534
2535	/* Real mode emulation using v86 mode. */
2536	if ( CPUMIsGuestInRealModeEx(pCtx)
2537	&& pVM->hm.s.vmx.pRealModeTSS)
2538	{
2539	/* Hide our emulation flags */
2540	pCtx->eflags.Bits.u1VM = 0;
2541
2542	/* Restore original IOPL setting as we always use 0. */
2543	pCtx->eflags.Bits.u2IOPL = pVCpu->hm.s.vmx.RealMode.eflags.Bits.u2IOPL;
2544
2545	/* Force a TR resync every time in case we switch modes. */
2546	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_TR;
2547	}
2548	else
2549	{
2550	/* In real mode we have a fake TSS, so only sync it back when it's supposed to be valid. */
2551	VMX_READ_SELREG(TR, tr);
2552	}
2553
2554	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
2555	/*
2556	* Save the possibly changed MSRs that we automatically restore and save during a world switch.
2557	*/
2558	for (unsigned i = 0; i < pVCpu->hm.s.vmx.cGuestMsrs; i++)
2559	{
2560	PVMXMSR pMsr = (PVMXMSR)pVCpu->hm.s.vmx.pvGuestMsr;
2561	pMsr += i;
2562
2563	switch (pMsr->u32IndexMSR)
2564	{
2565	case MSR_K8_LSTAR:
2566	pCtx->msrLSTAR = pMsr->u64Value;
2567	break;
2568	case MSR_K6_STAR:
2569	pCtx->msrSTAR = pMsr->u64Value;
2570	break;
2571	case MSR_K8_SF_MASK:
2572	pCtx->msrSFMASK = pMsr->u64Value;
2573	break;
2574	/* The KERNEL_GS_BASE MSR doesn't work reliably with auto load/store. See @bugref{6208} */
2575	#if 0
2576	case MSR_K8_KERNEL_GS_BASE:
2577	pCtx->msrKERNELGSBASE = pMsr->u64Value;
2578	break;
2579	#endif
2580	case MSR_K8_TSC_AUX:
2581	CPUMSetGuestMsr(pVCpu, MSR_K8_TSC_AUX, pMsr->u64Value);
2582	break;
2583
2584	case MSR_K6_EFER:
2585	/* EFER can't be changed without causing a VM-exit. */
2586	/* Assert(pCtx->msrEFER == pMsr->u64Value); */
2587	break;
2588
2589	default:
2590	AssertFailed();
2591	return VERR_HM_UNEXPECTED_LD_ST_MSR;
2592	}
2593	}
2594	#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
2595	return VINF_SUCCESS;
2596	}
2597
2598
2599	/**
2600	* Dummy placeholder for TLB flush handling before VM-entry. Used in the case
2601	* where neither EPT nor VPID is supported by the CPU.
2602	*
2603	* @param pVM Pointer to the VM.
2604	* @param pVCpu Pointer to the VMCPU.
2605	*/
2606	static DECLCALLBACK(void) hmR0VmxSetupTLBDummy(PVM pVM, PVMCPU pVCpu)
2607	{
2608	NOREF(pVM);
2609	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
2610	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
2611	pVCpu->hm.s.TlbShootdown.cPages = 0;
2612	return;
2613	}
2614
2615
2616	/**
2617	* Setup the tagged TLB for EPT+VPID.
2618	*
2619	* @param pVM Pointer to the VM.
2620	* @param pVCpu Pointer to the VMCPU.
2621	*/
2622	static DECLCALLBACK(void) hmR0VmxSetupTLBBoth(PVM pVM, PVMCPU pVCpu)
2623	{
2624	PHMGLOBLCPUINFO pCpu;
2625
2626	Assert(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid);
2627
2628	pCpu = HMR0GetCurrentCpu();
2629
2630	/*
2631	* Force a TLB flush for the first world switch if the current CPU differs from the one we ran on last
2632	* This can happen both for start & resume due to long jumps back to ring-3.
2633	* If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
2634	* or the host Cpu is online after a suspend/resume, so we cannot reuse the current ASID anymore.
2635	*/
2636	bool fNewAsid = false;
2637	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2638	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2639	{
2640	pVCpu->hm.s.fForceTLBFlush = true;
2641	fNewAsid = true;
2642	}
2643
2644	/*
2645	* Check for explicit TLB shootdowns.
2646	*/
2647	if (VMCPU_FF_TESTANDCLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2648	pVCpu->hm.s.fForceTLBFlush = true;
2649
2650	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2651
2652	if (pVCpu->hm.s.fForceTLBFlush)
2653	{
2654	if (fNewAsid)
2655	{
2656	++pCpu->uCurrentAsid;
2657	if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
2658	{
2659	pCpu->uCurrentAsid = 1; /* start at 1; host uses 0 */
2660	pCpu->cTlbFlushes++;
2661	pCpu->fFlushAsidBeforeUse = true;
2662	}
2663
2664	pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
2665	if (pCpu->fFlushAsidBeforeUse)
2666	hmR0VmxFlushVPID(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
2667	}
2668	else
2669	{
2670	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
2671	hmR0VmxFlushVPID(pVM, pVCpu, VMX_FLUSH_VPID_SINGLE_CONTEXT, 0 /* GCPtr */);
2672	else
2673	hmR0VmxFlushEPT(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
2674	}
2675
2676	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2677	pVCpu->hm.s.fForceTLBFlush = false;
2678	}
2679	else
2680	{
2681	AssertMsg(pVCpu->hm.s.uCurrentAsid && pCpu->uCurrentAsid,
2682	("hm->uCurrentAsid=%lu hm->cTlbFlushes=%lu cpu->uCurrentAsid=%lu cpu->cTlbFlushes=%lu\n",
2683	pVCpu->hm.s.uCurrentAsid, pVCpu->hm.s.cTlbFlushes,
2684	pCpu->uCurrentAsid, pCpu->cTlbFlushes));
2685
2686	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
2687	* not be executed. See hmQueueInvlPage() where it is commented
2688	* out. Support individual entry flushing someday. */
2689	if (VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
2690	{
2691	STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
2692
2693	/*
2694	* Flush individual guest entries using VPID from the TLB or as little as possible with EPT
2695	* as supported by the CPU.
2696	*/
2697	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
2698	{
2699	for (unsigned i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
2700	hmR0VmxFlushVPID(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
2701	}
2702	else
2703	hmR0VmxFlushEPT(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
2704	}
2705	else
2706	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
2707	}
2708
2709	pVCpu->hm.s.TlbShootdown.cPages = 0;
2710	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
2711
2712	AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
2713	("Flush count mismatch for cpu %d (%x vs %x)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
2714	AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2715	("cpu%d uCurrentAsid = %x\n", pCpu->idCpu, pCpu->uCurrentAsid));
2716	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2717	("cpu%d VM uCurrentAsid = %x\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2718
2719	/* Update VMCS with the VPID. */
2720	int rc = VMXWriteVmcs(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
2721	AssertRC(rc);
2722	}
2723
2724
2725	/**
2726	* Setup the tagged TLB for EPT only.
2727	*
2728	* @returns VBox status code.
2729	* @param pVM Pointer to the VM.
2730	* @param pVCpu Pointer to the VMCPU.
2731	*/
2732	static DECLCALLBACK(void) hmR0VmxSetupTLBEPT(PVM pVM, PVMCPU pVCpu)
2733	{
2734	PHMGLOBLCPUINFO pCpu;
2735
2736	Assert(pVM->hm.s.fNestedPaging);
2737	Assert(!pVM->hm.s.vmx.fVpid);
2738
2739	pCpu = HMR0GetCurrentCpu();
2740
2741	/*
2742	* Force a TLB flush for the first world switch if the current CPU differs from the one we ran on last
2743	* This can happen both for start & resume due to long jumps back to ring-3.
2744	* A change in the TLB flush count implies the host Cpu is online after a suspend/resume.
2745	*/
2746	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2747	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2748	{
2749	pVCpu->hm.s.fForceTLBFlush = true;
2750	}
2751
2752	/*
2753	* Check for explicit TLB shootdown flushes.
2754	*/
2755	if (VMCPU_FF_TESTANDCLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2756	pVCpu->hm.s.fForceTLBFlush = true;
2757
2758	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2759	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2760
2761	if (pVCpu->hm.s.fForceTLBFlush)
2762	hmR0VmxFlushEPT(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
2763	else
2764	{
2765	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
2766	* not be executed. See hmQueueInvlPage() where it is commented
2767	* out. Support individual entry flushing someday. */
2768	if (VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
2769	{
2770	/*
2771	* We cannot flush individual entries without VPID support. Flush using EPT.
2772	*/
2773	STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
2774	hmR0VmxFlushEPT(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
2775	}
2776	}
2777	pVCpu->hm.s.TlbShootdown.cPages= 0;
2778	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
2779
2780	#ifdef VBOX_WITH_STATISTICS
2781	/** @todo r=ramshankar: this is not accurate anymore with the VPID+EPT
2782	* handling. Should be fixed later. */
2783	if (pVCpu->hm.s.fForceTLBFlush)
2784	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2785	else
2786	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
2787	#endif
2788	}
2789
2790
2791	/**
2792	* Setup the tagged TLB for VPID.
2793	*
2794	* @returns VBox status code.
2795	* @param pVM Pointer to the VM.
2796	* @param pVCpu Pointer to the VMCPU.
2797	*/
2798	static DECLCALLBACK(void) hmR0VmxSetupTLBVPID(PVM pVM, PVMCPU pVCpu)
2799	{
2800	PHMGLOBLCPUINFO pCpu;
2801
2802	Assert(pVM->hm.s.vmx.fVpid);
2803	Assert(!pVM->hm.s.fNestedPaging);
2804
2805	pCpu = HMR0GetCurrentCpu();
2806
2807	/*
2808	* Force a TLB flush for the first world switch if the current CPU differs from the one we ran on last
2809	* This can happen both for start & resume due to long jumps back to ring-3.
2810	* If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
2811	* or the host Cpu is online after a suspend/resume, so we cannot reuse the current ASID anymore.
2812	*/
2813	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2814	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2815	{
2816	/* Force a TLB flush on VM entry. */
2817	pVCpu->hm.s.fForceTLBFlush = true;
2818	}
2819
2820	/*
2821	* Check for explicit TLB shootdown flushes.
2822	*/
2823	if (VMCPU_FF_TESTANDCLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2824	pVCpu->hm.s.fForceTLBFlush = true;
2825
2826	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2827
2828	if (pVCpu->hm.s.fForceTLBFlush)
2829	{
2830	++pCpu->uCurrentAsid;
2831	if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
2832	{
2833	pCpu->uCurrentAsid = 1; /* start at 1; host uses 0 */
2834	pCpu->cTlbFlushes++;
2835	pCpu->fFlushAsidBeforeUse = true;
2836	}
2837
2838	pVCpu->hm.s.fForceTLBFlush = false;
2839	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2840	pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
2841	if (pCpu->fFlushAsidBeforeUse)
2842	hmR0VmxFlushVPID(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
2843	}
2844	else
2845	{
2846	AssertMsg(pVCpu->hm.s.uCurrentAsid && pCpu->uCurrentAsid,
2847	("hm->uCurrentAsid=%lu hm->cTlbFlushes=%lu cpu->uCurrentAsid=%lu cpu->cTlbFlushes=%lu\n",
2848	pVCpu->hm.s.uCurrentAsid, pVCpu->hm.s.cTlbFlushes,
2849	pCpu->uCurrentAsid, pCpu->cTlbFlushes));
2850
2851	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
2852	* not be executed. See hmQueueInvlPage() where it is commented
2853	* out. Support individual entry flushing someday. */
2854	if (VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
2855	{
2856	/*
2857	* Flush individual guest entries using VPID from the TLB or as little as possible with EPT
2858	* as supported by the CPU.
2859	*/
2860	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
2861	{
2862	for (unsigned i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
2863	hmR0VmxFlushVPID(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
2864	}
2865	else
2866	hmR0VmxFlushVPID(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
2867	}
2868	}
2869	pVCpu->hm.s.TlbShootdown.cPages = 0;
2870	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
2871
2872	AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
2873	("Flush count mismatch for cpu %d (%x vs %x)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
2874	AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2875	("cpu%d uCurrentAsid = %x\n", pCpu->idCpu, pCpu->uCurrentAsid));
2876	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2877	("cpu%d VM uCurrentAsid = %x\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2878
2879	int rc = VMXWriteVmcs(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
2880	AssertRC(rc);
2881
2882	# ifdef VBOX_WITH_STATISTICS
2883	/** @todo r=ramshankar: this is not accurate anymore with EPT+VPID handling.
2884	* Should be fixed later. */
2885	if (pVCpu->hm.s.fForceTLBFlush)
2886	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2887	else
2888	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
2889	# endif
2890	}
2891
2892
2893	/**
2894	* Runs guest code in a VT-x VM.
2895	*
2896	* @returns VBox status code.
2897	* @param pVM Pointer to the VM.
2898	* @param pVCpu Pointer to the VMCPU.
2899	* @param pCtx Pointer to the guest CPU context.
2900	*/
2901	VMMR0DECL(int) VMXR0RunGuestCode(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
2902	{
2903	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
2904	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit1);
2905	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit2);
2906
2907	VBOXSTRICTRC rc = VINF_SUCCESS;
2908	int rc2;
2909	RTGCUINTREG val;
2910	RTGCUINTREG exitReason = (RTGCUINTREG)VMX_EXIT_INVALID;
2911	RTGCUINTREG instrError, cbInstr;
2912	RTGCUINTPTR exitQualification = 0;
2913	RTGCUINTPTR intInfo = 0; /* shut up buggy gcc 4 */
2914	RTGCUINTPTR errCode, instrInfo;
2915	bool fSetupTPRCaching = false;
2916	bool fNeedTscSetup = true;
2917	uint64_t u64OldLSTAR = 0;
2918	uint8_t u8LastTPR = 0;
2919	RTCCUINTREG uOldEFlags = ~(RTCCUINTREG)0;
2920	unsigned cResume = 0;
2921	#ifdef VBOX_STRICT
2922	RTCPUID idCpuCheck;
2923	bool fWasInLongMode = false;
2924	#endif
2925	#ifdef VBOX_HIGH_RES_TIMERS_HACK_IN_RING0
2926	uint64_t u64LastTime = RTTimeMilliTS();
2927	#endif
2928
2929	Assert(!(pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
2930	\|\| (pVCpu->hm.s.vmx.pbVirtApic && pVM->hm.s.vmx.pbApicAccess));
2931
2932	/*
2933	* Check if we need to use TPR shadowing.
2934	*/
2935	if ( CPUMIsGuestInLongModeEx(pCtx)
2936	\|\| ( (( pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
2937	\|\| pVM->hm.s.fTRPPatchingAllowed)
2938	&& pVM->hm.s.fHasIoApic)
2939	)
2940	{
2941	fSetupTPRCaching = true;
2942	}
2943
2944	Log2(("\nE"));
2945
2946	/* This is not ideal, but if we don't clear the event injection in the VMCS right here,
2947	* we may end up injecting some stale event into a VM, including injecting an event that
2948	* originated before a VM reset after the VM has been reset. See @bugref{6220}.
2949	*/
2950	VMXWriteVmcs(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, 0);
2951
2952	#ifdef VBOX_STRICT
2953	{
2954	RTCCUINTREG val2;
2955
2956	rc2 = VMXReadVmcs(VMX_VMCS32_CTRL_PIN_EXEC_CONTROLS, &val2);
2957	AssertRC(rc2);
2958	Log2(("VMX_VMCS_CTRL_PIN_EXEC_CONTROLS = %08x\n", val2));
2959
2960	/* allowed zero */
2961	if ((val2 & pVM->hm.s.vmx.msr.vmx_pin_ctls.n.disallowed0) != pVM->hm.s.vmx.msr.vmx_pin_ctls.n.disallowed0)
2962	Log(("Invalid VMX_VMCS_CTRL_PIN_EXEC_CONTROLS: zero\n"));
2963
2964	/* allowed one */
2965	if ((val2 & ~pVM->hm.s.vmx.msr.vmx_pin_ctls.n.allowed1) != 0)
2966	Log(("Invalid VMX_VMCS_CTRL_PIN_EXEC_CONTROLS: one\n"));
2967
2968	rc2 = VMXReadVmcs(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, &val2);
2969	AssertRC(rc2);
2970	Log2(("VMX_VMCS_CTRL_PROC_EXEC_CONTROLS = %08x\n", val2));
2971
2972	/*
2973	* Must be set according to the MSR, but can be cleared if nested paging is used.
2974	*/
2975	if (pVM->hm.s.fNestedPaging)
2976	{
2977	val2 \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INVLPG_EXIT
2978	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
2979	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT;
2980	}
2981
2982	/* allowed zero */
2983	if ((val2 & pVM->hm.s.vmx.msr.vmx_proc_ctls.n.disallowed0) != pVM->hm.s.vmx.msr.vmx_proc_ctls.n.disallowed0)
2984	Log(("Invalid VMX_VMCS_CTRL_PROC_EXEC_CONTROLS: zero\n"));
2985
2986	/* allowed one */
2987	if ((val2 & ~pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1) != 0)
2988	Log(("Invalid VMX_VMCS_CTRL_PROC_EXEC_CONTROLS: one\n"));
2989
2990	rc2 = VMXReadVmcs(VMX_VMCS32_CTRL_ENTRY_CONTROLS, &val2);
2991	AssertRC(rc2);
2992	Log2(("VMX_VMCS_CTRL_ENTRY_CONTROLS = %08x\n", val2));
2993
2994	/* allowed zero */
2995	if ((val2 & pVM->hm.s.vmx.msr.vmx_entry.n.disallowed0) != pVM->hm.s.vmx.msr.vmx_entry.n.disallowed0)
2996	Log(("Invalid VMX_VMCS_CTRL_ENTRY_CONTROLS: zero\n"));
2997
2998	/* allowed one */
2999	if ((val2 & ~pVM->hm.s.vmx.msr.vmx_entry.n.allowed1) != 0)
3000	Log(("Invalid VMX_VMCS_CTRL_ENTRY_CONTROLS: one\n"));
3001
3002	rc2 = VMXReadVmcs(VMX_VMCS32_CTRL_EXIT_CONTROLS, &val2);
3003	AssertRC(rc2);
3004	Log2(("VMX_VMCS_CTRL_EXIT_CONTROLS = %08x\n", val2));
3005
3006	/* allowed zero */
3007	if ((val2 & pVM->hm.s.vmx.msr.vmx_exit.n.disallowed0) != pVM->hm.s.vmx.msr.vmx_exit.n.disallowed0)
3008	Log(("Invalid VMX_VMCS_CTRL_EXIT_CONTROLS: zero\n"));
3009
3010	/* allowed one */
3011	if ((val2 & ~pVM->hm.s.vmx.msr.vmx_exit.n.allowed1) != 0)
3012	Log(("Invalid VMX_VMCS_CTRL_EXIT_CONTROLS: one\n"));
3013	}
3014	fWasInLongMode = CPUMIsGuestInLongModeEx(pCtx);
3015	#endif /* VBOX_STRICT */
3016
3017	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
3018	pVCpu->hm.s.vmx.VMCSCache.u64TimeEntry = RTTimeNanoTS();
3019	#endif
3020
3021	/*
3022	* We can jump to this point to resume execution after determining that a VM-exit is innocent.
3023	*/
3024	ResumeExecution:
3025	if (!STAM_REL_PROFILE_ADV_IS_RUNNING(&pVCpu->hm.s.StatEntry))
3026	STAM_REL_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit2, &pVCpu->hm.s.StatEntry, x);
3027	AssertMsg(pVCpu->hm.s.idEnteredCpu == RTMpCpuId(),
3028	("Expected %d, I'm %d; cResume=%d exitReason=%RGv exitQualification=%RGv\n",
3029	(int)pVCpu->hm.s.idEnteredCpu, (int)RTMpCpuId(), cResume, exitReason, exitQualification));
3030	Assert(!HMR0SuspendPending());
3031	/* Not allowed to switch modes without reloading the host state (32->64 switcher)!! */
3032	Assert(fWasInLongMode == CPUMIsGuestInLongModeEx(pCtx));
3033
3034	/*
3035	* Safety precaution; looping for too long here can have a very bad effect on the host.
3036	*/
3037	if (RT_UNLIKELY(++cResume > pVM->hm.s.cMaxResumeLoops))
3038	{
3039	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMaxResume);
3040	rc = VINF_EM_RAW_INTERRUPT;
3041	goto end;
3042	}
3043
3044	/*
3045	* Check for IRQ inhibition due to instruction fusing (sti, mov ss).
3046	*/
3047	if (VMCPU_FF_ISSET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3048	{
3049	Log(("VM_FF_INHIBIT_INTERRUPTS at %RGv successor %RGv\n", (RTGCPTR)pCtx->rip, EMGetInhibitInterruptsPC(pVCpu)));
3050	if (pCtx->rip != EMGetInhibitInterruptsPC(pVCpu))
3051	{
3052	/*
3053	* Note: we intentionally don't clear VM_FF_INHIBIT_INTERRUPTS here.
3054	* Before we are able to execute this instruction in raw mode (iret to guest code) an external interrupt might
3055	* force a world switch again. Possibly allowing a guest interrupt to be dispatched in the process. This could
3056	* break the guest. Sounds very unlikely, but such timing sensitive problems are not as rare as you might think.
3057	*/
3058	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
3059	/* Irq inhibition is no longer active; clear the corresponding VMX state. */
3060	rc2 = VMXWriteVmcs(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, 0);
3061	AssertRC(rc2);
3062	}
3063	}
3064	else
3065	{
3066	/* Irq inhibition is no longer active; clear the corresponding VMX state. */
3067	rc2 = VMXWriteVmcs(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, 0);
3068	AssertRC(rc2);
3069	}
3070
3071	#ifdef VBOX_HIGH_RES_TIMERS_HACK_IN_RING0
3072	if (RT_UNLIKELY((cResume & 0xf) == 0))
3073	{
3074	uint64_t u64CurTime = RTTimeMilliTS();
3075
3076	if (RT_UNLIKELY(u64CurTime > u64LastTime))
3077	{
3078	u64LastTime = u64CurTime;
3079	TMTimerPollVoid(pVM, pVCpu);
3080	}
3081	}
3082	#endif
3083
3084	/*
3085	* Check for pending actions that force us to go back to ring-3.
3086	*/
3087	if ( VM_FF_ISPENDING(pVM, VM_FF_HM_TO_R3_MASK \| VM_FF_REQUEST \| VM_FF_PGM_POOL_FLUSH_PENDING \| VM_FF_PDM_DMA)
3088	\|\| VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK \| VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL \| VMCPU_FF_REQUEST))
3089	{
3090	/* Check if a sync operation is pending. */
3091	if (VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
3092	{
3093	rc = PGMSyncCR3(pVCpu, pCtx->cr0, pCtx->cr3, pCtx->cr4, VMCPU_FF_ISSET(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
3094	if (rc != VINF_SUCCESS)
3095	{
3096	AssertRC(VBOXSTRICTRC_VAL(rc));
3097	Log(("Pending pool sync is forcing us back to ring 3; rc=%d\n", VBOXSTRICTRC_VAL(rc)));
3098	goto end;
3099	}
3100	}
3101
3102	#ifdef DEBUG
3103	/* Intercept X86_XCPT_DB if stepping is enabled */
3104	if (!DBGFIsStepping(pVCpu))
3105	#endif
3106	{
3107	if ( VM_FF_ISPENDING(pVM, VM_FF_HM_TO_R3_MASK)
3108	\|\| VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
3109	{
3110	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
3111	rc = RT_UNLIKELY(VM_FF_ISPENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
3112	goto end;
3113	}
3114	}
3115
3116	/* Pending request packets might contain actions that need immediate attention, such as pending hardware interrupts. */
3117	if ( VM_FF_ISPENDING(pVM, VM_FF_REQUEST)
3118	\|\| VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_REQUEST))
3119	{
3120	rc = VINF_EM_PENDING_REQUEST;
3121	goto end;
3122	}
3123
3124	/* Check if a pgm pool flush is in progress. */
3125	if (VM_FF_ISPENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
3126	{
3127	rc = VINF_PGM_POOL_FLUSH_PENDING;
3128	goto end;
3129	}
3130
3131	/* Check if DMA work is pending (2nd+ run). */
3132	if (VM_FF_ISPENDING(pVM, VM_FF_PDM_DMA) && cResume > 1)
3133	{
3134	rc = VINF_EM_RAW_TO_R3;
3135	goto end;
3136	}
3137	}
3138
3139	#ifdef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
3140	/*
3141	* Exit to ring-3 preemption/work is pending.
3142	*
3143	* Interrupts are disabled before the call to make sure we don't miss any interrupt
3144	* that would flag preemption (IPI, timer tick, ++). (Would've been nice to do this
3145	* further down, but hmR0VmxCheckPendingInterrupt makes that impossible.)
3146	*
3147	* Note! Interrupts must be disabled done before we check for TLB flushes; TLB
3148	* shootdowns rely on this.
3149	*/
3150	uOldEFlags = ASMIntDisableFlags();
3151	if (RTThreadPreemptIsPending(NIL_RTTHREAD))
3152	{
3153	STAM_COUNTER_INC(&pVCpu->hm.s.StatPendingHostIrq);
3154	rc = VINF_EM_RAW_INTERRUPT;
3155	goto end;
3156	}
3157	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
3158	#endif
3159
3160	/*
3161	* When external interrupts are pending, we should exit the VM when IF is set.
3162	* Note: After VM_FF_INHIBIT_INTERRUPTS check!
3163	*/
3164	rc = hmR0VmxCheckPendingInterrupt(pVM, pVCpu, pCtx);
3165	if (RT_FAILURE(rc))
3166	goto end;
3167
3168	/** @todo check timers?? */
3169
3170	/*
3171	* TPR caching using CR8 is only available in 64-bit mode.
3172	* Note: The 32-bit exception for AMD (X86_CPUID_AMD_FEATURE_ECX_CR8L), but this appears missing in Intel CPUs.
3173	* Note: We can't do this in LoadGuestState() as PDMApicGetTPR can jump back to ring-3 (lock)!! (no longer true) .
3174	*/
3175	/** @todo query and update the TPR only when it could have been changed (mmio
3176	* access & wrsmr (x2apic) */
3177	if (fSetupTPRCaching)
3178	{
3179	/* TPR caching in CR8 */
3180	bool fPending;
3181
3182	rc2 = PDMApicGetTPR(pVCpu, &u8LastTPR, &fPending);
3183	AssertRC(rc2);
3184	/* The TPR can be found at offset 0x80 in the APIC mmio page. */
3185	pVCpu->hm.s.vmx.pbVirtApic[0x80] = u8LastTPR;
3186
3187	/*
3188	* Two options here:
3189	* - external interrupt pending, but masked by the TPR value.
3190	* -> a CR8 update that lower the current TPR value should cause an exit
3191	* - no pending interrupts
3192	* -> We don't need to be explicitely notified. There are enough world switches for detecting pending interrupts.
3193	*/
3194
3195	/* cr8 bits 3-0 correspond to bits 7-4 of the task priority mmio register. */
3196	rc = VMXWriteVmcs(VMX_VMCS32_CTRL_TPR_THRESHOLD, (fPending) ? (u8LastTPR >> 4) : 0);
3197	AssertRC(VBOXSTRICTRC_VAL(rc));
3198
3199	if (pVM->hm.s.fTPRPatchingActive)
3200	{
3201	Assert(!CPUMIsGuestInLongModeEx(pCtx));
3202	/* Our patch code uses LSTAR for TPR caching. */
3203	pCtx->msrLSTAR = u8LastTPR;
3204
3205	/** @todo r=ramshankar: we should check for MSR-bitmap support here. */
3206	if (fPending)
3207	{
3208	/* A TPR change could activate a pending interrupt, so catch lstar writes. */
3209	hmR0VmxSetMSRPermission(pVCpu, MSR_K8_LSTAR, true, false);
3210	}
3211	else
3212	{
3213	/*
3214	* No interrupts are pending, so we don't need to be explicitely notified.
3215	* There are enough world switches for detecting pending interrupts.
3216	*/
3217	hmR0VmxSetMSRPermission(pVCpu, MSR_K8_LSTAR, true, true);
3218	}
3219	}
3220	}
3221
3222	#ifdef LOG_ENABLED
3223	if ( pVM->hm.s.fNestedPaging
3224	\|\| pVM->hm.s.vmx.fVpid)
3225	{
3226	PHMGLOBLCPUINFO pCpu = HMR0GetCurrentCpu();
3227	if (pVCpu->hm.s.idLastCpu != pCpu->idCpu)
3228	{
3229	LogFlow(("Force TLB flush due to rescheduling to a different cpu (%d vs %d)\n", pVCpu->hm.s.idLastCpu,
3230	pCpu->idCpu));
3231	}
3232	else if (pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
3233	{
3234	LogFlow(("Force TLB flush due to changed TLB flush count (%x vs %x)\n", pVCpu->hm.s.cTlbFlushes,
3235	pCpu->cTlbFlushes));
3236	}
3237	else if (VMCPU_FF_ISSET(pVCpu, VMCPU_FF_TLB_FLUSH))
3238	LogFlow(("Manual TLB flush\n"));
3239	}
3240	#endif
3241	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
3242	PGMRZDynMapFlushAutoSet(pVCpu);
3243	#endif
3244
3245	/*
3246	* NOTE: DO NOT DO ANYTHING AFTER THIS POINT THAT MIGHT JUMP BACK TO RING-3!
3247	* (until the actual world switch)
3248	*/
3249	#ifdef VBOX_STRICT
3250	idCpuCheck = RTMpCpuId();
3251	#endif
3252	#ifdef LOG_ENABLED
3253	VMMR0LogFlushDisable(pVCpu);
3254	#endif
3255
3256	/*
3257	* Save the host state first.
3258	*/
3259	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_HOST_CONTEXT)
3260	{
3261	rc = VMXR0SaveHostState(pVM, pVCpu);
3262	if (RT_UNLIKELY(rc != VINF_SUCCESS))
3263	{
3264	VMMR0LogFlushEnable(pVCpu);
3265	goto end;
3266	}
3267	}
3268
3269	/*
3270	* Load the guest state.
3271	*/
3272	if (!pVCpu->hm.s.fContextUseFlags)
3273	{
3274	VMXR0LoadMinimalGuestState(pVM, pVCpu, pCtx);
3275	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadMinimal);
3276	if (fNeedTscSetup)
3277	{
3278	VMXR0SetupTscOffsetAndPreemption(pVM, pVCpu);
3279	fNeedTscSetup = false;
3280	}
3281	}
3282	else
3283	{
3284	rc = VMXR0LoadGuestState(pVM, pVCpu, pCtx);
3285	if (RT_UNLIKELY(rc != VINF_SUCCESS))
3286	{
3287	VMMR0LogFlushEnable(pVCpu);
3288	goto end;
3289	}
3290	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadFull);
3291	VMXR0SetupTscOffsetAndPreemption(pVM, pVCpu);
3292	}
3293
3294	#ifndef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
3295	/*
3296	* Disable interrupts to make sure a poke will interrupt execution.
3297	* This must be done before we check for TLB flushes; TLB shootdowns rely on this.
3298	*/
3299	uOldEFlags = ASMIntDisableFlags();
3300	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
3301	#endif
3302
3303	/* Non-register state Guest Context */
3304	/** @todo change me according to cpu state */
3305	rc2 = VMXWriteVmcs(VMX_VMCS32_GUEST_ACTIVITY_STATE, VMX_VMCS_GUEST_ACTIVITY_ACTIVE);
3306	AssertRC(rc2);
3307
3308	/* Set TLB flush state as checked until we return from the world switch. */
3309	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true);
3310	/* Deal with tagged TLB setup and invalidation. */
3311	pVM->hm.s.vmx.pfnFlushTaggedTlb(pVM, pVCpu);
3312
3313	/*
3314	* Manual save and restore:
3315	* - General purpose registers except RIP, RSP
3316	*
3317	* Trashed:
3318	* - CR2 (we don't care)
3319	* - LDTR (reset to 0)
3320	* - DRx (presumably not changed at all)
3321	* - DR7 (reset to 0x400)
3322	* - EFLAGS (reset to RT_BIT(1); not relevant)
3323	*/
3324
3325	/* All done! Let's start VM execution. */
3326	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
3327	Assert(idCpuCheck == RTMpCpuId());
3328
3329	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
3330	pVCpu->hm.s.vmx.VMCSCache.cResume = cResume;
3331	pVCpu->hm.s.vmx.VMCSCache.u64TimeSwitch = RTTimeNanoTS();
3332	#endif
3333
3334	/*
3335	* Save the current TPR value in the LSTAR MSR so our patches can access it.
3336	*/
3337	if (pVM->hm.s.fTPRPatchingActive)
3338	{
3339	Assert(pVM->hm.s.fTPRPatchingActive);
3340	u64OldLSTAR = ASMRdMsr(MSR_K8_LSTAR);
3341	ASMWrMsr(MSR_K8_LSTAR, u8LastTPR);
3342	}
3343
3344	TMNotifyStartOfExecution(pVCpu);
3345
3346	#ifndef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
3347	/*
3348	* Save the current Host TSC_AUX and write the guest TSC_AUX to the host, so that
3349	* RDTSCPs (that don't cause exits) reads the guest MSR. See @bugref{3324}.
3350	*/
3351	if ( (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
3352	&& !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT))
3353	{
3354	pVCpu->hm.s.u64HostTSCAux = ASMRdMsr(MSR_K8_TSC_AUX);
3355	uint64_t u64GuestTSCAux = 0;
3356	rc2 = CPUMQueryGuestMsr(pVCpu, MSR_K8_TSC_AUX, &u64GuestTSCAux);
3357	AssertRC(rc2);
3358	ASMWrMsr(MSR_K8_TSC_AUX, u64GuestTSCAux);
3359	}
3360	#endif
3361
3362	#ifdef VBOX_WITH_KERNEL_USING_XMM
3363	rc = hmR0VMXStartVMWrapXMM(pVCpu->hm.s.fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hm.s.vmx.pfnStartVM);
3364	#else
3365	rc = pVCpu->hm.s.vmx.pfnStartVM(pVCpu->hm.s.fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu);
3366	#endif
3367	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false);
3368	ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits);
3369
3370	/* Possibly the last TSC value seen by the guest (too high) (only when we're in TSC offset mode). */
3371	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT))
3372	{
3373	#ifndef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
3374	/* Restore host's TSC_AUX. */
3375	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
3376	ASMWrMsr(MSR_K8_TSC_AUX, pVCpu->hm.s.u64HostTSCAux);
3377	#endif
3378
3379	TMCpuTickSetLastSeen(pVCpu,
3380	ASMReadTSC() + pVCpu->hm.s.vmx.u64TSCOffset - 0x400 /* guestimate of world switch overhead in clock ticks */);
3381	}
3382
3383	TMNotifyEndOfExecution(pVCpu);
3384	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED);
3385	Assert(!(ASMGetFlags() & X86_EFL_IF));
3386
3387	/*
3388	* Restore the host LSTAR MSR if the guest could have changed it.
3389	*/
3390	if (pVM->hm.s.fTPRPatchingActive)
3391	{
3392	Assert(pVM->hm.s.fTPRPatchingActive);
3393	pVCpu->hm.s.vmx.pbVirtApic[0x80] = pCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR);
3394	ASMWrMsr(MSR_K8_LSTAR, u64OldLSTAR);
3395	}
3396
3397	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatExit1, x);
3398	ASMSetFlags(uOldEFlags);
3399	#ifdef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
3400	uOldEFlags = ~(RTCCUINTREG)0;
3401	#endif
3402
3403	AssertMsg(!pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries, ("pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries=%d\n",
3404	pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries));
3405
3406	/* In case we execute a goto ResumeExecution later on. */
3407	pVCpu->hm.s.fResumeVM = true;
3408	pVCpu->hm.s.fForceTLBFlush = false;
3409
3410	/*
3411	* !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
3412	* IMPORTANT: WE CAN'T DO ANY LOGGING OR OPERATIONS THAT CAN DO A LONGJMP BACK TO RING 3 BEFORE WE'VE SYNCED BACK (MOST OF) THE GUEST STATE
3413	* !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
3414	*/
3415
3416	if (RT_UNLIKELY(rc != VINF_SUCCESS))
3417	{
3418	hmR0VmxReportWorldSwitchError(pVM, pVCpu, rc, pCtx);
3419	VMMR0LogFlushEnable(pVCpu);
3420	goto end;
3421	}
3422
3423	/* Success. Query the guest state and figure out what has happened. */
3424
3425	/* Investigate why there was a VM-exit. */
3426	rc2 = VMXReadCachedVmcs(VMX_VMCS32_RO_EXIT_REASON, &exitReason);
3427	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[exitReason & MASK_EXITREASON_STAT]);
3428
3429	exitReason &= 0xffff; /* bit 0-15 contain the exit code. */
3430	rc2 \|= VMXReadCachedVmcs(VMX_VMCS32_RO_VM_INSTR_ERROR, &instrError);
3431	rc2 \|= VMXReadCachedVmcs(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &cbInstr);
3432	rc2 \|= VMXReadCachedVmcs(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &intInfo);
3433	/* might not be valid; depends on VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID. */
3434	rc2 \|= VMXReadCachedVmcs(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &errCode);
3435	rc2 \|= VMXReadCachedVmcs(VMX_VMCS32_RO_EXIT_INSTR_INFO, &instrInfo);
3436	rc2 \|= VMXReadCachedVmcs(VMX_VMCS_RO_EXIT_QUALIFICATION, &exitQualification);
3437	AssertRC(rc2);
3438
3439	/*
3440	* Sync back the guest state.
3441	*/
3442	rc2 = VMXR0SaveGuestState(pVM, pVCpu, pCtx);
3443	AssertRC(rc2);
3444
3445	/* Note! NOW IT'S SAFE FOR LOGGING! */
3446	VMMR0LogFlushEnable(pVCpu);
3447	Log2(("Raw exit reason %08x\n", exitReason));
3448	#if ARCH_BITS == 64 /* for the time being */
3449	VBOXVMM_R0_HMVMX_VMEXIT(pVCpu, pCtx, exitReason);
3450	#endif
3451
3452	/*
3453	* Check if an injected event was interrupted prematurely.
3454	*/
3455	rc2 = VMXReadCachedVmcs(VMX_VMCS32_RO_IDT_INFO, &val);
3456	AssertRC(rc2);
3457	pVCpu->hm.s.Event.u64IntrInfo = VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(val);
3458	if ( VMX_EXIT_INTERRUPTION_INFO_VALID(pVCpu->hm.s.Event.u64IntrInfo)
3459	/* Ignore 'int xx' as they'll be restarted anyway. */
3460	&& VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hm.s.Event.u64IntrInfo) != VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT
3461	/* Ignore software exceptions (such as int3) as they'll reoccur when we restart the instruction anyway. */
3462	&& VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hm.s.Event.u64IntrInfo) != VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT)
3463	{
3464	Assert(!pVCpu->hm.s.Event.fPending);
3465	pVCpu->hm.s.Event.fPending = true;
3466	/* Error code present? */
3467	if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntrInfo))
3468	{
3469	rc2 = VMXReadCachedVmcs(VMX_VMCS32_RO_IDT_ERROR_CODE, &val);
3470	AssertRC(rc2);
3471	pVCpu->hm.s.Event.u32ErrCode = val;
3472	Log(("Pending inject %RX64 at %RGv exit=%08x intInfo=%08x exitQualification=%RGv pending error=%RX64\n",
3473	pVCpu->hm.s.Event.u64IntrInfo, (RTGCPTR)pCtx->rip, exitReason, intInfo, exitQualification, val));
3474	}
3475	else
3476	{
3477	Log(("Pending inject %RX64 at %RGv exit=%08x intInfo=%08x exitQualification=%RGv\n", pVCpu->hm.s.Event.u64IntrInfo,
3478	(RTGCPTR)pCtx->rip, exitReason, intInfo, exitQualification));
3479	pVCpu->hm.s.Event.u32ErrCode = 0;
3480	}
3481	}
3482	#ifdef VBOX_STRICT
3483	else if ( VMX_EXIT_INTERRUPTION_INFO_VALID(pVCpu->hm.s.Event.u64IntrInfo)
3484	/* Ignore software exceptions (such as int3) as they're reoccur when we restart the instruction anyway. */
3485	&& VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hm.s.Event.u64IntrInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT)
3486	{
3487	Log(("Ignore pending inject %RX64 at %RGv exit=%08x intInfo=%08x exitQualification=%RGv\n",
3488	pVCpu->hm.s.Event.u64IntrInfo, (RTGCPTR)pCtx->rip, exitReason, intInfo, exitQualification));
3489	}
3490
3491	if (exitReason == VMX_EXIT_ERR_INVALID_GUEST_STATE)
3492	HMDumpRegs(pVM, pVCpu, pCtx);
3493	#endif
3494
3495	Log2(("E%d: New EIP=%x:%RGv\n", (uint32_t)exitReason, pCtx->cs.Sel, (RTGCPTR)pCtx->rip));
3496	Log2(("Exit reason %d, exitQualification %RGv\n", (uint32_t)exitReason, exitQualification));
3497	Log2(("instrInfo=%d instrError=%d instr length=%d\n", (uint32_t)instrInfo, (uint32_t)instrError, (uint32_t)cbInstr));
3498	Log2(("Interruption error code %d\n", (uint32_t)errCode));
3499	Log2(("IntInfo = %08x\n", (uint32_t)intInfo));
3500
3501	/*
3502	* Sync back the TPR if it was changed.
3503	*/
3504	if ( fSetupTPRCaching
3505	&& u8LastTPR != pVCpu->hm.s.vmx.pbVirtApic[0x80])
3506	{
3507	rc2 = PDMApicSetTPR(pVCpu, pVCpu->hm.s.vmx.pbVirtApic[0x80]);
3508	AssertRC(rc2);
3509	}
3510
3511	#ifdef DBGFTRACE_ENABLED /** @todo DTrace later. */
3512	RTTraceBufAddMsgF(pVM->CTX_SUFF(hTraceBuf), "vmexit %08x %016RX64 at %04:%08RX64 %RX64",
3513	exitReason, (uint64_t)exitQualification, pCtx->cs.Sel, pCtx->rip, (uint64_t)intInfo);
3514	#endif
3515	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
3516
3517	/* Some cases don't need a complete resync of the guest CPU state; handle them here. */
3518	Assert(rc == VINF_SUCCESS); /* might consider VERR_IPE_UNINITIALIZED_STATUS here later... */
3519	switch (exitReason)
3520	{
3521	case VMX_EXIT_XCPT_NMI: /* 0 Exception or non-maskable interrupt (NMI). */
3522	case VMX_EXIT_EXT_INT: /* 1 External interrupt. */
3523	{
3524	uint32_t vector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(intInfo);
3525
3526	if (!VMX_EXIT_INTERRUPTION_INFO_VALID(intInfo))
3527	{
3528	Assert(exitReason == VMX_EXIT_EXT_INT);
3529	/* External interrupt; leave to allow it to be dispatched again. */
3530	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
3531	rc = VINF_EM_RAW_INTERRUPT;
3532	break;
3533	}
3534	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExit2Sub3, y3);
3535	switch (VMX_EXIT_INTERRUPTION_INFO_TYPE(intInfo))
3536	{
3537	case VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI: /* Non-maskable interrupt. */
3538	/* External interrupt; leave to allow it to be dispatched again. */
3539	rc = VINF_EM_RAW_INTERRUPT;
3540	break;
3541
3542	case VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT: /* External hardware interrupt. */
3543	AssertFailed(); /* can't come here; fails the first check. */
3544	break;
3545
3546	case VMX_EXIT_INTERRUPTION_INFO_TYPE_DB_XCPT: /* Unknown why we get this type for #DB */
3547	case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
3548	Assert(vector == 1 \|\| vector == 3 \|\| vector == 4);
3549	/* no break */
3550	case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT: /* Hardware exception. */
3551	Log2(("Hardware/software interrupt %d\n", vector));
3552	switch (vector)
3553	{
3554	case X86_XCPT_NM:
3555	{
3556	Log(("#NM fault at %RGv error code %x\n", (RTGCPTR)pCtx->rip, errCode));
3557
3558	/** @todo don't intercept #NM exceptions anymore when we've activated the guest FPU state. */
3559	/* If we sync the FPU/XMM state on-demand, then we can continue execution as if nothing has happened. */
3560	rc = CPUMR0LoadGuestFPU(pVM, pVCpu, pCtx);
3561	if (rc == VINF_SUCCESS)
3562	{
3563	Assert(CPUMIsGuestFPUStateActive(pVCpu));
3564
3565	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowNM);
3566
3567	/* Continue execution. */
3568	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
3569
3570	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2Sub3, y3);
3571	goto ResumeExecution;
3572	}
3573
3574	Log(("Forward #NM fault to the guest\n"));
3575	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
3576	rc2 = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo),
3577	cbInstr, 0);
3578	AssertRC(rc2);
3579	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2Sub3, y3);
3580	goto ResumeExecution;
3581	}
3582
3583	case X86_XCPT_PF: /* Page fault */
3584	{
3585	#ifdef VBOX_ALWAYS_TRAP_PF
3586	if (pVM->hm.s.fNestedPaging)
3587	{
3588	/*
3589	* A genuine pagefault. Forward the trap to the guest by injecting the exception and resuming execution.
3590	*/
3591	Log(("Guest page fault at %RGv cr2=%RGv error code %RGv rsp=%RGv\n", (RTGCPTR)pCtx->rip, exitQualification,
3592	errCode, (RTGCPTR)pCtx->rsp));
3593
3594	Assert(CPUMIsGuestInPagedProtectedModeEx(pCtx));
3595
3596	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
3597
3598	/* Now we must update CR2. */
3599	pCtx->cr2 = exitQualification;
3600	rc2 = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo),
3601	cbInstr, errCode);
3602	AssertRC(rc2);
3603
3604	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2Sub3, y3);
3605	goto ResumeExecution;
3606	}
3607	#else
3608	Assert(!pVM->hm.s.fNestedPaging);
3609	#endif
3610
3611	#ifdef VBOX_HM_WITH_GUEST_PATCHING
3612	/* Shortcut for APIC TPR reads and writes; 32 bits guests only */
3613	if ( pVM->hm.s.fTRPPatchingAllowed
3614	&& pVM->hm.s.pGuestPatchMem
3615	&& (exitQualification & 0xfff) == 0x080
3616	&& !(errCode & X86_TRAP_PF_P) /* not present */
3617	&& CPUMGetGuestCPL(pVCpu) == 0
3618	&& !CPUMIsGuestInLongModeEx(pCtx)
3619	&& pVM->hm.s.cPatches < RT_ELEMENTS(pVM->hm.s.aPatches))
3620	{
3621	RTGCPHYS GCPhysApicBase, GCPhys;
3622	GCPhysApicBase = pCtx->msrApicBase;
3623	GCPhysApicBase &= PAGE_BASE_GC_MASK;
3624
3625	rc = PGMGstGetPage(pVCpu, (RTGCPTR)exitQualification, NULL, &GCPhys);
3626	if ( rc == VINF_SUCCESS
3627	&& GCPhys == GCPhysApicBase)
3628	{
3629	/* Only attempt to patch the instruction once. */
3630	PHMTPRPATCH pPatch = (PHMTPRPATCH)RTAvloU32Get(&pVM->hm.s.PatchTree, (AVLOU32KEY)pCtx->eip);
3631	if (!pPatch)
3632	{
3633	rc = VINF_EM_HM_PATCH_TPR_INSTR;
3634	break;
3635	}
3636	}
3637	}
3638	#endif
3639
3640	Log2(("Page fault at %RGv error code %x\n", exitQualification, errCode));
3641	/* Exit qualification contains the linear address of the page fault. */
3642	TRPMAssertTrap(pVCpu, X86_XCPT_PF, TRPM_TRAP);
3643	TRPMSetErrorCode(pVCpu, errCode);
3644	TRPMSetFaultAddress(pVCpu, exitQualification);
3645
3646	/* Shortcut for APIC TPR reads and writes. */
3647	if ( (exitQualification & 0xfff) == 0x080
3648	&& !(errCode & X86_TRAP_PF_P) /* not present */
3649	&& fSetupTPRCaching
3650	&& (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC))
3651	{
3652	RTGCPHYS GCPhysApicBase, GCPhys;
3653	GCPhysApicBase = pCtx->msrApicBase;
3654	GCPhysApicBase &= PAGE_BASE_GC_MASK;
3655
3656	rc = PGMGstGetPage(pVCpu, (RTGCPTR)exitQualification, NULL, &GCPhys);
3657	if ( rc == VINF_SUCCESS
3658	&& GCPhys == GCPhysApicBase)
3659	{
3660	Log(("Enable VT-x virtual APIC access filtering\n"));
3661	rc2 = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess,
3662	X86_PTE_RW \| X86_PTE_P);
3663	AssertRC(rc2);
3664	}
3665	}
3666
3667	/* Forward it to our trap handler first, in case our shadow pages are out of sync. */
3668	rc = PGMTrap0eHandler(pVCpu, errCode, CPUMCTX2CORE(pCtx), (RTGCPTR)exitQualification);
3669	Log2(("PGMTrap0eHandler %RGv returned %Rrc\n", (RTGCPTR)pCtx->rip, VBOXSTRICTRC_VAL(rc)));
3670
3671	if (rc == VINF_SUCCESS)
3672	{ /* We've successfully synced our shadow pages, so let's just continue execution. */
3673	Log2(("Shadow page fault at %RGv cr2=%RGv error code %x\n", (RTGCPTR)pCtx->rip, exitQualification ,errCode));
3674	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
3675
3676	TRPMResetTrap(pVCpu);
3677	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2Sub3, y3);
3678	goto ResumeExecution;
3679	}
3680	else if (rc == VINF_EM_RAW_GUEST_TRAP)
3681	{
3682	/*
3683	* A genuine pagefault. Forward the trap to the guest by injecting the exception and resuming execution.
3684	*/
3685	Log2(("Forward page fault to the guest\n"));
3686
3687	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
3688	/* The error code might have been changed. */
3689	errCode = TRPMGetErrorCode(pVCpu);
3690
3691	TRPMResetTrap(pVCpu);
3692
3693	/* Now we must update CR2. */
3694	pCtx->cr2 = exitQualification;
3695	rc2 = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo),
3696	cbInstr, errCode);
3697	AssertRC(rc2);
3698
3699	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2Sub3, y3);
3700	goto ResumeExecution;
3701	}
3702	#ifdef VBOX_STRICT
3703	if (rc != VINF_EM_RAW_EMULATE_INSTR && rc != VINF_EM_RAW_EMULATE_IO_BLOCK)
3704	Log2(("PGMTrap0eHandler failed with %d\n", VBOXSTRICTRC_VAL(rc)));
3705	#endif
3706	/* Need to go back to the recompiler to emulate the instruction. */
3707	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
3708	TRPMResetTrap(pVCpu);
3709
3710	/* If event delivery caused the #PF (shadow or not), tell TRPM. */
3711	hmR0VmxCheckPendingEvent(pVCpu);
3712	break;
3713	}
3714
3715	case X86_XCPT_MF: /* Floating point exception. */
3716	{
3717	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
3718	if (!(pCtx->cr0 & X86_CR0_NE))
3719	{
3720	/* old style FPU error reporting needs some extra work. */
3721	/** @todo don't fall back to the recompiler, but do it manually. */
3722	rc = VINF_EM_RAW_EMULATE_INSTR;
3723	break;
3724	}
3725	Log(("Trap %x at %04X:%RGv\n", vector, pCtx->cs.Sel, (RTGCPTR)pCtx->rip));
3726	rc2 = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo),
3727	cbInstr, errCode);
3728	AssertRC(rc2);
3729
3730	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2Sub3, y3);
3731	goto ResumeExecution;
3732	}
3733
3734	case X86_XCPT_DB: /* Debug exception. */
3735	{
3736	uint64_t uDR6;
3737
3738	/*
3739	* DR6, DR7.GD and IA32_DEBUGCTL.LBR are not updated yet.
3740	*
3741	* Exit qualification bits:
3742	* 3:0 B0-B3 which breakpoint condition was met
3743	* 12:4 Reserved (0)
3744	* 13 BD - debug register access detected
3745	* 14 BS - single step execution or branch taken
3746	* 63:15 Reserved (0)
3747	*/
3748	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
3749
3750	/* Note that we don't support guest and host-initiated debugging at the same time. */
3751
3752	uDR6 = X86_DR6_INIT_VAL;
3753	uDR6 \|= (exitQualification & (X86_DR6_B0\|X86_DR6_B1\|X86_DR6_B2\|X86_DR6_B3\|X86_DR6_BD\|X86_DR6_BS));
3754	rc = DBGFRZTrap01Handler(pVM, pVCpu, CPUMCTX2CORE(pCtx), uDR6);
3755	if (rc == VINF_EM_RAW_GUEST_TRAP)
3756	{
3757	/* Update DR6 here. */
3758	pCtx->dr[6] = uDR6;
3759
3760	/* Resync DR6 if the debug state is active. */
3761	if (CPUMIsGuestDebugStateActive(pVCpu))
3762	ASMSetDR6(pCtx->dr[6]);
3763
3764	/* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
3765	pCtx->dr[7] &= ~X86_DR7_GD;
3766
3767	/* Paranoia. */
3768	pCtx->dr[7] &= 0xffffffff; /* upper 32 bits reserved */
3769	pCtx->dr[7] &= ~(RT_BIT(11) \| RT_BIT(12) \| RT_BIT(14) \| RT_BIT(15)); /* must be zero */
3770	pCtx->dr[7] \|= 0x400; /* must be one */
3771
3772	/* Resync DR7 */
3773	rc2 = VMXWriteVmcs64(VMX_VMCS_GUEST_DR7, pCtx->dr[7]);
3774	AssertRC(rc2);
3775
3776	Log(("Trap %x (debug) at %RGv exit qualification %RX64 dr6=%x dr7=%x\n", vector, (RTGCPTR)pCtx->rip,
3777	exitQualification, (uint32_t)pCtx->dr[6], (uint32_t)pCtx->dr[7]));
3778	rc2 = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo),
3779	cbInstr, errCode);
3780	AssertRC(rc2);
3781
3782	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2Sub3, y3);
3783	goto ResumeExecution;
3784	}
3785	/* Return to ring 3 to deal with the debug exit code. */
3786	Log(("Debugger hardware BP at %04x:%RGv (rc=%Rrc)\n", pCtx->cs.Sel, pCtx->rip, VBOXSTRICTRC_VAL(rc)));
3787	break;
3788	}
3789
3790	case X86_XCPT_BP: /* Breakpoint. */
3791	{
3792	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
3793	rc = DBGFRZTrap03Handler(pVM, pVCpu, CPUMCTX2CORE(pCtx));
3794	if (rc == VINF_EM_RAW_GUEST_TRAP)
3795	{
3796	Log(("Guest #BP at %04x:%RGv\n", pCtx->cs.Sel, pCtx->rip));
3797	rc2 = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo),
3798	cbInstr, errCode);
3799	AssertRC(rc2);
3800	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2Sub3, y3);
3801	goto ResumeExecution;
3802	}
3803	if (rc == VINF_SUCCESS)
3804	{
3805	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2Sub3, y3);
3806	goto ResumeExecution;
3807	}
3808	Log(("Debugger BP at %04x:%RGv (rc=%Rrc)\n", pCtx->cs.Sel, pCtx->rip, VBOXSTRICTRC_VAL(rc)));
3809	break;
3810	}
3811
3812	case X86_XCPT_GP: /* General protection failure exception. */
3813	{
3814	uint32_t cbOp;
3815	PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
3816
3817	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
3818	#ifdef VBOX_STRICT
3819	if ( !CPUMIsGuestInRealModeEx(pCtx)
3820	\|\| !pVM->hm.s.vmx.pRealModeTSS)
3821	{
3822	Log(("Trap %x at %04X:%RGv errorCode=%RGv\n", vector, pCtx->cs.Sel, (RTGCPTR)pCtx->rip, errCode));
3823	rc2 = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo),
3824	cbInstr, errCode);
3825	AssertRC(rc2);
3826	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2Sub3, y3);
3827	goto ResumeExecution;
3828	}
3829	#endif
3830	Assert(CPUMIsGuestInRealModeEx(pCtx));
3831
3832	LogFlow(("Real mode X86_XCPT_GP instruction emulation at %x:%RGv\n", pCtx->cs.Sel, (RTGCPTR)pCtx->rip));
3833
3834	rc2 = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbOp);
3835	if (RT_SUCCESS(rc2))
3836	{
3837	bool fUpdateRIP = true;
3838
3839	rc = VINF_SUCCESS;
3840	Assert(cbOp == pDis->cbInstr);
3841	switch (pDis->pCurInstr->uOpcode)
3842	{
3843	case OP_CLI:
3844	pCtx->eflags.Bits.u1IF = 0;
3845	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCli);
3846	break;
3847
3848	case OP_STI:
3849	pCtx->eflags.Bits.u1IF = 1;
3850	EMSetInhibitInterruptsPC(pVCpu, pCtx->rip + pDis->cbInstr);
3851	Assert(VMCPU_FF_ISSET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
3852	rc2 = VMXWriteVmcs(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE,
3853	VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
3854	AssertRC(rc2);
3855	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitSti);
3856	break;
3857
3858	case OP_HLT:
3859	fUpdateRIP = false;
3860	rc = VINF_EM_HALT;
3861	pCtx->rip += pDis->cbInstr;
3862	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
3863	break;
3864
3865	case OP_POPF:
3866	{
3867	RTGCPTR GCPtrStack;
3868	uint32_t cbParm;
3869	uint32_t uMask;
3870	X86EFLAGS eflags;
3871
3872	if (pDis->fPrefix & DISPREFIX_OPSIZE)
3873	{
3874	cbParm = 4;
3875	uMask = 0xffffffff;
3876	}
3877	else
3878	{
3879	cbParm = 2;
3880	uMask = 0xffff;
3881	}
3882
3883	rc2 = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pCtx), pCtx->esp & uMask, 0, &GCPtrStack);
3884	if (RT_FAILURE(rc2))
3885	{
3886	rc = VERR_EM_INTERPRETER;
3887	break;
3888	}
3889	eflags.u = 0;
3890	rc2 = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &eflags.u, cbParm);
3891	if (RT_FAILURE(rc2))
3892	{
3893	rc = VERR_EM_INTERPRETER;
3894	break;
3895	}
3896	LogFlow(("POPF %x -> %RGv mask=%x\n", eflags.u, pCtx->rsp, uMask));
3897	pCtx->eflags.u = (pCtx->eflags.u & ~(X86_EFL_POPF_BITS & uMask))
3898	\| (eflags.u & X86_EFL_POPF_BITS & uMask);
3899	/* RF cleared when popped in real mode; see pushf description in AMD manual. */
3900	pCtx->eflags.Bits.u1RF = 0;
3901	pCtx->esp += cbParm;
3902	pCtx->esp &= uMask;
3903
3904	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPopf);
3905	break;
3906	}
3907
3908	case OP_PUSHF:
3909	{
3910	RTGCPTR GCPtrStack;
3911	uint32_t cbParm;
3912	uint32_t uMask;
3913	X86EFLAGS eflags;
3914
3915	if (pDis->fPrefix & DISPREFIX_OPSIZE)
3916	{
3917	cbParm = 4;
3918	uMask = 0xffffffff;
3919	}
3920	else
3921	{
3922	cbParm = 2;
3923	uMask = 0xffff;
3924	}
3925
3926	rc2 = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pCtx), (pCtx->esp - cbParm) & uMask, 0,
3927	&GCPtrStack);
3928	if (RT_FAILURE(rc2))
3929	{
3930	rc = VERR_EM_INTERPRETER;
3931	break;
3932	}
3933	eflags = pCtx->eflags;
3934	/* RF & VM cleared when pushed in real mode; see pushf description in AMD manual. */
3935	eflags.Bits.u1RF = 0;
3936	eflags.Bits.u1VM = 0;
3937
3938	rc2 = PGMPhysWrite(pVM, (RTGCPHYS)GCPtrStack, &eflags.u, cbParm);
3939	if (RT_FAILURE(rc2))
3940	{
3941	rc = VERR_EM_INTERPRETER;
3942	break;
3943	}
3944	LogFlow(("PUSHF %x -> %RGv\n", eflags.u, GCPtrStack));
3945	pCtx->esp -= cbParm;
3946	pCtx->esp &= uMask;
3947	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPushf);
3948	break;
3949	}
3950
3951	case OP_IRET:
3952	{
3953	RTGCPTR GCPtrStack;
3954	uint32_t uMask = 0xffff;
3955	uint16_t aIretFrame[3];
3956
3957	if (pDis->fPrefix & (DISPREFIX_OPSIZE \| DISPREFIX_ADDRSIZE))
3958	{
3959	rc = VERR_EM_INTERPRETER;
3960	break;
3961	}
3962
3963	rc2 = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pCtx), pCtx->esp & uMask, 0, &GCPtrStack);
3964	if (RT_FAILURE(rc2))
3965	{
3966	rc = VERR_EM_INTERPRETER;
3967	break;
3968	}
3969	rc2 = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &aIretFrame[0], sizeof(aIretFrame));
3970	if (RT_FAILURE(rc2))
3971	{
3972	rc = VERR_EM_INTERPRETER;
3973	break;
3974	}
3975	pCtx->ip = aIretFrame[0];
3976	pCtx->cs.Sel = aIretFrame[1];
3977	pCtx->cs.ValidSel = aIretFrame[1];
3978	pCtx->cs.u64Base = (uint32_t)pCtx->cs.Sel << 4;
3979	pCtx->eflags.u = (pCtx->eflags.u & ~(X86_EFL_POPF_BITS & uMask))
3980	\| (aIretFrame[2] & X86_EFL_POPF_BITS & uMask);
3981	pCtx->sp += sizeof(aIretFrame);
3982
3983	LogFlow(("iret to %04x:%x\n", pCtx->cs.Sel, pCtx->ip));
3984	fUpdateRIP = false;
3985	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIret);
3986	break;
3987	}
3988
3989	case OP_INT:
3990	{
3991	uint32_t intInfo2;
3992
3993	LogFlow(("Realmode: INT %x\n", pDis->Param1.uValue & 0xff));
3994	intInfo2 = pDis->Param1.uValue & 0xff;
3995	intInfo2 \|= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
3996	intInfo2 \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
3997
3998	rc = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, intInfo2, cbOp, 0);
3999	AssertRC(VBOXSTRICTRC_VAL(rc));
4000	fUpdateRIP = false;
4001	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
4002	break;
4003	}
4004
4005	case OP_INTO:
4006	{
4007	if (pCtx->eflags.Bits.u1OF)
4008	{
4009	uint32_t intInfo2;
4010
4011	LogFlow(("Realmode: INTO\n"));
4012	intInfo2 = X86_XCPT_OF;
4013	intInfo2 \|= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
4014	intInfo2 \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
4015
4016	rc = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, intInfo2, cbOp, 0);
4017	AssertRC(VBOXSTRICTRC_VAL(rc));
4018	fUpdateRIP = false;
4019	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
4020	}
4021	break;
4022	}
4023
4024	case OP_INT3:
4025	{
4026	uint32_t intInfo2;
4027
4028	LogFlow(("Realmode: INT 3\n"));
4029	intInfo2 = 3;
4030	intInfo2 \|= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
4031	intInfo2 \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
4032
4033	rc = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, intInfo2, cbOp, 0);
4034	AssertRC(VBOXSTRICTRC_VAL(rc));
4035	fUpdateRIP = false;
4036	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
4037	break;
4038	}
4039
4040	default:
4041	rc = EMInterpretInstructionDisasState(pVCpu, pDis, CPUMCTX2CORE(pCtx), 0, EMCODETYPE_SUPERVISOR);
4042	fUpdateRIP = false;
4043	break;
4044	}
4045
4046	if (rc == VINF_SUCCESS)
4047	{
4048	if (fUpdateRIP)
4049	pCtx->rip += cbOp; /* Move on to the next instruction. */
4050
4051	/*
4052	* LIDT, LGDT can end up here. In the future CRx changes as well. Just reload the
4053	* whole context to be done with it.
4054	*/
4055	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_ALL;
4056
4057	/* Only resume if successful. */
4058	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2Sub3, y3);
4059	goto ResumeExecution;
4060	}
4061	}
4062	else
4063	rc = VERR_EM_INTERPRETER;
4064
4065	AssertMsg(rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_EM_HALT,
4066	("Unexpected rc=%Rrc\n", VBOXSTRICTRC_VAL(rc)));
4067	break;
4068	}
4069
4070	#ifdef VBOX_STRICT
4071	case X86_XCPT_XF: /* SIMD exception. */
4072	case X86_XCPT_DE: /* Divide error. */
4073	case X86_XCPT_UD: /* Unknown opcode exception. */
4074	case X86_XCPT_SS: /* Stack segment exception. */
4075	case X86_XCPT_NP: /* Segment not present exception. */
4076	{
4077	switch (vector)
4078	{
4079	case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE); break;
4080	case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD); break;
4081	case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS); break;
4082	case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP); break;
4083	case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF); break;
4084	}
4085
4086	Log(("Trap %x at %04X:%RGv\n", vector, pCtx->cs.Sel, (RTGCPTR)pCtx->rip));
4087	rc2 = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo),
4088	cbInstr, errCode);
4089	AssertRC(rc2);
4090
4091	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2Sub3, y3);
4092	goto ResumeExecution;
4093	}
4094	#endif
4095	default:
4096	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
4097	if ( CPUMIsGuestInRealModeEx(pCtx)
4098	&& pVM->hm.s.vmx.pRealModeTSS)
4099	{
4100	Log(("Real Mode Trap %x at %04x:%04X error code %x\n", vector, pCtx->cs.Sel, pCtx->eip, errCode));
4101	rc = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo),
4102	cbInstr, errCode);
4103	AssertRC(VBOXSTRICTRC_VAL(rc)); /* Strict RC check below. */
4104
4105	/* Go back to ring-3 in case of a triple fault. */
4106	if ( vector == X86_XCPT_DF
4107	&& rc == VINF_EM_RESET)
4108	{
4109	break;
4110	}
4111
4112	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2Sub3, y3);
4113	goto ResumeExecution;
4114	}
4115	AssertMsgFailed(("Unexpected vm-exit caused by exception %x\n", vector));
4116	rc = VERR_VMX_UNEXPECTED_EXCEPTION;
4117	break;
4118	} /* switch (vector) */
4119
4120	break;
4121
4122	default:
4123	rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_CODE;
4124	AssertMsgFailed(("Unexpected interruption code %x\n", intInfo));
4125	break;
4126	}
4127
4128	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2Sub3, y3);
4129	break;
4130	}
4131
4132	/*
4133	* 48 EPT violation. An attempt to access memory with a guest-physical address was disallowed
4134	* by the configuration of the EPT paging structures.
4135	*/
4136	case VMX_EXIT_EPT_VIOLATION:
4137	{
4138	RTGCPHYS GCPhys;
4139
4140	Assert(pVM->hm.s.fNestedPaging);
4141
4142	rc2 = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
4143	AssertRC(rc2);
4144	Assert(((exitQualification >> 7) & 3) != 2);
4145
4146	/* Determine the kind of violation. */
4147	errCode = 0;
4148	if (exitQualification & VMX_EXIT_QUALIFICATION_EPT_INSTR_FETCH)
4149	errCode \|= X86_TRAP_PF_ID;
4150
4151	if (exitQualification & VMX_EXIT_QUALIFICATION_EPT_DATA_WRITE)
4152	errCode \|= X86_TRAP_PF_RW;
4153
4154	/* If the page is present, then it's a page level protection fault. */
4155	if (exitQualification & VMX_EXIT_QUALIFICATION_EPT_ENTRY_PRESENT)
4156	errCode \|= X86_TRAP_PF_P;
4157	else
4158	{
4159	/* Shortcut for APIC TPR reads and writes. */
4160	if ( (GCPhys & 0xfff) == 0x080
4161	&& GCPhys > 0x1000000 /* to skip VGA frame buffer accesses */
4162	&& fSetupTPRCaching
4163	&& (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC))
4164	{
4165	RTGCPHYS GCPhysApicBase;
4166	GCPhysApicBase = pCtx->msrApicBase;
4167	GCPhysApicBase &= PAGE_BASE_GC_MASK;
4168	if (GCPhys == GCPhysApicBase + 0x80)
4169	{
4170	Log(("Enable VT-x virtual APIC access filtering\n"));
4171	rc2 = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess,
4172	X86_PTE_RW \| X86_PTE_P);
4173	AssertRC(rc2);
4174	}
4175	}
4176	}
4177	Log(("EPT Page fault %x at %RGp error code %x\n", (uint32_t)exitQualification, GCPhys, errCode));
4178
4179	/* GCPhys contains the guest physical address of the page fault. */
4180	TRPMAssertTrap(pVCpu, X86_XCPT_PF, TRPM_TRAP);
4181	TRPMSetErrorCode(pVCpu, errCode);
4182	TRPMSetFaultAddress(pVCpu, GCPhys);
4183
4184	/* Handle the pagefault trap for the nested shadow table. */
4185	rc = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, errCode, CPUMCTX2CORE(pCtx), GCPhys);
4186
4187	/*
4188	* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment below, @bugref{6043}.
4189	*/
4190	if ( rc == VINF_SUCCESS
4191	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
4192	\|\| rc == VERR_PAGE_NOT_PRESENT)
4193	{
4194	/* We've successfully synced our shadow pages, so let's just continue execution. */
4195	Log2(("Shadow page fault at %RGv cr2=%RGp error code %x\n", (RTGCPTR)pCtx->rip, exitQualification , errCode));
4196	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
4197
4198	TRPMResetTrap(pVCpu);
4199	goto ResumeExecution;
4200	}
4201
4202	#ifdef VBOX_STRICT
4203	if (rc != VINF_EM_RAW_EMULATE_INSTR)
4204	LogFlow(("PGMTrap0eHandlerNestedPaging at %RGv failed with %Rrc\n", (RTGCPTR)pCtx->rip, VBOXSTRICTRC_VAL(rc)));
4205	#endif
4206	/* Need to go back to the recompiler to emulate the instruction. */
4207	TRPMResetTrap(pVCpu);
4208	break;
4209	}
4210
4211	case VMX_EXIT_EPT_MISCONFIG:
4212	{
4213	RTGCPHYS GCPhys;
4214
4215	Assert(pVM->hm.s.fNestedPaging);
4216
4217	rc2 = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
4218	AssertRC(rc2);
4219	Log(("VMX_EXIT_EPT_MISCONFIG for %RGp\n", GCPhys));
4220
4221	/* Shortcut for APIC TPR reads and writes. */
4222	if ( (GCPhys & 0xfff) == 0x080
4223	&& GCPhys > 0x1000000 /* to skip VGA frame buffer accesses */
4224	&& fSetupTPRCaching
4225	&& (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC))
4226	{
4227	RTGCPHYS GCPhysApicBase = pCtx->msrApicBase;
4228	GCPhysApicBase &= PAGE_BASE_GC_MASK;
4229	if (GCPhys == GCPhysApicBase + 0x80)
4230	{
4231	Log(("Enable VT-x virtual APIC access filtering\n"));
4232	rc2 = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess,
4233	X86_PTE_RW \| X86_PTE_P);
4234	AssertRC(rc2);
4235	}
4236	}
4237
4238	rc = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pCtx), GCPhys, UINT32_MAX);
4239
4240	/*
4241	* If we succeed, resume execution.
4242	* Or, if fail in interpreting the instruction because we couldn't get the guest physical address
4243	* of the page containing the instruction via the guest's page tables (we would invalidate the guest page
4244	* in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
4245	* weird case. See @bugref{6043}.
4246	*/
4247	if ( rc == VINF_SUCCESS
4248	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
4249	\|\| rc == VERR_PAGE_NOT_PRESENT)
4250	{
4251	Log2(("PGMR0Trap0eHandlerNPMisconfig(,,,%RGp) at %RGv -> resume\n", GCPhys, (RTGCPTR)pCtx->rip));
4252	goto ResumeExecution;
4253	}
4254
4255	Log2(("PGMR0Trap0eHandlerNPMisconfig(,,,%RGp) at %RGv -> %Rrc\n", GCPhys, (RTGCPTR)pCtx->rip, VBOXSTRICTRC_VAL(rc)));
4256	break;
4257	}
4258
4259	case VMX_EXIT_INT_WINDOW: /* 7 Interrupt window exiting. */
4260	/* Clear VM-exit on IF=1 change. */
4261	LogFlow(("VMX_EXIT_INT_WINDOW %RGv pending=%d IF=%d\n", (RTGCPTR)pCtx->rip,
4262	VMCPU_FF_ISPENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC\|VMCPU_FF_INTERRUPT_PIC)), pCtx->eflags.Bits.u1IF));
4263	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INT_WINDOW_EXIT;
4264	rc2 = VMXWriteVmcs(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls);
4265	AssertRC(rc2);
4266	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
4267	goto ResumeExecution; /* we check for pending guest interrupts there */
4268
4269	case VMX_EXIT_WBINVD: /* 54 Guest software attempted to execute WBINVD. (conditional) */
4270	case VMX_EXIT_INVD: /* 13 Guest software attempted to execute INVD. (unconditional) */
4271	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvd);
4272	/* Skip instruction and continue directly. */
4273	pCtx->rip += cbInstr;
4274	/* Continue execution.*/
4275	goto ResumeExecution;
4276
4277	case VMX_EXIT_CPUID: /* 10 Guest software attempted to execute CPUID. */
4278	{
4279	Log2(("VMX: Cpuid %x\n", pCtx->eax));
4280	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCpuid);
4281	rc = EMInterpretCpuId(pVM, pVCpu, CPUMCTX2CORE(pCtx));
4282	if (rc == VINF_SUCCESS)
4283	{
4284	/* Update EIP and continue execution. */
4285	Assert(cbInstr == 2);
4286	pCtx->rip += cbInstr;
4287	goto ResumeExecution;
4288	}
4289	AssertMsgFailed(("EMU: cpuid failed with %Rrc\n", VBOXSTRICTRC_VAL(rc)));
4290	rc = VINF_EM_RAW_EMULATE_INSTR;
4291	break;
4292	}
4293
4294	case VMX_EXIT_RDPMC: /* 15 Guest software attempted to execute RDPMC. */
4295	{
4296	Log2(("VMX: Rdpmc %x\n", pCtx->ecx));
4297	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdpmc);
4298	rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pCtx));
4299	if (rc == VINF_SUCCESS)
4300	{
4301	/* Update EIP and continue execution. */
4302	Assert(cbInstr == 2);
4303	pCtx->rip += cbInstr;
4304	goto ResumeExecution;
4305	}
4306	rc = VINF_EM_RAW_EMULATE_INSTR;
4307	break;
4308	}
4309
4310	case VMX_EXIT_RDTSC: /* 16 Guest software attempted to execute RDTSC. */
4311	{
4312	Log2(("VMX: Rdtsc\n"));
4313	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
4314	rc = EMInterpretRdtsc(pVM, pVCpu, CPUMCTX2CORE(pCtx));
4315	if (rc == VINF_SUCCESS)
4316	{
4317	/* Update EIP and continue execution. */
4318	Assert(cbInstr == 2);
4319	pCtx->rip += cbInstr;
4320	fNeedTscSetup = true; /* See @bugref{6634}. */
4321	goto ResumeExecution;
4322	}
4323	rc = VINF_EM_RAW_EMULATE_INSTR;
4324	break;
4325	}
4326
4327	case VMX_EXIT_RDTSCP: /* 51 Guest software attempted to execute RDTSCP. */
4328	{
4329	Log2(("VMX: Rdtscp\n"));
4330	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtscp);
4331	rc = EMInterpretRdtscp(pVM, pVCpu, pCtx);
4332	if (rc == VINF_SUCCESS)
4333	{
4334	/* Update EIP and continue execution. */
4335	Assert(cbInstr == 3);
4336	pCtx->rip += cbInstr;
4337	fNeedTscSetup = true; /* See @bugref{6634}. */
4338	goto ResumeExecution;
4339	}
4340	rc = VINF_EM_RAW_EMULATE_INSTR;
4341	break;
4342	}
4343
4344	case VMX_EXIT_INVLPG: /* 14 Guest software attempted to execute INVLPG. */
4345	{
4346	Log2(("VMX: invlpg\n"));
4347	Assert(!pVM->hm.s.fNestedPaging);
4348
4349	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvlpg);
4350	rc = EMInterpretInvlpg(pVM, pVCpu, CPUMCTX2CORE(pCtx), exitQualification);
4351	if (rc == VINF_SUCCESS)
4352	{
4353	/* Update EIP and continue execution. */
4354	pCtx->rip += cbInstr;
4355	goto ResumeExecution;
4356	}
4357	AssertMsg(rc == VERR_EM_INTERPRETER, ("EMU: invlpg %RGv failed with %Rrc\n", exitQualification, VBOXSTRICTRC_VAL(rc)));
4358	break;
4359	}
4360
4361	case VMX_EXIT_MONITOR: /* 39 Guest software attempted to execute MONITOR. */
4362	{
4363	Log2(("VMX: monitor\n"));
4364
4365	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMonitor);
4366	rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pCtx));
4367	if (rc == VINF_SUCCESS)
4368	{
4369	/* Update EIP and continue execution. */
4370	pCtx->rip += cbInstr;
4371	goto ResumeExecution;
4372	}
4373	AssertMsg(rc == VERR_EM_INTERPRETER, ("EMU: monitor failed with %Rrc\n", VBOXSTRICTRC_VAL(rc)));
4374	break;
4375	}
4376
4377	case VMX_EXIT_WRMSR: /* 32 WRMSR. Guest software attempted to execute WRMSR. */
4378	/* When an interrupt is pending, we'll let MSR_K8_LSTAR writes fault in our TPR patch code. */
4379	if ( pVM->hm.s.fTPRPatchingActive
4380	&& pCtx->ecx == MSR_K8_LSTAR)
4381	{
4382	Assert(!CPUMIsGuestInLongModeEx(pCtx));
4383	if ((pCtx->eax & 0xff) != u8LastTPR)
4384	{
4385	Log(("VMX: Faulting MSR_K8_LSTAR write with new TPR value %x\n", pCtx->eax & 0xff));
4386
4387	/* Our patch code uses LSTAR for TPR caching. */
4388	rc2 = PDMApicSetTPR(pVCpu, pCtx->eax & 0xff);
4389	AssertRC(rc2);
4390	}
4391
4392	/* Skip the instruction and continue. */
4393	pCtx->rip += cbInstr; /* wrmsr = [0F 30] */
4394
4395	/* Only resume if successful. */
4396	goto ResumeExecution;
4397	}
4398	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_MSR;
4399	/* no break */
4400	case VMX_EXIT_RDMSR: /* 31 RDMSR. Guest software attempted to execute RDMSR. */
4401	{
4402	STAM_COUNTER_INC((exitReason == VMX_EXIT_RDMSR) ? &pVCpu->hm.s.StatExitRdmsr : &pVCpu->hm.s.StatExitWrmsr);
4403
4404	Log2(("VMX: %s\n", (exitReason == VMX_EXIT_RDMSR) ? "rdmsr" : "wrmsr"));
4405	rc = EMInterpretInstruction(pVCpu, CPUMCTX2CORE(pCtx), 0);
4406	if (rc == VINF_SUCCESS)
4407	{
4408	/* EIP has been updated already. */
4409	/* Only resume if successful. */
4410	goto ResumeExecution;
4411	}
4412	AssertMsg(rc == VERR_EM_INTERPRETER, ("EMU: %s failed with %Rrc\n",
4413	(exitReason == VMX_EXIT_RDMSR) ? "rdmsr" : "wrmsr", VBOXSTRICTRC_VAL(rc)));
4414	break;
4415	}
4416
4417	case VMX_EXIT_MOV_CRX: /* 28 Control-register accesses. */
4418	{
4419	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExit2Sub2, y2);
4420
4421	switch (VMX_EXIT_QUALIFICATION_CRX_ACCESS(exitQualification))
4422	{
4423	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_WRITE:
4424	{
4425	Log2(("VMX: %RGv mov cr%d, x\n", (RTGCPTR)pCtx->rip, VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification)));
4426	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxWrite[VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification)]);
4427	rc = EMInterpretCRxWrite(pVM, pVCpu, CPUMCTX2CORE(pCtx),
4428	VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification),
4429	VMX_EXIT_QUALIFICATION_CRX_GENREG(exitQualification));
4430	switch (VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification))
4431	{
4432	case 0:
4433	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0 \| HM_CHANGED_GUEST_CR3;
4434	break;
4435	case 2:
4436	break;
4437	case 3:
4438	Assert(!pVM->hm.s.fNestedPaging \|\| !CPUMIsGuestInPagedProtectedModeEx(pCtx));
4439	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR3;
4440	break;
4441	case 4:
4442	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR4;
4443	break;
4444	case 8:
4445	/* CR8 contains the APIC TPR */
4446	Assert(!(pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1
4447	& VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW));
4448	break;
4449
4450	default:
4451	AssertFailed();
4452	break;
4453	}
4454	break;
4455	}
4456
4457	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ:
4458	{
4459	Log2(("VMX: mov x, crx\n"));
4460	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxRead[VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification)]);
4461
4462	Assert( !pVM->hm.s.fNestedPaging
4463	\|\| !CPUMIsGuestInPagedProtectedModeEx(pCtx)
4464	\|\| VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification) != DISCREG_CR3);
4465
4466	/* CR8 reads only cause an exit when the TPR shadow feature isn't present. */
4467	Assert( VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification) != 8
4468	\|\| !(pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW));
4469
4470	rc = EMInterpretCRxRead(pVM, pVCpu, CPUMCTX2CORE(pCtx),
4471	VMX_EXIT_QUALIFICATION_CRX_GENREG(exitQualification),
4472	VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification));
4473	break;
4474	}
4475
4476	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_CLTS:
4477	{
4478	Log2(("VMX: clts\n"));
4479	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
4480	rc = EMInterpretCLTS(pVM, pVCpu);
4481	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
4482	break;
4483	}
4484
4485	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_LMSW:
4486	{
4487	Log2(("VMX: lmsw %x\n", VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(exitQualification)));
4488	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
4489	rc = EMInterpretLMSW(pVM, pVCpu, CPUMCTX2CORE(pCtx), VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(exitQualification));
4490	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
4491	break;
4492	}
4493	}
4494
4495	/* Update EIP if no error occurred. */
4496	if (RT_SUCCESS(rc))
4497	pCtx->rip += cbInstr;
4498
4499	if (rc == VINF_SUCCESS)
4500	{
4501	/* Only resume if successful. */
4502	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2Sub2, y2);
4503	goto ResumeExecution;
4504	}
4505	Assert(rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_PGM_SYNC_CR3);
4506	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2Sub2, y2);
4507	break;
4508	}
4509
4510	case VMX_EXIT_MOV_DRX: /* 29 Debug-register accesses. */
4511	{
4512	if ( !DBGFIsStepping(pVCpu)
4513	&& !CPUMIsHyperDebugStateActive(pVCpu))
4514	{
4515	/* Disable DRx move intercepts. */
4516	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT;
4517	rc2 = VMXWriteVmcs(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls);
4518	AssertRC(rc2);
4519
4520	/* Save the host and load the guest debug state. */
4521	rc2 = CPUMR0LoadGuestDebugState(pVM, pVCpu, pCtx, true /* include DR6 */);
4522	AssertRC(rc2);
4523
4524	#ifdef LOG_ENABLED
4525	if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(exitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
4526	{
4527	Log(("VMX_EXIT_MOV_DRX: write DR%d genreg %d\n", VMX_EXIT_QUALIFICATION_DRX_REGISTER(exitQualification),
4528	VMX_EXIT_QUALIFICATION_DRX_GENREG(exitQualification)));
4529	}
4530	else
4531	Log(("VMX_EXIT_MOV_DRX: read DR%d\n", VMX_EXIT_QUALIFICATION_DRX_REGISTER(exitQualification)));
4532	#endif
4533
4534	#ifdef VBOX_WITH_STATISTICS
4535	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
4536	if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(exitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
4537	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
4538	else
4539	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
4540	#endif
4541
4542	goto ResumeExecution;
4543	}
4544
4545	/** @todo clear VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT after the first
4546	* time and restore DRx registers afterwards */
4547	if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(exitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
4548	{
4549	Log2(("VMX: mov DRx%d, genreg%d\n", VMX_EXIT_QUALIFICATION_DRX_REGISTER(exitQualification),
4550	VMX_EXIT_QUALIFICATION_DRX_GENREG(exitQualification)));
4551	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
4552	rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pCtx),
4553	VMX_EXIT_QUALIFICATION_DRX_REGISTER(exitQualification),
4554	VMX_EXIT_QUALIFICATION_DRX_GENREG(exitQualification));
4555	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_DEBUG;
4556	Log2(("DR7=%08x\n", pCtx->dr[7]));
4557	}
4558	else
4559	{
4560	Log2(("VMX: mov x, DRx\n"));
4561	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
4562	rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pCtx),
4563	VMX_EXIT_QUALIFICATION_DRX_GENREG(exitQualification),
4564	VMX_EXIT_QUALIFICATION_DRX_REGISTER(exitQualification));
4565	}
4566	/* Update EIP if no error occurred. */
4567	if (RT_SUCCESS(rc))
4568	pCtx->rip += cbInstr;
4569
4570	if (rc == VINF_SUCCESS)
4571	{
4572	/* Only resume if successful. */
4573	goto ResumeExecution;
4574	}
4575	Assert(rc == VERR_EM_INTERPRETER);
4576	break;
4577	}
4578
4579	/* Note: We'll get a #GP if the IO instruction isn't allowed (IOPL or TSS bitmap); no need to double check. */
4580	case VMX_EXIT_IO_INSTR: /* 30 I/O instruction. */
4581	{
4582	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExit2Sub1, y1);
4583	uint32_t uPort;
4584	uint32_t uIOWidth = VMX_EXIT_QUALIFICATION_IO_WIDTH(exitQualification);
4585	bool fIOWrite = (VMX_EXIT_QUALIFICATION_IO_DIRECTION(exitQualification) == VMX_EXIT_QUALIFICATION_IO_DIRECTION_OUT);
4586
4587	/** @todo necessary to make the distinction? */
4588	if (VMX_EXIT_QUALIFICATION_IO_ENCODING(exitQualification) == VMX_EXIT_QUALIFICATION_IO_ENCODING_DX)
4589	uPort = pCtx->edx & 0xffff;
4590	else
4591	uPort = VMX_EXIT_QUALIFICATION_IO_PORT(exitQualification); /* Immediate encoding. */
4592
4593	if (RT_UNLIKELY(uIOWidth == 2 \|\| uIOWidth >= 4)) /* paranoia */
4594	{
4595	rc = fIOWrite ? VINF_IOM_R3_IOPORT_WRITE : VINF_IOM_R3_IOPORT_READ;
4596	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2Sub1, y1);
4597	break;
4598	}
4599
4600	uint32_t cbSize = g_aIOSize[uIOWidth];
4601	if (VMX_EXIT_QUALIFICATION_IO_STRING(exitQualification))
4602	{
4603	/* ins/outs */
4604	PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
4605
4606	/* Disassemble manually to deal with segment prefixes. */
4607	/** @todo VMX_VMCS_RO_EXIT_GUEST_LINEAR_ADDR contains the flat pointer
4608	* operand of the instruction. */
4609	/** @todo VMX_VMCS32_RO_EXIT_INSTR_INFO also contains segment prefix info. */
4610	rc2 = EMInterpretDisasCurrent(pVM, pVCpu, pDis, NULL);
4611	if (RT_SUCCESS(rc))
4612	{
4613	if (fIOWrite)
4614	{
4615	Log2(("IOMInterpretOUTSEx %RGv %x size=%d\n", (RTGCPTR)pCtx->rip, uPort, cbSize));
4616	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringWrite);
4617	rc = IOMInterpretOUTSEx(pVM, pVCpu, CPUMCTX2CORE(pCtx), uPort, pDis->fPrefix,
4618	(DISCPUMODE)pDis->uAddrMode, cbSize);
4619	}
4620	else
4621	{
4622	Log2(("IOMInterpretINSEx %RGv %x size=%d\n", (RTGCPTR)pCtx->rip, uPort, cbSize));
4623	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringRead);
4624	rc = IOMInterpretINSEx(pVM, pVCpu, CPUMCTX2CORE(pCtx), uPort, pDis->fPrefix,
4625	(DISCPUMODE)pDis->uAddrMode, cbSize);
4626	}
4627	}
4628	else
4629	rc = VINF_EM_RAW_EMULATE_INSTR;
4630	}
4631	else
4632	{
4633	/* Normal in/out */
4634	uint32_t uAndVal = g_aIOOpAnd[uIOWidth];
4635
4636	Assert(!VMX_EXIT_QUALIFICATION_IO_REP(exitQualification));
4637
4638	if (fIOWrite)
4639	{
4640	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
4641	rc = IOMIOPortWrite(pVM, pVCpu, uPort, pCtx->eax & uAndVal, cbSize);
4642	if (rc == VINF_IOM_R3_IOPORT_WRITE)
4643	HMR0SavePendingIOPortWrite(pVCpu, pCtx->rip, pCtx->rip + cbInstr, uPort, uAndVal, cbSize);
4644	}
4645	else
4646	{
4647	uint32_t u32Val = 0;
4648
4649	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
4650	rc = IOMIOPortRead(pVM, pVCpu, uPort, &u32Val, cbSize);
4651	if (IOM_SUCCESS(rc))
4652	{
4653	/* Write back to the EAX register. */
4654	pCtx->eax = (pCtx->eax & ~uAndVal) \| (u32Val & uAndVal);
4655	}
4656	else
4657	if (rc == VINF_IOM_R3_IOPORT_READ)
4658	HMR0SavePendingIOPortRead(pVCpu, pCtx->rip, pCtx->rip + cbInstr, uPort, uAndVal, cbSize);
4659	}
4660	}
4661
4662	/*
4663	* Handled the I/O return codes.
4664	* (The unhandled cases end up with rc == VINF_EM_RAW_EMULATE_INSTR.)
4665	*/
4666	if (IOM_SUCCESS(rc))
4667	{
4668	/* Update EIP and continue execution. */
4669	pCtx->rip += cbInstr;
4670	if (RT_LIKELY(rc == VINF_SUCCESS))
4671	{
4672	/* If any IO breakpoints are armed, then we should check if a debug trap needs to be generated. */
4673	if (pCtx->dr[7] & X86_DR7_ENABLED_MASK)
4674	{
4675	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
4676	for (unsigned i = 0; i < 4; i++)
4677	{
4678	unsigned uBPLen = g_aIOSize[X86_DR7_GET_LEN(pCtx->dr[7], i)];
4679
4680	if ( (uPort >= pCtx->dr[i] && uPort < pCtx->dr[i] + uBPLen)
4681	&& (pCtx->dr[7] & (X86_DR7_L(i) \| X86_DR7_G(i)))
4682	&& (pCtx->dr[7] & X86_DR7_RW(i, X86_DR7_RW_IO)) == X86_DR7_RW(i, X86_DR7_RW_IO))
4683	{
4684	uint64_t uDR6;
4685
4686	Assert(CPUMIsGuestDebugStateActive(pVCpu));
4687
4688	uDR6 = ASMGetDR6();
4689
4690	/* Clear all breakpoint status flags and set the one we just hit. */
4691	uDR6 &= ~(X86_DR6_B0\|X86_DR6_B1\|X86_DR6_B2\|X86_DR6_B3);
4692	uDR6 \|= (uint64_t)RT_BIT(i);
4693
4694	/*
4695	* Note: AMD64 Architecture Programmer's Manual 13.1:
4696	* Bits 15:13 of the DR6 register is never cleared by the processor and must
4697	* be cleared by software after the contents have been read.
4698	*/
4699	ASMSetDR6(uDR6);
4700
4701	/* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
4702	pCtx->dr[7] &= ~X86_DR7_GD;
4703
4704	/* Paranoia. */
4705	pCtx->dr[7] &= 0xffffffff; /* upper 32 bits reserved */
4706	pCtx->dr[7] &= ~(RT_BIT(11) \| RT_BIT(12) \| RT_BIT(14) \| RT_BIT(15)); /* must be zero */
4707	pCtx->dr[7] \|= 0x400; /* must be one */
4708
4709	/* Resync DR7 */
4710	rc2 = VMXWriteVmcs64(VMX_VMCS_GUEST_DR7, pCtx->dr[7]);
4711	AssertRC(rc2);
4712
4713	/* Construct inject info. */
4714	intInfo = X86_XCPT_DB;
4715	intInfo \|= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
4716	intInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
4717
4718	Log(("Inject IO debug trap at %RGv\n", (RTGCPTR)pCtx->rip));
4719	rc2 = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo),
4720	0 /* cbInstr /, 0 / errCode */);
4721	AssertRC(rc2);
4722
4723	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2Sub1, y1);
4724	goto ResumeExecution;
4725	}
4726	}
4727	}
4728	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2Sub1, y1);
4729	goto ResumeExecution;
4730	}
4731	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2Sub1, y1);
4732	break;
4733	}
4734
4735	#ifdef VBOX_STRICT
4736	if (rc == VINF_IOM_R3_IOPORT_READ)
4737	Assert(!fIOWrite);
4738	else if (rc == VINF_IOM_R3_IOPORT_WRITE)
4739	Assert(fIOWrite);
4740	else
4741	{
4742	AssertMsg( RT_FAILURE(rc)
4743	\|\| rc == VINF_EM_RAW_EMULATE_INSTR
4744	\|\| rc == VINF_EM_RAW_GUEST_TRAP
4745	\|\| rc == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", VBOXSTRICTRC_VAL(rc)));
4746	}
4747	#endif
4748	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2Sub1, y1);
4749	break;
4750	}
4751
4752	case VMX_EXIT_TPR_BELOW_THRESHOLD: /* 43 TPR below threshold. Guest software executed MOV to CR8. */
4753	LogFlow(("VMX_EXIT_TPR_BELOW_THRESHOLD\n"));
4754	/* RIP is already set to the next instruction and the TPR has been synced back. Just resume. */
4755	goto ResumeExecution;
4756
4757	case VMX_EXIT_APIC_ACCESS: /* 44 APIC access. Guest software attempted to access memory at a physical address
4758	on the APIC-access page. */
4759	{
4760	LogFlow(("VMX_EXIT_APIC_ACCESS\n"));
4761	unsigned uAccessType = VMX_EXIT_QUALIFICATION_APIC_ACCESS_TYPE(exitQualification);
4762
4763	switch (uAccessType)
4764	{
4765	case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
4766	case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
4767	{
4768	RTGCPHYS GCPhys = pCtx->msrApicBase;
4769	GCPhys &= PAGE_BASE_GC_MASK;
4770	GCPhys += VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(exitQualification);
4771
4772	LogFlow(("Apic access at %RGp\n", GCPhys));
4773	rc = IOMMMIOPhysHandler(pVM, pVCpu, (uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ) ? 0 : X86_TRAP_PF_RW,
4774	CPUMCTX2CORE(pCtx), GCPhys);
4775	if (rc == VINF_SUCCESS)
4776	goto ResumeExecution; /* rip already updated */
4777	break;
4778	}
4779
4780	default:
4781	rc = VINF_EM_RAW_EMULATE_INSTR;
4782	break;
4783	}
4784	break;
4785	}
4786
4787	case VMX_EXIT_PREEMPTION_TIMER: /* 52 VMX-preemption timer expired. The preemption timer counted down to zero. */
4788	if (!TMTimerPollBool(pVM, pVCpu))
4789	goto ResumeExecution;
4790	rc = VINF_EM_RAW_TIMER_PENDING;
4791	break;
4792
4793	default:
4794	/* The rest is handled after syncing the entire CPU state. */
4795	break;
4796	}
4797
4798
4799	/*
4800	* Note: The guest state is not entirely synced back at this stage!
4801	*/
4802
4803	/* Investigate why there was a VM-exit. (part 2) */
4804	switch (exitReason)
4805	{
4806	case VMX_EXIT_XCPT_NMI: /* 0 Exception or non-maskable interrupt (NMI). */
4807	case VMX_EXIT_EXT_INT: /* 1 External interrupt. */
4808	case VMX_EXIT_EPT_VIOLATION:
4809	case VMX_EXIT_EPT_MISCONFIG: /* 49 EPT misconfig is used by the PGM/MMIO optimizations. */
4810	case VMX_EXIT_PREEMPTION_TIMER: /* 52 VMX-preemption timer expired. The preemption timer counted down to zero. */
4811	/* Already handled above. */
4812	break;
4813
4814	case VMX_EXIT_TRIPLE_FAULT: /* 2 Triple fault. */
4815	rc = VINF_EM_RESET; /* Triple fault equals a reset. */
4816	break;
4817
4818	case VMX_EXIT_INIT_SIGNAL: /* 3 INIT signal. */
4819	case VMX_EXIT_SIPI: /* 4 Start-up IPI (SIPI). */
4820	rc = VINF_EM_RAW_INTERRUPT;
4821	AssertFailed(); /* Can't happen. Yet. */
4822	break;
4823
4824	case VMX_EXIT_IO_SMI: /* 5 I/O system-management interrupt (SMI). */
4825	case VMX_EXIT_SMI: /* 6 Other SMI. */
4826	rc = VINF_EM_RAW_INTERRUPT;
4827	AssertFailed(); /* Can't happen afaik. */
4828	break;
4829
4830	case VMX_EXIT_TASK_SWITCH: /* 9 Task switch: too complicated to emulate, so fall back to the recompiler */
4831	Log(("VMX_EXIT_TASK_SWITCH: exit=%RX64\n", exitQualification));
4832	if ( (VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE(exitQualification) == VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE_IDT)
4833	&& pVCpu->hm.s.Event.fPending)
4834	{
4835	/* Caused by an injected interrupt. */
4836	pVCpu->hm.s.Event.fPending = false;
4837
4838	Log(("VMX_EXIT_TASK_SWITCH: reassert trap %d\n", VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVCpu->hm.s.Event.u64IntrInfo)));
4839	Assert(!VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntrInfo));
4840	//@todo: Why do we assume this had to be a hardware interrupt? What about software interrupts or exceptions?
4841	rc2 = TRPMAssertTrap(pVCpu, VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVCpu->hm.s.Event.u64IntrInfo), TRPM_HARDWARE_INT);
4842	AssertRC(rc2);
4843	}
4844	/* else Exceptions and software interrupts can just be restarted. */
4845	rc = VERR_EM_INTERPRETER;
4846	break;
4847
4848	case VMX_EXIT_HLT: /* 12 Guest software attempted to execute HLT. */
4849	/* Check if external interrupts are pending; if so, don't switch back. */
4850	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
4851	pCtx->rip++; /* skip hlt */
4852	if (EMShouldContinueAfterHalt(pVCpu, pCtx))
4853	goto ResumeExecution;
4854
4855	rc = VINF_EM_HALT;
4856	break;
4857
4858	case VMX_EXIT_MWAIT: /* 36 Guest software executed MWAIT. */
4859	Log2(("VMX: mwait\n"));
4860	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMwait);
4861	rc = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pCtx));
4862	if ( rc == VINF_EM_HALT
4863	\|\| rc == VINF_SUCCESS)
4864	{
4865	/* Update EIP and continue execution. */
4866	pCtx->rip += cbInstr;
4867
4868	/* Check if external interrupts are pending; if so, don't switch back. */
4869	if ( rc == VINF_SUCCESS
4870	\|\| ( rc == VINF_EM_HALT
4871	&& EMShouldContinueAfterHalt(pVCpu, pCtx))
4872	)
4873	goto ResumeExecution;
4874	}
4875	AssertMsg(rc == VERR_EM_INTERPRETER \|\| rc == VINF_EM_HALT, ("EMU: mwait failed with %Rrc\n", VBOXSTRICTRC_VAL(rc)));
4876	break;
4877
4878	case VMX_EXIT_RSM: /* 17 Guest software attempted to execute RSM in SMM. */
4879	AssertFailed(); /* can't happen. */
4880	rc = VERR_EM_INTERPRETER;
4881	break;
4882
4883	case VMX_EXIT_MTF: /* 37 Exit due to Monitor Trap Flag. */
4884	LogFlow(("VMX_EXIT_MTF at %RGv\n", (RTGCPTR)pCtx->rip));
4885	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_TRAP_FLAG;
4886	rc2 = VMXWriteVmcs(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls);
4887	AssertRC(rc2);
4888	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMtf);
4889	#if 0
4890	DBGFDoneStepping(pVCpu);
4891	#endif
4892	rc = VINF_EM_DBG_STOP;
4893	break;
4894
4895	case VMX_EXIT_VMCALL: /* 18 Guest software executed VMCALL. */
4896	case VMX_EXIT_VMCLEAR: /* 19 Guest software executed VMCLEAR. */
4897	case VMX_EXIT_VMLAUNCH: /* 20 Guest software executed VMLAUNCH. */
4898	case VMX_EXIT_VMPTRLD: /* 21 Guest software executed VMPTRLD. */
4899	case VMX_EXIT_VMPTRST: /* 22 Guest software executed VMPTRST. */
4900	case VMX_EXIT_VMREAD: /* 23 Guest software executed VMREAD. */
4901	case VMX_EXIT_VMRESUME: /* 24 Guest software executed VMRESUME. */
4902	case VMX_EXIT_VMWRITE: /* 25 Guest software executed VMWRITE. */
4903	case VMX_EXIT_VMXOFF: /* 26 Guest software executed VMXOFF. */
4904	case VMX_EXIT_VMXON: /* 27 Guest software executed VMXON. */
4905	/** @todo inject #UD immediately */
4906	rc = VERR_EM_INTERPRETER;
4907	break;
4908
4909	case VMX_EXIT_CPUID: /* 10 Guest software attempted to execute CPUID. */
4910	case VMX_EXIT_RDTSC: /* 16 Guest software attempted to execute RDTSC. */
4911	case VMX_EXIT_INVLPG: /* 14 Guest software attempted to execute INVLPG. */
4912	case VMX_EXIT_MOV_CRX: /* 28 Control-register accesses. */
4913	case VMX_EXIT_MOV_DRX: /* 29 Debug-register accesses. */
4914	case VMX_EXIT_IO_INSTR: /* 30 I/O instruction. */
4915	case VMX_EXIT_RDPMC: /* 15 Guest software attempted to execute RDPMC. */
4916	case VMX_EXIT_RDTSCP: /* 51 Guest software attempted to execute RDTSCP. */
4917	/* already handled above */
4918	AssertMsg( rc == VINF_PGM_CHANGE_MODE
4919	\|\| rc == VINF_EM_RAW_INTERRUPT
4920	\|\| rc == VERR_EM_INTERPRETER
4921	\|\| rc == VINF_EM_RAW_EMULATE_INSTR
4922	\|\| rc == VINF_PGM_SYNC_CR3
4923	\|\| rc == VINF_IOM_R3_IOPORT_READ
4924	\|\| rc == VINF_IOM_R3_IOPORT_WRITE
4925	\|\| rc == VINF_EM_RAW_GUEST_TRAP
4926	\|\| rc == VINF_TRPM_XCPT_DISPATCHED
4927	\|\| rc == VINF_EM_RESCHEDULE_REM,
4928	("rc = %d\n", VBOXSTRICTRC_VAL(rc)));
4929	break;
4930
4931	case VMX_EXIT_TPR_BELOW_THRESHOLD: /* 43 TPR below threshold. Guest software executed MOV to CR8. */
4932	case VMX_EXIT_RDMSR: /* 31 RDMSR. Guest software attempted to execute RDMSR. */
4933	case VMX_EXIT_WRMSR: /* 32 WRMSR. Guest software attempted to execute WRMSR. */
4934	case VMX_EXIT_PAUSE: /* 40 Guest software attempted to execute PAUSE. */
4935	case VMX_EXIT_MONITOR: /* 39 Guest software attempted to execute MONITOR. */
4936	case VMX_EXIT_APIC_ACCESS: /* 44 APIC access. Guest software attempted to access memory at a physical address
4937	on the APIC-access page. */
4938	{
4939	/*
4940	* If we decided to emulate them here, then we must sync the MSRs that could have been changed (sysenter, FS/GS base)
4941	*/
4942	rc = VERR_EM_INTERPRETER;
4943	break;
4944	}
4945
4946	case VMX_EXIT_INT_WINDOW: /* 7 Interrupt window. */
4947	Assert(rc == VINF_EM_RAW_INTERRUPT);
4948	break;
4949
4950	case VMX_EXIT_ERR_INVALID_GUEST_STATE: /* 33 VM-entry failure due to invalid guest state. */
4951	{
4952	#ifdef VBOX_STRICT
4953	RTCCUINTREG val2 = 0;
4954
4955	Log(("VMX_EXIT_ERR_INVALID_GUEST_STATE\n"));
4956
4957	VMXReadVmcs(VMX_VMCS_GUEST_RIP, &val2);
4958	Log(("Old eip %RGv new %RGv\n", (RTGCPTR)pCtx->rip, (RTGCPTR)val2));
4959
4960	VMXReadVmcs(VMX_VMCS_GUEST_CR0, &val2);
4961	Log(("VMX_VMCS_GUEST_CR0 %RX64\n", (uint64_t)val2));
4962
4963	VMXReadVmcs(VMX_VMCS_GUEST_CR3, &val2);
4964	Log(("VMX_VMCS_GUEST_CR3 %RX64\n", (uint64_t)val2));
4965
4966	VMXReadVmcs(VMX_VMCS_GUEST_CR4, &val2);
4967	Log(("VMX_VMCS_GUEST_CR4 %RX64\n", (uint64_t)val2));
4968
4969	VMXReadVmcs(VMX_VMCS_GUEST_RFLAGS, &val2);
4970	Log(("VMX_VMCS_GUEST_RFLAGS %08x\n", val2));
4971
4972	VMX_LOG_SELREG(CS, "CS", val2);
4973	VMX_LOG_SELREG(DS, "DS", val2);
4974	VMX_LOG_SELREG(ES, "ES", val2);
4975	VMX_LOG_SELREG(FS, "FS", val2);
4976	VMX_LOG_SELREG(GS, "GS", val2);
4977	VMX_LOG_SELREG(SS, "SS", val2);
4978	VMX_LOG_SELREG(TR, "TR", val2);
4979	VMX_LOG_SELREG(LDTR, "LDTR", val2);
4980
4981	VMXReadVmcs(VMX_VMCS_GUEST_GDTR_BASE, &val2);
4982	Log(("VMX_VMCS_GUEST_GDTR_BASE %RX64\n", (uint64_t)val2));
4983	VMXReadVmcs(VMX_VMCS_GUEST_IDTR_BASE, &val2);
4984	Log(("VMX_VMCS_GUEST_IDTR_BASE %RX64\n", (uint64_t)val2));
4985	#endif /* VBOX_STRICT */
4986	rc = VERR_VMX_INVALID_GUEST_STATE;
4987	break;
4988	}
4989
4990	case VMX_EXIT_ERR_MSR_LOAD: /* 34 VM-entry failure due to MSR loading. */
4991	case VMX_EXIT_ERR_MACHINE_CHECK: /* 41 VM-entry failure due to machine-check. */
4992	default:
4993	rc = VERR_VMX_UNEXPECTED_EXIT_CODE;
4994	AssertMsgFailed(("Unexpected exit code %d\n", exitReason)); /* Can't happen. */
4995	break;
4996
4997	}
4998
4999	end:
5000	/* We now going back to ring-3, so clear the action flag. */
5001	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
5002
5003	/*
5004	* Signal changes for the recompiler.
5005	*/
5006	CPUMSetChangedFlags(pVCpu,
5007	CPUM_CHANGED_SYSENTER_MSR
5008	\| CPUM_CHANGED_LDTR
5009	\| CPUM_CHANGED_GDTR
5010	\| CPUM_CHANGED_IDTR
5011	\| CPUM_CHANGED_TR
5012	\| CPUM_CHANGED_HIDDEN_SEL_REGS);
5013
5014	/*
5015	* If we executed vmlaunch/vmresume and an external IRQ was pending, then we don't have to do a full sync the next time.
5016	*/
5017	if ( exitReason == VMX_EXIT_EXT_INT
5018	&& !VMX_EXIT_INTERRUPTION_INFO_VALID(intInfo))
5019	{
5020	STAM_COUNTER_INC(&pVCpu->hm.s.StatPendingHostIrq);
5021	/* On the next entry we'll only sync the host context. */
5022	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_HOST_CONTEXT;
5023	}
5024	else
5025	{
5026	/* On the next entry we'll sync everything. */
5027	/** @todo we can do better than this */
5028	/* Not in the VINF_PGM_CHANGE_MODE though! */
5029	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_ALL;
5030	}
5031
5032	/* Translate into a less severe return code */
5033	if (rc == VERR_EM_INTERPRETER)
5034	rc = VINF_EM_RAW_EMULATE_INSTR;
5035	else if (rc == VERR_VMX_INVALID_VMCS_PTR)
5036	{
5037	/* Try to extract more information about what might have gone wrong here. */
5038	VMXGetActivateVMCS(&pVCpu->hm.s.vmx.lasterror.u64VMCSPhys);
5039	pVCpu->hm.s.vmx.lasterror.u32VMCSRevision = (uint32_t )pVCpu->hm.s.vmx.pvVmcs;
5040	pVCpu->hm.s.vmx.lasterror.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
5041	pVCpu->hm.s.vmx.lasterror.idCurrentCpu = RTMpCpuId();
5042	}
5043
5044	/* Just set the correct state here instead of trying to catch every goto above. */
5045	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED, VMCPUSTATE_STARTED_EXEC);
5046
5047	#ifdef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
5048	/* Restore interrupts if we exited after disabling them. */
5049	if (uOldEFlags != ~(RTCCUINTREG)0)
5050	ASMSetFlags(uOldEFlags);
5051	#endif
5052
5053	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
5054	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
5055	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
5056	Log2(("X"));
5057	return VBOXSTRICTRC_TODO(rc);
5058	}
5059
5060
5061	/**
5062	* Enters the VT-x session.
5063	*
5064	* @returns VBox status code.
5065	* @param pVM Pointer to the VM.
5066	* @param pVCpu Pointer to the VMCPU.
5067	* @param pCpu Pointer to the CPU info struct.
5068	*/
5069	VMMR0DECL(int) VMXR0Enter(PVM pVM, PVMCPU pVCpu, PHMGLOBLCPUINFO pCpu)
5070	{
5071	Assert(pVM->hm.s.vmx.fSupported);
5072	NOREF(pCpu);
5073
5074	unsigned cr4 = ASMGetCR4();
5075	if (!(cr4 & X86_CR4_VMXE))
5076	{
5077	AssertMsgFailed(("X86_CR4_VMXE should be set!\n"));
5078	return VERR_VMX_X86_CR4_VMXE_CLEARED;
5079	}
5080
5081	/* Activate the VMCS. */
5082	int rc = VMXActivateVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
5083	if (RT_FAILURE(rc))
5084	return rc;
5085
5086	pVCpu->hm.s.fResumeVM = false;
5087	return VINF_SUCCESS;
5088	}
5089
5090
5091	/**
5092	* Leaves the VT-x session.
5093	*
5094	* @returns VBox status code.
5095	* @param pVM Pointer to the VM.
5096	* @param pVCpu Pointer to the VMCPU.
5097	* @param pCtx Pointer to the guests CPU context.
5098	*/
5099	VMMR0DECL(int) VMXR0Leave(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
5100	{
5101	Assert(pVM->hm.s.vmx.fSupported);
5102
5103	#ifdef DEBUG
5104	if (CPUMIsHyperDebugStateActive(pVCpu))
5105	{
5106	CPUMR0LoadHostDebugState(pVM, pVCpu);
5107	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT);
5108	}
5109	else
5110	#endif
5111
5112	/*
5113	* Save the guest debug state if necessary.
5114	*/
5115	if (CPUMIsGuestDebugStateActive(pVCpu))
5116	{
5117	CPUMR0SaveGuestDebugState(pVM, pVCpu, pCtx, true /* save DR6 */);
5118
5119	/* Enable DRx move intercepts again. */
5120	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT;
5121	int rc = VMXWriteVmcs(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls);
5122	AssertRC(rc);
5123
5124	/* Resync the debug registers the next time. */
5125	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_DEBUG;
5126	}
5127	else
5128	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT);
5129
5130	/*
5131	* Clear VMCS, marking it inactive, clearing implementation-specific data and writing
5132	* VMCS data back to memory.
5133	*/
5134	int rc = VMXClearVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
5135	AssertRC(rc);
5136
5137	return VINF_SUCCESS;
5138	}
5139
5140
5141	/**
5142	* Flush the TLB using EPT.
5143	*
5144	* @returns VBox status code.
5145	* @param pVM Pointer to the VM.
5146	* @param pVCpu Pointer to the VMCPU.
5147	* @param enmFlush Type of flush.
5148	*/
5149	static void hmR0VmxFlushEPT(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_EPT enmFlush)
5150	{
5151	uint64_t descriptor[2];
5152
5153	LogFlow(("hmR0VmxFlushEPT %d\n", enmFlush));
5154	Assert(pVM->hm.s.fNestedPaging);
5155	descriptor[0] = pVCpu->hm.s.vmx.GCPhysEPTP;
5156	descriptor[1] = 0; /* MBZ. Intel spec. 33.3 VMX Instructions */
5157	int rc = VMXR0InvEPT(enmFlush, &descriptor[0]);
5158	AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %x %RGv failed with %d\n", enmFlush, pVCpu->hm.s.vmx.GCPhysEPTP, rc));
5159	#ifdef VBOX_WITH_STATISTICS
5160	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
5161	#endif
5162	}
5163
5164
5165	/**
5166	* Flush the TLB using VPID.
5167	*
5168	* @returns VBox status code.
5169	* @param pVM Pointer to the VM.
5170	* @param pVCpu Pointer to the VMCPU (can be NULL depending on @a
5171	* enmFlush).
5172	* @param enmFlush Type of flush.
5173	* @param GCPtr Virtual address of the page to flush (can be 0 depending
5174	* on @a enmFlush).
5175	*/
5176	static void hmR0VmxFlushVPID(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_VPID enmFlush, RTGCPTR GCPtr)
5177	{
5178	uint64_t descriptor[2];
5179
5180	Assert(pVM->hm.s.vmx.fVpid);
5181	if (enmFlush == VMX_FLUSH_VPID_ALL_CONTEXTS)
5182	{
5183	descriptor[0] = 0;
5184	descriptor[1] = 0;
5185	}
5186	else
5187	{
5188	AssertPtr(pVCpu);
5189	AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
5190	AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
5191	descriptor[0] = pVCpu->hm.s.uCurrentAsid;
5192	descriptor[1] = GCPtr;
5193	}
5194	int rc = VMXR0InvVPID(enmFlush, &descriptor[0]); NOREF(rc);
5195	AssertMsg(rc == VINF_SUCCESS,
5196	("VMXR0InvVPID %x %x %RGv failed with %d\n", enmFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
5197	#ifdef VBOX_WITH_STATISTICS
5198	if (pVCpu)
5199	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
5200	#endif
5201	}
5202
5203
5204	/**
5205	* Invalidates a guest page by guest virtual address. Only relevant for
5206	* EPT/VPID, otherwise there is nothing really to invalidate.
5207	*
5208	* @returns VBox status code.
5209	* @param pVM Pointer to the VM.
5210	* @param pVCpu Pointer to the VMCPU.
5211	* @param GCVirt Guest virtual address of the page to invalidate.
5212	*/
5213	VMMR0DECL(int) VMXR0InvalidatePage(PVM pVM, PVMCPU pVCpu, RTGCPTR GCVirt)
5214	{
5215	bool fFlushPending = VMCPU_FF_ISSET(pVCpu, VMCPU_FF_TLB_FLUSH);
5216
5217	Log2(("VMXR0InvalidatePage %RGv\n", GCVirt));
5218
5219	if (!fFlushPending)
5220	{
5221	/*
5222	* We must invalidate the guest TLB entry in either case, we cannot ignore it even for the EPT case
5223	* See @bugref{6043} and @bugref{6177}
5224	*
5225	* Set the VMCPU_FF_TLB_FLUSH force flag and flush before VMENTRY in hmR0VmxSetupTLB*() as this
5226	* function maybe called in a loop with individual addresses.
5227	*/
5228	if (pVM->hm.s.vmx.fVpid)
5229	{
5230	/* If we can flush just this page do it, otherwise flush as little as possible. */
5231	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
5232	hmR0VmxFlushVPID(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, GCVirt);
5233	else
5234	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
5235	}
5236	else if (pVM->hm.s.fNestedPaging)
5237	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
5238	}
5239
5240	return VINF_SUCCESS;
5241	}
5242
5243
5244	/**
5245	* Invalidates a guest page by physical address. Only relevant for EPT/VPID,
5246	* otherwise there is nothing really to invalidate.
5247	*
5248	* NOTE: Assumes the current instruction references this physical page though a virtual address!!
5249	*
5250	* @returns VBox status code.
5251	* @param pVM Pointer to the VM.
5252	* @param pVCpu Pointer to the VMCPU.
5253	* @param GCPhys Guest physical address of the page to invalidate.
5254	*/
5255	VMMR0DECL(int) VMXR0InvalidatePhysPage(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys)
5256	{
5257	LogFlow(("VMXR0InvalidatePhysPage %RGp\n", GCPhys));
5258
5259	/*
5260	* We cannot flush a page by guest-physical address. invvpid takes only a linear address
5261	* while invept only flushes by EPT not individual addresses. We update the force flag here
5262	* and flush before VMENTRY in hmR0VmxSetupTLB*(). This function might be called in a loop.
5263	*/
5264	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
5265	return VINF_SUCCESS;
5266	}
5267
5268
5269	/**
5270	* Report world switch error and dump some useful debug info.
5271	*
5272	* @param pVM Pointer to the VM.
5273	* @param pVCpu Pointer to the VMCPU.
5274	* @param rc Return code.
5275	* @param pCtx Pointer to the current guest CPU context (not updated).
5276	*/
5277	static void hmR0VmxReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, VBOXSTRICTRC rc, PCPUMCTX pCtx)
5278	{
5279	NOREF(pVM);
5280
5281	switch (VBOXSTRICTRC_VAL(rc))
5282	{
5283	case VERR_VMX_INVALID_VMXON_PTR:
5284	AssertFailed();
5285	break;
5286
5287	case VERR_VMX_UNABLE_TO_START_VM:
5288	{
5289	int rc2;
5290	RTCCUINTREG exitReason, instrError;
5291
5292	rc2 = VMXReadVmcs(VMX_VMCS32_RO_EXIT_REASON, &exitReason);
5293	rc2 \|= VMXReadVmcs(VMX_VMCS32_RO_VM_INSTR_ERROR, &instrError);
5294	AssertRC(rc2);
5295	if (rc2 == VINF_SUCCESS)
5296	{
5297	Log(("Unable to start/resume VM for reason: %x. Instruction error %x\n", (uint32_t)exitReason,
5298	(uint32_t)instrError));
5299	Log(("Current stack %08x\n", &rc2));
5300
5301	pVCpu->hm.s.vmx.lasterror.u32InstrError = instrError;
5302	pVCpu->hm.s.vmx.lasterror.u32ExitReason = exitReason;
5303
5304	#ifdef VBOX_STRICT
5305	RTGDTR gdtr;
5306	PCX86DESCHC pDesc;
5307	RTCCUINTREG val;
5308
5309	ASMGetGDTR(&gdtr);
5310
5311	VMXReadVmcs(VMX_VMCS_GUEST_RIP, &val);
5312	Log(("Old eip %RGv new %RGv\n", (RTGCPTR)pCtx->rip, (RTGCPTR)val));
5313	VMXReadVmcs(VMX_VMCS32_CTRL_PIN_EXEC_CONTROLS, &val);
5314	Log(("VMX_VMCS_CTRL_PIN_EXEC_CONTROLS %08x\n", val));
5315	VMXReadVmcs(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, &val);
5316	Log(("VMX_VMCS_CTRL_PROC_EXEC_CONTROLS %08x\n", val));
5317	VMXReadVmcs(VMX_VMCS32_CTRL_ENTRY_CONTROLS, &val);
5318	Log(("VMX_VMCS_CTRL_ENTRY_CONTROLS %08x\n", val));
5319	VMXReadVmcs(VMX_VMCS32_CTRL_EXIT_CONTROLS, &val);
5320	Log(("VMX_VMCS_CTRL_EXIT_CONTROLS %08x\n", val));
5321
5322	VMXReadVmcs(VMX_VMCS_HOST_CR0, &val);
5323	Log(("VMX_VMCS_HOST_CR0 %08x\n", val));
5324	VMXReadVmcs(VMX_VMCS_HOST_CR3, &val);
5325	Log(("VMX_VMCS_HOST_CR3 %08x\n", val));
5326	VMXReadVmcs(VMX_VMCS_HOST_CR4, &val);
5327	Log(("VMX_VMCS_HOST_CR4 %08x\n", val));
5328
5329	VMXReadVmcs(VMX_VMCS16_HOST_FIELD_CS, &val);
5330	Log(("VMX_VMCS_HOST_FIELD_CS %08x\n", val));
5331	VMXReadVmcs(VMX_VMCS_GUEST_RFLAGS, &val);
5332	Log(("VMX_VMCS_GUEST_RFLAGS %08x\n", val));
5333
5334	if (val < gdtr.cbGdt)
5335	{
5336	pDesc = (PCX86DESCHC)(gdtr.pGdt + (val & X86_SEL_MASK));
5337	HMR0DumpDescriptor(pDesc, val, "CS: ");
5338	}
5339
5340	VMXReadVmcs(VMX_VMCS16_HOST_FIELD_DS, &val);
5341	Log(("VMX_VMCS_HOST_FIELD_DS %08x\n", val));
5342	if (val < gdtr.cbGdt)
5343	{
5344	pDesc = (PCX86DESCHC)(gdtr.pGdt + (val & X86_SEL_MASK));
5345	HMR0DumpDescriptor(pDesc, val, "DS: ");
5346	}
5347
5348	VMXReadVmcs(VMX_VMCS16_HOST_FIELD_ES, &val);
5349	Log(("VMX_VMCS_HOST_FIELD_ES %08x\n", val));
5350	if (val < gdtr.cbGdt)
5351	{
5352	pDesc = (PCX86DESCHC)(gdtr.pGdt + (val & X86_SEL_MASK));
5353	HMR0DumpDescriptor(pDesc, val, "ES: ");
5354	}
5355
5356	VMXReadVmcs(VMX_VMCS16_HOST_FIELD_FS, &val);
5357	Log(("VMX_VMCS16_HOST_FIELD_FS %08x\n", val));
5358	if (val < gdtr.cbGdt)
5359	{
5360	pDesc = (PCX86DESCHC)(gdtr.pGdt + (val & X86_SEL_MASK));
5361	HMR0DumpDescriptor(pDesc, val, "FS: ");
5362	}
5363
5364	VMXReadVmcs(VMX_VMCS16_HOST_FIELD_GS, &val);
5365	Log(("VMX_VMCS16_HOST_FIELD_GS %08x\n", val));
5366	if (val < gdtr.cbGdt)
5367	{
5368	pDesc = (PCX86DESCHC)(gdtr.pGdt + (val & X86_SEL_MASK));
5369	HMR0DumpDescriptor(pDesc, val, "GS: ");
5370	}
5371
5372	VMXReadVmcs(VMX_VMCS16_HOST_FIELD_SS, &val);
5373	Log(("VMX_VMCS16_HOST_FIELD_SS %08x\n", val));
5374	if (val < gdtr.cbGdt)
5375	{
5376	pDesc = (PCX86DESCHC)(gdtr.pGdt + (val & X86_SEL_MASK));
5377	HMR0DumpDescriptor(pDesc, val, "SS: ");
5378	}
5379
5380	VMXReadVmcs(VMX_VMCS16_HOST_FIELD_TR, &val);
5381	Log(("VMX_VMCS16_HOST_FIELD_TR %08x\n", val));
5382	if (val < gdtr.cbGdt)
5383	{
5384	pDesc = (PCX86DESCHC)(gdtr.pGdt + (val & X86_SEL_MASK));
5385	HMR0DumpDescriptor(pDesc, val, "TR: ");
5386	}
5387
5388	VMXReadVmcs(VMX_VMCS_HOST_TR_BASE, &val);
5389	Log(("VMX_VMCS_HOST_TR_BASE %RHv\n", val));
5390	VMXReadVmcs(VMX_VMCS_HOST_GDTR_BASE, &val);
5391	Log(("VMX_VMCS_HOST_GDTR_BASE %RHv\n", val));
5392	VMXReadVmcs(VMX_VMCS_HOST_IDTR_BASE, &val);
5393	Log(("VMX_VMCS_HOST_IDTR_BASE %RHv\n", val));
5394	VMXReadVmcs(VMX_VMCS32_HOST_SYSENTER_CS, &val);
5395	Log(("VMX_VMCS_HOST_SYSENTER_CS %08x\n", val));
5396	VMXReadVmcs(VMX_VMCS_HOST_SYSENTER_EIP, &val);
5397	Log(("VMX_VMCS_HOST_SYSENTER_EIP %RHv\n", val));
5398	VMXReadVmcs(VMX_VMCS_HOST_SYSENTER_ESP, &val);
5399	Log(("VMX_VMCS_HOST_SYSENTER_ESP %RHv\n", val));
5400	VMXReadVmcs(VMX_VMCS_HOST_RSP, &val);
5401	Log(("VMX_VMCS_HOST_RSP %RHv\n", val));
5402	VMXReadVmcs(VMX_VMCS_HOST_RIP, &val);
5403	Log(("VMX_VMCS_HOST_RIP %RHv\n", val));
5404	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
5405	if (VMX_IS_64BIT_HOST_MODE())
5406	{
5407	Log(("MSR_K6_EFER = %RX64\n", ASMRdMsr(MSR_K6_EFER)));
5408	Log(("MSR_K6_STAR = %RX64\n", ASMRdMsr(MSR_K6_STAR)));
5409	Log(("MSR_K8_LSTAR = %RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
5410	Log(("MSR_K8_CSTAR = %RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
5411	Log(("MSR_K8_SF_MASK = %RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
5412	Log(("MSR_K8_KERNEL_GS_BASE = %RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
5413	}
5414	# endif
5415	#endif /* VBOX_STRICT */
5416	}
5417	break;
5418	}
5419
5420	default:
5421	/* impossible */
5422	AssertMsgFailed(("%Rrc (%#x)\n", VBOXSTRICTRC_VAL(rc), VBOXSTRICTRC_VAL(rc)));
5423	break;
5424	}
5425	}
5426
5427
5428	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
5429	/**
5430	* Prepares for and executes VMLAUNCH (64 bits guest mode).
5431	*
5432	* @returns VBox status code.
5433	* @param fResume Whether to vmlauch/vmresume.
5434	* @param pCtx Pointer to the guest CPU context.
5435	* @param pCache Pointer to the VMCS cache.
5436	* @param pVM Pointer to the VM.
5437	* @param pVCpu Pointer to the VMCPU.
5438	*/
5439	DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
5440	{
5441	uint32_t aParam[6];
5442	PHMGLOBLCPUINFO pCpu;
5443	RTHCPHYS HCPhysCpuPage;
5444	int rc;
5445
5446	pCpu = HMR0GetCurrentCpu();
5447	HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
5448
5449	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5450	pCache->uPos = 1;
5451	pCache->interPD = PGMGetInterPaeCR3(pVM);
5452	pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
5453	#endif
5454
5455	#ifdef DEBUG
5456	pCache->TestIn.HCPhysCpuPage= 0;
5457	pCache->TestIn.HCPhysVmcs = 0;
5458	pCache->TestIn.pCache = 0;
5459	pCache->TestOut.HCPhysVmcs = 0;
5460	pCache->TestOut.pCache = 0;
5461	pCache->TestOut.pCtx = 0;
5462	pCache->TestOut.eflags = 0;
5463	#endif
5464
5465	aParam[0] = (uint32_t)(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
5466	aParam[1] = (uint32_t)(HCPhysCpuPage >> 32); /* Param 1: VMXON physical address - Hi. */
5467	aParam[2] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
5468	aParam[3] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs >> 32); /* Param 2: VMCS physical address - Hi. */
5469	aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache);
5470	aParam[5] = 0;
5471
5472	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5473	pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
5474	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
5475	#endif
5476	rc = VMXR0Execute64BitsHandler(pVM, pVCpu, pCtx, pVM->hm.s.pfnVMXGCStartVM64, 6, &aParam[0]);
5477
5478	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5479	Assert((uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
5480	Assert(pCtx->dr[4] == 10);
5481	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
5482	#endif
5483
5484	#ifdef DEBUG
5485	AssertMsg(pCache->TestIn.HCPhysCpuPage== HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
5486	AssertMsg(pCache->TestIn.HCPhysVmcs == pVCpu->hm.s.vmx.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5487	pVCpu->hm.s.vmx.HCPhysVmcs));
5488	AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5489	pCache->TestOut.HCPhysVmcs));
5490	AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
5491	pCache->TestOut.pCache));
5492	AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache),
5493	("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache)));
5494	AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
5495	pCache->TestOut.pCtx));
5496	Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
5497	#endif
5498	return rc;
5499	}
5500
5501
5502	#ifdef VBOX_STRICT
5503	static bool hmR0VmxIsValidReadField(uint32_t idxField)
5504	{
5505	switch (idxField)
5506	{
5507	case VMX_VMCS_GUEST_RIP:
5508	case VMX_VMCS_GUEST_RSP:
5509	case VMX_VMCS_GUEST_RFLAGS:
5510	case VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE:
5511	case VMX_VMCS_CTRL_CR0_READ_SHADOW:
5512	case VMX_VMCS_GUEST_CR0:
5513	case VMX_VMCS_CTRL_CR4_READ_SHADOW:
5514	case VMX_VMCS_GUEST_CR4:
5515	case VMX_VMCS_GUEST_DR7:
5516	case VMX_VMCS32_GUEST_SYSENTER_CS:
5517	case VMX_VMCS_GUEST_SYSENTER_EIP:
5518	case VMX_VMCS_GUEST_SYSENTER_ESP:
5519	case VMX_VMCS32_GUEST_GDTR_LIMIT:
5520	case VMX_VMCS_GUEST_GDTR_BASE:
5521	case VMX_VMCS32_GUEST_IDTR_LIMIT:
5522	case VMX_VMCS_GUEST_IDTR_BASE:
5523	case VMX_VMCS16_GUEST_FIELD_CS:
5524	case VMX_VMCS32_GUEST_CS_LIMIT:
5525	case VMX_VMCS_GUEST_CS_BASE:
5526	case VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS:
5527	case VMX_VMCS16_GUEST_FIELD_DS:
5528	case VMX_VMCS32_GUEST_DS_LIMIT:
5529	case VMX_VMCS_GUEST_DS_BASE:
5530	case VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS:
5531	case VMX_VMCS16_GUEST_FIELD_ES:
5532	case VMX_VMCS32_GUEST_ES_LIMIT:
5533	case VMX_VMCS_GUEST_ES_BASE:
5534	case VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS:
5535	case VMX_VMCS16_GUEST_FIELD_FS:
5536	case VMX_VMCS32_GUEST_FS_LIMIT:
5537	case VMX_VMCS_GUEST_FS_BASE:
5538	case VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS:
5539	case VMX_VMCS16_GUEST_FIELD_GS:
5540	case VMX_VMCS32_GUEST_GS_LIMIT:
5541	case VMX_VMCS_GUEST_GS_BASE:
5542	case VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS:
5543	case VMX_VMCS16_GUEST_FIELD_SS:
5544	case VMX_VMCS32_GUEST_SS_LIMIT:
5545	case VMX_VMCS_GUEST_SS_BASE:
5546	case VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS:
5547	case VMX_VMCS16_GUEST_FIELD_LDTR:
5548	case VMX_VMCS32_GUEST_LDTR_LIMIT:
5549	case VMX_VMCS_GUEST_LDTR_BASE:
5550	case VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS:
5551	case VMX_VMCS16_GUEST_FIELD_TR:
5552	case VMX_VMCS32_GUEST_TR_LIMIT:
5553	case VMX_VMCS_GUEST_TR_BASE:
5554	case VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS:
5555	case VMX_VMCS32_RO_EXIT_REASON:
5556	case VMX_VMCS32_RO_VM_INSTR_ERROR:
5557	case VMX_VMCS32_RO_EXIT_INSTR_LENGTH:
5558	case VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE:
5559	case VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO:
5560	case VMX_VMCS32_RO_EXIT_INSTR_INFO:
5561	case VMX_VMCS_RO_EXIT_QUALIFICATION:
5562	case VMX_VMCS32_RO_IDT_INFO:
5563	case VMX_VMCS32_RO_IDT_ERROR_CODE:
5564	case VMX_VMCS_GUEST_CR3:
5565	case VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL:
5566	return true;
5567	}
5568	return false;
5569	}
5570
5571
5572	static bool hmR0VmxIsValidWriteField(uint32_t idxField)
5573	{
5574	switch (idxField)
5575	{
5576	case VMX_VMCS_GUEST_LDTR_BASE:
5577	case VMX_VMCS_GUEST_TR_BASE:
5578	case VMX_VMCS_GUEST_GDTR_BASE:
5579	case VMX_VMCS_GUEST_IDTR_BASE:
5580	case VMX_VMCS_GUEST_SYSENTER_EIP:
5581	case VMX_VMCS_GUEST_SYSENTER_ESP:
5582	case VMX_VMCS_GUEST_CR0:
5583	case VMX_VMCS_GUEST_CR4:
5584	case VMX_VMCS_GUEST_CR3:
5585	case VMX_VMCS_GUEST_DR7:
5586	case VMX_VMCS_GUEST_RIP:
5587	case VMX_VMCS_GUEST_RSP:
5588	case VMX_VMCS_GUEST_CS_BASE:
5589	case VMX_VMCS_GUEST_DS_BASE:
5590	case VMX_VMCS_GUEST_ES_BASE:
5591	case VMX_VMCS_GUEST_FS_BASE:
5592	case VMX_VMCS_GUEST_GS_BASE:
5593	case VMX_VMCS_GUEST_SS_BASE:
5594	return true;
5595	}
5596	return false;
5597	}
5598	#endif /* VBOX_STRICT */
5599
5600
5601	/**
5602	* Executes the specified handler in 64-bit mode.
5603	*
5604	* @returns VBox status code.
5605	* @param pVM Pointer to the VM.
5606	* @param pVCpu Pointer to the VMCPU.
5607	* @param pCtx Pointer to the guest CPU context.
5608	* @param pfnHandler Pointer to the RC handler function.
5609	* @param cbParam Number of parameters.
5610	* @param paParam Array of 32-bit parameters.
5611	*/
5612	VMMR0DECL(int) VMXR0Execute64BitsHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, RTRCPTR pfnHandler, uint32_t cbParam,
5613	uint32_t *paParam)
5614	{
5615	int rc, rc2;
5616	PHMGLOBLCPUINFO pCpu;
5617	RTHCPHYS HCPhysCpuPage;
5618	RTHCUINTREG uOldEFlags;
5619
5620	AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
5621	Assert(pfnHandler);
5622	Assert(pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Write.aField));
5623	Assert(pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Read.aField));
5624
5625	#ifdef VBOX_STRICT
5626	for (unsigned i=0;i<pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries;i++)
5627	Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VMCSCache.Write.aField[i]));
5628
5629	for (unsigned i=0;i<pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries;i++)
5630	Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VMCSCache.Read.aField[i]));
5631	#endif
5632
5633	/* Disable interrupts. */
5634	uOldEFlags = ASMIntDisableFlags();
5635
5636	#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
5637	RTCPUID idHostCpu = RTMpCpuId();
5638	CPUMR0SetLApic(pVM, idHostCpu);
5639	#endif
5640
5641	pCpu = HMR0GetCurrentCpu();
5642	HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
5643
5644	/* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
5645	VMXClearVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
5646
5647	/* Leave VMX Root Mode. */
5648	VMXDisable();
5649
5650	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
5651
5652	CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
5653	CPUMSetHyperEIP(pVCpu, pfnHandler);
5654	for (int i=(int)cbParam-1;i>=0;i--)
5655	CPUMPushHyper(pVCpu, paParam[i]);
5656
5657	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
5658
5659	/* Call switcher. */
5660	rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_OFFSETOF(VM, aCpus[pVCpu->idCpu].cpum) - RT_OFFSETOF(VM, cpum));
5661	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
5662
5663	/* Make sure the VMX instructions don't cause #UD faults. */
5664	ASMSetCR4(ASMGetCR4() \| X86_CR4_VMXE);
5665
5666	/* Enter VMX Root Mode */
5667	rc2 = VMXEnable(HCPhysCpuPage);
5668	if (RT_FAILURE(rc2))
5669	{
5670	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
5671	ASMSetFlags(uOldEFlags);
5672	return VERR_VMX_VMXON_FAILED;
5673	}
5674
5675	rc2 = VMXActivateVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
5676	AssertRC(rc2);
5677	Assert(!(ASMGetFlags() & X86_EFL_IF));
5678	ASMSetFlags(uOldEFlags);
5679	return rc;
5680	}
5681	#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL) */
5682
5683
5684	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
5685	/**
5686	* Executes VMWRITE.
5687	*
5688	* @returns VBox status code
5689	* @param pVCpu Pointer to the VMCPU.
5690	* @param idxField VMCS field index.
5691	* @param u64Val 16, 32 or 64 bits value.
5692	*/
5693	VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5694	{
5695	int rc;
5696	switch (idxField)
5697	{
5698	case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
5699	case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
5700	case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
5701	case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
5702	case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
5703	case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
5704	case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
5705	case VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL:
5706	case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
5707	case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
5708	case VMX_VMCS64_GUEST_PDPTE0_FULL:
5709	case VMX_VMCS64_GUEST_PDPTE1_FULL:
5710	case VMX_VMCS64_GUEST_PDPTE2_FULL:
5711	case VMX_VMCS64_GUEST_PDPTE3_FULL:
5712	case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
5713	case VMX_VMCS64_GUEST_EFER_FULL:
5714	case VMX_VMCS64_CTRL_EPTP_FULL:
5715	/* These fields consist of two parts, which are both writable in 32 bits mode. */
5716	rc = VMXWriteVmcs32(idxField, u64Val);
5717	rc \|= VMXWriteVmcs32(idxField + 1, (uint32_t)(u64Val >> 32ULL));
5718	AssertRC(rc);
5719	return rc;
5720
5721	case VMX_VMCS_GUEST_LDTR_BASE:
5722	case VMX_VMCS_GUEST_TR_BASE:
5723	case VMX_VMCS_GUEST_GDTR_BASE:
5724	case VMX_VMCS_GUEST_IDTR_BASE:
5725	case VMX_VMCS_GUEST_SYSENTER_EIP:
5726	case VMX_VMCS_GUEST_SYSENTER_ESP:
5727	case VMX_VMCS_GUEST_CR0:
5728	case VMX_VMCS_GUEST_CR4:
5729	case VMX_VMCS_GUEST_CR3:
5730	case VMX_VMCS_GUEST_DR7:
5731	case VMX_VMCS_GUEST_RIP:
5732	case VMX_VMCS_GUEST_RSP:
5733	case VMX_VMCS_GUEST_CS_BASE:
5734	case VMX_VMCS_GUEST_DS_BASE:
5735	case VMX_VMCS_GUEST_ES_BASE:
5736	case VMX_VMCS_GUEST_FS_BASE:
5737	case VMX_VMCS_GUEST_GS_BASE:
5738	case VMX_VMCS_GUEST_SS_BASE:
5739	/* Queue a 64 bits value as we can't set it in 32 bits host mode. */
5740	if (u64Val >> 32ULL)
5741	rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
5742	else
5743	rc = VMXWriteVmcs32(idxField, (uint32_t)u64Val);
5744
5745	return rc;
5746
5747	default:
5748	AssertMsgFailed(("Unexpected field %x\n", idxField));
5749	return VERR_INVALID_PARAMETER;
5750	}
5751	}
5752
5753
5754	/**
5755	* Cache VMCS writes for running 64 bits guests on 32 bits hosts.
5756	*
5757	* @param pVCpu Pointer to the VMCPU.
5758	* @param idxField VMCS field index.
5759	* @param u64Val 16, 32 or 64 bits value.
5760	*/
5761	VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5762	{
5763	PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5764
5765	AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1,
5766	("entries=%x\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
5767
5768	/* Make sure there are no duplicates. */
5769	for (unsigned i = 0; i < pCache->Write.cValidEntries; i++)
5770	{
5771	if (pCache->Write.aField[i] == idxField)
5772	{
5773	pCache->Write.aFieldVal[i] = u64Val;
5774	return VINF_SUCCESS;
5775	}
5776	}
5777
5778	pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
5779	pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
5780	pCache->Write.cValidEntries++;
5781	return VINF_SUCCESS;
5782	}
5783
5784	#endif /* HC_ARCH_BITS == 32 && !VBOX_WITH_HYBRID_32BIT_KERNEL */
5785

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMR0/HWVMXR0.cpp@ 45474

Download in other formats: