HWVMXR0.cpp@ 45216

Last change on this file since 45216 was 45216, checked in by vboxsync, 12 years ago
VMM: HM bits.
Property svn:eol-style set to `native` Property svn:keywords set to `Id Revision`
File size: 228.3 KB

Line
1	/* $Id: HWVMXR0.cpp 45216 2013-03-27 19:08:28Z vboxsync $ */
2	/** @file
3	* HM VMX (VT-x) - Host Context Ring-0.
4	*/
5
6	/*
7	* Copyright (C) 2006-2013 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18
19	/*******************************************************************************
20	* Header Files *
21	*******************************************************************************/
22	#define LOG_GROUP LOG_GROUP_HM
23	#include <iprt/asm-amd64-x86.h>
24	#include <VBox/vmm/hm.h>
25	#include <VBox/vmm/pgm.h>
26	#include <VBox/vmm/dbgf.h>
27	#include <VBox/vmm/dbgftrace.h>
28	#include <VBox/vmm/selm.h>
29	#include <VBox/vmm/iom.h>
30	#ifdef VBOX_WITH_REM
31	# include <VBox/vmm/rem.h>
32	#endif
33	#include <VBox/vmm/tm.h>
34	#include "HMInternal.h"
35	#include <VBox/vmm/vm.h>
36	#include <VBox/vmm/pdmapi.h>
37	#include <VBox/err.h>
38	#include <VBox/log.h>
39	#include <iprt/assert.h>
40	#include <iprt/param.h>
41	#include <iprt/string.h>
42	#include <iprt/time.h>
43	#ifdef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
44	# include <iprt/thread.h>
45	#endif
46	#include <iprt/x86.h>
47	#include "HWVMXR0.h"
48
49	#include "dtrace/VBoxVMM.h"
50
51
52	/*******************************************************************************
53	* Defined Constants And Macros *
54	*******************************************************************************/
55	#if defined(RT_ARCH_AMD64)
56	# define VMX_IS_64BIT_HOST_MODE() (true)
57	#elif defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
58	# define VMX_IS_64BIT_HOST_MODE() (g_fVMXIs64bitHost != 0)
59	#else
60	# define VMX_IS_64BIT_HOST_MODE() (false)
61	#endif
62
63	# define VMX_WRITE_SELREG(REG, reg) \
64	do \
65	{ \
66	rc = VMXWriteVmcs(VMX_VMCS16_GUEST_FIELD_##REG, pCtx->reg.Sel); \
67	rc \|= VMXWriteVmcs(VMX_VMCS32_GUEST_##REG##_LIMIT, pCtx->reg.u32Limit); \
68	rc \|= VMXWriteVmcs64(VMX_VMCS_GUEST_##REG##_BASE, pCtx->reg.u64Base); \
69	if ((pCtx->eflags.u32 & X86_EFL_VM)) \
70	{ \
71	/* Must override this or else VT-x will fail with invalid guest state errors. */ \
72	/* DPL=3, present, code/data, r/w/accessed. */ \
73	/** @todo we shouldn't have to do this, if it is not 0xf3 it means we screwed up elsewhere (recompiler). */ \
74	/** @todo VT-x docs explicitly mentions 0xF3. Why not just val = 0xf3 ??. */ \
75	val = (pCtx->reg.Attr.u & ~0xFF) \| 0xF3; \
76	} \
77	else \
78	if ( CPUMIsGuestInRealModeEx(pCtx) \
79	&& !pVM->hm.s.vmx.fUnrestrictedGuest) \
80	{ \
81	/** @todo shouldn't the 'if' condition above check for 'pRealModeTSS' ? */ \
82	/* Must override this or else VT-x will fail with invalid guest state errors. */ \
83	/* DPL=3, present, code/data, r/w/accessed. */ \
84	val = 0xf3; \
85	} \
86	else \
87	if ( ( pCtx->reg.Sel \
88	\|\| !CPUMIsGuestInPagedProtectedModeEx(pCtx) \
89	\|\| (!pCtx->cs.Attr.n.u1DefBig && !CPUMIsGuestIn64BitCodeEx(pCtx)) \
90	) \
91	&& pCtx->reg.Attr.n.u1Present == 1) \
92	{ \
93	val = pCtx->reg.Attr.u \| X86_SEL_TYPE_ACCESSED; \
94	} \
95	else \
96	val = 0x10000; /* Invalid guest state error otherwise. (BIT(16) = Unusable) */ \
97	\
98	rc \|= VMXWriteVmcs(VMX_VMCS32_GUEST_##REG##_ACCESS_RIGHTS, val); \
99	} while (0)
100
101	# define VMX_READ_SELREG(REG, reg) \
102	do \
103	{ \
104	VMXReadCachedVmcs(VMX_VMCS16_GUEST_FIELD_##REG, &val); \
105	pCtx->reg.Sel = val; \
106	pCtx->reg.ValidSel = val; \
107	pCtx->reg.fFlags = CPUMSELREG_FLAGS_VALID; \
108	VMXReadCachedVmcs(VMX_VMCS32_GUEST_##REG##_LIMIT, &val); \
109	pCtx->reg.u32Limit = val; \
110	VMXReadCachedVmcs(VMX_VMCS_GUEST_##REG##_BASE, &val); \
111	pCtx->reg.u64Base = val; \
112	VMXReadCachedVmcs(VMX_VMCS32_GUEST_##REG##_ACCESS_RIGHTS, &val); \
113	pCtx->reg.Attr.u = val; \
114	} while (0)
115
116	/* Don't read from the cache in this macro; used only in case of failure where the cache is out of sync. */
117	# define VMX_LOG_SELREG(REG, szSelReg, val) \
118	do \
119	{ \
120	VMXReadVmcs(VMX_VMCS16_GUEST_FIELD_##REG, &(val)); \
121	Log(("%s Selector %x\n", szSelReg, (val))); \
122	VMXReadVmcs(VMX_VMCS32_GUEST_##REG##_LIMIT, &(val)); \
123	Log(("%s Limit %x\n", szSelReg, (val))); \
124	VMXReadVmcs(VMX_VMCS_GUEST_##REG##_BASE, &(val)); \
125	Log(("%s Base %RX64\n", szSelReg, (uint64_t)(val))); \
126	VMXReadVmcs(VMX_VMCS32_GUEST_##REG##_ACCESS_RIGHTS, &(val)); \
127	Log(("%s Attributes %x\n", szSelReg, (val))); \
128	} while (0)
129
130	#define VMXSetupCachedReadVmcs(pCache, idxField) \
131	{ \
132	Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
133	pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
134	pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
135	}
136	#define VMX_SETUP_SELREG(REG, pCache) \
137	{ \
138	VMXSetupCachedReadVmcs(pCache, VMX_VMCS16_GUEST_FIELD_##REG); \
139	VMXSetupCachedReadVmcs(pCache, VMX_VMCS32_GUEST_##REG##_LIMIT); \
140	VMXSetupCachedReadVmcs(pCache, VMX_VMCS_GUEST_##REG##_BASE); \
141	VMXSetupCachedReadVmcs(pCache, VMX_VMCS32_GUEST_##REG##_ACCESS_RIGHTS); \
142	}
143
144
145	/*******************************************************************************
146	* Global Variables *
147	*******************************************************************************/
148	/* IO operation lookup arrays. */
149	static uint32_t const g_aIOSize[4] = {1, 2, 0, 4};
150	static uint32_t const g_aIOOpAnd[4] = {0xff, 0xffff, 0, 0xffffffff};
151
152	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
153	/** See HMR0A.asm. */
154	extern "C" uint32_t g_fVMXIs64bitHost;
155	#endif
156
157
158	/*******************************************************************************
159	* Local Functions *
160	*******************************************************************************/
161	static DECLCALLBACK(void) hmR0VmxSetupTLBEPT(PVM pVM, PVMCPU pVCpu);
162	static DECLCALLBACK(void) hmR0VmxSetupTLBVPID(PVM pVM, PVMCPU pVCpu);
163	static DECLCALLBACK(void) hmR0VmxSetupTLBBoth(PVM pVM, PVMCPU pVCpu);
164	static DECLCALLBACK(void) hmR0VmxSetupTLBDummy(PVM pVM, PVMCPU pVCpu);
165	static void hmR0VmxFlushEPT(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_EPT enmFlush);
166	static void hmR0VmxFlushVPID(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_VPID enmFlush, RTGCPTR GCPtr);
167	static void hmR0VmxUpdateExceptionBitmap(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx);
168	static void hmR0VmxSetMSRPermission(PVMCPU pVCpu, unsigned ulMSR, bool fRead, bool fWrite);
169	static void hmR0VmxReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, VBOXSTRICTRC rc, PCPUMCTX pCtx);
170
171
172	/**
173	* Updates error from VMCS to HMCPU's lasterror record.
174	*
175	* @param pVM Pointer to the VM.
176	* @param pVCpu Pointer to the VMCPU.
177	* @param rc The error code.
178	*/
179	static void hmR0VmxCheckError(PVM pVM, PVMCPU pVCpu, int rc)
180	{
181	if (rc == VERR_VMX_GENERIC)
182	{
183	RTCCUINTREG instrError;
184
185	VMXReadVmcs(VMX_VMCS32_RO_VM_INSTR_ERROR, &instrError);
186	pVCpu->hm.s.vmx.lasterror.u32InstrError = instrError;
187	}
188	pVM->hm.s.lLastError = rc;
189	}
190
191
192	/**
193	* Sets up and activates VT-x on the current CPU.
194	*
195	* @returns VBox status code.
196	* @param pCpu Pointer to the CPU info struct.
197	* @param pVM Pointer to the VM. (can be NULL after a resume!!)
198	* @param pvCpuPage Pointer to the global CPU page.
199	* @param HCPhysCpuPage Physical address of the global CPU page.
200	* @param fEnabledByHost Set if SUPR0EnableVTx or similar was used to enable
201	* VT-x/AMD-V on the host.
202	*/
203	VMMR0DECL(int) VMXR0EnableCpu(PHMGLOBLCPUINFO pCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost)
204	{
205	if (!fEnabledByHost)
206	{
207	AssertReturn(HCPhysCpuPage != 0 && HCPhysCpuPage != NIL_RTHCPHYS, VERR_INVALID_PARAMETER);
208	AssertReturn(pvCpuPage, VERR_INVALID_PARAMETER);
209
210	if (pVM)
211	{
212	/* Set revision dword at the beginning of the VMXON structure. */
213	(uint32_t )pvCpuPage = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.msr.vmx_basic_info);
214	}
215
216	/** @todo we should unmap the two pages from the virtual address space in order to prevent accidental corruption.
217	* (which can have very bad consequences!!!)
218	*/
219
220	/** @todo r=bird: Why is this code different than the probing code earlier
221	* on? It just sets VMXE if needed and doesn't check that it isn't
222	* set. Mac OS X host_vmxoff may leave this set and we'll fail here
223	* and debug-assert in the calling code. This is what caused the
224	* "regression" after backing out the SUPR0EnableVTx code hours before
225	* 4.2.0GA (reboot fixed the issue). I've changed here to do the same
226	* as the init code. */
227	uint64_t uCr4 = ASMGetCR4();
228	if (!(uCr4 & X86_CR4_VMXE))
229	ASMSetCR4(ASMGetCR4() \| X86_CR4_VMXE); /* Make sure the VMX instructions don't cause #UD faults. */
230
231	/*
232	* Enter VM root mode.
233	*/
234	int rc = VMXEnable(HCPhysCpuPage);
235	if (RT_FAILURE(rc))
236	{
237	ASMSetCR4(uCr4);
238	return VERR_VMX_VMXON_FAILED;
239	}
240	}
241
242	/*
243	* Flush all VPIDs (in case we or any other hypervisor have been using VPIDs) so that
244	* we can avoid an explicit flush while using new VPIDs. We would still need to flush
245	* each time while reusing a VPID after hitting the MaxASID limit once.
246	*/
247	if ( pVM
248	&& pVM->hm.s.vmx.fVpid
249	&& (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS))
250	{
251	hmR0VmxFlushVPID(pVM, NULL /* pvCpu /, VMX_FLUSH_VPID_ALL_CONTEXTS, 0 / GCPtr */);
252	pCpu->fFlushAsidBeforeUse = false;
253	}
254	else
255	pCpu->fFlushAsidBeforeUse = true;
256
257	/*
258	* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}.
259	*/
260	++pCpu->cTlbFlushes;
261
262	return VINF_SUCCESS;
263	}
264
265
266	/**
267	* Deactivates VT-x on the current CPU.
268	*
269	* @returns VBox status code.
270	* @param pCpu Pointer to the CPU info struct.
271	* @param pvCpuPage Pointer to the global CPU page.
272	* @param HCPhysCpuPage Physical address of the global CPU page.
273	*/
274	VMMR0DECL(int) VMXR0DisableCpu(PHMGLOBLCPUINFO pCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
275	{
276	AssertReturn(HCPhysCpuPage != 0 && HCPhysCpuPage != NIL_RTHCPHYS, VERR_INVALID_PARAMETER);
277	AssertReturn(pvCpuPage, VERR_INVALID_PARAMETER);
278	NOREF(pCpu);
279
280	/* If we're somehow not in VMX root mode, then we shouldn't dare leaving it. */
281	if (!(ASMGetCR4() & X86_CR4_VMXE))
282	return VERR_VMX_NOT_IN_VMX_ROOT_MODE;
283
284	/* Leave VMX Root Mode. */
285	VMXDisable();
286
287	/* And clear the X86_CR4_VMXE bit. */
288	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
289	return VINF_SUCCESS;
290	}
291
292	VMMR0DECL(int) VMXR0GlobalInit(void)
293	{
294	/* Nothing to do. */
295	return VINF_SUCCESS;
296	}
297
298	VMMR0DECL(void) VMXR0GlobalTerm(void)
299	{
300	/* Nothing to do. */
301	}
302
303	/**
304	* Does Ring-0 per VM VT-x initialization.
305	*
306	* @returns VBox status code.
307	* @param pVM Pointer to the VM.
308	*/
309	VMMR0DECL(int) VMXR0InitVM(PVM pVM)
310	{
311	int rc;
312
313	#ifdef LOG_ENABLED
314	SUPR0Printf("VMXR0InitVM %p\n", pVM);
315	#endif
316
317	pVM->hm.s.vmx.hMemObjApicAccess = NIL_RTR0MEMOBJ;
318
319	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
320	{
321	/* Allocate one page for the APIC physical page (serves for filtering accesses). */
322	rc = RTR0MemObjAllocCont(&pVM->hm.s.vmx.hMemObjApicAccess, PAGE_SIZE, false /* fExecutable */);
323	AssertRC(rc);
324	if (RT_FAILURE(rc))
325	return rc;
326
327	pVM->hm.s.vmx.pbApicAccess = (uint8_t *)RTR0MemObjAddress(pVM->hm.s.vmx.hMemObjApicAccess);
328	pVM->hm.s.vmx.HCPhysApicAccess = RTR0MemObjGetPagePhysAddr(pVM->hm.s.vmx.hMemObjApicAccess, 0);
329	ASMMemZero32(pVM->hm.s.vmx.pbApicAccess, PAGE_SIZE);
330	}
331	else
332	{
333	pVM->hm.s.vmx.hMemObjApicAccess = 0;
334	pVM->hm.s.vmx.pbApicAccess = 0;
335	pVM->hm.s.vmx.HCPhysApicAccess = 0;
336	}
337
338	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
339	{
340	rc = RTR0MemObjAllocCont(&pVM->hm.s.vmx.hMemObjScratch, PAGE_SIZE, false /* fExecutable */);
341	AssertRC(rc);
342	if (RT_FAILURE(rc))
343	return rc;
344
345	pVM->hm.s.vmx.pScratch = (uint8_t *)RTR0MemObjAddress(pVM->hm.s.vmx.hMemObjScratch);
346	pVM->hm.s.vmx.pScratchPhys = RTR0MemObjGetPagePhysAddr(pVM->hm.s.vmx.hMemObjScratch, 0);
347
348	ASMMemZero32(pVM->hm.s.vmx.pbScratch, PAGE_SIZE);
349	strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
350	(uint64_t )(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xDEADBEEFDEADBEEF);
351	}
352	#endif
353
354	/* Allocate VMCSs for all guest CPUs. */
355	for (VMCPUID i = 0; i < pVM->cCpus; i++)
356	{
357	PVMCPU pVCpu = &pVM->aCpus[i];
358
359	pVCpu->hm.s.vmx.hMemObjVmcs = NIL_RTR0MEMOBJ;
360
361	/* Allocate one page for the VM control structure (VMCS). */
362	rc = RTR0MemObjAllocCont(&pVCpu->hm.s.vmx.hMemObjVmcs, PAGE_SIZE, false /* fExecutable */);
363	AssertRC(rc);
364	if (RT_FAILURE(rc))
365	return rc;
366
367	pVCpu->hm.s.vmx.pvVmcs = RTR0MemObjAddress(pVCpu->hm.s.vmx.hMemObjVmcs);
368	pVCpu->hm.s.vmx.HCPhysVmcs = RTR0MemObjGetPagePhysAddr(pVCpu->hm.s.vmx.hMemObjVmcs, 0);
369	ASMMemZeroPage(pVCpu->hm.s.vmx.pvVmcs);
370
371	pVCpu->hm.s.vmx.cr0_mask = 0;
372	pVCpu->hm.s.vmx.cr4_mask = 0;
373
374	/* Allocate one page for the virtual APIC page for TPR caching. */
375	rc = RTR0MemObjAllocCont(&pVCpu->hm.s.vmx.hMemObjVirtApic, PAGE_SIZE, false /* fExecutable */);
376	AssertRC(rc);
377	if (RT_FAILURE(rc))
378	return rc;
379
380	pVCpu->hm.s.vmx.pbVirtApic = (uint8_t *)RTR0MemObjAddress(pVCpu->hm.s.vmx.hMemObjVirtApic);
381	pVCpu->hm.s.vmx.HCPhysVirtApic = RTR0MemObjGetPagePhysAddr(pVCpu->hm.s.vmx.hMemObjVirtApic, 0);
382	ASMMemZeroPage(pVCpu->hm.s.vmx.pbVirtApic);
383
384	/* Allocate the MSR bitmap if this feature is supported. */
385	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
386	{
387	rc = RTR0MemObjAllocCont(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, PAGE_SIZE, false /* fExecutable */);
388	AssertRC(rc);
389	if (RT_FAILURE(rc))
390	return rc;
391
392	pVCpu->hm.s.vmx.pvMsrBitmap = (uint8_t *)RTR0MemObjAddress(pVCpu->hm.s.vmx.hMemObjMsrBitmap);
393	pVCpu->hm.s.vmx.HCPhysMsrBitmap = RTR0MemObjGetPagePhysAddr(pVCpu->hm.s.vmx.hMemObjMsrBitmap, 0);
394	memset(pVCpu->hm.s.vmx.pvMsrBitmap, 0xff, PAGE_SIZE);
395	}
396
397	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
398	/* Allocate one page for the guest MSR load area (for preloading guest MSRs during the world switch). */
399	rc = RTR0MemObjAllocCont(&pVCpu->hm.s.vmx.hMemObjGuestMsr, PAGE_SIZE, false /* fExecutable */);
400	AssertRC(rc);
401	if (RT_FAILURE(rc))
402	return rc;
403
404	pVCpu->hm.s.vmx.pvGuestMsr = (uint8_t *)RTR0MemObjAddress(pVCpu->hm.s.vmx.hMemObjGuestMsr);
405	pVCpu->hm.s.vmx.HCPhysGuestMsr = RTR0MemObjGetPagePhysAddr(pVCpu->hm.s.vmx.hMemObjGuestMsr, 0);
406	Assert(!(pVCpu->hm.s.vmx.HCPhysGuestMsr & 0xf));
407	memset(pVCpu->hm.s.vmx.pvGuestMsr, 0, PAGE_SIZE);
408
409	/* Allocate one page for the host MSR load area (for restoring host MSRs after the world switch back). */
410	rc = RTR0MemObjAllocCont(&pVCpu->hm.s.vmx.hMemObjHostMsr, PAGE_SIZE, false /* fExecutable */);
411	AssertRC(rc);
412	if (RT_FAILURE(rc))
413	return rc;
414
415	pVCpu->hm.s.vmx.pvHostMsr = (uint8_t *)RTR0MemObjAddress(pVCpu->hm.s.vmx.hMemObjHostMsr);
416	pVCpu->hm.s.vmx.HCPhysHostMsr = RTR0MemObjGetPagePhysAddr(pVCpu->hm.s.vmx.hMemObjHostMsr, 0);
417	Assert(!(pVCpu->hm.s.vmx.HCPhysHostMsr & 0xf));
418	memset(pVCpu->hm.s.vmx.pvHostMsr, 0, PAGE_SIZE);
419	#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
420
421	/* Current guest paging mode. */
422	pVCpu->hm.s.vmx.enmLastSeenGuestMode = PGMMODE_REAL;
423
424	#ifdef LOG_ENABLED
425	SUPR0Printf("VMXR0InitVM %x VMCS=%x (%x)\n", pVM, pVCpu->hm.s.vmx.pvVmcs, (uint32_t)pVCpu->hm.s.vmx.HCPhysVmcs);
426	#endif
427	}
428
429	return VINF_SUCCESS;
430	}
431
432
433	/**
434	* Does Ring-0 per VM VT-x termination.
435	*
436	* @returns VBox status code.
437	* @param pVM Pointer to the VM.
438	*/
439	VMMR0DECL(int) VMXR0TermVM(PVM pVM)
440	{
441	for (VMCPUID i = 0; i < pVM->cCpus; i++)
442	{
443	PVMCPU pVCpu = &pVM->aCpus[i];
444
445	if (pVCpu->hm.s.vmx.hMemObjVmcs != NIL_RTR0MEMOBJ)
446	{
447	RTR0MemObjFree(pVCpu->hm.s.vmx.hMemObjVmcs, false);
448	pVCpu->hm.s.vmx.hMemObjVmcs = NIL_RTR0MEMOBJ;
449	pVCpu->hm.s.vmx.pvVmcs = 0;
450	pVCpu->hm.s.vmx.HCPhysVmcs = 0;
451	}
452	if (pVCpu->hm.s.vmx.hMemObjVirtApic != NIL_RTR0MEMOBJ)
453	{
454	RTR0MemObjFree(pVCpu->hm.s.vmx.hMemObjVirtApic, false);
455	pVCpu->hm.s.vmx.hMemObjVirtApic = NIL_RTR0MEMOBJ;
456	pVCpu->hm.s.vmx.pbVirtApic = 0;
457	pVCpu->hm.s.vmx.HCPhysVirtApic = 0;
458	}
459	if (pVCpu->hm.s.vmx.hMemObjMsrBitmap != NIL_RTR0MEMOBJ)
460	{
461	RTR0MemObjFree(pVCpu->hm.s.vmx.hMemObjMsrBitmap, false);
462	pVCpu->hm.s.vmx.hMemObjMsrBitmap = NIL_RTR0MEMOBJ;
463	pVCpu->hm.s.vmx.pvMsrBitmap = 0;
464	pVCpu->hm.s.vmx.HCPhysMsrBitmap = 0;
465	}
466	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
467	if (pVCpu->hm.s.vmx.hMemObjHostMsr != NIL_RTR0MEMOBJ)
468	{
469	RTR0MemObjFree(pVCpu->hm.s.vmx.hMemObjHostMsr, false);
470	pVCpu->hm.s.vmx.hMemObjHostMsr = NIL_RTR0MEMOBJ;
471	pVCpu->hm.s.vmx.pvHostMsr = 0;
472	pVCpu->hm.s.vmx.HCPhysHostMsr = 0;
473	}
474	if (pVCpu->hm.s.vmx.hMemObjGuestMsr != NIL_RTR0MEMOBJ)
475	{
476	RTR0MemObjFree(pVCpu->hm.s.vmx.hMemObjGuestMsr, false);
477	pVCpu->hm.s.vmx.hMemObjGuestMsr = NIL_RTR0MEMOBJ;
478	pVCpu->hm.s.vmx.pvGuestMsr = 0;
479	pVCpu->hm.s.vmx.HCPhysGuestMsr = 0;
480	}
481	#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
482	}
483	if (pVM->hm.s.vmx.hMemObjApicAccess != NIL_RTR0MEMOBJ)
484	{
485	RTR0MemObjFree(pVM->hm.s.vmx.hMemObjApicAccess, false);
486	pVM->hm.s.vmx.hMemObjApicAccess = NIL_RTR0MEMOBJ;
487	pVM->hm.s.vmx.pbApicAccess = 0;
488	pVM->hm.s.vmx.HCPhysApicAccess = 0;
489	}
490	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
491	if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
492	{
493	ASMMemZero32(pVM->hm.s.vmx.pScratch, PAGE_SIZE);
494	RTR0MemObjFree(pVM->hm.s.vmx.hMemObjScratch, false);
495	pVM->hm.s.vmx.hMemObjScratch = NIL_RTR0MEMOBJ;
496	pVM->hm.s.vmx.pScratch = 0;
497	pVM->hm.s.vmx.pScratchPhys = 0;
498	}
499	#endif
500	return VINF_SUCCESS;
501	}
502
503
504	/**
505	* Sets up VT-x for the specified VM.
506	*
507	* @returns VBox status code.
508	* @param pVM Pointer to the VM.
509	*/
510	VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
511	{
512	int rc = VINF_SUCCESS;
513	uint32_t val;
514
515	AssertReturn(pVM, VERR_INVALID_PARAMETER);
516
517	/* Initialize these always, see hmR3InitFinalizeR0().*/
518	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NONE;
519	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NONE;
520
521	/* Determine optimal flush type for EPT. */
522	if (pVM->hm.s.fNestedPaging)
523	{
524	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
525	{
526	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
527	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_SINGLE_CONTEXT;
528	else if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
529	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_ALL_CONTEXTS;
530	else
531	{
532	/*
533	* Should never really happen. EPT is supported but no suitable flush types supported.
534	* We cannot ignore EPT at this point as we've already setup Unrestricted Guest execution.
535	*/
536	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
537	return VERR_VMX_GENERIC;
538	}
539	}
540	else
541	{
542	/*
543	* Should never really happen. EPT is supported but INVEPT instruction is not supported.
544	*/
545	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
546	return VERR_VMX_GENERIC;
547	}
548	}
549
550	/* Determine optimal flush type for VPID. */
551	if (pVM->hm.s.vmx.fVpid)
552	{
553	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
554	{
555	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
556	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_SINGLE_CONTEXT;
557	else if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
558	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_ALL_CONTEXTS;
559	else
560	{
561	/*
562	* Neither SINGLE nor ALL context flush types for VPID supported by the CPU.
563	* We do not handle other flush type combinations, ignore VPID capabilities.
564	*/
565	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
566	Log(("VMXR0SetupVM: Only VMX_FLUSH_VPID_INDIV_ADDR supported. Ignoring VPID.\n"));
567	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
568	Log(("VMXR0SetupVM: Only VMX_FLUSH_VPID_SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
569	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NOT_SUPPORTED;
570	pVM->hm.s.vmx.fVpid = false;
571	}
572	}
573	else
574	{
575	/*
576	* Should not really happen. EPT is supported but INVEPT is not supported.
577	* Ignore VPID capabilities as our code relies on using INVEPT for selective flushing.
578	*/
579	Log(("VMXR0SetupVM: VPID supported without INVEPT support. Ignoring VPID.\n"));
580	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NOT_SUPPORTED;
581	pVM->hm.s.vmx.fVpid = false;
582	}
583	}
584
585	for (VMCPUID i = 0; i < pVM->cCpus; i++)
586	{
587	PVMCPU pVCpu = &pVM->aCpus[i];
588
589	AssertPtr(pVCpu->hm.s.vmx.pvVmcs);
590
591	/* Set revision dword at the beginning of the VMCS structure. */
592	(uint32_t )pVCpu->hm.s.vmx.pvVmcs = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.msr.vmx_basic_info);
593
594	/*
595	* Clear and activate the VMCS.
596	*/
597	Log(("HCPhysVmcs = %RHp\n", pVCpu->hm.s.vmx.HCPhysVmcs));
598	rc = VMXClearVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
599	if (RT_FAILURE(rc))
600	goto vmx_end;
601
602	rc = VMXActivateVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
603	if (RT_FAILURE(rc))
604	goto vmx_end;
605
606	/*
607	* VMX_VMCS_CTRL_PIN_EXEC_CONTROLS
608	* Set required bits to one and zero according to the MSR capabilities.
609	*/
610	val = pVM->hm.s.vmx.msr.vmx_pin_ctls.n.disallowed0;
611	val \|= VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_EXT_INT_EXIT /* External interrupts */
612	\| VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_NMI_EXIT; /* Non-maskable interrupts */
613
614	/*
615	* Enable the VMX preemption timer.
616	*/
617	if (pVM->hm.s.vmx.fUsePreemptTimer)
618	val \|= VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_PREEMPT_TIMER;
619	val &= pVM->hm.s.vmx.msr.vmx_pin_ctls.n.allowed1;
620
621	rc = VMXWriteVmcs(VMX_VMCS32_CTRL_PIN_EXEC_CONTROLS, val);
622	AssertRC(rc);
623	pVCpu->hm.s.vmx.u32PinCtls = val;
624
625	/*
626	* VMX_VMCS_CTRL_PROC_EXEC_CONTROLS
627	* Set required bits to one and zero according to the MSR capabilities.
628	*/
629	val = pVM->hm.s.vmx.msr.vmx_proc_ctls.n.disallowed0;
630	/* Program which event cause VM-exits and which features we want to use. */
631	val \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_HLT_EXIT
632	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TSC_OFFSETTING
633	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT
634	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_UNCOND_IO_EXIT
635	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDPMC_EXIT
636	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_EXIT
637	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MWAIT_EXIT; /* don't execute mwait or else we'll idle inside
638	the guest (host thinks the cpu load is high) */
639
640	/* Without nested paging we should intercept invlpg and cr3 mov instructions. */
641	if (!pVM->hm.s.fNestedPaging)
642	{
643	val \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INVLPG_EXIT
644	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
645	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT;
646	}
647
648	/*
649	* VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MWAIT_EXIT might cause a vmlaunch
650	* failure with an invalid control fields error. (combined with some other exit reasons)
651	*/
652	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
653	{
654	/* CR8 reads from the APIC shadow page; writes cause an exit is they lower the TPR below the threshold */
655	val \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW;
656	Assert(pVM->hm.s.vmx.pbApicAccess);
657	}
658	else
659	/* Exit on CR8 reads & writes in case the TPR shadow feature isn't present. */
660	val \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR8_STORE_EXIT \| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR8_LOAD_EXIT;
661
662	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
663	{
664	Assert(pVCpu->hm.s.vmx.HCPhysMsrBitmap);
665	val \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS;
666	}
667
668	/* We will use the secondary control if it's present. */
669	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
670
671	/* Mask away the bits that the CPU doesn't support */
672	/** @todo make sure they don't conflict with the above requirements. */
673	val &= pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1;
674	pVCpu->hm.s.vmx.u32ProcCtls = val;
675
676	rc = VMXWriteVmcs(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, val);
677	AssertRC(rc);
678
679	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
680	{
681	/*
682	* VMX_VMCS_CTRL_PROC_EXEC_CONTROLS2
683	* Set required bits to one and zero according to the MSR capabilities.
684	*/
685	val = pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.disallowed0;
686	val \|= VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT;
687
688	if (pVM->hm.s.fNestedPaging)
689	val \|= VMX_VMCS_CTRL_PROC_EXEC2_EPT;
690
691	if (pVM->hm.s.vmx.fVpid)
692	val \|= VMX_VMCS_CTRL_PROC_EXEC2_VPID;
693
694	if (pVM->hm.s.fHasIoApic)
695	val \|= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC;
696
697	if (pVM->hm.s.vmx.fUnrestrictedGuest)
698	val \|= VMX_VMCS_CTRL_PROC_EXEC2_UNRESTRICTED_GUEST;
699
700	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
701	val \|= VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP;
702
703	/* Mask away the bits that the CPU doesn't support */
704	/** @todo make sure they don't conflict with the above requirements. */
705	val &= pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1;
706	pVCpu->hm.s.vmx.u32ProcCtls2 = val;
707	rc = VMXWriteVmcs(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS2, val);
708	AssertRC(rc);
709	}
710
711	/*
712	* VMX_VMCS_CTRL_CR3_TARGET_COUNT
713	* Set required bits to one and zero according to the MSR capabilities.
714	*/
715	rc = VMXWriteVmcs(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, 0);
716	AssertRC(rc);
717
718	/*
719	* Forward all exception except #NM & #PF to the guest.
720	* We always need to check pagefaults since our shadow page table can be out of sync.
721	* And we always lazily sync the FPU & XMM state. .
722	*/
723
724	/** @todo Possible optimization:
725	* Keep the FPU and XMM state current in the EM thread. That way there's no need to
726	* lazily sync anything, but the downside is that we can't use the FPU stack or XMM
727	* registers ourselves of course.
728	*
729	* Note: only possible if the current state is actually ours (X86_CR0_TS flag)
730	*/
731
732	/*
733	* Don't filter page faults, all of them should cause a world switch.
734	*/
735	rc = VMXWriteVmcs(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, 0);
736	AssertRC(rc);
737	rc = VMXWriteVmcs(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, 0);
738	AssertRC(rc);
739
740	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, 0);
741	AssertRC(rc);
742	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_A_FULL, 0);
743	AssertRC(rc);
744	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_B_FULL, 0);
745	AssertRC(rc);
746
747	/*
748	* Set the MSR bitmap address.
749	*/
750	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
751	{
752	Assert(pVCpu->hm.s.vmx.HCPhysMsrBitmap);
753
754	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, pVCpu->hm.s.vmx.HCPhysMsrBitmap);
755	AssertRC(rc);
756
757	/*
758	* Allow the guest to directly modify these MSRs; they are loaded/stored automatically
759	* using MSR-load/store areas in the VMCS.
760	*/
761	hmR0VmxSetMSRPermission(pVCpu, MSR_IA32_SYSENTER_CS, true, true);
762	hmR0VmxSetMSRPermission(pVCpu, MSR_IA32_SYSENTER_ESP, true, true);
763	hmR0VmxSetMSRPermission(pVCpu, MSR_IA32_SYSENTER_EIP, true, true);
764	hmR0VmxSetMSRPermission(pVCpu, MSR_K8_LSTAR, true, true);
765	hmR0VmxSetMSRPermission(pVCpu, MSR_K6_STAR, true, true);
766	hmR0VmxSetMSRPermission(pVCpu, MSR_K8_SF_MASK, true, true);
767	hmR0VmxSetMSRPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, true, true);
768	hmR0VmxSetMSRPermission(pVCpu, MSR_K8_GS_BASE, true, true);
769	hmR0VmxSetMSRPermission(pVCpu, MSR_K8_FS_BASE, true, true);
770	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
771	hmR0VmxSetMSRPermission(pVCpu, MSR_K8_TSC_AUX, true, true);
772	}
773
774	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
775	/*
776	* Set the guest & host MSR load/store physical addresses.
777	*/
778	Assert(pVCpu->hm.s.vmx.HCPhysGuestMsr);
779	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
780	AssertRC(rc);
781	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
782	AssertRC(rc);
783	Assert(pVCpu->hm.s.vmx.HCPhysHostMsr);
784	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysHostMsr);
785	AssertRC(rc);
786	#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
787
788	rc = VMXWriteVmcs(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, 0);
789	AssertRC(rc);
790	rc = VMXWriteVmcs(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, 0);
791	AssertRC(rc);
792	rc = VMXWriteVmcs(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, 0);
793	AssertRC(rc);
794
795	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
796	{
797	Assert(pVM->hm.s.vmx.hMemObjApicAccess);
798	/* Optional */
799	rc = VMXWriteVmcs(VMX_VMCS32_CTRL_TPR_THRESHOLD, 0);
800	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL, pVCpu->hm.s.vmx.HCPhysVirtApic);
801
802	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
803	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, pVM->hm.s.vmx.HCPhysApicAccess);
804
805	AssertRC(rc);
806	}
807
808	/* Set link pointer to -1. Not currently used. */
809	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, 0xFFFFFFFFFFFFFFFFULL);
810	AssertRC(rc);
811
812	/*
813	* Clear VMCS, marking it inactive. Clear implementation specific data and writing back
814	* VMCS data back to memory.
815	*/
816	rc = VMXClearVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
817	AssertRC(rc);
818
819	/*
820	* Configure the VMCS read cache.
821	*/
822	PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
823
824	VMXSetupCachedReadVmcs(pCache, VMX_VMCS_GUEST_RIP);
825	VMXSetupCachedReadVmcs(pCache, VMX_VMCS_GUEST_RSP);
826	VMXSetupCachedReadVmcs(pCache, VMX_VMCS_GUEST_RFLAGS);
827	VMXSetupCachedReadVmcs(pCache, VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE);
828	VMXSetupCachedReadVmcs(pCache, VMX_VMCS_CTRL_CR0_READ_SHADOW);
829	VMXSetupCachedReadVmcs(pCache, VMX_VMCS_GUEST_CR0);
830	VMXSetupCachedReadVmcs(pCache, VMX_VMCS_CTRL_CR4_READ_SHADOW);
831	VMXSetupCachedReadVmcs(pCache, VMX_VMCS_GUEST_CR4);
832	VMXSetupCachedReadVmcs(pCache, VMX_VMCS_GUEST_DR7);
833	VMXSetupCachedReadVmcs(pCache, VMX_VMCS32_GUEST_SYSENTER_CS);
834	VMXSetupCachedReadVmcs(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
835	VMXSetupCachedReadVmcs(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
836	VMXSetupCachedReadVmcs(pCache, VMX_VMCS32_GUEST_GDTR_LIMIT);
837	VMXSetupCachedReadVmcs(pCache, VMX_VMCS_GUEST_GDTR_BASE);
838	VMXSetupCachedReadVmcs(pCache, VMX_VMCS32_GUEST_IDTR_LIMIT);
839	VMXSetupCachedReadVmcs(pCache, VMX_VMCS_GUEST_IDTR_BASE);
840
841	VMX_SETUP_SELREG(ES, pCache);
842	VMX_SETUP_SELREG(SS, pCache);
843	VMX_SETUP_SELREG(CS, pCache);
844	VMX_SETUP_SELREG(DS, pCache);
845	VMX_SETUP_SELREG(FS, pCache);
846	VMX_SETUP_SELREG(GS, pCache);
847	VMX_SETUP_SELREG(LDTR, pCache);
848	VMX_SETUP_SELREG(TR, pCache);
849
850	/*
851	* Status code VMCS reads.
852	*/
853	VMXSetupCachedReadVmcs(pCache, VMX_VMCS32_RO_EXIT_REASON);
854	VMXSetupCachedReadVmcs(pCache, VMX_VMCS32_RO_VM_INSTR_ERROR);
855	VMXSetupCachedReadVmcs(pCache, VMX_VMCS32_RO_EXIT_INSTR_LENGTH);
856	VMXSetupCachedReadVmcs(pCache, VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE);
857	VMXSetupCachedReadVmcs(pCache, VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO);
858	VMXSetupCachedReadVmcs(pCache, VMX_VMCS32_RO_EXIT_INSTR_INFO);
859	VMXSetupCachedReadVmcs(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
860	VMXSetupCachedReadVmcs(pCache, VMX_VMCS32_RO_IDT_INFO);
861	VMXSetupCachedReadVmcs(pCache, VMX_VMCS32_RO_IDT_ERROR_CODE);
862
863	if (pVM->hm.s.fNestedPaging)
864	{
865	VMXSetupCachedReadVmcs(pCache, VMX_VMCS_GUEST_CR3);
866	VMXSetupCachedReadVmcs(pCache, VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL);
867	pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
868	}
869	else
870	pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
871	} /* for each VMCPU */
872
873	/*
874	* Setup the right TLB function based on CPU capabilities.
875	*/
876	if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
877	pVM->hm.s.vmx.pfnFlushTaggedTlb = hmR0VmxSetupTLBBoth;
878	else if (pVM->hm.s.fNestedPaging)
879	pVM->hm.s.vmx.pfnFlushTaggedTlb = hmR0VmxSetupTLBEPT;
880	else if (pVM->hm.s.vmx.fVpid)
881	pVM->hm.s.vmx.pfnFlushTaggedTlb = hmR0VmxSetupTLBVPID;
882	else
883	pVM->hm.s.vmx.pfnFlushTaggedTlb = hmR0VmxSetupTLBDummy;
884
885	vmx_end:
886	hmR0VmxCheckError(pVM, &pVM->aCpus[0], rc);
887	return rc;
888	}
889
890
891	/**
892	* Sets the permission bits for the specified MSR.
893	*
894	* @param pVCpu Pointer to the VMCPU.
895	* @param ulMSR The MSR value.
896	* @param fRead Whether reading is allowed.
897	* @param fWrite Whether writing is allowed.
898	*/
899	static void hmR0VmxSetMSRPermission(PVMCPU pVCpu, unsigned ulMSR, bool fRead, bool fWrite)
900	{
901	unsigned ulBit;
902	uint8_t pvMsrBitmap = (uint8_t )pVCpu->hm.s.vmx.pvMsrBitmap;
903
904	/*
905	* Layout:
906	* 0x000 - 0x3ff - Low MSR read bits
907	* 0x400 - 0x7ff - High MSR read bits
908	* 0x800 - 0xbff - Low MSR write bits
909	* 0xc00 - 0xfff - High MSR write bits
910	*/
911	if (ulMSR <= 0x00001FFF)
912	{
913	/* Pentium-compatible MSRs */
914	ulBit = ulMSR;
915	}
916	else if ( ulMSR >= 0xC0000000
917	&& ulMSR <= 0xC0001FFF)
918	{
919	/* AMD Sixth Generation x86 Processor MSRs */
920	ulBit = (ulMSR - 0xC0000000);
921	pvMsrBitmap += 0x400;
922	}
923	else
924	{
925	AssertFailed();
926	return;
927	}
928
929	Assert(ulBit <= 0x1fff);
930	if (fRead)
931	ASMBitClear(pvMsrBitmap, ulBit);
932	else
933	ASMBitSet(pvMsrBitmap, ulBit);
934
935	if (fWrite)
936	ASMBitClear(pvMsrBitmap + 0x800, ulBit);
937	else
938	ASMBitSet(pvMsrBitmap + 0x800, ulBit);
939	}
940
941
942	/**
943	* Injects an event (trap or external interrupt).
944	*
945	* @returns VBox status code. Note that it may return VINF_EM_RESET to
946	* indicate a triple fault when injecting X86_XCPT_DF.
947	*
948	* @param pVM Pointer to the VM.
949	* @param pVCpu Pointer to the VMCPU.
950	* @param pCtx Pointer to the guest CPU Context.
951	* @param intInfo VMX interrupt info.
952	* @param cbInstr Opcode length of faulting instruction.
953	* @param errCode Error code (optional).
954	*/
955	static int hmR0VmxInjectEvent(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, uint32_t intInfo, uint32_t cbInstr, uint32_t errCode)
956	{
957	int rc;
958	uint32_t iGate = VMX_EXIT_INTERRUPTION_INFO_VECTOR(intInfo);
959
960	#ifdef VBOX_WITH_STATISTICS
961	STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[iGate & MASK_INJECT_IRQ_STAT]);
962	#endif
963
964	#ifdef VBOX_STRICT
965	if (iGate == 0xE)
966	{
967	LogFlow(("hmR0VmxInjectEvent: Injecting interrupt %d at %RGv error code=%08x CR2=%RGv intInfo=%08x\n", iGate,
968	(RTGCPTR)pCtx->rip, errCode, pCtx->cr2, intInfo));
969	}
970	else if (iGate < 0x20)
971	{
972	LogFlow(("hmR0VmxInjectEvent: Injecting interrupt %d at %RGv error code=%08x\n", iGate, (RTGCPTR)pCtx->rip,
973	errCode));
974	}
975	else
976	{
977	LogFlow(("INJ-EI: %x at %RGv\n", iGate, (RTGCPTR)pCtx->rip));
978	Assert( VMX_EXIT_INTERRUPTION_INFO_TYPE(intInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT
979	\|\| !VMCPU_FF_ISSET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
980	Assert( VMX_EXIT_INTERRUPTION_INFO_TYPE(intInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT
981	\|\| pCtx->eflags.u32 & X86_EFL_IF);
982	}
983	#endif
984
985	if ( CPUMIsGuestInRealModeEx(pCtx)
986	&& pVM->hm.s.vmx.pRealModeTSS)
987	{
988	RTGCPHYS GCPhysHandler;
989	uint16_t offset, ip;
990	RTSEL sel;
991
992	/*
993	* Injecting events doesn't work right with real mode emulation.
994	* (#GP if we try to inject external hardware interrupts)
995	* Inject the interrupt or trap directly instead.
996	*
997	* ASSUMES no access handlers for the bits we read or write below (should be safe).
998	*/
999	Log(("Manual interrupt/trap '%x' inject (real mode)\n", iGate));
1000
1001	/*
1002	* Check if the interrupt handler is present.
1003	*/
1004	if (iGate * 4 + 3 > pCtx->idtr.cbIdt)
1005	{
1006	Log(("IDT cbIdt violation\n"));
1007	if (iGate != X86_XCPT_DF)
1008	{
1009	uint32_t intInfo2;
1010
1011	intInfo2 = (iGate == X86_XCPT_GP) ? (uint32_t)X86_XCPT_DF : (uint32_t)X86_XCPT_GP;
1012	intInfo2 \|= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
1013	intInfo2 \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
1014	intInfo2 \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
1015
1016	return hmR0VmxInjectEvent(pVM, pVCpu, pCtx, intInfo2, 0, 0 /* no error code according to the Intel docs */);
1017	}
1018	Log(("Triple fault -> reset the VM!\n"));
1019	return VINF_EM_RESET;
1020	}
1021	if ( VMX_EXIT_INTERRUPTION_INFO_TYPE(intInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT
1022	\|\| iGate == 3 /* Both #BP and #OF point to the instruction after. */
1023	\|\| iGate == 4)
1024	{
1025	ip = pCtx->ip + cbInstr;
1026	}
1027	else
1028	ip = pCtx->ip;
1029
1030	/*
1031	* Read the selector:offset pair of the interrupt handler.
1032	*/
1033	GCPhysHandler = (RTGCPHYS)pCtx->idtr.pIdt + iGate * 4;
1034	rc = PGMPhysSimpleReadGCPhys(pVM, &offset, GCPhysHandler, sizeof(offset)); AssertRC(rc);
1035	rc = PGMPhysSimpleReadGCPhys(pVM, &sel, GCPhysHandler + 2, sizeof(sel)); AssertRC(rc);
1036
1037	LogFlow(("IDT handler %04X:%04X\n", sel, offset));
1038
1039	/*
1040	* Construct the stack frame.
1041	*/
1042	/** @todo Check stack limit. */
1043	pCtx->sp -= 2;
1044	LogFlow(("ss:sp %04X:%04X eflags=%x\n", pCtx->ss.Sel, pCtx->sp, pCtx->eflags.u));
1045	rc = PGMPhysSimpleWriteGCPhys(pVM, pCtx->ss.u64Base + pCtx->sp, &pCtx->eflags, sizeof(uint16_t)); AssertRC(rc);
1046	pCtx->sp -= 2;
1047	LogFlow(("ss:sp %04X:%04X cs=%x\n", pCtx->ss.Sel, pCtx->sp, pCtx->cs.Sel));
1048	rc = PGMPhysSimpleWriteGCPhys(pVM, pCtx->ss.u64Base + pCtx->sp, &pCtx->cs, sizeof(uint16_t)); AssertRC(rc);
1049	pCtx->sp -= 2;
1050	LogFlow(("ss:sp %04X:%04X ip=%x\n", pCtx->ss.Sel, pCtx->sp, ip));
1051	rc = PGMPhysSimpleWriteGCPhys(pVM, pCtx->ss.u64Base + pCtx->sp, &ip, sizeof(ip)); AssertRC(rc);
1052
1053	/*
1054	* Update the CPU state for executing the handler.
1055	*/
1056	pCtx->rip = offset;
1057	pCtx->cs.Sel = sel;
1058	pCtx->cs.u64Base = sel << 4;
1059	pCtx->eflags.u &= ~(X86_EFL_IF \| X86_EFL_TF \| X86_EFL_RF \| X86_EFL_AC);
1060
1061	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SEGMENT_REGS;
1062	return VINF_SUCCESS;
1063	}
1064
1065	/*
1066	* Set event injection state.
1067	*/
1068	rc = VMXWriteVmcs(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, intInfo \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT));
1069	rc \|= VMXWriteVmcs(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
1070	rc \|= VMXWriteVmcs(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, errCode);
1071
1072	AssertRC(rc);
1073	return rc;
1074	}
1075
1076
1077	/**
1078	* Checks for pending guest interrupts and injects them.
1079	*
1080	* @returns VBox status code.
1081	* @param pVM Pointer to the VM.
1082	* @param pVCpu Pointer to the VMCPU.
1083	* @param pCtx Pointer to the guest CPU context.
1084	*/
1085	static int hmR0VmxCheckPendingInterrupt(PVM pVM, PVMCPU pVCpu, CPUMCTX *pCtx)
1086	{
1087	int rc;
1088
1089	/*
1090	* Dispatch any pending interrupts (injected before, but a VM exit occurred prematurely).
1091	*/
1092	if (pVCpu->hm.s.Event.fPending)
1093	{
1094	Log(("CPU%d: Reinjecting event %RX64 %08x at %RGv cr2=%RX64\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntrInfo,
1095	pVCpu->hm.s.Event.u32ErrCode, (RTGCPTR)pCtx->rip, pCtx->cr2));
1096	STAM_COUNTER_INC(&pVCpu->hm.s.StatIntReinject);
1097	rc = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, pVCpu->hm.s.Event.u64IntrInfo, 0, pVCpu->hm.s.Event.u32ErrCode);
1098	AssertRC(rc);
1099
1100	pVCpu->hm.s.Event.fPending = false;
1101	return VINF_SUCCESS;
1102	}
1103
1104	/*
1105	* If an active trap is already pending, we must forward it first!
1106	*/
1107	if (!TRPMHasTrap(pVCpu))
1108	{
1109	if (VMCPU_FF_TESTANDCLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI))
1110	{
1111	RTGCUINTPTR intInfo;
1112
1113	Log(("CPU%d: injecting #NMI\n", pVCpu->idCpu));
1114
1115	intInfo = X86_XCPT_NMI;
1116	intInfo \|= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
1117	intInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
1118
1119	rc = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, intInfo, 0, 0);
1120	AssertRC(rc);
1121
1122	return VINF_SUCCESS;
1123	}
1124
1125	/** @todo SMI interrupts. */
1126
1127	/*
1128	* When external interrupts are pending, we should exit the VM when IF is set.
1129	*/
1130	if (VMCPU_FF_ISPENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC\|VMCPU_FF_INTERRUPT_PIC)))
1131	{
1132	if (!(pCtx->eflags.u32 & X86_EFL_IF))
1133	{
1134	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INT_WINDOW_EXIT))
1135	{
1136	LogFlow(("Enable irq window exit!\n"));
1137	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INT_WINDOW_EXIT;
1138	rc = VMXWriteVmcs(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls);
1139	AssertRC(rc);
1140	}
1141	/* else nothing to do but wait */
1142	}
1143	else if (!VMCPU_FF_ISSET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
1144	{
1145	uint8_t u8Interrupt;
1146
1147	rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
1148	Log(("CPU%d: Dispatch interrupt: u8Interrupt=%x (%d) rc=%Rrc cs:rip=%04X:%RGv\n", pVCpu->idCpu,
1149	u8Interrupt, u8Interrupt, rc, pCtx->cs.Sel, (RTGCPTR)pCtx->rip));
1150	if (RT_SUCCESS(rc))
1151	{
1152	rc = TRPMAssertTrap(pVCpu, u8Interrupt, TRPM_HARDWARE_INT);
1153	AssertRC(rc);
1154	}
1155	else
1156	{
1157	/* Can only happen in rare cases where a pending interrupt is cleared behind our back */
1158	Assert(!VMCPU_FF_ISPENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC\|VMCPU_FF_INTERRUPT_PIC)));
1159	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
1160	/* Just continue */
1161	}
1162	}
1163	else
1164	Log(("Pending interrupt blocked at %RGv by VM_FF_INHIBIT_INTERRUPTS!!\n", (RTGCPTR)pCtx->rip));
1165	}
1166	}
1167
1168	#ifdef VBOX_STRICT
1169	if (TRPMHasTrap(pVCpu))
1170	{
1171	uint8_t u8Vector;
1172	rc = TRPMQueryTrapAll(pVCpu, &u8Vector, 0, 0, 0);
1173	AssertRC(rc);
1174	}
1175	#endif
1176
1177	if ( (pCtx->eflags.u32 & X86_EFL_IF)
1178	&& (!VMCPU_FF_ISSET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
1179	&& TRPMHasTrap(pVCpu)
1180	)
1181	{
1182	uint8_t u8Vector;
1183	TRPMEVENT enmType;
1184	RTGCUINTPTR intInfo;
1185	RTGCUINT errCode;
1186
1187	/*
1188	* If a new event is pending, dispatch it now.
1189	*/
1190	rc = TRPMQueryTrapAll(pVCpu, &u8Vector, &enmType, &errCode, 0);
1191	AssertRC(rc);
1192	Assert(pCtx->eflags.Bits.u1IF == 1 \|\| enmType == TRPM_TRAP);
1193	Assert(enmType != TRPM_SOFTWARE_INT);
1194
1195	/*
1196	* Clear the pending trap.
1197	*/
1198	rc = TRPMResetTrap(pVCpu);
1199	AssertRC(rc);
1200
1201	intInfo = u8Vector;
1202	intInfo \|= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
1203
1204	if (enmType == TRPM_TRAP)
1205	{
1206	switch (u8Vector)
1207	{
1208	case X86_XCPT_DF:
1209	case X86_XCPT_TS:
1210	case X86_XCPT_NP:
1211	case X86_XCPT_SS:
1212	case X86_XCPT_GP:
1213	case X86_XCPT_PF:
1214	case X86_XCPT_AC:
1215	{
1216	/** @todo r=ramshankar: setting this bit would blow up for real-mode guests with
1217	* unrestricted guest execution. */
1218	/* Valid error codes. */
1219	intInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
1220	break;
1221	}
1222
1223	default:
1224	break;
1225	}
1226
1227	if ( u8Vector == X86_XCPT_BP
1228	\|\| u8Vector == X86_XCPT_OF)
1229	{
1230	intInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
1231	}
1232	else
1233	intInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
1234	}
1235	else
1236	intInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
1237
1238	STAM_COUNTER_INC(&pVCpu->hm.s.StatIntInject);
1239	rc = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, intInfo, 0, errCode);
1240	AssertRC(rc);
1241	} /* if (interrupts can be dispatched) */
1242
1243	return VINF_SUCCESS;
1244	}
1245
1246	/**
1247	* Checks for pending VMX events and converts them to TRPM. Before we execute any instruction
1248	* outside of VMX, any pending VMX event must be converted so that it can be delivered properly.
1249	*
1250	* @returns VBox status code.
1251	* @param pVCpu Pointer to the VMCPU.
1252	*/
1253	static int hmR0VmxCheckPendingEvent(PVMCPU pVCpu)
1254	{
1255	if (pVCpu->hm.s.Event.fPending)
1256	{
1257	TRPMEVENT enmTrapType;
1258
1259	/* If a trap was already pending, we did something wrong! */
1260	Assert((TRPMQueryTrap(pVCpu, NULL, NULL) == VERR_TRPM_NO_ACTIVE_TRAP));
1261
1262	/*
1263	* Clear the pending event and move it over to TRPM for the rest
1264	* of the world to see.
1265	*/
1266	pVCpu->hm.s.Event.fPending = false;
1267	switch (VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hm.s.Event.u64IntrInfo))
1268	{
1269	case VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT:
1270	case VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI:
1271	enmTrapType = TRPM_HARDWARE_INT;
1272	break;
1273	case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT:
1274	case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT: /** @todo Is classifying #BP, #OF as TRPM_SOFTWARE_INT correct? */
1275	case VMX_EXIT_INTERRUPTION_INFO_TYPE_DB_XCPT:
1276	enmTrapType = TRPM_SOFTWARE_INT;
1277	break;
1278	case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
1279	enmTrapType = TRPM_TRAP;
1280	break;
1281	default:
1282	enmTrapType = TRPM_32BIT_HACK; /* Can't get here. */
1283	AssertFailed();
1284	}
1285	TRPMAssertTrap(pVCpu, VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVCpu->hm.s.Event.u64IntrInfo), enmTrapType);
1286	if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntrInfo))
1287	TRPMSetErrorCode(pVCpu, pVCpu->hm.s.Event.u32ErrCode);
1288	//@todo: Is there any situation where we need to call TRPMSetFaultAddress()?
1289	}
1290	return VINF_SUCCESS;
1291	}
1292
1293	/**
1294	* Save the host state into the VMCS.
1295	*
1296	* @returns VBox status code.
1297	* @param pVM Pointer to the VM.
1298	* @param pVCpu Pointer to the VMCPU.
1299	*/
1300	VMMR0DECL(int) VMXR0SaveHostState(PVM pVM, PVMCPU pVCpu)
1301	{
1302	int rc = VINF_SUCCESS;
1303	NOREF(pVM);
1304
1305	/*
1306	* Host CPU Context.
1307	*/
1308	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_HOST_CONTEXT)
1309	{
1310	RTIDTR idtr;
1311	RTGDTR gdtr;
1312	RTSEL SelTR;
1313	PCX86DESCHC pDesc;
1314	uintptr_t trBase;
1315	RTSEL cs;
1316	RTSEL ss;
1317	uint64_t cr3;
1318
1319	/*
1320	* Control registers.
1321	*/
1322	rc = VMXWriteVmcs(VMX_VMCS_HOST_CR0, ASMGetCR0());
1323	Log2(("VMX_VMCS_HOST_CR0 %08x\n", ASMGetCR0()));
1324	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
1325	if (VMX_IS_64BIT_HOST_MODE())
1326	{
1327	cr3 = hmR0Get64bitCR3();
1328	rc \|= VMXWriteVmcs64(VMX_VMCS_HOST_CR3, cr3);
1329	}
1330	else
1331	#endif
1332	{
1333	cr3 = ASMGetCR3();
1334	rc \|= VMXWriteVmcs(VMX_VMCS_HOST_CR3, cr3);
1335	}
1336	Log2(("VMX_VMCS_HOST_CR3 %08RX64\n", cr3));
1337	rc \|= VMXWriteVmcs(VMX_VMCS_HOST_CR4, ASMGetCR4());
1338	Log2(("VMX_VMCS_HOST_CR4 %08x\n", ASMGetCR4()));
1339	AssertRC(rc);
1340
1341	/*
1342	* Selector registers.
1343	*/
1344	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
1345	if (VMX_IS_64BIT_HOST_MODE())
1346	{
1347	cs = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelCS;
1348	ss = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelSS;
1349	}
1350	else
1351	{
1352	/* sysenter loads LDT cs & ss, VMX doesn't like this. Load the GDT ones (safe). */
1353	cs = (RTSEL)(uintptr_t)&SUPR0AbsKernelCS;
1354	ss = (RTSEL)(uintptr_t)&SUPR0AbsKernelSS;
1355	}
1356	#else
1357	cs = ASMGetCS();
1358	ss = ASMGetSS();
1359	#endif
1360	Assert(!(cs & X86_SEL_LDT)); Assert((cs & X86_SEL_RPL) == 0);
1361	Assert(!(ss & X86_SEL_LDT)); Assert((ss & X86_SEL_RPL) == 0);
1362	rc = VMXWriteVmcs(VMX_VMCS16_HOST_FIELD_CS, cs);
1363	/* Note: VMX is (again) very picky about the RPL of the selectors here; we'll restore them manually. */
1364	rc \|= VMXWriteVmcs(VMX_VMCS16_HOST_FIELD_DS, 0);
1365	rc \|= VMXWriteVmcs(VMX_VMCS16_HOST_FIELD_ES, 0);
1366	#if HC_ARCH_BITS == 32
1367	if (!VMX_IS_64BIT_HOST_MODE())
1368	{
1369	rc \|= VMXWriteVmcs(VMX_VMCS16_HOST_FIELD_FS, 0);
1370	rc \|= VMXWriteVmcs(VMX_VMCS16_HOST_FIELD_GS, 0);
1371	}
1372	#endif
1373	rc \|= VMXWriteVmcs(VMX_VMCS16_HOST_FIELD_SS, ss);
1374	SelTR = ASMGetTR();
1375	rc \|= VMXWriteVmcs(VMX_VMCS16_HOST_FIELD_TR, SelTR);
1376	AssertRC(rc);
1377	Log2(("VMX_VMCS_HOST_FIELD_CS %08x (%08x)\n", cs, ASMGetSS()));
1378	Log2(("VMX_VMCS_HOST_FIELD_DS 00000000 (%08x)\n", ASMGetDS()));
1379	Log2(("VMX_VMCS_HOST_FIELD_ES 00000000 (%08x)\n", ASMGetES()));
1380	Log2(("VMX_VMCS_HOST_FIELD_FS 00000000 (%08x)\n", ASMGetFS()));
1381	Log2(("VMX_VMCS_HOST_FIELD_GS 00000000 (%08x)\n", ASMGetGS()));
1382	Log2(("VMX_VMCS_HOST_FIELD_SS %08x (%08x)\n", ss, ASMGetSS()));
1383	Log2(("VMX_VMCS_HOST_FIELD_TR %08x\n", ASMGetTR()));
1384
1385	/*
1386	* GDTR & IDTR.
1387	*/
1388	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
1389	if (VMX_IS_64BIT_HOST_MODE())
1390	{
1391	X86XDTR64 gdtr64, idtr64;
1392	hmR0Get64bitGdtrAndIdtr(&gdtr64, &idtr64);
1393	rc = VMXWriteVmcs64(VMX_VMCS_HOST_GDTR_BASE, gdtr64.uAddr);
1394	rc \|= VMXWriteVmcs64(VMX_VMCS_HOST_IDTR_BASE, idtr64.uAddr);
1395	AssertRC(rc);
1396	Log2(("VMX_VMCS_HOST_GDTR_BASE %RX64\n", gdtr64.uAddr));
1397	Log2(("VMX_VMCS_HOST_IDTR_BASE %RX64\n", idtr64.uAddr));
1398	gdtr.cbGdt = gdtr64.cb;
1399	gdtr.pGdt = (uintptr_t)gdtr64.uAddr;
1400	}
1401	else
1402	#endif
1403	{
1404	ASMGetGDTR(&gdtr);
1405	rc = VMXWriteVmcs(VMX_VMCS_HOST_GDTR_BASE, gdtr.pGdt);
1406	ASMGetIDTR(&idtr);
1407	rc \|= VMXWriteVmcs(VMX_VMCS_HOST_IDTR_BASE, idtr.pIdt);
1408	AssertRC(rc);
1409	Log2(("VMX_VMCS_HOST_GDTR_BASE %RHv\n", gdtr.pGdt));
1410	Log2(("VMX_VMCS_HOST_IDTR_BASE %RHv\n", idtr.pIdt));
1411	}
1412
1413	/*
1414	* Save the base address of the TR selector.
1415	*/
1416	if (SelTR > gdtr.cbGdt)
1417	{
1418	AssertMsgFailed(("Invalid TR selector %x. GDTR.cbGdt=%x\n", SelTR, gdtr.cbGdt));
1419	return VERR_VMX_INVALID_HOST_STATE;
1420	}
1421
1422	pDesc = (PCX86DESCHC)(gdtr.pGdt + (SelTR & X86_SEL_MASK));
1423	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
1424	if (VMX_IS_64BIT_HOST_MODE())
1425	{
1426	uint64_t trBase64 = X86DESC64_BASE((PX86DESC64)pDesc);
1427	rc = VMXWriteVmcs64(VMX_VMCS_HOST_TR_BASE, trBase64);
1428	Log2(("VMX_VMCS_HOST_TR_BASE %RX64\n", trBase64));
1429	AssertRC(rc);
1430	}
1431	else
1432	#endif
1433	{
1434	#if HC_ARCH_BITS == 64
1435	trBase = X86DESC64_BASE(pDesc);
1436	#else
1437	trBase = X86DESC_BASE(pDesc);
1438	#endif
1439	rc = VMXWriteVmcs(VMX_VMCS_HOST_TR_BASE, trBase);
1440	AssertRC(rc);
1441	Log2(("VMX_VMCS_HOST_TR_BASE %RHv\n", trBase));
1442	}
1443
1444	/*
1445	* FS base and GS base.
1446	*/
1447	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
1448	if (VMX_IS_64BIT_HOST_MODE())
1449	{
1450	Log2(("MSR_K8_FS_BASE = %RX64\n", ASMRdMsr(MSR_K8_FS_BASE)));
1451	Log2(("MSR_K8_GS_BASE = %RX64\n", ASMRdMsr(MSR_K8_GS_BASE)));
1452	rc = VMXWriteVmcs64(VMX_VMCS_HOST_FS_BASE, ASMRdMsr(MSR_K8_FS_BASE));
1453	rc \|= VMXWriteVmcs64(VMX_VMCS_HOST_GS_BASE, ASMRdMsr(MSR_K8_GS_BASE));
1454	}
1455	#endif
1456	AssertRC(rc);
1457
1458	/*
1459	* Sysenter MSRs.
1460	*/
1461	/** @todo expensive!! */
1462	rc = VMXWriteVmcs(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
1463	Log2(("VMX_VMCS_HOST_SYSENTER_CS %08x\n", ASMRdMsr_Low(MSR_IA32_SYSENTER_CS)));
1464	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
1465	if (VMX_IS_64BIT_HOST_MODE())
1466	{
1467	Log2(("VMX_VMCS_HOST_SYSENTER_EIP %RX64\n", ASMRdMsr(MSR_IA32_SYSENTER_EIP)));
1468	Log2(("VMX_VMCS_HOST_SYSENTER_ESP %RX64\n", ASMRdMsr(MSR_IA32_SYSENTER_ESP)));
1469	rc \|= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
1470	rc \|= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
1471	}
1472	else
1473	{
1474	rc \|= VMXWriteVmcs(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
1475	rc \|= VMXWriteVmcs(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
1476	Log2(("VMX_VMCS_HOST_SYSENTER_EIP %RX32\n", ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP)));
1477	Log2(("VMX_VMCS_HOST_SYSENTER_ESP %RX32\n", ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP)));
1478	}
1479	#elif HC_ARCH_BITS == 32
1480	rc \|= VMXWriteVmcs(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
1481	rc \|= VMXWriteVmcs(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
1482	Log2(("VMX_VMCS_HOST_SYSENTER_EIP %RX32\n", ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP)));
1483	Log2(("VMX_VMCS_HOST_SYSENTER_ESP %RX32\n", ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP)));
1484	#else
1485	Log2(("VMX_VMCS_HOST_SYSENTER_EIP %RX64\n", ASMRdMsr(MSR_IA32_SYSENTER_EIP)));
1486	Log2(("VMX_VMCS_HOST_SYSENTER_ESP %RX64\n", ASMRdMsr(MSR_IA32_SYSENTER_ESP)));
1487	rc \|= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
1488	rc \|= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
1489	#endif
1490	AssertRC(rc);
1491
1492
1493	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
1494	/*
1495	* Store all host MSRs in the VM-Exit load area, so they will be reloaded after
1496	* the world switch back to the host.
1497	*/
1498	PVMXMSR pMsr = (PVMXMSR)pVCpu->hm.s.vmx.pvHostMsr;
1499	unsigned idxMsr = 0;
1500
1501	uint32_t u32HostExtFeatures = ASMCpuId_EDX(0x80000001);
1502	if (u32HostExtFeatures & (X86_CPUID_EXT_FEATURE_EDX_NX \| X86_CPUID_EXT_FEATURE_EDX_LONG_MODE))
1503	{
1504	pMsr->u32IndexMSR = MSR_K6_EFER;
1505	pMsr->u32Reserved = 0;
1506	# if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
1507	if (CPUMIsGuestInLongMode(pVCpu))
1508	{
1509	/* Must match the EFER value in our 64 bits switcher. */
1510	pMsr->u64Value = ASMRdMsr(MSR_K6_EFER) \| MSR_K6_EFER_LME \| MSR_K6_EFER_SCE \| MSR_K6_EFER_NXE;
1511	}
1512	else
1513	# endif
1514	pMsr->u64Value = ASMRdMsr(MSR_K6_EFER);
1515	pMsr++; idxMsr++;
1516	}
1517
1518	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
1519	if (VMX_IS_64BIT_HOST_MODE())
1520	{
1521	pMsr->u32IndexMSR = MSR_K6_STAR;
1522	pMsr->u32Reserved = 0;
1523	pMsr->u64Value = ASMRdMsr(MSR_K6_STAR); /* legacy syscall eip, cs & ss */
1524	pMsr++; idxMsr++;
1525	pMsr->u32IndexMSR = MSR_K8_LSTAR;
1526	pMsr->u32Reserved = 0;
1527	pMsr->u64Value = ASMRdMsr(MSR_K8_LSTAR); /* 64 bits mode syscall rip */
1528	pMsr++; idxMsr++;
1529	pMsr->u32IndexMSR = MSR_K8_SF_MASK;
1530	pMsr->u32Reserved = 0;
1531	pMsr->u64Value = ASMRdMsr(MSR_K8_SF_MASK); /* syscall flag mask */
1532	pMsr++; idxMsr++;
1533
1534	/* The KERNEL_GS_BASE MSR doesn't work reliably with auto load/store. See @bugref{6208} */
1535	#if 0
1536	pMsr->u32IndexMSR = MSR_K8_KERNEL_GS_BASE;
1537	pMsr->u32Reserved = 0;
1538	pMsr->u64Value = ASMRdMsr(MSR_K8_KERNEL_GS_BASE); /* swapgs exchange value */
1539	pMsr++; idxMsr++;
1540	#endif
1541	}
1542	# endif
1543
1544	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
1545	{
1546	pMsr->u32IndexMSR = MSR_K8_TSC_AUX;
1547	pMsr->u32Reserved = 0;
1548	pMsr->u64Value = ASMRdMsr(MSR_K8_TSC_AUX);
1549	pMsr++; idxMsr++;
1550	}
1551
1552	/** @todo r=ramshankar: check IA32_VMX_MISC bits 27:25 for valid idxMsr
1553	* range. */
1554	rc = VMXWriteVmcs(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, idxMsr);
1555	AssertRC(rc);
1556	#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
1557
1558	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_HOST_CONTEXT;
1559	}
1560	return rc;
1561	}
1562
1563
1564	/**
1565	* Loads the 4 PDPEs into the guest state when nested paging is used and the
1566	* guest operates in PAE mode.
1567	*
1568	* @returns VBox status code.
1569	* @param pVCpu Pointer to the VMCPU.
1570	* @param pCtx Pointer to the guest CPU context.
1571	*/
1572	static int hmR0VmxLoadPaePdpes(PVMCPU pVCpu, PCPUMCTX pCtx)
1573	{
1574	if (CPUMIsGuestInPAEModeEx(pCtx))
1575	{
1576	X86PDPE aPdpes[4];
1577	int rc = PGMGstGetPaePdpes(pVCpu, &aPdpes[0]);
1578	AssertRCReturn(rc, rc);
1579
1580	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, aPdpes[0].u); AssertRCReturn(rc, rc);
1581	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, aPdpes[1].u); AssertRCReturn(rc, rc);
1582	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, aPdpes[2].u); AssertRCReturn(rc, rc);
1583	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, aPdpes[3].u); AssertRCReturn(rc, rc);
1584	}
1585	return VINF_SUCCESS;
1586	}
1587
1588
1589	/**
1590	* Saves the 4 PDPEs into the guest state when nested paging is used and the
1591	* guest operates in PAE mode.
1592	*
1593	* @returns VBox status code.
1594	* @param pVCpu Pointer to the VM CPU.
1595	* @param pCtx Pointer to the guest CPU context.
1596	*
1597	* @remarks Tell PGM about CR3 changes before calling this helper.
1598	*/
1599	static int hmR0VmxSavePaePdpes(PVMCPU pVCpu, PCPUMCTX pCtx)
1600	{
1601	if (CPUMIsGuestInPAEModeEx(pCtx))
1602	{
1603	int rc;
1604	X86PDPE aPdpes[4];
1605	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &aPdpes[0].u); AssertRCReturn(rc, rc);
1606	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &aPdpes[1].u); AssertRCReturn(rc, rc);
1607	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &aPdpes[2].u); AssertRCReturn(rc, rc);
1608	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &aPdpes[3].u); AssertRCReturn(rc, rc);
1609
1610	rc = PGMGstUpdatePaePdpes(pVCpu, &aPdpes[0]);
1611	AssertRCReturn(rc, rc);
1612	}
1613	return VINF_SUCCESS;
1614	}
1615
1616
1617	/**
1618	* Update the exception bitmap according to the current CPU state.
1619	*
1620	* @param pVM Pointer to the VM.
1621	* @param pVCpu Pointer to the VMCPU.
1622	* @param pCtx Pointer to the guest CPU context.
1623	*/
1624	static void hmR0VmxUpdateExceptionBitmap(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
1625	{
1626	uint32_t u32TrapMask;
1627	Assert(pCtx);
1628
1629	/*
1630	* Set up a mask for intercepting traps.
1631	*/
1632	/** @todo Do we really need to always intercept #DB? */
1633	u32TrapMask = RT_BIT(X86_XCPT_DB)
1634	\| RT_BIT(X86_XCPT_NM)
1635	#ifdef VBOX_ALWAYS_TRAP_PF
1636	\| RT_BIT(X86_XCPT_PF)
1637	#endif
1638	#ifdef VBOX_STRICT
1639	\| RT_BIT(X86_XCPT_BP)
1640	\| RT_BIT(X86_XCPT_DB)
1641	\| RT_BIT(X86_XCPT_DE)
1642	\| RT_BIT(X86_XCPT_NM)
1643	\| RT_BIT(X86_XCPT_UD)
1644	\| RT_BIT(X86_XCPT_NP)
1645	\| RT_BIT(X86_XCPT_SS)
1646	\| RT_BIT(X86_XCPT_GP)
1647	\| RT_BIT(X86_XCPT_MF)
1648	#endif
1649	;
1650
1651	/*
1652	* Without nested paging, #PF must be intercepted to implement shadow paging.
1653	*/
1654	/** @todo NP state won't change so maybe we should build the initial trap mask up front? */
1655	if (!pVM->hm.s.fNestedPaging)
1656	u32TrapMask \|= RT_BIT(X86_XCPT_PF);
1657
1658	/* Catch floating point exceptions if we need to report them to the guest in a different way. */
1659	if (!(pCtx->cr0 & X86_CR0_NE))
1660	u32TrapMask \|= RT_BIT(X86_XCPT_MF);
1661
1662	#ifdef VBOX_STRICT
1663	Assert(u32TrapMask & RT_BIT(X86_XCPT_GP));
1664	#endif
1665
1666	/*
1667	* Intercept all exceptions in real mode as none of them can be injected directly (#GP otherwise).
1668	*/
1669	/** @todo Despite the claim to intercept everything, with NP we do not intercept #PF. Should we? */
1670	if ( CPUMIsGuestInRealModeEx(pCtx)
1671	&& pVM->hm.s.vmx.pRealModeTSS)
1672	{
1673	u32TrapMask \|= RT_BIT(X86_XCPT_DE)
1674	\| RT_BIT(X86_XCPT_DB)
1675	\| RT_BIT(X86_XCPT_NMI)
1676	\| RT_BIT(X86_XCPT_BP)
1677	\| RT_BIT(X86_XCPT_OF)
1678	\| RT_BIT(X86_XCPT_BR)
1679	\| RT_BIT(X86_XCPT_UD)
1680	\| RT_BIT(X86_XCPT_DF)
1681	\| RT_BIT(X86_XCPT_CO_SEG_OVERRUN)
1682	\| RT_BIT(X86_XCPT_TS)
1683	\| RT_BIT(X86_XCPT_NP)
1684	\| RT_BIT(X86_XCPT_SS)
1685	\| RT_BIT(X86_XCPT_GP)
1686	\| RT_BIT(X86_XCPT_MF)
1687	\| RT_BIT(X86_XCPT_AC)
1688	\| RT_BIT(X86_XCPT_MC)
1689	\| RT_BIT(X86_XCPT_XF)
1690	;
1691	}
1692
1693	int rc = VMXWriteVmcs(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32TrapMask);
1694	AssertRC(rc);
1695	}
1696
1697
1698	/**
1699	* Loads a minimal guest state.
1700	*
1701	* NOTE: Don't do anything here that can cause a jump back to ring 3!!!!!
1702	*
1703	* @param pVM Pointer to the VM.
1704	* @param pVCpu Pointer to the VMCPU.
1705	* @param pCtx Pointer to the guest CPU context.
1706	*/
1707	VMMR0DECL(void) VMXR0LoadMinimalGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
1708	{
1709	int rc;
1710	X86EFLAGS eflags;
1711
1712	Assert(!(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_ALL_GUEST));
1713
1714	/*
1715	* Load EIP, ESP and EFLAGS.
1716	*/
1717	rc = VMXWriteVmcs64(VMX_VMCS_GUEST_RIP, pCtx->rip);
1718	rc \|= VMXWriteVmcs64(VMX_VMCS_GUEST_RSP, pCtx->rsp);
1719	AssertRC(rc);
1720
1721	/*
1722	* Bits 22-31, 15, 5 & 3 must be zero. Bit 1 must be 1.
1723	*/
1724	eflags = pCtx->eflags;
1725	eflags.u32 &= VMX_EFLAGS_RESERVED_0;
1726	eflags.u32 \|= VMX_EFLAGS_RESERVED_1;
1727
1728	/*
1729	* Check if real mode emulation using v86 mode.
1730	*/
1731	if ( CPUMIsGuestInRealModeEx(pCtx)
1732	&& pVM->hm.s.vmx.pRealModeTSS)
1733	{
1734	pVCpu->hm.s.vmx.RealMode.eflags = eflags;
1735
1736	eflags.Bits.u1VM = 1;
1737	eflags.Bits.u2IOPL = 0; /* must always be 0 or else certain instructions won't cause faults. */
1738	}
1739	rc = VMXWriteVmcs(VMX_VMCS_GUEST_RFLAGS, eflags.u32);
1740	AssertRC(rc);
1741	}
1742
1743
1744	/**
1745	* Sets up TSC offsetting and VMX preemption, if supported/configured/available.
1746	*
1747	* NOTE: This function reads the host TSC value. Therefore it must be executed very
1748	* shortly before a VM entry and execution MUST NOT be rescheduled between a call to
1749	* this function and a VM entry without calling this function again.
1750	*
1751	* @returns VBox status code.
1752	* @param pVM Pointer to the VM.
1753	* @param pVCpu Pointer to the VMCPU.
1754	*/
1755	VMMR0DECL(int) VMXR0SetupTscOffsetAndPreemption(PVM pVM, PVMCPU pVCpu)
1756	{
1757	int rc = VINF_SUCCESS;
1758	bool fOffsettedTsc;
1759
1760	if (pVM->hm.s.vmx.fUsePreemptTimer)
1761	{
1762	uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVCpu, &fOffsettedTsc, &pVCpu->hm.s.vmx.u64TSCOffset);
1763
1764	/* Make sure the returned values have sane upper and lower boundaries. */
1765	uint64_t u64CpuHz = SUPGetCpuHzFromGIP(g_pSUPGlobalInfoPage);
1766
1767	cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64 of a second */
1768	cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
1769
1770	cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
1771	uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
1772	rc = VMXWriteVmcs(VMX_VMCS32_GUEST_PREEMPTION_TIMER_VALUE, cPreemptionTickCount);
1773	AssertRC(rc);
1774	}
1775	else
1776	fOffsettedTsc = TMCpuTickCanUseRealTSC(pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset);
1777
1778	if (fOffsettedTsc)
1779	{
1780	uint64_t u64CurTSC = ASMReadTSC();
1781	if (u64CurTSC + pVCpu->hm.s.vmx.u64TSCOffset > TMCpuTickGetLastSeen(pVCpu))
1782	{
1783	/* Note: VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT takes precedence over TSC_OFFSET, applies to RDTSCP too. */
1784	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, pVCpu->hm.s.vmx.u64TSCOffset);
1785	AssertRC(rc);
1786
1787	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT;
1788	rc = VMXWriteVmcs(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls);
1789	AssertRC(rc);
1790	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
1791	}
1792	else
1793	{
1794	/* Fall back to rdtsc, rdtscp emulation as we would otherwise pass decreasing tsc values to the guest. */
1795	LogFlow(("TSC %RX64 offset %RX64 time=%RX64 last=%RX64 (diff=%RX64, virt_tsc=%RX64)\n", u64CurTSC,
1796	pVCpu->hm.s.vmx.u64TSCOffset, u64CurTSC + pVCpu->hm.s.vmx.u64TSCOffset,
1797	TMCpuTickGetLastSeen(pVCpu), TMCpuTickGetLastSeen(pVCpu) - u64CurTSC - pVCpu->hm.s.vmx.u64TSCOffset,
1798	TMCpuTickGet(pVCpu)));
1799	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT;
1800	rc = VMXWriteVmcs(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls);
1801	AssertRC(rc);
1802	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscInterceptOverFlow);
1803	}
1804	}
1805	else
1806	{
1807	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT;
1808	rc = VMXWriteVmcs(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls);
1809	AssertRC(rc);
1810	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
1811	}
1812	return rc;
1813	}
1814
1815	/**
1816	* Loads the guest state.
1817	*
1818	* NOTE: Don't do anything here that can cause a jump back to ring 3!!!!!
1819	*
1820	* @returns VBox status code.
1821	* @param pVM Pointer to the VM.
1822	* @param pVCpu Pointer to the VMCPU.
1823	* @param pCtx Pointer to the guest CPU context.
1824	*/
1825	VMMR0DECL(int) VMXR0LoadGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
1826	{
1827	int rc = VINF_SUCCESS;
1828	RTGCUINTPTR val;
1829
1830	/*
1831	* VMX_VMCS_CTRL_ENTRY_CONTROLS
1832	* Set required bits to one and zero according to the MSR capabilities.
1833	*/
1834	val = pVM->hm.s.vmx.msr.vmx_entry.n.disallowed0;
1835
1836	/*
1837	* Load guest debug controls (DR7 & IA32_DEBUGCTL_MSR).
1838	* Forced to 1 on the 'first' VT-x capable CPUs; this actually includes the newest Nehalem CPUs
1839	*/
1840	val \|= VMX_VMCS_CTRL_ENTRY_CONTROLS_LOAD_DEBUG;
1841
1842	if (CPUMIsGuestInLongModeEx(pCtx))
1843	val \|= VMX_VMCS_CTRL_ENTRY_CONTROLS_IA32E_MODE_GUEST;
1844	/* else Must be zero when AMD64 is not available. */
1845
1846	/*
1847	* Mask away the bits that the CPU doesn't support.
1848	*/
1849	val &= pVM->hm.s.vmx.msr.vmx_entry.n.allowed1;
1850	rc = VMXWriteVmcs(VMX_VMCS32_CTRL_ENTRY_CONTROLS, val);
1851	AssertRC(rc);
1852
1853	/*
1854	* VMX_VMCS_CTRL_EXIT_CONTROLS
1855	* Set required bits to one and zero according to the MSR capabilities.
1856	*/
1857	val = pVM->hm.s.vmx.msr.vmx_exit.n.disallowed0;
1858
1859	/*
1860	* Save debug controls (DR7 & IA32_DEBUGCTL_MSR)
1861	* Forced to 1 on the 'first' VT-x capable CPUs; this actually includes the newest Nehalem CPUs
1862	*/
1863	val \|= VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_DEBUG;
1864
1865	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
1866	if (VMX_IS_64BIT_HOST_MODE())
1867	val \|= VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_ADDR_SPACE_SIZE;
1868	/* else Must be zero when AMD64 is not available. */
1869	#elif HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
1870	if (CPUMIsGuestInLongModeEx(pCtx))
1871	val \|= VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_ADDR_SPACE_SIZE; /* our switcher goes to long mode */
1872	else
1873	Assert(!(val & VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_ADDR_SPACE_SIZE));
1874	#endif
1875	val &= pVM->hm.s.vmx.msr.vmx_exit.n.allowed1;
1876
1877	/*
1878	* Don't acknowledge external interrupts on VM-exit.
1879	*/
1880	rc = VMXWriteVmcs(VMX_VMCS32_CTRL_EXIT_CONTROLS, val);
1881	AssertRC(rc);
1882
1883	/*
1884	* Guest CPU context: ES, CS, SS, DS, FS, GS.
1885	*/
1886	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SEGMENT_REGS)
1887	{
1888	if (pVM->hm.s.vmx.pRealModeTSS)
1889	{
1890	PGMMODE enmGuestMode = PGMGetGuestMode(pVCpu);
1891	if (pVCpu->hm.s.vmx.enmLastSeenGuestMode != enmGuestMode)
1892	{
1893	/*
1894	* Correct weird requirements for switching to protected mode.
1895	*/
1896	if ( pVCpu->hm.s.vmx.enmLastSeenGuestMode == PGMMODE_REAL
1897	&& enmGuestMode >= PGMMODE_PROTECTED)
1898	{
1899	#ifdef VBOX_WITH_REM
1900	/*
1901	* Flush the recompiler code cache as it's not unlikely the guest will rewrite code
1902	* it will later execute in real mode (OpenBSD 4.0 is one such example)
1903	*/
1904	REMFlushTBs(pVM);
1905	#endif
1906
1907	/*
1908	* DPL of all hidden selector registers must match the current CPL (0).
1909	*/
1910	pCtx->cs.Attr.n.u2Dpl = 0;
1911	pCtx->cs.Attr.n.u4Type = X86_SEL_TYPE_CODE \| X86_SEL_TYPE_RW_ACC;
1912
1913	pCtx->ds.Attr.n.u2Dpl = 0;
1914	pCtx->es.Attr.n.u2Dpl = 0;
1915	pCtx->fs.Attr.n.u2Dpl = 0;
1916	pCtx->gs.Attr.n.u2Dpl = 0;
1917	pCtx->ss.Attr.n.u2Dpl = 0;
1918	}
1919	pVCpu->hm.s.vmx.enmLastSeenGuestMode = enmGuestMode;
1920	}
1921	}
1922
1923	VMX_WRITE_SELREG(ES, es);
1924	AssertRC(rc);
1925
1926	VMX_WRITE_SELREG(CS, cs);
1927	AssertRC(rc);
1928
1929	VMX_WRITE_SELREG(SS, ss);
1930	AssertRC(rc);
1931
1932	VMX_WRITE_SELREG(DS, ds);
1933	AssertRC(rc);
1934
1935	VMX_WRITE_SELREG(FS, fs);
1936	AssertRC(rc);
1937
1938	VMX_WRITE_SELREG(GS, gs);
1939	AssertRC(rc);
1940	}
1941
1942	/*
1943	* Guest CPU context: LDTR.
1944	*/
1945	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_LDTR)
1946	{
1947	if (pCtx->ldtr.Sel == 0)
1948	{
1949	rc = VMXWriteVmcs(VMX_VMCS16_GUEST_FIELD_LDTR, 0);
1950	rc \|= VMXWriteVmcs(VMX_VMCS32_GUEST_LDTR_LIMIT, 0);
1951	rc \|= VMXWriteVmcs64(VMX_VMCS_GUEST_LDTR_BASE, 0); /* @todo removing "64" in the function should be the same. */
1952	/* Note: vmlaunch will fail with 0 or just 0x02. No idea why. */
1953	rc \|= VMXWriteVmcs(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, 0x82 /* present, LDT */);
1954	}
1955	else
1956	{
1957	rc = VMXWriteVmcs(VMX_VMCS16_GUEST_FIELD_LDTR, pCtx->ldtr.Sel);
1958	rc \|= VMXWriteVmcs(VMX_VMCS32_GUEST_LDTR_LIMIT, pCtx->ldtr.u32Limit);
1959	rc \|= VMXWriteVmcs64(VMX_VMCS_GUEST_LDTR_BASE, pCtx->ldtr.u64Base); /* @todo removing "64" and it should be the same */
1960	rc \|= VMXWriteVmcs(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, pCtx->ldtr.Attr.u);
1961	}
1962	AssertRC(rc);
1963	}
1964
1965	/*
1966	* Guest CPU context: TR.
1967	*/
1968	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_TR)
1969	{
1970	/*
1971	* Real mode emulation using v86 mode with CR4.VME (interrupt redirection
1972	* using the int bitmap in the TSS).
1973	*/
1974	if ( CPUMIsGuestInRealModeEx(pCtx)
1975	&& pVM->hm.s.vmx.pRealModeTSS)
1976	{
1977	RTGCPHYS GCPhys;
1978
1979	/* We convert it here every time as PCI regions could be reconfigured. */
1980	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
1981	AssertRC(rc);
1982
1983	rc = VMXWriteVmcs(VMX_VMCS16_GUEST_FIELD_TR, 0);
1984	rc \|= VMXWriteVmcs(VMX_VMCS32_GUEST_TR_LIMIT, HM_VTX_TSS_SIZE);
1985	rc \|= VMXWriteVmcs64(VMX_VMCS_GUEST_TR_BASE, GCPhys /* phys = virt in this mode */);
1986
1987	X86DESCATTR attr;
1988
1989	attr.u = 0;
1990	attr.n.u1Present = 1;
1991	attr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
1992	val = attr.u;
1993	}
1994	else
1995	{
1996	rc = VMXWriteVmcs(VMX_VMCS16_GUEST_FIELD_TR, pCtx->tr.Sel);
1997	rc \|= VMXWriteVmcs(VMX_VMCS32_GUEST_TR_LIMIT, pCtx->tr.u32Limit);
1998	rc \|= VMXWriteVmcs64(VMX_VMCS_GUEST_TR_BASE, pCtx->tr.u64Base);
1999
2000	val = pCtx->tr.Attr.u;
2001
2002	/* The TSS selector must be busy (REM bugs? see defect #XXXX). */
2003	if (!(val & X86_SEL_TYPE_SYS_TSS_BUSY_MASK))
2004	{
2005	if (val & 0xf)
2006	val \|= X86_SEL_TYPE_SYS_TSS_BUSY_MASK;
2007	else
2008	/* Default if no TR selector has been set (otherwise vmlaunch will fail!) */
2009	val = (val & ~0xF) \| X86_SEL_TYPE_SYS_386_TSS_BUSY;
2010	}
2011	AssertMsg((val & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY \|\| (val & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY,
2012	("%#x\n", val));
2013	}
2014	rc \|= VMXWriteVmcs(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, val);
2015	AssertRC(rc);
2016	}
2017
2018	/*
2019	* Guest CPU context: GDTR.
2020	*/
2021	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_GDTR)
2022	{
2023	rc = VMXWriteVmcs(VMX_VMCS32_GUEST_GDTR_LIMIT, pCtx->gdtr.cbGdt);
2024	rc \|= VMXWriteVmcs64(VMX_VMCS_GUEST_GDTR_BASE, pCtx->gdtr.pGdt);
2025	AssertRC(rc);
2026	}
2027
2028	/*
2029	* Guest CPU context: IDTR.
2030	*/
2031	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_IDTR)
2032	{
2033	rc = VMXWriteVmcs(VMX_VMCS32_GUEST_IDTR_LIMIT, pCtx->idtr.cbIdt);
2034	rc \|= VMXWriteVmcs64(VMX_VMCS_GUEST_IDTR_BASE, pCtx->idtr.pIdt);
2035	AssertRC(rc);
2036	}
2037
2038	/*
2039	* Sysenter MSRs.
2040	*/
2041	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_MSR)
2042	{
2043	rc = VMXWriteVmcs(VMX_VMCS32_GUEST_SYSENTER_CS, pCtx->SysEnter.cs);
2044	rc \|= VMXWriteVmcs64(VMX_VMCS_GUEST_SYSENTER_EIP, pCtx->SysEnter.eip);
2045	rc \|= VMXWriteVmcs64(VMX_VMCS_GUEST_SYSENTER_ESP, pCtx->SysEnter.esp);
2046	AssertRC(rc);
2047	}
2048
2049	/*
2050	* Guest CPU context: Control registers.
2051	*/
2052	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR0)
2053	{
2054	val = pCtx->cr0;
2055	rc = VMXWriteVmcs(VMX_VMCS_CTRL_CR0_READ_SHADOW, val);
2056	Log2(("Guest CR0-shadow %08x\n", val));
2057	if (CPUMIsGuestFPUStateActive(pVCpu) == false)
2058	{
2059	/* Always use #NM exceptions to load the FPU/XMM state on demand. */
2060	val \|= X86_CR0_TS \| X86_CR0_ET \| X86_CR0_NE \| X86_CR0_MP;
2061	}
2062	else
2063	{
2064	/** @todo check if we support the old style mess correctly. */
2065	if (!(val & X86_CR0_NE))
2066	Log(("Forcing X86_CR0_NE!!!\n"));
2067
2068	val \|= X86_CR0_NE; /* always turn on the native mechanism to report FPU errors (old style uses interrupts) */
2069	}
2070	/* Protected mode & paging are always enabled; we use them for emulating real and protected mode without paging too. */
2071	if (!pVM->hm.s.vmx.fUnrestrictedGuest)
2072	val \|= X86_CR0_PE \| X86_CR0_PG;
2073
2074	if (pVM->hm.s.fNestedPaging)
2075	{
2076	if (CPUMIsGuestInPagedProtectedModeEx(pCtx))
2077	{
2078	/* Disable CR3 read/write monitoring as we don't need it for EPT. */
2079	pVCpu->hm.s.vmx.u32ProcCtls &= ~( VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
2080	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT);
2081	}
2082	else
2083	{
2084	/* Reenable CR3 read/write monitoring as our identity mapped page table is active. */
2085	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
2086	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT;
2087	}
2088	rc = VMXWriteVmcs(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls);
2089	AssertRC(rc);
2090	}
2091	else
2092	{
2093	/* Note: We must also set this as we rely on protecting various pages for which supervisor writes must be caught. */
2094	val \|= X86_CR0_WP;
2095	}
2096
2097	/* Always enable caching. */
2098	val &= ~(X86_CR0_CD\|X86_CR0_NW);
2099
2100	rc \|= VMXWriteVmcs64(VMX_VMCS_GUEST_CR0, val);
2101	Log2(("Guest CR0 %08x\n", val));
2102
2103	/*
2104	* CR0 flags owned by the host; if the guests attempts to change them, then the VM will exit.
2105	*/
2106	val = X86_CR0_PE /* Must monitor this bit (assumptions are made for real mode emulation) */
2107	\| X86_CR0_WP /* Must monitor this bit (it must always be enabled). */
2108	\| X86_CR0_PG /* Must monitor this bit (assumptions are made for real mode & protected mode without paging emulation) */
2109	\| X86_CR0_CD /* Bit not restored during VM-exit! */
2110	\| X86_CR0_NW /* Bit not restored during VM-exit! */
2111	\| X86_CR0_NE;
2112
2113	/*
2114	* When the guest's FPU state is active, then we no longer care about the FPU related bits.
2115	*/
2116	if (CPUMIsGuestFPUStateActive(pVCpu) == false)
2117	val \|= X86_CR0_TS \| X86_CR0_ET \| X86_CR0_MP;
2118
2119	pVCpu->hm.s.vmx.cr0_mask = val;
2120
2121	rc \|= VMXWriteVmcs(VMX_VMCS_CTRL_CR0_MASK, val);
2122	Log2(("Guest CR0-mask %08x\n", val));
2123	AssertRC(rc);
2124	}
2125
2126	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR4)
2127	{
2128	rc = VMXWriteVmcs(VMX_VMCS_CTRL_CR4_READ_SHADOW, pCtx->cr4);
2129	Log2(("Guest CR4-shadow %08x\n", pCtx->cr4));
2130	/* Set the required bits in cr4 too (currently X86_CR4_VMXE). */
2131	val = pCtx->cr4 \| (uint32_t)pVM->hm.s.vmx.msr.vmx_cr4_fixed0;
2132
2133	if (!pVM->hm.s.fNestedPaging)
2134	{
2135	switch (pVCpu->hm.s.enmShadowMode)
2136	{
2137	case PGMMODE_REAL: /* Real mode -> emulated using v86 mode */
2138	case PGMMODE_PROTECTED: /* Protected mode, no paging -> emulated using identity mapping. */
2139	case PGMMODE_32_BIT: /* 32-bit paging. */
2140	val &= ~X86_CR4_PAE;
2141	break;
2142
2143	case PGMMODE_PAE: /* PAE paging. */
2144	case PGMMODE_PAE_NX: /* PAE paging with NX enabled. */
2145	/** Must use PAE paging as we could use physical memory > 4 GB */
2146	val \|= X86_CR4_PAE;
2147	break;
2148
2149	case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
2150	case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
2151	#ifdef VBOX_ENABLE_64_BITS_GUESTS
2152	break;
2153	#else
2154	AssertFailed();
2155	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
2156	#endif
2157	default: /* shut up gcc */
2158	AssertFailed();
2159	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
2160	}
2161	}
2162	else if ( !CPUMIsGuestInPagedProtectedModeEx(pCtx)
2163	&& !pVM->hm.s.vmx.fUnrestrictedGuest)
2164	{
2165	/* We use 4 MB pages in our identity mapping page table for real and protected mode without paging. */
2166	val \|= X86_CR4_PSE;
2167	/* Our identity mapping is a 32 bits page directory. */
2168	val &= ~X86_CR4_PAE;
2169	}
2170
2171	/*
2172	* Turn off VME if we're in emulated real mode.
2173	*/
2174	if ( CPUMIsGuestInRealModeEx(pCtx)
2175	&& pVM->hm.s.vmx.pRealModeTSS)
2176	{
2177	val &= ~X86_CR4_VME;
2178	}
2179
2180	rc \|= VMXWriteVmcs64(VMX_VMCS_GUEST_CR4, val);
2181	Log2(("Guest CR4 %08x\n", val));
2182
2183	/*
2184	* CR4 flags owned by the host; if the guests attempts to change them, then the VM will exit.
2185	*/
2186	val = 0
2187	\| X86_CR4_VME
2188	\| X86_CR4_PAE
2189	\| X86_CR4_PGE
2190	\| X86_CR4_PSE
2191	\| X86_CR4_VMXE;
2192	pVCpu->hm.s.vmx.cr4_mask = val;
2193
2194	rc \|= VMXWriteVmcs(VMX_VMCS_CTRL_CR4_MASK, val);
2195	Log2(("Guest CR4-mask %08x\n", val));
2196	AssertRC(rc);
2197	}
2198
2199	#if 0
2200	/* Enable single stepping if requested and CPU supports it. */
2201	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_TRAP_FLAG)
2202	if (DBGFIsStepping(pVCpu))
2203	{
2204	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_TRAP_FLAG;
2205	rc = VMXWriteVmcs(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls);
2206	AssertRC(rc);
2207	}
2208	#endif
2209
2210	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR3)
2211	{
2212	if (pVM->hm.s.fNestedPaging)
2213	{
2214	Assert(PGMGetHyperCR3(pVCpu));
2215	pVCpu->hm.s.vmx.GCPhysEPTP = PGMGetHyperCR3(pVCpu);
2216
2217	Assert(!(pVCpu->hm.s.vmx.GCPhysEPTP & 0xfff));
2218	/** @todo Check the IA32_VMX_EPT_VPID_CAP MSR for other supported memory types. */
2219	pVCpu->hm.s.vmx.GCPhysEPTP \|= VMX_EPT_MEMTYPE_WB
2220	\| (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
2221
2222	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVCpu->hm.s.vmx.GCPhysEPTP);
2223	AssertRC(rc);
2224
2225	if ( !CPUMIsGuestInPagedProtectedModeEx(pCtx)
2226	&& !pVM->hm.s.vmx.fUnrestrictedGuest)
2227	{
2228	RTGCPHYS GCPhys;
2229
2230	/* We convert it here every time as PCI regions could be reconfigured. */
2231	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
2232	AssertMsgRC(rc, ("pNonPagingModeEPTPageTable = %RGv\n", pVM->hm.s.vmx.pNonPagingModeEPTPageTable));
2233
2234	/*
2235	* We use our identity mapping page table here as we need to map guest virtual to
2236	* guest physical addresses; EPT will take care of the translation to host physical addresses.
2237	*/
2238	val = GCPhys;
2239	}
2240	else
2241	{
2242	/* Save the real guest CR3 in VMX_VMCS_GUEST_CR3 */
2243	val = pCtx->cr3;
2244	rc = hmR0VmxLoadPaePdpes(pVCpu, pCtx);
2245	AssertRCReturn(rc, rc);
2246	}
2247	}
2248	else
2249	{
2250	val = PGMGetHyperCR3(pVCpu);
2251	Assert(val \|\| VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL));
2252	}
2253
2254	/* Save our shadow CR3 register. */
2255	rc = VMXWriteVmcs64(VMX_VMCS_GUEST_CR3, val);
2256	AssertRC(rc);
2257	}
2258
2259	/*
2260	* Guest CPU context: Debug registers.
2261	*/
2262	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_DEBUG)
2263	{
2264	pCtx->dr[6] \|= X86_DR6_INIT_VAL; /* set all reserved bits to 1. */
2265	pCtx->dr[6] &= ~RT_BIT(12); /* must be zero. */
2266
2267	pCtx->dr[7] &= 0xffffffff; /* upper 32 bits reserved */
2268	pCtx->dr[7] &= ~(RT_BIT(11) \| RT_BIT(12) \| RT_BIT(14) \| RT_BIT(15)); /* must be zero */
2269	pCtx->dr[7] \|= 0x400; /* must be one */
2270
2271	/* Resync DR7 */
2272	rc = VMXWriteVmcs64(VMX_VMCS_GUEST_DR7, pCtx->dr[7]);
2273	AssertRC(rc);
2274
2275	#ifdef DEBUG
2276	/* Sync the hypervisor debug state now if any breakpoint is armed. */
2277	if ( CPUMGetHyperDR7(pVCpu) & (X86_DR7_ENABLED_MASK\|X86_DR7_GD)
2278	&& !CPUMIsHyperDebugStateActive(pVCpu)
2279	&& !DBGFIsStepping(pVCpu))
2280	{
2281	/* Save the host and load the hypervisor debug state. */
2282	rc = CPUMR0LoadHyperDebugState(pVM, pVCpu, pCtx, true /* include DR6 */);
2283	AssertRC(rc);
2284
2285	/* DRx intercepts remain enabled. */
2286
2287	/* Override dr7 with the hypervisor value. */
2288	rc = VMXWriteVmcs64(VMX_VMCS_GUEST_DR7, CPUMGetHyperDR7(pVCpu));
2289	AssertRC(rc);
2290	}
2291	else
2292	#endif
2293	/* Sync the debug state now if any breakpoint is armed. */
2294	if ( (pCtx->dr[7] & (X86_DR7_ENABLED_MASK\|X86_DR7_GD))
2295	&& !CPUMIsGuestDebugStateActive(pVCpu)
2296	&& !DBGFIsStepping(pVCpu))
2297	{
2298	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
2299
2300	/* Disable DRx move intercepts. */
2301	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT;
2302	rc = VMXWriteVmcs(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls);
2303	AssertRC(rc);
2304
2305	/* Save the host and load the guest debug state. */
2306	rc = CPUMR0LoadGuestDebugState(pVM, pVCpu, pCtx, true /* include DR6 */);
2307	AssertRC(rc);
2308	}
2309
2310	/* IA32_DEBUGCTL MSR. */
2311	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, 0);
2312	AssertRC(rc);
2313
2314	/** @todo do we really ever need this? */
2315	rc \|= VMXWriteVmcs(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
2316	AssertRC(rc);
2317	}
2318
2319	/*
2320	* 64-bit guest mode.
2321	*/
2322	if (CPUMIsGuestInLongModeEx(pCtx))
2323	{
2324	#if !defined(VBOX_ENABLE_64_BITS_GUESTS)
2325	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
2326	#elif HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2327	pVCpu->hm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
2328	#else
2329	# ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2330	if (!pVM->hm.s.fAllow64BitGuests)
2331	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
2332	# endif
2333	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM64;
2334	#endif
2335	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_MSR)
2336	{
2337	/* Update these as wrmsr might have changed them. */
2338	rc = VMXWriteVmcs64(VMX_VMCS_GUEST_FS_BASE, pCtx->fs.u64Base);
2339	AssertRC(rc);
2340	rc = VMXWriteVmcs64(VMX_VMCS_GUEST_GS_BASE, pCtx->gs.u64Base);
2341	AssertRC(rc);
2342	}
2343	}
2344	else
2345	{
2346	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
2347	}
2348
2349	hmR0VmxUpdateExceptionBitmap(pVM, pVCpu, pCtx);
2350
2351	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
2352	/*
2353	* Store all guest MSRs in the VM-entry load area, so they will be loaded
2354	* during VM-entry and restored into the VM-exit store area during VM-exit.
2355	*/
2356	PVMXMSR pMsr = (PVMXMSR)pVCpu->hm.s.vmx.pvGuestMsr;
2357	unsigned idxMsr = 0;
2358
2359	uint32_t u32GstExtFeatures;
2360	uint32_t u32Temp;
2361	CPUMGetGuestCpuId(pVCpu, 0x80000001, &u32Temp, &u32Temp, &u32Temp, &u32GstExtFeatures);
2362
2363	if (u32GstExtFeatures & (X86_CPUID_EXT_FEATURE_EDX_NX \| X86_CPUID_EXT_FEATURE_EDX_LONG_MODE))
2364	{
2365	pMsr->u32IndexMSR = MSR_K6_EFER;
2366	pMsr->u32Reserved = 0;
2367	pMsr->u64Value = pCtx->msrEFER;
2368	/* VT-x will complain if only MSR_K6_EFER_LME is set. */
2369	if (!CPUMIsGuestInLongModeEx(pCtx))
2370	pMsr->u64Value &= ~(MSR_K6_EFER_LMA \| MSR_K6_EFER_LME);
2371	pMsr++; idxMsr++;
2372
2373	if (u32GstExtFeatures & X86_CPUID_EXT_FEATURE_EDX_LONG_MODE)
2374	{
2375	pMsr->u32IndexMSR = MSR_K8_LSTAR;
2376	pMsr->u32Reserved = 0;
2377	pMsr->u64Value = pCtx->msrLSTAR; /* 64 bits mode syscall rip */
2378	pMsr++; idxMsr++;
2379	pMsr->u32IndexMSR = MSR_K6_STAR;
2380	pMsr->u32Reserved = 0;
2381	pMsr->u64Value = pCtx->msrSTAR; /* legacy syscall eip, cs & ss */
2382	pMsr++; idxMsr++;
2383	pMsr->u32IndexMSR = MSR_K8_SF_MASK;
2384	pMsr->u32Reserved = 0;
2385	pMsr->u64Value = pCtx->msrSFMASK; /* syscall flag mask */
2386	pMsr++; idxMsr++;
2387
2388	/* The KERNEL_GS_BASE MSR doesn't work reliably with auto load/store. See @bugref{6208} */
2389	#if 0
2390	pMsr->u32IndexMSR = MSR_K8_KERNEL_GS_BASE;
2391	pMsr->u32Reserved = 0;
2392	pMsr->u64Value = pCtx->msrKERNELGSBASE; /* swapgs exchange value */
2393	pMsr++; idxMsr++;
2394	#endif
2395	}
2396	}
2397
2398	if ( pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP
2399	&& (u32GstExtFeatures & X86_CPUID_EXT_FEATURE_EDX_RDTSCP))
2400	{
2401	pMsr->u32IndexMSR = MSR_K8_TSC_AUX;
2402	pMsr->u32Reserved = 0;
2403	rc = CPUMQueryGuestMsr(pVCpu, MSR_K8_TSC_AUX, &pMsr->u64Value);
2404	AssertRC(rc);
2405	pMsr++; idxMsr++;
2406	}
2407
2408	pVCpu->hm.s.vmx.cGuestMsrs = idxMsr;
2409
2410	rc = VMXWriteVmcs(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, idxMsr);
2411	AssertRC(rc);
2412
2413	rc = VMXWriteVmcs(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, idxMsr);
2414	AssertRC(rc);
2415	#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
2416
2417	/* Done with the major changes */
2418	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_ALL_GUEST;
2419
2420	/* Minimal guest state update (ESP, EIP, EFLAGS mostly) */
2421	VMXR0LoadMinimalGuestState(pVM, pVCpu, pCtx);
2422	return rc;
2423	}
2424
2425
2426	/**
2427	* Syncs back the guest state from VMCS.
2428	*
2429	* @returns VBox status code.
2430	* @param pVM Pointer to the VM.
2431	* @param pVCpu Pointer to the VMCPU.
2432	* @param pCtx Pointer to the guest CPU context.
2433	*/
2434	DECLINLINE(int) VMXR0SaveGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
2435	{
2436	RTGCUINTREG val, valShadow;
2437	RTGCUINTPTR uInterruptState;
2438	int rc;
2439
2440	/* First sync back EIP, ESP, and EFLAGS. */
2441	rc = VMXReadCachedVmcs(VMX_VMCS_GUEST_RIP, &val);
2442	AssertRC(rc);
2443	pCtx->rip = val;
2444	rc = VMXReadCachedVmcs(VMX_VMCS_GUEST_RSP, &val);
2445	AssertRC(rc);
2446	pCtx->rsp = val;
2447	rc = VMXReadCachedVmcs(VMX_VMCS_GUEST_RFLAGS, &val);
2448	AssertRC(rc);
2449	pCtx->eflags.u32 = val;
2450
2451	/* Take care of instruction fusing (sti, mov ss) */
2452	rc \|= VMXReadCachedVmcs(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &val);
2453	uInterruptState = val;
2454	if (uInterruptState != 0)
2455	{
2456	Assert(uInterruptState <= 2); /* only sti & mov ss */
2457	Log(("uInterruptState %x eip=%RGv\n", (uint32_t)uInterruptState, pCtx->rip));
2458	EMSetInhibitInterruptsPC(pVCpu, pCtx->rip);
2459	}
2460	else
2461	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
2462
2463	/* Control registers. */
2464	VMXReadCachedVmcs(VMX_VMCS_CTRL_CR0_READ_SHADOW, &valShadow);
2465	VMXReadCachedVmcs(VMX_VMCS_GUEST_CR0, &val);
2466	val = (valShadow & pVCpu->hm.s.vmx.cr0_mask) \| (val & ~pVCpu->hm.s.vmx.cr0_mask);
2467	CPUMSetGuestCR0(pVCpu, val);
2468
2469	VMXReadCachedVmcs(VMX_VMCS_CTRL_CR4_READ_SHADOW, &valShadow);
2470	VMXReadCachedVmcs(VMX_VMCS_GUEST_CR4, &val);
2471	val = (valShadow & pVCpu->hm.s.vmx.cr4_mask) \| (val & ~pVCpu->hm.s.vmx.cr4_mask);
2472	CPUMSetGuestCR4(pVCpu, val);
2473
2474	/*
2475	* No reason to sync back the CRx registers. They can't be changed by the guest unless in
2476	* the nested paging case where CR3 & CR4 can be changed by the guest.
2477	*/
2478	if ( pVM->hm.s.fNestedPaging
2479	&& CPUMIsGuestInPagedProtectedModeEx(pCtx)) /** @todo check if we will always catch mode switches and such... */
2480	{
2481	PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
2482
2483	/* Can be updated behind our back in the nested paging case. */
2484	CPUMSetGuestCR2(pVCpu, pCache->cr2);
2485
2486	VMXReadCachedVmcs(VMX_VMCS_GUEST_CR3, &val);
2487
2488	if (val != pCtx->cr3)
2489	{
2490	CPUMSetGuestCR3(pVCpu, val);
2491	PGMUpdateCR3(pVCpu, val);
2492	}
2493	rc = hmR0VmxSavePaePdpes(pVCpu, pCtx);
2494	AssertRCReturn(rc, rc);
2495	}
2496
2497	/* Sync back DR7. */
2498	VMXReadCachedVmcs(VMX_VMCS_GUEST_DR7, &val);
2499	pCtx->dr[7] = val;
2500
2501	/* Guest CPU context: ES, CS, SS, DS, FS, GS. */
2502	VMX_READ_SELREG(ES, es);
2503	VMX_READ_SELREG(SS, ss);
2504	VMX_READ_SELREG(CS, cs);
2505	VMX_READ_SELREG(DS, ds);
2506	VMX_READ_SELREG(FS, fs);
2507	VMX_READ_SELREG(GS, gs);
2508
2509	/* System MSRs */
2510	VMXReadCachedVmcs(VMX_VMCS32_GUEST_SYSENTER_CS, &val);
2511	pCtx->SysEnter.cs = val;
2512	VMXReadCachedVmcs(VMX_VMCS_GUEST_SYSENTER_EIP, &val);
2513	pCtx->SysEnter.eip = val;
2514	VMXReadCachedVmcs(VMX_VMCS_GUEST_SYSENTER_ESP, &val);
2515	pCtx->SysEnter.esp = val;
2516
2517	/* Misc. registers; must sync everything otherwise we can get out of sync when jumping to ring 3. */
2518	VMX_READ_SELREG(LDTR, ldtr);
2519
2520	VMXReadCachedVmcs(VMX_VMCS32_GUEST_GDTR_LIMIT, &val);
2521	pCtx->gdtr.cbGdt = val;
2522	VMXReadCachedVmcs(VMX_VMCS_GUEST_GDTR_BASE, &val);
2523	pCtx->gdtr.pGdt = val;
2524
2525	VMXReadCachedVmcs(VMX_VMCS32_GUEST_IDTR_LIMIT, &val);
2526	pCtx->idtr.cbIdt = val;
2527	VMXReadCachedVmcs(VMX_VMCS_GUEST_IDTR_BASE, &val);
2528	pCtx->idtr.pIdt = val;
2529
2530	/* Real mode emulation using v86 mode. */
2531	if ( CPUMIsGuestInRealModeEx(pCtx)
2532	&& pVM->hm.s.vmx.pRealModeTSS)
2533	{
2534	/* Hide our emulation flags */
2535	pCtx->eflags.Bits.u1VM = 0;
2536
2537	/* Restore original IOPL setting as we always use 0. */
2538	pCtx->eflags.Bits.u2IOPL = pVCpu->hm.s.vmx.RealMode.eflags.Bits.u2IOPL;
2539
2540	/* Force a TR resync every time in case we switch modes. */
2541	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_TR;
2542	}
2543	else
2544	{
2545	/* In real mode we have a fake TSS, so only sync it back when it's supposed to be valid. */
2546	VMX_READ_SELREG(TR, tr);
2547	}
2548
2549	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
2550	/*
2551	* Save the possibly changed MSRs that we automatically restore and save during a world switch.
2552	*/
2553	for (unsigned i = 0; i < pVCpu->hm.s.vmx.cGuestMsrs; i++)
2554	{
2555	PVMXMSR pMsr = (PVMXMSR)pVCpu->hm.s.vmx.pvGuestMsr;
2556	pMsr += i;
2557
2558	switch (pMsr->u32IndexMSR)
2559	{
2560	case MSR_K8_LSTAR:
2561	pCtx->msrLSTAR = pMsr->u64Value;
2562	break;
2563	case MSR_K6_STAR:
2564	pCtx->msrSTAR = pMsr->u64Value;
2565	break;
2566	case MSR_K8_SF_MASK:
2567	pCtx->msrSFMASK = pMsr->u64Value;
2568	break;
2569	/* The KERNEL_GS_BASE MSR doesn't work reliably with auto load/store. See @bugref{6208} */
2570	#if 0
2571	case MSR_K8_KERNEL_GS_BASE:
2572	pCtx->msrKERNELGSBASE = pMsr->u64Value;
2573	break;
2574	#endif
2575	case MSR_K8_TSC_AUX:
2576	CPUMSetGuestMsr(pVCpu, MSR_K8_TSC_AUX, pMsr->u64Value);
2577	break;
2578
2579	case MSR_K6_EFER:
2580	/* EFER can't be changed without causing a VM-exit. */
2581	/* Assert(pCtx->msrEFER == pMsr->u64Value); */
2582	break;
2583
2584	default:
2585	AssertFailed();
2586	return VERR_HM_UNEXPECTED_LD_ST_MSR;
2587	}
2588	}
2589	#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
2590	return VINF_SUCCESS;
2591	}
2592
2593
2594	/**
2595	* Dummy placeholder for TLB flush handling before VM-entry. Used in the case
2596	* where neither EPT nor VPID is supported by the CPU.
2597	*
2598	* @param pVM Pointer to the VM.
2599	* @param pVCpu Pointer to the VMCPU.
2600	*/
2601	static DECLCALLBACK(void) hmR0VmxSetupTLBDummy(PVM pVM, PVMCPU pVCpu)
2602	{
2603	NOREF(pVM);
2604	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
2605	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
2606	pVCpu->hm.s.TlbShootdown.cPages = 0;
2607	return;
2608	}
2609
2610
2611	/**
2612	* Setup the tagged TLB for EPT+VPID.
2613	*
2614	* @param pVM Pointer to the VM.
2615	* @param pVCpu Pointer to the VMCPU.
2616	*/
2617	static DECLCALLBACK(void) hmR0VmxSetupTLBBoth(PVM pVM, PVMCPU pVCpu)
2618	{
2619	PHMGLOBLCPUINFO pCpu;
2620
2621	Assert(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid);
2622
2623	pCpu = HMR0GetCurrentCpu();
2624
2625	/*
2626	* Force a TLB flush for the first world switch if the current CPU differs from the one we ran on last
2627	* This can happen both for start & resume due to long jumps back to ring-3.
2628	* If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
2629	* or the host Cpu is online after a suspend/resume, so we cannot reuse the current ASID anymore.
2630	*/
2631	bool fNewAsid = false;
2632	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2633	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2634	{
2635	pVCpu->hm.s.fForceTLBFlush = true;
2636	fNewAsid = true;
2637	}
2638
2639	/*
2640	* Check for explicit TLB shootdowns.
2641	*/
2642	if (VMCPU_FF_TESTANDCLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2643	pVCpu->hm.s.fForceTLBFlush = true;
2644
2645	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2646
2647	if (pVCpu->hm.s.fForceTLBFlush)
2648	{
2649	if (fNewAsid)
2650	{
2651	++pCpu->uCurrentAsid;
2652	if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
2653	{
2654	pCpu->uCurrentAsid = 1; /* start at 1; host uses 0 */
2655	pCpu->cTlbFlushes++;
2656	pCpu->fFlushAsidBeforeUse = true;
2657	}
2658
2659	pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
2660	if (pCpu->fFlushAsidBeforeUse)
2661	hmR0VmxFlushVPID(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
2662	}
2663	else
2664	{
2665	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
2666	hmR0VmxFlushVPID(pVM, pVCpu, VMX_FLUSH_VPID_SINGLE_CONTEXT, 0 /* GCPtr */);
2667	else
2668	hmR0VmxFlushEPT(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
2669	}
2670
2671	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2672	pVCpu->hm.s.fForceTLBFlush = false;
2673	}
2674	else
2675	{
2676	AssertMsg(pVCpu->hm.s.uCurrentAsid && pCpu->uCurrentAsid,
2677	("hm->uCurrentAsid=%lu hm->cTlbFlushes=%lu cpu->uCurrentAsid=%lu cpu->cTlbFlushes=%lu\n",
2678	pVCpu->hm.s.uCurrentAsid, pVCpu->hm.s.cTlbFlushes,
2679	pCpu->uCurrentAsid, pCpu->cTlbFlushes));
2680
2681	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
2682	* not be executed. See hmQueueInvlPage() where it is commented
2683	* out. Support individual entry flushing someday. */
2684	if (VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
2685	{
2686	STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
2687
2688	/*
2689	* Flush individual guest entries using VPID from the TLB or as little as possible with EPT
2690	* as supported by the CPU.
2691	*/
2692	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
2693	{
2694	for (unsigned i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
2695	hmR0VmxFlushVPID(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
2696	}
2697	else
2698	hmR0VmxFlushEPT(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
2699	}
2700	else
2701	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
2702	}
2703
2704	pVCpu->hm.s.TlbShootdown.cPages = 0;
2705	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
2706
2707	AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
2708	("Flush count mismatch for cpu %d (%x vs %x)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
2709	AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2710	("cpu%d uCurrentAsid = %x\n", pCpu->idCpu, pCpu->uCurrentAsid));
2711	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2712	("cpu%d VM uCurrentAsid = %x\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2713
2714	/* Update VMCS with the VPID. */
2715	int rc = VMXWriteVmcs(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
2716	AssertRC(rc);
2717	}
2718
2719
2720	/**
2721	* Setup the tagged TLB for EPT only.
2722	*
2723	* @returns VBox status code.
2724	* @param pVM Pointer to the VM.
2725	* @param pVCpu Pointer to the VMCPU.
2726	*/
2727	static DECLCALLBACK(void) hmR0VmxSetupTLBEPT(PVM pVM, PVMCPU pVCpu)
2728	{
2729	PHMGLOBLCPUINFO pCpu;
2730
2731	Assert(pVM->hm.s.fNestedPaging);
2732	Assert(!pVM->hm.s.vmx.fVpid);
2733
2734	pCpu = HMR0GetCurrentCpu();
2735
2736	/*
2737	* Force a TLB flush for the first world switch if the current CPU differs from the one we ran on last
2738	* This can happen both for start & resume due to long jumps back to ring-3.
2739	* A change in the TLB flush count implies the host Cpu is online after a suspend/resume.
2740	*/
2741	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2742	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2743	{
2744	pVCpu->hm.s.fForceTLBFlush = true;
2745	}
2746
2747	/*
2748	* Check for explicit TLB shootdown flushes.
2749	*/
2750	if (VMCPU_FF_TESTANDCLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2751	pVCpu->hm.s.fForceTLBFlush = true;
2752
2753	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2754	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2755
2756	if (pVCpu->hm.s.fForceTLBFlush)
2757	hmR0VmxFlushEPT(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
2758	else
2759	{
2760	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
2761	* not be executed. See hmQueueInvlPage() where it is commented
2762	* out. Support individual entry flushing someday. */
2763	if (VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
2764	{
2765	/*
2766	* We cannot flush individual entries without VPID support. Flush using EPT.
2767	*/
2768	STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
2769	hmR0VmxFlushEPT(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
2770	}
2771	}
2772	pVCpu->hm.s.TlbShootdown.cPages= 0;
2773	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
2774
2775	#ifdef VBOX_WITH_STATISTICS
2776	/** @todo r=ramshankar: this is not accurate anymore with the VPID+EPT
2777	* handling. Should be fixed later. */
2778	if (pVCpu->hm.s.fForceTLBFlush)
2779	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2780	else
2781	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
2782	#endif
2783	}
2784
2785
2786	/**
2787	* Setup the tagged TLB for VPID.
2788	*
2789	* @returns VBox status code.
2790	* @param pVM Pointer to the VM.
2791	* @param pVCpu Pointer to the VMCPU.
2792	*/
2793	static DECLCALLBACK(void) hmR0VmxSetupTLBVPID(PVM pVM, PVMCPU pVCpu)
2794	{
2795	PHMGLOBLCPUINFO pCpu;
2796
2797	Assert(pVM->hm.s.vmx.fVpid);
2798	Assert(!pVM->hm.s.fNestedPaging);
2799
2800	pCpu = HMR0GetCurrentCpu();
2801
2802	/*
2803	* Force a TLB flush for the first world switch if the current CPU differs from the one we ran on last
2804	* This can happen both for start & resume due to long jumps back to ring-3.
2805	* If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
2806	* or the host Cpu is online after a suspend/resume, so we cannot reuse the current ASID anymore.
2807	*/
2808	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2809	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2810	{
2811	/* Force a TLB flush on VM entry. */
2812	pVCpu->hm.s.fForceTLBFlush = true;
2813	}
2814
2815	/*
2816	* Check for explicit TLB shootdown flushes.
2817	*/
2818	if (VMCPU_FF_TESTANDCLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2819	pVCpu->hm.s.fForceTLBFlush = true;
2820
2821	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2822
2823	if (pVCpu->hm.s.fForceTLBFlush)
2824	{
2825	++pCpu->uCurrentAsid;
2826	if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
2827	{
2828	pCpu->uCurrentAsid = 1; /* start at 1; host uses 0 */
2829	pCpu->cTlbFlushes++;
2830	pCpu->fFlushAsidBeforeUse = true;
2831	}
2832
2833	pVCpu->hm.s.fForceTLBFlush = false;
2834	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2835	pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
2836	if (pCpu->fFlushAsidBeforeUse)
2837	hmR0VmxFlushVPID(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
2838	}
2839	else
2840	{
2841	AssertMsg(pVCpu->hm.s.uCurrentAsid && pCpu->uCurrentAsid,
2842	("hm->uCurrentAsid=%lu hm->cTlbFlushes=%lu cpu->uCurrentAsid=%lu cpu->cTlbFlushes=%lu\n",
2843	pVCpu->hm.s.uCurrentAsid, pVCpu->hm.s.cTlbFlushes,
2844	pCpu->uCurrentAsid, pCpu->cTlbFlushes));
2845
2846	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
2847	* not be executed. See hmQueueInvlPage() where it is commented
2848	* out. Support individual entry flushing someday. */
2849	if (VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
2850	{
2851	/*
2852	* Flush individual guest entries using VPID from the TLB or as little as possible with EPT
2853	* as supported by the CPU.
2854	*/
2855	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
2856	{
2857	for (unsigned i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
2858	hmR0VmxFlushVPID(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
2859	}
2860	else
2861	hmR0VmxFlushVPID(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
2862	}
2863	}
2864	pVCpu->hm.s.TlbShootdown.cPages = 0;
2865	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
2866
2867	AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
2868	("Flush count mismatch for cpu %d (%x vs %x)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
2869	AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2870	("cpu%d uCurrentAsid = %x\n", pCpu->idCpu, pCpu->uCurrentAsid));
2871	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2872	("cpu%d VM uCurrentAsid = %x\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2873
2874	int rc = VMXWriteVmcs(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
2875	AssertRC(rc);
2876
2877	# ifdef VBOX_WITH_STATISTICS
2878	/** @todo r=ramshankar: this is not accurate anymore with EPT+VPID handling.
2879	* Should be fixed later. */
2880	if (pVCpu->hm.s.fForceTLBFlush)
2881	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2882	else
2883	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
2884	# endif
2885	}
2886
2887
2888	/**
2889	* Runs guest code in a VT-x VM.
2890	*
2891	* @returns VBox status code.
2892	* @param pVM Pointer to the VM.
2893	* @param pVCpu Pointer to the VMCPU.
2894	* @param pCtx Pointer to the guest CPU context.
2895	*/
2896	VMMR0DECL(int) VMXR0RunGuestCode(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
2897	{
2898	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
2899	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit1);
2900	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit2);
2901
2902	VBOXSTRICTRC rc = VINF_SUCCESS;
2903	int rc2;
2904	RTGCUINTREG val;
2905	RTGCUINTREG exitReason = (RTGCUINTREG)VMX_EXIT_INVALID;
2906	RTGCUINTREG instrError, cbInstr;
2907	RTGCUINTPTR exitQualification = 0;
2908	RTGCUINTPTR intInfo = 0; /* shut up buggy gcc 4 */
2909	RTGCUINTPTR errCode, instrInfo;
2910	bool fSetupTPRCaching = false;
2911	bool fNeedTscSetup = true;
2912	uint64_t u64OldLSTAR = 0;
2913	uint8_t u8LastTPR = 0;
2914	RTCCUINTREG uOldEFlags = ~(RTCCUINTREG)0;
2915	unsigned cResume = 0;
2916	#ifdef VBOX_STRICT
2917	RTCPUID idCpuCheck;
2918	bool fWasInLongMode = false;
2919	#endif
2920	#ifdef VBOX_HIGH_RES_TIMERS_HACK_IN_RING0
2921	uint64_t u64LastTime = RTTimeMilliTS();
2922	#endif
2923
2924	Assert(!(pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
2925	\|\| (pVCpu->hm.s.vmx.pbVirtApic && pVM->hm.s.vmx.pbApicAccess));
2926
2927	/*
2928	* Check if we need to use TPR shadowing.
2929	*/
2930	if ( CPUMIsGuestInLongModeEx(pCtx)
2931	\|\| ( (( pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
2932	\|\| pVM->hm.s.fTRPPatchingAllowed)
2933	&& pVM->hm.s.fHasIoApic)
2934	)
2935	{
2936	fSetupTPRCaching = true;
2937	}
2938
2939	Log2(("\nE"));
2940
2941	/* This is not ideal, but if we don't clear the event injection in the VMCS right here,
2942	* we may end up injecting some stale event into a VM, including injecting an event that
2943	* originated before a VM reset after the VM has been reset. See @bugref{6220}.
2944	*/
2945	VMXWriteVmcs(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, 0);
2946
2947	#ifdef VBOX_STRICT
2948	{
2949	RTCCUINTREG val2;
2950
2951	rc2 = VMXReadVmcs(VMX_VMCS32_CTRL_PIN_EXEC_CONTROLS, &val2);
2952	AssertRC(rc2);
2953	Log2(("VMX_VMCS_CTRL_PIN_EXEC_CONTROLS = %08x\n", val2));
2954
2955	/* allowed zero */
2956	if ((val2 & pVM->hm.s.vmx.msr.vmx_pin_ctls.n.disallowed0) != pVM->hm.s.vmx.msr.vmx_pin_ctls.n.disallowed0)
2957	Log(("Invalid VMX_VMCS_CTRL_PIN_EXEC_CONTROLS: zero\n"));
2958
2959	/* allowed one */
2960	if ((val2 & ~pVM->hm.s.vmx.msr.vmx_pin_ctls.n.allowed1) != 0)
2961	Log(("Invalid VMX_VMCS_CTRL_PIN_EXEC_CONTROLS: one\n"));
2962
2963	rc2 = VMXReadVmcs(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, &val2);
2964	AssertRC(rc2);
2965	Log2(("VMX_VMCS_CTRL_PROC_EXEC_CONTROLS = %08x\n", val2));
2966
2967	/*
2968	* Must be set according to the MSR, but can be cleared if nested paging is used.
2969	*/
2970	if (pVM->hm.s.fNestedPaging)
2971	{
2972	val2 \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INVLPG_EXIT
2973	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
2974	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT;
2975	}
2976
2977	/* allowed zero */
2978	if ((val2 & pVM->hm.s.vmx.msr.vmx_proc_ctls.n.disallowed0) != pVM->hm.s.vmx.msr.vmx_proc_ctls.n.disallowed0)
2979	Log(("Invalid VMX_VMCS_CTRL_PROC_EXEC_CONTROLS: zero\n"));
2980
2981	/* allowed one */
2982	if ((val2 & ~pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1) != 0)
2983	Log(("Invalid VMX_VMCS_CTRL_PROC_EXEC_CONTROLS: one\n"));
2984
2985	rc2 = VMXReadVmcs(VMX_VMCS32_CTRL_ENTRY_CONTROLS, &val2);
2986	AssertRC(rc2);
2987	Log2(("VMX_VMCS_CTRL_ENTRY_CONTROLS = %08x\n", val2));
2988
2989	/* allowed zero */
2990	if ((val2 & pVM->hm.s.vmx.msr.vmx_entry.n.disallowed0) != pVM->hm.s.vmx.msr.vmx_entry.n.disallowed0)
2991	Log(("Invalid VMX_VMCS_CTRL_ENTRY_CONTROLS: zero\n"));
2992
2993	/* allowed one */
2994	if ((val2 & ~pVM->hm.s.vmx.msr.vmx_entry.n.allowed1) != 0)
2995	Log(("Invalid VMX_VMCS_CTRL_ENTRY_CONTROLS: one\n"));
2996
2997	rc2 = VMXReadVmcs(VMX_VMCS32_CTRL_EXIT_CONTROLS, &val2);
2998	AssertRC(rc2);
2999	Log2(("VMX_VMCS_CTRL_EXIT_CONTROLS = %08x\n", val2));
3000
3001	/* allowed zero */
3002	if ((val2 & pVM->hm.s.vmx.msr.vmx_exit.n.disallowed0) != pVM->hm.s.vmx.msr.vmx_exit.n.disallowed0)
3003	Log(("Invalid VMX_VMCS_CTRL_EXIT_CONTROLS: zero\n"));
3004
3005	/* allowed one */
3006	if ((val2 & ~pVM->hm.s.vmx.msr.vmx_exit.n.allowed1) != 0)
3007	Log(("Invalid VMX_VMCS_CTRL_EXIT_CONTROLS: one\n"));
3008	}
3009	fWasInLongMode = CPUMIsGuestInLongModeEx(pCtx);
3010	#endif /* VBOX_STRICT */
3011
3012	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
3013	pVCpu->hm.s.vmx.VMCSCache.u64TimeEntry = RTTimeNanoTS();
3014	#endif
3015
3016	/*
3017	* We can jump to this point to resume execution after determining that a VM-exit is innocent.
3018	*/
3019	ResumeExecution:
3020	if (!STAM_REL_PROFILE_ADV_IS_RUNNING(&pVCpu->hm.s.StatEntry))
3021	STAM_REL_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit2, &pVCpu->hm.s.StatEntry, x);
3022	AssertMsg(pVCpu->hm.s.idEnteredCpu == RTMpCpuId(),
3023	("Expected %d, I'm %d; cResume=%d exitReason=%RGv exitQualification=%RGv\n",
3024	(int)pVCpu->hm.s.idEnteredCpu, (int)RTMpCpuId(), cResume, exitReason, exitQualification));
3025	Assert(!HMR0SuspendPending());
3026	/* Not allowed to switch modes without reloading the host state (32->64 switcher)!! */
3027	Assert(fWasInLongMode == CPUMIsGuestInLongModeEx(pCtx));
3028
3029	/*
3030	* Safety precaution; looping for too long here can have a very bad effect on the host.
3031	*/
3032	if (RT_UNLIKELY(++cResume > pVM->hm.s.cMaxResumeLoops))
3033	{
3034	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMaxResume);
3035	rc = VINF_EM_RAW_INTERRUPT;
3036	goto end;
3037	}
3038
3039	/*
3040	* Check for IRQ inhibition due to instruction fusing (sti, mov ss).
3041	*/
3042	if (VMCPU_FF_ISSET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3043	{
3044	Log(("VM_FF_INHIBIT_INTERRUPTS at %RGv successor %RGv\n", (RTGCPTR)pCtx->rip, EMGetInhibitInterruptsPC(pVCpu)));
3045	if (pCtx->rip != EMGetInhibitInterruptsPC(pVCpu))
3046	{
3047	/*
3048	* Note: we intentionally don't clear VM_FF_INHIBIT_INTERRUPTS here.
3049	* Before we are able to execute this instruction in raw mode (iret to guest code) an external interrupt might
3050	* force a world switch again. Possibly allowing a guest interrupt to be dispatched in the process. This could
3051	* break the guest. Sounds very unlikely, but such timing sensitive problems are not as rare as you might think.
3052	*/
3053	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
3054	/* Irq inhibition is no longer active; clear the corresponding VMX state. */
3055	rc2 = VMXWriteVmcs(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, 0);
3056	AssertRC(rc2);
3057	}
3058	}
3059	else
3060	{
3061	/* Irq inhibition is no longer active; clear the corresponding VMX state. */
3062	rc2 = VMXWriteVmcs(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, 0);
3063	AssertRC(rc2);
3064	}
3065
3066	#ifdef VBOX_HIGH_RES_TIMERS_HACK_IN_RING0
3067	if (RT_UNLIKELY((cResume & 0xf) == 0))
3068	{
3069	uint64_t u64CurTime = RTTimeMilliTS();
3070
3071	if (RT_UNLIKELY(u64CurTime > u64LastTime))
3072	{
3073	u64LastTime = u64CurTime;
3074	TMTimerPollVoid(pVM, pVCpu);
3075	}
3076	}
3077	#endif
3078
3079	/*
3080	* Check for pending actions that force us to go back to ring-3.
3081	*/
3082	if ( VM_FF_ISPENDING(pVM, VM_FF_HM_TO_R3_MASK \| VM_FF_REQUEST \| VM_FF_PGM_POOL_FLUSH_PENDING \| VM_FF_PDM_DMA)
3083	\|\| VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK \| VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL \| VMCPU_FF_REQUEST))
3084	{
3085	/* Check if a sync operation is pending. */
3086	if (VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
3087	{
3088	rc = PGMSyncCR3(pVCpu, pCtx->cr0, pCtx->cr3, pCtx->cr4, VMCPU_FF_ISSET(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
3089	if (rc != VINF_SUCCESS)
3090	{
3091	AssertRC(VBOXSTRICTRC_VAL(rc));
3092	Log(("Pending pool sync is forcing us back to ring 3; rc=%d\n", VBOXSTRICTRC_VAL(rc)));
3093	goto end;
3094	}
3095	}
3096
3097	#ifdef DEBUG
3098	/* Intercept X86_XCPT_DB if stepping is enabled */
3099	if (!DBGFIsStepping(pVCpu))
3100	#endif
3101	{
3102	if ( VM_FF_ISPENDING(pVM, VM_FF_HM_TO_R3_MASK)
3103	\|\| VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
3104	{
3105	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchToR3);
3106	rc = RT_UNLIKELY(VM_FF_ISPENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
3107	goto end;
3108	}
3109	}
3110
3111	/* Pending request packets might contain actions that need immediate attention, such as pending hardware interrupts. */
3112	if ( VM_FF_ISPENDING(pVM, VM_FF_REQUEST)
3113	\|\| VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_REQUEST))
3114	{
3115	rc = VINF_EM_PENDING_REQUEST;
3116	goto end;
3117	}
3118
3119	/* Check if a pgm pool flush is in progress. */
3120	if (VM_FF_ISPENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
3121	{
3122	rc = VINF_PGM_POOL_FLUSH_PENDING;
3123	goto end;
3124	}
3125
3126	/* Check if DMA work is pending (2nd+ run). */
3127	if (VM_FF_ISPENDING(pVM, VM_FF_PDM_DMA) && cResume > 1)
3128	{
3129	rc = VINF_EM_RAW_TO_R3;
3130	goto end;
3131	}
3132	}
3133
3134	#ifdef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
3135	/*
3136	* Exit to ring-3 preemption/work is pending.
3137	*
3138	* Interrupts are disabled before the call to make sure we don't miss any interrupt
3139	* that would flag preemption (IPI, timer tick, ++). (Would've been nice to do this
3140	* further down, but hmR0VmxCheckPendingInterrupt makes that impossible.)
3141	*
3142	* Note! Interrupts must be disabled done before we check for TLB flushes; TLB
3143	* shootdowns rely on this.
3144	*/
3145	uOldEFlags = ASMIntDisableFlags();
3146	if (RTThreadPreemptIsPending(NIL_RTTHREAD))
3147	{
3148	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptPending);
3149	rc = VINF_EM_RAW_INTERRUPT;
3150	goto end;
3151	}
3152	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
3153	#endif
3154
3155	/*
3156	* When external interrupts are pending, we should exit the VM when IF is set.
3157	* Note: After VM_FF_INHIBIT_INTERRUPTS check!
3158	*/
3159	rc = hmR0VmxCheckPendingInterrupt(pVM, pVCpu, pCtx);
3160	if (RT_FAILURE(rc))
3161	goto end;
3162
3163	/** @todo check timers?? */
3164
3165	/*
3166	* TPR caching using CR8 is only available in 64-bit mode.
3167	* Note: The 32-bit exception for AMD (X86_CPUID_AMD_FEATURE_ECX_CR8L), but this appears missing in Intel CPUs.
3168	* Note: We can't do this in LoadGuestState() as PDMApicGetTPR can jump back to ring-3 (lock)!! (no longer true) .
3169	*/
3170	/** @todo query and update the TPR only when it could have been changed (mmio
3171	* access & wrsmr (x2apic) */
3172	if (fSetupTPRCaching)
3173	{
3174	/* TPR caching in CR8 */
3175	bool fPending;
3176
3177	rc2 = PDMApicGetTPR(pVCpu, &u8LastTPR, &fPending);
3178	AssertRC(rc2);
3179	/* The TPR can be found at offset 0x80 in the APIC mmio page. */
3180	pVCpu->hm.s.vmx.pbVirtApic[0x80] = u8LastTPR;
3181
3182	/*
3183	* Two options here:
3184	* - external interrupt pending, but masked by the TPR value.
3185	* -> a CR8 update that lower the current TPR value should cause an exit
3186	* - no pending interrupts
3187	* -> We don't need to be explicitely notified. There are enough world switches for detecting pending interrupts.
3188	*/
3189
3190	/* cr8 bits 3-0 correspond to bits 7-4 of the task priority mmio register. */
3191	rc = VMXWriteVmcs(VMX_VMCS32_CTRL_TPR_THRESHOLD, (fPending) ? (u8LastTPR >> 4) : 0);
3192	AssertRC(VBOXSTRICTRC_VAL(rc));
3193
3194	if (pVM->hm.s.fTPRPatchingActive)
3195	{
3196	Assert(!CPUMIsGuestInLongModeEx(pCtx));
3197	/* Our patch code uses LSTAR for TPR caching. */
3198	pCtx->msrLSTAR = u8LastTPR;
3199
3200	/** @todo r=ramshankar: we should check for MSR-bitmap support here. */
3201	if (fPending)
3202	{
3203	/* A TPR change could activate a pending interrupt, so catch lstar writes. */
3204	hmR0VmxSetMSRPermission(pVCpu, MSR_K8_LSTAR, true, false);
3205	}
3206	else
3207	{
3208	/*
3209	* No interrupts are pending, so we don't need to be explicitely notified.
3210	* There are enough world switches for detecting pending interrupts.
3211	*/
3212	hmR0VmxSetMSRPermission(pVCpu, MSR_K8_LSTAR, true, true);
3213	}
3214	}
3215	}
3216
3217	#ifdef LOG_ENABLED
3218	if ( pVM->hm.s.fNestedPaging
3219	\|\| pVM->hm.s.vmx.fVpid)
3220	{
3221	PHMGLOBLCPUINFO pCpu = HMR0GetCurrentCpu();
3222	if (pVCpu->hm.s.idLastCpu != pCpu->idCpu)
3223	{
3224	LogFlow(("Force TLB flush due to rescheduling to a different cpu (%d vs %d)\n", pVCpu->hm.s.idLastCpu,
3225	pCpu->idCpu));
3226	}
3227	else if (pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
3228	{
3229	LogFlow(("Force TLB flush due to changed TLB flush count (%x vs %x)\n", pVCpu->hm.s.cTlbFlushes,
3230	pCpu->cTlbFlushes));
3231	}
3232	else if (VMCPU_FF_ISSET(pVCpu, VMCPU_FF_TLB_FLUSH))
3233	LogFlow(("Manual TLB flush\n"));
3234	}
3235	#endif
3236	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
3237	PGMRZDynMapFlushAutoSet(pVCpu);
3238	#endif
3239
3240	/*
3241	* NOTE: DO NOT DO ANYTHING AFTER THIS POINT THAT MIGHT JUMP BACK TO RING-3!
3242	* (until the actual world switch)
3243	*/
3244	#ifdef VBOX_STRICT
3245	idCpuCheck = RTMpCpuId();
3246	#endif
3247	#ifdef LOG_ENABLED
3248	VMMR0LogFlushDisable(pVCpu);
3249	#endif
3250
3251	/*
3252	* Save the host state first.
3253	*/
3254	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_HOST_CONTEXT)
3255	{
3256	rc = VMXR0SaveHostState(pVM, pVCpu);
3257	if (RT_UNLIKELY(rc != VINF_SUCCESS))
3258	{
3259	VMMR0LogFlushEnable(pVCpu);
3260	goto end;
3261	}
3262	}
3263
3264	/*
3265	* Load the guest state.
3266	*/
3267	if (!pVCpu->hm.s.fContextUseFlags)
3268	{
3269	VMXR0LoadMinimalGuestState(pVM, pVCpu, pCtx);
3270	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadMinimal);
3271	if (fNeedTscSetup)
3272	{
3273	VMXR0SetupTscOffsetAndPreemption(pVM, pVCpu);
3274	fNeedTscSetup = false;
3275	}
3276	}
3277	else
3278	{
3279	rc = VMXR0LoadGuestState(pVM, pVCpu, pCtx);
3280	if (RT_UNLIKELY(rc != VINF_SUCCESS))
3281	{
3282	VMMR0LogFlushEnable(pVCpu);
3283	goto end;
3284	}
3285	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadFull);
3286	VMXR0SetupTscOffsetAndPreemption(pVM, pVCpu);
3287	}
3288
3289	#ifndef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
3290	/*
3291	* Disable interrupts to make sure a poke will interrupt execution.
3292	* This must be done before we check for TLB flushes; TLB shootdowns rely on this.
3293	*/
3294	uOldEFlags = ASMIntDisableFlags();
3295	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
3296	#endif
3297
3298	/* Non-register state Guest Context */
3299	/** @todo change me according to cpu state */
3300	rc2 = VMXWriteVmcs(VMX_VMCS32_GUEST_ACTIVITY_STATE, VMX_VMCS_GUEST_ACTIVITY_ACTIVE);
3301	AssertRC(rc2);
3302
3303	/* Set TLB flush state as checked until we return from the world switch. */
3304	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true);
3305	/* Deal with tagged TLB setup and invalidation. */
3306	pVM->hm.s.vmx.pfnFlushTaggedTlb(pVM, pVCpu);
3307
3308	/*
3309	* Manual save and restore:
3310	* - General purpose registers except RIP, RSP
3311	*
3312	* Trashed:
3313	* - CR2 (we don't care)
3314	* - LDTR (reset to 0)
3315	* - DRx (presumably not changed at all)
3316	* - DR7 (reset to 0x400)
3317	* - EFLAGS (reset to RT_BIT(1); not relevant)
3318	*/
3319
3320	/* All done! Let's start VM execution. */
3321	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
3322	Assert(idCpuCheck == RTMpCpuId());
3323
3324	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
3325	pVCpu->hm.s.vmx.VMCSCache.cResume = cResume;
3326	pVCpu->hm.s.vmx.VMCSCache.u64TimeSwitch = RTTimeNanoTS();
3327	#endif
3328
3329	/*
3330	* Save the current TPR value in the LSTAR MSR so our patches can access it.
3331	*/
3332	if (pVM->hm.s.fTPRPatchingActive)
3333	{
3334	Assert(pVM->hm.s.fTPRPatchingActive);
3335	u64OldLSTAR = ASMRdMsr(MSR_K8_LSTAR);
3336	ASMWrMsr(MSR_K8_LSTAR, u8LastTPR);
3337	}
3338
3339	TMNotifyStartOfExecution(pVCpu);
3340
3341	#ifndef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
3342	/*
3343	* Save the current Host TSC_AUX and write the guest TSC_AUX to the host, so that
3344	* RDTSCPs (that don't cause exits) reads the guest MSR. See @bugref{3324}.
3345	*/
3346	if ( (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
3347	&& !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT))
3348	{
3349	pVCpu->hm.s.u64HostTSCAux = ASMRdMsr(MSR_K8_TSC_AUX);
3350	uint64_t u64GuestTSCAux = 0;
3351	rc2 = CPUMQueryGuestMsr(pVCpu, MSR_K8_TSC_AUX, &u64GuestTSCAux);
3352	AssertRC(rc2);
3353	ASMWrMsr(MSR_K8_TSC_AUX, u64GuestTSCAux);
3354	}
3355	#endif
3356
3357	#ifdef VBOX_WITH_KERNEL_USING_XMM
3358	rc = hmR0VMXStartVMWrapXMM(pVCpu->hm.s.fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hm.s.vmx.pfnStartVM);
3359	#else
3360	rc = pVCpu->hm.s.vmx.pfnStartVM(pVCpu->hm.s.fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu);
3361	#endif
3362	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false);
3363	ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits);
3364
3365	/* Possibly the last TSC value seen by the guest (too high) (only when we're in TSC offset mode). */
3366	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT))
3367	{
3368	#ifndef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
3369	/* Restore host's TSC_AUX. */
3370	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
3371	ASMWrMsr(MSR_K8_TSC_AUX, pVCpu->hm.s.u64HostTSCAux);
3372	#endif
3373
3374	TMCpuTickSetLastSeen(pVCpu,
3375	ASMReadTSC() + pVCpu->hm.s.vmx.u64TSCOffset - 0x400 /* guestimate of world switch overhead in clock ticks */);
3376	}
3377
3378	TMNotifyEndOfExecution(pVCpu);
3379	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED);
3380	Assert(!(ASMGetFlags() & X86_EFL_IF));
3381
3382	/*
3383	* Restore the host LSTAR MSR if the guest could have changed it.
3384	*/
3385	if (pVM->hm.s.fTPRPatchingActive)
3386	{
3387	Assert(pVM->hm.s.fTPRPatchingActive);
3388	pVCpu->hm.s.vmx.pbVirtApic[0x80] = pCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR);
3389	ASMWrMsr(MSR_K8_LSTAR, u64OldLSTAR);
3390	}
3391
3392	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatExit1, x);
3393	ASMSetFlags(uOldEFlags);
3394	#ifdef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
3395	uOldEFlags = ~(RTCCUINTREG)0;
3396	#endif
3397
3398	AssertMsg(!pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries, ("pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries=%d\n",
3399	pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries));
3400
3401	/* In case we execute a goto ResumeExecution later on. */
3402	pVCpu->hm.s.fResumeVM = true;
3403	pVCpu->hm.s.fForceTLBFlush = false;
3404
3405	/*
3406	* !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
3407	* IMPORTANT: WE CAN'T DO ANY LOGGING OR OPERATIONS THAT CAN DO A LONGJMP BACK TO RING 3 BEFORE WE'VE SYNCED BACK (MOST OF) THE GUEST STATE
3408	* !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
3409	*/
3410
3411	if (RT_UNLIKELY(rc != VINF_SUCCESS))
3412	{
3413	hmR0VmxReportWorldSwitchError(pVM, pVCpu, rc, pCtx);
3414	VMMR0LogFlushEnable(pVCpu);
3415	goto end;
3416	}
3417
3418	/* Success. Query the guest state and figure out what has happened. */
3419
3420	/* Investigate why there was a VM-exit. */
3421	rc2 = VMXReadCachedVmcs(VMX_VMCS32_RO_EXIT_REASON, &exitReason);
3422	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[exitReason & MASK_EXITREASON_STAT]);
3423
3424	exitReason &= 0xffff; /* bit 0-15 contain the exit code. */
3425	rc2 \|= VMXReadCachedVmcs(VMX_VMCS32_RO_VM_INSTR_ERROR, &instrError);
3426	rc2 \|= VMXReadCachedVmcs(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &cbInstr);
3427	rc2 \|= VMXReadCachedVmcs(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &intInfo);
3428	/* might not be valid; depends on VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID. */
3429	rc2 \|= VMXReadCachedVmcs(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &errCode);
3430	rc2 \|= VMXReadCachedVmcs(VMX_VMCS32_RO_EXIT_INSTR_INFO, &instrInfo);
3431	rc2 \|= VMXReadCachedVmcs(VMX_VMCS_RO_EXIT_QUALIFICATION, &exitQualification);
3432	AssertRC(rc2);
3433
3434	/*
3435	* Sync back the guest state.
3436	*/
3437	rc2 = VMXR0SaveGuestState(pVM, pVCpu, pCtx);
3438	AssertRC(rc2);
3439
3440	/* Note! NOW IT'S SAFE FOR LOGGING! */
3441	VMMR0LogFlushEnable(pVCpu);
3442	Log2(("Raw exit reason %08x\n", exitReason));
3443	#if ARCH_BITS == 64 /* for the time being */
3444	VBOXVMM_R0_HMVMX_VMEXIT(pVCpu, pCtx, exitReason);
3445	#endif
3446
3447	/*
3448	* Check if an injected event was interrupted prematurely.
3449	*/
3450	rc2 = VMXReadCachedVmcs(VMX_VMCS32_RO_IDT_INFO, &val);
3451	AssertRC(rc2);
3452	pVCpu->hm.s.Event.u64IntrInfo = VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(val);
3453	if ( VMX_EXIT_INTERRUPTION_INFO_VALID(pVCpu->hm.s.Event.u64IntrInfo)
3454	/* Ignore 'int xx' as they'll be restarted anyway. */
3455	&& VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hm.s.Event.u64IntrInfo) != VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT
3456	/* Ignore software exceptions (such as int3) as they'll reoccur when we restart the instruction anyway. */
3457	&& VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hm.s.Event.u64IntrInfo) != VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT)
3458	{
3459	Assert(!pVCpu->hm.s.Event.fPending);
3460	pVCpu->hm.s.Event.fPending = true;
3461	/* Error code present? */
3462	if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntrInfo))
3463	{
3464	rc2 = VMXReadCachedVmcs(VMX_VMCS32_RO_IDT_ERROR_CODE, &val);
3465	AssertRC(rc2);
3466	pVCpu->hm.s.Event.u32ErrCode = val;
3467	Log(("Pending inject %RX64 at %RGv exit=%08x intInfo=%08x exitQualification=%RGv pending error=%RX64\n",
3468	pVCpu->hm.s.Event.u64IntrInfo, (RTGCPTR)pCtx->rip, exitReason, intInfo, exitQualification, val));
3469	}
3470	else
3471	{
3472	Log(("Pending inject %RX64 at %RGv exit=%08x intInfo=%08x exitQualification=%RGv\n", pVCpu->hm.s.Event.u64IntrInfo,
3473	(RTGCPTR)pCtx->rip, exitReason, intInfo, exitQualification));
3474	pVCpu->hm.s.Event.u32ErrCode = 0;
3475	}
3476	}
3477	#ifdef VBOX_STRICT
3478	else if ( VMX_EXIT_INTERRUPTION_INFO_VALID(pVCpu->hm.s.Event.u64IntrInfo)
3479	/* Ignore software exceptions (such as int3) as they're reoccur when we restart the instruction anyway. */
3480	&& VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hm.s.Event.u64IntrInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT)
3481	{
3482	Log(("Ignore pending inject %RX64 at %RGv exit=%08x intInfo=%08x exitQualification=%RGv\n",
3483	pVCpu->hm.s.Event.u64IntrInfo, (RTGCPTR)pCtx->rip, exitReason, intInfo, exitQualification));
3484	}
3485
3486	if (exitReason == VMX_EXIT_ERR_INVALID_GUEST_STATE)
3487	HMDumpRegs(pVM, pVCpu, pCtx);
3488	#endif
3489
3490	Log2(("E%d: New EIP=%x:%RGv\n", (uint32_t)exitReason, pCtx->cs.Sel, (RTGCPTR)pCtx->rip));
3491	Log2(("Exit reason %d, exitQualification %RGv\n", (uint32_t)exitReason, exitQualification));
3492	Log2(("instrInfo=%d instrError=%d instr length=%d\n", (uint32_t)instrInfo, (uint32_t)instrError, (uint32_t)cbInstr));
3493	Log2(("Interruption error code %d\n", (uint32_t)errCode));
3494	Log2(("IntInfo = %08x\n", (uint32_t)intInfo));
3495
3496	/*
3497	* Sync back the TPR if it was changed.
3498	*/
3499	if ( fSetupTPRCaching
3500	&& u8LastTPR != pVCpu->hm.s.vmx.pbVirtApic[0x80])
3501	{
3502	rc2 = PDMApicSetTPR(pVCpu, pVCpu->hm.s.vmx.pbVirtApic[0x80]);
3503	AssertRC(rc2);
3504	}
3505
3506	#ifdef DBGFTRACE_ENABLED /** @todo DTrace later. */
3507	RTTraceBufAddMsgF(pVM->CTX_SUFF(hTraceBuf), "vmexit %08x %016RX64 at %04:%08RX64 %RX64",
3508	exitReason, (uint64_t)exitQualification, pCtx->cs.Sel, pCtx->rip, (uint64_t)intInfo);
3509	#endif
3510	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
3511
3512	/* Some cases don't need a complete resync of the guest CPU state; handle them here. */
3513	Assert(rc == VINF_SUCCESS); /* might consider VERR_IPE_UNINITIALIZED_STATUS here later... */
3514	switch (exitReason)
3515	{
3516	case VMX_EXIT_XCPT_NMI: /* 0 Exception or non-maskable interrupt (NMI). */
3517	case VMX_EXIT_EXT_INT: /* 1 External interrupt. */
3518	{
3519	uint32_t vector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(intInfo);
3520
3521	if (!VMX_EXIT_INTERRUPTION_INFO_VALID(intInfo))
3522	{
3523	Assert(exitReason == VMX_EXIT_EXT_INT);
3524	/* External interrupt; leave to allow it to be dispatched again. */
3525	rc = VINF_EM_RAW_INTERRUPT;
3526	break;
3527	}
3528	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExit2Sub3, y3);
3529	switch (VMX_EXIT_INTERRUPTION_INFO_TYPE(intInfo))
3530	{
3531	case VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI: /* Non-maskable interrupt. */
3532	/* External interrupt; leave to allow it to be dispatched again. */
3533	rc = VINF_EM_RAW_INTERRUPT;
3534	break;
3535
3536	case VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT: /* External hardware interrupt. */
3537	AssertFailed(); /* can't come here; fails the first check. */
3538	break;
3539
3540	case VMX_EXIT_INTERRUPTION_INFO_TYPE_DB_XCPT: /* Unknown why we get this type for #DB */
3541	case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
3542	Assert(vector == 1 \|\| vector == 3 \|\| vector == 4);
3543	/* no break */
3544	case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT: /* Hardware exception. */
3545	Log2(("Hardware/software interrupt %d\n", vector));
3546	switch (vector)
3547	{
3548	case X86_XCPT_NM:
3549	{
3550	Log(("#NM fault at %RGv error code %x\n", (RTGCPTR)pCtx->rip, errCode));
3551
3552	/** @todo don't intercept #NM exceptions anymore when we've activated the guest FPU state. */
3553	/* If we sync the FPU/XMM state on-demand, then we can continue execution as if nothing has happened. */
3554	rc = CPUMR0LoadGuestFPU(pVM, pVCpu, pCtx);
3555	if (rc == VINF_SUCCESS)
3556	{
3557	Assert(CPUMIsGuestFPUStateActive(pVCpu));
3558
3559	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowNM);
3560
3561	/* Continue execution. */
3562	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
3563
3564	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2Sub3, y3);
3565	goto ResumeExecution;
3566	}
3567
3568	Log(("Forward #NM fault to the guest\n"));
3569	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
3570	rc2 = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo),
3571	cbInstr, 0);
3572	AssertRC(rc2);
3573	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2Sub3, y3);
3574	goto ResumeExecution;
3575	}
3576
3577	case X86_XCPT_PF: /* Page fault */
3578	{
3579	#ifdef VBOX_ALWAYS_TRAP_PF
3580	if (pVM->hm.s.fNestedPaging)
3581	{
3582	/*
3583	* A genuine pagefault. Forward the trap to the guest by injecting the exception and resuming execution.
3584	*/
3585	Log(("Guest page fault at %RGv cr2=%RGv error code %RGv rsp=%RGv\n", (RTGCPTR)pCtx->rip, exitQualification,
3586	errCode, (RTGCPTR)pCtx->rsp));
3587
3588	Assert(CPUMIsGuestInPagedProtectedModeEx(pCtx));
3589
3590	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
3591
3592	/* Now we must update CR2. */
3593	pCtx->cr2 = exitQualification;
3594	rc2 = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo),
3595	cbInstr, errCode);
3596	AssertRC(rc2);
3597
3598	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2Sub3, y3);
3599	goto ResumeExecution;
3600	}
3601	#else
3602	Assert(!pVM->hm.s.fNestedPaging);
3603	#endif
3604
3605	#ifdef VBOX_HM_WITH_GUEST_PATCHING
3606	/* Shortcut for APIC TPR reads and writes; 32 bits guests only */
3607	if ( pVM->hm.s.fTRPPatchingAllowed
3608	&& pVM->hm.s.pGuestPatchMem
3609	&& (exitQualification & 0xfff) == 0x080
3610	&& !(errCode & X86_TRAP_PF_P) /* not present */
3611	&& CPUMGetGuestCPL(pVCpu) == 0
3612	&& !CPUMIsGuestInLongModeEx(pCtx)
3613	&& pVM->hm.s.cPatches < RT_ELEMENTS(pVM->hm.s.aPatches))
3614	{
3615	RTGCPHYS GCPhysApicBase, GCPhys;
3616	GCPhysApicBase = pCtx->msrApicBase;
3617	GCPhysApicBase &= PAGE_BASE_GC_MASK;
3618
3619	rc = PGMGstGetPage(pVCpu, (RTGCPTR)exitQualification, NULL, &GCPhys);
3620	if ( rc == VINF_SUCCESS
3621	&& GCPhys == GCPhysApicBase)
3622	{
3623	/* Only attempt to patch the instruction once. */
3624	PHMTPRPATCH pPatch = (PHMTPRPATCH)RTAvloU32Get(&pVM->hm.s.PatchTree, (AVLOU32KEY)pCtx->eip);
3625	if (!pPatch)
3626	{
3627	rc = VINF_EM_HM_PATCH_TPR_INSTR;
3628	break;
3629	}
3630	}
3631	}
3632	#endif
3633
3634	Log2(("Page fault at %RGv error code %x\n", exitQualification, errCode));
3635	/* Exit qualification contains the linear address of the page fault. */
3636	TRPMAssertTrap(pVCpu, X86_XCPT_PF, TRPM_TRAP);
3637	TRPMSetErrorCode(pVCpu, errCode);
3638	TRPMSetFaultAddress(pVCpu, exitQualification);
3639
3640	/* Shortcut for APIC TPR reads and writes. */
3641	if ( (exitQualification & 0xfff) == 0x080
3642	&& !(errCode & X86_TRAP_PF_P) /* not present */
3643	&& fSetupTPRCaching
3644	&& (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC))
3645	{
3646	RTGCPHYS GCPhysApicBase, GCPhys;
3647	GCPhysApicBase = pCtx->msrApicBase;
3648	GCPhysApicBase &= PAGE_BASE_GC_MASK;
3649
3650	rc = PGMGstGetPage(pVCpu, (RTGCPTR)exitQualification, NULL, &GCPhys);
3651	if ( rc == VINF_SUCCESS
3652	&& GCPhys == GCPhysApicBase)
3653	{
3654	Log(("Enable VT-x virtual APIC access filtering\n"));
3655	rc2 = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess,
3656	X86_PTE_RW \| X86_PTE_P);
3657	AssertRC(rc2);
3658	}
3659	}
3660
3661	/* Forward it to our trap handler first, in case our shadow pages are out of sync. */
3662	rc = PGMTrap0eHandler(pVCpu, errCode, CPUMCTX2CORE(pCtx), (RTGCPTR)exitQualification);
3663	Log2(("PGMTrap0eHandler %RGv returned %Rrc\n", (RTGCPTR)pCtx->rip, VBOXSTRICTRC_VAL(rc)));
3664
3665	if (rc == VINF_SUCCESS)
3666	{ /* We've successfully synced our shadow pages, so let's just continue execution. */
3667	Log2(("Shadow page fault at %RGv cr2=%RGv error code %x\n", (RTGCPTR)pCtx->rip, exitQualification ,errCode));
3668	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
3669
3670	TRPMResetTrap(pVCpu);
3671	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2Sub3, y3);
3672	goto ResumeExecution;
3673	}
3674	else if (rc == VINF_EM_RAW_GUEST_TRAP)
3675	{
3676	/*
3677	* A genuine pagefault. Forward the trap to the guest by injecting the exception and resuming execution.
3678	*/
3679	Log2(("Forward page fault to the guest\n"));
3680
3681	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
3682	/* The error code might have been changed. */
3683	errCode = TRPMGetErrorCode(pVCpu);
3684
3685	TRPMResetTrap(pVCpu);
3686
3687	/* Now we must update CR2. */
3688	pCtx->cr2 = exitQualification;
3689	rc2 = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo),
3690	cbInstr, errCode);
3691	AssertRC(rc2);
3692
3693	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2Sub3, y3);
3694	goto ResumeExecution;
3695	}
3696	#ifdef VBOX_STRICT
3697	if (rc != VINF_EM_RAW_EMULATE_INSTR && rc != VINF_EM_RAW_EMULATE_IO_BLOCK)
3698	Log2(("PGMTrap0eHandler failed with %d\n", VBOXSTRICTRC_VAL(rc)));
3699	#endif
3700	/* Need to go back to the recompiler to emulate the instruction. */
3701	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
3702	TRPMResetTrap(pVCpu);
3703
3704	/* If event delivery caused the #PF (shadow or not), tell TRPM. */
3705	hmR0VmxCheckPendingEvent(pVCpu);
3706	break;
3707	}
3708
3709	case X86_XCPT_MF: /* Floating point exception. */
3710	{
3711	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
3712	if (!(pCtx->cr0 & X86_CR0_NE))
3713	{
3714	/* old style FPU error reporting needs some extra work. */
3715	/** @todo don't fall back to the recompiler, but do it manually. */
3716	rc = VINF_EM_RAW_EMULATE_INSTR;
3717	break;
3718	}
3719	Log(("Trap %x at %04X:%RGv\n", vector, pCtx->cs.Sel, (RTGCPTR)pCtx->rip));
3720	rc2 = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo),
3721	cbInstr, errCode);
3722	AssertRC(rc2);
3723
3724	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2Sub3, y3);
3725	goto ResumeExecution;
3726	}
3727
3728	case X86_XCPT_DB: /* Debug exception. */
3729	{
3730	uint64_t uDR6;
3731
3732	/*
3733	* DR6, DR7.GD and IA32_DEBUGCTL.LBR are not updated yet.
3734	*
3735	* Exit qualification bits:
3736	* 3:0 B0-B3 which breakpoint condition was met
3737	* 12:4 Reserved (0)
3738	* 13 BD - debug register access detected
3739	* 14 BS - single step execution or branch taken
3740	* 63:15 Reserved (0)
3741	*/
3742	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
3743
3744	/* Note that we don't support guest and host-initiated debugging at the same time. */
3745
3746	uDR6 = X86_DR6_INIT_VAL;
3747	uDR6 \|= (exitQualification & (X86_DR6_B0\|X86_DR6_B1\|X86_DR6_B2\|X86_DR6_B3\|X86_DR6_BD\|X86_DR6_BS));
3748	rc = DBGFRZTrap01Handler(pVM, pVCpu, CPUMCTX2CORE(pCtx), uDR6);
3749	if (rc == VINF_EM_RAW_GUEST_TRAP)
3750	{
3751	/* Update DR6 here. */
3752	pCtx->dr[6] = uDR6;
3753
3754	/* Resync DR6 if the debug state is active. */
3755	if (CPUMIsGuestDebugStateActive(pVCpu))
3756	ASMSetDR6(pCtx->dr[6]);
3757
3758	/* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
3759	pCtx->dr[7] &= ~X86_DR7_GD;
3760
3761	/* Paranoia. */
3762	pCtx->dr[7] &= 0xffffffff; /* upper 32 bits reserved */
3763	pCtx->dr[7] &= ~(RT_BIT(11) \| RT_BIT(12) \| RT_BIT(14) \| RT_BIT(15)); /* must be zero */
3764	pCtx->dr[7] \|= 0x400; /* must be one */
3765
3766	/* Resync DR7 */
3767	rc2 = VMXWriteVmcs64(VMX_VMCS_GUEST_DR7, pCtx->dr[7]);
3768	AssertRC(rc2);
3769
3770	Log(("Trap %x (debug) at %RGv exit qualification %RX64 dr6=%x dr7=%x\n", vector, (RTGCPTR)pCtx->rip,
3771	exitQualification, (uint32_t)pCtx->dr[6], (uint32_t)pCtx->dr[7]));
3772	rc2 = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo),
3773	cbInstr, errCode);
3774	AssertRC(rc2);
3775
3776	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2Sub3, y3);
3777	goto ResumeExecution;
3778	}
3779	/* Return to ring 3 to deal with the debug exit code. */
3780	Log(("Debugger hardware BP at %04x:%RGv (rc=%Rrc)\n", pCtx->cs.Sel, pCtx->rip, VBOXSTRICTRC_VAL(rc)));
3781	break;
3782	}
3783
3784	case X86_XCPT_BP: /* Breakpoint. */
3785	{
3786	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
3787	rc = DBGFRZTrap03Handler(pVM, pVCpu, CPUMCTX2CORE(pCtx));
3788	if (rc == VINF_EM_RAW_GUEST_TRAP)
3789	{
3790	Log(("Guest #BP at %04x:%RGv\n", pCtx->cs.Sel, pCtx->rip));
3791	rc2 = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo),
3792	cbInstr, errCode);
3793	AssertRC(rc2);
3794	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2Sub3, y3);
3795	goto ResumeExecution;
3796	}
3797	if (rc == VINF_SUCCESS)
3798	{
3799	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2Sub3, y3);
3800	goto ResumeExecution;
3801	}
3802	Log(("Debugger BP at %04x:%RGv (rc=%Rrc)\n", pCtx->cs.Sel, pCtx->rip, VBOXSTRICTRC_VAL(rc)));
3803	break;
3804	}
3805
3806	case X86_XCPT_GP: /* General protection failure exception. */
3807	{
3808	uint32_t cbOp;
3809	PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
3810
3811	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
3812	#ifdef VBOX_STRICT
3813	if ( !CPUMIsGuestInRealModeEx(pCtx)
3814	\|\| !pVM->hm.s.vmx.pRealModeTSS)
3815	{
3816	Log(("Trap %x at %04X:%RGv errorCode=%RGv\n", vector, pCtx->cs.Sel, (RTGCPTR)pCtx->rip, errCode));
3817	rc2 = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo),
3818	cbInstr, errCode);
3819	AssertRC(rc2);
3820	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2Sub3, y3);
3821	goto ResumeExecution;
3822	}
3823	#endif
3824	Assert(CPUMIsGuestInRealModeEx(pCtx));
3825
3826	LogFlow(("Real mode X86_XCPT_GP instruction emulation at %x:%RGv\n", pCtx->cs.Sel, (RTGCPTR)pCtx->rip));
3827
3828	rc2 = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbOp);
3829	if (RT_SUCCESS(rc2))
3830	{
3831	bool fUpdateRIP = true;
3832
3833	rc = VINF_SUCCESS;
3834	Assert(cbOp == pDis->cbInstr);
3835	switch (pDis->pCurInstr->uOpcode)
3836	{
3837	case OP_CLI:
3838	pCtx->eflags.Bits.u1IF = 0;
3839	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCli);
3840	break;
3841
3842	case OP_STI:
3843	pCtx->eflags.Bits.u1IF = 1;
3844	EMSetInhibitInterruptsPC(pVCpu, pCtx->rip + pDis->cbInstr);
3845	Assert(VMCPU_FF_ISSET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
3846	rc2 = VMXWriteVmcs(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE,
3847	VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
3848	AssertRC(rc2);
3849	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitSti);
3850	break;
3851
3852	case OP_HLT:
3853	fUpdateRIP = false;
3854	rc = VINF_EM_HALT;
3855	pCtx->rip += pDis->cbInstr;
3856	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
3857	break;
3858
3859	case OP_POPF:
3860	{
3861	RTGCPTR GCPtrStack;
3862	uint32_t cbParm;
3863	uint32_t uMask;
3864	X86EFLAGS eflags;
3865
3866	if (pDis->fPrefix & DISPREFIX_OPSIZE)
3867	{
3868	cbParm = 4;
3869	uMask = 0xffffffff;
3870	}
3871	else
3872	{
3873	cbParm = 2;
3874	uMask = 0xffff;
3875	}
3876
3877	rc2 = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pCtx), pCtx->esp & uMask, 0, &GCPtrStack);
3878	if (RT_FAILURE(rc2))
3879	{
3880	rc = VERR_EM_INTERPRETER;
3881	break;
3882	}
3883	eflags.u = 0;
3884	rc2 = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &eflags.u, cbParm);
3885	if (RT_FAILURE(rc2))
3886	{
3887	rc = VERR_EM_INTERPRETER;
3888	break;
3889	}
3890	LogFlow(("POPF %x -> %RGv mask=%x\n", eflags.u, pCtx->rsp, uMask));
3891	pCtx->eflags.u = (pCtx->eflags.u & ~(X86_EFL_POPF_BITS & uMask))
3892	\| (eflags.u & X86_EFL_POPF_BITS & uMask);
3893	/* RF cleared when popped in real mode; see pushf description in AMD manual. */
3894	pCtx->eflags.Bits.u1RF = 0;
3895	pCtx->esp += cbParm;
3896	pCtx->esp &= uMask;
3897
3898	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPopf);
3899	break;
3900	}
3901
3902	case OP_PUSHF:
3903	{
3904	RTGCPTR GCPtrStack;
3905	uint32_t cbParm;
3906	uint32_t uMask;
3907	X86EFLAGS eflags;
3908
3909	if (pDis->fPrefix & DISPREFIX_OPSIZE)
3910	{
3911	cbParm = 4;
3912	uMask = 0xffffffff;
3913	}
3914	else
3915	{
3916	cbParm = 2;
3917	uMask = 0xffff;
3918	}
3919
3920	rc2 = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pCtx), (pCtx->esp - cbParm) & uMask, 0,
3921	&GCPtrStack);
3922	if (RT_FAILURE(rc2))
3923	{
3924	rc = VERR_EM_INTERPRETER;
3925	break;
3926	}
3927	eflags = pCtx->eflags;
3928	/* RF & VM cleared when pushed in real mode; see pushf description in AMD manual. */
3929	eflags.Bits.u1RF = 0;
3930	eflags.Bits.u1VM = 0;
3931
3932	rc2 = PGMPhysWrite(pVM, (RTGCPHYS)GCPtrStack, &eflags.u, cbParm);
3933	if (RT_FAILURE(rc2))
3934	{
3935	rc = VERR_EM_INTERPRETER;
3936	break;
3937	}
3938	LogFlow(("PUSHF %x -> %RGv\n", eflags.u, GCPtrStack));
3939	pCtx->esp -= cbParm;
3940	pCtx->esp &= uMask;
3941	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPushf);
3942	break;
3943	}
3944
3945	case OP_IRET:
3946	{
3947	RTGCPTR GCPtrStack;
3948	uint32_t uMask = 0xffff;
3949	uint16_t aIretFrame[3];
3950
3951	if (pDis->fPrefix & (DISPREFIX_OPSIZE \| DISPREFIX_ADDRSIZE))
3952	{
3953	rc = VERR_EM_INTERPRETER;
3954	break;
3955	}
3956
3957	rc2 = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pCtx), pCtx->esp & uMask, 0, &GCPtrStack);
3958	if (RT_FAILURE(rc2))
3959	{
3960	rc = VERR_EM_INTERPRETER;
3961	break;
3962	}
3963	rc2 = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &aIretFrame[0], sizeof(aIretFrame));
3964	if (RT_FAILURE(rc2))
3965	{
3966	rc = VERR_EM_INTERPRETER;
3967	break;
3968	}
3969	pCtx->ip = aIretFrame[0];
3970	pCtx->cs.Sel = aIretFrame[1];
3971	pCtx->cs.ValidSel = aIretFrame[1];
3972	pCtx->cs.u64Base = (uint32_t)pCtx->cs.Sel << 4;
3973	pCtx->eflags.u = (pCtx->eflags.u & ~(X86_EFL_POPF_BITS & uMask))
3974	\| (aIretFrame[2] & X86_EFL_POPF_BITS & uMask);
3975	pCtx->sp += sizeof(aIretFrame);
3976
3977	LogFlow(("iret to %04x:%x\n", pCtx->cs.Sel, pCtx->ip));
3978	fUpdateRIP = false;
3979	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIret);
3980	break;
3981	}
3982
3983	case OP_INT:
3984	{
3985	uint32_t intInfo2;
3986
3987	LogFlow(("Realmode: INT %x\n", pDis->Param1.uValue & 0xff));
3988	intInfo2 = pDis->Param1.uValue & 0xff;
3989	intInfo2 \|= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
3990	intInfo2 \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
3991
3992	rc = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, intInfo2, cbOp, 0);
3993	AssertRC(VBOXSTRICTRC_VAL(rc));
3994	fUpdateRIP = false;
3995	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
3996	break;
3997	}
3998
3999	case OP_INTO:
4000	{
4001	if (pCtx->eflags.Bits.u1OF)
4002	{
4003	uint32_t intInfo2;
4004
4005	LogFlow(("Realmode: INTO\n"));
4006	intInfo2 = X86_XCPT_OF;
4007	intInfo2 \|= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
4008	intInfo2 \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
4009
4010	rc = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, intInfo2, cbOp, 0);
4011	AssertRC(VBOXSTRICTRC_VAL(rc));
4012	fUpdateRIP = false;
4013	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
4014	}
4015	break;
4016	}
4017
4018	case OP_INT3:
4019	{
4020	uint32_t intInfo2;
4021
4022	LogFlow(("Realmode: INT 3\n"));
4023	intInfo2 = 3;
4024	intInfo2 \|= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
4025	intInfo2 \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
4026
4027	rc = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, intInfo2, cbOp, 0);
4028	AssertRC(VBOXSTRICTRC_VAL(rc));
4029	fUpdateRIP = false;
4030	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
4031	break;
4032	}
4033
4034	default:
4035	rc = EMInterpretInstructionDisasState(pVCpu, pDis, CPUMCTX2CORE(pCtx), 0, EMCODETYPE_SUPERVISOR);
4036	fUpdateRIP = false;
4037	break;
4038	}
4039
4040	if (rc == VINF_SUCCESS)
4041	{
4042	if (fUpdateRIP)
4043	pCtx->rip += cbOp; /* Move on to the next instruction. */
4044
4045	/*
4046	* LIDT, LGDT can end up here. In the future CRx changes as well. Just reload the
4047	* whole context to be done with it.
4048	*/
4049	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_ALL;
4050
4051	/* Only resume if successful. */
4052	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2Sub3, y3);
4053	goto ResumeExecution;
4054	}
4055	}
4056	else
4057	rc = VERR_EM_INTERPRETER;
4058
4059	AssertMsg(rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_EM_HALT,
4060	("Unexpected rc=%Rrc\n", VBOXSTRICTRC_VAL(rc)));
4061	break;
4062	}
4063
4064	#ifdef VBOX_STRICT
4065	case X86_XCPT_XF: /* SIMD exception. */
4066	case X86_XCPT_DE: /* Divide error. */
4067	case X86_XCPT_UD: /* Unknown opcode exception. */
4068	case X86_XCPT_SS: /* Stack segment exception. */
4069	case X86_XCPT_NP: /* Segment not present exception. */
4070	{
4071	switch (vector)
4072	{
4073	case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE); break;
4074	case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD); break;
4075	case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS); break;
4076	case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP); break;
4077	case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF); break;
4078	}
4079
4080	Log(("Trap %x at %04X:%RGv\n", vector, pCtx->cs.Sel, (RTGCPTR)pCtx->rip));
4081	rc2 = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo),
4082	cbInstr, errCode);
4083	AssertRC(rc2);
4084
4085	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2Sub3, y3);
4086	goto ResumeExecution;
4087	}
4088	#endif
4089	default:
4090	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
4091	if ( CPUMIsGuestInRealModeEx(pCtx)
4092	&& pVM->hm.s.vmx.pRealModeTSS)
4093	{
4094	Log(("Real Mode Trap %x at %04x:%04X error code %x\n", vector, pCtx->cs.Sel, pCtx->eip, errCode));
4095	rc = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo),
4096	cbInstr, errCode);
4097	AssertRC(VBOXSTRICTRC_VAL(rc)); /* Strict RC check below. */
4098
4099	/* Go back to ring-3 in case of a triple fault. */
4100	if ( vector == X86_XCPT_DF
4101	&& rc == VINF_EM_RESET)
4102	{
4103	break;
4104	}
4105
4106	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2Sub3, y3);
4107	goto ResumeExecution;
4108	}
4109	AssertMsgFailed(("Unexpected vm-exit caused by exception %x\n", vector));
4110	rc = VERR_VMX_UNEXPECTED_EXCEPTION;
4111	break;
4112	} /* switch (vector) */
4113
4114	break;
4115
4116	default:
4117	rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_CODE;
4118	AssertMsgFailed(("Unexpected interruption code %x\n", intInfo));
4119	break;
4120	}
4121
4122	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2Sub3, y3);
4123	break;
4124	}
4125
4126	/*
4127	* 48 EPT violation. An attempt to access memory with a guest-physical address was disallowed
4128	* by the configuration of the EPT paging structures.
4129	*/
4130	case VMX_EXIT_EPT_VIOLATION:
4131	{
4132	RTGCPHYS GCPhys;
4133
4134	Assert(pVM->hm.s.fNestedPaging);
4135
4136	rc2 = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
4137	AssertRC(rc2);
4138	Assert(((exitQualification >> 7) & 3) != 2);
4139
4140	/* Determine the kind of violation. */
4141	errCode = 0;
4142	if (exitQualification & VMX_EXIT_QUALIFICATION_EPT_INSTR_FETCH)
4143	errCode \|= X86_TRAP_PF_ID;
4144
4145	if (exitQualification & VMX_EXIT_QUALIFICATION_EPT_DATA_WRITE)
4146	errCode \|= X86_TRAP_PF_RW;
4147
4148	/* If the page is present, then it's a page level protection fault. */
4149	if (exitQualification & VMX_EXIT_QUALIFICATION_EPT_ENTRY_PRESENT)
4150	errCode \|= X86_TRAP_PF_P;
4151	else
4152	{
4153	/* Shortcut for APIC TPR reads and writes. */
4154	if ( (GCPhys & 0xfff) == 0x080
4155	&& GCPhys > 0x1000000 /* to skip VGA frame buffer accesses */
4156	&& fSetupTPRCaching
4157	&& (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC))
4158	{
4159	RTGCPHYS GCPhysApicBase;
4160	GCPhysApicBase = pCtx->msrApicBase;
4161	GCPhysApicBase &= PAGE_BASE_GC_MASK;
4162	if (GCPhys == GCPhysApicBase + 0x80)
4163	{
4164	Log(("Enable VT-x virtual APIC access filtering\n"));
4165	rc2 = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess,
4166	X86_PTE_RW \| X86_PTE_P);
4167	AssertRC(rc2);
4168	}
4169	}
4170	}
4171	Log(("EPT Page fault %x at %RGp error code %x\n", (uint32_t)exitQualification, GCPhys, errCode));
4172
4173	/* GCPhys contains the guest physical address of the page fault. */
4174	TRPMAssertTrap(pVCpu, X86_XCPT_PF, TRPM_TRAP);
4175	TRPMSetErrorCode(pVCpu, errCode);
4176	TRPMSetFaultAddress(pVCpu, GCPhys);
4177
4178	/* Handle the pagefault trap for the nested shadow table. */
4179	rc = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, errCode, CPUMCTX2CORE(pCtx), GCPhys);
4180
4181	/*
4182	* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment below, @bugref{6043}.
4183	*/
4184	if ( rc == VINF_SUCCESS
4185	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
4186	\|\| rc == VERR_PAGE_NOT_PRESENT)
4187	{
4188	/* We've successfully synced our shadow pages, so let's just continue execution. */
4189	Log2(("Shadow page fault at %RGv cr2=%RGp error code %x\n", (RTGCPTR)pCtx->rip, exitQualification , errCode));
4190	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
4191
4192	TRPMResetTrap(pVCpu);
4193	goto ResumeExecution;
4194	}
4195
4196	#ifdef VBOX_STRICT
4197	if (rc != VINF_EM_RAW_EMULATE_INSTR)
4198	LogFlow(("PGMTrap0eHandlerNestedPaging at %RGv failed with %Rrc\n", (RTGCPTR)pCtx->rip, VBOXSTRICTRC_VAL(rc)));
4199	#endif
4200	/* Need to go back to the recompiler to emulate the instruction. */
4201	TRPMResetTrap(pVCpu);
4202	break;
4203	}
4204
4205	case VMX_EXIT_EPT_MISCONFIG:
4206	{
4207	RTGCPHYS GCPhys;
4208
4209	Assert(pVM->hm.s.fNestedPaging);
4210
4211	rc2 = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
4212	AssertRC(rc2);
4213	Log(("VMX_EXIT_EPT_MISCONFIG for %RGp\n", GCPhys));
4214
4215	/* Shortcut for APIC TPR reads and writes. */
4216	if ( (GCPhys & 0xfff) == 0x080
4217	&& GCPhys > 0x1000000 /* to skip VGA frame buffer accesses */
4218	&& fSetupTPRCaching
4219	&& (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC))
4220	{
4221	RTGCPHYS GCPhysApicBase = pCtx->msrApicBase;
4222	GCPhysApicBase &= PAGE_BASE_GC_MASK;
4223	if (GCPhys == GCPhysApicBase + 0x80)
4224	{
4225	Log(("Enable VT-x virtual APIC access filtering\n"));
4226	rc2 = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess,
4227	X86_PTE_RW \| X86_PTE_P);
4228	AssertRC(rc2);
4229	}
4230	}
4231
4232	rc = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pCtx), GCPhys, UINT32_MAX);
4233
4234	/*
4235	* If we succeed, resume execution.
4236	* Or, if fail in interpreting the instruction because we couldn't get the guest physical address
4237	* of the page containing the instruction via the guest's page tables (we would invalidate the guest page
4238	* in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
4239	* weird case. See @bugref{6043}.
4240	*/
4241	if ( rc == VINF_SUCCESS
4242	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
4243	\|\| rc == VERR_PAGE_NOT_PRESENT)
4244	{
4245	Log2(("PGMR0Trap0eHandlerNPMisconfig(,,,%RGp) at %RGv -> resume\n", GCPhys, (RTGCPTR)pCtx->rip));
4246	goto ResumeExecution;
4247	}
4248
4249	Log2(("PGMR0Trap0eHandlerNPMisconfig(,,,%RGp) at %RGv -> %Rrc\n", GCPhys, (RTGCPTR)pCtx->rip, VBOXSTRICTRC_VAL(rc)));
4250	break;
4251	}
4252
4253	case VMX_EXIT_INT_WINDOW: /* 7 Interrupt window exiting. */
4254	/* Clear VM-exit on IF=1 change. */
4255	LogFlow(("VMX_EXIT_INT_WINDOW %RGv pending=%d IF=%d\n", (RTGCPTR)pCtx->rip,
4256	VMCPU_FF_ISPENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC\|VMCPU_FF_INTERRUPT_PIC)), pCtx->eflags.Bits.u1IF));
4257	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INT_WINDOW_EXIT;
4258	rc2 = VMXWriteVmcs(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls);
4259	AssertRC(rc2);
4260	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
4261	goto ResumeExecution; /* we check for pending guest interrupts there */
4262
4263	case VMX_EXIT_WBINVD: /* 54 Guest software attempted to execute WBINVD. (conditional) */
4264	case VMX_EXIT_INVD: /* 13 Guest software attempted to execute INVD. (unconditional) */
4265	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvd);
4266	/* Skip instruction and continue directly. */
4267	pCtx->rip += cbInstr;
4268	/* Continue execution.*/
4269	goto ResumeExecution;
4270
4271	case VMX_EXIT_CPUID: /* 10 Guest software attempted to execute CPUID. */
4272	{
4273	Log2(("VMX: Cpuid %x\n", pCtx->eax));
4274	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCpuid);
4275	rc = EMInterpretCpuId(pVM, pVCpu, CPUMCTX2CORE(pCtx));
4276	if (rc == VINF_SUCCESS)
4277	{
4278	/* Update EIP and continue execution. */
4279	Assert(cbInstr == 2);
4280	pCtx->rip += cbInstr;
4281	goto ResumeExecution;
4282	}
4283	AssertMsgFailed(("EMU: cpuid failed with %Rrc\n", VBOXSTRICTRC_VAL(rc)));
4284	rc = VINF_EM_RAW_EMULATE_INSTR;
4285	break;
4286	}
4287
4288	case VMX_EXIT_RDPMC: /* 15 Guest software attempted to execute RDPMC. */
4289	{
4290	Log2(("VMX: Rdpmc %x\n", pCtx->ecx));
4291	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdpmc);
4292	rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pCtx));
4293	if (rc == VINF_SUCCESS)
4294	{
4295	/* Update EIP and continue execution. */
4296	Assert(cbInstr == 2);
4297	pCtx->rip += cbInstr;
4298	goto ResumeExecution;
4299	}
4300	rc = VINF_EM_RAW_EMULATE_INSTR;
4301	break;
4302	}
4303
4304	case VMX_EXIT_RDTSC: /* 16 Guest software attempted to execute RDTSC. */
4305	{
4306	Log2(("VMX: Rdtsc\n"));
4307	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
4308	rc = EMInterpretRdtsc(pVM, pVCpu, CPUMCTX2CORE(pCtx));
4309	if (rc == VINF_SUCCESS)
4310	{
4311	/* Update EIP and continue execution. */
4312	Assert(cbInstr == 2);
4313	pCtx->rip += cbInstr;
4314	fNeedTscSetup = true; /* See @bugref{6634}. */
4315	goto ResumeExecution;
4316	}
4317	rc = VINF_EM_RAW_EMULATE_INSTR;
4318	break;
4319	}
4320
4321	case VMX_EXIT_RDTSCP: /* 51 Guest software attempted to execute RDTSCP. */
4322	{
4323	Log2(("VMX: Rdtscp\n"));
4324	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtscp);
4325	rc = EMInterpretRdtscp(pVM, pVCpu, pCtx);
4326	if (rc == VINF_SUCCESS)
4327	{
4328	/* Update EIP and continue execution. */
4329	Assert(cbInstr == 3);
4330	pCtx->rip += cbInstr;
4331	fNeedTscSetup = true; /* See @bugref{6634}. */
4332	goto ResumeExecution;
4333	}
4334	rc = VINF_EM_RAW_EMULATE_INSTR;
4335	break;
4336	}
4337
4338	case VMX_EXIT_INVLPG: /* 14 Guest software attempted to execute INVLPG. */
4339	{
4340	Log2(("VMX: invlpg\n"));
4341	Assert(!pVM->hm.s.fNestedPaging);
4342
4343	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvlpg);
4344	rc = EMInterpretInvlpg(pVM, pVCpu, CPUMCTX2CORE(pCtx), exitQualification);
4345	if (rc == VINF_SUCCESS)
4346	{
4347	/* Update EIP and continue execution. */
4348	pCtx->rip += cbInstr;
4349	goto ResumeExecution;
4350	}
4351	AssertMsg(rc == VERR_EM_INTERPRETER, ("EMU: invlpg %RGv failed with %Rrc\n", exitQualification, VBOXSTRICTRC_VAL(rc)));
4352	break;
4353	}
4354
4355	case VMX_EXIT_MONITOR: /* 39 Guest software attempted to execute MONITOR. */
4356	{
4357	Log2(("VMX: monitor\n"));
4358
4359	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMonitor);
4360	rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pCtx));
4361	if (rc == VINF_SUCCESS)
4362	{
4363	/* Update EIP and continue execution. */
4364	pCtx->rip += cbInstr;
4365	goto ResumeExecution;
4366	}
4367	AssertMsg(rc == VERR_EM_INTERPRETER, ("EMU: monitor failed with %Rrc\n", VBOXSTRICTRC_VAL(rc)));
4368	break;
4369	}
4370
4371	case VMX_EXIT_WRMSR: /* 32 WRMSR. Guest software attempted to execute WRMSR. */
4372	/* When an interrupt is pending, we'll let MSR_K8_LSTAR writes fault in our TPR patch code. */
4373	if ( pVM->hm.s.fTPRPatchingActive
4374	&& pCtx->ecx == MSR_K8_LSTAR)
4375	{
4376	Assert(!CPUMIsGuestInLongModeEx(pCtx));
4377	if ((pCtx->eax & 0xff) != u8LastTPR)
4378	{
4379	Log(("VMX: Faulting MSR_K8_LSTAR write with new TPR value %x\n", pCtx->eax & 0xff));
4380
4381	/* Our patch code uses LSTAR for TPR caching. */
4382	rc2 = PDMApicSetTPR(pVCpu, pCtx->eax & 0xff);
4383	AssertRC(rc2);
4384	}
4385
4386	/* Skip the instruction and continue. */
4387	pCtx->rip += cbInstr; /* wrmsr = [0F 30] */
4388
4389	/* Only resume if successful. */
4390	goto ResumeExecution;
4391	}
4392	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_MSR;
4393	/* no break */
4394	case VMX_EXIT_RDMSR: /* 31 RDMSR. Guest software attempted to execute RDMSR. */
4395	{
4396	STAM_COUNTER_INC((exitReason == VMX_EXIT_RDMSR) ? &pVCpu->hm.s.StatExitRdmsr : &pVCpu->hm.s.StatExitWrmsr);
4397
4398	Log2(("VMX: %s\n", (exitReason == VMX_EXIT_RDMSR) ? "rdmsr" : "wrmsr"));
4399	rc = EMInterpretInstruction(pVCpu, CPUMCTX2CORE(pCtx), 0);
4400	if (rc == VINF_SUCCESS)
4401	{
4402	/* EIP has been updated already. */
4403	/* Only resume if successful. */
4404	goto ResumeExecution;
4405	}
4406	AssertMsg(rc == VERR_EM_INTERPRETER, ("EMU: %s failed with %Rrc\n",
4407	(exitReason == VMX_EXIT_RDMSR) ? "rdmsr" : "wrmsr", VBOXSTRICTRC_VAL(rc)));
4408	break;
4409	}
4410
4411	case VMX_EXIT_MOV_CRX: /* 28 Control-register accesses. */
4412	{
4413	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExit2Sub2, y2);
4414
4415	switch (VMX_EXIT_QUALIFICATION_CRX_ACCESS(exitQualification))
4416	{
4417	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_WRITE:
4418	{
4419	Log2(("VMX: %RGv mov cr%d, x\n", (RTGCPTR)pCtx->rip, VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification)));
4420	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxWrite[VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification)]);
4421	rc = EMInterpretCRxWrite(pVM, pVCpu, CPUMCTX2CORE(pCtx),
4422	VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification),
4423	VMX_EXIT_QUALIFICATION_CRX_GENREG(exitQualification));
4424	switch (VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification))
4425	{
4426	case 0:
4427	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0 \| HM_CHANGED_GUEST_CR3;
4428	break;
4429	case 2:
4430	break;
4431	case 3:
4432	Assert(!pVM->hm.s.fNestedPaging \|\| !CPUMIsGuestInPagedProtectedModeEx(pCtx));
4433	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR3;
4434	break;
4435	case 4:
4436	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR4;
4437	break;
4438	case 8:
4439	/* CR8 contains the APIC TPR */
4440	Assert(!(pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1
4441	& VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW));
4442	break;
4443
4444	default:
4445	AssertFailed();
4446	break;
4447	}
4448	break;
4449	}
4450
4451	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ:
4452	{
4453	Log2(("VMX: mov x, crx\n"));
4454	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxRead[VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification)]);
4455
4456	Assert( !pVM->hm.s.fNestedPaging
4457	\|\| !CPUMIsGuestInPagedProtectedModeEx(pCtx)
4458	\|\| VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification) != DISCREG_CR3);
4459
4460	/* CR8 reads only cause an exit when the TPR shadow feature isn't present. */
4461	Assert( VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification) != 8
4462	\|\| !(pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW));
4463
4464	rc = EMInterpretCRxRead(pVM, pVCpu, CPUMCTX2CORE(pCtx),
4465	VMX_EXIT_QUALIFICATION_CRX_GENREG(exitQualification),
4466	VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification));
4467	break;
4468	}
4469
4470	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_CLTS:
4471	{
4472	Log2(("VMX: clts\n"));
4473	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
4474	rc = EMInterpretCLTS(pVM, pVCpu);
4475	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
4476	break;
4477	}
4478
4479	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_LMSW:
4480	{
4481	Log2(("VMX: lmsw %x\n", VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(exitQualification)));
4482	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
4483	rc = EMInterpretLMSW(pVM, pVCpu, CPUMCTX2CORE(pCtx), VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(exitQualification));
4484	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
4485	break;
4486	}
4487	}
4488
4489	/* Update EIP if no error occurred. */
4490	if (RT_SUCCESS(rc))
4491	pCtx->rip += cbInstr;
4492
4493	if (rc == VINF_SUCCESS)
4494	{
4495	/* Only resume if successful. */
4496	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2Sub2, y2);
4497	goto ResumeExecution;
4498	}
4499	Assert(rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_PGM_SYNC_CR3);
4500	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2Sub2, y2);
4501	break;
4502	}
4503
4504	case VMX_EXIT_MOV_DRX: /* 29 Debug-register accesses. */
4505	{
4506	if ( !DBGFIsStepping(pVCpu)
4507	&& !CPUMIsHyperDebugStateActive(pVCpu))
4508	{
4509	/* Disable DRx move intercepts. */
4510	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT;
4511	rc2 = VMXWriteVmcs(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls);
4512	AssertRC(rc2);
4513
4514	/* Save the host and load the guest debug state. */
4515	rc2 = CPUMR0LoadGuestDebugState(pVM, pVCpu, pCtx, true /* include DR6 */);
4516	AssertRC(rc2);
4517
4518	#ifdef LOG_ENABLED
4519	if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(exitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
4520	{
4521	Log(("VMX_EXIT_MOV_DRX: write DR%d genreg %d\n", VMX_EXIT_QUALIFICATION_DRX_REGISTER(exitQualification),
4522	VMX_EXIT_QUALIFICATION_DRX_GENREG(exitQualification)));
4523	}
4524	else
4525	Log(("VMX_EXIT_MOV_DRX: read DR%d\n", VMX_EXIT_QUALIFICATION_DRX_REGISTER(exitQualification)));
4526	#endif
4527
4528	#ifdef VBOX_WITH_STATISTICS
4529	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
4530	if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(exitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
4531	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
4532	else
4533	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
4534	#endif
4535
4536	goto ResumeExecution;
4537	}
4538
4539	/** @todo clear VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT after the first
4540	* time and restore DRx registers afterwards */
4541	if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(exitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
4542	{
4543	Log2(("VMX: mov DRx%d, genreg%d\n", VMX_EXIT_QUALIFICATION_DRX_REGISTER(exitQualification),
4544	VMX_EXIT_QUALIFICATION_DRX_GENREG(exitQualification)));
4545	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
4546	rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pCtx),
4547	VMX_EXIT_QUALIFICATION_DRX_REGISTER(exitQualification),
4548	VMX_EXIT_QUALIFICATION_DRX_GENREG(exitQualification));
4549	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_DEBUG;
4550	Log2(("DR7=%08x\n", pCtx->dr[7]));
4551	}
4552	else
4553	{
4554	Log2(("VMX: mov x, DRx\n"));
4555	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
4556	rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pCtx),
4557	VMX_EXIT_QUALIFICATION_DRX_GENREG(exitQualification),
4558	VMX_EXIT_QUALIFICATION_DRX_REGISTER(exitQualification));
4559	}
4560	/* Update EIP if no error occurred. */
4561	if (RT_SUCCESS(rc))
4562	pCtx->rip += cbInstr;
4563
4564	if (rc == VINF_SUCCESS)
4565	{
4566	/* Only resume if successful. */
4567	goto ResumeExecution;
4568	}
4569	Assert(rc == VERR_EM_INTERPRETER);
4570	break;
4571	}
4572
4573	/* Note: We'll get a #GP if the IO instruction isn't allowed (IOPL or TSS bitmap); no need to double check. */
4574	case VMX_EXIT_IO_INSTR: /* 30 I/O instruction. */
4575	{
4576	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExit2Sub1, y1);
4577	uint32_t uPort;
4578	uint32_t uIOWidth = VMX_EXIT_QUALIFICATION_IO_WIDTH(exitQualification);
4579	bool fIOWrite = (VMX_EXIT_QUALIFICATION_IO_DIRECTION(exitQualification) == VMX_EXIT_QUALIFICATION_IO_DIRECTION_OUT);
4580
4581	/** @todo necessary to make the distinction? */
4582	if (VMX_EXIT_QUALIFICATION_IO_ENCODING(exitQualification) == VMX_EXIT_QUALIFICATION_IO_ENCODING_DX)
4583	uPort = pCtx->edx & 0xffff;
4584	else
4585	uPort = VMX_EXIT_QUALIFICATION_IO_PORT(exitQualification); /* Immediate encoding. */
4586
4587	if (RT_UNLIKELY(uIOWidth == 2 \|\| uIOWidth >= 4)) /* paranoia */
4588	{
4589	rc = fIOWrite ? VINF_IOM_R3_IOPORT_WRITE : VINF_IOM_R3_IOPORT_READ;
4590	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2Sub1, y1);
4591	break;
4592	}
4593
4594	uint32_t cbSize = g_aIOSize[uIOWidth];
4595	if (VMX_EXIT_QUALIFICATION_IO_STRING(exitQualification))
4596	{
4597	/* ins/outs */
4598	PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
4599
4600	/* Disassemble manually to deal with segment prefixes. */
4601	/** @todo VMX_VMCS_RO_EXIT_GUEST_LINEAR_ADDR contains the flat pointer
4602	* operand of the instruction. */
4603	/** @todo VMX_VMCS32_RO_EXIT_INSTR_INFO also contains segment prefix info. */
4604	rc2 = EMInterpretDisasCurrent(pVM, pVCpu, pDis, NULL);
4605	if (RT_SUCCESS(rc))
4606	{
4607	if (fIOWrite)
4608	{
4609	Log2(("IOMInterpretOUTSEx %RGv %x size=%d\n", (RTGCPTR)pCtx->rip, uPort, cbSize));
4610	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringWrite);
4611	rc = IOMInterpretOUTSEx(pVM, CPUMCTX2CORE(pCtx), uPort, pDis->fPrefix, (DISCPUMODE)pDis->uAddrMode, cbSize);
4612	}
4613	else
4614	{
4615	Log2(("IOMInterpretINSEx %RGv %x size=%d\n", (RTGCPTR)pCtx->rip, uPort, cbSize));
4616	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringRead);
4617	rc = IOMInterpretINSEx(pVM, CPUMCTX2CORE(pCtx), uPort, pDis->fPrefix, (DISCPUMODE)pDis->uAddrMode, cbSize);
4618	}
4619	}
4620	else
4621	rc = VINF_EM_RAW_EMULATE_INSTR;
4622	}
4623	else
4624	{
4625	/* Normal in/out */
4626	uint32_t uAndVal = g_aIOOpAnd[uIOWidth];
4627
4628	Assert(!VMX_EXIT_QUALIFICATION_IO_REP(exitQualification));
4629
4630	if (fIOWrite)
4631	{
4632	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
4633	rc = IOMIOPortWrite(pVM, uPort, pCtx->eax & uAndVal, cbSize);
4634	if (rc == VINF_IOM_R3_IOPORT_WRITE)
4635	HMR0SavePendingIOPortWrite(pVCpu, pCtx->rip, pCtx->rip + cbInstr, uPort, uAndVal, cbSize);
4636	}
4637	else
4638	{
4639	uint32_t u32Val = 0;
4640
4641	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
4642	rc = IOMIOPortRead(pVM, uPort, &u32Val, cbSize);
4643	if (IOM_SUCCESS(rc))
4644	{
4645	/* Write back to the EAX register. */
4646	pCtx->eax = (pCtx->eax & ~uAndVal) \| (u32Val & uAndVal);
4647	}
4648	else
4649	if (rc == VINF_IOM_R3_IOPORT_READ)
4650	HMR0SavePendingIOPortRead(pVCpu, pCtx->rip, pCtx->rip + cbInstr, uPort, uAndVal, cbSize);
4651	}
4652	}
4653
4654	/*
4655	* Handled the I/O return codes.
4656	* (The unhandled cases end up with rc == VINF_EM_RAW_EMULATE_INSTR.)
4657	*/
4658	if (IOM_SUCCESS(rc))
4659	{
4660	/* Update EIP and continue execution. */
4661	pCtx->rip += cbInstr;
4662	if (RT_LIKELY(rc == VINF_SUCCESS))
4663	{
4664	/* If any IO breakpoints are armed, then we should check if a debug trap needs to be generated. */
4665	if (pCtx->dr[7] & X86_DR7_ENABLED_MASK)
4666	{
4667	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
4668	for (unsigned i = 0; i < 4; i++)
4669	{
4670	unsigned uBPLen = g_aIOSize[X86_DR7_GET_LEN(pCtx->dr[7], i)];
4671
4672	if ( (uPort >= pCtx->dr[i] && uPort < pCtx->dr[i] + uBPLen)
4673	&& (pCtx->dr[7] & (X86_DR7_L(i) \| X86_DR7_G(i)))
4674	&& (pCtx->dr[7] & X86_DR7_RW(i, X86_DR7_RW_IO)) == X86_DR7_RW(i, X86_DR7_RW_IO))
4675	{
4676	uint64_t uDR6;
4677
4678	Assert(CPUMIsGuestDebugStateActive(pVCpu));
4679
4680	uDR6 = ASMGetDR6();
4681
4682	/* Clear all breakpoint status flags and set the one we just hit. */
4683	uDR6 &= ~(X86_DR6_B0\|X86_DR6_B1\|X86_DR6_B2\|X86_DR6_B3);
4684	uDR6 \|= (uint64_t)RT_BIT(i);
4685
4686	/*
4687	* Note: AMD64 Architecture Programmer's Manual 13.1:
4688	* Bits 15:13 of the DR6 register is never cleared by the processor and must
4689	* be cleared by software after the contents have been read.
4690	*/
4691	ASMSetDR6(uDR6);
4692
4693	/* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
4694	pCtx->dr[7] &= ~X86_DR7_GD;
4695
4696	/* Paranoia. */
4697	pCtx->dr[7] &= 0xffffffff; /* upper 32 bits reserved */
4698	pCtx->dr[7] &= ~(RT_BIT(11) \| RT_BIT(12) \| RT_BIT(14) \| RT_BIT(15)); /* must be zero */
4699	pCtx->dr[7] \|= 0x400; /* must be one */
4700
4701	/* Resync DR7 */
4702	rc2 = VMXWriteVmcs64(VMX_VMCS_GUEST_DR7, pCtx->dr[7]);
4703	AssertRC(rc2);
4704
4705	/* Construct inject info. */
4706	intInfo = X86_XCPT_DB;
4707	intInfo \|= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
4708	intInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
4709
4710	Log(("Inject IO debug trap at %RGv\n", (RTGCPTR)pCtx->rip));
4711	rc2 = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo),
4712	0 /* cbInstr /, 0 / errCode */);
4713	AssertRC(rc2);
4714
4715	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2Sub1, y1);
4716	goto ResumeExecution;
4717	}
4718	}
4719	}
4720	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2Sub1, y1);
4721	goto ResumeExecution;
4722	}
4723	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2Sub1, y1);
4724	break;
4725	}
4726
4727	#ifdef VBOX_STRICT
4728	if (rc == VINF_IOM_R3_IOPORT_READ)
4729	Assert(!fIOWrite);
4730	else if (rc == VINF_IOM_R3_IOPORT_WRITE)
4731	Assert(fIOWrite);
4732	else
4733	{
4734	AssertMsg( RT_FAILURE(rc)
4735	\|\| rc == VINF_EM_RAW_EMULATE_INSTR
4736	\|\| rc == VINF_EM_RAW_GUEST_TRAP
4737	\|\| rc == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", VBOXSTRICTRC_VAL(rc)));
4738	}
4739	#endif
4740	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2Sub1, y1);
4741	break;
4742	}
4743
4744	case VMX_EXIT_TPR_BELOW_THRESHOLD: /* 43 TPR below threshold. Guest software executed MOV to CR8. */
4745	LogFlow(("VMX_EXIT_TPR_BELOW_THRESHOLD\n"));
4746	/* RIP is already set to the next instruction and the TPR has been synced back. Just resume. */
4747	goto ResumeExecution;
4748
4749	case VMX_EXIT_APIC_ACCESS: /* 44 APIC access. Guest software attempted to access memory at a physical address
4750	on the APIC-access page. */
4751	{
4752	LogFlow(("VMX_EXIT_APIC_ACCESS\n"));
4753	unsigned uAccessType = VMX_EXIT_QUALIFICATION_APIC_ACCESS_TYPE(exitQualification);
4754
4755	switch (uAccessType)
4756	{
4757	case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
4758	case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
4759	{
4760	RTGCPHYS GCPhys = pCtx->msrApicBase;
4761	GCPhys &= PAGE_BASE_GC_MASK;
4762	GCPhys += VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(exitQualification);
4763
4764	LogFlow(("Apic access at %RGp\n", GCPhys));
4765	rc = IOMMMIOPhysHandler(pVM, (uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ) ? 0 : X86_TRAP_PF_RW,
4766	CPUMCTX2CORE(pCtx), GCPhys);
4767	if (rc == VINF_SUCCESS)
4768	goto ResumeExecution; /* rip already updated */
4769	break;
4770	}
4771
4772	default:
4773	rc = VINF_EM_RAW_EMULATE_INSTR;
4774	break;
4775	}
4776	break;
4777	}
4778
4779	case VMX_EXIT_PREEMPTION_TIMER: /* 52 VMX-preemption timer expired. The preemption timer counted down to zero. */
4780	if (!TMTimerPollBool(pVM, pVCpu))
4781	goto ResumeExecution;
4782	rc = VINF_EM_RAW_TIMER_PENDING;
4783	break;
4784
4785	default:
4786	/* The rest is handled after syncing the entire CPU state. */
4787	break;
4788	}
4789
4790
4791	/*
4792	* Note: The guest state is not entirely synced back at this stage!
4793	*/
4794
4795	/* Investigate why there was a VM-exit. (part 2) */
4796	switch (exitReason)
4797	{
4798	case VMX_EXIT_XCPT_NMI: /* 0 Exception or non-maskable interrupt (NMI). */
4799	case VMX_EXIT_EXT_INT: /* 1 External interrupt. */
4800	case VMX_EXIT_EPT_VIOLATION:
4801	case VMX_EXIT_EPT_MISCONFIG: /* 49 EPT misconfig is used by the PGM/MMIO optimizations. */
4802	case VMX_EXIT_PREEMPTION_TIMER: /* 52 VMX-preemption timer expired. The preemption timer counted down to zero. */
4803	/* Already handled above. */
4804	break;
4805
4806	case VMX_EXIT_TRIPLE_FAULT: /* 2 Triple fault. */
4807	rc = VINF_EM_RESET; /* Triple fault equals a reset. */
4808	break;
4809
4810	case VMX_EXIT_INIT_SIGNAL: /* 3 INIT signal. */
4811	case VMX_EXIT_SIPI: /* 4 Start-up IPI (SIPI). */
4812	rc = VINF_EM_RAW_INTERRUPT;
4813	AssertFailed(); /* Can't happen. Yet. */
4814	break;
4815
4816	case VMX_EXIT_IO_SMI: /* 5 I/O system-management interrupt (SMI). */
4817	case VMX_EXIT_SMI: /* 6 Other SMI. */
4818	rc = VINF_EM_RAW_INTERRUPT;
4819	AssertFailed(); /* Can't happen afaik. */
4820	break;
4821
4822	case VMX_EXIT_TASK_SWITCH: /* 9 Task switch: too complicated to emulate, so fall back to the recompiler */
4823	Log(("VMX_EXIT_TASK_SWITCH: exit=%RX64\n", exitQualification));
4824	if ( (VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE(exitQualification) == VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE_IDT)
4825	&& pVCpu->hm.s.Event.fPending)
4826	{
4827	/* Caused by an injected interrupt. */
4828	pVCpu->hm.s.Event.fPending = false;
4829
4830	Log(("VMX_EXIT_TASK_SWITCH: reassert trap %d\n", VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVCpu->hm.s.Event.u64IntrInfo)));
4831	Assert(!VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntrInfo));
4832	//@todo: Why do we assume this had to be a hardware interrupt? What about software interrupts or exceptions?
4833	rc2 = TRPMAssertTrap(pVCpu, VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVCpu->hm.s.Event.u64IntrInfo), TRPM_HARDWARE_INT);
4834	AssertRC(rc2);
4835	}
4836	/* else Exceptions and software interrupts can just be restarted. */
4837	rc = VERR_EM_INTERPRETER;
4838	break;
4839
4840	case VMX_EXIT_HLT: /* 12 Guest software attempted to execute HLT. */
4841	/* Check if external interrupts are pending; if so, don't switch back. */
4842	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
4843	pCtx->rip++; /* skip hlt */
4844	if (EMShouldContinueAfterHalt(pVCpu, pCtx))
4845	goto ResumeExecution;
4846
4847	rc = VINF_EM_HALT;
4848	break;
4849
4850	case VMX_EXIT_MWAIT: /* 36 Guest software executed MWAIT. */
4851	Log2(("VMX: mwait\n"));
4852	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMwait);
4853	rc = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pCtx));
4854	if ( rc == VINF_EM_HALT
4855	\|\| rc == VINF_SUCCESS)
4856	{
4857	/* Update EIP and continue execution. */
4858	pCtx->rip += cbInstr;
4859
4860	/* Check if external interrupts are pending; if so, don't switch back. */
4861	if ( rc == VINF_SUCCESS
4862	\|\| ( rc == VINF_EM_HALT
4863	&& EMShouldContinueAfterHalt(pVCpu, pCtx))
4864	)
4865	goto ResumeExecution;
4866	}
4867	AssertMsg(rc == VERR_EM_INTERPRETER \|\| rc == VINF_EM_HALT, ("EMU: mwait failed with %Rrc\n", VBOXSTRICTRC_VAL(rc)));
4868	break;
4869
4870	case VMX_EXIT_RSM: /* 17 Guest software attempted to execute RSM in SMM. */
4871	AssertFailed(); /* can't happen. */
4872	rc = VERR_EM_INTERPRETER;
4873	break;
4874
4875	case VMX_EXIT_MTF: /* 37 Exit due to Monitor Trap Flag. */
4876	LogFlow(("VMX_EXIT_MTF at %RGv\n", (RTGCPTR)pCtx->rip));
4877	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_TRAP_FLAG;
4878	rc2 = VMXWriteVmcs(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls);
4879	AssertRC(rc2);
4880	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMtf);
4881	#if 0
4882	DBGFDoneStepping(pVCpu);
4883	#endif
4884	rc = VINF_EM_DBG_STOP;
4885	break;
4886
4887	case VMX_EXIT_VMCALL: /* 18 Guest software executed VMCALL. */
4888	case VMX_EXIT_VMCLEAR: /* 19 Guest software executed VMCLEAR. */
4889	case VMX_EXIT_VMLAUNCH: /* 20 Guest software executed VMLAUNCH. */
4890	case VMX_EXIT_VMPTRLD: /* 21 Guest software executed VMPTRLD. */
4891	case VMX_EXIT_VMPTRST: /* 22 Guest software executed VMPTRST. */
4892	case VMX_EXIT_VMREAD: /* 23 Guest software executed VMREAD. */
4893	case VMX_EXIT_VMRESUME: /* 24 Guest software executed VMRESUME. */
4894	case VMX_EXIT_VMWRITE: /* 25 Guest software executed VMWRITE. */
4895	case VMX_EXIT_VMXOFF: /* 26 Guest software executed VMXOFF. */
4896	case VMX_EXIT_VMXON: /* 27 Guest software executed VMXON. */
4897	/** @todo inject #UD immediately */
4898	rc = VERR_EM_INTERPRETER;
4899	break;
4900
4901	case VMX_EXIT_CPUID: /* 10 Guest software attempted to execute CPUID. */
4902	case VMX_EXIT_RDTSC: /* 16 Guest software attempted to execute RDTSC. */
4903	case VMX_EXIT_INVLPG: /* 14 Guest software attempted to execute INVLPG. */
4904	case VMX_EXIT_MOV_CRX: /* 28 Control-register accesses. */
4905	case VMX_EXIT_MOV_DRX: /* 29 Debug-register accesses. */
4906	case VMX_EXIT_IO_INSTR: /* 30 I/O instruction. */
4907	case VMX_EXIT_RDPMC: /* 15 Guest software attempted to execute RDPMC. */
4908	case VMX_EXIT_RDTSCP: /* 51 Guest software attempted to execute RDTSCP. */
4909	/* already handled above */
4910	AssertMsg( rc == VINF_PGM_CHANGE_MODE
4911	\|\| rc == VINF_EM_RAW_INTERRUPT
4912	\|\| rc == VERR_EM_INTERPRETER
4913	\|\| rc == VINF_EM_RAW_EMULATE_INSTR
4914	\|\| rc == VINF_PGM_SYNC_CR3
4915	\|\| rc == VINF_IOM_R3_IOPORT_READ
4916	\|\| rc == VINF_IOM_R3_IOPORT_WRITE
4917	\|\| rc == VINF_EM_RAW_GUEST_TRAP
4918	\|\| rc == VINF_TRPM_XCPT_DISPATCHED
4919	\|\| rc == VINF_EM_RESCHEDULE_REM,
4920	("rc = %d\n", VBOXSTRICTRC_VAL(rc)));
4921	break;
4922
4923	case VMX_EXIT_TPR_BELOW_THRESHOLD: /* 43 TPR below threshold. Guest software executed MOV to CR8. */
4924	case VMX_EXIT_RDMSR: /* 31 RDMSR. Guest software attempted to execute RDMSR. */
4925	case VMX_EXIT_WRMSR: /* 32 WRMSR. Guest software attempted to execute WRMSR. */
4926	case VMX_EXIT_PAUSE: /* 40 Guest software attempted to execute PAUSE. */
4927	case VMX_EXIT_MONITOR: /* 39 Guest software attempted to execute MONITOR. */
4928	case VMX_EXIT_APIC_ACCESS: /* 44 APIC access. Guest software attempted to access memory at a physical address
4929	on the APIC-access page. */
4930	{
4931	/*
4932	* If we decided to emulate them here, then we must sync the MSRs that could have been changed (sysenter, FS/GS base)
4933	*/
4934	rc = VERR_EM_INTERPRETER;
4935	break;
4936	}
4937
4938	case VMX_EXIT_INT_WINDOW: /* 7 Interrupt window. */
4939	Assert(rc == VINF_EM_RAW_INTERRUPT);
4940	break;
4941
4942	case VMX_EXIT_ERR_INVALID_GUEST_STATE: /* 33 VM-entry failure due to invalid guest state. */
4943	{
4944	#ifdef VBOX_STRICT
4945	RTCCUINTREG val2 = 0;
4946
4947	Log(("VMX_EXIT_ERR_INVALID_GUEST_STATE\n"));
4948
4949	VMXReadVmcs(VMX_VMCS_GUEST_RIP, &val2);
4950	Log(("Old eip %RGv new %RGv\n", (RTGCPTR)pCtx->rip, (RTGCPTR)val2));
4951
4952	VMXReadVmcs(VMX_VMCS_GUEST_CR0, &val2);
4953	Log(("VMX_VMCS_GUEST_CR0 %RX64\n", (uint64_t)val2));
4954
4955	VMXReadVmcs(VMX_VMCS_GUEST_CR3, &val2);
4956	Log(("VMX_VMCS_GUEST_CR3 %RX64\n", (uint64_t)val2));
4957
4958	VMXReadVmcs(VMX_VMCS_GUEST_CR4, &val2);
4959	Log(("VMX_VMCS_GUEST_CR4 %RX64\n", (uint64_t)val2));
4960
4961	VMXReadVmcs(VMX_VMCS_GUEST_RFLAGS, &val2);
4962	Log(("VMX_VMCS_GUEST_RFLAGS %08x\n", val2));
4963
4964	VMX_LOG_SELREG(CS, "CS", val2);
4965	VMX_LOG_SELREG(DS, "DS", val2);
4966	VMX_LOG_SELREG(ES, "ES", val2);
4967	VMX_LOG_SELREG(FS, "FS", val2);
4968	VMX_LOG_SELREG(GS, "GS", val2);
4969	VMX_LOG_SELREG(SS, "SS", val2);
4970	VMX_LOG_SELREG(TR, "TR", val2);
4971	VMX_LOG_SELREG(LDTR, "LDTR", val2);
4972
4973	VMXReadVmcs(VMX_VMCS_GUEST_GDTR_BASE, &val2);
4974	Log(("VMX_VMCS_GUEST_GDTR_BASE %RX64\n", (uint64_t)val2));
4975	VMXReadVmcs(VMX_VMCS_GUEST_IDTR_BASE, &val2);
4976	Log(("VMX_VMCS_GUEST_IDTR_BASE %RX64\n", (uint64_t)val2));
4977	#endif /* VBOX_STRICT */
4978	rc = VERR_VMX_INVALID_GUEST_STATE;
4979	break;
4980	}
4981
4982	case VMX_EXIT_ERR_MSR_LOAD: /* 34 VM-entry failure due to MSR loading. */
4983	case VMX_EXIT_ERR_MACHINE_CHECK: /* 41 VM-entry failure due to machine-check. */
4984	default:
4985	rc = VERR_VMX_UNEXPECTED_EXIT_CODE;
4986	AssertMsgFailed(("Unexpected exit code %d\n", exitReason)); /* Can't happen. */
4987	break;
4988
4989	}
4990
4991	end:
4992	/* We now going back to ring-3, so clear the action flag. */
4993	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
4994
4995	/*
4996	* Signal changes for the recompiler.
4997	*/
4998	CPUMSetChangedFlags(pVCpu,
4999	CPUM_CHANGED_SYSENTER_MSR
5000	\| CPUM_CHANGED_LDTR
5001	\| CPUM_CHANGED_GDTR
5002	\| CPUM_CHANGED_IDTR
5003	\| CPUM_CHANGED_TR
5004	\| CPUM_CHANGED_HIDDEN_SEL_REGS);
5005
5006	/*
5007	* If we executed vmlaunch/vmresume and an external IRQ was pending, then we don't have to do a full sync the next time.
5008	*/
5009	if ( exitReason == VMX_EXIT_EXT_INT
5010	&& !VMX_EXIT_INTERRUPTION_INFO_VALID(intInfo))
5011	{
5012	STAM_COUNTER_INC(&pVCpu->hm.s.StatPendingHostIrq);
5013	/* On the next entry we'll only sync the host context. */
5014	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_HOST_CONTEXT;
5015	}
5016	else
5017	{
5018	/* On the next entry we'll sync everything. */
5019	/** @todo we can do better than this */
5020	/* Not in the VINF_PGM_CHANGE_MODE though! */
5021	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_ALL;
5022	}
5023
5024	/* Translate into a less severe return code */
5025	if (rc == VERR_EM_INTERPRETER)
5026	rc = VINF_EM_RAW_EMULATE_INSTR;
5027	else if (rc == VERR_VMX_INVALID_VMCS_PTR)
5028	{
5029	/* Try to extract more information about what might have gone wrong here. */
5030	VMXGetActivateVMCS(&pVCpu->hm.s.vmx.lasterror.u64VMCSPhys);
5031	pVCpu->hm.s.vmx.lasterror.u32VMCSRevision = (uint32_t )pVCpu->hm.s.vmx.pvVmcs;
5032	pVCpu->hm.s.vmx.lasterror.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
5033	pVCpu->hm.s.vmx.lasterror.idCurrentCpu = RTMpCpuId();
5034	}
5035
5036	/* Just set the correct state here instead of trying to catch every goto above. */
5037	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED, VMCPUSTATE_STARTED_EXEC);
5038
5039	#ifdef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
5040	/* Restore interrupts if we exited after disabling them. */
5041	if (uOldEFlags != ~(RTCCUINTREG)0)
5042	ASMSetFlags(uOldEFlags);
5043	#endif
5044
5045	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
5046	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
5047	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
5048	Log2(("X"));
5049	return VBOXSTRICTRC_TODO(rc);
5050	}
5051
5052
5053	/**
5054	* Enters the VT-x session.
5055	*
5056	* @returns VBox status code.
5057	* @param pVM Pointer to the VM.
5058	* @param pVCpu Pointer to the VMCPU.
5059	* @param pCpu Pointer to the CPU info struct.
5060	*/
5061	VMMR0DECL(int) VMXR0Enter(PVM pVM, PVMCPU pVCpu, PHMGLOBLCPUINFO pCpu)
5062	{
5063	Assert(pVM->hm.s.vmx.fSupported);
5064	NOREF(pCpu);
5065
5066	unsigned cr4 = ASMGetCR4();
5067	if (!(cr4 & X86_CR4_VMXE))
5068	{
5069	AssertMsgFailed(("X86_CR4_VMXE should be set!\n"));
5070	return VERR_VMX_X86_CR4_VMXE_CLEARED;
5071	}
5072
5073	/* Activate the VMCS. */
5074	int rc = VMXActivateVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
5075	if (RT_FAILURE(rc))
5076	return rc;
5077
5078	pVCpu->hm.s.fResumeVM = false;
5079	return VINF_SUCCESS;
5080	}
5081
5082
5083	/**
5084	* Leaves the VT-x session.
5085	*
5086	* @returns VBox status code.
5087	* @param pVM Pointer to the VM.
5088	* @param pVCpu Pointer to the VMCPU.
5089	* @param pCtx Pointer to the guests CPU context.
5090	*/
5091	VMMR0DECL(int) VMXR0Leave(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
5092	{
5093	Assert(pVM->hm.s.vmx.fSupported);
5094
5095	#ifdef DEBUG
5096	if (CPUMIsHyperDebugStateActive(pVCpu))
5097	{
5098	CPUMR0LoadHostDebugState(pVM, pVCpu);
5099	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT);
5100	}
5101	else
5102	#endif
5103
5104	/*
5105	* Save the guest debug state if necessary.
5106	*/
5107	if (CPUMIsGuestDebugStateActive(pVCpu))
5108	{
5109	CPUMR0SaveGuestDebugState(pVM, pVCpu, pCtx, true /* save DR6 */);
5110
5111	/* Enable DRx move intercepts again. */
5112	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT;
5113	int rc = VMXWriteVmcs(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls);
5114	AssertRC(rc);
5115
5116	/* Resync the debug registers the next time. */
5117	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_DEBUG;
5118	}
5119	else
5120	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT);
5121
5122	/*
5123	* Clear VMCS, marking it inactive, clearing implementation-specific data and writing
5124	* VMCS data back to memory.
5125	*/
5126	int rc = VMXClearVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
5127	AssertRC(rc);
5128
5129	return VINF_SUCCESS;
5130	}
5131
5132
5133	/**
5134	* Flush the TLB using EPT.
5135	*
5136	* @returns VBox status code.
5137	* @param pVM Pointer to the VM.
5138	* @param pVCpu Pointer to the VMCPU.
5139	* @param enmFlush Type of flush.
5140	*/
5141	static void hmR0VmxFlushEPT(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_EPT enmFlush)
5142	{
5143	uint64_t descriptor[2];
5144
5145	LogFlow(("hmR0VmxFlushEPT %d\n", enmFlush));
5146	Assert(pVM->hm.s.fNestedPaging);
5147	descriptor[0] = pVCpu->hm.s.vmx.GCPhysEPTP;
5148	descriptor[1] = 0; /* MBZ. Intel spec. 33.3 VMX Instructions */
5149	int rc = VMXR0InvEPT(enmFlush, &descriptor[0]);
5150	AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %x %RGv failed with %d\n", enmFlush, pVCpu->hm.s.vmx.GCPhysEPTP, rc));
5151	#ifdef VBOX_WITH_STATISTICS
5152	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
5153	#endif
5154	}
5155
5156
5157	/**
5158	* Flush the TLB using VPID.
5159	*
5160	* @returns VBox status code.
5161	* @param pVM Pointer to the VM.
5162	* @param pVCpu Pointer to the VMCPU (can be NULL depending on @a
5163	* enmFlush).
5164	* @param enmFlush Type of flush.
5165	* @param GCPtr Virtual address of the page to flush (can be 0 depending
5166	* on @a enmFlush).
5167	*/
5168	static void hmR0VmxFlushVPID(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_VPID enmFlush, RTGCPTR GCPtr)
5169	{
5170	uint64_t descriptor[2];
5171
5172	Assert(pVM->hm.s.vmx.fVpid);
5173	if (enmFlush == VMX_FLUSH_VPID_ALL_CONTEXTS)
5174	{
5175	descriptor[0] = 0;
5176	descriptor[1] = 0;
5177	}
5178	else
5179	{
5180	AssertPtr(pVCpu);
5181	AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
5182	AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
5183	descriptor[0] = pVCpu->hm.s.uCurrentAsid;
5184	descriptor[1] = GCPtr;
5185	}
5186	int rc = VMXR0InvVPID(enmFlush, &descriptor[0]); NOREF(rc);
5187	AssertMsg(rc == VINF_SUCCESS,
5188	("VMXR0InvVPID %x %x %RGv failed with %d\n", enmFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
5189	#ifdef VBOX_WITH_STATISTICS
5190	if (pVCpu)
5191	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
5192	#endif
5193	}
5194
5195
5196	/**
5197	* Invalidates a guest page by guest virtual address. Only relevant for
5198	* EPT/VPID, otherwise there is nothing really to invalidate.
5199	*
5200	* @returns VBox status code.
5201	* @param pVM Pointer to the VM.
5202	* @param pVCpu Pointer to the VMCPU.
5203	* @param GCVirt Guest virtual address of the page to invalidate.
5204	*/
5205	VMMR0DECL(int) VMXR0InvalidatePage(PVM pVM, PVMCPU pVCpu, RTGCPTR GCVirt)
5206	{
5207	bool fFlushPending = VMCPU_FF_ISSET(pVCpu, VMCPU_FF_TLB_FLUSH);
5208
5209	Log2(("VMXR0InvalidatePage %RGv\n", GCVirt));
5210
5211	if (!fFlushPending)
5212	{
5213	/*
5214	* We must invalidate the guest TLB entry in either case, we cannot ignore it even for the EPT case
5215	* See @bugref{6043} and @bugref{6177}
5216	*
5217	* Set the VMCPU_FF_TLB_FLUSH force flag and flush before VMENTRY in hmR0VmxSetupTLB*() as this
5218	* function maybe called in a loop with individual addresses.
5219	*/
5220	if (pVM->hm.s.vmx.fVpid)
5221	{
5222	/* If we can flush just this page do it, otherwise flush as little as possible. */
5223	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
5224	hmR0VmxFlushVPID(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, GCVirt);
5225	else
5226	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
5227	}
5228	else if (pVM->hm.s.fNestedPaging)
5229	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
5230	}
5231
5232	return VINF_SUCCESS;
5233	}
5234
5235
5236	/**
5237	* Invalidates a guest page by physical address. Only relevant for EPT/VPID,
5238	* otherwise there is nothing really to invalidate.
5239	*
5240	* NOTE: Assumes the current instruction references this physical page though a virtual address!!
5241	*
5242	* @returns VBox status code.
5243	* @param pVM Pointer to the VM.
5244	* @param pVCpu Pointer to the VMCPU.
5245	* @param GCPhys Guest physical address of the page to invalidate.
5246	*/
5247	VMMR0DECL(int) VMXR0InvalidatePhysPage(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys)
5248	{
5249	LogFlow(("VMXR0InvalidatePhysPage %RGp\n", GCPhys));
5250
5251	/*
5252	* We cannot flush a page by guest-physical address. invvpid takes only a linear address
5253	* while invept only flushes by EPT not individual addresses. We update the force flag here
5254	* and flush before VMENTRY in hmR0VmxSetupTLB*(). This function might be called in a loop.
5255	*/
5256	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
5257	return VINF_SUCCESS;
5258	}
5259
5260
5261	/**
5262	* Report world switch error and dump some useful debug info.
5263	*
5264	* @param pVM Pointer to the VM.
5265	* @param pVCpu Pointer to the VMCPU.
5266	* @param rc Return code.
5267	* @param pCtx Pointer to the current guest CPU context (not updated).
5268	*/
5269	static void hmR0VmxReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, VBOXSTRICTRC rc, PCPUMCTX pCtx)
5270	{
5271	NOREF(pVM);
5272
5273	switch (VBOXSTRICTRC_VAL(rc))
5274	{
5275	case VERR_VMX_INVALID_VMXON_PTR:
5276	AssertFailed();
5277	break;
5278
5279	case VERR_VMX_UNABLE_TO_START_VM:
5280	case VERR_VMX_UNABLE_TO_RESUME_VM:
5281	{
5282	int rc2;
5283	RTCCUINTREG exitReason, instrError;
5284
5285	rc2 = VMXReadVmcs(VMX_VMCS32_RO_EXIT_REASON, &exitReason);
5286	rc2 \|= VMXReadVmcs(VMX_VMCS32_RO_VM_INSTR_ERROR, &instrError);
5287	AssertRC(rc2);
5288	if (rc2 == VINF_SUCCESS)
5289	{
5290	Log(("Unable to start/resume VM for reason: %x. Instruction error %x\n", (uint32_t)exitReason,
5291	(uint32_t)instrError));
5292	Log(("Current stack %08x\n", &rc2));
5293
5294	pVCpu->hm.s.vmx.lasterror.u32InstrError = instrError;
5295	pVCpu->hm.s.vmx.lasterror.u32ExitReason = exitReason;
5296
5297	#ifdef VBOX_STRICT
5298	RTGDTR gdtr;
5299	PCX86DESCHC pDesc;
5300	RTCCUINTREG val;
5301
5302	ASMGetGDTR(&gdtr);
5303
5304	VMXReadVmcs(VMX_VMCS_GUEST_RIP, &val);
5305	Log(("Old eip %RGv new %RGv\n", (RTGCPTR)pCtx->rip, (RTGCPTR)val));
5306	VMXReadVmcs(VMX_VMCS32_CTRL_PIN_EXEC_CONTROLS, &val);
5307	Log(("VMX_VMCS_CTRL_PIN_EXEC_CONTROLS %08x\n", val));
5308	VMXReadVmcs(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, &val);
5309	Log(("VMX_VMCS_CTRL_PROC_EXEC_CONTROLS %08x\n", val));
5310	VMXReadVmcs(VMX_VMCS32_CTRL_ENTRY_CONTROLS, &val);
5311	Log(("VMX_VMCS_CTRL_ENTRY_CONTROLS %08x\n", val));
5312	VMXReadVmcs(VMX_VMCS32_CTRL_EXIT_CONTROLS, &val);
5313	Log(("VMX_VMCS_CTRL_EXIT_CONTROLS %08x\n", val));
5314
5315	VMXReadVmcs(VMX_VMCS_HOST_CR0, &val);
5316	Log(("VMX_VMCS_HOST_CR0 %08x\n", val));
5317	VMXReadVmcs(VMX_VMCS_HOST_CR3, &val);
5318	Log(("VMX_VMCS_HOST_CR3 %08x\n", val));
5319	VMXReadVmcs(VMX_VMCS_HOST_CR4, &val);
5320	Log(("VMX_VMCS_HOST_CR4 %08x\n", val));
5321
5322	VMXReadVmcs(VMX_VMCS16_HOST_FIELD_CS, &val);
5323	Log(("VMX_VMCS_HOST_FIELD_CS %08x\n", val));
5324	VMXReadVmcs(VMX_VMCS_GUEST_RFLAGS, &val);
5325	Log(("VMX_VMCS_GUEST_RFLAGS %08x\n", val));
5326
5327	if (val < gdtr.cbGdt)
5328	{
5329	pDesc = (PCX86DESCHC)(gdtr.pGdt + (val & X86_SEL_MASK));
5330	HMR0DumpDescriptor(pDesc, val, "CS: ");
5331	}
5332
5333	VMXReadVmcs(VMX_VMCS16_HOST_FIELD_DS, &val);
5334	Log(("VMX_VMCS_HOST_FIELD_DS %08x\n", val));
5335	if (val < gdtr.cbGdt)
5336	{
5337	pDesc = (PCX86DESCHC)(gdtr.pGdt + (val & X86_SEL_MASK));
5338	HMR0DumpDescriptor(pDesc, val, "DS: ");
5339	}
5340
5341	VMXReadVmcs(VMX_VMCS16_HOST_FIELD_ES, &val);
5342	Log(("VMX_VMCS_HOST_FIELD_ES %08x\n", val));
5343	if (val < gdtr.cbGdt)
5344	{
5345	pDesc = (PCX86DESCHC)(gdtr.pGdt + (val & X86_SEL_MASK));
5346	HMR0DumpDescriptor(pDesc, val, "ES: ");
5347	}
5348
5349	VMXReadVmcs(VMX_VMCS16_HOST_FIELD_FS, &val);
5350	Log(("VMX_VMCS16_HOST_FIELD_FS %08x\n", val));
5351	if (val < gdtr.cbGdt)
5352	{
5353	pDesc = (PCX86DESCHC)(gdtr.pGdt + (val & X86_SEL_MASK));
5354	HMR0DumpDescriptor(pDesc, val, "FS: ");
5355	}
5356
5357	VMXReadVmcs(VMX_VMCS16_HOST_FIELD_GS, &val);
5358	Log(("VMX_VMCS16_HOST_FIELD_GS %08x\n", val));
5359	if (val < gdtr.cbGdt)
5360	{
5361	pDesc = (PCX86DESCHC)(gdtr.pGdt + (val & X86_SEL_MASK));
5362	HMR0DumpDescriptor(pDesc, val, "GS: ");
5363	}
5364
5365	VMXReadVmcs(VMX_VMCS16_HOST_FIELD_SS, &val);
5366	Log(("VMX_VMCS16_HOST_FIELD_SS %08x\n", val));
5367	if (val < gdtr.cbGdt)
5368	{
5369	pDesc = (PCX86DESCHC)(gdtr.pGdt + (val & X86_SEL_MASK));
5370	HMR0DumpDescriptor(pDesc, val, "SS: ");
5371	}
5372
5373	VMXReadVmcs(VMX_VMCS16_HOST_FIELD_TR, &val);
5374	Log(("VMX_VMCS16_HOST_FIELD_TR %08x\n", val));
5375	if (val < gdtr.cbGdt)
5376	{
5377	pDesc = (PCX86DESCHC)(gdtr.pGdt + (val & X86_SEL_MASK));
5378	HMR0DumpDescriptor(pDesc, val, "TR: ");
5379	}
5380
5381	VMXReadVmcs(VMX_VMCS_HOST_TR_BASE, &val);
5382	Log(("VMX_VMCS_HOST_TR_BASE %RHv\n", val));
5383	VMXReadVmcs(VMX_VMCS_HOST_GDTR_BASE, &val);
5384	Log(("VMX_VMCS_HOST_GDTR_BASE %RHv\n", val));
5385	VMXReadVmcs(VMX_VMCS_HOST_IDTR_BASE, &val);
5386	Log(("VMX_VMCS_HOST_IDTR_BASE %RHv\n", val));
5387	VMXReadVmcs(VMX_VMCS32_HOST_SYSENTER_CS, &val);
5388	Log(("VMX_VMCS_HOST_SYSENTER_CS %08x\n", val));
5389	VMXReadVmcs(VMX_VMCS_HOST_SYSENTER_EIP, &val);
5390	Log(("VMX_VMCS_HOST_SYSENTER_EIP %RHv\n", val));
5391	VMXReadVmcs(VMX_VMCS_HOST_SYSENTER_ESP, &val);
5392	Log(("VMX_VMCS_HOST_SYSENTER_ESP %RHv\n", val));
5393	VMXReadVmcs(VMX_VMCS_HOST_RSP, &val);
5394	Log(("VMX_VMCS_HOST_RSP %RHv\n", val));
5395	VMXReadVmcs(VMX_VMCS_HOST_RIP, &val);
5396	Log(("VMX_VMCS_HOST_RIP %RHv\n", val));
5397	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
5398	if (VMX_IS_64BIT_HOST_MODE())
5399	{
5400	Log(("MSR_K6_EFER = %RX64\n", ASMRdMsr(MSR_K6_EFER)));
5401	Log(("MSR_K6_STAR = %RX64\n", ASMRdMsr(MSR_K6_STAR)));
5402	Log(("MSR_K8_LSTAR = %RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
5403	Log(("MSR_K8_CSTAR = %RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
5404	Log(("MSR_K8_SF_MASK = %RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
5405	Log(("MSR_K8_KERNEL_GS_BASE = %RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
5406	}
5407	# endif
5408	#endif /* VBOX_STRICT */
5409	}
5410	break;
5411	}
5412
5413	default:
5414	/* impossible */
5415	AssertMsgFailed(("%Rrc (%#x)\n", VBOXSTRICTRC_VAL(rc), VBOXSTRICTRC_VAL(rc)));
5416	break;
5417	}
5418	}
5419
5420
5421	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
5422	/**
5423	* Prepares for and executes VMLAUNCH (64 bits guest mode).
5424	*
5425	* @returns VBox status code.
5426	* @param fResume Whether to vmlauch/vmresume.
5427	* @param pCtx Pointer to the guest CPU context.
5428	* @param pCache Pointer to the VMCS cache.
5429	* @param pVM Pointer to the VM.
5430	* @param pVCpu Pointer to the VMCPU.
5431	*/
5432	DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
5433	{
5434	uint32_t aParam[6];
5435	PHMGLOBLCPUINFO pCpu;
5436	RTHCPHYS HCPhysCpuPage;
5437	int rc;
5438
5439	pCpu = HMR0GetCurrentCpu();
5440	HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
5441
5442	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5443	pCache->uPos = 1;
5444	pCache->interPD = PGMGetInterPaeCR3(pVM);
5445	pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
5446	#endif
5447
5448	#ifdef DEBUG
5449	pCache->TestIn.HCPhysCpuPage= 0;
5450	pCache->TestIn.HCPhysVmcs = 0;
5451	pCache->TestIn.pCache = 0;
5452	pCache->TestOut.HCPhysVmcs = 0;
5453	pCache->TestOut.pCache = 0;
5454	pCache->TestOut.pCtx = 0;
5455	pCache->TestOut.eflags = 0;
5456	#endif
5457
5458	aParam[0] = (uint32_t)(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
5459	aParam[1] = (uint32_t)(HCPhysCpuPage >> 32); /* Param 1: VMXON physical address - Hi. */
5460	aParam[2] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
5461	aParam[3] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs >> 32); /* Param 2: VMCS physical address - Hi. */
5462	aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache);
5463	aParam[5] = 0;
5464
5465	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5466	pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
5467	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
5468	#endif
5469	rc = VMXR0Execute64BitsHandler(pVM, pVCpu, pCtx, pVM->hm.s.pfnVMXGCStartVM64, 6, &aParam[0]);
5470
5471	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5472	Assert((uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
5473	Assert(pCtx->dr[4] == 10);
5474	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
5475	#endif
5476
5477	#ifdef DEBUG
5478	AssertMsg(pCache->TestIn.HCPhysCpuPage== HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
5479	AssertMsg(pCache->TestIn.HCPhysVmcs == pVCpu->hm.s.vmx.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5480	pVCpu->hm.s.vmx.HCPhysVmcs));
5481	AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5482	pCache->TestOut.HCPhysVmcs));
5483	AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
5484	pCache->TestOut.pCache));
5485	AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache),
5486	("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache)));
5487	AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
5488	pCache->TestOut.pCtx));
5489	Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
5490	#endif
5491	return rc;
5492	}
5493
5494
5495	# ifdef VBOX_STRICT
5496	static bool hmR0VmxIsValidReadField(uint32_t idxField)
5497	{
5498	switch (idxField)
5499	{
5500	case VMX_VMCS_GUEST_RIP:
5501	case VMX_VMCS_GUEST_RSP:
5502	case VMX_VMCS_GUEST_RFLAGS:
5503	case VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE:
5504	case VMX_VMCS_CTRL_CR0_READ_SHADOW:
5505	case VMX_VMCS_GUEST_CR0:
5506	case VMX_VMCS_CTRL_CR4_READ_SHADOW:
5507	case VMX_VMCS_GUEST_CR4:
5508	case VMX_VMCS_GUEST_DR7:
5509	case VMX_VMCS32_GUEST_SYSENTER_CS:
5510	case VMX_VMCS_GUEST_SYSENTER_EIP:
5511	case VMX_VMCS_GUEST_SYSENTER_ESP:
5512	case VMX_VMCS32_GUEST_GDTR_LIMIT:
5513	case VMX_VMCS_GUEST_GDTR_BASE:
5514	case VMX_VMCS32_GUEST_IDTR_LIMIT:
5515	case VMX_VMCS_GUEST_IDTR_BASE:
5516	case VMX_VMCS16_GUEST_FIELD_CS:
5517	case VMX_VMCS32_GUEST_CS_LIMIT:
5518	case VMX_VMCS_GUEST_CS_BASE:
5519	case VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS:
5520	case VMX_VMCS16_GUEST_FIELD_DS:
5521	case VMX_VMCS32_GUEST_DS_LIMIT:
5522	case VMX_VMCS_GUEST_DS_BASE:
5523	case VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS:
5524	case VMX_VMCS16_GUEST_FIELD_ES:
5525	case VMX_VMCS32_GUEST_ES_LIMIT:
5526	case VMX_VMCS_GUEST_ES_BASE:
5527	case VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS:
5528	case VMX_VMCS16_GUEST_FIELD_FS:
5529	case VMX_VMCS32_GUEST_FS_LIMIT:
5530	case VMX_VMCS_GUEST_FS_BASE:
5531	case VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS:
5532	case VMX_VMCS16_GUEST_FIELD_GS:
5533	case VMX_VMCS32_GUEST_GS_LIMIT:
5534	case VMX_VMCS_GUEST_GS_BASE:
5535	case VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS:
5536	case VMX_VMCS16_GUEST_FIELD_SS:
5537	case VMX_VMCS32_GUEST_SS_LIMIT:
5538	case VMX_VMCS_GUEST_SS_BASE:
5539	case VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS:
5540	case VMX_VMCS16_GUEST_FIELD_LDTR:
5541	case VMX_VMCS32_GUEST_LDTR_LIMIT:
5542	case VMX_VMCS_GUEST_LDTR_BASE:
5543	case VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS:
5544	case VMX_VMCS16_GUEST_FIELD_TR:
5545	case VMX_VMCS32_GUEST_TR_LIMIT:
5546	case VMX_VMCS_GUEST_TR_BASE:
5547	case VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS:
5548	case VMX_VMCS32_RO_EXIT_REASON:
5549	case VMX_VMCS32_RO_VM_INSTR_ERROR:
5550	case VMX_VMCS32_RO_EXIT_INSTR_LENGTH:
5551	case VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE:
5552	case VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO:
5553	case VMX_VMCS32_RO_EXIT_INSTR_INFO:
5554	case VMX_VMCS_RO_EXIT_QUALIFICATION:
5555	case VMX_VMCS32_RO_IDT_INFO:
5556	case VMX_VMCS32_RO_IDT_ERROR_CODE:
5557	case VMX_VMCS_GUEST_CR3:
5558	case VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL:
5559	return true;
5560	}
5561	return false;
5562	}
5563
5564
5565	static bool hmR0VmxIsValidWriteField(uint32_t idxField)
5566	{
5567	switch (idxField)
5568	{
5569	case VMX_VMCS_GUEST_LDTR_BASE:
5570	case VMX_VMCS_GUEST_TR_BASE:
5571	case VMX_VMCS_GUEST_GDTR_BASE:
5572	case VMX_VMCS_GUEST_IDTR_BASE:
5573	case VMX_VMCS_GUEST_SYSENTER_EIP:
5574	case VMX_VMCS_GUEST_SYSENTER_ESP:
5575	case VMX_VMCS_GUEST_CR0:
5576	case VMX_VMCS_GUEST_CR4:
5577	case VMX_VMCS_GUEST_CR3:
5578	case VMX_VMCS_GUEST_DR7:
5579	case VMX_VMCS_GUEST_RIP:
5580	case VMX_VMCS_GUEST_RSP:
5581	case VMX_VMCS_GUEST_CS_BASE:
5582	case VMX_VMCS_GUEST_DS_BASE:
5583	case VMX_VMCS_GUEST_ES_BASE:
5584	case VMX_VMCS_GUEST_FS_BASE:
5585	case VMX_VMCS_GUEST_GS_BASE:
5586	case VMX_VMCS_GUEST_SS_BASE:
5587	return true;
5588	}
5589	return false;
5590	}
5591	# endif /* VBOX_STRICT */
5592
5593
5594	/**
5595	* Executes the specified handler in 64-bit mode.
5596	*
5597	* @returns VBox status code.
5598	* @param pVM Pointer to the VM.
5599	* @param pVCpu Pointer to the VMCPU.
5600	* @param pCtx Pointer to the guest CPU context.
5601	* @param pfnHandler Pointer to the RC handler function.
5602	* @param cbParam Number of parameters.
5603	* @param paParam Array of 32-bit parameters.
5604	*/
5605	VMMR0DECL(int) VMXR0Execute64BitsHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, RTRCPTR pfnHandler, uint32_t cbParam,
5606	uint32_t *paParam)
5607	{
5608	int rc, rc2;
5609	PHMGLOBLCPUINFO pCpu;
5610	RTHCPHYS HCPhysCpuPage;
5611	RTHCUINTREG uOldEFlags;
5612
5613	AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
5614	Assert(pfnHandler);
5615	Assert(pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Write.aField));
5616	Assert(pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Read.aField));
5617
5618	#ifdef VBOX_STRICT
5619	for (unsigned i=0;i<pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries;i++)
5620	Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VMCSCache.Write.aField[i]));
5621
5622	for (unsigned i=0;i<pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries;i++)
5623	Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VMCSCache.Read.aField[i]));
5624	#endif
5625
5626	/* Disable interrupts. */
5627	uOldEFlags = ASMIntDisableFlags();
5628
5629	#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
5630	RTCPUID idHostCpu = RTMpCpuId();
5631	CPUMR0SetLApic(pVM, idHostCpu);
5632	#endif
5633
5634	pCpu = HMR0GetCurrentCpu();
5635	HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
5636
5637	/* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
5638	VMXClearVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
5639
5640	/* Leave VMX Root Mode. */
5641	VMXDisable();
5642
5643	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
5644
5645	CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
5646	CPUMSetHyperEIP(pVCpu, pfnHandler);
5647	for (int i=(int)cbParam-1;i>=0;i--)
5648	CPUMPushHyper(pVCpu, paParam[i]);
5649
5650	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
5651
5652	/* Call switcher. */
5653	rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_OFFSETOF(VM, aCpus[pVCpu->idCpu].cpum) - RT_OFFSETOF(VM, cpum));
5654	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
5655
5656	/* Make sure the VMX instructions don't cause #UD faults. */
5657	ASMSetCR4(ASMGetCR4() \| X86_CR4_VMXE);
5658
5659	/* Enter VMX Root Mode */
5660	rc2 = VMXEnable(HCPhysCpuPage);
5661	if (RT_FAILURE(rc2))
5662	{
5663	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
5664	ASMSetFlags(uOldEFlags);
5665	return VERR_VMX_VMXON_FAILED;
5666	}
5667
5668	rc2 = VMXActivateVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
5669	AssertRC(rc2);
5670	Assert(!(ASMGetFlags() & X86_EFL_IF));
5671	ASMSetFlags(uOldEFlags);
5672	return rc;
5673	}
5674	#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL) */
5675
5676
5677	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
5678	/**
5679	* Executes VMWRITE.
5680	*
5681	* @returns VBox status code
5682	* @param pVCpu Pointer to the VMCPU.
5683	* @param idxField VMCS field index.
5684	* @param u64Val 16, 32 or 64 bits value.
5685	*/
5686	VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5687	{
5688	int rc;
5689	switch (idxField)
5690	{
5691	case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
5692	case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
5693	case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
5694	case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
5695	case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
5696	case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
5697	case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
5698	case VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL:
5699	case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
5700	case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
5701	case VMX_VMCS64_GUEST_PDPTE0_FULL:
5702	case VMX_VMCS64_GUEST_PDPTE1_FULL:
5703	case VMX_VMCS64_GUEST_PDPTE2_FULL:
5704	case VMX_VMCS64_GUEST_PDPTE3_FULL:
5705	case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
5706	case VMX_VMCS64_GUEST_EFER_FULL:
5707	case VMX_VMCS64_CTRL_EPTP_FULL:
5708	/* These fields consist of two parts, which are both writable in 32 bits mode. */
5709	rc = VMXWriteVmcs32(idxField, u64Val);
5710	rc \|= VMXWriteVmcs32(idxField + 1, (uint32_t)(u64Val >> 32ULL));
5711	AssertRC(rc);
5712	return rc;
5713
5714	case VMX_VMCS_GUEST_LDTR_BASE:
5715	case VMX_VMCS_GUEST_TR_BASE:
5716	case VMX_VMCS_GUEST_GDTR_BASE:
5717	case VMX_VMCS_GUEST_IDTR_BASE:
5718	case VMX_VMCS_GUEST_SYSENTER_EIP:
5719	case VMX_VMCS_GUEST_SYSENTER_ESP:
5720	case VMX_VMCS_GUEST_CR0:
5721	case VMX_VMCS_GUEST_CR4:
5722	case VMX_VMCS_GUEST_CR3:
5723	case VMX_VMCS_GUEST_DR7:
5724	case VMX_VMCS_GUEST_RIP:
5725	case VMX_VMCS_GUEST_RSP:
5726	case VMX_VMCS_GUEST_CS_BASE:
5727	case VMX_VMCS_GUEST_DS_BASE:
5728	case VMX_VMCS_GUEST_ES_BASE:
5729	case VMX_VMCS_GUEST_FS_BASE:
5730	case VMX_VMCS_GUEST_GS_BASE:
5731	case VMX_VMCS_GUEST_SS_BASE:
5732	/* Queue a 64 bits value as we can't set it in 32 bits host mode. */
5733	if (u64Val >> 32ULL)
5734	rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
5735	else
5736	rc = VMXWriteVmcs32(idxField, (uint32_t)u64Val);
5737
5738	return rc;
5739
5740	default:
5741	AssertMsgFailed(("Unexpected field %x\n", idxField));
5742	return VERR_INVALID_PARAMETER;
5743	}
5744	}
5745
5746
5747	/**
5748	* Cache VMCS writes for running 64 bits guests on 32 bits hosts.
5749	*
5750	* @param pVCpu Pointer to the VMCPU.
5751	* @param idxField VMCS field index.
5752	* @param u64Val 16, 32 or 64 bits value.
5753	*/
5754	VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5755	{
5756	PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5757
5758	AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1,
5759	("entries=%x\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
5760
5761	/* Make sure there are no duplicates. */
5762	for (unsigned i = 0; i < pCache->Write.cValidEntries; i++)
5763	{
5764	if (pCache->Write.aField[i] == idxField)
5765	{
5766	pCache->Write.aFieldVal[i] = u64Val;
5767	return VINF_SUCCESS;
5768	}
5769	}
5770
5771	pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
5772	pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
5773	pCache->Write.cValidEntries++;
5774	return VINF_SUCCESS;
5775	}
5776
5777	#endif /* HC_ARCH_BITS == 32 && !VBOX_WITH_HYBRID_32BIT_KERNEL */
5778

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMR0/HWVMXR0.cpp@ 45216

Download in other formats: