HWVMXR0.cpp@ 42463

Last change on this file since 42463 was 42407, checked in by vboxsync, 13 years ago
VMM: Futher work on dealing with hidden segment register, esp. when going stale.
Property svn:eol-style set to `native` Property svn:keywords set to `Id Revision`
File size: 220.7 KB

Line
1	/* $Id: HWVMXR0.cpp 42407 2012-07-26 11:41:35Z vboxsync $ */
2	/** @file
3	* HM VMX (VT-x) - Host Context Ring-0.
4	*/
5
6	/*
7	* Copyright (C) 2006-2012 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18
19	/*******************************************************************************
20	* Header Files *
21	*******************************************************************************/
22	#define LOG_GROUP LOG_GROUP_HWACCM
23	#include <iprt/asm-amd64-x86.h>
24	#include <VBox/vmm/hwaccm.h>
25	#include <VBox/vmm/pgm.h>
26	#include <VBox/vmm/dbgf.h>
27	#include <VBox/vmm/dbgftrace.h>
28	#include <VBox/vmm/selm.h>
29	#include <VBox/vmm/iom.h>
30	#ifdef VBOX_WITH_REM
31	# include <VBox/vmm/rem.h>
32	#endif
33	#include <VBox/vmm/tm.h>
34	#include "HWACCMInternal.h"
35	#include <VBox/vmm/vm.h>
36	#include <VBox/vmm/pdmapi.h>
37	#include <VBox/err.h>
38	#include <VBox/log.h>
39	#include <iprt/assert.h>
40	#include <iprt/param.h>
41	#include <iprt/string.h>
42	#include <iprt/time.h>
43	#ifdef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
44	# include <iprt/thread.h>
45	#endif
46	#include <iprt/x86.h>
47	#include "HWVMXR0.h"
48
49	#include "dtrace/VBoxVMM.h"
50
51
52	/*******************************************************************************
53	* Defined Constants And Macros *
54	*******************************************************************************/
55	#if defined(RT_ARCH_AMD64)
56	# define VMX_IS_64BIT_HOST_MODE() (true)
57	#elif defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
58	# define VMX_IS_64BIT_HOST_MODE() (g_fVMXIs64bitHost != 0)
59	#else
60	# define VMX_IS_64BIT_HOST_MODE() (false)
61	#endif
62
63
64	/*******************************************************************************
65	* Global Variables *
66	*******************************************************************************/
67	/* IO operation lookup arrays. */
68	static uint32_t const g_aIOSize[4] = {1, 2, 0, 4};
69	static uint32_t const g_aIOOpAnd[4] = {0xff, 0xffff, 0, 0xffffffff};
70
71	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
72	/** See HWACCMR0A.asm. */
73	extern "C" uint32_t g_fVMXIs64bitHost;
74	#endif
75
76
77	/*******************************************************************************
78	* Local Functions *
79	*******************************************************************************/
80	static DECLCALLBACK(void) hmR0VmxSetupTLBEPT(PVM pVM, PVMCPU pVCpu);
81	static DECLCALLBACK(void) hmR0VmxSetupTLBVPID(PVM pVM, PVMCPU pVCpu);
82	static DECLCALLBACK(void) hmR0VmxSetupTLBBoth(PVM pVM, PVMCPU pVCpu);
83	static DECLCALLBACK(void) hmR0VmxSetupTLBDummy(PVM pVM, PVMCPU pVCpu);
84	static void hmR0VmxFlushEPT(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_EPT enmFlush);
85	static void hmR0VmxFlushVPID(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_VPID enmFlush, RTGCPTR GCPtr);
86	static void hmR0VmxUpdateExceptionBitmap(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx);
87	static void hmR0VmxSetMSRPermission(PVMCPU pVCpu, unsigned ulMSR, bool fRead, bool fWrite);
88	static void hmR0VmxReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, VBOXSTRICTRC rc, PCPUMCTX pCtx);
89
90
91	/**
92	* Updates error from VMCS to HWACCMCPU's lasterror record.
93	*
94	* @param pVM Pointer to the VM.
95	* @param pVCpu Pointer to the VMCPU.
96	* @param rc The error code.
97	*/
98	static void hmR0VmxCheckError(PVM pVM, PVMCPU pVCpu, int rc)
99	{
100	if (rc == VERR_VMX_GENERIC)
101	{
102	RTCCUINTREG instrError;
103
104	VMXReadVMCS(VMX_VMCS32_RO_VM_INSTR_ERROR, &instrError);
105	pVCpu->hwaccm.s.vmx.lasterror.ulInstrError = instrError;
106	}
107	pVM->hwaccm.s.lLastError = rc;
108	}
109
110
111	/**
112	* Sets up and activates VT-x on the current CPU.
113	*
114	* @returns VBox status code.
115	* @param pCpu Pointer to the CPU info struct.
116	* @param pVM Pointer to the VM. (can be NULL after a resume!!)
117	* @param pvCpuPage Pointer to the global CPU page.
118	* @param HCPhysCpuPage Physical address of the global CPU page.
119	*/
120	VMMR0DECL(int) VMXR0EnableCpu(PHMGLOBLCPUINFO pCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
121	{
122	AssertReturn(HCPhysCpuPage != 0 && HCPhysCpuPage != NIL_RTHCPHYS, VERR_INVALID_PARAMETER);
123	AssertReturn(pvCpuPage, VERR_INVALID_PARAMETER);
124
125	if (pVM)
126	{
127	/* Set revision dword at the beginning of the VMXON structure. */
128	(uint32_t )pvCpuPage = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hwaccm.s.vmx.msr.vmx_basic_info);
129	}
130
131	/** @todo we should unmap the two pages from the virtual address space in order to prevent accidental corruption.
132	* (which can have very bad consequences!!!)
133	*/
134
135	if (ASMGetCR4() & X86_CR4_VMXE)
136	return VERR_VMX_IN_VMX_ROOT_MODE;
137
138	ASMSetCR4(ASMGetCR4() \| X86_CR4_VMXE); /* Make sure the VMX instructions don't cause #UD faults. */
139
140	/*
141	* Enter VM root mode.
142	*/
143	int rc = VMXEnable(HCPhysCpuPage);
144	if (RT_FAILURE(rc))
145	{
146	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
147	return VERR_VMX_VMXON_FAILED;
148	}
149
150	/*
151	* Flush all VPIDs (in case we or any other hypervisor have been using VPIDs) so that
152	* we can avoid an explicit flush while using new VPIDs. We would still need to flush
153	* each time while reusing a VPID after hitting the MaxASID limit once.
154	*/
155	if ( pVM
156	&& pVM->hwaccm.s.vmx.fVPID
157	&& (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVVPID_CAPS_ALL_CONTEXTS))
158	{
159	hmR0VmxFlushVPID(pVM, NULL /* pvCpu /, VMX_FLUSH_VPID_ALL_CONTEXTS, 0 / GCPtr */);
160	pCpu->fFlushASIDBeforeUse = false;
161	}
162	else
163	pCpu->fFlushASIDBeforeUse = true;
164
165	/*
166	* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}.
167	*/
168	++pCpu->cTLBFlushes;
169
170	return VINF_SUCCESS;
171	}
172
173
174	/**
175	* Deactivates VT-x on the current CPU.
176	*
177	* @returns VBox status code.
178	* @param pCpu Pointer to the CPU info struct.
179	* @param pvCpuPage Pointer to the global CPU page.
180	* @param HCPhysCpuPage Physical address of the global CPU page.
181	*/
182	VMMR0DECL(int) VMXR0DisableCpu(PHMGLOBLCPUINFO pCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
183	{
184	AssertReturn(HCPhysCpuPage != 0 && HCPhysCpuPage != NIL_RTHCPHYS, VERR_INVALID_PARAMETER);
185	AssertReturn(pvCpuPage, VERR_INVALID_PARAMETER);
186	NOREF(pCpu);
187
188	/* If we're somehow not in VMX root mode, then we shouldn't dare leaving it. */
189	if (!(ASMGetCR4() & X86_CR4_VMXE))
190	return VERR_VMX_NOT_IN_VMX_ROOT_MODE;
191
192	/* Leave VMX Root Mode. */
193	VMXDisable();
194
195	/* And clear the X86_CR4_VMXE bit. */
196	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
197	return VINF_SUCCESS;
198	}
199
200
201	/**
202	* Does Ring-0 per VM VT-x initialization.
203	*
204	* @returns VBox status code.
205	* @param pVM Pointer to the VM.
206	*/
207	VMMR0DECL(int) VMXR0InitVM(PVM pVM)
208	{
209	int rc;
210
211	#ifdef LOG_ENABLED
212	SUPR0Printf("VMXR0InitVM %p\n", pVM);
213	#endif
214
215	pVM->hwaccm.s.vmx.pMemObjAPIC = NIL_RTR0MEMOBJ;
216
217	if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
218	{
219	/* Allocate one page for the APIC physical page (serves for filtering accesses). */
220	rc = RTR0MemObjAllocCont(&pVM->hwaccm.s.vmx.pMemObjAPIC, PAGE_SIZE, true /* executable R0 mapping */);
221	AssertRC(rc);
222	if (RT_FAILURE(rc))
223	return rc;
224
225	pVM->hwaccm.s.vmx.pAPIC = (uint8_t *)RTR0MemObjAddress(pVM->hwaccm.s.vmx.pMemObjAPIC);
226	pVM->hwaccm.s.vmx.pAPICPhys = RTR0MemObjGetPagePhysAddr(pVM->hwaccm.s.vmx.pMemObjAPIC, 0);
227	ASMMemZero32(pVM->hwaccm.s.vmx.pAPIC, PAGE_SIZE);
228	}
229	else
230	{
231	pVM->hwaccm.s.vmx.pMemObjAPIC = 0;
232	pVM->hwaccm.s.vmx.pAPIC = 0;
233	pVM->hwaccm.s.vmx.pAPICPhys = 0;
234	}
235
236	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
237	{
238	rc = RTR0MemObjAllocCont(&pVM->hwaccm.s.vmx.pMemObjScratch, PAGE_SIZE, true /* executable R0 mapping */);
239	AssertRC(rc);
240	if (RT_FAILURE(rc))
241	return rc;
242
243	pVM->hwaccm.s.vmx.pScratch = (uint8_t *)RTR0MemObjAddress(pVM->hwaccm.s.vmx.pMemObjScratch);
244	pVM->hwaccm.s.vmx.pScratchPhys = RTR0MemObjGetPagePhysAddr(pVM->hwaccm.s.vmx.pMemObjScratch, 0);
245
246	ASMMemZero32(pVM->hwaccm.s.vmx.pScratch, PAGE_SIZE);
247	strcpy((char *)pVM->hwaccm.s.vmx.pScratch, "SCRATCH Magic");
248	(uint64_t )(pVM->hwaccm.s.vmx.pScratch + 16) = UINT64_C(0xDEADBEEFDEADBEEF);
249	}
250	#endif
251
252	/* Allocate VMCSs for all guest CPUs. */
253	for (VMCPUID i = 0; i < pVM->cCpus; i++)
254	{
255	PVMCPU pVCpu = &pVM->aCpus[i];
256
257	pVCpu->hwaccm.s.vmx.hMemObjVMCS = NIL_RTR0MEMOBJ;
258
259	/* Allocate one page for the VM control structure (VMCS). */
260	rc = RTR0MemObjAllocCont(&pVCpu->hwaccm.s.vmx.hMemObjVMCS, PAGE_SIZE, true /* executable R0 mapping */);
261	AssertRC(rc);
262	if (RT_FAILURE(rc))
263	return rc;
264
265	pVCpu->hwaccm.s.vmx.pvVMCS = RTR0MemObjAddress(pVCpu->hwaccm.s.vmx.hMemObjVMCS);
266	pVCpu->hwaccm.s.vmx.HCPhysVMCS = RTR0MemObjGetPagePhysAddr(pVCpu->hwaccm.s.vmx.hMemObjVMCS, 0);
267	ASMMemZeroPage(pVCpu->hwaccm.s.vmx.pvVMCS);
268
269	pVCpu->hwaccm.s.vmx.cr0_mask = 0;
270	pVCpu->hwaccm.s.vmx.cr4_mask = 0;
271
272	/* Allocate one page for the virtual APIC page for TPR caching. */
273	rc = RTR0MemObjAllocCont(&pVCpu->hwaccm.s.vmx.hMemObjVAPIC, PAGE_SIZE, true /* executable R0 mapping */);
274	AssertRC(rc);
275	if (RT_FAILURE(rc))
276	return rc;
277
278	pVCpu->hwaccm.s.vmx.pbVAPIC = (uint8_t *)RTR0MemObjAddress(pVCpu->hwaccm.s.vmx.hMemObjVAPIC);
279	pVCpu->hwaccm.s.vmx.HCPhysVAPIC = RTR0MemObjGetPagePhysAddr(pVCpu->hwaccm.s.vmx.hMemObjVAPIC, 0);
280	ASMMemZeroPage(pVCpu->hwaccm.s.vmx.pbVAPIC);
281
282	/* Allocate the MSR bitmap if this feature is supported. */
283	if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
284	{
285	rc = RTR0MemObjAllocCont(&pVCpu->hwaccm.s.vmx.pMemObjMSRBitmap, PAGE_SIZE, true /* executable R0 mapping */);
286	AssertRC(rc);
287	if (RT_FAILURE(rc))
288	return rc;
289
290	pVCpu->hwaccm.s.vmx.pMSRBitmap = (uint8_t *)RTR0MemObjAddress(pVCpu->hwaccm.s.vmx.pMemObjMSRBitmap);
291	pVCpu->hwaccm.s.vmx.pMSRBitmapPhys = RTR0MemObjGetPagePhysAddr(pVCpu->hwaccm.s.vmx.pMemObjMSRBitmap, 0);
292	memset(pVCpu->hwaccm.s.vmx.pMSRBitmap, 0xff, PAGE_SIZE);
293	}
294
295	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
296	/* Allocate one page for the guest MSR load area (for preloading guest MSRs during the world switch). */
297	rc = RTR0MemObjAllocCont(&pVCpu->hwaccm.s.vmx.pMemObjGuestMSR, PAGE_SIZE, true /* executable R0 mapping */);
298	AssertRC(rc);
299	if (RT_FAILURE(rc))
300	return rc;
301
302	pVCpu->hwaccm.s.vmx.pGuestMSR = (uint8_t *)RTR0MemObjAddress(pVCpu->hwaccm.s.vmx.pMemObjGuestMSR);
303	pVCpu->hwaccm.s.vmx.pGuestMSRPhys = RTR0MemObjGetPagePhysAddr(pVCpu->hwaccm.s.vmx.pMemObjGuestMSR, 0);
304	memset(pVCpu->hwaccm.s.vmx.pGuestMSR, 0, PAGE_SIZE);
305
306	/* Allocate one page for the host MSR load area (for restoring host MSRs after the world switch back). */
307	rc = RTR0MemObjAllocCont(&pVCpu->hwaccm.s.vmx.pMemObjHostMSR, PAGE_SIZE, true /* executable R0 mapping */);
308	AssertRC(rc);
309	if (RT_FAILURE(rc))
310	return rc;
311
312	pVCpu->hwaccm.s.vmx.pHostMSR = (uint8_t *)RTR0MemObjAddress(pVCpu->hwaccm.s.vmx.pMemObjHostMSR);
313	pVCpu->hwaccm.s.vmx.pHostMSRPhys = RTR0MemObjGetPagePhysAddr(pVCpu->hwaccm.s.vmx.pMemObjHostMSR, 0);
314	memset(pVCpu->hwaccm.s.vmx.pHostMSR, 0, PAGE_SIZE);
315	#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
316
317	/* Current guest paging mode. */
318	pVCpu->hwaccm.s.vmx.enmLastSeenGuestMode = PGMMODE_REAL;
319
320	#ifdef LOG_ENABLED
321	SUPR0Printf("VMXR0InitVM %x VMCS=%x (%x)\n", pVM, pVCpu->hwaccm.s.vmx.pvVMCS, (uint32_t)pVCpu->hwaccm.s.vmx.HCPhysVMCS);
322	#endif
323	}
324
325	return VINF_SUCCESS;
326	}
327
328
329	/**
330	* Does Ring-0 per VM VT-x termination.
331	*
332	* @returns VBox status code.
333	* @param pVM Pointer to the VM.
334	*/
335	VMMR0DECL(int) VMXR0TermVM(PVM pVM)
336	{
337	for (VMCPUID i = 0; i < pVM->cCpus; i++)
338	{
339	PVMCPU pVCpu = &pVM->aCpus[i];
340
341	if (pVCpu->hwaccm.s.vmx.hMemObjVMCS != NIL_RTR0MEMOBJ)
342	{
343	RTR0MemObjFree(pVCpu->hwaccm.s.vmx.hMemObjVMCS, false);
344	pVCpu->hwaccm.s.vmx.hMemObjVMCS = NIL_RTR0MEMOBJ;
345	pVCpu->hwaccm.s.vmx.pvVMCS = 0;
346	pVCpu->hwaccm.s.vmx.HCPhysVMCS = 0;
347	}
348	if (pVCpu->hwaccm.s.vmx.hMemObjVAPIC != NIL_RTR0MEMOBJ)
349	{
350	RTR0MemObjFree(pVCpu->hwaccm.s.vmx.hMemObjVAPIC, false);
351	pVCpu->hwaccm.s.vmx.hMemObjVAPIC = NIL_RTR0MEMOBJ;
352	pVCpu->hwaccm.s.vmx.pbVAPIC = 0;
353	pVCpu->hwaccm.s.vmx.HCPhysVAPIC = 0;
354	}
355	if (pVCpu->hwaccm.s.vmx.pMemObjMSRBitmap != NIL_RTR0MEMOBJ)
356	{
357	RTR0MemObjFree(pVCpu->hwaccm.s.vmx.pMemObjMSRBitmap, false);
358	pVCpu->hwaccm.s.vmx.pMemObjMSRBitmap = NIL_RTR0MEMOBJ;
359	pVCpu->hwaccm.s.vmx.pMSRBitmap = 0;
360	pVCpu->hwaccm.s.vmx.pMSRBitmapPhys = 0;
361	}
362	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
363	if (pVCpu->hwaccm.s.vmx.pMemObjHostMSR != NIL_RTR0MEMOBJ)
364	{
365	RTR0MemObjFree(pVCpu->hwaccm.s.vmx.pMemObjHostMSR, false);
366	pVCpu->hwaccm.s.vmx.pMemObjHostMSR = NIL_RTR0MEMOBJ;
367	pVCpu->hwaccm.s.vmx.pHostMSR = 0;
368	pVCpu->hwaccm.s.vmx.pHostMSRPhys = 0;
369	}
370	if (pVCpu->hwaccm.s.vmx.pMemObjGuestMSR != NIL_RTR0MEMOBJ)
371	{
372	RTR0MemObjFree(pVCpu->hwaccm.s.vmx.pMemObjGuestMSR, false);
373	pVCpu->hwaccm.s.vmx.pMemObjGuestMSR = NIL_RTR0MEMOBJ;
374	pVCpu->hwaccm.s.vmx.pGuestMSR = 0;
375	pVCpu->hwaccm.s.vmx.pGuestMSRPhys = 0;
376	}
377	#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
378	}
379	if (pVM->hwaccm.s.vmx.pMemObjAPIC != NIL_RTR0MEMOBJ)
380	{
381	RTR0MemObjFree(pVM->hwaccm.s.vmx.pMemObjAPIC, false);
382	pVM->hwaccm.s.vmx.pMemObjAPIC = NIL_RTR0MEMOBJ;
383	pVM->hwaccm.s.vmx.pAPIC = 0;
384	pVM->hwaccm.s.vmx.pAPICPhys = 0;
385	}
386	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
387	if (pVM->hwaccm.s.vmx.pMemObjScratch != NIL_RTR0MEMOBJ)
388	{
389	ASMMemZero32(pVM->hwaccm.s.vmx.pScratch, PAGE_SIZE);
390	RTR0MemObjFree(pVM->hwaccm.s.vmx.pMemObjScratch, false);
391	pVM->hwaccm.s.vmx.pMemObjScratch = NIL_RTR0MEMOBJ;
392	pVM->hwaccm.s.vmx.pScratch = 0;
393	pVM->hwaccm.s.vmx.pScratchPhys = 0;
394	}
395	#endif
396	return VINF_SUCCESS;
397	}
398
399
400	/**
401	* Sets up VT-x for the specified VM.
402	*
403	* @returns VBox status code.
404	* @param pVM Pointer to the VM.
405	*/
406	VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
407	{
408	int rc = VINF_SUCCESS;
409	uint32_t val;
410
411	AssertReturn(pVM, VERR_INVALID_PARAMETER);
412
413	/* Initialize these always, see hwaccmR3InitFinalizeR0().*/
414	pVM->hwaccm.s.vmx.enmFlushEPT = VMX_FLUSH_EPT_NONE;
415	pVM->hwaccm.s.vmx.enmFlushVPID = VMX_FLUSH_VPID_NONE;
416
417	/* Determine optimal flush type for EPT. */
418	if (pVM->hwaccm.s.fNestedPaging)
419	{
420	if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVEPT_CAPS_SINGLE_CONTEXT)
421	pVM->hwaccm.s.vmx.enmFlushEPT = VMX_FLUSH_EPT_SINGLE_CONTEXT;
422	else if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVEPT_CAPS_ALL_CONTEXTS)
423	pVM->hwaccm.s.vmx.enmFlushEPT = VMX_FLUSH_EPT_ALL_CONTEXTS;
424	else
425	{
426	/*
427	* Should never really happen. EPT is supported but no suitable flush types supported.
428	* We cannot ignore EPT at this point as we've already setup Unrestricted Guest execution.
429	*/
430	pVM->hwaccm.s.vmx.enmFlushEPT = VMX_FLUSH_EPT_NOT_SUPPORTED;
431	return VERR_VMX_GENERIC;
432	}
433	}
434
435	/* Determine optimal flush type for VPID. */
436	if (pVM->hwaccm.s.vmx.fVPID)
437	{
438	if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVVPID_CAPS_SINGLE_CONTEXT)
439	pVM->hwaccm.s.vmx.enmFlushVPID = VMX_FLUSH_VPID_SINGLE_CONTEXT;
440	else if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVVPID_CAPS_ALL_CONTEXTS)
441	pVM->hwaccm.s.vmx.enmFlushVPID = VMX_FLUSH_VPID_ALL_CONTEXTS;
442	else
443	{
444	/*
445	* Neither SINGLE nor ALL context flush types for VPID supported by the CPU.
446	* We do not handle other flush type combinations, ignore VPID capabilities.
447	*/
448	if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVVPID_CAPS_INDIV_ADDR)
449	Log(("VMXR0SetupVM: Only VMX_FLUSH_VPID_INDIV_ADDR supported. Ignoring VPID.\n"));
450	if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVVPID_CAPS_SINGLE_CONTEXT_RETAIN_GLOBALS)
451	Log(("VMXR0SetupVM: Only VMX_FLUSH_VPID_SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
452	pVM->hwaccm.s.vmx.enmFlushVPID = VMX_FLUSH_VPID_NOT_SUPPORTED;
453	pVM->hwaccm.s.vmx.fVPID = false;
454	}
455	}
456
457	for (VMCPUID i = 0; i < pVM->cCpus; i++)
458	{
459	PVMCPU pVCpu = &pVM->aCpus[i];
460
461	AssertPtr(pVCpu->hwaccm.s.vmx.pvVMCS);
462
463	/* Set revision dword at the beginning of the VMCS structure. */
464	(uint32_t )pVCpu->hwaccm.s.vmx.pvVMCS = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hwaccm.s.vmx.msr.vmx_basic_info);
465
466	/*
467	* Clear and activate the VMCS.
468	*/
469	Log(("HCPhysVMCS = %RHp\n", pVCpu->hwaccm.s.vmx.HCPhysVMCS));
470	rc = VMXClearVMCS(pVCpu->hwaccm.s.vmx.HCPhysVMCS);
471	if (RT_FAILURE(rc))
472	goto vmx_end;
473
474	rc = VMXActivateVMCS(pVCpu->hwaccm.s.vmx.HCPhysVMCS);
475	if (RT_FAILURE(rc))
476	goto vmx_end;
477
478	/*
479	* VMX_VMCS_CTRL_PIN_EXEC_CONTROLS
480	* Set required bits to one and zero according to the MSR capabilities.
481	*/
482	val = pVM->hwaccm.s.vmx.msr.vmx_pin_ctls.n.disallowed0;
483	val \|= VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_EXT_INT_EXIT /* External interrupts */
484	\| VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_NMI_EXIT; /* Non-maskable interrupts */
485
486	/*
487	* Enable the VMX preemption timer.
488	*/
489	if (pVM->hwaccm.s.vmx.fUsePreemptTimer)
490	val \|= VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_PREEMPT_TIMER;
491	val &= pVM->hwaccm.s.vmx.msr.vmx_pin_ctls.n.allowed1;
492
493	rc = VMXWriteVMCS(VMX_VMCS_CTRL_PIN_EXEC_CONTROLS, val);
494	AssertRC(rc);
495
496	/*
497	* VMX_VMCS_CTRL_PROC_EXEC_CONTROLS
498	* Set required bits to one and zero according to the MSR capabilities.
499	*/
500	val = pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.disallowed0;
501	/* Program which event cause VM-exits and which features we want to use. */
502	val \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_HLT_EXIT
503	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_TSC_OFFSET
504	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT
505	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_UNCOND_IO_EXIT
506	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDPMC_EXIT
507	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_EXIT
508	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MWAIT_EXIT; /* don't execute mwait or else we'll idle inside
509	the guest (host thinks the cpu load is high) */
510
511	/* Without nested paging we should intercept invlpg and cr3 mov instructions. */
512	if (!pVM->hwaccm.s.fNestedPaging)
513	{
514	val \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INVLPG_EXIT
515	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
516	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT;
517	}
518
519	/*
520	* VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MWAIT_EXIT might cause a vmlaunch
521	* failure with an invalid control fields error. (combined with some other exit reasons)
522	*/
523	if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
524	{
525	/* CR8 reads from the APIC shadow page; writes cause an exit is they lower the TPR below the threshold */
526	val \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW;
527	Assert(pVM->hwaccm.s.vmx.pAPIC);
528	}
529	else
530	/* Exit on CR8 reads & writes in case the TPR shadow feature isn't present. */
531	val \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR8_STORE_EXIT \| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR8_LOAD_EXIT;
532
533	if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
534	{
535	Assert(pVCpu->hwaccm.s.vmx.pMSRBitmapPhys);
536	val \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS;
537	}
538
539	/* We will use the secondary control if it's present. */
540	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
541
542	/* Mask away the bits that the CPU doesn't support */
543	/** @todo make sure they don't conflict with the above requirements. */
544	val &= pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1;
545	pVCpu->hwaccm.s.vmx.proc_ctls = val;
546
547	rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, val);
548	AssertRC(rc);
549
550	if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
551	{
552	/*
553	* VMX_VMCS_CTRL_PROC_EXEC_CONTROLS2
554	* Set required bits to one and zero according to the MSR capabilities.
555	*/
556	val = pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2.n.disallowed0;
557	val \|= VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT;
558
559	if (pVM->hwaccm.s.fNestedPaging)
560	val \|= VMX_VMCS_CTRL_PROC_EXEC2_EPT;
561
562	if (pVM->hwaccm.s.vmx.fVPID)
563	val \|= VMX_VMCS_CTRL_PROC_EXEC2_VPID;
564
565	if (pVM->hwaccm.s.fHasIoApic)
566	val \|= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC;
567
568	if (pVM->hwaccm.s.vmx.fUnrestrictedGuest)
569	val \|= VMX_VMCS_CTRL_PROC_EXEC2_REAL_MODE;
570
571	if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
572	val \|= VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP;
573
574	/* Mask away the bits that the CPU doesn't support */
575	/** @todo make sure they don't conflict with the above requirements. */
576	val &= pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2.n.allowed1;
577	pVCpu->hwaccm.s.vmx.proc_ctls2 = val;
578	rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS2, val);
579	AssertRC(rc);
580	}
581
582	/*
583	* VMX_VMCS_CTRL_CR3_TARGET_COUNT
584	* Set required bits to one and zero according to the MSR capabilities.
585	*/
586	rc = VMXWriteVMCS(VMX_VMCS_CTRL_CR3_TARGET_COUNT, 0);
587	AssertRC(rc);
588
589	/*
590	* Forward all exception except #NM & #PF to the guest.
591	* We always need to check pagefaults since our shadow page table can be out of sync.
592	* And we always lazily sync the FPU & XMM state. .
593	*/
594
595	/** @todo Possible optimization:
596	* Keep the FPU and XMM state current in the EM thread. That way there's no need to
597	* lazily sync anything, but the downside is that we can't use the FPU stack or XMM
598	* registers ourselves of course.
599	*
600	* Note: only possible if the current state is actually ours (X86_CR0_TS flag)
601	*/
602
603	/*
604	* Don't filter page faults, all of them should cause a world switch.
605	*/
606	rc = VMXWriteVMCS(VMX_VMCS_CTRL_PAGEFAULT_ERROR_MASK, 0);
607	AssertRC(rc);
608	rc = VMXWriteVMCS(VMX_VMCS_CTRL_PAGEFAULT_ERROR_MATCH, 0);
609	AssertRC(rc);
610
611	rc = VMXWriteVMCS64(VMX_VMCS_CTRL_TSC_OFFSET_FULL, 0);
612	AssertRC(rc);
613	rc = VMXWriteVMCS64(VMX_VMCS_CTRL_IO_BITMAP_A_FULL, 0);
614	AssertRC(rc);
615	rc = VMXWriteVMCS64(VMX_VMCS_CTRL_IO_BITMAP_B_FULL, 0);
616	AssertRC(rc);
617
618	/*
619	* Set the MSR bitmap address.
620	*/
621	if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
622	{
623	Assert(pVCpu->hwaccm.s.vmx.pMSRBitmapPhys);
624
625	rc = VMXWriteVMCS64(VMX_VMCS_CTRL_MSR_BITMAP_FULL, pVCpu->hwaccm.s.vmx.pMSRBitmapPhys);
626	AssertRC(rc);
627
628	/*
629	* Allow the guest to directly modify these MSRs; they are loaded/stored automatically
630	* using MSR-load/store areas in the VMCS.
631	*/
632	hmR0VmxSetMSRPermission(pVCpu, MSR_IA32_SYSENTER_CS, true, true);
633	hmR0VmxSetMSRPermission(pVCpu, MSR_IA32_SYSENTER_ESP, true, true);
634	hmR0VmxSetMSRPermission(pVCpu, MSR_IA32_SYSENTER_EIP, true, true);
635	hmR0VmxSetMSRPermission(pVCpu, MSR_K8_LSTAR, true, true);
636	hmR0VmxSetMSRPermission(pVCpu, MSR_K6_STAR, true, true);
637	hmR0VmxSetMSRPermission(pVCpu, MSR_K8_SF_MASK, true, true);
638	hmR0VmxSetMSRPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, true, true);
639	hmR0VmxSetMSRPermission(pVCpu, MSR_K8_GS_BASE, true, true);
640	hmR0VmxSetMSRPermission(pVCpu, MSR_K8_FS_BASE, true, true);
641	if (pVCpu->hwaccm.s.vmx.proc_ctls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
642	hmR0VmxSetMSRPermission(pVCpu, MSR_K8_TSC_AUX, true, true);
643	}
644
645	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
646	/*
647	* Set the guest & host MSR load/store physical addresses.
648	*/
649	Assert(pVCpu->hwaccm.s.vmx.pGuestMSRPhys);
650	rc = VMXWriteVMCS64(VMX_VMCS_CTRL_VMENTRY_MSR_LOAD_FULL, pVCpu->hwaccm.s.vmx.pGuestMSRPhys);
651	AssertRC(rc);
652	rc = VMXWriteVMCS64(VMX_VMCS_CTRL_VMEXIT_MSR_STORE_FULL, pVCpu->hwaccm.s.vmx.pGuestMSRPhys);
653	AssertRC(rc);
654	Assert(pVCpu->hwaccm.s.vmx.pHostMSRPhys);
655	rc = VMXWriteVMCS64(VMX_VMCS_CTRL_VMEXIT_MSR_LOAD_FULL, pVCpu->hwaccm.s.vmx.pHostMSRPhys);
656	AssertRC(rc);
657	#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
658
659	rc = VMXWriteVMCS(VMX_VMCS_CTRL_ENTRY_MSR_LOAD_COUNT, 0);
660	AssertRC(rc);
661	rc = VMXWriteVMCS(VMX_VMCS_CTRL_EXIT_MSR_STORE_COUNT, 0);
662	AssertRC(rc);
663	rc = VMXWriteVMCS(VMX_VMCS_CTRL_EXIT_MSR_LOAD_COUNT, 0);
664	AssertRC(rc);
665
666	if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
667	{
668	Assert(pVM->hwaccm.s.vmx.pMemObjAPIC);
669	/* Optional */
670	rc = VMXWriteVMCS(VMX_VMCS_CTRL_TPR_THRESHOLD, 0);
671	rc \|= VMXWriteVMCS64(VMX_VMCS_CTRL_VAPIC_PAGEADDR_FULL, pVCpu->hwaccm.s.vmx.HCPhysVAPIC);
672
673	if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
674	rc \|= VMXWriteVMCS64(VMX_VMCS_CTRL_APIC_ACCESSADDR_FULL, pVM->hwaccm.s.vmx.pAPICPhys);
675
676	AssertRC(rc);
677	}
678
679	/* Set link pointer to -1. Not currently used. */
680	rc = VMXWriteVMCS64(VMX_VMCS_GUEST_LINK_PTR_FULL, 0xFFFFFFFFFFFFFFFFULL);
681	AssertRC(rc);
682
683	/*
684	* Clear VMCS, marking it inactive. Clear implementation specific data and writing back
685	* VMCS data back to memory.
686	*/
687	rc = VMXClearVMCS(pVCpu->hwaccm.s.vmx.HCPhysVMCS);
688	AssertRC(rc);
689
690	/*
691	* Configure the VMCS read cache.
692	*/
693	PVMCSCACHE pCache = &pVCpu->hwaccm.s.vmx.VMCSCache;
694
695	VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_RIP);
696	VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_RSP);
697	VMXSetupCachedReadVMCS(pCache, VMX_VMCS_GUEST_RFLAGS);
698	VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE);
699	VMXSetupCachedReadVMCS(pCache, VMX_VMCS_CTRL_CR0_READ_SHADOW);
700	VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_CR0);
701	VMXSetupCachedReadVMCS(pCache, VMX_VMCS_CTRL_CR4_READ_SHADOW);
702	VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_CR4);
703	VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_DR7);
704	VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_GUEST_SYSENTER_CS);
705	VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_SYSENTER_EIP);
706	VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_SYSENTER_ESP);
707	VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_GUEST_GDTR_LIMIT);
708	VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_GDTR_BASE);
709	VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_GUEST_IDTR_LIMIT);
710	VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_IDTR_BASE);
711
712	VMX_SETUP_SELREG(ES, pCache);
713	VMX_SETUP_SELREG(SS, pCache);
714	VMX_SETUP_SELREG(CS, pCache);
715	VMX_SETUP_SELREG(DS, pCache);
716	VMX_SETUP_SELREG(FS, pCache);
717	VMX_SETUP_SELREG(GS, pCache);
718	VMX_SETUP_SELREG(LDTR, pCache);
719	VMX_SETUP_SELREG(TR, pCache);
720
721	/*
722	* Status code VMCS reads.
723	*/
724	VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_EXIT_REASON);
725	VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_VM_INSTR_ERROR);
726	VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_EXIT_INSTR_LENGTH);
727	VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_EXIT_INTERRUPTION_ERRCODE);
728	VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO);
729	VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_EXIT_INSTR_INFO);
730	VMXSetupCachedReadVMCS(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
731	VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_IDT_INFO);
732	VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_IDT_ERRCODE);
733
734	if (pVM->hwaccm.s.fNestedPaging)
735	{
736	VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_CR3);
737	VMXSetupCachedReadVMCS(pCache, VMX_VMCS_EXIT_PHYS_ADDR_FULL);
738	pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
739	}
740	else
741	pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
742	} /* for each VMCPU */
743
744	/*
745	* Setup the right TLB function based on CPU capabilities.
746	*/
747	if (pVM->hwaccm.s.fNestedPaging && pVM->hwaccm.s.vmx.fVPID)
748	pVM->hwaccm.s.vmx.pfnSetupTaggedTLB = hmR0VmxSetupTLBBoth;
749	else if (pVM->hwaccm.s.fNestedPaging)
750	pVM->hwaccm.s.vmx.pfnSetupTaggedTLB = hmR0VmxSetupTLBEPT;
751	else if (pVM->hwaccm.s.vmx.fVPID)
752	pVM->hwaccm.s.vmx.pfnSetupTaggedTLB = hmR0VmxSetupTLBVPID;
753	else
754	pVM->hwaccm.s.vmx.pfnSetupTaggedTLB = hmR0VmxSetupTLBDummy;
755
756	vmx_end:
757	hmR0VmxCheckError(pVM, &pVM->aCpus[0], rc);
758	return rc;
759	}
760
761
762	/**
763	* Sets the permission bits for the specified MSR.
764	*
765	* @param pVCpu Pointer to the VMCPU.
766	* @param ulMSR The MSR value.
767	* @param fRead Whether reading is allowed.
768	* @param fWrite Whether writing is allowed.
769	*/
770	static void hmR0VmxSetMSRPermission(PVMCPU pVCpu, unsigned ulMSR, bool fRead, bool fWrite)
771	{
772	unsigned ulBit;
773	uint8_t pMSRBitmap = (uint8_t )pVCpu->hwaccm.s.vmx.pMSRBitmap;
774
775	/*
776	* Layout:
777	* 0x000 - 0x3ff - Low MSR read bits
778	* 0x400 - 0x7ff - High MSR read bits
779	* 0x800 - 0xbff - Low MSR write bits
780	* 0xc00 - 0xfff - High MSR write bits
781	*/
782	if (ulMSR <= 0x00001FFF)
783	{
784	/* Pentium-compatible MSRs */
785	ulBit = ulMSR;
786	}
787	else if ( ulMSR >= 0xC0000000
788	&& ulMSR <= 0xC0001FFF)
789	{
790	/* AMD Sixth Generation x86 Processor MSRs */
791	ulBit = (ulMSR - 0xC0000000);
792	pMSRBitmap += 0x400;
793	}
794	else
795	{
796	AssertFailed();
797	return;
798	}
799
800	Assert(ulBit <= 0x1fff);
801	if (fRead)
802	ASMBitClear(pMSRBitmap, ulBit);
803	else
804	ASMBitSet(pMSRBitmap, ulBit);
805
806	if (fWrite)
807	ASMBitClear(pMSRBitmap + 0x800, ulBit);
808	else
809	ASMBitSet(pMSRBitmap + 0x800, ulBit);
810	}
811
812
813	/**
814	* Injects an event (trap or external interrupt).
815	*
816	* @returns VBox status code. Note that it may return VINF_EM_RESET to
817	* indicate a triple fault when injecting X86_XCPT_DF.
818	*
819	* @param pVM Pointer to the VM.
820	* @param pVCpu Pointer to the VMCPU.
821	* @param pCtx Pointer to the guest CPU Context.
822	* @param intInfo VMX interrupt info.
823	* @param cbInstr Opcode length of faulting instruction.
824	* @param errCode Error code (optional).
825	*/
826	static int hmR0VmxInjectEvent(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, uint32_t intInfo, uint32_t cbInstr, uint32_t errCode)
827	{
828	int rc;
829	uint32_t iGate = VMX_EXIT_INTERRUPTION_INFO_VECTOR(intInfo);
830
831	#ifdef VBOX_WITH_STATISTICS
832	STAM_COUNTER_INC(&pVCpu->hwaccm.s.paStatInjectedIrqsR0[iGate & MASK_INJECT_IRQ_STAT]);
833	#endif
834
835	#ifdef VBOX_STRICT
836	if (iGate == 0xE)
837	{
838	LogFlow(("hmR0VmxInjectEvent: Injecting interrupt %d at %RGv error code=%08x CR2=%RGv intInfo=%08x\n", iGate,
839	(RTGCPTR)pCtx->rip, errCode, pCtx->cr2, intInfo));
840	}
841	else if (iGate < 0x20)
842	{
843	LogFlow(("hmR0VmxInjectEvent: Injecting interrupt %d at %RGv error code=%08x\n", iGate, (RTGCPTR)pCtx->rip,
844	errCode));
845	}
846	else
847	{
848	LogFlow(("INJ-EI: %x at %RGv\n", iGate, (RTGCPTR)pCtx->rip));
849	Assert( VMX_EXIT_INTERRUPTION_INFO_TYPE(intInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW
850	\|\| !VMCPU_FF_ISSET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
851	Assert( VMX_EXIT_INTERRUPTION_INFO_TYPE(intInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW
852	\|\| pCtx->eflags.u32 & X86_EFL_IF);
853	}
854	#endif
855
856	if ( CPUMIsGuestInRealModeEx(pCtx)
857	&& pVM->hwaccm.s.vmx.pRealModeTSS)
858	{
859	RTGCPHYS GCPhysHandler;
860	uint16_t offset, ip;
861	RTSEL sel;
862
863	/*
864	* Injecting events doesn't work right with real mode emulation.
865	* (#GP if we try to inject external hardware interrupts)
866	* Inject the interrupt or trap directly instead.
867	*
868	* ASSUMES no access handlers for the bits we read or write below (should be safe).
869	*/
870	Log(("Manual interrupt/trap '%x' inject (real mode)\n", iGate));
871
872	/*
873	* Check if the interrupt handler is present.
874	*/
875	if (iGate * 4 + 3 > pCtx->idtr.cbIdt)
876	{
877	Log(("IDT cbIdt violation\n"));
878	if (iGate != X86_XCPT_DF)
879	{
880	uint32_t intInfo2;
881
882	intInfo2 = (iGate == X86_XCPT_GP) ? (uint32_t)X86_XCPT_DF : iGate;
883	intInfo2 \|= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
884	intInfo2 \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
885	intInfo2 \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HWEXCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
886
887	return hmR0VmxInjectEvent(pVM, pVCpu, pCtx, intInfo2, 0, 0 /* no error code according to the Intel docs */);
888	}
889	Log(("Triple fault -> reset the VM!\n"));
890	return VINF_EM_RESET;
891	}
892	if ( VMX_EXIT_INTERRUPTION_INFO_TYPE(intInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW
893	\|\| iGate == 3 /* Both #BP and #OF point to the instruction after. */
894	\|\| iGate == 4)
895	{
896	ip = pCtx->ip + cbInstr;
897	}
898	else
899	ip = pCtx->ip;
900
901	/*
902	* Read the selector:offset pair of the interrupt handler.
903	*/
904	GCPhysHandler = (RTGCPHYS)pCtx->idtr.pIdt + iGate * 4;
905	rc = PGMPhysSimpleReadGCPhys(pVM, &offset, GCPhysHandler, sizeof(offset)); AssertRC(rc);
906	rc = PGMPhysSimpleReadGCPhys(pVM, &sel, GCPhysHandler + 2, sizeof(sel)); AssertRC(rc);
907
908	LogFlow(("IDT handler %04X:%04X\n", sel, offset));
909
910	/*
911	* Construct the stack frame.
912	*/
913	/** @todo Check stack limit. */
914	pCtx->sp -= 2;
915	LogFlow(("ss:sp %04X:%04X eflags=%x\n", pCtx->ss.Sel, pCtx->sp, pCtx->eflags.u));
916	rc = PGMPhysSimpleWriteGCPhys(pVM, pCtx->ss.u64Base + pCtx->sp, &pCtx->eflags, sizeof(uint16_t)); AssertRC(rc);
917	pCtx->sp -= 2;
918	LogFlow(("ss:sp %04X:%04X cs=%x\n", pCtx->ss.Sel, pCtx->sp, pCtx->cs.Sel));
919	rc = PGMPhysSimpleWriteGCPhys(pVM, pCtx->ss.u64Base + pCtx->sp, &pCtx->cs, sizeof(uint16_t)); AssertRC(rc);
920	pCtx->sp -= 2;
921	LogFlow(("ss:sp %04X:%04X ip=%x\n", pCtx->ss.Sel, pCtx->sp, ip));
922	rc = PGMPhysSimpleWriteGCPhys(pVM, pCtx->ss.u64Base + pCtx->sp, &ip, sizeof(ip)); AssertRC(rc);
923
924	/*
925	* Update the CPU state for executing the handler.
926	*/
927	pCtx->rip = offset;
928	pCtx->cs.Sel = sel;
929	pCtx->cs.u64Base = sel << 4;
930	pCtx->eflags.u &= ~(X86_EFL_IF \| X86_EFL_TF \| X86_EFL_RF \| X86_EFL_AC);
931
932	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_GUEST_SEGMENT_REGS;
933	return VINF_SUCCESS;
934	}
935
936	/*
937	* Set event injection state.
938	*/
939	rc = VMXWriteVMCS(VMX_VMCS_CTRL_ENTRY_IRQ_INFO, intInfo \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT));
940	rc \|= VMXWriteVMCS(VMX_VMCS_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
941	rc \|= VMXWriteVMCS(VMX_VMCS_CTRL_ENTRY_EXCEPTION_ERRCODE, errCode);
942
943	AssertRC(rc);
944	return rc;
945	}
946
947
948	/**
949	* Checks for pending guest interrupts and injects them.
950	*
951	* @returns VBox status code.
952	* @param pVM Pointer to the VM.
953	* @param pVCpu Pointer to the VMCPU.
954	* @param pCtx Pointer to the guest CPU context.
955	*/
956	static int hmR0VmxCheckPendingInterrupt(PVM pVM, PVMCPU pVCpu, CPUMCTX *pCtx)
957	{
958	int rc;
959
960	/*
961	* Dispatch any pending interrupts (injected before, but a VM exit occurred prematurely).
962	*/
963	if (pVCpu->hwaccm.s.Event.fPending)
964	{
965	Log(("CPU%d: Reinjecting event %RX64 %08x at %RGv cr2=%RX64\n", pVCpu->idCpu, pVCpu->hwaccm.s.Event.intInfo,
966	pVCpu->hwaccm.s.Event.errCode, (RTGCPTR)pCtx->rip, pCtx->cr2));
967	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatIntReinject);
968	rc = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, pVCpu->hwaccm.s.Event.intInfo, 0, pVCpu->hwaccm.s.Event.errCode);
969	AssertRC(rc);
970
971	pVCpu->hwaccm.s.Event.fPending = false;
972	return VINF_SUCCESS;
973	}
974
975	/*
976	* If an active trap is already pending, we must forward it first!
977	*/
978	if (!TRPMHasTrap(pVCpu))
979	{
980	if (VMCPU_FF_TESTANDCLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI))
981	{
982	RTGCUINTPTR intInfo;
983
984	Log(("CPU%d: injecting #NMI\n", pVCpu->idCpu));
985
986	intInfo = X86_XCPT_NMI;
987	intInfo \|= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
988	intInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
989
990	rc = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, intInfo, 0, 0);
991	AssertRC(rc);
992
993	return VINF_SUCCESS;
994	}
995
996	/** @todo SMI interrupts. */
997
998	/*
999	* When external interrupts are pending, we should exit the VM when IF is set.
1000	*/
1001	if (VMCPU_FF_ISPENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC\|VMCPU_FF_INTERRUPT_PIC)))
1002	{
1003	if (!(pCtx->eflags.u32 & X86_EFL_IF))
1004	{
1005	if (!(pVCpu->hwaccm.s.vmx.proc_ctls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_IRQ_WINDOW_EXIT))
1006	{
1007	LogFlow(("Enable irq window exit!\n"));
1008	pVCpu->hwaccm.s.vmx.proc_ctls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_IRQ_WINDOW_EXIT;
1009	rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
1010	AssertRC(rc);
1011	}
1012	/* else nothing to do but wait */
1013	}
1014	else if (!VMCPU_FF_ISSET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
1015	{
1016	uint8_t u8Interrupt;
1017
1018	rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
1019	Log(("CPU%d: Dispatch interrupt: u8Interrupt=%x (%d) rc=%Rrc cs:rip=%04X:%RGv\n", pVCpu->idCpu,
1020	u8Interrupt, u8Interrupt, rc, pCtx->cs.Sel, (RTGCPTR)pCtx->rip));
1021	if (RT_SUCCESS(rc))
1022	{
1023	rc = TRPMAssertTrap(pVCpu, u8Interrupt, TRPM_HARDWARE_INT);
1024	AssertRC(rc);
1025	}
1026	else
1027	{
1028	/* Can only happen in rare cases where a pending interrupt is cleared behind our back */
1029	Assert(!VMCPU_FF_ISPENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC\|VMCPU_FF_INTERRUPT_PIC)));
1030	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatSwitchGuestIrq);
1031	/* Just continue */
1032	}
1033	}
1034	else
1035	Log(("Pending interrupt blocked at %RGv by VM_FF_INHIBIT_INTERRUPTS!!\n", (RTGCPTR)pCtx->rip));
1036	}
1037	}
1038
1039	#ifdef VBOX_STRICT
1040	if (TRPMHasTrap(pVCpu))
1041	{
1042	uint8_t u8Vector;
1043	rc = TRPMQueryTrapAll(pVCpu, &u8Vector, 0, 0, 0);
1044	AssertRC(rc);
1045	}
1046	#endif
1047
1048	if ( (pCtx->eflags.u32 & X86_EFL_IF)
1049	&& (!VMCPU_FF_ISSET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
1050	&& TRPMHasTrap(pVCpu)
1051	)
1052	{
1053	uint8_t u8Vector;
1054	TRPMEVENT enmType;
1055	RTGCUINTPTR intInfo;
1056	RTGCUINT errCode;
1057
1058	/*
1059	* If a new event is pending, dispatch it now.
1060	*/
1061	rc = TRPMQueryTrapAll(pVCpu, &u8Vector, &enmType, &errCode, 0);
1062	AssertRC(rc);
1063	Assert(pCtx->eflags.Bits.u1IF == 1 \|\| enmType == TRPM_TRAP);
1064	Assert(enmType != TRPM_SOFTWARE_INT);
1065
1066	/*
1067	* Clear the pending trap.
1068	*/
1069	rc = TRPMResetTrap(pVCpu);
1070	AssertRC(rc);
1071
1072	intInfo = u8Vector;
1073	intInfo \|= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
1074
1075	if (enmType == TRPM_TRAP)
1076	{
1077	switch (u8Vector)
1078	{
1079	case X86_XCPT_DF:
1080	case X86_XCPT_TS:
1081	case X86_XCPT_NP:
1082	case X86_XCPT_SS:
1083	case X86_XCPT_GP:
1084	case X86_XCPT_PF:
1085	case X86_XCPT_AC:
1086	{
1087	/* Valid error codes. */
1088	intInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
1089	break;
1090	}
1091
1092	default:
1093	break;
1094	}
1095
1096	if ( u8Vector == X86_XCPT_BP
1097	\|\| u8Vector == X86_XCPT_OF)
1098	{
1099	intInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SWEXCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
1100	}
1101	else
1102	intInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HWEXCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
1103	}
1104	else
1105	intInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
1106
1107	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatIntInject);
1108	rc = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, intInfo, 0, errCode);
1109	AssertRC(rc);
1110	} /* if (interrupts can be dispatched) */
1111
1112	return VINF_SUCCESS;
1113	}
1114
1115
1116	/**
1117	* Save the host state into the VMCS.
1118	*
1119	* @returns VBox status code.
1120	* @param pVM Pointer to the VM.
1121	* @param pVCpu Pointer to the VMCPU.
1122	*/
1123	VMMR0DECL(int) VMXR0SaveHostState(PVM pVM, PVMCPU pVCpu)
1124	{
1125	int rc = VINF_SUCCESS;
1126	NOREF(pVM);
1127
1128	/*
1129	* Host CPU Context.
1130	*/
1131	if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_HOST_CONTEXT)
1132	{
1133	RTIDTR idtr;
1134	RTGDTR gdtr;
1135	RTSEL SelTR;
1136	PCX86DESCHC pDesc;
1137	uintptr_t trBase;
1138	RTSEL cs;
1139	RTSEL ss;
1140	uint64_t cr3;
1141
1142	/*
1143	* Control registers.
1144	*/
1145	rc = VMXWriteVMCS(VMX_VMCS_HOST_CR0, ASMGetCR0());
1146	Log2(("VMX_VMCS_HOST_CR0 %08x\n", ASMGetCR0()));
1147	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
1148	if (VMX_IS_64BIT_HOST_MODE())
1149	{
1150	cr3 = hwaccmR0Get64bitCR3();
1151	rc \|= VMXWriteVMCS64(VMX_VMCS_HOST_CR3, cr3);
1152	}
1153	else
1154	#endif
1155	{
1156	cr3 = ASMGetCR3();
1157	rc \|= VMXWriteVMCS(VMX_VMCS_HOST_CR3, cr3);
1158	}
1159	Log2(("VMX_VMCS_HOST_CR3 %08RX64\n", cr3));
1160	rc \|= VMXWriteVMCS(VMX_VMCS_HOST_CR4, ASMGetCR4());
1161	Log2(("VMX_VMCS_HOST_CR4 %08x\n", ASMGetCR4()));
1162	AssertRC(rc);
1163
1164	/*
1165	* Selector registers.
1166	*/
1167	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
1168	if (VMX_IS_64BIT_HOST_MODE())
1169	{
1170	cs = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelCS;
1171	ss = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelSS;
1172	}
1173	else
1174	{
1175	/* sysenter loads LDT cs & ss, VMX doesn't like this. Load the GDT ones (safe). */
1176	cs = (RTSEL)(uintptr_t)&SUPR0AbsKernelCS;
1177	ss = (RTSEL)(uintptr_t)&SUPR0AbsKernelSS;
1178	}
1179	#else
1180	cs = ASMGetCS();
1181	ss = ASMGetSS();
1182	#endif
1183	Assert(!(cs & X86_SEL_LDT)); Assert((cs & X86_SEL_RPL) == 0);
1184	Assert(!(ss & X86_SEL_LDT)); Assert((ss & X86_SEL_RPL) == 0);
1185	rc = VMXWriteVMCS(VMX_VMCS16_HOST_FIELD_CS, cs);
1186	/* Note: VMX is (again) very picky about the RPL of the selectors here; we'll restore them manually. */
1187	rc \|= VMXWriteVMCS(VMX_VMCS16_HOST_FIELD_DS, 0);
1188	rc \|= VMXWriteVMCS(VMX_VMCS16_HOST_FIELD_ES, 0);
1189	#if HC_ARCH_BITS == 32
1190	if (!VMX_IS_64BIT_HOST_MODE())
1191	{
1192	rc \|= VMXWriteVMCS(VMX_VMCS16_HOST_FIELD_FS, 0);
1193	rc \|= VMXWriteVMCS(VMX_VMCS16_HOST_FIELD_GS, 0);
1194	}
1195	#endif
1196	rc \|= VMXWriteVMCS(VMX_VMCS16_HOST_FIELD_SS, ss);
1197	SelTR = ASMGetTR();
1198	rc \|= VMXWriteVMCS(VMX_VMCS16_HOST_FIELD_TR, SelTR);
1199	AssertRC(rc);
1200	Log2(("VMX_VMCS_HOST_FIELD_CS %08x (%08x)\n", cs, ASMGetSS()));
1201	Log2(("VMX_VMCS_HOST_FIELD_DS 00000000 (%08x)\n", ASMGetDS()));
1202	Log2(("VMX_VMCS_HOST_FIELD_ES 00000000 (%08x)\n", ASMGetES()));
1203	Log2(("VMX_VMCS_HOST_FIELD_FS 00000000 (%08x)\n", ASMGetFS()));
1204	Log2(("VMX_VMCS_HOST_FIELD_GS 00000000 (%08x)\n", ASMGetGS()));
1205	Log2(("VMX_VMCS_HOST_FIELD_SS %08x (%08x)\n", ss, ASMGetSS()));
1206	Log2(("VMX_VMCS_HOST_FIELD_TR %08x\n", ASMGetTR()));
1207
1208	/*
1209	* GDTR & IDTR.
1210	*/
1211	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
1212	if (VMX_IS_64BIT_HOST_MODE())
1213	{
1214	X86XDTR64 gdtr64, idtr64;
1215	hwaccmR0Get64bitGDTRandIDTR(&gdtr64, &idtr64);
1216	rc = VMXWriteVMCS64(VMX_VMCS_HOST_GDTR_BASE, gdtr64.uAddr);
1217	rc \|= VMXWriteVMCS64(VMX_VMCS_HOST_IDTR_BASE, gdtr64.uAddr);
1218	AssertRC(rc);
1219	Log2(("VMX_VMCS_HOST_GDTR_BASE %RX64\n", gdtr64.uAddr));
1220	Log2(("VMX_VMCS_HOST_IDTR_BASE %RX64\n", idtr64.uAddr));
1221	gdtr.cbGdt = gdtr64.cb;
1222	gdtr.pGdt = (uintptr_t)gdtr64.uAddr;
1223	}
1224	else
1225	#endif
1226	{
1227	ASMGetGDTR(&gdtr);
1228	rc = VMXWriteVMCS(VMX_VMCS_HOST_GDTR_BASE, gdtr.pGdt);
1229	ASMGetIDTR(&idtr);
1230	rc \|= VMXWriteVMCS(VMX_VMCS_HOST_IDTR_BASE, idtr.pIdt);
1231	AssertRC(rc);
1232	Log2(("VMX_VMCS_HOST_GDTR_BASE %RHv\n", gdtr.pGdt));
1233	Log2(("VMX_VMCS_HOST_IDTR_BASE %RHv\n", idtr.pIdt));
1234	}
1235
1236	/*
1237	* Save the base address of the TR selector.
1238	*/
1239	if (SelTR > gdtr.cbGdt)
1240	{
1241	AssertMsgFailed(("Invalid TR selector %x. GDTR.cbGdt=%x\n", SelTR, gdtr.cbGdt));
1242	return VERR_VMX_INVALID_HOST_STATE;
1243	}
1244
1245	pDesc = (PCX86DESCHC)(gdtr.pGdt + (SelTR & X86_SEL_MASK));
1246	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
1247	if (VMX_IS_64BIT_HOST_MODE())
1248	{
1249	uint64_t trBase64 = X86DESC64_BASE((PX86DESC64)pDesc);
1250	rc = VMXWriteVMCS64(VMX_VMCS_HOST_TR_BASE, trBase64);
1251	Log2(("VMX_VMCS_HOST_TR_BASE %RX64\n", trBase64));
1252	AssertRC(rc);
1253	}
1254	else
1255	#endif
1256	{
1257	#if HC_ARCH_BITS == 64
1258	trBase = X86DESC64_BASE(pDesc);
1259	#else
1260	trBase = X86DESC_BASE(pDesc);
1261	#endif
1262	rc = VMXWriteVMCS(VMX_VMCS_HOST_TR_BASE, trBase);
1263	AssertRC(rc);
1264	Log2(("VMX_VMCS_HOST_TR_BASE %RHv\n", trBase));
1265	}
1266
1267	/*
1268	* FS base and GS base.
1269	*/
1270	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
1271	if (VMX_IS_64BIT_HOST_MODE())
1272	{
1273	Log2(("MSR_K8_FS_BASE = %RX64\n", ASMRdMsr(MSR_K8_FS_BASE)));
1274	Log2(("MSR_K8_GS_BASE = %RX64\n", ASMRdMsr(MSR_K8_GS_BASE)));
1275	rc = VMXWriteVMCS64(VMX_VMCS_HOST_FS_BASE, ASMRdMsr(MSR_K8_FS_BASE));
1276	rc \|= VMXWriteVMCS64(VMX_VMCS_HOST_GS_BASE, ASMRdMsr(MSR_K8_GS_BASE));
1277	}
1278	#endif
1279	AssertRC(rc);
1280
1281	/*
1282	* Sysenter MSRs.
1283	*/
1284	/** @todo expensive!! */
1285	rc = VMXWriteVMCS(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
1286	Log2(("VMX_VMCS_HOST_SYSENTER_CS %08x\n", ASMRdMsr_Low(MSR_IA32_SYSENTER_CS)));
1287	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
1288	if (VMX_IS_64BIT_HOST_MODE())
1289	{
1290	Log2(("VMX_VMCS_HOST_SYSENTER_EIP %RX64\n", ASMRdMsr(MSR_IA32_SYSENTER_EIP)));
1291	Log2(("VMX_VMCS_HOST_SYSENTER_ESP %RX64\n", ASMRdMsr(MSR_IA32_SYSENTER_ESP)));
1292	rc \|= VMXWriteVMCS64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
1293	rc \|= VMXWriteVMCS64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
1294	}
1295	else
1296	{
1297	rc \|= VMXWriteVMCS(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
1298	rc \|= VMXWriteVMCS(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
1299	Log2(("VMX_VMCS_HOST_SYSENTER_EIP %RX32\n", ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP)));
1300	Log2(("VMX_VMCS_HOST_SYSENTER_ESP %RX32\n", ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP)));
1301	}
1302	#elif HC_ARCH_BITS == 32
1303	rc \|= VMXWriteVMCS(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
1304	rc \|= VMXWriteVMCS(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
1305	Log2(("VMX_VMCS_HOST_SYSENTER_EIP %RX32\n", ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP)));
1306	Log2(("VMX_VMCS_HOST_SYSENTER_ESP %RX32\n", ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP)));
1307	#else
1308	Log2(("VMX_VMCS_HOST_SYSENTER_EIP %RX64\n", ASMRdMsr(MSR_IA32_SYSENTER_EIP)));
1309	Log2(("VMX_VMCS_HOST_SYSENTER_ESP %RX64\n", ASMRdMsr(MSR_IA32_SYSENTER_ESP)));
1310	rc \|= VMXWriteVMCS64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
1311	rc \|= VMXWriteVMCS64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
1312	#endif
1313	AssertRC(rc);
1314
1315
1316	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
1317	/*
1318	* Store all host MSRs in the VM-Exit load area, so they will be reloaded after
1319	* the world switch back to the host.
1320	*/
1321	PVMXMSR pMsr = (PVMXMSR)pVCpu->hwaccm.s.vmx.pHostMSR;
1322	unsigned idxMsr = 0;
1323
1324	/*
1325	* Check if EFER MSR present.
1326	*/
1327	uint32_t u32HostExtFeatures = ASMCpuId_EDX(0x80000001);
1328	if (u32HostExtFeatures & (X86_CPUID_EXT_FEATURE_EDX_NX \| X86_CPUID_EXT_FEATURE_EDX_LONG_MODE))
1329	{
1330	if (u32HostExtFeatures & X86_CPUID_EXT_FEATURE_EDX_SYSCALL)
1331	{
1332	pMsr->u32IndexMSR = MSR_K6_STAR;
1333	pMsr->u32Reserved = 0;
1334	pMsr->u64Value = ASMRdMsr(MSR_K6_STAR); /* legacy syscall eip, cs & ss */
1335	pMsr++; idxMsr++;
1336	}
1337
1338	pMsr->u32IndexMSR = MSR_K6_EFER;
1339	pMsr->u32Reserved = 0;
1340	# if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
1341	if (CPUMIsGuestInLongMode(pVCpu))
1342	{
1343	/* Must match the EFER value in our 64 bits switcher. */
1344	pMsr->u64Value = ASMRdMsr(MSR_K6_EFER) \| MSR_K6_EFER_LME \| MSR_K6_EFER_SCE \| MSR_K6_EFER_NXE;
1345	}
1346	else
1347	# endif
1348	pMsr->u64Value = ASMRdMsr(MSR_K6_EFER);
1349	pMsr++; idxMsr++;
1350	}
1351
1352	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
1353	if (VMX_IS_64BIT_HOST_MODE())
1354	{
1355	pMsr->u32IndexMSR = MSR_K8_LSTAR;
1356	pMsr->u32Reserved = 0;
1357	pMsr->u64Value = ASMRdMsr(MSR_K8_LSTAR); /* 64 bits mode syscall rip */
1358	pMsr++; idxMsr++;
1359	pMsr->u32IndexMSR = MSR_K8_SF_MASK;
1360	pMsr->u32Reserved = 0;
1361	pMsr->u64Value = ASMRdMsr(MSR_K8_SF_MASK); /* syscall flag mask */
1362	pMsr++; idxMsr++;
1363
1364	/* The KERNEL_GS_BASE MSR doesn't work reliably with auto load/store. See @bugref{6208} */
1365	#if 0
1366	pMsr->u32IndexMSR = MSR_K8_KERNEL_GS_BASE;
1367	pMsr->u32Reserved = 0;
1368	pMsr->u64Value = ASMRdMsr(MSR_K8_KERNEL_GS_BASE); /* swapgs exchange value */
1369	pMsr++; idxMsr++;
1370	#endif
1371	}
1372	# endif
1373
1374	if (pVCpu->hwaccm.s.vmx.proc_ctls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
1375	{
1376	pMsr->u32IndexMSR = MSR_K8_TSC_AUX;
1377	pMsr->u32Reserved = 0;
1378	pMsr->u64Value = ASMRdMsr(MSR_K8_TSC_AUX);
1379	pMsr++; idxMsr++;
1380	}
1381
1382	/** @todo r=ramshankar: check IA32_VMX_MISC bits 27:25 for valid idxMsr
1383	* range. */
1384	rc = VMXWriteVMCS(VMX_VMCS_CTRL_EXIT_MSR_LOAD_COUNT, idxMsr);
1385	AssertRC(rc);
1386	#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
1387
1388	pVCpu->hwaccm.s.fContextUseFlags &= ~HWACCM_CHANGED_HOST_CONTEXT;
1389	}
1390	return rc;
1391	}
1392
1393
1394	/**
1395	* Loads the 4 PDPEs into the guest state when nested paging is used and the
1396	* guest operates in PAE mode.
1397	*
1398	* @returns VBox status code.
1399	* @param pVCpu Pointer to the VMCPU.
1400	* @param pCtx Pointer to the guest CPU context.
1401	*/
1402	static int hmR0VmxLoadPaePdpes(PVMCPU pVCpu, PCPUMCTX pCtx)
1403	{
1404	if (CPUMIsGuestInPAEModeEx(pCtx))
1405	{
1406	X86PDPE aPdpes[4];
1407	int rc = PGMGstGetPaePdpes(pVCpu, &aPdpes[0]);
1408	AssertRCReturn(rc, rc);
1409
1410	rc = VMXWriteVMCS64(VMX_VMCS_GUEST_PDPTR0_FULL, aPdpes[0].u); AssertRCReturn(rc, rc);
1411	rc = VMXWriteVMCS64(VMX_VMCS_GUEST_PDPTR1_FULL, aPdpes[1].u); AssertRCReturn(rc, rc);
1412	rc = VMXWriteVMCS64(VMX_VMCS_GUEST_PDPTR2_FULL, aPdpes[2].u); AssertRCReturn(rc, rc);
1413	rc = VMXWriteVMCS64(VMX_VMCS_GUEST_PDPTR3_FULL, aPdpes[3].u); AssertRCReturn(rc, rc);
1414	}
1415	return VINF_SUCCESS;
1416	}
1417
1418
1419	/**
1420	* Saves the 4 PDPEs into the guest state when nested paging is used and the
1421	* guest operates in PAE mode.
1422	*
1423	* @returns VBox status code.
1424	* @param pVCpu Pointer to the VM CPU.
1425	* @param pCtx Pointer to the guest CPU context.
1426	*
1427	* @remarks Tell PGM about CR3 changes before calling this helper.
1428	*/
1429	static int hmR0VmxSavePaePdpes(PVMCPU pVCpu, PCPUMCTX pCtx)
1430	{
1431	if (CPUMIsGuestInPAEModeEx(pCtx))
1432	{
1433	int rc;
1434	X86PDPE aPdpes[4];
1435	rc = VMXReadVMCS64(VMX_VMCS_GUEST_PDPTR0_FULL, &aPdpes[0].u); AssertRCReturn(rc, rc);
1436	rc = VMXReadVMCS64(VMX_VMCS_GUEST_PDPTR1_FULL, &aPdpes[1].u); AssertRCReturn(rc, rc);
1437	rc = VMXReadVMCS64(VMX_VMCS_GUEST_PDPTR2_FULL, &aPdpes[2].u); AssertRCReturn(rc, rc);
1438	rc = VMXReadVMCS64(VMX_VMCS_GUEST_PDPTR3_FULL, &aPdpes[3].u); AssertRCReturn(rc, rc);
1439
1440	rc = PGMGstUpdatePaePdpes(pVCpu, &aPdpes[0]);
1441	AssertRCReturn(rc, rc);
1442	}
1443	return VINF_SUCCESS;
1444	}
1445
1446
1447	/**
1448	* Update the exception bitmap according to the current CPU state.
1449	*
1450	* @param pVM Pointer to the VM.
1451	* @param pVCpu Pointer to the VMCPU.
1452	* @param pCtx Pointer to the guest CPU context.
1453	*/
1454	static void hmR0VmxUpdateExceptionBitmap(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
1455	{
1456	uint32_t u32TrapMask;
1457	Assert(pCtx);
1458
1459	/*
1460	* Set up a mask for intercepting traps.
1461	*/
1462	/** @todo Do we really need to always intercept #DB? */
1463	u32TrapMask = RT_BIT(X86_XCPT_DB)
1464	\| RT_BIT(X86_XCPT_NM)
1465	#ifdef VBOX_ALWAYS_TRAP_PF
1466	\| RT_BIT(X86_XCPT_PF)
1467	#endif
1468	#ifdef VBOX_STRICT
1469	\| RT_BIT(X86_XCPT_BP)
1470	\| RT_BIT(X86_XCPT_DB)
1471	\| RT_BIT(X86_XCPT_DE)
1472	\| RT_BIT(X86_XCPT_NM)
1473	\| RT_BIT(X86_XCPT_UD)
1474	\| RT_BIT(X86_XCPT_NP)
1475	\| RT_BIT(X86_XCPT_SS)
1476	\| RT_BIT(X86_XCPT_GP)
1477	\| RT_BIT(X86_XCPT_MF)
1478	#endif
1479	;
1480
1481	/*
1482	* Without nested paging, #PF must be intercepted to implement shadow paging.
1483	*/
1484	/** @todo NP state won't change so maybe we should build the initial trap mask up front? */
1485	if (!pVM->hwaccm.s.fNestedPaging)
1486	u32TrapMask \|= RT_BIT(X86_XCPT_PF);
1487
1488	/* Catch floating point exceptions if we need to report them to the guest in a different way. */
1489	if (!(pCtx->cr0 & X86_CR0_NE))
1490	u32TrapMask \|= RT_BIT(X86_XCPT_MF);
1491
1492	#ifdef VBOX_STRICT
1493	Assert(u32TrapMask & RT_BIT(X86_XCPT_GP));
1494	#endif
1495
1496	/*
1497	* Intercept all exceptions in real mode as none of them can be injected directly (#GP otherwise).
1498	*/
1499	/** @todo Despite the claim to intercept everything, with NP we do not intercept #PF. Should we? */
1500	if ( CPUMIsGuestInRealModeEx(pCtx)
1501	&& pVM->hwaccm.s.vmx.pRealModeTSS)
1502	{
1503	u32TrapMask \|= RT_BIT(X86_XCPT_DE)
1504	\| RT_BIT(X86_XCPT_DB)
1505	\| RT_BIT(X86_XCPT_NMI)
1506	\| RT_BIT(X86_XCPT_BP)
1507	\| RT_BIT(X86_XCPT_OF)
1508	\| RT_BIT(X86_XCPT_BR)
1509	\| RT_BIT(X86_XCPT_UD)
1510	\| RT_BIT(X86_XCPT_DF)
1511	\| RT_BIT(X86_XCPT_CO_SEG_OVERRUN)
1512	\| RT_BIT(X86_XCPT_TS)
1513	\| RT_BIT(X86_XCPT_NP)
1514	\| RT_BIT(X86_XCPT_SS)
1515	\| RT_BIT(X86_XCPT_GP)
1516	\| RT_BIT(X86_XCPT_MF)
1517	\| RT_BIT(X86_XCPT_AC)
1518	\| RT_BIT(X86_XCPT_MC)
1519	\| RT_BIT(X86_XCPT_XF)
1520	;
1521	}
1522
1523	int rc = VMXWriteVMCS(VMX_VMCS_CTRL_EXCEPTION_BITMAP, u32TrapMask);
1524	AssertRC(rc);
1525	}
1526
1527
1528	/**
1529	* Loads a minimal guest state.
1530	*
1531	* NOTE: Don't do anything here that can cause a jump back to ring 3!!!!!
1532	*
1533	* @param pVM Pointer to the VM.
1534	* @param pVCpu Pointer to the VMCPU.
1535	* @param pCtx Pointer to the guest CPU context.
1536	*/
1537	VMMR0DECL(void) VMXR0LoadMinimalGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
1538	{
1539	int rc;
1540	X86EFLAGS eflags;
1541
1542	Assert(!(pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_ALL_GUEST));
1543
1544	/*
1545	* Load EIP, ESP and EFLAGS.
1546	*/
1547	rc = VMXWriteVMCS64(VMX_VMCS64_GUEST_RIP, pCtx->rip);
1548	rc \|= VMXWriteVMCS64(VMX_VMCS64_GUEST_RSP, pCtx->rsp);
1549	AssertRC(rc);
1550
1551	/*
1552	* Bits 22-31, 15, 5 & 3 must be zero. Bit 1 must be 1.
1553	*/
1554	eflags = pCtx->eflags;
1555	eflags.u32 &= VMX_EFLAGS_RESERVED_0;
1556	eflags.u32 \|= VMX_EFLAGS_RESERVED_1;
1557
1558	/*
1559	* Check if real mode emulation using v86 mode.
1560	*/
1561	if ( CPUMIsGuestInRealModeEx(pCtx)
1562	&& pVM->hwaccm.s.vmx.pRealModeTSS)
1563	{
1564	pVCpu->hwaccm.s.vmx.RealMode.eflags = eflags;
1565
1566	eflags.Bits.u1VM = 1;
1567	eflags.Bits.u2IOPL = 0; /* must always be 0 or else certain instructions won't cause faults. */
1568	}
1569	rc = VMXWriteVMCS(VMX_VMCS_GUEST_RFLAGS, eflags.u32);
1570	AssertRC(rc);
1571	}
1572
1573
1574	/**
1575	* Loads the guest state.
1576	*
1577	* NOTE: Don't do anything here that can cause a jump back to ring 3!!!!!
1578	*
1579	* @returns VBox status code.
1580	* @param pVM Pointer to the VM.
1581	* @param pVCpu Pointer to the VMCPU.
1582	* @param pCtx Pointer to the guest CPU context.
1583	*/
1584	VMMR0DECL(int) VMXR0LoadGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
1585	{
1586	int rc = VINF_SUCCESS;
1587	RTGCUINTPTR val;
1588
1589	/*
1590	* VMX_VMCS_CTRL_ENTRY_CONTROLS
1591	* Set required bits to one and zero according to the MSR capabilities.
1592	*/
1593	val = pVM->hwaccm.s.vmx.msr.vmx_entry.n.disallowed0;
1594
1595	/*
1596	* Load guest debug controls (DR7 & IA32_DEBUGCTL_MSR).
1597	* Forced to 1 on the 'first' VT-x capable CPUs; this actually includes the newest Nehalem CPUs
1598	*/
1599	val \|= VMX_VMCS_CTRL_ENTRY_CONTROLS_LOAD_DEBUG;
1600
1601	if (CPUMIsGuestInLongModeEx(pCtx))
1602	val \|= VMX_VMCS_CTRL_ENTRY_CONTROLS_IA64_MODE;
1603	/* else Must be zero when AMD64 is not available. */
1604
1605	/*
1606	* Mask away the bits that the CPU doesn't support.
1607	*/
1608	val &= pVM->hwaccm.s.vmx.msr.vmx_entry.n.allowed1;
1609	rc = VMXWriteVMCS(VMX_VMCS_CTRL_ENTRY_CONTROLS, val);
1610	AssertRC(rc);
1611
1612	/*
1613	* VMX_VMCS_CTRL_EXIT_CONTROLS
1614	* Set required bits to one and zero according to the MSR capabilities.
1615	*/
1616	val = pVM->hwaccm.s.vmx.msr.vmx_exit.n.disallowed0;
1617
1618	/*
1619	* Save debug controls (DR7 & IA32_DEBUGCTL_MSR)
1620	* Forced to 1 on the 'first' VT-x capable CPUs; this actually includes the newest Nehalem CPUs
1621	*/
1622	val \|= VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_DEBUG;
1623
1624	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
1625	if (VMX_IS_64BIT_HOST_MODE())
1626	val \|= VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_AMD64;
1627	/* else Must be zero when AMD64 is not available. */
1628	#elif HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
1629	if (CPUMIsGuestInLongModeEx(pCtx))
1630	val \|= VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_AMD64; /* our switcher goes to long mode */
1631	else
1632	Assert(!(val & VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_AMD64));
1633	#endif
1634	val &= pVM->hwaccm.s.vmx.msr.vmx_exit.n.allowed1;
1635
1636	/*
1637	* Don't acknowledge external interrupts on VM-exit.
1638	*/
1639	rc = VMXWriteVMCS(VMX_VMCS_CTRL_EXIT_CONTROLS, val);
1640	AssertRC(rc);
1641
1642	/*
1643	* Guest CPU context: ES, CS, SS, DS, FS, GS.
1644	*/
1645	if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_SEGMENT_REGS)
1646	{
1647	if (pVM->hwaccm.s.vmx.pRealModeTSS)
1648	{
1649	PGMMODE enmGuestMode = PGMGetGuestMode(pVCpu);
1650	if (pVCpu->hwaccm.s.vmx.enmLastSeenGuestMode != enmGuestMode)
1651	{
1652	/*
1653	* Correct weird requirements for switching to protected mode.
1654	*/
1655	if ( pVCpu->hwaccm.s.vmx.enmLastSeenGuestMode == PGMMODE_REAL
1656	&& enmGuestMode >= PGMMODE_PROTECTED)
1657	{
1658	#ifdef VBOX_WITH_REM
1659	/*
1660	* Flush the recompiler code cache as it's not unlikely the guest will rewrite code
1661	* it will later execute in real mode (OpenBSD 4.0 is one such example)
1662	*/
1663	REMFlushTBs(pVM);
1664	#endif
1665
1666	/*
1667	* DPL of all hidden selector registers must match the current CPL (0).
1668	*/
1669	pCtx->cs.Attr.n.u2Dpl = 0;
1670	pCtx->cs.Attr.n.u4Type = X86_SEL_TYPE_CODE \| X86_SEL_TYPE_RW_ACC;
1671
1672	pCtx->ds.Attr.n.u2Dpl = 0;
1673	pCtx->es.Attr.n.u2Dpl = 0;
1674	pCtx->fs.Attr.n.u2Dpl = 0;
1675	pCtx->gs.Attr.n.u2Dpl = 0;
1676	pCtx->ss.Attr.n.u2Dpl = 0;
1677	}
1678	pVCpu->hwaccm.s.vmx.enmLastSeenGuestMode = enmGuestMode;
1679	}
1680	else if ( CPUMIsGuestInRealModeEx(pCtx)
1681	&& pCtx->cs.u64Base == 0xffff0000)
1682	{
1683	/* VT-x will fail with a guest invalid state otherwise... (CPU state after a reset) */
1684	pCtx->cs.u64Base = 0xf0000;
1685	pCtx->cs.Sel = 0xf000;
1686	}
1687	}
1688
1689	VMX_WRITE_SELREG(ES, es);
1690	AssertRC(rc);
1691
1692	VMX_WRITE_SELREG(CS, cs);
1693	AssertRC(rc);
1694
1695	VMX_WRITE_SELREG(SS, ss);
1696	AssertRC(rc);
1697
1698	VMX_WRITE_SELREG(DS, ds);
1699	AssertRC(rc);
1700
1701	VMX_WRITE_SELREG(FS, fs);
1702	AssertRC(rc);
1703
1704	VMX_WRITE_SELREG(GS, gs);
1705	AssertRC(rc);
1706	}
1707
1708	/*
1709	* Guest CPU context: LDTR.
1710	*/
1711	if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_LDTR)
1712	{
1713	if (pCtx->ldtr.Sel == 0)
1714	{
1715	rc = VMXWriteVMCS(VMX_VMCS16_GUEST_FIELD_LDTR, 0);
1716	rc \|= VMXWriteVMCS(VMX_VMCS32_GUEST_LDTR_LIMIT, 0);
1717	rc \|= VMXWriteVMCS64(VMX_VMCS64_GUEST_LDTR_BASE, 0);
1718	/* Note: vmlaunch will fail with 0 or just 0x02. No idea why. */
1719	rc \|= VMXWriteVMCS(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, 0x82 /* present, LDT */);
1720	}
1721	else
1722	{
1723	rc = VMXWriteVMCS(VMX_VMCS16_GUEST_FIELD_LDTR, pCtx->ldtr.Sel);
1724	rc \|= VMXWriteVMCS(VMX_VMCS32_GUEST_LDTR_LIMIT, pCtx->ldtr.u32Limit);
1725	rc \|= VMXWriteVMCS64(VMX_VMCS64_GUEST_LDTR_BASE, pCtx->ldtr.u64Base);
1726	rc \|= VMXWriteVMCS(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, pCtx->ldtr.Attr.u);
1727	}
1728	AssertRC(rc);
1729	}
1730
1731	/*
1732	* Guest CPU context: TR.
1733	*/
1734	if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_TR)
1735	{
1736	/*
1737	* Real mode emulation using v86 mode with CR4.VME (interrupt redirection
1738	* using the int bitmap in the TSS).
1739	*/
1740	if ( CPUMIsGuestInRealModeEx(pCtx)
1741	&& pVM->hwaccm.s.vmx.pRealModeTSS)
1742	{
1743	RTGCPHYS GCPhys;
1744
1745	/* We convert it here every time as PCI regions could be reconfigured. */
1746	rc = PDMVMMDevHeapR3ToGCPhys(pVM, pVM->hwaccm.s.vmx.pRealModeTSS, &GCPhys);
1747	AssertRC(rc);
1748
1749	rc = VMXWriteVMCS(VMX_VMCS16_GUEST_FIELD_TR, 0);
1750	rc \|= VMXWriteVMCS(VMX_VMCS32_GUEST_TR_LIMIT, HWACCM_VTX_TSS_SIZE);
1751	rc \|= VMXWriteVMCS64(VMX_VMCS64_GUEST_TR_BASE, GCPhys /* phys = virt in this mode */);
1752
1753	X86DESCATTR attr;
1754
1755	attr.u = 0;
1756	attr.n.u1Present = 1;
1757	attr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
1758	val = attr.u;
1759	}
1760	else
1761	{
1762	rc = VMXWriteVMCS(VMX_VMCS16_GUEST_FIELD_TR, pCtx->tr.Sel);
1763	rc \|= VMXWriteVMCS(VMX_VMCS32_GUEST_TR_LIMIT, pCtx->tr.u32Limit);
1764	rc \|= VMXWriteVMCS64(VMX_VMCS64_GUEST_TR_BASE, pCtx->tr.u64Base);
1765
1766	val = pCtx->tr.Attr.u;
1767
1768	/* The TSS selector must be busy (REM bugs? see defect #XXXX). */
1769	if (!(val & X86_SEL_TYPE_SYS_TSS_BUSY_MASK))
1770	{
1771	if (val & 0xf)
1772	val \|= X86_SEL_TYPE_SYS_TSS_BUSY_MASK;
1773	else
1774	/* Default if no TR selector has been set (otherwise vmlaunch will fail!) */
1775	val = (val & ~0xF) \| X86_SEL_TYPE_SYS_386_TSS_BUSY;
1776	}
1777	AssertMsg((val & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY \|\| (val & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY,
1778	("%#x\n", val));
1779	}
1780	rc \|= VMXWriteVMCS(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, val);
1781	AssertRC(rc);
1782	}
1783
1784	/*
1785	* Guest CPU context: GDTR.
1786	*/
1787	if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_GDTR)
1788	{
1789	rc = VMXWriteVMCS(VMX_VMCS32_GUEST_GDTR_LIMIT, pCtx->gdtr.cbGdt);
1790	rc \|= VMXWriteVMCS64(VMX_VMCS64_GUEST_GDTR_BASE, pCtx->gdtr.pGdt);
1791	AssertRC(rc);
1792	}
1793
1794	/*
1795	* Guest CPU context: IDTR.
1796	*/
1797	if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_IDTR)
1798	{
1799	rc = VMXWriteVMCS(VMX_VMCS32_GUEST_IDTR_LIMIT, pCtx->idtr.cbIdt);
1800	rc \|= VMXWriteVMCS64(VMX_VMCS64_GUEST_IDTR_BASE, pCtx->idtr.pIdt);
1801	AssertRC(rc);
1802	}
1803
1804	/*
1805	* Sysenter MSRs.
1806	*/
1807	if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_MSR)
1808	{
1809	rc = VMXWriteVMCS(VMX_VMCS32_GUEST_SYSENTER_CS, pCtx->SysEnter.cs);
1810	rc \|= VMXWriteVMCS64(VMX_VMCS64_GUEST_SYSENTER_EIP, pCtx->SysEnter.eip);
1811	rc \|= VMXWriteVMCS64(VMX_VMCS64_GUEST_SYSENTER_ESP, pCtx->SysEnter.esp);
1812	AssertRC(rc);
1813	}
1814
1815	/*
1816	* Guest CPU context: Control registers.
1817	*/
1818	if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_CR0)
1819	{
1820	val = pCtx->cr0;
1821	rc = VMXWriteVMCS(VMX_VMCS_CTRL_CR0_READ_SHADOW, val);
1822	Log2(("Guest CR0-shadow %08x\n", val));
1823	if (CPUMIsGuestFPUStateActive(pVCpu) == false)
1824	{
1825	/* Always use #NM exceptions to load the FPU/XMM state on demand. */
1826	val \|= X86_CR0_TS \| X86_CR0_ET \| X86_CR0_NE \| X86_CR0_MP;
1827	}
1828	else
1829	{
1830	/** @todo check if we support the old style mess correctly. */
1831	if (!(val & X86_CR0_NE))
1832	Log(("Forcing X86_CR0_NE!!!\n"));
1833
1834	val \|= X86_CR0_NE; /* always turn on the native mechanism to report FPU errors (old style uses interrupts) */
1835	}
1836	/* Protected mode & paging are always enabled; we use them for emulating real and protected mode without paging too. */
1837	if (!pVM->hwaccm.s.vmx.fUnrestrictedGuest)
1838	val \|= X86_CR0_PE \| X86_CR0_PG;
1839
1840	if (pVM->hwaccm.s.fNestedPaging)
1841	{
1842	if (CPUMIsGuestInPagedProtectedModeEx(pCtx))
1843	{
1844	/* Disable CR3 read/write monitoring as we don't need it for EPT. */
1845	pVCpu->hwaccm.s.vmx.proc_ctls &= ~( VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
1846	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT);
1847	}
1848	else
1849	{
1850	/* Reenable CR3 read/write monitoring as our identity mapped page table is active. */
1851	pVCpu->hwaccm.s.vmx.proc_ctls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
1852	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT;
1853	}
1854	rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
1855	AssertRC(rc);
1856	}
1857	else
1858	{
1859	/* Note: We must also set this as we rely on protecting various pages for which supervisor writes must be caught. */
1860	val \|= X86_CR0_WP;
1861	}
1862
1863	/* Always enable caching. */
1864	val &= ~(X86_CR0_CD\|X86_CR0_NW);
1865
1866	rc \|= VMXWriteVMCS64(VMX_VMCS64_GUEST_CR0, val);
1867	Log2(("Guest CR0 %08x\n", val));
1868
1869	/*
1870	* CR0 flags owned by the host; if the guests attempts to change them, then the VM will exit.
1871	*/
1872	val = X86_CR0_PE /* Must monitor this bit (assumptions are made for real mode emulation) */
1873	\| X86_CR0_WP /* Must monitor this bit (it must always be enabled). */
1874	\| X86_CR0_PG /* Must monitor this bit (assumptions are made for real mode & protected mode without paging emulation) */
1875	\| X86_CR0_CD /* Bit not restored during VM-exit! */
1876	\| X86_CR0_NW /* Bit not restored during VM-exit! */
1877	\| X86_CR0_NE;
1878
1879	/*
1880	* When the guest's FPU state is active, then we no longer care about the FPU related bits.
1881	*/
1882	if (CPUMIsGuestFPUStateActive(pVCpu) == false)
1883	val \|= X86_CR0_TS \| X86_CR0_ET \| X86_CR0_MP;
1884
1885	pVCpu->hwaccm.s.vmx.cr0_mask = val;
1886
1887	rc \|= VMXWriteVMCS(VMX_VMCS_CTRL_CR0_MASK, val);
1888	Log2(("Guest CR0-mask %08x\n", val));
1889	AssertRC(rc);
1890	}
1891
1892	if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_CR4)
1893	{
1894	rc = VMXWriteVMCS(VMX_VMCS_CTRL_CR4_READ_SHADOW, pCtx->cr4);
1895	Log2(("Guest CR4-shadow %08x\n", pCtx->cr4));
1896	/* Set the required bits in cr4 too (currently X86_CR4_VMXE). */
1897	val = pCtx->cr4 \| (uint32_t)pVM->hwaccm.s.vmx.msr.vmx_cr4_fixed0;
1898
1899	if (!pVM->hwaccm.s.fNestedPaging)
1900	{
1901	switch(pVCpu->hwaccm.s.enmShadowMode)
1902	{
1903	case PGMMODE_REAL: /* Real mode -> emulated using v86 mode */
1904	case PGMMODE_PROTECTED: /* Protected mode, no paging -> emulated using identity mapping. */
1905	case PGMMODE_32_BIT: /* 32-bit paging. */
1906	val &= ~X86_CR4_PAE;
1907	break;
1908
1909	case PGMMODE_PAE: /* PAE paging. */
1910	case PGMMODE_PAE_NX: /* PAE paging with NX enabled. */
1911	/** Must use PAE paging as we could use physical memory > 4 GB */
1912	val \|= X86_CR4_PAE;
1913	break;
1914
1915	case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
1916	case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
1917	#ifdef VBOX_ENABLE_64_BITS_GUESTS
1918	break;
1919	#else
1920	AssertFailed();
1921	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
1922	#endif
1923	default: /* shut up gcc */
1924	AssertFailed();
1925	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
1926	}
1927	}
1928	else if ( !CPUMIsGuestInPagedProtectedModeEx(pCtx)
1929	&& !pVM->hwaccm.s.vmx.fUnrestrictedGuest)
1930	{
1931	/* We use 4 MB pages in our identity mapping page table for real and protected mode without paging. */
1932	val \|= X86_CR4_PSE;
1933	/* Our identity mapping is a 32 bits page directory. */
1934	val &= ~X86_CR4_PAE;
1935	}
1936
1937	/*
1938	* Turn off VME if we're in emulated real mode.
1939	*/
1940	if ( CPUMIsGuestInRealModeEx(pCtx)
1941	&& pVM->hwaccm.s.vmx.pRealModeTSS)
1942	{
1943	val &= ~X86_CR4_VME;
1944	}
1945
1946	rc \|= VMXWriteVMCS64(VMX_VMCS64_GUEST_CR4, val);
1947	Log2(("Guest CR4 %08x\n", val));
1948
1949	/*
1950	* CR4 flags owned by the host; if the guests attempts to change them, then the VM will exit.
1951	*/
1952	val = 0
1953	\| X86_CR4_VME
1954	\| X86_CR4_PAE
1955	\| X86_CR4_PGE
1956	\| X86_CR4_PSE
1957	\| X86_CR4_VMXE;
1958	pVCpu->hwaccm.s.vmx.cr4_mask = val;
1959
1960	rc \|= VMXWriteVMCS(VMX_VMCS_CTRL_CR4_MASK, val);
1961	Log2(("Guest CR4-mask %08x\n", val));
1962	AssertRC(rc);
1963	}
1964
1965	#if 0
1966	/* Enable single stepping if requested and CPU supports it. */
1967	if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_TRAP_FLAG)
1968	if (DBGFIsStepping(pVCpu))
1969	{
1970	pVCpu->hwaccm.s.vmx.proc_ctls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_TRAP_FLAG;
1971	rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
1972	AssertRC(rc);
1973	}
1974	#endif
1975
1976	if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_CR3)
1977	{
1978	if (pVM->hwaccm.s.fNestedPaging)
1979	{
1980	Assert(PGMGetHyperCR3(pVCpu));
1981	pVCpu->hwaccm.s.vmx.GCPhysEPTP = PGMGetHyperCR3(pVCpu);
1982
1983	Assert(!(pVCpu->hwaccm.s.vmx.GCPhysEPTP & 0xfff));
1984	/** @todo Check the IA32_VMX_EPT_VPID_CAP MSR for other supported memory types. */
1985	pVCpu->hwaccm.s.vmx.GCPhysEPTP \|= VMX_EPT_MEMTYPE_WB
1986	\| (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
1987
1988	rc = VMXWriteVMCS64(VMX_VMCS_CTRL_EPTP_FULL, pVCpu->hwaccm.s.vmx.GCPhysEPTP);
1989	AssertRC(rc);
1990
1991	if ( !CPUMIsGuestInPagedProtectedModeEx(pCtx)
1992	&& !pVM->hwaccm.s.vmx.fUnrestrictedGuest)
1993	{
1994	RTGCPHYS GCPhys;
1995
1996	/* We convert it here every time as PCI regions could be reconfigured. */
1997	rc = PDMVMMDevHeapR3ToGCPhys(pVM, pVM->hwaccm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
1998	AssertMsgRC(rc, ("pNonPagingModeEPTPageTable = %RGv\n", pVM->hwaccm.s.vmx.pNonPagingModeEPTPageTable));
1999
2000	/*
2001	* We use our identity mapping page table here as we need to map guest virtual to
2002	* guest physical addresses; EPT will take care of the translation to host physical addresses.
2003	*/
2004	val = GCPhys;
2005	}
2006	else
2007	{
2008	/* Save the real guest CR3 in VMX_VMCS_GUEST_CR3 */
2009	val = pCtx->cr3;
2010	rc = hmR0VmxLoadPaePdpes(pVCpu, pCtx);
2011	AssertRCReturn(rc, rc);
2012	}
2013	}
2014	else
2015	{
2016	val = PGMGetHyperCR3(pVCpu);
2017	Assert(val \|\| VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL));
2018	}
2019
2020	/* Save our shadow CR3 register. */
2021	rc = VMXWriteVMCS64(VMX_VMCS64_GUEST_CR3, val);
2022	AssertRC(rc);
2023	}
2024
2025	/*
2026	* Guest CPU context: Debug registers.
2027	*/
2028	if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_DEBUG)
2029	{
2030	pCtx->dr[6] \|= X86_DR6_INIT_VAL; /* set all reserved bits to 1. */
2031	pCtx->dr[6] &= ~RT_BIT(12); /* must be zero. */
2032
2033	pCtx->dr[7] &= 0xffffffff; /* upper 32 bits reserved */
2034	pCtx->dr[7] &= ~(RT_BIT(11) \| RT_BIT(12) \| RT_BIT(14) \| RT_BIT(15)); /* must be zero */
2035	pCtx->dr[7] \|= 0x400; /* must be one */
2036
2037	/* Resync DR7 */
2038	rc = VMXWriteVMCS64(VMX_VMCS64_GUEST_DR7, pCtx->dr[7]);
2039	AssertRC(rc);
2040
2041	#ifdef DEBUG
2042	/* Sync the hypervisor debug state now if any breakpoint is armed. */
2043	if ( CPUMGetHyperDR7(pVCpu) & (X86_DR7_ENABLED_MASK\|X86_DR7_GD)
2044	&& !CPUMIsHyperDebugStateActive(pVCpu)
2045	&& !DBGFIsStepping(pVCpu))
2046	{
2047	/* Save the host and load the hypervisor debug state. */
2048	rc = CPUMR0LoadHyperDebugState(pVM, pVCpu, pCtx, true /* include DR6 */);
2049	AssertRC(rc);
2050
2051	/* DRx intercepts remain enabled. */
2052
2053	/* Override dr7 with the hypervisor value. */
2054	rc = VMXWriteVMCS64(VMX_VMCS64_GUEST_DR7, CPUMGetHyperDR7(pVCpu));
2055	AssertRC(rc);
2056	}
2057	else
2058	#endif
2059	/* Sync the debug state now if any breakpoint is armed. */
2060	if ( (pCtx->dr[7] & (X86_DR7_ENABLED_MASK\|X86_DR7_GD))
2061	&& !CPUMIsGuestDebugStateActive(pVCpu)
2062	&& !DBGFIsStepping(pVCpu))
2063	{
2064	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatDRxArmed);
2065
2066	/* Disable DRx move intercepts. */
2067	pVCpu->hwaccm.s.vmx.proc_ctls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT;
2068	rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
2069	AssertRC(rc);
2070
2071	/* Save the host and load the guest debug state. */
2072	rc = CPUMR0LoadGuestDebugState(pVM, pVCpu, pCtx, true /* include DR6 */);
2073	AssertRC(rc);
2074	}
2075
2076	/* IA32_DEBUGCTL MSR. */
2077	rc = VMXWriteVMCS64(VMX_VMCS_GUEST_DEBUGCTL_FULL, 0);
2078	AssertRC(rc);
2079
2080	/** @todo do we really ever need this? */
2081	rc \|= VMXWriteVMCS(VMX_VMCS_GUEST_DEBUG_EXCEPTIONS, 0);
2082	AssertRC(rc);
2083	}
2084
2085	/*
2086	* 64-bit guest mode.
2087	*/
2088	if (CPUMIsGuestInLongModeEx(pCtx))
2089	{
2090	#if !defined(VBOX_ENABLE_64_BITS_GUESTS)
2091	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
2092	#elif HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2093	pVCpu->hwaccm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
2094	#else
2095	# ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2096	if (!pVM->hwaccm.s.fAllow64BitGuests)
2097	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
2098	# endif
2099	pVCpu->hwaccm.s.vmx.pfnStartVM = VMXR0StartVM64;
2100	#endif
2101	if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_MSR)
2102	{
2103	/* Update these as wrmsr might have changed them. */
2104	rc = VMXWriteVMCS64(VMX_VMCS64_GUEST_FS_BASE, pCtx->fs.u64Base);
2105	AssertRC(rc);
2106	rc = VMXWriteVMCS64(VMX_VMCS64_GUEST_GS_BASE, pCtx->gs.u64Base);
2107	AssertRC(rc);
2108	}
2109	}
2110	else
2111	{
2112	pVCpu->hwaccm.s.vmx.pfnStartVM = VMXR0StartVM32;
2113	}
2114
2115	hmR0VmxUpdateExceptionBitmap(pVM, pVCpu, pCtx);
2116
2117	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
2118	/*
2119	* Store all guest MSRs in the VM-entry load area, so they will be loaded
2120	* during VM-entry and restored into the VM-exit store area during VM-exit.
2121	*/
2122	PVMXMSR pMsr = (PVMXMSR)pVCpu->hwaccm.s.vmx.pGuestMSR;
2123	unsigned idxMsr = 0;
2124
2125	uint32_t u32GstExtFeatures;
2126	uint32_t u32Temp;
2127	CPUMGetGuestCpuId(pVCpu, 0x80000001, &u32Temp, &u32Temp, &u32Temp, &u32GstExtFeatures);
2128
2129	/*
2130	* Check if EFER MSR present.
2131	*/
2132	if (u32GstExtFeatures & (X86_CPUID_EXT_FEATURE_EDX_NX \| X86_CPUID_EXT_FEATURE_EDX_LONG_MODE))
2133	{
2134	pMsr->u32IndexMSR = MSR_K6_EFER;
2135	pMsr->u32Reserved = 0;
2136	pMsr->u64Value = pCtx->msrEFER;
2137	/* VT-x will complain if only MSR_K6_EFER_LME is set. */
2138	if (!CPUMIsGuestInLongModeEx(pCtx))
2139	pMsr->u64Value &= ~(MSR_K6_EFER_LMA \| MSR_K6_EFER_LME);
2140	pMsr++; idxMsr++;
2141
2142	if (u32GstExtFeatures & X86_CPUID_EXT_FEATURE_EDX_LONG_MODE)
2143	{
2144	pMsr->u32IndexMSR = MSR_K8_LSTAR;
2145	pMsr->u32Reserved = 0;
2146	pMsr->u64Value = pCtx->msrLSTAR; /* 64 bits mode syscall rip */
2147	pMsr++; idxMsr++;
2148	pMsr->u32IndexMSR = MSR_K6_STAR;
2149	pMsr->u32Reserved = 0;
2150	pMsr->u64Value = pCtx->msrSTAR; /* legacy syscall eip, cs & ss */
2151	pMsr++; idxMsr++;
2152	pMsr->u32IndexMSR = MSR_K8_SF_MASK;
2153	pMsr->u32Reserved = 0;
2154	pMsr->u64Value = pCtx->msrSFMASK; /* syscall flag mask */
2155	pMsr++; idxMsr++;
2156
2157	/* The KERNEL_GS_BASE MSR doesn't work reliably with auto load/store. See @bugref{6208} */
2158	#if 0
2159	pMsr->u32IndexMSR = MSR_K8_KERNEL_GS_BASE;
2160	pMsr->u32Reserved = 0;
2161	pMsr->u64Value = pCtx->msrKERNELGSBASE; /* swapgs exchange value */
2162	pMsr++; idxMsr++;
2163	#endif
2164	}
2165	}
2166
2167	if ( pVCpu->hwaccm.s.vmx.proc_ctls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP
2168	&& (u32GstExtFeatures & X86_CPUID_EXT_FEATURE_EDX_RDTSCP))
2169	{
2170	pMsr->u32IndexMSR = MSR_K8_TSC_AUX;
2171	pMsr->u32Reserved = 0;
2172	rc = CPUMQueryGuestMsr(pVCpu, MSR_K8_TSC_AUX, &pMsr->u64Value);
2173	AssertRC(rc);
2174	pMsr++; idxMsr++;
2175	}
2176
2177	pVCpu->hwaccm.s.vmx.cCachedMSRs = idxMsr;
2178
2179	rc = VMXWriteVMCS(VMX_VMCS_CTRL_ENTRY_MSR_LOAD_COUNT, idxMsr);
2180	AssertRC(rc);
2181
2182	rc = VMXWriteVMCS(VMX_VMCS_CTRL_EXIT_MSR_STORE_COUNT, idxMsr);
2183	AssertRC(rc);
2184	#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
2185
2186	bool fOffsettedTsc;
2187	if (pVM->hwaccm.s.vmx.fUsePreemptTimer)
2188	{
2189	uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVCpu, &fOffsettedTsc, &pVCpu->hwaccm.s.vmx.u64TSCOffset);
2190
2191	/* Make sure the returned values have sane upper and lower boundaries. */
2192	uint64_t u64CpuHz = SUPGetCpuHzFromGIP(g_pSUPGlobalInfoPage);
2193
2194	cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64 of a second */
2195	cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
2196
2197	cTicksToDeadline >>= pVM->hwaccm.s.vmx.cPreemptTimerShift;
2198	uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
2199	rc = VMXWriteVMCS(VMX_VMCS32_GUEST_PREEMPTION_TIMER_VALUE, cPreemptionTickCount);
2200	AssertRC(rc);
2201	}
2202	else
2203	fOffsettedTsc = TMCpuTickCanUseRealTSC(pVCpu, &pVCpu->hwaccm.s.vmx.u64TSCOffset);
2204
2205	if (fOffsettedTsc)
2206	{
2207	uint64_t u64CurTSC = ASMReadTSC();
2208	if (u64CurTSC + pVCpu->hwaccm.s.vmx.u64TSCOffset >= TMCpuTickGetLastSeen(pVCpu))
2209	{
2210	/* Note: VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT takes precedence over TSC_OFFSET, applies to RDTSCP too. */
2211	rc = VMXWriteVMCS64(VMX_VMCS_CTRL_TSC_OFFSET_FULL, pVCpu->hwaccm.s.vmx.u64TSCOffset);
2212	AssertRC(rc);
2213
2214	pVCpu->hwaccm.s.vmx.proc_ctls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT;
2215	rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
2216	AssertRC(rc);
2217	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatTSCOffset);
2218	}
2219	else
2220	{
2221	/* Fall back to rdtsc, rdtscp emulation as we would otherwise pass decreasing tsc values to the guest. */
2222	LogFlow(("TSC %RX64 offset %RX64 time=%RX64 last=%RX64 (diff=%RX64, virt_tsc=%RX64)\n", u64CurTSC,
2223	pVCpu->hwaccm.s.vmx.u64TSCOffset, u64CurTSC + pVCpu->hwaccm.s.vmx.u64TSCOffset,
2224	TMCpuTickGetLastSeen(pVCpu), TMCpuTickGetLastSeen(pVCpu) - u64CurTSC - pVCpu->hwaccm.s.vmx.u64TSCOffset,
2225	TMCpuTickGet(pVCpu)));
2226	pVCpu->hwaccm.s.vmx.proc_ctls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT;
2227	rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
2228	AssertRC(rc);
2229	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatTSCInterceptOverFlow);
2230	}
2231	}
2232	else
2233	{
2234	pVCpu->hwaccm.s.vmx.proc_ctls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT;
2235	rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
2236	AssertRC(rc);
2237	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatTSCIntercept);
2238	}
2239
2240	/* Done with the major changes */
2241	pVCpu->hwaccm.s.fContextUseFlags &= ~HWACCM_CHANGED_ALL_GUEST;
2242
2243	/* Minimal guest state update (ESP, EIP, EFLAGS mostly) */
2244	VMXR0LoadMinimalGuestState(pVM, pVCpu, pCtx);
2245	return rc;
2246	}
2247
2248
2249	/**
2250	* Syncs back the guest state from VMCS.
2251	*
2252	* @returns VBox status code.
2253	* @param pVM Pointer to the VM.
2254	* @param pVCpu Pointer to the VMCPU.
2255	* @param pCtx Pointer to the guest CPU context.
2256	*/
2257	DECLINLINE(int) VMXR0SaveGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
2258	{
2259	RTGCUINTREG val, valShadow;
2260	RTGCUINTPTR uInterruptState;
2261	int rc;
2262
2263	/* First sync back EIP, ESP, and EFLAGS. */
2264	rc = VMXReadCachedVMCS(VMX_VMCS64_GUEST_RIP, &val);
2265	AssertRC(rc);
2266	pCtx->rip = val;
2267	rc = VMXReadCachedVMCS(VMX_VMCS64_GUEST_RSP, &val);
2268	AssertRC(rc);
2269	pCtx->rsp = val;
2270	rc = VMXReadCachedVMCS(VMX_VMCS_GUEST_RFLAGS, &val);
2271	AssertRC(rc);
2272	pCtx->eflags.u32 = val;
2273
2274	/* Take care of instruction fusing (sti, mov ss) */
2275	rc \|= VMXReadCachedVMCS(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &val);
2276	uInterruptState = val;
2277	if (uInterruptState != 0)
2278	{
2279	Assert(uInterruptState <= 2); /* only sti & mov ss */
2280	Log(("uInterruptState %x eip=%RGv\n", (uint32_t)uInterruptState, pCtx->rip));
2281	EMSetInhibitInterruptsPC(pVCpu, pCtx->rip);
2282	}
2283	else
2284	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
2285
2286	/* Control registers. */
2287	VMXReadCachedVMCS(VMX_VMCS_CTRL_CR0_READ_SHADOW, &valShadow);
2288	VMXReadCachedVMCS(VMX_VMCS64_GUEST_CR0, &val);
2289	val = (valShadow & pVCpu->hwaccm.s.vmx.cr0_mask) \| (val & ~pVCpu->hwaccm.s.vmx.cr0_mask);
2290	CPUMSetGuestCR0(pVCpu, val);
2291
2292	VMXReadCachedVMCS(VMX_VMCS_CTRL_CR4_READ_SHADOW, &valShadow);
2293	VMXReadCachedVMCS(VMX_VMCS64_GUEST_CR4, &val);
2294	val = (valShadow & pVCpu->hwaccm.s.vmx.cr4_mask) \| (val & ~pVCpu->hwaccm.s.vmx.cr4_mask);
2295	CPUMSetGuestCR4(pVCpu, val);
2296
2297	/*
2298	* No reason to sync back the CRx registers. They can't be changed by the guest unless in
2299	* the nested paging case where CR3 & CR4 can be changed by the guest.
2300	*/
2301	if ( pVM->hwaccm.s.fNestedPaging
2302	&& CPUMIsGuestInPagedProtectedModeEx(pCtx)) /** @todo check if we will always catch mode switches and such... */
2303	{
2304	PVMCSCACHE pCache = &pVCpu->hwaccm.s.vmx.VMCSCache;
2305
2306	/* Can be updated behind our back in the nested paging case. */
2307	CPUMSetGuestCR2(pVCpu, pCache->cr2);
2308
2309	VMXReadCachedVMCS(VMX_VMCS64_GUEST_CR3, &val);
2310
2311	if (val != pCtx->cr3)
2312	{
2313	CPUMSetGuestCR3(pVCpu, val);
2314	PGMUpdateCR3(pVCpu, val);
2315	}
2316	rc = hmR0VmxSavePaePdpes(pVCpu, pCtx);
2317	AssertRCReturn(rc, rc);
2318	}
2319
2320	/* Sync back DR7. */
2321	VMXReadCachedVMCS(VMX_VMCS64_GUEST_DR7, &val);
2322	pCtx->dr[7] = val;
2323
2324	/* Guest CPU context: ES, CS, SS, DS, FS, GS. */
2325	VMX_READ_SELREG(ES, es);
2326	VMX_READ_SELREG(SS, ss);
2327	VMX_READ_SELREG(CS, cs);
2328	VMX_READ_SELREG(DS, ds);
2329	VMX_READ_SELREG(FS, fs);
2330	VMX_READ_SELREG(GS, gs);
2331
2332	/* System MSRs */
2333	VMXReadCachedVMCS(VMX_VMCS32_GUEST_SYSENTER_CS, &val);
2334	pCtx->SysEnter.cs = val;
2335	VMXReadCachedVMCS(VMX_VMCS64_GUEST_SYSENTER_EIP, &val);
2336	pCtx->SysEnter.eip = val;
2337	VMXReadCachedVMCS(VMX_VMCS64_GUEST_SYSENTER_ESP, &val);
2338	pCtx->SysEnter.esp = val;
2339
2340	/* Misc. registers; must sync everything otherwise we can get out of sync when jumping to ring 3. */
2341	VMX_READ_SELREG(LDTR, ldtr);
2342
2343	VMXReadCachedVMCS(VMX_VMCS32_GUEST_GDTR_LIMIT, &val);
2344	pCtx->gdtr.cbGdt = val;
2345	VMXReadCachedVMCS(VMX_VMCS64_GUEST_GDTR_BASE, &val);
2346	pCtx->gdtr.pGdt = val;
2347
2348	VMXReadCachedVMCS(VMX_VMCS32_GUEST_IDTR_LIMIT, &val);
2349	pCtx->idtr.cbIdt = val;
2350	VMXReadCachedVMCS(VMX_VMCS64_GUEST_IDTR_BASE, &val);
2351	pCtx->idtr.pIdt = val;
2352
2353	/* Real mode emulation using v86 mode. */
2354	if ( CPUMIsGuestInRealModeEx(pCtx)
2355	&& pVM->hwaccm.s.vmx.pRealModeTSS)
2356	{
2357	/* Hide our emulation flags */
2358	pCtx->eflags.Bits.u1VM = 0;
2359
2360	/* Restore original IOPL setting as we always use 0. */
2361	pCtx->eflags.Bits.u2IOPL = pVCpu->hwaccm.s.vmx.RealMode.eflags.Bits.u2IOPL;
2362
2363	/* Force a TR resync every time in case we switch modes. */
2364	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_GUEST_TR;
2365	}
2366	else
2367	{
2368	/* In real mode we have a fake TSS, so only sync it back when it's supposed to be valid. */
2369	VMX_READ_SELREG(TR, tr);
2370	}
2371
2372	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
2373	/*
2374	* Save the possibly changed MSRs that we automatically restore and save during a world switch.
2375	*/
2376	for (unsigned i = 0; i < pVCpu->hwaccm.s.vmx.cCachedMSRs; i++)
2377	{
2378	PVMXMSR pMsr = (PVMXMSR)pVCpu->hwaccm.s.vmx.pGuestMSR;
2379	pMsr += i;
2380
2381	switch (pMsr->u32IndexMSR)
2382	{
2383	case MSR_K8_LSTAR:
2384	pCtx->msrLSTAR = pMsr->u64Value;
2385	break;
2386	case MSR_K6_STAR:
2387	pCtx->msrSTAR = pMsr->u64Value;
2388	break;
2389	case MSR_K8_SF_MASK:
2390	pCtx->msrSFMASK = pMsr->u64Value;
2391	break;
2392
2393	/* The KERNEL_GS_BASE MSR doesn't work reliably with auto load/store. See @bugref{6208} */
2394	#if 0
2395	case MSR_K8_KERNEL_GS_BASE:
2396	pCtx->msrKERNELGSBASE = pMsr->u64Value;
2397	break;
2398	#endif
2399	case MSR_K8_TSC_AUX:
2400	CPUMSetGuestMsr(pVCpu, MSR_K8_TSC_AUX, pMsr->u64Value);
2401	break;
2402
2403	case MSR_K6_EFER:
2404	/* EFER can't be changed without causing a VM-exit. */
2405	/* Assert(pCtx->msrEFER == pMsr->u64Value); */
2406	break;
2407	default:
2408	AssertFailed();
2409	return VERR_HM_UNEXPECTED_LD_ST_MSR;
2410	}
2411	}
2412	#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
2413	return VINF_SUCCESS;
2414	}
2415
2416
2417	/**
2418	* Dummy placeholder for TLB flush handling before VM-entry. Used in the case
2419	* where neither EPT nor VPID is supported by the CPU.
2420	*
2421	* @param pVM Pointer to the VM.
2422	* @param pVCpu Pointer to the VMCPU.
2423	*/
2424	static DECLCALLBACK(void) hmR0VmxSetupTLBDummy(PVM pVM, PVMCPU pVCpu)
2425	{
2426	NOREF(pVM);
2427	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
2428	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
2429	pVCpu->hwaccm.s.TlbShootdown.cPages = 0;
2430	return;
2431	}
2432
2433
2434	/**
2435	* Setup the tagged TLB for EPT+VPID.
2436	*
2437	* @param pVM Pointer to the VM.
2438	* @param pVCpu Pointer to the VMCPU.
2439	*/
2440	static DECLCALLBACK(void) hmR0VmxSetupTLBBoth(PVM pVM, PVMCPU pVCpu)
2441	{
2442	PHMGLOBLCPUINFO pCpu;
2443
2444	Assert(pVM->hwaccm.s.fNestedPaging && pVM->hwaccm.s.vmx.fVPID);
2445
2446	pCpu = HWACCMR0GetCurrentCpu();
2447
2448	/*
2449	* Force a TLB flush for the first world switch if the current CPU differs from the one we ran on last
2450	* This can happen both for start & resume due to long jumps back to ring-3.
2451	* If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
2452	* or the host Cpu is online after a suspend/resume, so we cannot reuse the current ASID anymore.
2453	*/
2454	bool fNewASID = false;
2455	if ( pVCpu->hwaccm.s.idLastCpu != pCpu->idCpu
2456	\|\| pVCpu->hwaccm.s.cTLBFlushes != pCpu->cTLBFlushes)
2457	{
2458	pVCpu->hwaccm.s.fForceTLBFlush = true;
2459	fNewASID = true;
2460	}
2461
2462	/*
2463	* Check for explicit TLB shootdowns.
2464	*/
2465	if (VMCPU_FF_TESTANDCLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2466	pVCpu->hwaccm.s.fForceTLBFlush = true;
2467
2468	pVCpu->hwaccm.s.idLastCpu = pCpu->idCpu;
2469
2470	if (pVCpu->hwaccm.s.fForceTLBFlush)
2471	{
2472	if (fNewASID)
2473	{
2474	++pCpu->uCurrentASID;
2475	if (pCpu->uCurrentASID >= pVM->hwaccm.s.uMaxASID)
2476	{
2477	pCpu->uCurrentASID = 1; /* start at 1; host uses 0 */
2478	pCpu->cTLBFlushes++;
2479	pCpu->fFlushASIDBeforeUse = true;
2480	}
2481
2482	pVCpu->hwaccm.s.uCurrentASID = pCpu->uCurrentASID;
2483	if (pCpu->fFlushASIDBeforeUse)
2484	{
2485	hmR0VmxFlushVPID(pVM, pVCpu, pVM->hwaccm.s.vmx.enmFlushVPID, 0 /* GCPtr */);
2486	#ifdef VBOX_WITH_STATISTICS
2487	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatFlushASID);
2488	#endif
2489	}
2490	}
2491	else
2492	{
2493	if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVVPID_CAPS_SINGLE_CONTEXT)
2494	hmR0VmxFlushVPID(pVM, pVCpu, VMX_FLUSH_VPID_SINGLE_CONTEXT, 0 /* GCPtr */);
2495	else
2496	hmR0VmxFlushEPT(pVM, pVCpu, pVM->hwaccm.s.vmx.enmFlushEPT);
2497
2498	#ifdef VBOX_WITH_STATISTICS
2499	/*
2500	* This is not terribly accurate (i.e. we don't have any StatFlushEPT counter). We currently count these
2501	* as ASID flushes too, better than including them under StatFlushTLBWorldSwitch.
2502	*/
2503	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatFlushASID);
2504	#endif
2505	}
2506
2507	pVCpu->hwaccm.s.cTLBFlushes = pCpu->cTLBFlushes;
2508	pVCpu->hwaccm.s.fForceTLBFlush = false;
2509	}
2510	else
2511	{
2512	AssertMsg(pVCpu->hwaccm.s.uCurrentASID && pCpu->uCurrentASID,
2513	("hwaccm->uCurrentASID=%lu hwaccm->cTLBFlushes=%lu cpu->uCurrentASID=%lu cpu->cTLBFlushes=%lu\n",
2514	pVCpu->hwaccm.s.uCurrentASID, pVCpu->hwaccm.s.cTLBFlushes,
2515	pCpu->uCurrentASID, pCpu->cTLBFlushes));
2516
2517	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
2518	* not be executed. See hwaccmQueueInvlPage() where it is commented
2519	* out. Support individual entry flushing someday. */
2520	if (VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
2521	{
2522	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatTlbShootdown);
2523
2524	/*
2525	* Flush individual guest entries using VPID from the TLB or as little as possible with EPT
2526	* as supported by the CPU.
2527	*/
2528	if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVVPID_CAPS_INDIV_ADDR)
2529	{
2530	for (unsigned i = 0; i < pVCpu->hwaccm.s.TlbShootdown.cPages; i++)
2531	hmR0VmxFlushVPID(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, pVCpu->hwaccm.s.TlbShootdown.aPages[i]);
2532	}
2533	else
2534	hmR0VmxFlushEPT(pVM, pVCpu, pVM->hwaccm.s.vmx.enmFlushEPT);
2535	}
2536	else
2537	{
2538	#ifdef VBOX_WITH_STATISTICS
2539	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatNoFlushTLBWorldSwitch);
2540	#endif
2541	}
2542	}
2543	pVCpu->hwaccm.s.TlbShootdown.cPages = 0;
2544	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
2545
2546	AssertMsg(pVCpu->hwaccm.s.cTLBFlushes == pCpu->cTLBFlushes,
2547	("Flush count mismatch for cpu %d (%x vs %x)\n", pCpu->idCpu, pVCpu->hwaccm.s.cTLBFlushes, pCpu->cTLBFlushes));
2548	AssertMsg(pCpu->uCurrentASID >= 1 && pCpu->uCurrentASID < pVM->hwaccm.s.uMaxASID,
2549	("cpu%d uCurrentASID = %x\n", pCpu->idCpu, pCpu->uCurrentASID));
2550	AssertMsg(pVCpu->hwaccm.s.uCurrentASID >= 1 && pVCpu->hwaccm.s.uCurrentASID < pVM->hwaccm.s.uMaxASID,
2551	("cpu%d VM uCurrentASID = %x\n", pCpu->idCpu, pVCpu->hwaccm.s.uCurrentASID));
2552
2553	/* Update VMCS with the VPID. */
2554	int rc = VMXWriteVMCS(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hwaccm.s.uCurrentASID);
2555	AssertRC(rc);
2556	}
2557
2558
2559	/**
2560	* Setup the tagged TLB for EPT only.
2561	*
2562	* @returns VBox status code.
2563	* @param pVM Pointer to the VM.
2564	* @param pVCpu Pointer to the VMCPU.
2565	*/
2566	static DECLCALLBACK(void) hmR0VmxSetupTLBEPT(PVM pVM, PVMCPU pVCpu)
2567	{
2568	PHMGLOBLCPUINFO pCpu;
2569
2570	Assert(pVM->hwaccm.s.fNestedPaging);
2571	Assert(!pVM->hwaccm.s.vmx.fVPID);
2572
2573	pCpu = HWACCMR0GetCurrentCpu();
2574
2575	/*
2576	* Force a TLB flush for the first world switch if the current CPU differs from the one we ran on last
2577	* This can happen both for start & resume due to long jumps back to ring-3.
2578	* A change in the TLB flush count implies the host Cpu is online after a suspend/resume.
2579	*/
2580	if ( pVCpu->hwaccm.s.idLastCpu != pCpu->idCpu
2581	\|\| pVCpu->hwaccm.s.cTLBFlushes != pCpu->cTLBFlushes)
2582	{
2583	pVCpu->hwaccm.s.fForceTLBFlush = true;
2584	}
2585
2586	/*
2587	* Check for explicit TLB shootdown flushes.
2588	*/
2589	if (VMCPU_FF_TESTANDCLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2590	pVCpu->hwaccm.s.fForceTLBFlush = true;
2591
2592	pVCpu->hwaccm.s.idLastCpu = pCpu->idCpu;
2593	pVCpu->hwaccm.s.cTLBFlushes = pCpu->cTLBFlushes;
2594
2595	if (pVCpu->hwaccm.s.fForceTLBFlush)
2596	hmR0VmxFlushEPT(pVM, pVCpu, pVM->hwaccm.s.vmx.enmFlushEPT);
2597	else
2598	{
2599	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
2600	* not be executed. See hwaccmQueueInvlPage() where it is commented
2601	* out. Support individual entry flushing someday. */
2602	if (VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
2603	{
2604	/*
2605	* We cannot flush individual entries without VPID support. Flush using EPT.
2606	*/
2607	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatTlbShootdown);
2608	hmR0VmxFlushEPT(pVM, pVCpu, pVM->hwaccm.s.vmx.enmFlushEPT);
2609	}
2610	}
2611	pVCpu->hwaccm.s.TlbShootdown.cPages= 0;
2612	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
2613
2614	#ifdef VBOX_WITH_STATISTICS
2615	if (pVCpu->hwaccm.s.fForceTLBFlush)
2616	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatFlushTLBWorldSwitch);
2617	else
2618	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatNoFlushTLBWorldSwitch);
2619	#endif
2620	}
2621
2622
2623	/**
2624	* Setup the tagged TLB for VPID.
2625	*
2626	* @returns VBox status code.
2627	* @param pVM Pointer to the VM.
2628	* @param pVCpu Pointer to the VMCPU.
2629	*/
2630	static DECLCALLBACK(void) hmR0VmxSetupTLBVPID(PVM pVM, PVMCPU pVCpu)
2631	{
2632	PHMGLOBLCPUINFO pCpu;
2633
2634	Assert(pVM->hwaccm.s.vmx.fVPID);
2635	Assert(!pVM->hwaccm.s.fNestedPaging);
2636
2637	pCpu = HWACCMR0GetCurrentCpu();
2638
2639	/*
2640	* Force a TLB flush for the first world switch if the current CPU differs from the one we ran on last
2641	* This can happen both for start & resume due to long jumps back to ring-3.
2642	* If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
2643	* or the host Cpu is online after a suspend/resume, so we cannot reuse the current ASID anymore.
2644	*/
2645	if ( pVCpu->hwaccm.s.idLastCpu != pCpu->idCpu
2646	\|\| pVCpu->hwaccm.s.cTLBFlushes != pCpu->cTLBFlushes)
2647	{
2648	/* Force a TLB flush on VM entry. */
2649	pVCpu->hwaccm.s.fForceTLBFlush = true;
2650	}
2651
2652	/*
2653	* Check for explicit TLB shootdown flushes.
2654	*/
2655	if (VMCPU_FF_TESTANDCLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2656	pVCpu->hwaccm.s.fForceTLBFlush = true;
2657
2658	pVCpu->hwaccm.s.idLastCpu = pCpu->idCpu;
2659
2660	if (pVCpu->hwaccm.s.fForceTLBFlush)
2661	{
2662	++pCpu->uCurrentASID;
2663	if (pCpu->uCurrentASID >= pVM->hwaccm.s.uMaxASID)
2664	{
2665	pCpu->uCurrentASID = 1; /* start at 1; host uses 0 */
2666	pCpu->cTLBFlushes++;
2667	pCpu->fFlushASIDBeforeUse = true;
2668	}
2669	else
2670	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatFlushASID);
2671
2672	pVCpu->hwaccm.s.fForceTLBFlush = false;
2673	pVCpu->hwaccm.s.cTLBFlushes = pCpu->cTLBFlushes;
2674	pVCpu->hwaccm.s.uCurrentASID = pCpu->uCurrentASID;
2675	if (pCpu->fFlushASIDBeforeUse)
2676	hmR0VmxFlushVPID(pVM, pVCpu, pVM->hwaccm.s.vmx.enmFlushVPID, 0 /* GCPtr */);
2677	}
2678	else
2679	{
2680	AssertMsg(pVCpu->hwaccm.s.uCurrentASID && pCpu->uCurrentASID,
2681	("hwaccm->uCurrentASID=%lu hwaccm->cTLBFlushes=%lu cpu->uCurrentASID=%lu cpu->cTLBFlushes=%lu\n",
2682	pVCpu->hwaccm.s.uCurrentASID, pVCpu->hwaccm.s.cTLBFlushes,
2683	pCpu->uCurrentASID, pCpu->cTLBFlushes));
2684
2685	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
2686	* not be executed. See hwaccmQueueInvlPage() where it is commented
2687	* out. Support individual entry flushing someday. */
2688	if (VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
2689	{
2690	/*
2691	* Flush individual guest entries using VPID from the TLB or as little as possible with EPT
2692	* as supported by the CPU.
2693	*/
2694	if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVVPID_CAPS_INDIV_ADDR)
2695	{
2696	for (unsigned i = 0; i < pVCpu->hwaccm.s.TlbShootdown.cPages; i++)
2697	hmR0VmxFlushVPID(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, pVCpu->hwaccm.s.TlbShootdown.aPages[i]);
2698	}
2699	else
2700	hmR0VmxFlushVPID(pVM, pVCpu, pVM->hwaccm.s.vmx.enmFlushVPID, 0 /* GCPtr */);
2701	}
2702	}
2703	pVCpu->hwaccm.s.TlbShootdown.cPages = 0;
2704	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
2705
2706	AssertMsg(pVCpu->hwaccm.s.cTLBFlushes == pCpu->cTLBFlushes,
2707	("Flush count mismatch for cpu %d (%x vs %x)\n", pCpu->idCpu, pVCpu->hwaccm.s.cTLBFlushes, pCpu->cTLBFlushes));
2708	AssertMsg(pCpu->uCurrentASID >= 1 && pCpu->uCurrentASID < pVM->hwaccm.s.uMaxASID,
2709	("cpu%d uCurrentASID = %x\n", pCpu->idCpu, pCpu->uCurrentASID));
2710	AssertMsg(pVCpu->hwaccm.s.uCurrentASID >= 1 && pVCpu->hwaccm.s.uCurrentASID < pVM->hwaccm.s.uMaxASID,
2711	("cpu%d VM uCurrentASID = %x\n", pCpu->idCpu, pVCpu->hwaccm.s.uCurrentASID));
2712
2713	int rc = VMXWriteVMCS(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hwaccm.s.uCurrentASID);
2714	AssertRC(rc);
2715
2716	# ifdef VBOX_WITH_STATISTICS
2717	if (pVCpu->hwaccm.s.fForceTLBFlush)
2718	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatFlushTLBWorldSwitch);
2719	else
2720	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatNoFlushTLBWorldSwitch);
2721	# endif
2722	}
2723
2724
2725	/**
2726	* Runs guest code in a VT-x VM.
2727	*
2728	* @returns VBox status code.
2729	* @param pVM Pointer to the VM.
2730	* @param pVCpu Pointer to the VMCPU.
2731	* @param pCtx Pointer to the guest CPU context.
2732	*/
2733	VMMR0DECL(int) VMXR0RunGuestCode(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
2734	{
2735	STAM_PROFILE_ADV_START(&pVCpu->hwaccm.s.StatEntry, x);
2736	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hwaccm.s.StatExit1);
2737	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hwaccm.s.StatExit2);
2738
2739	VBOXSTRICTRC rc = VINF_SUCCESS;
2740	int rc2;
2741	RTGCUINTREG val;
2742	RTGCUINTREG exitReason = (RTGCUINTREG)VMX_EXIT_INVALID;
2743	RTGCUINTREG instrError, cbInstr;
2744	RTGCUINTPTR exitQualification = 0;
2745	RTGCUINTPTR intInfo = 0; /* shut up buggy gcc 4 */
2746	RTGCUINTPTR errCode, instrInfo;
2747	bool fSetupTPRCaching = false;
2748	uint64_t u64OldLSTAR = 0;
2749	uint8_t u8LastTPR = 0;
2750	RTCCUINTREG uOldEFlags = ~(RTCCUINTREG)0;
2751	unsigned cResume = 0;
2752	#ifdef VBOX_STRICT
2753	RTCPUID idCpuCheck;
2754	bool fWasInLongMode = false;
2755	#endif
2756	#ifdef VBOX_HIGH_RES_TIMERS_HACK_IN_RING0
2757	uint64_t u64LastTime = RTTimeMilliTS();
2758	#endif
2759
2760	Assert(!(pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
2761	\|\| (pVCpu->hwaccm.s.vmx.pbVAPIC && pVM->hwaccm.s.vmx.pAPIC));
2762
2763	/*
2764	* Check if we need to use TPR shadowing.
2765	*/
2766	if ( CPUMIsGuestInLongModeEx(pCtx)
2767	\|\| ( (( pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
2768	\|\| pVM->hwaccm.s.fTRPPatchingAllowed)
2769	&& pVM->hwaccm.s.fHasIoApic)
2770	)
2771	{
2772	fSetupTPRCaching = true;
2773	}
2774
2775	Log2(("\nE"));
2776
2777	#ifdef VBOX_STRICT
2778	{
2779	RTCCUINTREG val2;
2780
2781	rc2 = VMXReadVMCS(VMX_VMCS_CTRL_PIN_EXEC_CONTROLS, &val2);
2782	AssertRC(rc2);
2783	Log2(("VMX_VMCS_CTRL_PIN_EXEC_CONTROLS = %08x\n", val2));
2784
2785	/* allowed zero */
2786	if ((val2 & pVM->hwaccm.s.vmx.msr.vmx_pin_ctls.n.disallowed0) != pVM->hwaccm.s.vmx.msr.vmx_pin_ctls.n.disallowed0)
2787	Log(("Invalid VMX_VMCS_CTRL_PIN_EXEC_CONTROLS: zero\n"));
2788
2789	/* allowed one */
2790	if ((val2 & ~pVM->hwaccm.s.vmx.msr.vmx_pin_ctls.n.allowed1) != 0)
2791	Log(("Invalid VMX_VMCS_CTRL_PIN_EXEC_CONTROLS: one\n"));
2792
2793	rc2 = VMXReadVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, &val2);
2794	AssertRC(rc2);
2795	Log2(("VMX_VMCS_CTRL_PROC_EXEC_CONTROLS = %08x\n", val2));
2796
2797	/*
2798	* Must be set according to the MSR, but can be cleared if nested paging is used.
2799	*/
2800	if (pVM->hwaccm.s.fNestedPaging)
2801	{
2802	val2 \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INVLPG_EXIT
2803	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
2804	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT;
2805	}
2806
2807	/* allowed zero */
2808	if ((val2 & pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.disallowed0) != pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.disallowed0)
2809	Log(("Invalid VMX_VMCS_CTRL_PROC_EXEC_CONTROLS: zero\n"));
2810
2811	/* allowed one */
2812	if ((val2 & ~pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1) != 0)
2813	Log(("Invalid VMX_VMCS_CTRL_PROC_EXEC_CONTROLS: one\n"));
2814
2815	rc2 = VMXReadVMCS(VMX_VMCS_CTRL_ENTRY_CONTROLS, &val2);
2816	AssertRC(rc2);
2817	Log2(("VMX_VMCS_CTRL_ENTRY_CONTROLS = %08x\n", val2));
2818
2819	/* allowed zero */
2820	if ((val2 & pVM->hwaccm.s.vmx.msr.vmx_entry.n.disallowed0) != pVM->hwaccm.s.vmx.msr.vmx_entry.n.disallowed0)
2821	Log(("Invalid VMX_VMCS_CTRL_ENTRY_CONTROLS: zero\n"));
2822
2823	/* allowed one */
2824	if ((val2 & ~pVM->hwaccm.s.vmx.msr.vmx_entry.n.allowed1) != 0)
2825	Log(("Invalid VMX_VMCS_CTRL_ENTRY_CONTROLS: one\n"));
2826
2827	rc2 = VMXReadVMCS(VMX_VMCS_CTRL_EXIT_CONTROLS, &val2);
2828	AssertRC(rc2);
2829	Log2(("VMX_VMCS_CTRL_EXIT_CONTROLS = %08x\n", val2));
2830
2831	/* allowed zero */
2832	if ((val2 & pVM->hwaccm.s.vmx.msr.vmx_exit.n.disallowed0) != pVM->hwaccm.s.vmx.msr.vmx_exit.n.disallowed0)
2833	Log(("Invalid VMX_VMCS_CTRL_EXIT_CONTROLS: zero\n"));
2834
2835	/* allowed one */
2836	if ((val2 & ~pVM->hwaccm.s.vmx.msr.vmx_exit.n.allowed1) != 0)
2837	Log(("Invalid VMX_VMCS_CTRL_EXIT_CONTROLS: one\n"));
2838	}
2839	fWasInLongMode = CPUMIsGuestInLongModeEx(pCtx);
2840	#endif /* VBOX_STRICT */
2841
2842	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
2843	pVCpu->hwaccm.s.vmx.VMCSCache.u64TimeEntry = RTTimeNanoTS();
2844	#endif
2845
2846	/*
2847	* We can jump to this point to resume execution after determining that a VM-exit is innocent.
2848	*/
2849	ResumeExecution:
2850	if (!STAM_REL_PROFILE_ADV_IS_RUNNING(&pVCpu->hwaccm.s.StatEntry))
2851	STAM_REL_PROFILE_ADV_STOP_START(&pVCpu->hwaccm.s.StatExit2, &pVCpu->hwaccm.s.StatEntry, x);
2852	AssertMsg(pVCpu->hwaccm.s.idEnteredCpu == RTMpCpuId(),
2853	("Expected %d, I'm %d; cResume=%d exitReason=%RGv exitQualification=%RGv\n",
2854	(int)pVCpu->hwaccm.s.idEnteredCpu, (int)RTMpCpuId(), cResume, exitReason, exitQualification));
2855	Assert(!HWACCMR0SuspendPending());
2856	/* Not allowed to switch modes without reloading the host state (32->64 switcher)!! */
2857	Assert(fWasInLongMode == CPUMIsGuestInLongModeEx(pCtx));
2858
2859	/*
2860	* Safety precaution; looping for too long here can have a very bad effect on the host.
2861	*/
2862	if (RT_UNLIKELY(++cResume > pVM->hwaccm.s.cMaxResumeLoops))
2863	{
2864	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitMaxResume);
2865	rc = VINF_EM_RAW_INTERRUPT;
2866	goto end;
2867	}
2868
2869	/*
2870	* Check for IRQ inhibition due to instruction fusing (sti, mov ss).
2871	*/
2872	if (VMCPU_FF_ISSET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
2873	{
2874	Log(("VM_FF_INHIBIT_INTERRUPTS at %RGv successor %RGv\n", (RTGCPTR)pCtx->rip, EMGetInhibitInterruptsPC(pVCpu)));
2875	if (pCtx->rip != EMGetInhibitInterruptsPC(pVCpu))
2876	{
2877	/*
2878	* Note: we intentionally don't clear VM_FF_INHIBIT_INTERRUPTS here.
2879	* Before we are able to execute this instruction in raw mode (iret to guest code) an external interrupt might
2880	* force a world switch again. Possibly allowing a guest interrupt to be dispatched in the process. This could
2881	* break the guest. Sounds very unlikely, but such timing sensitive problems are not as rare as you might think.
2882	*/
2883	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
2884	/* Irq inhibition is no longer active; clear the corresponding VMX state. */
2885	rc2 = VMXWriteVMCS(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, 0);
2886	AssertRC(rc2);
2887	}
2888	}
2889	else
2890	{
2891	/* Irq inhibition is no longer active; clear the corresponding VMX state. */
2892	rc2 = VMXWriteVMCS(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, 0);
2893	AssertRC(rc2);
2894	}
2895
2896	#ifdef VBOX_HIGH_RES_TIMERS_HACK_IN_RING0
2897	if (RT_UNLIKELY((cResume & 0xf) == 0))
2898	{
2899	uint64_t u64CurTime = RTTimeMilliTS();
2900
2901	if (RT_UNLIKELY(u64CurTime > u64LastTime))
2902	{
2903	u64LastTime = u64CurTime;
2904	TMTimerPollVoid(pVM, pVCpu);
2905	}
2906	}
2907	#endif
2908
2909	/*
2910	* Check for pending actions that force us to go back to ring-3.
2911	*/
2912	if ( VM_FF_ISPENDING(pVM, VM_FF_HWACCM_TO_R3_MASK \| VM_FF_REQUEST \| VM_FF_PGM_POOL_FLUSH_PENDING \| VM_FF_PDM_DMA)
2913	\|\| VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_HWACCM_TO_R3_MASK \| VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL \| VMCPU_FF_REQUEST))
2914	{
2915	/* Check if a sync operation is pending. */
2916	if (VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
2917	{
2918	rc = PGMSyncCR3(pVCpu, pCtx->cr0, pCtx->cr3, pCtx->cr4, VMCPU_FF_ISSET(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
2919	if (rc != VINF_SUCCESS)
2920	{
2921	AssertRC(VBOXSTRICTRC_VAL(rc));
2922	Log(("Pending pool sync is forcing us back to ring 3; rc=%d\n", VBOXSTRICTRC_VAL(rc)));
2923	goto end;
2924	}
2925	}
2926
2927	#ifdef DEBUG
2928	/* Intercept X86_XCPT_DB if stepping is enabled */
2929	if (!DBGFIsStepping(pVCpu))
2930	#endif
2931	{
2932	if ( VM_FF_ISPENDING(pVM, VM_FF_HWACCM_TO_R3_MASK)
2933	\|\| VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_HWACCM_TO_R3_MASK))
2934	{
2935	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatSwitchToR3);
2936	rc = RT_UNLIKELY(VM_FF_ISPENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
2937	goto end;
2938	}
2939	}
2940
2941	/* Pending request packets might contain actions that need immediate attention, such as pending hardware interrupts. */
2942	if ( VM_FF_ISPENDING(pVM, VM_FF_REQUEST)
2943	\|\| VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_REQUEST))
2944	{
2945	rc = VINF_EM_PENDING_REQUEST;
2946	goto end;
2947	}
2948
2949	/* Check if a pgm pool flush is in progress. */
2950	if (VM_FF_ISPENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
2951	{
2952	rc = VINF_PGM_POOL_FLUSH_PENDING;
2953	goto end;
2954	}
2955
2956	/* Check if DMA work is pending (2nd+ run). */
2957	if (VM_FF_ISPENDING(pVM, VM_FF_PDM_DMA) && cResume > 1)
2958	{
2959	rc = VINF_EM_RAW_TO_R3;
2960	goto end;
2961	}
2962	}
2963
2964	#ifdef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
2965	/*
2966	* Exit to ring-3 preemption/work is pending.
2967	*
2968	* Interrupts are disabled before the call to make sure we don't miss any interrupt
2969	* that would flag preemption (IPI, timer tick, ++). (Would've been nice to do this
2970	* further down, but hmR0VmxCheckPendingInterrupt makes that impossible.)
2971	*
2972	* Note! Interrupts must be disabled done before we check for TLB flushes; TLB
2973	* shootdowns rely on this.
2974	*/
2975	uOldEFlags = ASMIntDisableFlags();
2976	if (RTThreadPreemptIsPending(NIL_RTTHREAD))
2977	{
2978	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitPreemptPending);
2979	rc = VINF_EM_RAW_INTERRUPT;
2980	goto end;
2981	}
2982	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
2983	#endif
2984
2985	/*
2986	* When external interrupts are pending, we should exit the VM when IF is et.
2987	* Note: After VM_FF_INHIBIT_INTERRUPTS check!
2988	*/
2989	rc = hmR0VmxCheckPendingInterrupt(pVM, pVCpu, pCtx);
2990	if (RT_FAILURE(rc))
2991	goto end;
2992
2993	/** @todo check timers?? */
2994
2995	/*
2996	* TPR caching using CR8 is only available in 64-bit mode.
2997	* Note: The 32-bit exception for AMD (X86_CPUID_AMD_FEATURE_ECX_CR8L), but this appears missing in Intel CPUs.
2998	* Note: We can't do this in LoadGuestState() as PDMApicGetTPR can jump back to ring-3 (lock)!! (no longer true) .
2999	*/
3000	/** @todo query and update the TPR only when it could have been changed (mmio
3001	* access & wrsmr (x2apic) */
3002	if (fSetupTPRCaching)
3003	{
3004	/* TPR caching in CR8 */
3005	bool fPending;
3006
3007	rc2 = PDMApicGetTPR(pVCpu, &u8LastTPR, &fPending);
3008	AssertRC(rc2);
3009	/* The TPR can be found at offset 0x80 in the APIC mmio page. */
3010	pVCpu->hwaccm.s.vmx.pbVAPIC[0x80] = u8LastTPR;
3011
3012	/*
3013	* Two options here:
3014	* - external interrupt pending, but masked by the TPR value.
3015	* -> a CR8 update that lower the current TPR value should cause an exit
3016	* - no pending interrupts
3017	* -> We don't need to be explicitely notified. There are enough world switches for detecting pending interrupts.
3018	*/
3019
3020	/* cr8 bits 3-0 correspond to bits 7-4 of the task priority mmio register. */
3021	rc = VMXWriteVMCS(VMX_VMCS_CTRL_TPR_THRESHOLD, (fPending) ? (u8LastTPR >> 4) : 0);
3022	AssertRC(VBOXSTRICTRC_VAL(rc));
3023
3024	if (pVM->hwaccm.s.fTPRPatchingActive)
3025	{
3026	Assert(!CPUMIsGuestInLongModeEx(pCtx));
3027	/* Our patch code uses LSTAR for TPR caching. */
3028	pCtx->msrLSTAR = u8LastTPR;
3029
3030	/** @todo r=ramshankar: we should check for MSR-bitmap support here. */
3031	if (fPending)
3032	{
3033	/* A TPR change could activate a pending interrupt, so catch lstar writes. */
3034	hmR0VmxSetMSRPermission(pVCpu, MSR_K8_LSTAR, true, false);
3035	}
3036	else
3037	{
3038	/*
3039	* No interrupts are pending, so we don't need to be explicitely notified.
3040	* There are enough world switches for detecting pending interrupts.
3041	*/
3042	hmR0VmxSetMSRPermission(pVCpu, MSR_K8_LSTAR, true, true);
3043	}
3044	}
3045	}
3046
3047	#ifdef LOG_ENABLED
3048	if ( pVM->hwaccm.s.fNestedPaging
3049	\|\| pVM->hwaccm.s.vmx.fVPID)
3050	{
3051	PHMGLOBLCPUINFO pCpu = HWACCMR0GetCurrentCpu();
3052	if (pVCpu->hwaccm.s.idLastCpu != pCpu->idCpu)
3053	{
3054	LogFlow(("Force TLB flush due to rescheduling to a different cpu (%d vs %d)\n", pVCpu->hwaccm.s.idLastCpu,
3055	pCpu->idCpu));
3056	}
3057	else if (pVCpu->hwaccm.s.cTLBFlushes != pCpu->cTLBFlushes)
3058	{
3059	LogFlow(("Force TLB flush due to changed TLB flush count (%x vs %x)\n", pVCpu->hwaccm.s.cTLBFlushes,
3060	pCpu->cTLBFlushes));
3061	}
3062	else if (VMCPU_FF_ISSET(pVCpu, VMCPU_FF_TLB_FLUSH))
3063	LogFlow(("Manual TLB flush\n"));
3064	}
3065	#endif
3066	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
3067	PGMRZDynMapFlushAutoSet(pVCpu);
3068	#endif
3069
3070	/*
3071	* NOTE: DO NOT DO ANYTHING AFTER THIS POINT THAT MIGHT JUMP BACK TO RING-3!
3072	* (until the actual world switch)
3073	*/
3074	#ifdef VBOX_STRICT
3075	idCpuCheck = RTMpCpuId();
3076	#endif
3077	#ifdef LOG_ENABLED
3078	VMMR0LogFlushDisable(pVCpu);
3079	#endif
3080
3081	/*
3082	* Save the host state first.
3083	*/
3084	if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_HOST_CONTEXT)
3085	{
3086	rc = VMXR0SaveHostState(pVM, pVCpu);
3087	if (RT_UNLIKELY(rc != VINF_SUCCESS))
3088	{
3089	VMMR0LogFlushEnable(pVCpu);
3090	goto end;
3091	}
3092	}
3093
3094	/*
3095	* Load the guest state.
3096	*/
3097	if (!pVCpu->hwaccm.s.fContextUseFlags)
3098	{
3099	VMXR0LoadMinimalGuestState(pVM, pVCpu, pCtx);
3100	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatLoadMinimal);
3101	}
3102	else
3103	{
3104	rc = VMXR0LoadGuestState(pVM, pVCpu, pCtx);
3105	if (RT_UNLIKELY(rc != VINF_SUCCESS))
3106	{
3107	VMMR0LogFlushEnable(pVCpu);
3108	goto end;
3109	}
3110	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatLoadFull);
3111	}
3112
3113	#ifndef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
3114	/*
3115	* Disable interrupts to make sure a poke will interrupt execution.
3116	* This must be done before we check for TLB flushes; TLB shootdowns rely on this.
3117	*/
3118	uOldEFlags = ASMIntDisableFlags();
3119	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
3120	#endif
3121
3122	/* Non-register state Guest Context */
3123	/** @todo change me according to cpu state */
3124	rc2 = VMXWriteVMCS(VMX_VMCS32_GUEST_ACTIVITY_STATE, VMX_CMS_GUEST_ACTIVITY_ACTIVE);
3125	AssertRC(rc2);
3126
3127	/* Set TLB flush state as checked until we return from the world switch. */
3128	ASMAtomicWriteBool(&pVCpu->hwaccm.s.fCheckedTLBFlush, true);
3129	/* Deal with tagged TLB setup and invalidation. */
3130	pVM->hwaccm.s.vmx.pfnSetupTaggedTLB(pVM, pVCpu);
3131
3132	/*
3133	* Manual save and restore:
3134	* - General purpose registers except RIP, RSP
3135	*
3136	* Trashed:
3137	* - CR2 (we don't care)
3138	* - LDTR (reset to 0)
3139	* - DRx (presumably not changed at all)
3140	* - DR7 (reset to 0x400)
3141	* - EFLAGS (reset to RT_BIT(1); not relevant)
3142	*/
3143
3144	/* All done! Let's start VM execution. */
3145	STAM_PROFILE_ADV_STOP_START(&pVCpu->hwaccm.s.StatEntry, &pVCpu->hwaccm.s.StatInGC, x);
3146	Assert(idCpuCheck == RTMpCpuId());
3147
3148	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
3149	pVCpu->hwaccm.s.vmx.VMCSCache.cResume = cResume;
3150	pVCpu->hwaccm.s.vmx.VMCSCache.u64TimeSwitch = RTTimeNanoTS();
3151	#endif
3152
3153	/*
3154	* Save the current TPR value in the LSTAR MSR so our patches can access it.
3155	*/
3156	if (pVM->hwaccm.s.fTPRPatchingActive)
3157	{
3158	Assert(pVM->hwaccm.s.fTPRPatchingActive);
3159	u64OldLSTAR = ASMRdMsr(MSR_K8_LSTAR);
3160	ASMWrMsr(MSR_K8_LSTAR, u8LastTPR);
3161	}
3162
3163	TMNotifyStartOfExecution(pVCpu);
3164
3165	#ifndef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
3166	/*
3167	* Save the current Host TSC_AUX and write the guest TSC_AUX to the host, so that
3168	* RDTSCPs (that don't cause exits) reads the guest MSR. See @bugref{3324}.
3169	*/
3170	if ( (pVCpu->hwaccm.s.vmx.proc_ctls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
3171	&& !(pVCpu->hwaccm.s.vmx.proc_ctls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT))
3172	{
3173	pVCpu->hwaccm.s.u64HostTSCAux = ASMRdMsr(MSR_K8_TSC_AUX);
3174	uint64_t u64GuestTSCAux = 0;
3175	rc2 = CPUMQueryGuestMsr(pVCpu, MSR_K8_TSC_AUX, &u64GuestTSCAux);
3176	AssertRC(rc2);
3177	ASMWrMsr(MSR_K8_TSC_AUX, u64GuestTSCAux);
3178	}
3179	#endif
3180
3181	#ifdef VBOX_WITH_KERNEL_USING_XMM
3182	rc = hwaccmR0VMXStartVMWrapXMM(pVCpu->hwaccm.s.fResumeVM, pCtx, &pVCpu->hwaccm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hwaccm.s.vmx.pfnStartVM);
3183	#else
3184	rc = pVCpu->hwaccm.s.vmx.pfnStartVM(pVCpu->hwaccm.s.fResumeVM, pCtx, &pVCpu->hwaccm.s.vmx.VMCSCache, pVM, pVCpu);
3185	#endif
3186	ASMAtomicWriteBool(&pVCpu->hwaccm.s.fCheckedTLBFlush, false);
3187	ASMAtomicIncU32(&pVCpu->hwaccm.s.cWorldSwitchExits);
3188
3189	/* Possibly the last TSC value seen by the guest (too high) (only when we're in TSC offset mode). */
3190	if (!(pVCpu->hwaccm.s.vmx.proc_ctls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT))
3191	{
3192	#ifndef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
3193	/* Restore host's TSC_AUX. */
3194	if (pVCpu->hwaccm.s.vmx.proc_ctls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
3195	ASMWrMsr(MSR_K8_TSC_AUX, pVCpu->hwaccm.s.u64HostTSCAux);
3196	#endif
3197
3198	TMCpuTickSetLastSeen(pVCpu,
3199	ASMReadTSC() + pVCpu->hwaccm.s.vmx.u64TSCOffset - 0x400 /* guestimate of world switch overhead in clock ticks */);
3200	}
3201
3202	TMNotifyEndOfExecution(pVCpu);
3203	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED);
3204	Assert(!(ASMGetFlags() & X86_EFL_IF));
3205
3206	/*
3207	* Restore the host LSTAR MSR if the guest could have changed it.
3208	*/
3209	if (pVM->hwaccm.s.fTPRPatchingActive)
3210	{
3211	Assert(pVM->hwaccm.s.fTPRPatchingActive);
3212	pVCpu->hwaccm.s.vmx.pbVAPIC[0x80] = pCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR);
3213	ASMWrMsr(MSR_K8_LSTAR, u64OldLSTAR);
3214	}
3215
3216	STAM_PROFILE_ADV_STOP_START(&pVCpu->hwaccm.s.StatInGC, &pVCpu->hwaccm.s.StatExit1, x);
3217	ASMSetFlags(uOldEFlags);
3218	#ifdef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
3219	uOldEFlags = ~(RTCCUINTREG)0;
3220	#endif
3221
3222	AssertMsg(!pVCpu->hwaccm.s.vmx.VMCSCache.Write.cValidEntries, ("pVCpu->hwaccm.s.vmx.VMCSCache.Write.cValidEntries=%d\n",
3223	pVCpu->hwaccm.s.vmx.VMCSCache.Write.cValidEntries));
3224
3225	/* In case we execute a goto ResumeExecution later on. */
3226	pVCpu->hwaccm.s.fResumeVM = true;
3227	pVCpu->hwaccm.s.fForceTLBFlush = false;
3228
3229	/*
3230	* !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
3231	* IMPORTANT: WE CAN'T DO ANY LOGGING OR OPERATIONS THAT CAN DO A LONGJMP BACK TO RING 3 BEFORE WE'VE SYNCED BACK (MOST OF) THE GUEST STATE
3232	* !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
3233	*/
3234
3235	if (RT_UNLIKELY(rc != VINF_SUCCESS))
3236	{
3237	hmR0VmxReportWorldSwitchError(pVM, pVCpu, rc, pCtx);
3238	VMMR0LogFlushEnable(pVCpu);
3239	goto end;
3240	}
3241
3242	/* Success. Query the guest state and figure out what has happened. */
3243
3244	/* Investigate why there was a VM-exit. */
3245	rc2 = VMXReadCachedVMCS(VMX_VMCS32_RO_EXIT_REASON, &exitReason);
3246	STAM_COUNTER_INC(&pVCpu->hwaccm.s.paStatExitReasonR0[exitReason & MASK_EXITREASON_STAT]);
3247
3248	exitReason &= 0xffff; /* bit 0-15 contain the exit code. */
3249	rc2 \|= VMXReadCachedVMCS(VMX_VMCS32_RO_VM_INSTR_ERROR, &instrError);
3250	rc2 \|= VMXReadCachedVMCS(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &cbInstr);
3251	rc2 \|= VMXReadCachedVMCS(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &intInfo);
3252	/* might not be valid; depends on VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID. */
3253	rc2 \|= VMXReadCachedVMCS(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERRCODE, &errCode);
3254	rc2 \|= VMXReadCachedVMCS(VMX_VMCS32_RO_EXIT_INSTR_INFO, &instrInfo);
3255	rc2 \|= VMXReadCachedVMCS(VMX_VMCS_RO_EXIT_QUALIFICATION, &exitQualification);
3256	AssertRC(rc2);
3257
3258	/*
3259	* Sync back the guest state.
3260	*/
3261	rc2 = VMXR0SaveGuestState(pVM, pVCpu, pCtx);
3262	AssertRC(rc2);
3263
3264	/* Note! NOW IT'S SAFE FOR LOGGING! */
3265	VMMR0LogFlushEnable(pVCpu);
3266	Log2(("Raw exit reason %08x\n", exitReason));
3267	#if ARCH_BITS == 64 /* for the time being */
3268	VBOXVMM_R0_HMVMX_VMEXIT(pVCpu, pCtx, exitReason);
3269	#endif
3270
3271	/*
3272	* Check if an injected event was interrupted prematurely.
3273	*/
3274	rc2 = VMXReadCachedVMCS(VMX_VMCS32_RO_IDT_INFO, &val);
3275	AssertRC(rc2);
3276	pVCpu->hwaccm.s.Event.intInfo = VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(val);
3277	if ( VMX_EXIT_INTERRUPTION_INFO_VALID(pVCpu->hwaccm.s.Event.intInfo)
3278	/* Ignore 'int xx' as they'll be restarted anyway. */
3279	&& VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hwaccm.s.Event.intInfo) != VMX_EXIT_INTERRUPTION_INFO_TYPE_SW
3280	/* Ignore software exceptions (such as int3) as they'll reoccur when we restart the instruction anyway. */
3281	&& VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hwaccm.s.Event.intInfo) != VMX_EXIT_INTERRUPTION_INFO_TYPE_SWEXCPT)
3282	{
3283	Assert(!pVCpu->hwaccm.s.Event.fPending);
3284	pVCpu->hwaccm.s.Event.fPending = true;
3285	/* Error code present? */
3286	if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(pVCpu->hwaccm.s.Event.intInfo))
3287	{
3288	rc2 = VMXReadCachedVMCS(VMX_VMCS32_RO_IDT_ERRCODE, &val);
3289	AssertRC(rc2);
3290	pVCpu->hwaccm.s.Event.errCode = val;
3291	Log(("Pending inject %RX64 at %RGv exit=%08x intInfo=%08x exitQualification=%RGv pending error=%RX64\n",
3292	pVCpu->hwaccm.s.Event.intInfo, (RTGCPTR)pCtx->rip, exitReason, intInfo, exitQualification, val));
3293	}
3294	else
3295	{
3296	Log(("Pending inject %RX64 at %RGv exit=%08x intInfo=%08x exitQualification=%RGv\n", pVCpu->hwaccm.s.Event.intInfo,
3297	(RTGCPTR)pCtx->rip, exitReason, intInfo, exitQualification));
3298	pVCpu->hwaccm.s.Event.errCode = 0;
3299	}
3300	}
3301	#ifdef VBOX_STRICT
3302	else if ( VMX_EXIT_INTERRUPTION_INFO_VALID(pVCpu->hwaccm.s.Event.intInfo)
3303	/* Ignore software exceptions (such as int3) as they're reoccur when we restart the instruction anyway. */
3304	&& VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hwaccm.s.Event.intInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_SWEXCPT)
3305	{
3306	Log(("Ignore pending inject %RX64 at %RGv exit=%08x intInfo=%08x exitQualification=%RGv\n",
3307	pVCpu->hwaccm.s.Event.intInfo, (RTGCPTR)pCtx->rip, exitReason, intInfo, exitQualification));
3308	}
3309
3310	if (exitReason == VMX_EXIT_ERR_INVALID_GUEST_STATE)
3311	HWACCMDumpRegs(pVM, pVCpu, pCtx);
3312	#endif
3313
3314	Log2(("E%d: New EIP=%x:%RGv\n", (uint32_t)exitReason, pCtx->cs.Sel, (RTGCPTR)pCtx->rip));
3315	Log2(("Exit reason %d, exitQualification %RGv\n", (uint32_t)exitReason, exitQualification));
3316	Log2(("instrInfo=%d instrError=%d instr length=%d\n", (uint32_t)instrInfo, (uint32_t)instrError, (uint32_t)cbInstr));
3317	Log2(("Interruption error code %d\n", (uint32_t)errCode));
3318	Log2(("IntInfo = %08x\n", (uint32_t)intInfo));
3319
3320	/*
3321	* Sync back the TPR if it was changed.
3322	*/
3323	if ( fSetupTPRCaching
3324	&& u8LastTPR != pVCpu->hwaccm.s.vmx.pbVAPIC[0x80])
3325	{
3326	rc2 = PDMApicSetTPR(pVCpu, pVCpu->hwaccm.s.vmx.pbVAPIC[0x80]);
3327	AssertRC(rc2);
3328	}
3329
3330	#ifdef DBGFTRACE_ENABLED /** @todo DTrace later. */
3331	RTTraceBufAddMsgF(pVM->CTX_SUFF(hTraceBuf), "vmexit %08x %016RX64 at %04:%08RX64 %RX64",
3332	exitReason, (uint64_t)exitQualification, pCtx->cs.Sel, pCtx->rip, (uint64_t)intInfo);
3333	#endif
3334	STAM_PROFILE_ADV_STOP_START(&pVCpu->hwaccm.s.StatExit1, &pVCpu->hwaccm.s.StatExit2, x);
3335
3336	/* Some cases don't need a complete resync of the guest CPU state; handle them here. */
3337	Assert(rc == VINF_SUCCESS); /* might consider VERR_IPE_UNINITIALIZED_STATUS here later... */
3338	switch (exitReason)
3339	{
3340	case VMX_EXIT_EXCEPTION: /* 0 Exception or non-maskable interrupt (NMI). */
3341	case VMX_EXIT_EXTERNAL_IRQ: /* 1 External interrupt. */
3342	{
3343	uint32_t vector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(intInfo);
3344
3345	if (!VMX_EXIT_INTERRUPTION_INFO_VALID(intInfo))
3346	{
3347	Assert(exitReason == VMX_EXIT_EXTERNAL_IRQ);
3348	#if 0 //def VBOX_WITH_VMMR0_DISABLE_PREEMPTION
3349	if ( RTThreadPreemptIsPendingTrusty()
3350	&& !RTThreadPreemptIsPending(NIL_RTTHREAD))
3351	goto ResumeExecution;
3352	#endif
3353	/* External interrupt; leave to allow it to be dispatched again. */
3354	rc = VINF_EM_RAW_INTERRUPT;
3355	break;
3356	}
3357	STAM_PROFILE_ADV_START(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3358	switch (VMX_EXIT_INTERRUPTION_INFO_TYPE(intInfo))
3359	{
3360	case VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI: /* Non-maskable interrupt. */
3361	/* External interrupt; leave to allow it to be dispatched again. */
3362	rc = VINF_EM_RAW_INTERRUPT;
3363	break;
3364
3365	case VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT: /* External hardware interrupt. */
3366	AssertFailed(); /* can't come here; fails the first check. */
3367	break;
3368
3369	case VMX_EXIT_INTERRUPTION_INFO_TYPE_DBEXCPT: /* Unknown why we get this type for #DB */
3370	case VMX_EXIT_INTERRUPTION_INFO_TYPE_SWEXCPT: /* Software exception. (#BP or #OF) */
3371	Assert(vector == 1 \|\| vector == 3 \|\| vector == 4);
3372	/* no break */
3373	case VMX_EXIT_INTERRUPTION_INFO_TYPE_HWEXCPT: /* Hardware exception. */
3374	Log2(("Hardware/software interrupt %d\n", vector));
3375	switch (vector)
3376	{
3377	case X86_XCPT_NM:
3378	{
3379	Log(("#NM fault at %RGv error code %x\n", (RTGCPTR)pCtx->rip, errCode));
3380
3381	/** @todo don't intercept #NM exceptions anymore when we've activated the guest FPU state. */
3382	/* If we sync the FPU/XMM state on-demand, then we can continue execution as if nothing has happened. */
3383	rc = CPUMR0LoadGuestFPU(pVM, pVCpu, pCtx);
3384	if (rc == VINF_SUCCESS)
3385	{
3386	Assert(CPUMIsGuestFPUStateActive(pVCpu));
3387
3388	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitShadowNM);
3389
3390	/* Continue execution. */
3391	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_GUEST_CR0;
3392
3393	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3394	goto ResumeExecution;
3395	}
3396
3397	Log(("Forward #NM fault to the guest\n"));
3398	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestNM);
3399	rc2 = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo),
3400	cbInstr, 0);
3401	AssertRC(rc2);
3402	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3403	goto ResumeExecution;
3404	}
3405
3406	case X86_XCPT_PF: /* Page fault */
3407	{
3408	#ifdef VBOX_ALWAYS_TRAP_PF
3409	if (pVM->hwaccm.s.fNestedPaging)
3410	{
3411	/*
3412	* A genuine pagefault. Forward the trap to the guest by injecting the exception and resuming execution.
3413	*/
3414	Log(("Guest page fault at %RGv cr2=%RGv error code %RGv rsp=%RGv\n", (RTGCPTR)pCtx->rip, exitQualification,
3415	errCode, (RTGCPTR)pCtx->rsp));
3416
3417	Assert(CPUMIsGuestInPagedProtectedModeEx(pCtx));
3418
3419	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestPF);
3420
3421	/* Now we must update CR2. */
3422	pCtx->cr2 = exitQualification;
3423	rc2 = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo),
3424	cbInstr, errCode);
3425	AssertRC(rc2);
3426
3427	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3428	goto ResumeExecution;
3429	}
3430	#else
3431	Assert(!pVM->hwaccm.s.fNestedPaging);
3432	#endif
3433
3434	#ifdef VBOX_HWACCM_WITH_GUEST_PATCHING
3435	/* Shortcut for APIC TPR reads and writes; 32 bits guests only */
3436	if ( pVM->hwaccm.s.fTRPPatchingAllowed
3437	&& pVM->hwaccm.s.pGuestPatchMem
3438	&& (exitQualification & 0xfff) == 0x080
3439	&& !(errCode & X86_TRAP_PF_P) /* not present */
3440	&& CPUMGetGuestCPL(pVCpu) == 0
3441	&& !CPUMIsGuestInLongModeEx(pCtx)
3442	&& pVM->hwaccm.s.cPatches < RT_ELEMENTS(pVM->hwaccm.s.aPatches))
3443	{
3444	RTGCPHYS GCPhysApicBase, GCPhys;
3445	PDMApicGetBase(pVM, &GCPhysApicBase); /** @todo cache this */
3446	GCPhysApicBase &= PAGE_BASE_GC_MASK;
3447
3448	rc = PGMGstGetPage(pVCpu, (RTGCPTR)exitQualification, NULL, &GCPhys);
3449	if ( rc == VINF_SUCCESS
3450	&& GCPhys == GCPhysApicBase)
3451	{
3452	/* Only attempt to patch the instruction once. */
3453	PHWACCMTPRPATCH pPatch = (PHWACCMTPRPATCH)RTAvloU32Get(&pVM->hwaccm.s.PatchTree, (AVLOU32KEY)pCtx->eip);
3454	if (!pPatch)
3455	{
3456	rc = VINF_EM_HWACCM_PATCH_TPR_INSTR;
3457	break;
3458	}
3459	}
3460	}
3461	#endif
3462
3463	Log2(("Page fault at %RGv error code %x\n", exitQualification, errCode));
3464	/* Exit qualification contains the linear address of the page fault. */
3465	TRPMAssertTrap(pVCpu, X86_XCPT_PF, TRPM_TRAP);
3466	TRPMSetErrorCode(pVCpu, errCode);
3467	TRPMSetFaultAddress(pVCpu, exitQualification);
3468
3469	/* Shortcut for APIC TPR reads and writes. */
3470	if ( (exitQualification & 0xfff) == 0x080
3471	&& !(errCode & X86_TRAP_PF_P) /* not present */
3472	&& fSetupTPRCaching
3473	&& (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC))
3474	{
3475	RTGCPHYS GCPhysApicBase, GCPhys;
3476	PDMApicGetBase(pVM, &GCPhysApicBase); /* @todo cache this */
3477	GCPhysApicBase &= PAGE_BASE_GC_MASK;
3478
3479	rc = PGMGstGetPage(pVCpu, (RTGCPTR)exitQualification, NULL, &GCPhys);
3480	if ( rc == VINF_SUCCESS
3481	&& GCPhys == GCPhysApicBase)
3482	{
3483	Log(("Enable VT-x virtual APIC access filtering\n"));
3484	rc2 = IOMMMIOMapMMIOHCPage(pVM, GCPhysApicBase, pVM->hwaccm.s.vmx.pAPICPhys, X86_PTE_RW \| X86_PTE_P);
3485	AssertRC(rc2);
3486	}
3487	}
3488
3489	/* Forward it to our trap handler first, in case our shadow pages are out of sync. */
3490	rc = PGMTrap0eHandler(pVCpu, errCode, CPUMCTX2CORE(pCtx), (RTGCPTR)exitQualification);
3491	Log2(("PGMTrap0eHandler %RGv returned %Rrc\n", (RTGCPTR)pCtx->rip, VBOXSTRICTRC_VAL(rc)));
3492
3493	if (rc == VINF_SUCCESS)
3494	{ /* We've successfully synced our shadow pages, so let's just continue execution. */
3495	Log2(("Shadow page fault at %RGv cr2=%RGv error code %x\n", (RTGCPTR)pCtx->rip, exitQualification ,errCode));
3496	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitShadowPF);
3497
3498	TRPMResetTrap(pVCpu);
3499	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3500	goto ResumeExecution;
3501	}
3502	else if (rc == VINF_EM_RAW_GUEST_TRAP)
3503	{
3504	/*
3505	* A genuine pagefault. Forward the trap to the guest by injecting the exception and resuming execution.
3506	*/
3507	Log2(("Forward page fault to the guest\n"));
3508
3509	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestPF);
3510	/* The error code might have been changed. */
3511	errCode = TRPMGetErrorCode(pVCpu);
3512
3513	TRPMResetTrap(pVCpu);
3514
3515	/* Now we must update CR2. */
3516	pCtx->cr2 = exitQualification;
3517	rc2 = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo),
3518	cbInstr, errCode);
3519	AssertRC(rc2);
3520
3521	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3522	goto ResumeExecution;
3523	}
3524	#ifdef VBOX_STRICT
3525	if (rc != VINF_EM_RAW_EMULATE_INSTR && rc != VINF_EM_RAW_EMULATE_IO_BLOCK)
3526	Log2(("PGMTrap0eHandler failed with %d\n", VBOXSTRICTRC_VAL(rc)));
3527	#endif
3528	/* Need to go back to the recompiler to emulate the instruction. */
3529	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitShadowPFEM);
3530	TRPMResetTrap(pVCpu);
3531	break;
3532	}
3533
3534	case X86_XCPT_MF: /* Floating point exception. */
3535	{
3536	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestMF);
3537	if (!(pCtx->cr0 & X86_CR0_NE))
3538	{
3539	/* old style FPU error reporting needs some extra work. */
3540	/** @todo don't fall back to the recompiler, but do it manually. */
3541	rc = VINF_EM_RAW_EMULATE_INSTR;
3542	break;
3543	}
3544	Log(("Trap %x at %04X:%RGv\n", vector, pCtx->cs.Sel, (RTGCPTR)pCtx->rip));
3545	rc2 = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo),
3546	cbInstr, errCode);
3547	AssertRC(rc2);
3548
3549	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3550	goto ResumeExecution;
3551	}
3552
3553	case X86_XCPT_DB: /* Debug exception. */
3554	{
3555	uint64_t uDR6;
3556
3557	/*
3558	* DR6, DR7.GD and IA32_DEBUGCTL.LBR are not updated yet.
3559	*
3560	* Exit qualification bits:
3561	* 3:0 B0-B3 which breakpoint condition was met
3562	* 12:4 Reserved (0)
3563	* 13 BD - debug register access detected
3564	* 14 BS - single step execution or branch taken
3565	* 63:15 Reserved (0)
3566	*/
3567	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestDB);
3568
3569	/* Note that we don't support guest and host-initiated debugging at the same time. */
3570
3571	uDR6 = X86_DR6_INIT_VAL;
3572	uDR6 \|= (exitQualification & (X86_DR6_B0\|X86_DR6_B1\|X86_DR6_B2\|X86_DR6_B3\|X86_DR6_BD\|X86_DR6_BS));
3573	rc = DBGFRZTrap01Handler(pVM, pVCpu, CPUMCTX2CORE(pCtx), uDR6);
3574	if (rc == VINF_EM_RAW_GUEST_TRAP)
3575	{
3576	/* Update DR6 here. */
3577	pCtx->dr[6] = uDR6;
3578
3579	/* Resync DR6 if the debug state is active. */
3580	if (CPUMIsGuestDebugStateActive(pVCpu))
3581	ASMSetDR6(pCtx->dr[6]);
3582
3583	/* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
3584	pCtx->dr[7] &= ~X86_DR7_GD;
3585
3586	/* Paranoia. */
3587	pCtx->dr[7] &= 0xffffffff; /* upper 32 bits reserved */
3588	pCtx->dr[7] &= ~(RT_BIT(11) \| RT_BIT(12) \| RT_BIT(14) \| RT_BIT(15)); /* must be zero */
3589	pCtx->dr[7] \|= 0x400; /* must be one */
3590
3591	/* Resync DR7 */
3592	rc2 = VMXWriteVMCS64(VMX_VMCS64_GUEST_DR7, pCtx->dr[7]);
3593	AssertRC(rc2);
3594
3595	Log(("Trap %x (debug) at %RGv exit qualification %RX64 dr6=%x dr7=%x\n", vector, (RTGCPTR)pCtx->rip,
3596	exitQualification, (uint32_t)pCtx->dr[6], (uint32_t)pCtx->dr[7]));
3597	rc2 = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo),
3598	cbInstr, errCode);
3599	AssertRC(rc2);
3600
3601	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3602	goto ResumeExecution;
3603	}
3604	/* Return to ring 3 to deal with the debug exit code. */
3605	Log(("Debugger hardware BP at %04x:%RGv (rc=%Rrc)\n", pCtx->cs.Sel, pCtx->rip, VBOXSTRICTRC_VAL(rc)));
3606	break;
3607	}
3608
3609	case X86_XCPT_BP: /* Breakpoint. */
3610	{
3611	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestBP);
3612	rc = DBGFRZTrap03Handler(pVM, pVCpu, CPUMCTX2CORE(pCtx));
3613	if (rc == VINF_EM_RAW_GUEST_TRAP)
3614	{
3615	Log(("Guest #BP at %04x:%RGv\n", pCtx->cs.Sel, pCtx->rip));
3616	rc2 = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo),
3617	cbInstr, errCode);
3618	AssertRC(rc2);
3619	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3620	goto ResumeExecution;
3621	}
3622	if (rc == VINF_SUCCESS)
3623	{
3624	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3625	goto ResumeExecution;
3626	}
3627	Log(("Debugger BP at %04x:%RGv (rc=%Rrc)\n", pCtx->cs.Sel, pCtx->rip, VBOXSTRICTRC_VAL(rc)));
3628	break;
3629	}
3630
3631	case X86_XCPT_GP: /* General protection failure exception. */
3632	{
3633	uint32_t cbOp;
3634	PDISCPUSTATE pDis = &pVCpu->hwaccm.s.DisState;
3635
3636	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestGP);
3637	#ifdef VBOX_STRICT
3638	if ( !CPUMIsGuestInRealModeEx(pCtx)
3639	\|\| !pVM->hwaccm.s.vmx.pRealModeTSS)
3640	{
3641	Log(("Trap %x at %04X:%RGv errorCode=%RGv\n", vector, pCtx->cs.Sel, (RTGCPTR)pCtx->rip, errCode));
3642	rc2 = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo),
3643	cbInstr, errCode);
3644	AssertRC(rc2);
3645	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3646	goto ResumeExecution;
3647	}
3648	#endif
3649	Assert(CPUMIsGuestInRealModeEx(pCtx));
3650
3651	LogFlow(("Real mode X86_XCPT_GP instruction emulation at %x:%RGv\n", pCtx->cs.Sel, (RTGCPTR)pCtx->rip));
3652
3653	rc2 = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbOp);
3654	if (RT_SUCCESS(rc2))
3655	{
3656	bool fUpdateRIP = true;
3657
3658	rc = VINF_SUCCESS;
3659	Assert(cbOp == pDis->cbInstr);
3660	switch (pDis->pCurInstr->uOpcode)
3661	{
3662	case OP_CLI:
3663	pCtx->eflags.Bits.u1IF = 0;
3664	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitCli);
3665	break;
3666
3667	case OP_STI:
3668	pCtx->eflags.Bits.u1IF = 1;
3669	EMSetInhibitInterruptsPC(pVCpu, pCtx->rip + pDis->cbInstr);
3670	Assert(VMCPU_FF_ISSET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
3671	rc2 = VMXWriteVMCS(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE,
3672	VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
3673	AssertRC(rc2);
3674	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitSti);
3675	break;
3676
3677	case OP_HLT:
3678	fUpdateRIP = false;
3679	rc = VINF_EM_HALT;
3680	pCtx->rip += pDis->cbInstr;
3681	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitHlt);
3682	break;
3683
3684	case OP_POPF:
3685	{
3686	RTGCPTR GCPtrStack;
3687	uint32_t cbParm;
3688	uint32_t uMask;
3689	X86EFLAGS eflags;
3690
3691	if (pDis->fPrefix & DISPREFIX_OPSIZE)
3692	{
3693	cbParm = 4;
3694	uMask = 0xffffffff;
3695	}
3696	else
3697	{
3698	cbParm = 2;
3699	uMask = 0xffff;
3700	}
3701
3702	rc2 = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pCtx), pCtx->esp & uMask, 0, &GCPtrStack);
3703	if (RT_FAILURE(rc2))
3704	{
3705	rc = VERR_EM_INTERPRETER;
3706	break;
3707	}
3708	eflags.u = 0;
3709	rc2 = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &eflags.u, cbParm);
3710	if (RT_FAILURE(rc2))
3711	{
3712	rc = VERR_EM_INTERPRETER;
3713	break;
3714	}
3715	LogFlow(("POPF %x -> %RGv mask=%x\n", eflags.u, pCtx->rsp, uMask));
3716	pCtx->eflags.u = (pCtx->eflags.u & ~(X86_EFL_POPF_BITS & uMask))
3717	\| (eflags.u & X86_EFL_POPF_BITS & uMask);
3718	/* RF cleared when popped in real mode; see pushf description in AMD manual. */
3719	pCtx->eflags.Bits.u1RF = 0;
3720	pCtx->esp += cbParm;
3721	pCtx->esp &= uMask;
3722
3723	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitPopf);
3724	break;
3725	}
3726
3727	case OP_PUSHF:
3728	{
3729	RTGCPTR GCPtrStack;
3730	uint32_t cbParm;
3731	uint32_t uMask;
3732	X86EFLAGS eflags;
3733
3734	if (pDis->fPrefix & DISPREFIX_OPSIZE)
3735	{
3736	cbParm = 4;
3737	uMask = 0xffffffff;
3738	}
3739	else
3740	{
3741	cbParm = 2;
3742	uMask = 0xffff;
3743	}
3744
3745	rc2 = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pCtx), (pCtx->esp - cbParm) & uMask, 0,
3746	&GCPtrStack);
3747	if (RT_FAILURE(rc2))
3748	{
3749	rc = VERR_EM_INTERPRETER;
3750	break;
3751	}
3752	eflags = pCtx->eflags;
3753	/* RF & VM cleared when pushed in real mode; see pushf description in AMD manual. */
3754	eflags.Bits.u1RF = 0;
3755	eflags.Bits.u1VM = 0;
3756
3757	rc2 = PGMPhysWrite(pVM, (RTGCPHYS)GCPtrStack, &eflags.u, cbParm);
3758	if (RT_FAILURE(rc2))
3759	{
3760	rc = VERR_EM_INTERPRETER;
3761	break;
3762	}
3763	LogFlow(("PUSHF %x -> %RGv\n", eflags.u, GCPtrStack));
3764	pCtx->esp -= cbParm;
3765	pCtx->esp &= uMask;
3766	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitPushf);
3767	break;
3768	}
3769
3770	case OP_IRET:
3771	{
3772	RTGCPTR GCPtrStack;
3773	uint32_t uMask = 0xffff;
3774	uint16_t aIretFrame[3];
3775
3776	if (pDis->fPrefix & (DISPREFIX_OPSIZE \| DISPREFIX_ADDRSIZE))
3777	{
3778	rc = VERR_EM_INTERPRETER;
3779	break;
3780	}
3781
3782	rc2 = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pCtx), pCtx->esp & uMask, 0, &GCPtrStack);
3783	if (RT_FAILURE(rc2))
3784	{
3785	rc = VERR_EM_INTERPRETER;
3786	break;
3787	}
3788	rc2 = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &aIretFrame[0], sizeof(aIretFrame));
3789	if (RT_FAILURE(rc2))
3790	{
3791	rc = VERR_EM_INTERPRETER;
3792	break;
3793	}
3794	pCtx->ip = aIretFrame[0];
3795	pCtx->cs.Sel = aIretFrame[1];
3796	pCtx->cs.ValidSel = aIretFrame[1];
3797	pCtx->cs.u64Base = (uint32_t)pCtx->cs.Sel << 4;
3798	pCtx->eflags.u = (pCtx->eflags.u & ~(X86_EFL_POPF_BITS & uMask))
3799	\| (aIretFrame[2] & X86_EFL_POPF_BITS & uMask);
3800	pCtx->sp += sizeof(aIretFrame);
3801
3802	LogFlow(("iret to %04x:%x\n", pCtx->cs.Sel, pCtx->ip));
3803	fUpdateRIP = false;
3804	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitIret);
3805	break;
3806	}
3807
3808	case OP_INT:
3809	{
3810	uint32_t intInfo2;
3811
3812	LogFlow(("Realmode: INT %x\n", pDis->Param1.uValue & 0xff));
3813	intInfo2 = pDis->Param1.uValue & 0xff;
3814	intInfo2 \|= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
3815	intInfo2 \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
3816
3817	rc = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, intInfo2, cbOp, 0);
3818	AssertRC(VBOXSTRICTRC_VAL(rc));
3819	fUpdateRIP = false;
3820	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitInt);
3821	break;
3822	}
3823
3824	case OP_INTO:
3825	{
3826	if (pCtx->eflags.Bits.u1OF)
3827	{
3828	uint32_t intInfo2;
3829
3830	LogFlow(("Realmode: INTO\n"));
3831	intInfo2 = X86_XCPT_OF;
3832	intInfo2 \|= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
3833	intInfo2 \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
3834
3835	rc = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, intInfo2, cbOp, 0);
3836	AssertRC(VBOXSTRICTRC_VAL(rc));
3837	fUpdateRIP = false;
3838	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitInt);
3839	}
3840	break;
3841	}
3842
3843	case OP_INT3:
3844	{
3845	uint32_t intInfo2;
3846
3847	LogFlow(("Realmode: INT 3\n"));
3848	intInfo2 = 3;
3849	intInfo2 \|= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
3850	intInfo2 \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
3851
3852	rc = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, intInfo2, cbOp, 0);
3853	AssertRC(VBOXSTRICTRC_VAL(rc));
3854	fUpdateRIP = false;
3855	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitInt);
3856	break;
3857	}
3858
3859	default:
3860	rc = EMInterpretInstructionDisasState(pVCpu, pDis, CPUMCTX2CORE(pCtx), 0, EMCODETYPE_SUPERVISOR);
3861	fUpdateRIP = false;
3862	break;
3863	}
3864
3865	if (rc == VINF_SUCCESS)
3866	{
3867	if (fUpdateRIP)
3868	pCtx->rip += cbOp; /* Move on to the next instruction. */
3869
3870	/*
3871	* LIDT, LGDT can end up here. In the future CRx changes as well. Just reload the
3872	* whole context to be done with it.
3873	*/
3874	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_ALL;
3875
3876	/* Only resume if successful. */
3877	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3878	goto ResumeExecution;
3879	}
3880	}
3881	else
3882	rc = VERR_EM_INTERPRETER;
3883
3884	AssertMsg(rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_EM_HALT,
3885	("Unexpected rc=%Rrc\n", VBOXSTRICTRC_VAL(rc)));
3886	break;
3887	}
3888
3889	#ifdef VBOX_STRICT
3890	case X86_XCPT_XF: /* SIMD exception. */
3891	case X86_XCPT_DE: /* Divide error. */
3892	case X86_XCPT_UD: /* Unknown opcode exception. */
3893	case X86_XCPT_SS: /* Stack segment exception. */
3894	case X86_XCPT_NP: /* Segment not present exception. */
3895	{
3896	switch (vector)
3897	{
3898	case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestDE); break;
3899	case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestUD); break;
3900	case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestSS); break;
3901	case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestNP); break;
3902	case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestXF); break;
3903	}
3904
3905	Log(("Trap %x at %04X:%RGv\n", vector, pCtx->cs.Sel, (RTGCPTR)pCtx->rip));
3906	rc2 = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo),
3907	cbInstr, errCode);
3908	AssertRC(rc2);
3909
3910	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3911	goto ResumeExecution;
3912	}
3913	#endif
3914	default:
3915	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestXcpUnk);
3916	if ( CPUMIsGuestInRealModeEx(pCtx)
3917	&& pVM->hwaccm.s.vmx.pRealModeTSS)
3918	{
3919	Log(("Real Mode Trap %x at %04x:%04X error code %x\n", vector, pCtx->cs.Sel, pCtx->eip, errCode));
3920	rc = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo),
3921	cbInstr, errCode);
3922	AssertRC(VBOXSTRICTRC_VAL(rc)); /* Strict RC check below. */
3923
3924	/* Go back to ring-3 in case of a triple fault. */
3925	if ( vector == X86_XCPT_DF
3926	&& rc == VINF_EM_RESET)
3927	{
3928	break;
3929	}
3930
3931	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3932	goto ResumeExecution;
3933	}
3934	AssertMsgFailed(("Unexpected vm-exit caused by exception %x\n", vector));
3935	rc = VERR_VMX_UNEXPECTED_EXCEPTION;
3936	break;
3937	} /* switch (vector) */
3938
3939	break;
3940
3941	default:
3942	rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_CODE;
3943	AssertMsgFailed(("Unexpected interruption code %x\n", intInfo));
3944	break;
3945	}
3946
3947	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3948	break;
3949	}
3950
3951	/*
3952	* 48 EPT violation. An attempt to access memory with a guest-physical address was disallowed
3953	* by the configuration of the EPT paging structures.
3954	*/
3955	case VMX_EXIT_EPT_VIOLATION:
3956	{
3957	RTGCPHYS GCPhys;
3958
3959	Assert(pVM->hwaccm.s.fNestedPaging);
3960
3961	rc2 = VMXReadVMCS64(VMX_VMCS_EXIT_PHYS_ADDR_FULL, &GCPhys);
3962	AssertRC(rc2);
3963	Assert(((exitQualification >> 7) & 3) != 2);
3964
3965	/* Determine the kind of violation. */
3966	errCode = 0;
3967	if (exitQualification & VMX_EXIT_QUALIFICATION_EPT_INSTR_FETCH)
3968	errCode \|= X86_TRAP_PF_ID;
3969
3970	if (exitQualification & VMX_EXIT_QUALIFICATION_EPT_DATA_WRITE)
3971	errCode \|= X86_TRAP_PF_RW;
3972
3973	/* If the page is present, then it's a page level protection fault. */
3974	if (exitQualification & VMX_EXIT_QUALIFICATION_EPT_ENTRY_PRESENT)
3975	errCode \|= X86_TRAP_PF_P;
3976	else
3977	{
3978	/* Shortcut for APIC TPR reads and writes. */
3979	if ( (GCPhys & 0xfff) == 0x080
3980	&& GCPhys > 0x1000000 /* to skip VGA frame buffer accesses */
3981	&& fSetupTPRCaching
3982	&& (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC))
3983	{
3984	RTGCPHYS GCPhysApicBase;
3985	PDMApicGetBase(pVM, &GCPhysApicBase); /* @todo cache this */
3986	GCPhysApicBase &= PAGE_BASE_GC_MASK;
3987	if (GCPhys == GCPhysApicBase + 0x80)
3988	{
3989	Log(("Enable VT-x virtual APIC access filtering\n"));
3990	rc2 = IOMMMIOMapMMIOHCPage(pVM, GCPhysApicBase, pVM->hwaccm.s.vmx.pAPICPhys, X86_PTE_RW \| X86_PTE_P);
3991	AssertRC(rc2);
3992	}
3993	}
3994	}
3995	Log(("EPT Page fault %x at %RGp error code %x\n", (uint32_t)exitQualification, GCPhys, errCode));
3996
3997	/* GCPhys contains the guest physical address of the page fault. */
3998	TRPMAssertTrap(pVCpu, X86_XCPT_PF, TRPM_TRAP);
3999	TRPMSetErrorCode(pVCpu, errCode);
4000	TRPMSetFaultAddress(pVCpu, GCPhys);
4001
4002	/* Handle the pagefault trap for the nested shadow table. */
4003	rc = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, errCode, CPUMCTX2CORE(pCtx), GCPhys);
4004
4005	/*
4006	* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment below, @bugref{6043}.
4007	*/
4008	if ( rc == VINF_SUCCESS
4009	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
4010	\|\| rc == VERR_PAGE_NOT_PRESENT)
4011	{
4012	/* We've successfully synced our shadow pages, so let's just continue execution. */
4013	Log2(("Shadow page fault at %RGv cr2=%RGp error code %x\n", (RTGCPTR)pCtx->rip, exitQualification , errCode));
4014	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitReasonNPF);
4015
4016	TRPMResetTrap(pVCpu);
4017	goto ResumeExecution;
4018	}
4019
4020	#ifdef VBOX_STRICT
4021	if (rc != VINF_EM_RAW_EMULATE_INSTR)
4022	LogFlow(("PGMTrap0eHandlerNestedPaging at %RGv failed with %Rrc\n", (RTGCPTR)pCtx->rip, VBOXSTRICTRC_VAL(rc)));
4023	#endif
4024	/* Need to go back to the recompiler to emulate the instruction. */
4025	TRPMResetTrap(pVCpu);
4026	break;
4027	}
4028
4029	case VMX_EXIT_EPT_MISCONFIG:
4030	{
4031	RTGCPHYS GCPhys;
4032
4033	Assert(pVM->hwaccm.s.fNestedPaging);
4034
4035	rc2 = VMXReadVMCS64(VMX_VMCS_EXIT_PHYS_ADDR_FULL, &GCPhys);
4036	AssertRC(rc2);
4037	Log(("VMX_EXIT_EPT_MISCONFIG for %RGp\n", GCPhys));
4038
4039	/* Shortcut for APIC TPR reads and writes. */
4040	if ( (GCPhys & 0xfff) == 0x080
4041	&& GCPhys > 0x1000000 /* to skip VGA frame buffer accesses */
4042	&& fSetupTPRCaching
4043	&& (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC))
4044	{
4045	RTGCPHYS GCPhysApicBase;
4046	PDMApicGetBase(pVM, &GCPhysApicBase); /* @todo cache this */
4047	GCPhysApicBase &= PAGE_BASE_GC_MASK;
4048	if (GCPhys == GCPhysApicBase + 0x80)
4049	{
4050	Log(("Enable VT-x virtual APIC access filtering\n"));
4051	rc2 = IOMMMIOMapMMIOHCPage(pVM, GCPhysApicBase, pVM->hwaccm.s.vmx.pAPICPhys, X86_PTE_RW \| X86_PTE_P);
4052	AssertRC(rc2);
4053	}
4054	}
4055
4056	rc = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pCtx), GCPhys, UINT32_MAX);
4057
4058	/*
4059	* If we succeed, resume execution.
4060	* Or, if fail in interpreting the instruction because we couldn't get the guest physical address
4061	* of the page containing the instruction via the guest's page tables (we would invalidate the guest page
4062	* in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
4063	* weird case. See @bugref{6043}.
4064	*/
4065	if ( rc == VINF_SUCCESS
4066	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
4067	\|\| rc == VERR_PAGE_NOT_PRESENT)
4068	{
4069	Log2(("PGMR0Trap0eHandlerNPMisconfig(,,,%RGp) at %RGv -> resume\n", GCPhys, (RTGCPTR)pCtx->rip));
4070	goto ResumeExecution;
4071	}
4072
4073	Log2(("PGMR0Trap0eHandlerNPMisconfig(,,,%RGp) at %RGv -> %Rrc\n", GCPhys, (RTGCPTR)pCtx->rip, VBOXSTRICTRC_VAL(rc)));
4074	break;
4075	}
4076
4077	case VMX_EXIT_IRQ_WINDOW: /* 7 Interrupt window. */
4078	/* Clear VM-exit on IF=1 change. */
4079	LogFlow(("VMX_EXIT_IRQ_WINDOW %RGv pending=%d IF=%d\n", (RTGCPTR)pCtx->rip,
4080	VMCPU_FF_ISPENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC\|VMCPU_FF_INTERRUPT_PIC)), pCtx->eflags.Bits.u1IF));
4081	pVCpu->hwaccm.s.vmx.proc_ctls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_IRQ_WINDOW_EXIT;
4082	rc2 = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
4083	AssertRC(rc2);
4084	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitIrqWindow);
4085	goto ResumeExecution; /* we check for pending guest interrupts there */
4086
4087	case VMX_EXIT_WBINVD: /* 54 Guest software attempted to execute WBINVD. (conditional) */
4088	case VMX_EXIT_INVD: /* 13 Guest software attempted to execute INVD. (unconditional) */
4089	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitInvd);
4090	/* Skip instruction and continue directly. */
4091	pCtx->rip += cbInstr;
4092	/* Continue execution.*/
4093	goto ResumeExecution;
4094
4095	case VMX_EXIT_CPUID: /* 10 Guest software attempted to execute CPUID. */
4096	{
4097	Log2(("VMX: Cpuid %x\n", pCtx->eax));
4098	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitCpuid);
4099	rc = EMInterpretCpuId(pVM, pVCpu, CPUMCTX2CORE(pCtx));
4100	if (rc == VINF_SUCCESS)
4101	{
4102	/* Update EIP and continue execution. */
4103	Assert(cbInstr == 2);
4104	pCtx->rip += cbInstr;
4105	goto ResumeExecution;
4106	}
4107	AssertMsgFailed(("EMU: cpuid failed with %Rrc\n", VBOXSTRICTRC_VAL(rc)));
4108	rc = VINF_EM_RAW_EMULATE_INSTR;
4109	break;
4110	}
4111
4112	case VMX_EXIT_RDPMC: /* 15 Guest software attempted to execute RDPMC. */
4113	{
4114	Log2(("VMX: Rdpmc %x\n", pCtx->ecx));
4115	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitRdpmc);
4116	rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pCtx));
4117	if (rc == VINF_SUCCESS)
4118	{
4119	/* Update EIP and continue execution. */
4120	Assert(cbInstr == 2);
4121	pCtx->rip += cbInstr;
4122	goto ResumeExecution;
4123	}
4124	rc = VINF_EM_RAW_EMULATE_INSTR;
4125	break;
4126	}
4127
4128	case VMX_EXIT_RDTSC: /* 16 Guest software attempted to execute RDTSC. */
4129	{
4130	Log2(("VMX: Rdtsc\n"));
4131	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitRdtsc);
4132	rc = EMInterpretRdtsc(pVM, pVCpu, CPUMCTX2CORE(pCtx));
4133	if (rc == VINF_SUCCESS)
4134	{
4135	/* Update EIP and continue execution. */
4136	Assert(cbInstr == 2);
4137	pCtx->rip += cbInstr;
4138	goto ResumeExecution;
4139	}
4140	rc = VINF_EM_RAW_EMULATE_INSTR;
4141	break;
4142	}
4143
4144	case VMX_EXIT_RDTSCP: /* 51 Guest software attempted to execute RDTSCP. */
4145	{
4146	Log2(("VMX: Rdtscp\n"));
4147	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitRdtscp);
4148	rc = EMInterpretRdtscp(pVM, pVCpu, pCtx);
4149	if (rc == VINF_SUCCESS)
4150	{
4151	/* Update EIP and continue execution. */
4152	Assert(cbInstr == 3);
4153	pCtx->rip += cbInstr;
4154	goto ResumeExecution;
4155	}
4156	rc = VINF_EM_RAW_EMULATE_INSTR;
4157	break;
4158	}
4159
4160	case VMX_EXIT_INVLPG: /* 14 Guest software attempted to execute INVLPG. */
4161	{
4162	Log2(("VMX: invlpg\n"));
4163	Assert(!pVM->hwaccm.s.fNestedPaging);
4164
4165	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitInvlpg);
4166	rc = EMInterpretInvlpg(pVM, pVCpu, CPUMCTX2CORE(pCtx), exitQualification);
4167	if (rc == VINF_SUCCESS)
4168	{
4169	/* Update EIP and continue execution. */
4170	pCtx->rip += cbInstr;
4171	goto ResumeExecution;
4172	}
4173	AssertMsg(rc == VERR_EM_INTERPRETER, ("EMU: invlpg %RGv failed with %Rrc\n", exitQualification, VBOXSTRICTRC_VAL(rc)));
4174	break;
4175	}
4176
4177	case VMX_EXIT_MONITOR: /* 39 Guest software attempted to execute MONITOR. */
4178	{
4179	Log2(("VMX: monitor\n"));
4180
4181	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitMonitor);
4182	rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pCtx));
4183	if (rc == VINF_SUCCESS)
4184	{
4185	/* Update EIP and continue execution. */
4186	pCtx->rip += cbInstr;
4187	goto ResumeExecution;
4188	}
4189	AssertMsg(rc == VERR_EM_INTERPRETER, ("EMU: monitor failed with %Rrc\n", VBOXSTRICTRC_VAL(rc)));
4190	break;
4191	}
4192
4193	case VMX_EXIT_WRMSR: /* 32 WRMSR. Guest software attempted to execute WRMSR. */
4194	/* When an interrupt is pending, we'll let MSR_K8_LSTAR writes fault in our TPR patch code. */
4195	if ( pVM->hwaccm.s.fTPRPatchingActive
4196	&& pCtx->ecx == MSR_K8_LSTAR)
4197	{
4198	Assert(!CPUMIsGuestInLongModeEx(pCtx));
4199	if ((pCtx->eax & 0xff) != u8LastTPR)
4200	{
4201	Log(("VMX: Faulting MSR_K8_LSTAR write with new TPR value %x\n", pCtx->eax & 0xff));
4202
4203	/* Our patch code uses LSTAR for TPR caching. */
4204	rc2 = PDMApicSetTPR(pVCpu, pCtx->eax & 0xff);
4205	AssertRC(rc2);
4206	}
4207
4208	/* Skip the instruction and continue. */
4209	pCtx->rip += cbInstr; /* wrmsr = [0F 30] */
4210
4211	/* Only resume if successful. */
4212	goto ResumeExecution;
4213	}
4214	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_GUEST_MSR;
4215	/* no break */
4216	case VMX_EXIT_RDMSR: /* 31 RDMSR. Guest software attempted to execute RDMSR. */
4217	{
4218	STAM_COUNTER_INC((exitReason == VMX_EXIT_RDMSR) ? &pVCpu->hwaccm.s.StatExitRdmsr : &pVCpu->hwaccm.s.StatExitWrmsr);
4219
4220	/*
4221	* Note: The Intel spec. claims there's an REX version of RDMSR that's slightly different,
4222	* so we play safe by completely disassembling the instruction.
4223	*/
4224	Log2(("VMX: %s\n", (exitReason == VMX_EXIT_RDMSR) ? "rdmsr" : "wrmsr"));
4225	rc = EMInterpretInstruction(pVCpu, CPUMCTX2CORE(pCtx), 0);
4226	if (rc == VINF_SUCCESS)
4227	{
4228	/* EIP has been updated already. */
4229	/* Only resume if successful. */
4230	goto ResumeExecution;
4231	}
4232	AssertMsg(rc == VERR_EM_INTERPRETER, ("EMU: %s failed with %Rrc\n",
4233	(exitReason == VMX_EXIT_RDMSR) ? "rdmsr" : "wrmsr", VBOXSTRICTRC_VAL(rc)));
4234	break;
4235	}
4236
4237	case VMX_EXIT_CRX_MOVE: /* 28 Control-register accesses. */
4238	{
4239	STAM_PROFILE_ADV_START(&pVCpu->hwaccm.s.StatExit2Sub2, y2);
4240
4241	switch (VMX_EXIT_QUALIFICATION_CRX_ACCESS(exitQualification))
4242	{
4243	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_WRITE:
4244	{
4245	Log2(("VMX: %RGv mov cr%d, x\n", (RTGCPTR)pCtx->rip, VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification)));
4246	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitCRxWrite[VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification)]);
4247	rc = EMInterpretCRxWrite(pVM, pVCpu, CPUMCTX2CORE(pCtx),
4248	VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification),
4249	VMX_EXIT_QUALIFICATION_CRX_GENREG(exitQualification));
4250	switch (VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification))
4251	{
4252	case 0:
4253	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_GUEST_CR0 \| HWACCM_CHANGED_GUEST_CR3;
4254	break;
4255	case 2:
4256	break;
4257	case 3:
4258	Assert(!pVM->hwaccm.s.fNestedPaging \|\| !CPUMIsGuestInPagedProtectedModeEx(pCtx));
4259	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_GUEST_CR3;
4260	break;
4261	case 4:
4262	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_GUEST_CR4;
4263	break;
4264	case 8:
4265	/* CR8 contains the APIC TPR */
4266	Assert(!(pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1
4267	& VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW));
4268	break;
4269
4270	default:
4271	AssertFailed();
4272	break;
4273	}
4274	break;
4275	}
4276
4277	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ:
4278	{
4279	Log2(("VMX: mov x, crx\n"));
4280	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitCRxRead[VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification)]);
4281
4282	Assert( !pVM->hwaccm.s.fNestedPaging
4283	\|\| !CPUMIsGuestInPagedProtectedModeEx(pCtx)
4284	\|\| VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification) != DISCREG_CR3);
4285
4286	/* CR8 reads only cause an exit when the TPR shadow feature isn't present. */
4287	Assert( VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification) != 8
4288	\|\| !(pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW));
4289
4290	rc = EMInterpretCRxRead(pVM, pVCpu, CPUMCTX2CORE(pCtx),
4291	VMX_EXIT_QUALIFICATION_CRX_GENREG(exitQualification),
4292	VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification));
4293	break;
4294	}
4295
4296	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_CLTS:
4297	{
4298	Log2(("VMX: clts\n"));
4299	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitCLTS);
4300	rc = EMInterpretCLTS(pVM, pVCpu);
4301	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_GUEST_CR0;
4302	break;
4303	}
4304
4305	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_LMSW:
4306	{
4307	Log2(("VMX: lmsw %x\n", VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(exitQualification)));
4308	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitLMSW);
4309	rc = EMInterpretLMSW(pVM, pVCpu, CPUMCTX2CORE(pCtx), VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(exitQualification));
4310	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_GUEST_CR0;
4311	break;
4312	}
4313	}
4314
4315	/* Update EIP if no error occurred. */
4316	if (RT_SUCCESS(rc))
4317	pCtx->rip += cbInstr;
4318
4319	if (rc == VINF_SUCCESS)
4320	{
4321	/* Only resume if successful. */
4322	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub2, y2);
4323	goto ResumeExecution;
4324	}
4325	Assert(rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_PGM_SYNC_CR3);
4326	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub2, y2);
4327	break;
4328	}
4329
4330	case VMX_EXIT_DRX_MOVE: /* 29 Debug-register accesses. */
4331	{
4332	if ( !DBGFIsStepping(pVCpu)
4333	&& !CPUMIsHyperDebugStateActive(pVCpu))
4334	{
4335	/* Disable DRx move intercepts. */
4336	pVCpu->hwaccm.s.vmx.proc_ctls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT;
4337	rc2 = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
4338	AssertRC(rc2);
4339
4340	/* Save the host and load the guest debug state. */
4341	rc2 = CPUMR0LoadGuestDebugState(pVM, pVCpu, pCtx, true /* include DR6 */);
4342	AssertRC(rc2);
4343
4344	#ifdef LOG_ENABLED
4345	if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(exitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
4346	{
4347	Log(("VMX_EXIT_DRX_MOVE: write DR%d genreg %d\n", VMX_EXIT_QUALIFICATION_DRX_REGISTER(exitQualification),
4348	VMX_EXIT_QUALIFICATION_DRX_GENREG(exitQualification)));
4349	}
4350	else
4351	Log(("VMX_EXIT_DRX_MOVE: read DR%d\n", VMX_EXIT_QUALIFICATION_DRX_REGISTER(exitQualification)));
4352	#endif
4353
4354	#ifdef VBOX_WITH_STATISTICS
4355	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatDRxContextSwitch);
4356	if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(exitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
4357	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitDRxWrite);
4358	else
4359	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitDRxRead);
4360	#endif
4361
4362	goto ResumeExecution;
4363	}
4364
4365	/** @todo clear VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT after the first
4366	* time and restore DRx registers afterwards */
4367	if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(exitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
4368	{
4369	Log2(("VMX: mov DRx%d, genreg%d\n", VMX_EXIT_QUALIFICATION_DRX_REGISTER(exitQualification),
4370	VMX_EXIT_QUALIFICATION_DRX_GENREG(exitQualification)));
4371	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitDRxWrite);
4372	rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pCtx),
4373	VMX_EXIT_QUALIFICATION_DRX_REGISTER(exitQualification),
4374	VMX_EXIT_QUALIFICATION_DRX_GENREG(exitQualification));
4375	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_GUEST_DEBUG;
4376	Log2(("DR7=%08x\n", pCtx->dr[7]));
4377	}
4378	else
4379	{
4380	Log2(("VMX: mov x, DRx\n"));
4381	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitDRxRead);
4382	rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pCtx),
4383	VMX_EXIT_QUALIFICATION_DRX_GENREG(exitQualification),
4384	VMX_EXIT_QUALIFICATION_DRX_REGISTER(exitQualification));
4385	}
4386	/* Update EIP if no error occurred. */
4387	if (RT_SUCCESS(rc))
4388	pCtx->rip += cbInstr;
4389
4390	if (rc == VINF_SUCCESS)
4391	{
4392	/* Only resume if successful. */
4393	goto ResumeExecution;
4394	}
4395	Assert(rc == VERR_EM_INTERPRETER);
4396	break;
4397	}
4398
4399	/* Note: We'll get a #GP if the IO instruction isn't allowed (IOPL or TSS bitmap); no need to double check. */
4400	case VMX_EXIT_PORT_IO: /* 30 I/O instruction. */
4401	{
4402	STAM_PROFILE_ADV_START(&pVCpu->hwaccm.s.StatExit2Sub1, y1);
4403	uint32_t uPort;
4404	uint32_t uIOWidth = VMX_EXIT_QUALIFICATION_IO_WIDTH(exitQualification);
4405	bool fIOWrite = (VMX_EXIT_QUALIFICATION_IO_DIRECTION(exitQualification) == VMX_EXIT_QUALIFICATION_IO_DIRECTION_OUT);
4406
4407	/** @todo necessary to make the distinction? */
4408	if (VMX_EXIT_QUALIFICATION_IO_ENCODING(exitQualification) == VMX_EXIT_QUALIFICATION_IO_ENCODING_DX)
4409	uPort = pCtx->edx & 0xffff;
4410	else
4411	uPort = VMX_EXIT_QUALIFICATION_IO_PORT(exitQualification); /* Immediate encoding. */
4412
4413	if (RT_UNLIKELY(uIOWidth == 2 \|\| uIOWidth >= 4)) /* paranoia */
4414	{
4415	rc = fIOWrite ? VINF_IOM_R3_IOPORT_WRITE : VINF_IOM_R3_IOPORT_READ;
4416	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub1, y1);
4417	break;
4418	}
4419
4420	uint32_t cbSize = g_aIOSize[uIOWidth];
4421	if (VMX_EXIT_QUALIFICATION_IO_STRING(exitQualification))
4422	{
4423	/* ins/outs */
4424	PDISCPUSTATE pDis = &pVCpu->hwaccm.s.DisState;
4425
4426	/* Disassemble manually to deal with segment prefixes. */
4427	/** @todo VMX_VMCS_EXIT_GUEST_LINEAR_ADDR contains the flat pointer operand of the instruction. */
4428	/** @todo VMX_VMCS32_RO_EXIT_INSTR_INFO also contains segment prefix info. */
4429	rc2 = EMInterpretDisasCurrent(pVM, pVCpu, pDis, NULL);
4430	if (RT_SUCCESS(rc))
4431	{
4432	if (fIOWrite)
4433	{
4434	Log2(("IOMInterpretOUTSEx %RGv %x size=%d\n", (RTGCPTR)pCtx->rip, uPort, cbSize));
4435	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitIOStringWrite);
4436	rc = IOMInterpretOUTSEx(pVM, CPUMCTX2CORE(pCtx), uPort, pDis->fPrefix, (DISCPUMODE)pDis->uAddrMode, cbSize);
4437	}
4438	else
4439	{
4440	Log2(("IOMInterpretINSEx %RGv %x size=%d\n", (RTGCPTR)pCtx->rip, uPort, cbSize));
4441	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitIOStringRead);
4442	rc = IOMInterpretINSEx(pVM, CPUMCTX2CORE(pCtx), uPort, pDis->fPrefix, (DISCPUMODE)pDis->uAddrMode, cbSize);
4443	}
4444	}
4445	else
4446	rc = VINF_EM_RAW_EMULATE_INSTR;
4447	}
4448	else
4449	{
4450	/* Normal in/out */
4451	uint32_t uAndVal = g_aIOOpAnd[uIOWidth];
4452
4453	Assert(!VMX_EXIT_QUALIFICATION_IO_REP(exitQualification));
4454
4455	if (fIOWrite)
4456	{
4457	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitIOWrite);
4458	rc = IOMIOPortWrite(pVM, uPort, pCtx->eax & uAndVal, cbSize);
4459	if (rc == VINF_IOM_R3_IOPORT_WRITE)
4460	HWACCMR0SavePendingIOPortWrite(pVCpu, pCtx->rip, pCtx->rip + cbInstr, uPort, uAndVal, cbSize);
4461	}
4462	else
4463	{
4464	uint32_t u32Val = 0;
4465
4466	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitIORead);
4467	rc = IOMIOPortRead(pVM, uPort, &u32Val, cbSize);
4468	if (IOM_SUCCESS(rc))
4469	{
4470	/* Write back to the EAX register. */
4471	pCtx->eax = (pCtx->eax & ~uAndVal) \| (u32Val & uAndVal);
4472	}
4473	else
4474	if (rc == VINF_IOM_R3_IOPORT_READ)
4475	HWACCMR0SavePendingIOPortRead(pVCpu, pCtx->rip, pCtx->rip + cbInstr, uPort, uAndVal, cbSize);
4476	}
4477	}
4478
4479	/*
4480	* Handled the I/O return codes.
4481	* (The unhandled cases end up with rc == VINF_EM_RAW_EMULATE_INSTR.)
4482	*/
4483	if (IOM_SUCCESS(rc))
4484	{
4485	/* Update EIP and continue execution. */
4486	pCtx->rip += cbInstr;
4487	if (RT_LIKELY(rc == VINF_SUCCESS))
4488	{
4489	/* If any IO breakpoints are armed, then we should check if a debug trap needs to be generated. */
4490	if (pCtx->dr[7] & X86_DR7_ENABLED_MASK)
4491	{
4492	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatDRxIOCheck);
4493	for (unsigned i = 0; i < 4; i++)
4494	{
4495	unsigned uBPLen = g_aIOSize[X86_DR7_GET_LEN(pCtx->dr[7], i)];
4496
4497	if ( (uPort >= pCtx->dr[i] && uPort < pCtx->dr[i] + uBPLen)
4498	&& (pCtx->dr[7] & (X86_DR7_L(i) \| X86_DR7_G(i)))
4499	&& (pCtx->dr[7] & X86_DR7_RW(i, X86_DR7_RW_IO)) == X86_DR7_RW(i, X86_DR7_RW_IO))
4500	{
4501	uint64_t uDR6;
4502
4503	Assert(CPUMIsGuestDebugStateActive(pVCpu));
4504
4505	uDR6 = ASMGetDR6();
4506
4507	/* Clear all breakpoint status flags and set the one we just hit. */
4508	uDR6 &= ~(X86_DR6_B0\|X86_DR6_B1\|X86_DR6_B2\|X86_DR6_B3);
4509	uDR6 \|= (uint64_t)RT_BIT(i);
4510
4511	/*
4512	* Note: AMD64 Architecture Programmer's Manual 13.1:
4513	* Bits 15:13 of the DR6 register is never cleared by the processor and must
4514	* be cleared by software after the contents have been read.
4515	*/
4516	ASMSetDR6(uDR6);
4517
4518	/* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
4519	pCtx->dr[7] &= ~X86_DR7_GD;
4520
4521	/* Paranoia. */
4522	pCtx->dr[7] &= 0xffffffff; /* upper 32 bits reserved */
4523	pCtx->dr[7] &= ~(RT_BIT(11) \| RT_BIT(12) \| RT_BIT(14) \| RT_BIT(15)); /* must be zero */
4524	pCtx->dr[7] \|= 0x400; /* must be one */
4525
4526	/* Resync DR7 */
4527	rc2 = VMXWriteVMCS64(VMX_VMCS64_GUEST_DR7, pCtx->dr[7]);
4528	AssertRC(rc2);
4529
4530	/* Construct inject info. */
4531	intInfo = X86_XCPT_DB;
4532	intInfo \|= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
4533	intInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HWEXCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
4534
4535	Log(("Inject IO debug trap at %RGv\n", (RTGCPTR)pCtx->rip));
4536	rc2 = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo),
4537	0 /* cbInstr /, 0 / errCode */);
4538	AssertRC(rc2);
4539
4540	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub1, y1);
4541	goto ResumeExecution;
4542	}
4543	}
4544	}
4545	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub1, y1);
4546	goto ResumeExecution;
4547	}
4548	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub1, y1);
4549	break;
4550	}
4551
4552	#ifdef VBOX_STRICT
4553	if (rc == VINF_IOM_R3_IOPORT_READ)
4554	Assert(!fIOWrite);
4555	else if (rc == VINF_IOM_R3_IOPORT_WRITE)
4556	Assert(fIOWrite);
4557	else
4558	{
4559	AssertMsg( RT_FAILURE(rc)
4560	\|\| rc == VINF_EM_RAW_EMULATE_INSTR
4561	\|\| rc == VINF_EM_RAW_GUEST_TRAP
4562	\|\| rc == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", VBOXSTRICTRC_VAL(rc)));
4563	}
4564	#endif
4565	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub1, y1);
4566	break;
4567	}
4568
4569	case VMX_EXIT_TPR: /* 43 TPR below threshold. Guest software executed MOV to CR8. */
4570	LogFlow(("VMX_EXIT_TPR\n"));
4571	/* RIP is already set to the next instruction and the TPR has been synced back. Just resume. */
4572	goto ResumeExecution;
4573
4574	case VMX_EXIT_APIC_ACCESS: /* 44 APIC access. Guest software attempted to access memory at a physical address
4575	on the APIC-access page. */
4576	{
4577	LogFlow(("VMX_EXIT_APIC_ACCESS\n"));
4578	unsigned uAccessType = VMX_EXIT_QUALIFICATION_APIC_ACCESS_TYPE(exitQualification);
4579
4580	switch(uAccessType)
4581	{
4582	case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
4583	case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
4584	{
4585	RTGCPHYS GCPhys;
4586	PDMApicGetBase(pVM, &GCPhys);
4587	GCPhys &= PAGE_BASE_GC_MASK;
4588	GCPhys += VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(exitQualification);
4589
4590	LogFlow(("Apic access at %RGp\n", GCPhys));
4591	rc = IOMMMIOPhysHandler(pVM, (uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ) ? 0 : X86_TRAP_PF_RW,
4592	CPUMCTX2CORE(pCtx), GCPhys);
4593	if (rc == VINF_SUCCESS)
4594	goto ResumeExecution; /* rip already updated */
4595	break;
4596	}
4597
4598	default:
4599	rc = VINF_EM_RAW_EMULATE_INSTR;
4600	break;
4601	}
4602	break;
4603	}
4604
4605	case VMX_EXIT_PREEMPTION_TIMER: /* 52 VMX-preemption timer expired. The preemption timer counted down to zero. */
4606	if (!TMTimerPollBool(pVM, pVCpu))
4607	goto ResumeExecution;
4608	rc = VINF_EM_RAW_TIMER_PENDING;
4609	break;
4610
4611	default:
4612	/* The rest is handled after syncing the entire CPU state. */
4613	break;
4614	}
4615
4616
4617	/*
4618	* Note: The guest state is not entirely synced back at this stage!
4619	*/
4620
4621	/* Investigate why there was a VM-exit. (part 2) */
4622	switch (exitReason)
4623	{
4624	case VMX_EXIT_EXCEPTION: /* 0 Exception or non-maskable interrupt (NMI). */
4625	case VMX_EXIT_EXTERNAL_IRQ: /* 1 External interrupt. */
4626	case VMX_EXIT_EPT_VIOLATION:
4627	case VMX_EXIT_EPT_MISCONFIG: /* 49 EPT misconfig is used by the PGM/MMIO optimizations. */
4628	case VMX_EXIT_PREEMPTION_TIMER: /* 52 VMX-preemption timer expired. The preemption timer counted down to zero. */
4629	/* Already handled above. */
4630	break;
4631
4632	case VMX_EXIT_TRIPLE_FAULT: /* 2 Triple fault. */
4633	rc = VINF_EM_RESET; /* Triple fault equals a reset. */
4634	break;
4635
4636	case VMX_EXIT_INIT_SIGNAL: /* 3 INIT signal. */
4637	case VMX_EXIT_SIPI: /* 4 Start-up IPI (SIPI). */
4638	rc = VINF_EM_RAW_INTERRUPT;
4639	AssertFailed(); /* Can't happen. Yet. */
4640	break;
4641
4642	case VMX_EXIT_IO_SMI_IRQ: /* 5 I/O system-management interrupt (SMI). */
4643	case VMX_EXIT_SMI_IRQ: /* 6 Other SMI. */
4644	rc = VINF_EM_RAW_INTERRUPT;
4645	AssertFailed(); /* Can't happen afaik. */
4646	break;
4647
4648	case VMX_EXIT_TASK_SWITCH: /* 9 Task switch: too complicated to emulate, so fall back to the recompiler */
4649	Log(("VMX_EXIT_TASK_SWITCH: exit=%RX64\n", exitQualification));
4650	if ( (VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE(exitQualification) == VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE_IDT)
4651	&& pVCpu->hwaccm.s.Event.fPending)
4652	{
4653	/* Caused by an injected interrupt. */
4654	pVCpu->hwaccm.s.Event.fPending = false;
4655
4656	Log(("VMX_EXIT_TASK_SWITCH: reassert trap %d\n", VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVCpu->hwaccm.s.Event.intInfo)));
4657	Assert(!VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(pVCpu->hwaccm.s.Event.intInfo));
4658	rc2 = TRPMAssertTrap(pVCpu, VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVCpu->hwaccm.s.Event.intInfo), TRPM_HARDWARE_INT);
4659	AssertRC(rc2);
4660	}
4661	/* else Exceptions and software interrupts can just be restarted. */
4662	rc = VERR_EM_INTERPRETER;
4663	break;
4664
4665	case VMX_EXIT_HLT: /* 12 Guest software attempted to execute HLT. */
4666	/* Check if external interrupts are pending; if so, don't switch back. */
4667	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitHlt);
4668	pCtx->rip++; /* skip hlt */
4669	if (EMShouldContinueAfterHalt(pVCpu, pCtx))
4670	goto ResumeExecution;
4671
4672	rc = VINF_EM_HALT;
4673	break;
4674
4675	case VMX_EXIT_MWAIT: /* 36 Guest software executed MWAIT. */
4676	Log2(("VMX: mwait\n"));
4677	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitMwait);
4678	rc = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pCtx));
4679	if ( rc == VINF_EM_HALT
4680	\|\| rc == VINF_SUCCESS)
4681	{
4682	/* Update EIP and continue execution. */
4683	pCtx->rip += cbInstr;
4684
4685	/* Check if external interrupts are pending; if so, don't switch back. */
4686	if ( rc == VINF_SUCCESS
4687	\|\| ( rc == VINF_EM_HALT
4688	&& EMShouldContinueAfterHalt(pVCpu, pCtx))
4689	)
4690	goto ResumeExecution;
4691	}
4692	AssertMsg(rc == VERR_EM_INTERPRETER \|\| rc == VINF_EM_HALT, ("EMU: mwait failed with %Rrc\n", VBOXSTRICTRC_VAL(rc)));
4693	break;
4694
4695	case VMX_EXIT_RSM: /* 17 Guest software attempted to execute RSM in SMM. */
4696	AssertFailed(); /* can't happen. */
4697	rc = VERR_EM_INTERPRETER;
4698	break;
4699
4700	case VMX_EXIT_MTF: /* 37 Exit due to Monitor Trap Flag. */
4701	LogFlow(("VMX_EXIT_MTF at %RGv\n", (RTGCPTR)pCtx->rip));
4702	pVCpu->hwaccm.s.vmx.proc_ctls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_TRAP_FLAG;
4703	rc2 = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
4704	AssertRC(rc2);
4705	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitMTF);
4706	#if 0
4707	DBGFDoneStepping(pVCpu);
4708	#endif
4709	rc = VINF_EM_DBG_STOP;
4710	break;
4711
4712	case VMX_EXIT_VMCALL: /* 18 Guest software executed VMCALL. */
4713	case VMX_EXIT_VMCLEAR: /* 19 Guest software executed VMCLEAR. */
4714	case VMX_EXIT_VMLAUNCH: /* 20 Guest software executed VMLAUNCH. */
4715	case VMX_EXIT_VMPTRLD: /* 21 Guest software executed VMPTRLD. */
4716	case VMX_EXIT_VMPTRST: /* 22 Guest software executed VMPTRST. */
4717	case VMX_EXIT_VMREAD: /* 23 Guest software executed VMREAD. */
4718	case VMX_EXIT_VMRESUME: /* 24 Guest software executed VMRESUME. */
4719	case VMX_EXIT_VMWRITE: /* 25 Guest software executed VMWRITE. */
4720	case VMX_EXIT_VMXOFF: /* 26 Guest software executed VMXOFF. */
4721	case VMX_EXIT_VMXON: /* 27 Guest software executed VMXON. */
4722	/** @todo inject #UD immediately */
4723	rc = VERR_EM_INTERPRETER;
4724	break;
4725
4726	case VMX_EXIT_CPUID: /* 10 Guest software attempted to execute CPUID. */
4727	case VMX_EXIT_RDTSC: /* 16 Guest software attempted to execute RDTSC. */
4728	case VMX_EXIT_INVLPG: /* 14 Guest software attempted to execute INVLPG. */
4729	case VMX_EXIT_CRX_MOVE: /* 28 Control-register accesses. */
4730	case VMX_EXIT_DRX_MOVE: /* 29 Debug-register accesses. */
4731	case VMX_EXIT_PORT_IO: /* 30 I/O instruction. */
4732	case VMX_EXIT_RDPMC: /* 15 Guest software attempted to execute RDPMC. */
4733	case VMX_EXIT_RDTSCP: /* 51 Guest software attempted to execute RDTSCP. */
4734	/* already handled above */
4735	AssertMsg( rc == VINF_PGM_CHANGE_MODE
4736	\|\| rc == VINF_EM_RAW_INTERRUPT
4737	\|\| rc == VERR_EM_INTERPRETER
4738	\|\| rc == VINF_EM_RAW_EMULATE_INSTR
4739	\|\| rc == VINF_PGM_SYNC_CR3
4740	\|\| rc == VINF_IOM_R3_IOPORT_READ
4741	\|\| rc == VINF_IOM_R3_IOPORT_WRITE
4742	\|\| rc == VINF_EM_RAW_GUEST_TRAP
4743	\|\| rc == VINF_TRPM_XCPT_DISPATCHED
4744	\|\| rc == VINF_EM_RESCHEDULE_REM,
4745	("rc = %d\n", VBOXSTRICTRC_VAL(rc)));
4746	break;
4747
4748	case VMX_EXIT_TPR: /* 43 TPR below threshold. Guest software executed MOV to CR8. */
4749	case VMX_EXIT_RDMSR: /* 31 RDMSR. Guest software attempted to execute RDMSR. */
4750	case VMX_EXIT_WRMSR: /* 32 WRMSR. Guest software attempted to execute WRMSR. */
4751	case VMX_EXIT_PAUSE: /* 40 Guest software attempted to execute PAUSE. */
4752	case VMX_EXIT_MONITOR: /* 39 Guest software attempted to execute MONITOR. */
4753	case VMX_EXIT_APIC_ACCESS: /* 44 APIC access. Guest software attempted to access memory at a physical address
4754	on the APIC-access page. */
4755	{
4756	/*
4757	* If we decided to emulate them here, then we must sync the MSRs that could have been changed (sysenter, FS/GS base)
4758	*/
4759	rc = VERR_EM_INTERPRETER;
4760	break;
4761	}
4762
4763	case VMX_EXIT_IRQ_WINDOW: /* 7 Interrupt window. */
4764	Assert(rc == VINF_EM_RAW_INTERRUPT);
4765	break;
4766
4767	case VMX_EXIT_ERR_INVALID_GUEST_STATE: /* 33 VM-entry failure due to invalid guest state. */
4768	{
4769	#ifdef VBOX_STRICT
4770	RTCCUINTREG val2 = 0;
4771
4772	Log(("VMX_EXIT_ERR_INVALID_GUEST_STATE\n"));
4773
4774	VMXReadVMCS(VMX_VMCS64_GUEST_RIP, &val2);
4775	Log(("Old eip %RGv new %RGv\n", (RTGCPTR)pCtx->rip, (RTGCPTR)val2));
4776
4777	VMXReadVMCS(VMX_VMCS64_GUEST_CR0, &val2);
4778	Log(("VMX_VMCS_GUEST_CR0 %RX64\n", (uint64_t)val2));
4779
4780	VMXReadVMCS(VMX_VMCS64_GUEST_CR3, &val2);
4781	Log(("VMX_VMCS_GUEST_CR3 %RX64\n", (uint64_t)val2));
4782
4783	VMXReadVMCS(VMX_VMCS64_GUEST_CR4, &val2);
4784	Log(("VMX_VMCS_GUEST_CR4 %RX64\n", (uint64_t)val2));
4785
4786	VMXReadVMCS(VMX_VMCS_GUEST_RFLAGS, &val2);
4787	Log(("VMX_VMCS_GUEST_RFLAGS %08x\n", val2));
4788
4789	VMX_LOG_SELREG(CS, "CS", val2);
4790	VMX_LOG_SELREG(DS, "DS", val2);
4791	VMX_LOG_SELREG(ES, "ES", val2);
4792	VMX_LOG_SELREG(FS, "FS", val2);
4793	VMX_LOG_SELREG(GS, "GS", val2);
4794	VMX_LOG_SELREG(SS, "SS", val2);
4795	VMX_LOG_SELREG(TR, "TR", val2);
4796	VMX_LOG_SELREG(LDTR, "LDTR", val2);
4797
4798	VMXReadVMCS(VMX_VMCS64_GUEST_GDTR_BASE, &val2);
4799	Log(("VMX_VMCS_GUEST_GDTR_BASE %RX64\n", (uint64_t)val2));
4800	VMXReadVMCS(VMX_VMCS64_GUEST_IDTR_BASE, &val2);
4801	Log(("VMX_VMCS_GUEST_IDTR_BASE %RX64\n", (uint64_t)val2));
4802	#endif /* VBOX_STRICT */
4803	rc = VERR_VMX_INVALID_GUEST_STATE;
4804	break;
4805	}
4806
4807	case VMX_EXIT_ERR_MSR_LOAD: /* 34 VM-entry failure due to MSR loading. */
4808	case VMX_EXIT_ERR_MACHINE_CHECK: /* 41 VM-entry failure due to machine-check. */
4809	default:
4810	rc = VERR_VMX_UNEXPECTED_EXIT_CODE;
4811	AssertMsgFailed(("Unexpected exit code %d\n", exitReason)); /* Can't happen. */
4812	break;
4813
4814	}
4815
4816	end:
4817	/* We now going back to ring-3, so clear the action flag. */
4818	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
4819
4820	/*
4821	* Signal changes for the recompiler.
4822	*/
4823	CPUMSetChangedFlags(pVCpu,
4824	CPUM_CHANGED_SYSENTER_MSR
4825	\| CPUM_CHANGED_LDTR
4826	\| CPUM_CHANGED_GDTR
4827	\| CPUM_CHANGED_IDTR
4828	\| CPUM_CHANGED_TR
4829	\| CPUM_CHANGED_HIDDEN_SEL_REGS);
4830
4831	/*
4832	* If we executed vmlaunch/vmresume and an external IRQ was pending, then we don't have to do a full sync the next time.
4833	*/
4834	if ( exitReason == VMX_EXIT_EXTERNAL_IRQ
4835	&& !VMX_EXIT_INTERRUPTION_INFO_VALID(intInfo))
4836	{
4837	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatPendingHostIrq);
4838	/* On the next entry we'll only sync the host context. */
4839	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_HOST_CONTEXT;
4840	}
4841	else
4842	{
4843	/* On the next entry we'll sync everything. */
4844	/** @todo we can do better than this */
4845	/* Not in the VINF_PGM_CHANGE_MODE though! */
4846	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_ALL;
4847	}
4848
4849	/* Translate into a less severe return code */
4850	if (rc == VERR_EM_INTERPRETER)
4851	rc = VINF_EM_RAW_EMULATE_INSTR;
4852	else if (rc == VERR_VMX_INVALID_VMCS_PTR)
4853	{
4854	/* Try to extract more information about what might have gone wrong here. */
4855	VMXGetActivateVMCS(&pVCpu->hwaccm.s.vmx.lasterror.u64VMCSPhys);
4856	pVCpu->hwaccm.s.vmx.lasterror.ulVMCSRevision = (uint32_t )pVCpu->hwaccm.s.vmx.pvVMCS;
4857	pVCpu->hwaccm.s.vmx.lasterror.idEnteredCpu = pVCpu->hwaccm.s.idEnteredCpu;
4858	pVCpu->hwaccm.s.vmx.lasterror.idCurrentCpu = RTMpCpuId();
4859	}
4860
4861	/* Just set the correct state here instead of trying to catch every goto above. */
4862	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED, VMCPUSTATE_STARTED_EXEC);
4863
4864	#ifdef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
4865	/* Restore interrupts if we exited after disabling them. */
4866	if (uOldEFlags != ~(RTCCUINTREG)0)
4867	ASMSetFlags(uOldEFlags);
4868	#endif
4869
4870	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2, x);
4871	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit1, x);
4872	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatEntry, x);
4873	Log2(("X"));
4874	return VBOXSTRICTRC_TODO(rc);
4875	}
4876
4877
4878	/**
4879	* Enters the VT-x session.
4880	*
4881	* @returns VBox status code.
4882	* @param pVM Pointer to the VM.
4883	* @param pVCpu Pointer to the VMCPU.
4884	* @param pCpu Pointer to the CPU info struct.
4885	*/
4886	VMMR0DECL(int) VMXR0Enter(PVM pVM, PVMCPU pVCpu, PHMGLOBLCPUINFO pCpu)
4887	{
4888	Assert(pVM->hwaccm.s.vmx.fSupported);
4889	NOREF(pCpu);
4890
4891	unsigned cr4 = ASMGetCR4();
4892	if (!(cr4 & X86_CR4_VMXE))
4893	{
4894	AssertMsgFailed(("X86_CR4_VMXE should be set!\n"));
4895	return VERR_VMX_X86_CR4_VMXE_CLEARED;
4896	}
4897
4898	/* Activate the VMCS. */
4899	int rc = VMXActivateVMCS(pVCpu->hwaccm.s.vmx.HCPhysVMCS);
4900	if (RT_FAILURE(rc))
4901	return rc;
4902
4903	pVCpu->hwaccm.s.fResumeVM = false;
4904	return VINF_SUCCESS;
4905	}
4906
4907
4908	/**
4909	* Leaves the VT-x session.
4910	*
4911	* @returns VBox status code.
4912	* @param pVM Pointer to the VM.
4913	* @param pVCpu Pointer to the VMCPU.
4914	* @param pCtx Pointer to the guests CPU context.
4915	*/
4916	VMMR0DECL(int) VMXR0Leave(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
4917	{
4918	Assert(pVM->hwaccm.s.vmx.fSupported);
4919
4920	#ifdef DEBUG
4921	if (CPUMIsHyperDebugStateActive(pVCpu))
4922	{
4923	CPUMR0LoadHostDebugState(pVM, pVCpu);
4924	Assert(pVCpu->hwaccm.s.vmx.proc_ctls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT);
4925	}
4926	else
4927	#endif
4928
4929	/*
4930	* Save the guest debug state if necessary.
4931	*/
4932	if (CPUMIsGuestDebugStateActive(pVCpu))
4933	{
4934	CPUMR0SaveGuestDebugState(pVM, pVCpu, pCtx, true /* save DR6 */);
4935
4936	/* Enable DRx move intercepts again. */
4937	pVCpu->hwaccm.s.vmx.proc_ctls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT;
4938	int rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
4939	AssertRC(rc);
4940
4941	/* Resync the debug registers the next time. */
4942	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_GUEST_DEBUG;
4943	}
4944	else
4945	Assert(pVCpu->hwaccm.s.vmx.proc_ctls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT);
4946
4947	/*
4948	* Clear VMCS, marking it inactive, clearing implementation-specific data and writing
4949	* VMCS data back to memory.
4950	*/
4951	int rc = VMXClearVMCS(pVCpu->hwaccm.s.vmx.HCPhysVMCS);
4952	AssertRC(rc);
4953
4954	return VINF_SUCCESS;
4955	}
4956
4957
4958	/**
4959	* Flush the TLB using EPT.
4960	*
4961	* @returns VBox status code.
4962	* @param pVM Pointer to the VM.
4963	* @param pVCpu Pointer to the VMCPU.
4964	* @param enmFlush Type of flush.
4965	*/
4966	static void hmR0VmxFlushEPT(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_EPT enmFlush)
4967	{
4968	uint64_t descriptor[2];
4969
4970	LogFlow(("hmR0VmxFlushEPT %d\n", enmFlush));
4971	Assert(pVM->hwaccm.s.fNestedPaging);
4972	descriptor[0] = pVCpu->hwaccm.s.vmx.GCPhysEPTP;
4973	descriptor[1] = 0; /* MBZ. Intel spec. 33.3 VMX Instructions */
4974	int rc = VMXR0InvEPT(enmFlush, &descriptor[0]);
4975	AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %x %RGv failed with %d\n", enmFlush, pVCpu->hwaccm.s.vmx.GCPhysEPTP, rc));
4976	}
4977
4978
4979	/**
4980	* Flush the TLB using VPID.
4981	*
4982	* @returns VBox status code.
4983	* @param pVM Pointer to the VM.
4984	* @param pVCpu Pointer to the VMCPU (can be NULL depending on @a
4985	* enmFlush).
4986	* @param enmFlush Type of flush.
4987	* @param GCPtr Virtual address of the page to flush (can be 0 depending
4988	* on @a enmFlush).
4989	*/
4990	static void hmR0VmxFlushVPID(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_VPID enmFlush, RTGCPTR GCPtr)
4991	{
4992	#if HC_ARCH_BITS == 32
4993	/*
4994	* If we get a flush in 64-bit guest mode, then force a full TLB flush. invvpid probably takes only 32-bit addresses.
4995	*/
4996	if ( CPUMIsGuestInLongMode(pVCpu)
4997	&& !VMX_IS_64BIT_HOST_MODE())
4998	{
4999	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
5000	}
5001	else
5002	#endif
5003	{
5004	uint64_t descriptor[2];
5005
5006	Assert(pVM->hwaccm.s.vmx.fVPID);
5007	if (enmFlush == VMX_FLUSH_VPID_ALL_CONTEXTS)
5008	{
5009	descriptor[0] = 0;
5010	descriptor[1] = 0;
5011	}
5012	else
5013	{
5014	AssertPtr(pVCpu);
5015	Assert(pVCpu->hwaccm.s.uCurrentASID != 0);
5016	descriptor[0] = pVCpu->hwaccm.s.uCurrentASID;
5017	descriptor[1] = GCPtr;
5018	}
5019	int rc = VMXR0InvVPID(enmFlush, &descriptor[0]); NOREF(rc);
5020	AssertMsg(rc == VINF_SUCCESS,
5021	("VMXR0InvVPID %x %x %RGv failed with %d\n", enmFlush, pVCpu ? pVCpu->hwaccm.s.uCurrentASID : 0, GCPtr, rc));
5022	}
5023	}
5024
5025
5026	/**
5027	* Invalidates a guest page by guest virtual address. Only relevant for
5028	* EPT/VPID, otherwise there is nothing really to invalidate.
5029	*
5030	* @returns VBox status code.
5031	* @param pVM Pointer to the VM.
5032	* @param pVCpu Pointer to the VMCPU.
5033	* @param GCVirt Guest virtual address of the page to invalidate.
5034	*/
5035	VMMR0DECL(int) VMXR0InvalidatePage(PVM pVM, PVMCPU pVCpu, RTGCPTR GCVirt)
5036	{
5037	bool fFlushPending = VMCPU_FF_ISSET(pVCpu, VMCPU_FF_TLB_FLUSH);
5038
5039	Log2(("VMXR0InvalidatePage %RGv\n", GCVirt));
5040
5041	if (!fFlushPending)
5042	{
5043	/*
5044	* We must invalidate the guest TLB entry in either case, we cannot ignore it even for the EPT case
5045	* See @bugref{6043} and @bugref{6177}
5046	*
5047	* Set the VMCPU_FF_TLB_FLUSH force flag and flush before VMENTRY in hmR0VmxSetupTLB*() as this
5048	* function maybe called in a loop with individual addresses.
5049	*/
5050	if (pVM->hwaccm.s.vmx.fVPID)
5051	{
5052	/* If we can flush just this page do it, otherwise flush as little as possible. */
5053	if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVVPID_CAPS_INDIV_ADDR)
5054	hmR0VmxFlushVPID(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, GCVirt);
5055	else
5056	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
5057	}
5058	else if (pVM->hwaccm.s.fNestedPaging)
5059	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
5060	}
5061
5062	return VINF_SUCCESS;
5063	}
5064
5065
5066	/**
5067	* Invalidates a guest page by physical address. Only relevant for EPT/VPID,
5068	* otherwise there is nothing really to invalidate.
5069	*
5070	* NOTE: Assumes the current instruction references this physical page though a virtual address!!
5071	*
5072	* @returns VBox status code.
5073	* @param pVM Pointer to the VM.
5074	* @param pVCpu Pointer to the VMCPU.
5075	* @param GCPhys Guest physical address of the page to invalidate.
5076	*/
5077	VMMR0DECL(int) VMXR0InvalidatePhysPage(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys)
5078	{
5079	LogFlow(("VMXR0InvalidatePhysPage %RGp\n", GCPhys));
5080
5081	/*
5082	* We cannot flush a page by guest-physical address. invvpid takes only a linear address
5083	* while invept only flushes by EPT not individual addresses. We update the force flag here
5084	* and flush before VMENTRY in hmR0VmxSetupTLB*(). This function might be called in a loop.
5085	*/
5086	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
5087	return VINF_SUCCESS;
5088	}
5089
5090
5091	/**
5092	* Report world switch error and dump some useful debug info.
5093	*
5094	* @param pVM Pointer to the VM.
5095	* @param pVCpu Pointer to the VMCPU.
5096	* @param rc Return code.
5097	* @param pCtx Pointer to the current guest CPU context (not updated).
5098	*/
5099	static void hmR0VmxReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, VBOXSTRICTRC rc, PCPUMCTX pCtx)
5100	{
5101	NOREF(pVM);
5102
5103	switch (VBOXSTRICTRC_VAL(rc))
5104	{
5105	case VERR_VMX_INVALID_VMXON_PTR:
5106	AssertFailed();
5107	break;
5108
5109	case VERR_VMX_UNABLE_TO_START_VM:
5110	case VERR_VMX_UNABLE_TO_RESUME_VM:
5111	{
5112	int rc2;
5113	RTCCUINTREG exitReason, instrError;
5114
5115	rc2 = VMXReadVMCS(VMX_VMCS32_RO_EXIT_REASON, &exitReason);
5116	rc2 \|= VMXReadVMCS(VMX_VMCS32_RO_VM_INSTR_ERROR, &instrError);
5117	AssertRC(rc2);
5118	if (rc2 == VINF_SUCCESS)
5119	{
5120	Log(("Unable to start/resume VM for reason: %x. Instruction error %x\n", (uint32_t)exitReason,
5121	(uint32_t)instrError));
5122	Log(("Current stack %08x\n", &rc2));
5123
5124	pVCpu->hwaccm.s.vmx.lasterror.ulInstrError = instrError;
5125	pVCpu->hwaccm.s.vmx.lasterror.ulExitReason = exitReason;
5126
5127	#ifdef VBOX_STRICT
5128	RTGDTR gdtr;
5129	PCX86DESCHC pDesc;
5130	RTCCUINTREG val;
5131
5132	ASMGetGDTR(&gdtr);
5133
5134	VMXReadVMCS(VMX_VMCS64_GUEST_RIP, &val);
5135	Log(("Old eip %RGv new %RGv\n", (RTGCPTR)pCtx->rip, (RTGCPTR)val));
5136	VMXReadVMCS(VMX_VMCS_CTRL_PIN_EXEC_CONTROLS, &val);
5137	Log(("VMX_VMCS_CTRL_PIN_EXEC_CONTROLS %08x\n", val));
5138	VMXReadVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, &val);
5139	Log(("VMX_VMCS_CTRL_PROC_EXEC_CONTROLS %08x\n", val));
5140	VMXReadVMCS(VMX_VMCS_CTRL_ENTRY_CONTROLS, &val);
5141	Log(("VMX_VMCS_CTRL_ENTRY_CONTROLS %08x\n", val));
5142	VMXReadVMCS(VMX_VMCS_CTRL_EXIT_CONTROLS, &val);
5143	Log(("VMX_VMCS_CTRL_EXIT_CONTROLS %08x\n", val));
5144
5145	VMXReadVMCS(VMX_VMCS_HOST_CR0, &val);
5146	Log(("VMX_VMCS_HOST_CR0 %08x\n", val));
5147	VMXReadVMCS(VMX_VMCS_HOST_CR3, &val);
5148	Log(("VMX_VMCS_HOST_CR3 %08x\n", val));
5149	VMXReadVMCS(VMX_VMCS_HOST_CR4, &val);
5150	Log(("VMX_VMCS_HOST_CR4 %08x\n", val));
5151
5152	VMXReadVMCS(VMX_VMCS16_HOST_FIELD_CS, &val);
5153	Log(("VMX_VMCS_HOST_FIELD_CS %08x\n", val));
5154	VMXReadVMCS(VMX_VMCS_GUEST_RFLAGS, &val);
5155	Log(("VMX_VMCS_GUEST_RFLAGS %08x\n", val));
5156
5157	if (val < gdtr.cbGdt)
5158	{
5159	pDesc = (PCX86DESCHC)(gdtr.pGdt + (val & X86_SEL_MASK));
5160	HWACCMR0DumpDescriptor(pDesc, val, "CS: ");
5161	}
5162
5163	VMXReadVMCS(VMX_VMCS16_HOST_FIELD_DS, &val);
5164	Log(("VMX_VMCS_HOST_FIELD_DS %08x\n", val));
5165	if (val < gdtr.cbGdt)
5166	{
5167	pDesc = (PCX86DESCHC)(gdtr.pGdt + (val & X86_SEL_MASK));
5168	HWACCMR0DumpDescriptor(pDesc, val, "DS: ");
5169	}
5170
5171	VMXReadVMCS(VMX_VMCS16_HOST_FIELD_ES, &val);
5172	Log(("VMX_VMCS_HOST_FIELD_ES %08x\n", val));
5173	if (val < gdtr.cbGdt)
5174	{
5175	pDesc = (PCX86DESCHC)(gdtr.pGdt + (val & X86_SEL_MASK));
5176	HWACCMR0DumpDescriptor(pDesc, val, "ES: ");
5177	}
5178
5179	VMXReadVMCS(VMX_VMCS16_HOST_FIELD_FS, &val);
5180	Log(("VMX_VMCS16_HOST_FIELD_FS %08x\n", val));
5181	if (val < gdtr.cbGdt)
5182	{
5183	pDesc = (PCX86DESCHC)(gdtr.pGdt + (val & X86_SEL_MASK));
5184	HWACCMR0DumpDescriptor(pDesc, val, "FS: ");
5185	}
5186
5187	VMXReadVMCS(VMX_VMCS16_HOST_FIELD_GS, &val);
5188	Log(("VMX_VMCS16_HOST_FIELD_GS %08x\n", val));
5189	if (val < gdtr.cbGdt)
5190	{
5191	pDesc = (PCX86DESCHC)(gdtr.pGdt + (val & X86_SEL_MASK));
5192	HWACCMR0DumpDescriptor(pDesc, val, "GS: ");
5193	}
5194
5195	VMXReadVMCS(VMX_VMCS16_HOST_FIELD_SS, &val);
5196	Log(("VMX_VMCS16_HOST_FIELD_SS %08x\n", val));
5197	if (val < gdtr.cbGdt)
5198	{
5199	pDesc = (PCX86DESCHC)(gdtr.pGdt + (val & X86_SEL_MASK));
5200	HWACCMR0DumpDescriptor(pDesc, val, "SS: ");
5201	}
5202
5203	VMXReadVMCS(VMX_VMCS16_HOST_FIELD_TR, &val);
5204	Log(("VMX_VMCS16_HOST_FIELD_TR %08x\n", val));
5205	if (val < gdtr.cbGdt)
5206	{
5207	pDesc = (PCX86DESCHC)(gdtr.pGdt + (val & X86_SEL_MASK));
5208	HWACCMR0DumpDescriptor(pDesc, val, "TR: ");
5209	}
5210
5211	VMXReadVMCS(VMX_VMCS_HOST_TR_BASE, &val);
5212	Log(("VMX_VMCS_HOST_TR_BASE %RHv\n", val));
5213	VMXReadVMCS(VMX_VMCS_HOST_GDTR_BASE, &val);
5214	Log(("VMX_VMCS_HOST_GDTR_BASE %RHv\n", val));
5215	VMXReadVMCS(VMX_VMCS_HOST_IDTR_BASE, &val);
5216	Log(("VMX_VMCS_HOST_IDTR_BASE %RHv\n", val));
5217	VMXReadVMCS(VMX_VMCS32_HOST_SYSENTER_CS, &val);
5218	Log(("VMX_VMCS_HOST_SYSENTER_CS %08x\n", val));
5219	VMXReadVMCS(VMX_VMCS_HOST_SYSENTER_EIP, &val);
5220	Log(("VMX_VMCS_HOST_SYSENTER_EIP %RHv\n", val));
5221	VMXReadVMCS(VMX_VMCS_HOST_SYSENTER_ESP, &val);
5222	Log(("VMX_VMCS_HOST_SYSENTER_ESP %RHv\n", val));
5223	VMXReadVMCS(VMX_VMCS_HOST_RSP, &val);
5224	Log(("VMX_VMCS_HOST_RSP %RHv\n", val));
5225	VMXReadVMCS(VMX_VMCS_HOST_RIP, &val);
5226	Log(("VMX_VMCS_HOST_RIP %RHv\n", val));
5227	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
5228	if (VMX_IS_64BIT_HOST_MODE())
5229	{
5230	Log(("MSR_K6_EFER = %RX64\n", ASMRdMsr(MSR_K6_EFER)));
5231	Log(("MSR_K6_STAR = %RX64\n", ASMRdMsr(MSR_K6_STAR)));
5232	Log(("MSR_K8_LSTAR = %RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
5233	Log(("MSR_K8_CSTAR = %RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
5234	Log(("MSR_K8_SF_MASK = %RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
5235	Log(("MSR_K8_KERNEL_GS_BASE = %RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
5236	}
5237	# endif
5238	#endif /* VBOX_STRICT */
5239	}
5240	break;
5241	}
5242
5243	default:
5244	/* impossible */
5245	AssertMsgFailed(("%Rrc (%#x)\n", VBOXSTRICTRC_VAL(rc), VBOXSTRICTRC_VAL(rc)));
5246	break;
5247	}
5248	}
5249
5250
5251	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
5252	/**
5253	* Prepares for and executes VMLAUNCH (64 bits guest mode).
5254	*
5255	* @returns VBox status code.
5256	* @param fResume Whether to vmlauch/vmresume.
5257	* @param pCtx Pointer to the guest CPU context.
5258	* @param pCache Pointer to the VMCS cache.
5259	* @param pVM Pointer to the VM.
5260	* @param pVCpu Pointer to the VMCPU.
5261	*/
5262	DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
5263	{
5264	uint32_t aParam[6];
5265	PHMGLOBLCPUINFO pCpu;
5266	RTHCPHYS HCPhysCpuPage;
5267	int rc;
5268
5269	pCpu = HWACCMR0GetCurrentCpu();
5270	HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
5271
5272	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5273	pCache->uPos = 1;
5274	pCache->interPD = PGMGetInterPaeCR3(pVM);
5275	pCache->pSwitcher = (uint64_t)pVM->hwaccm.s.pfnHost32ToGuest64R0;
5276	#endif
5277
5278	#ifdef DEBUG
5279	pCache->TestIn.HCPhysCpuPage= 0;
5280	pCache->TestIn.HCPhysVMCS = 0;
5281	pCache->TestIn.pCache = 0;
5282	pCache->TestOut.HCPhysVMCS = 0;
5283	pCache->TestOut.pCache = 0;
5284	pCache->TestOut.pCtx = 0;
5285	pCache->TestOut.eflags = 0;
5286	#endif
5287
5288	aParam[0] = (uint32_t)(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
5289	aParam[1] = (uint32_t)(HCPhysCpuPage >> 32); /* Param 1: VMXON physical address - Hi. */
5290	aParam[2] = (uint32_t)(pVCpu->hwaccm.s.vmx.HCPhysVMCS); /* Param 2: VMCS physical address - Lo. */
5291	aParam[3] = (uint32_t)(pVCpu->hwaccm.s.vmx.HCPhysVMCS >> 32); /* Param 2: VMCS physical address - Hi. */
5292	aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hwaccm.s.vmx.VMCSCache);
5293	aParam[5] = 0;
5294
5295	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5296	pCtx->dr[4] = pVM->hwaccm.s.vmx.pScratchPhys + 16 + 8;
5297	(uint32_t )(pVM->hwaccm.s.vmx.pScratch + 16 + 8) = 1;
5298	#endif
5299	rc = VMXR0Execute64BitsHandler(pVM, pVCpu, pCtx, pVM->hwaccm.s.pfnVMXGCStartVM64, 6, &aParam[0]);
5300
5301	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5302	Assert((uint32_t )(pVM->hwaccm.s.vmx.pScratch + 16 + 8) == 5);
5303	Assert(pCtx->dr[4] == 10);
5304	(uint32_t )(pVM->hwaccm.s.vmx.pScratch + 16 + 8) = 0xff;
5305	#endif
5306
5307	#ifdef DEBUG
5308	AssertMsg(pCache->TestIn.HCPhysCpuPage== HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
5309	AssertMsg(pCache->TestIn.HCPhysVMCS == pVCpu->hwaccm.s.vmx.HCPhysVMCS, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVMCS,
5310	pVCpu->hwaccm.s.vmx.HCPhysVMCS));
5311	AssertMsg(pCache->TestIn.HCPhysVMCS == pCache->TestOut.HCPhysVMCS, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVMCS,
5312	pCache->TestOut.HCPhysVMCS));
5313	AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
5314	pCache->TestOut.pCache));
5315	AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hwaccm.s.vmx.VMCSCache),
5316	("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hwaccm.s.vmx.VMCSCache)));
5317	AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
5318	pCache->TestOut.pCtx));
5319	Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
5320	#endif
5321	return rc;
5322	}
5323
5324
5325	# ifdef VBOX_STRICT
5326	static bool hmR0VmxIsValidReadField(uint32_t idxField)
5327	{
5328	switch (idxField)
5329	{
5330	case VMX_VMCS64_GUEST_RIP:
5331	case VMX_VMCS64_GUEST_RSP:
5332	case VMX_VMCS_GUEST_RFLAGS:
5333	case VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE:
5334	case VMX_VMCS_CTRL_CR0_READ_SHADOW:
5335	case VMX_VMCS64_GUEST_CR0:
5336	case VMX_VMCS_CTRL_CR4_READ_SHADOW:
5337	case VMX_VMCS64_GUEST_CR4:
5338	case VMX_VMCS64_GUEST_DR7:
5339	case VMX_VMCS32_GUEST_SYSENTER_CS:
5340	case VMX_VMCS64_GUEST_SYSENTER_EIP:
5341	case VMX_VMCS64_GUEST_SYSENTER_ESP:
5342	case VMX_VMCS32_GUEST_GDTR_LIMIT:
5343	case VMX_VMCS64_GUEST_GDTR_BASE:
5344	case VMX_VMCS32_GUEST_IDTR_LIMIT:
5345	case VMX_VMCS64_GUEST_IDTR_BASE:
5346	case VMX_VMCS16_GUEST_FIELD_CS:
5347	case VMX_VMCS32_GUEST_CS_LIMIT:
5348	case VMX_VMCS64_GUEST_CS_BASE:
5349	case VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS:
5350	case VMX_VMCS16_GUEST_FIELD_DS:
5351	case VMX_VMCS32_GUEST_DS_LIMIT:
5352	case VMX_VMCS64_GUEST_DS_BASE:
5353	case VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS:
5354	case VMX_VMCS16_GUEST_FIELD_ES:
5355	case VMX_VMCS32_GUEST_ES_LIMIT:
5356	case VMX_VMCS64_GUEST_ES_BASE:
5357	case VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS:
5358	case VMX_VMCS16_GUEST_FIELD_FS:
5359	case VMX_VMCS32_GUEST_FS_LIMIT:
5360	case VMX_VMCS64_GUEST_FS_BASE:
5361	case VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS:
5362	case VMX_VMCS16_GUEST_FIELD_GS:
5363	case VMX_VMCS32_GUEST_GS_LIMIT:
5364	case VMX_VMCS64_GUEST_GS_BASE:
5365	case VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS:
5366	case VMX_VMCS16_GUEST_FIELD_SS:
5367	case VMX_VMCS32_GUEST_SS_LIMIT:
5368	case VMX_VMCS64_GUEST_SS_BASE:
5369	case VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS:
5370	case VMX_VMCS16_GUEST_FIELD_LDTR:
5371	case VMX_VMCS32_GUEST_LDTR_LIMIT:
5372	case VMX_VMCS64_GUEST_LDTR_BASE:
5373	case VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS:
5374	case VMX_VMCS16_GUEST_FIELD_TR:
5375	case VMX_VMCS32_GUEST_TR_LIMIT:
5376	case VMX_VMCS64_GUEST_TR_BASE:
5377	case VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS:
5378	case VMX_VMCS32_RO_EXIT_REASON:
5379	case VMX_VMCS32_RO_VM_INSTR_ERROR:
5380	case VMX_VMCS32_RO_EXIT_INSTR_LENGTH:
5381	case VMX_VMCS32_RO_EXIT_INTERRUPTION_ERRCODE:
5382	case VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO:
5383	case VMX_VMCS32_RO_EXIT_INSTR_INFO:
5384	case VMX_VMCS_RO_EXIT_QUALIFICATION:
5385	case VMX_VMCS32_RO_IDT_INFO:
5386	case VMX_VMCS32_RO_IDT_ERRCODE:
5387	case VMX_VMCS64_GUEST_CR3:
5388	case VMX_VMCS_EXIT_PHYS_ADDR_FULL:
5389	return true;
5390	}
5391	return false;
5392	}
5393
5394
5395	static bool hmR0VmxIsValidWriteField(uint32_t idxField)
5396	{
5397	switch (idxField)
5398	{
5399	case VMX_VMCS64_GUEST_LDTR_BASE:
5400	case VMX_VMCS64_GUEST_TR_BASE:
5401	case VMX_VMCS64_GUEST_GDTR_BASE:
5402	case VMX_VMCS64_GUEST_IDTR_BASE:
5403	case VMX_VMCS64_GUEST_SYSENTER_EIP:
5404	case VMX_VMCS64_GUEST_SYSENTER_ESP:
5405	case VMX_VMCS64_GUEST_CR0:
5406	case VMX_VMCS64_GUEST_CR4:
5407	case VMX_VMCS64_GUEST_CR3:
5408	case VMX_VMCS64_GUEST_DR7:
5409	case VMX_VMCS64_GUEST_RIP:
5410	case VMX_VMCS64_GUEST_RSP:
5411	case VMX_VMCS64_GUEST_CS_BASE:
5412	case VMX_VMCS64_GUEST_DS_BASE:
5413	case VMX_VMCS64_GUEST_ES_BASE:
5414	case VMX_VMCS64_GUEST_FS_BASE:
5415	case VMX_VMCS64_GUEST_GS_BASE:
5416	case VMX_VMCS64_GUEST_SS_BASE:
5417	return true;
5418	}
5419	return false;
5420	}
5421	# endif /* VBOX_STRICT */
5422
5423
5424	/**
5425	* Executes the specified handler in 64-bit mode.
5426	*
5427	* @returns VBox status code.
5428	* @param pVM Pointer to the VM.
5429	* @param pVCpu Pointer to the VMCPU.
5430	* @param pCtx Pointer to the guest CPU context.
5431	* @param pfnHandler Pointer to the RC handler function.
5432	* @param cbParam Number of parameters.
5433	* @param paParam Array of 32-bit parameters.
5434	*/
5435	VMMR0DECL(int) VMXR0Execute64BitsHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, RTRCPTR pfnHandler, uint32_t cbParam,
5436	uint32_t *paParam)
5437	{
5438	int rc, rc2;
5439	PHMGLOBLCPUINFO pCpu;
5440	RTHCPHYS HCPhysCpuPage;
5441	RTHCUINTREG uOldEFlags;
5442
5443	AssertReturn(pVM->hwaccm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
5444	Assert(pfnHandler);
5445	Assert(pVCpu->hwaccm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hwaccm.s.vmx.VMCSCache.Write.aField));
5446	Assert(pVCpu->hwaccm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hwaccm.s.vmx.VMCSCache.Read.aField));
5447
5448	#ifdef VBOX_STRICT
5449	for (unsigned i=0;i<pVCpu->hwaccm.s.vmx.VMCSCache.Write.cValidEntries;i++)
5450	Assert(hmR0VmxIsValidWriteField(pVCpu->hwaccm.s.vmx.VMCSCache.Write.aField[i]));
5451
5452	for (unsigned i=0;i<pVCpu->hwaccm.s.vmx.VMCSCache.Read.cValidEntries;i++)
5453	Assert(hmR0VmxIsValidReadField(pVCpu->hwaccm.s.vmx.VMCSCache.Read.aField[i]));
5454	#endif
5455
5456	/* Disable interrupts. */
5457	uOldEFlags = ASMIntDisableFlags();
5458
5459	#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
5460	RTCPUID idHostCpu = RTMpCpuId();
5461	CPUMR0SetLApic(pVM, idHostCpu);
5462	#endif
5463
5464	pCpu = HWACCMR0GetCurrentCpu();
5465	HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
5466
5467	/* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
5468	VMXClearVMCS(pVCpu->hwaccm.s.vmx.HCPhysVMCS);
5469
5470	/* Leave VMX Root Mode. */
5471	VMXDisable();
5472
5473	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
5474
5475	CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
5476	CPUMSetHyperEIP(pVCpu, pfnHandler);
5477	for (int i=(int)cbParam-1;i>=0;i--)
5478	CPUMPushHyper(pVCpu, paParam[i]);
5479
5480	STAM_PROFILE_ADV_START(&pVCpu->hwaccm.s.StatWorldSwitch3264, z);
5481
5482	/* Call switcher. */
5483	rc = pVM->hwaccm.s.pfnHost32ToGuest64R0(pVM, RT_OFFSETOF(VM, aCpus[pVCpu->idCpu].cpum) - RT_OFFSETOF(VM, cpum));
5484	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatWorldSwitch3264, z);
5485
5486	/* Make sure the VMX instructions don't cause #UD faults. */
5487	ASMSetCR4(ASMGetCR4() \| X86_CR4_VMXE);
5488
5489	/* Enter VMX Root Mode */
5490	rc2 = VMXEnable(HCPhysCpuPage);
5491	if (RT_FAILURE(rc2))
5492	{
5493	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
5494	ASMSetFlags(uOldEFlags);
5495	return VERR_VMX_VMXON_FAILED;
5496	}
5497
5498	rc2 = VMXActivateVMCS(pVCpu->hwaccm.s.vmx.HCPhysVMCS);
5499	AssertRC(rc2);
5500	Assert(!(ASMGetFlags() & X86_EFL_IF));
5501	ASMSetFlags(uOldEFlags);
5502	return rc;
5503	}
5504	#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL) */
5505
5506
5507	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
5508	/**
5509	* Executes VMWRITE.
5510	*
5511	* @returns VBox status code
5512	* @param pVCpu Pointer to the VMCPU.
5513	* @param idxField VMCS field index.
5514	* @param u64Val 16, 32 or 64 bits value.
5515	*/
5516	VMMR0DECL(int) VMXWriteVMCS64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5517	{
5518	int rc;
5519	switch (idxField)
5520	{
5521	case VMX_VMCS_CTRL_TSC_OFFSET_FULL:
5522	case VMX_VMCS_CTRL_IO_BITMAP_A_FULL:
5523	case VMX_VMCS_CTRL_IO_BITMAP_B_FULL:
5524	case VMX_VMCS_CTRL_MSR_BITMAP_FULL:
5525	case VMX_VMCS_CTRL_VMEXIT_MSR_STORE_FULL:
5526	case VMX_VMCS_CTRL_VMEXIT_MSR_LOAD_FULL:
5527	case VMX_VMCS_CTRL_VMENTRY_MSR_LOAD_FULL:
5528	case VMX_VMCS_CTRL_VAPIC_PAGEADDR_FULL:
5529	case VMX_VMCS_CTRL_APIC_ACCESSADDR_FULL:
5530	case VMX_VMCS_GUEST_LINK_PTR_FULL:
5531	case VMX_VMCS_GUEST_PDPTR0_FULL:
5532	case VMX_VMCS_GUEST_PDPTR1_FULL:
5533	case VMX_VMCS_GUEST_PDPTR2_FULL:
5534	case VMX_VMCS_GUEST_PDPTR3_FULL:
5535	case VMX_VMCS_GUEST_DEBUGCTL_FULL:
5536	case VMX_VMCS_GUEST_EFER_FULL:
5537	case VMX_VMCS_CTRL_EPTP_FULL:
5538	/* These fields consist of two parts, which are both writable in 32 bits mode. */
5539	rc = VMXWriteVMCS32(idxField, u64Val);
5540	rc \|= VMXWriteVMCS32(idxField + 1, (uint32_t)(u64Val >> 32ULL));
5541	AssertRC(rc);
5542	return rc;
5543
5544	case VMX_VMCS64_GUEST_LDTR_BASE:
5545	case VMX_VMCS64_GUEST_TR_BASE:
5546	case VMX_VMCS64_GUEST_GDTR_BASE:
5547	case VMX_VMCS64_GUEST_IDTR_BASE:
5548	case VMX_VMCS64_GUEST_SYSENTER_EIP:
5549	case VMX_VMCS64_GUEST_SYSENTER_ESP:
5550	case VMX_VMCS64_GUEST_CR0:
5551	case VMX_VMCS64_GUEST_CR4:
5552	case VMX_VMCS64_GUEST_CR3:
5553	case VMX_VMCS64_GUEST_DR7:
5554	case VMX_VMCS64_GUEST_RIP:
5555	case VMX_VMCS64_GUEST_RSP:
5556	case VMX_VMCS64_GUEST_CS_BASE:
5557	case VMX_VMCS64_GUEST_DS_BASE:
5558	case VMX_VMCS64_GUEST_ES_BASE:
5559	case VMX_VMCS64_GUEST_FS_BASE:
5560	case VMX_VMCS64_GUEST_GS_BASE:
5561	case VMX_VMCS64_GUEST_SS_BASE:
5562	/* Queue a 64 bits value as we can't set it in 32 bits host mode. */
5563	if (u64Val >> 32ULL)
5564	rc = VMXWriteCachedVMCSEx(pVCpu, idxField, u64Val);
5565	else
5566	rc = VMXWriteVMCS32(idxField, (uint32_t)u64Val);
5567
5568	return rc;
5569
5570	default:
5571	AssertMsgFailed(("Unexpected field %x\n", idxField));
5572	return VERR_INVALID_PARAMETER;
5573	}
5574	}
5575
5576
5577	/**
5578	* Cache VMCS writes for performance reasons (Darwin) and for running 64 bits guests on 32 bits hosts.
5579	*
5580	* @param pVCpu Pointer to the VMCPU.
5581	* @param idxField VMCS field index.
5582	* @param u64Val 16, 32 or 64 bits value.
5583	*/
5584	VMMR0DECL(int) VMXWriteCachedVMCSEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5585	{
5586	PVMCSCACHE pCache = &pVCpu->hwaccm.s.vmx.VMCSCache;
5587
5588	AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1,
5589	("entries=%x\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
5590
5591	/* Make sure there are no duplicates. */
5592	for (unsigned i = 0; i < pCache->Write.cValidEntries; i++)
5593	{
5594	if (pCache->Write.aField[i] == idxField)
5595	{
5596	pCache->Write.aFieldVal[i] = u64Val;
5597	return VINF_SUCCESS;
5598	}
5599	}
5600
5601	pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
5602	pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
5603	pCache->Write.cValidEntries++;
5604	return VINF_SUCCESS;
5605	}
5606
5607	#endif /* HC_ARCH_BITS == 32 && !VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0 */
5608

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMR0/HWVMXR0.cpp@ 42463

Download in other formats: