HWVMXR0.cpp@ 43303

Last change on this file since 43303 was 43150, checked in by vboxsync, 12 years ago
VMM/VMMR0: fix OS X kernel panic due to skipped auto load/store of STAR MSR on 32-bit OS X while running 64-bit guests. See #6313 for details.
Property svn:eol-style set to `native` Property svn:keywords set to `Id Revision`
File size: 221.5 KB

Line
1	/* $Id: HWVMXR0.cpp 43150 2012-09-03 11:44:17Z vboxsync $ */
2	/** @file
3	* HM VMX (VT-x) - Host Context Ring-0.
4	*/
5
6	/*
7	* Copyright (C) 2006-2012 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18
19	/*******************************************************************************
20	* Header Files *
21	*******************************************************************************/
22	#define LOG_GROUP LOG_GROUP_HWACCM
23	#include <iprt/asm-amd64-x86.h>
24	#include <VBox/vmm/hwaccm.h>
25	#include <VBox/vmm/pgm.h>
26	#include <VBox/vmm/dbgf.h>
27	#include <VBox/vmm/dbgftrace.h>
28	#include <VBox/vmm/selm.h>
29	#include <VBox/vmm/iom.h>
30	#ifdef VBOX_WITH_REM
31	# include <VBox/vmm/rem.h>
32	#endif
33	#include <VBox/vmm/tm.h>
34	#include "HWACCMInternal.h"
35	#include <VBox/vmm/vm.h>
36	#include <VBox/vmm/pdmapi.h>
37	#include <VBox/err.h>
38	#include <VBox/log.h>
39	#include <iprt/assert.h>
40	#include <iprt/param.h>
41	#include <iprt/string.h>
42	#include <iprt/time.h>
43	#ifdef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
44	# include <iprt/thread.h>
45	#endif
46	#include <iprt/x86.h>
47	#include "HWVMXR0.h"
48
49	#include "dtrace/VBoxVMM.h"
50
51
52	/*******************************************************************************
53	* Defined Constants And Macros *
54	*******************************************************************************/
55	#if defined(RT_ARCH_AMD64)
56	# define VMX_IS_64BIT_HOST_MODE() (true)
57	#elif defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
58	# define VMX_IS_64BIT_HOST_MODE() (g_fVMXIs64bitHost != 0)
59	#else
60	# define VMX_IS_64BIT_HOST_MODE() (false)
61	#endif
62
63
64	/*******************************************************************************
65	* Global Variables *
66	*******************************************************************************/
67	/* IO operation lookup arrays. */
68	static uint32_t const g_aIOSize[4] = {1, 2, 0, 4};
69	static uint32_t const g_aIOOpAnd[4] = {0xff, 0xffff, 0, 0xffffffff};
70
71	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
72	/** See HWACCMR0A.asm. */
73	extern "C" uint32_t g_fVMXIs64bitHost;
74	#endif
75
76
77	/*******************************************************************************
78	* Local Functions *
79	*******************************************************************************/
80	static DECLCALLBACK(void) hmR0VmxSetupTLBEPT(PVM pVM, PVMCPU pVCpu);
81	static DECLCALLBACK(void) hmR0VmxSetupTLBVPID(PVM pVM, PVMCPU pVCpu);
82	static DECLCALLBACK(void) hmR0VmxSetupTLBBoth(PVM pVM, PVMCPU pVCpu);
83	static DECLCALLBACK(void) hmR0VmxSetupTLBDummy(PVM pVM, PVMCPU pVCpu);
84	static void hmR0VmxFlushEPT(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_EPT enmFlush);
85	static void hmR0VmxFlushVPID(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_VPID enmFlush, RTGCPTR GCPtr);
86	static void hmR0VmxUpdateExceptionBitmap(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx);
87	static void hmR0VmxSetMSRPermission(PVMCPU pVCpu, unsigned ulMSR, bool fRead, bool fWrite);
88	static void hmR0VmxReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, VBOXSTRICTRC rc, PCPUMCTX pCtx);
89
90
91	/**
92	* Updates error from VMCS to HWACCMCPU's lasterror record.
93	*
94	* @param pVM Pointer to the VM.
95	* @param pVCpu Pointer to the VMCPU.
96	* @param rc The error code.
97	*/
98	static void hmR0VmxCheckError(PVM pVM, PVMCPU pVCpu, int rc)
99	{
100	if (rc == VERR_VMX_GENERIC)
101	{
102	RTCCUINTREG instrError;
103
104	VMXReadVMCS(VMX_VMCS32_RO_VM_INSTR_ERROR, &instrError);
105	pVCpu->hwaccm.s.vmx.lasterror.ulInstrError = instrError;
106	}
107	pVM->hwaccm.s.lLastError = rc;
108	}
109
110
111	/**
112	* Sets up and activates VT-x on the current CPU.
113	*
114	* @returns VBox status code.
115	* @param pCpu Pointer to the CPU info struct.
116	* @param pVM Pointer to the VM. (can be NULL after a resume!!)
117	* @param pvCpuPage Pointer to the global CPU page.
118	* @param HCPhysCpuPage Physical address of the global CPU page.
119	*/
120	VMMR0DECL(int) VMXR0EnableCpu(PHMGLOBLCPUINFO pCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
121	{
122	AssertReturn(HCPhysCpuPage != 0 && HCPhysCpuPage != NIL_RTHCPHYS, VERR_INVALID_PARAMETER);
123	AssertReturn(pvCpuPage, VERR_INVALID_PARAMETER);
124
125	if (pVM)
126	{
127	/* Set revision dword at the beginning of the VMXON structure. */
128	(uint32_t )pvCpuPage = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hwaccm.s.vmx.msr.vmx_basic_info);
129	}
130
131	/** @todo we should unmap the two pages from the virtual address space in order to prevent accidental corruption.
132	* (which can have very bad consequences!!!)
133	*/
134
135	if (ASMGetCR4() & X86_CR4_VMXE)
136	return VERR_VMX_IN_VMX_ROOT_MODE;
137
138	ASMSetCR4(ASMGetCR4() \| X86_CR4_VMXE); /* Make sure the VMX instructions don't cause #UD faults. */
139
140	/*
141	* Enter VM root mode.
142	*/
143	int rc = VMXEnable(HCPhysCpuPage);
144	if (RT_FAILURE(rc))
145	{
146	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
147	return VERR_VMX_VMXON_FAILED;
148	}
149
150	/*
151	* Flush all VPIDs (in case we or any other hypervisor have been using VPIDs) so that
152	* we can avoid an explicit flush while using new VPIDs. We would still need to flush
153	* each time while reusing a VPID after hitting the MaxASID limit once.
154	*/
155	if ( pVM
156	&& pVM->hwaccm.s.vmx.fVPID
157	&& (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVVPID_CAPS_ALL_CONTEXTS))
158	{
159	hmR0VmxFlushVPID(pVM, NULL /* pvCpu /, VMX_FLUSH_VPID_ALL_CONTEXTS, 0 / GCPtr */);
160	pCpu->fFlushASIDBeforeUse = false;
161	}
162	else
163	pCpu->fFlushASIDBeforeUse = true;
164
165	/*
166	* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}.
167	*/
168	++pCpu->cTLBFlushes;
169
170	return VINF_SUCCESS;
171	}
172
173
174	/**
175	* Deactivates VT-x on the current CPU.
176	*
177	* @returns VBox status code.
178	* @param pCpu Pointer to the CPU info struct.
179	* @param pvCpuPage Pointer to the global CPU page.
180	* @param HCPhysCpuPage Physical address of the global CPU page.
181	*/
182	VMMR0DECL(int) VMXR0DisableCpu(PHMGLOBLCPUINFO pCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
183	{
184	AssertReturn(HCPhysCpuPage != 0 && HCPhysCpuPage != NIL_RTHCPHYS, VERR_INVALID_PARAMETER);
185	AssertReturn(pvCpuPage, VERR_INVALID_PARAMETER);
186	NOREF(pCpu);
187
188	/* If we're somehow not in VMX root mode, then we shouldn't dare leaving it. */
189	if (!(ASMGetCR4() & X86_CR4_VMXE))
190	return VERR_VMX_NOT_IN_VMX_ROOT_MODE;
191
192	/* Leave VMX Root Mode. */
193	VMXDisable();
194
195	/* And clear the X86_CR4_VMXE bit. */
196	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
197	return VINF_SUCCESS;
198	}
199
200
201	/**
202	* Does Ring-0 per VM VT-x initialization.
203	*
204	* @returns VBox status code.
205	* @param pVM Pointer to the VM.
206	*/
207	VMMR0DECL(int) VMXR0InitVM(PVM pVM)
208	{
209	int rc;
210
211	#ifdef LOG_ENABLED
212	SUPR0Printf("VMXR0InitVM %p\n", pVM);
213	#endif
214
215	pVM->hwaccm.s.vmx.pMemObjAPIC = NIL_RTR0MEMOBJ;
216
217	if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
218	{
219	/* Allocate one page for the APIC physical page (serves for filtering accesses). */
220	rc = RTR0MemObjAllocCont(&pVM->hwaccm.s.vmx.pMemObjAPIC, PAGE_SIZE, true /* executable R0 mapping */);
221	AssertRC(rc);
222	if (RT_FAILURE(rc))
223	return rc;
224
225	pVM->hwaccm.s.vmx.pAPIC = (uint8_t *)RTR0MemObjAddress(pVM->hwaccm.s.vmx.pMemObjAPIC);
226	pVM->hwaccm.s.vmx.pAPICPhys = RTR0MemObjGetPagePhysAddr(pVM->hwaccm.s.vmx.pMemObjAPIC, 0);
227	ASMMemZero32(pVM->hwaccm.s.vmx.pAPIC, PAGE_SIZE);
228	}
229	else
230	{
231	pVM->hwaccm.s.vmx.pMemObjAPIC = 0;
232	pVM->hwaccm.s.vmx.pAPIC = 0;
233	pVM->hwaccm.s.vmx.pAPICPhys = 0;
234	}
235
236	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
237	{
238	rc = RTR0MemObjAllocCont(&pVM->hwaccm.s.vmx.pMemObjScratch, PAGE_SIZE, true /* executable R0 mapping */);
239	AssertRC(rc);
240	if (RT_FAILURE(rc))
241	return rc;
242
243	pVM->hwaccm.s.vmx.pScratch = (uint8_t *)RTR0MemObjAddress(pVM->hwaccm.s.vmx.pMemObjScratch);
244	pVM->hwaccm.s.vmx.pScratchPhys = RTR0MemObjGetPagePhysAddr(pVM->hwaccm.s.vmx.pMemObjScratch, 0);
245
246	ASMMemZero32(pVM->hwaccm.s.vmx.pScratch, PAGE_SIZE);
247	strcpy((char *)pVM->hwaccm.s.vmx.pScratch, "SCRATCH Magic");
248	(uint64_t )(pVM->hwaccm.s.vmx.pScratch + 16) = UINT64_C(0xDEADBEEFDEADBEEF);
249	}
250	#endif
251
252	/* Allocate VMCSs for all guest CPUs. */
253	for (VMCPUID i = 0; i < pVM->cCpus; i++)
254	{
255	PVMCPU pVCpu = &pVM->aCpus[i];
256
257	pVCpu->hwaccm.s.vmx.hMemObjVMCS = NIL_RTR0MEMOBJ;
258
259	/* Allocate one page for the VM control structure (VMCS). */
260	rc = RTR0MemObjAllocCont(&pVCpu->hwaccm.s.vmx.hMemObjVMCS, PAGE_SIZE, true /* executable R0 mapping */);
261	AssertRC(rc);
262	if (RT_FAILURE(rc))
263	return rc;
264
265	pVCpu->hwaccm.s.vmx.pvVMCS = RTR0MemObjAddress(pVCpu->hwaccm.s.vmx.hMemObjVMCS);
266	pVCpu->hwaccm.s.vmx.HCPhysVMCS = RTR0MemObjGetPagePhysAddr(pVCpu->hwaccm.s.vmx.hMemObjVMCS, 0);
267	ASMMemZeroPage(pVCpu->hwaccm.s.vmx.pvVMCS);
268
269	pVCpu->hwaccm.s.vmx.cr0_mask = 0;
270	pVCpu->hwaccm.s.vmx.cr4_mask = 0;
271
272	/* Allocate one page for the virtual APIC page for TPR caching. */
273	rc = RTR0MemObjAllocCont(&pVCpu->hwaccm.s.vmx.hMemObjVAPIC, PAGE_SIZE, true /* executable R0 mapping */);
274	AssertRC(rc);
275	if (RT_FAILURE(rc))
276	return rc;
277
278	pVCpu->hwaccm.s.vmx.pbVAPIC = (uint8_t *)RTR0MemObjAddress(pVCpu->hwaccm.s.vmx.hMemObjVAPIC);
279	pVCpu->hwaccm.s.vmx.HCPhysVAPIC = RTR0MemObjGetPagePhysAddr(pVCpu->hwaccm.s.vmx.hMemObjVAPIC, 0);
280	ASMMemZeroPage(pVCpu->hwaccm.s.vmx.pbVAPIC);
281
282	/* Allocate the MSR bitmap if this feature is supported. */
283	if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
284	{
285	rc = RTR0MemObjAllocCont(&pVCpu->hwaccm.s.vmx.pMemObjMSRBitmap, PAGE_SIZE, true /* executable R0 mapping */);
286	AssertRC(rc);
287	if (RT_FAILURE(rc))
288	return rc;
289
290	pVCpu->hwaccm.s.vmx.pMSRBitmap = (uint8_t *)RTR0MemObjAddress(pVCpu->hwaccm.s.vmx.pMemObjMSRBitmap);
291	pVCpu->hwaccm.s.vmx.pMSRBitmapPhys = RTR0MemObjGetPagePhysAddr(pVCpu->hwaccm.s.vmx.pMemObjMSRBitmap, 0);
292	memset(pVCpu->hwaccm.s.vmx.pMSRBitmap, 0xff, PAGE_SIZE);
293	}
294
295	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
296	/* Allocate one page for the guest MSR load area (for preloading guest MSRs during the world switch). */
297	rc = RTR0MemObjAllocCont(&pVCpu->hwaccm.s.vmx.pMemObjGuestMSR, PAGE_SIZE, true /* executable R0 mapping */);
298	AssertRC(rc);
299	if (RT_FAILURE(rc))
300	return rc;
301
302	pVCpu->hwaccm.s.vmx.pGuestMSR = (uint8_t *)RTR0MemObjAddress(pVCpu->hwaccm.s.vmx.pMemObjGuestMSR);
303	pVCpu->hwaccm.s.vmx.pGuestMSRPhys = RTR0MemObjGetPagePhysAddr(pVCpu->hwaccm.s.vmx.pMemObjGuestMSR, 0);
304	Assert(!(pVCpu->hwaccm.s.vmx.pGuestMSRPhys & 0xf));
305	memset(pVCpu->hwaccm.s.vmx.pGuestMSR, 0, PAGE_SIZE);
306
307	/* Allocate one page for the host MSR load area (for restoring host MSRs after the world switch back). */
308	rc = RTR0MemObjAllocCont(&pVCpu->hwaccm.s.vmx.pMemObjHostMSR, PAGE_SIZE, true /* executable R0 mapping */);
309	AssertRC(rc);
310	if (RT_FAILURE(rc))
311	return rc;
312
313	pVCpu->hwaccm.s.vmx.pHostMSR = (uint8_t *)RTR0MemObjAddress(pVCpu->hwaccm.s.vmx.pMemObjHostMSR);
314	pVCpu->hwaccm.s.vmx.pHostMSRPhys = RTR0MemObjGetPagePhysAddr(pVCpu->hwaccm.s.vmx.pMemObjHostMSR, 0);
315	Assert(!(pVCpu->hwaccm.s.vmx.pHostMSRPhys & 0xf));
316	memset(pVCpu->hwaccm.s.vmx.pHostMSR, 0, PAGE_SIZE);
317	#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
318
319	/* Current guest paging mode. */
320	pVCpu->hwaccm.s.vmx.enmLastSeenGuestMode = PGMMODE_REAL;
321
322	#ifdef LOG_ENABLED
323	SUPR0Printf("VMXR0InitVM %x VMCS=%x (%x)\n", pVM, pVCpu->hwaccm.s.vmx.pvVMCS, (uint32_t)pVCpu->hwaccm.s.vmx.HCPhysVMCS);
324	#endif
325	}
326
327	return VINF_SUCCESS;
328	}
329
330
331	/**
332	* Does Ring-0 per VM VT-x termination.
333	*
334	* @returns VBox status code.
335	* @param pVM Pointer to the VM.
336	*/
337	VMMR0DECL(int) VMXR0TermVM(PVM pVM)
338	{
339	for (VMCPUID i = 0; i < pVM->cCpus; i++)
340	{
341	PVMCPU pVCpu = &pVM->aCpus[i];
342
343	if (pVCpu->hwaccm.s.vmx.hMemObjVMCS != NIL_RTR0MEMOBJ)
344	{
345	RTR0MemObjFree(pVCpu->hwaccm.s.vmx.hMemObjVMCS, false);
346	pVCpu->hwaccm.s.vmx.hMemObjVMCS = NIL_RTR0MEMOBJ;
347	pVCpu->hwaccm.s.vmx.pvVMCS = 0;
348	pVCpu->hwaccm.s.vmx.HCPhysVMCS = 0;
349	}
350	if (pVCpu->hwaccm.s.vmx.hMemObjVAPIC != NIL_RTR0MEMOBJ)
351	{
352	RTR0MemObjFree(pVCpu->hwaccm.s.vmx.hMemObjVAPIC, false);
353	pVCpu->hwaccm.s.vmx.hMemObjVAPIC = NIL_RTR0MEMOBJ;
354	pVCpu->hwaccm.s.vmx.pbVAPIC = 0;
355	pVCpu->hwaccm.s.vmx.HCPhysVAPIC = 0;
356	}
357	if (pVCpu->hwaccm.s.vmx.pMemObjMSRBitmap != NIL_RTR0MEMOBJ)
358	{
359	RTR0MemObjFree(pVCpu->hwaccm.s.vmx.pMemObjMSRBitmap, false);
360	pVCpu->hwaccm.s.vmx.pMemObjMSRBitmap = NIL_RTR0MEMOBJ;
361	pVCpu->hwaccm.s.vmx.pMSRBitmap = 0;
362	pVCpu->hwaccm.s.vmx.pMSRBitmapPhys = 0;
363	}
364	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
365	if (pVCpu->hwaccm.s.vmx.pMemObjHostMSR != NIL_RTR0MEMOBJ)
366	{
367	RTR0MemObjFree(pVCpu->hwaccm.s.vmx.pMemObjHostMSR, false);
368	pVCpu->hwaccm.s.vmx.pMemObjHostMSR = NIL_RTR0MEMOBJ;
369	pVCpu->hwaccm.s.vmx.pHostMSR = 0;
370	pVCpu->hwaccm.s.vmx.pHostMSRPhys = 0;
371	}
372	if (pVCpu->hwaccm.s.vmx.pMemObjGuestMSR != NIL_RTR0MEMOBJ)
373	{
374	RTR0MemObjFree(pVCpu->hwaccm.s.vmx.pMemObjGuestMSR, false);
375	pVCpu->hwaccm.s.vmx.pMemObjGuestMSR = NIL_RTR0MEMOBJ;
376	pVCpu->hwaccm.s.vmx.pGuestMSR = 0;
377	pVCpu->hwaccm.s.vmx.pGuestMSRPhys = 0;
378	}
379	#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
380	}
381	if (pVM->hwaccm.s.vmx.pMemObjAPIC != NIL_RTR0MEMOBJ)
382	{
383	RTR0MemObjFree(pVM->hwaccm.s.vmx.pMemObjAPIC, false);
384	pVM->hwaccm.s.vmx.pMemObjAPIC = NIL_RTR0MEMOBJ;
385	pVM->hwaccm.s.vmx.pAPIC = 0;
386	pVM->hwaccm.s.vmx.pAPICPhys = 0;
387	}
388	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
389	if (pVM->hwaccm.s.vmx.pMemObjScratch != NIL_RTR0MEMOBJ)
390	{
391	ASMMemZero32(pVM->hwaccm.s.vmx.pScratch, PAGE_SIZE);
392	RTR0MemObjFree(pVM->hwaccm.s.vmx.pMemObjScratch, false);
393	pVM->hwaccm.s.vmx.pMemObjScratch = NIL_RTR0MEMOBJ;
394	pVM->hwaccm.s.vmx.pScratch = 0;
395	pVM->hwaccm.s.vmx.pScratchPhys = 0;
396	}
397	#endif
398	return VINF_SUCCESS;
399	}
400
401
402	/**
403	* Sets up VT-x for the specified VM.
404	*
405	* @returns VBox status code.
406	* @param pVM Pointer to the VM.
407	*/
408	VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
409	{
410	int rc = VINF_SUCCESS;
411	uint32_t val;
412
413	AssertReturn(pVM, VERR_INVALID_PARAMETER);
414
415	/* Initialize these always, see hwaccmR3InitFinalizeR0().*/
416	pVM->hwaccm.s.vmx.enmFlushEPT = VMX_FLUSH_EPT_NONE;
417	pVM->hwaccm.s.vmx.enmFlushVPID = VMX_FLUSH_VPID_NONE;
418
419	/* Determine optimal flush type for EPT. */
420	if (pVM->hwaccm.s.fNestedPaging)
421	{
422	if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVEPT)
423	{
424	if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVEPT_CAPS_SINGLE_CONTEXT)
425	pVM->hwaccm.s.vmx.enmFlushEPT = VMX_FLUSH_EPT_SINGLE_CONTEXT;
426	else if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVEPT_CAPS_ALL_CONTEXTS)
427	pVM->hwaccm.s.vmx.enmFlushEPT = VMX_FLUSH_EPT_ALL_CONTEXTS;
428	else
429	{
430	/*
431	* Should never really happen. EPT is supported but no suitable flush types supported.
432	* We cannot ignore EPT at this point as we've already setup Unrestricted Guest execution.
433	*/
434	pVM->hwaccm.s.vmx.enmFlushEPT = VMX_FLUSH_EPT_NOT_SUPPORTED;
435	return VERR_VMX_GENERIC;
436	}
437	}
438	else
439	{
440	/*
441	* Should never really happen. EPT is supported but INVEPT instruction is not supported.
442	*/
443	pVM->hwaccm.s.vmx.enmFlushEPT = VMX_FLUSH_EPT_NOT_SUPPORTED;
444	return VERR_VMX_GENERIC;
445	}
446	}
447
448	/* Determine optimal flush type for VPID. */
449	if (pVM->hwaccm.s.vmx.fVPID)
450	{
451	if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVVPID)
452	{
453	if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVVPID_CAPS_SINGLE_CONTEXT)
454	pVM->hwaccm.s.vmx.enmFlushVPID = VMX_FLUSH_VPID_SINGLE_CONTEXT;
455	else if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVVPID_CAPS_ALL_CONTEXTS)
456	pVM->hwaccm.s.vmx.enmFlushVPID = VMX_FLUSH_VPID_ALL_CONTEXTS;
457	else
458	{
459	/*
460	* Neither SINGLE nor ALL context flush types for VPID supported by the CPU.
461	* We do not handle other flush type combinations, ignore VPID capabilities.
462	*/
463	if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVVPID_CAPS_INDIV_ADDR)
464	Log(("VMXR0SetupVM: Only VMX_FLUSH_VPID_INDIV_ADDR supported. Ignoring VPID.\n"));
465	if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVVPID_CAPS_SINGLE_CONTEXT_RETAIN_GLOBALS)
466	Log(("VMXR0SetupVM: Only VMX_FLUSH_VPID_SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
467	pVM->hwaccm.s.vmx.enmFlushVPID = VMX_FLUSH_VPID_NOT_SUPPORTED;
468	pVM->hwaccm.s.vmx.fVPID = false;
469	}
470	}
471	else
472	{
473	/*
474	* Should not really happen. EPT is supported but INVEPT is not supported.
475	* Ignore VPID capabilities as our code relies on using INVEPT for selective flushing.
476	*/
477	Log(("VMXR0SetupVM: VPID supported without INVEPT support. Ignoring VPID.\n"));
478	pVM->hwaccm.s.vmx.enmFlushVPID = VMX_FLUSH_VPID_NOT_SUPPORTED;
479	pVM->hwaccm.s.vmx.fVPID = false;
480	}
481	}
482
483	for (VMCPUID i = 0; i < pVM->cCpus; i++)
484	{
485	PVMCPU pVCpu = &pVM->aCpus[i];
486
487	AssertPtr(pVCpu->hwaccm.s.vmx.pvVMCS);
488
489	/* Set revision dword at the beginning of the VMCS structure. */
490	(uint32_t )pVCpu->hwaccm.s.vmx.pvVMCS = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hwaccm.s.vmx.msr.vmx_basic_info);
491
492	/*
493	* Clear and activate the VMCS.
494	*/
495	Log(("HCPhysVMCS = %RHp\n", pVCpu->hwaccm.s.vmx.HCPhysVMCS));
496	rc = VMXClearVMCS(pVCpu->hwaccm.s.vmx.HCPhysVMCS);
497	if (RT_FAILURE(rc))
498	goto vmx_end;
499
500	rc = VMXActivateVMCS(pVCpu->hwaccm.s.vmx.HCPhysVMCS);
501	if (RT_FAILURE(rc))
502	goto vmx_end;
503
504	/*
505	* VMX_VMCS_CTRL_PIN_EXEC_CONTROLS
506	* Set required bits to one and zero according to the MSR capabilities.
507	*/
508	val = pVM->hwaccm.s.vmx.msr.vmx_pin_ctls.n.disallowed0;
509	val \|= VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_EXT_INT_EXIT /* External interrupts */
510	\| VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_NMI_EXIT; /* Non-maskable interrupts */
511
512	/*
513	* Enable the VMX preemption timer.
514	*/
515	if (pVM->hwaccm.s.vmx.fUsePreemptTimer)
516	val \|= VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_PREEMPT_TIMER;
517	val &= pVM->hwaccm.s.vmx.msr.vmx_pin_ctls.n.allowed1;
518
519	rc = VMXWriteVMCS(VMX_VMCS_CTRL_PIN_EXEC_CONTROLS, val);
520	AssertRC(rc);
521
522	/*
523	* VMX_VMCS_CTRL_PROC_EXEC_CONTROLS
524	* Set required bits to one and zero according to the MSR capabilities.
525	*/
526	val = pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.disallowed0;
527	/* Program which event cause VM-exits and which features we want to use. */
528	val \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_HLT_EXIT
529	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_TSC_OFFSET
530	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT
531	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_UNCOND_IO_EXIT
532	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDPMC_EXIT
533	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_EXIT
534	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MWAIT_EXIT; /* don't execute mwait or else we'll idle inside
535	the guest (host thinks the cpu load is high) */
536
537	/* Without nested paging we should intercept invlpg and cr3 mov instructions. */
538	if (!pVM->hwaccm.s.fNestedPaging)
539	{
540	val \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INVLPG_EXIT
541	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
542	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT;
543	}
544
545	/*
546	* VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MWAIT_EXIT might cause a vmlaunch
547	* failure with an invalid control fields error. (combined with some other exit reasons)
548	*/
549	if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
550	{
551	/* CR8 reads from the APIC shadow page; writes cause an exit is they lower the TPR below the threshold */
552	val \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW;
553	Assert(pVM->hwaccm.s.vmx.pAPIC);
554	}
555	else
556	/* Exit on CR8 reads & writes in case the TPR shadow feature isn't present. */
557	val \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR8_STORE_EXIT \| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR8_LOAD_EXIT;
558
559	if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
560	{
561	Assert(pVCpu->hwaccm.s.vmx.pMSRBitmapPhys);
562	val \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS;
563	}
564
565	/* We will use the secondary control if it's present. */
566	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
567
568	/* Mask away the bits that the CPU doesn't support */
569	/** @todo make sure they don't conflict with the above requirements. */
570	val &= pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1;
571	pVCpu->hwaccm.s.vmx.proc_ctls = val;
572
573	rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, val);
574	AssertRC(rc);
575
576	if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
577	{
578	/*
579	* VMX_VMCS_CTRL_PROC_EXEC_CONTROLS2
580	* Set required bits to one and zero according to the MSR capabilities.
581	*/
582	val = pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2.n.disallowed0;
583	val \|= VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT;
584
585	if (pVM->hwaccm.s.fNestedPaging)
586	val \|= VMX_VMCS_CTRL_PROC_EXEC2_EPT;
587
588	if (pVM->hwaccm.s.vmx.fVPID)
589	val \|= VMX_VMCS_CTRL_PROC_EXEC2_VPID;
590
591	if (pVM->hwaccm.s.fHasIoApic)
592	val \|= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC;
593
594	if (pVM->hwaccm.s.vmx.fUnrestrictedGuest)
595	val \|= VMX_VMCS_CTRL_PROC_EXEC2_REAL_MODE;
596
597	if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
598	val \|= VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP;
599
600	/* Mask away the bits that the CPU doesn't support */
601	/** @todo make sure they don't conflict with the above requirements. */
602	val &= pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2.n.allowed1;
603	pVCpu->hwaccm.s.vmx.proc_ctls2 = val;
604	rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS2, val);
605	AssertRC(rc);
606	}
607
608	/*
609	* VMX_VMCS_CTRL_CR3_TARGET_COUNT
610	* Set required bits to one and zero according to the MSR capabilities.
611	*/
612	rc = VMXWriteVMCS(VMX_VMCS_CTRL_CR3_TARGET_COUNT, 0);
613	AssertRC(rc);
614
615	/*
616	* Forward all exception except #NM & #PF to the guest.
617	* We always need to check pagefaults since our shadow page table can be out of sync.
618	* And we always lazily sync the FPU & XMM state. .
619	*/
620
621	/** @todo Possible optimization:
622	* Keep the FPU and XMM state current in the EM thread. That way there's no need to
623	* lazily sync anything, but the downside is that we can't use the FPU stack or XMM
624	* registers ourselves of course.
625	*
626	* Note: only possible if the current state is actually ours (X86_CR0_TS flag)
627	*/
628
629	/*
630	* Don't filter page faults, all of them should cause a world switch.
631	*/
632	rc = VMXWriteVMCS(VMX_VMCS_CTRL_PAGEFAULT_ERROR_MASK, 0);
633	AssertRC(rc);
634	rc = VMXWriteVMCS(VMX_VMCS_CTRL_PAGEFAULT_ERROR_MATCH, 0);
635	AssertRC(rc);
636
637	rc = VMXWriteVMCS64(VMX_VMCS_CTRL_TSC_OFFSET_FULL, 0);
638	AssertRC(rc);
639	rc = VMXWriteVMCS64(VMX_VMCS_CTRL_IO_BITMAP_A_FULL, 0);
640	AssertRC(rc);
641	rc = VMXWriteVMCS64(VMX_VMCS_CTRL_IO_BITMAP_B_FULL, 0);
642	AssertRC(rc);
643
644	/*
645	* Set the MSR bitmap address.
646	*/
647	if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
648	{
649	Assert(pVCpu->hwaccm.s.vmx.pMSRBitmapPhys);
650
651	rc = VMXWriteVMCS64(VMX_VMCS_CTRL_MSR_BITMAP_FULL, pVCpu->hwaccm.s.vmx.pMSRBitmapPhys);
652	AssertRC(rc);
653
654	/*
655	* Allow the guest to directly modify these MSRs; they are loaded/stored automatically
656	* using MSR-load/store areas in the VMCS.
657	*/
658	hmR0VmxSetMSRPermission(pVCpu, MSR_IA32_SYSENTER_CS, true, true);
659	hmR0VmxSetMSRPermission(pVCpu, MSR_IA32_SYSENTER_ESP, true, true);
660	hmR0VmxSetMSRPermission(pVCpu, MSR_IA32_SYSENTER_EIP, true, true);
661	hmR0VmxSetMSRPermission(pVCpu, MSR_K8_LSTAR, true, true);
662	hmR0VmxSetMSRPermission(pVCpu, MSR_K6_STAR, true, true);
663	hmR0VmxSetMSRPermission(pVCpu, MSR_K8_SF_MASK, true, true);
664	hmR0VmxSetMSRPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, true, true);
665	hmR0VmxSetMSRPermission(pVCpu, MSR_K8_GS_BASE, true, true);
666	hmR0VmxSetMSRPermission(pVCpu, MSR_K8_FS_BASE, true, true);
667	if (pVCpu->hwaccm.s.vmx.proc_ctls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
668	hmR0VmxSetMSRPermission(pVCpu, MSR_K8_TSC_AUX, true, true);
669	}
670
671	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
672	/*
673	* Set the guest & host MSR load/store physical addresses.
674	*/
675	Assert(pVCpu->hwaccm.s.vmx.pGuestMSRPhys);
676	rc = VMXWriteVMCS64(VMX_VMCS_CTRL_VMENTRY_MSR_LOAD_FULL, pVCpu->hwaccm.s.vmx.pGuestMSRPhys);
677	AssertRC(rc);
678	rc = VMXWriteVMCS64(VMX_VMCS_CTRL_VMEXIT_MSR_STORE_FULL, pVCpu->hwaccm.s.vmx.pGuestMSRPhys);
679	AssertRC(rc);
680	Assert(pVCpu->hwaccm.s.vmx.pHostMSRPhys);
681	rc = VMXWriteVMCS64(VMX_VMCS_CTRL_VMEXIT_MSR_LOAD_FULL, pVCpu->hwaccm.s.vmx.pHostMSRPhys);
682	AssertRC(rc);
683	#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
684
685	rc = VMXWriteVMCS(VMX_VMCS_CTRL_ENTRY_MSR_LOAD_COUNT, 0);
686	AssertRC(rc);
687	rc = VMXWriteVMCS(VMX_VMCS_CTRL_EXIT_MSR_STORE_COUNT, 0);
688	AssertRC(rc);
689	rc = VMXWriteVMCS(VMX_VMCS_CTRL_EXIT_MSR_LOAD_COUNT, 0);
690	AssertRC(rc);
691
692	if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
693	{
694	Assert(pVM->hwaccm.s.vmx.pMemObjAPIC);
695	/* Optional */
696	rc = VMXWriteVMCS(VMX_VMCS_CTRL_TPR_THRESHOLD, 0);
697	rc \|= VMXWriteVMCS64(VMX_VMCS_CTRL_VAPIC_PAGEADDR_FULL, pVCpu->hwaccm.s.vmx.HCPhysVAPIC);
698
699	if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
700	rc \|= VMXWriteVMCS64(VMX_VMCS_CTRL_APIC_ACCESSADDR_FULL, pVM->hwaccm.s.vmx.pAPICPhys);
701
702	AssertRC(rc);
703	}
704
705	/* Set link pointer to -1. Not currently used. */
706	rc = VMXWriteVMCS64(VMX_VMCS_GUEST_LINK_PTR_FULL, 0xFFFFFFFFFFFFFFFFULL);
707	AssertRC(rc);
708
709	/*
710	* Clear VMCS, marking it inactive. Clear implementation specific data and writing back
711	* VMCS data back to memory.
712	*/
713	rc = VMXClearVMCS(pVCpu->hwaccm.s.vmx.HCPhysVMCS);
714	AssertRC(rc);
715
716	/*
717	* Configure the VMCS read cache.
718	*/
719	PVMCSCACHE pCache = &pVCpu->hwaccm.s.vmx.VMCSCache;
720
721	VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_RIP);
722	VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_RSP);
723	VMXSetupCachedReadVMCS(pCache, VMX_VMCS_GUEST_RFLAGS);
724	VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE);
725	VMXSetupCachedReadVMCS(pCache, VMX_VMCS_CTRL_CR0_READ_SHADOW);
726	VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_CR0);
727	VMXSetupCachedReadVMCS(pCache, VMX_VMCS_CTRL_CR4_READ_SHADOW);
728	VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_CR4);
729	VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_DR7);
730	VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_GUEST_SYSENTER_CS);
731	VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_SYSENTER_EIP);
732	VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_SYSENTER_ESP);
733	VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_GUEST_GDTR_LIMIT);
734	VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_GDTR_BASE);
735	VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_GUEST_IDTR_LIMIT);
736	VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_IDTR_BASE);
737
738	VMX_SETUP_SELREG(ES, pCache);
739	VMX_SETUP_SELREG(SS, pCache);
740	VMX_SETUP_SELREG(CS, pCache);
741	VMX_SETUP_SELREG(DS, pCache);
742	VMX_SETUP_SELREG(FS, pCache);
743	VMX_SETUP_SELREG(GS, pCache);
744	VMX_SETUP_SELREG(LDTR, pCache);
745	VMX_SETUP_SELREG(TR, pCache);
746
747	/*
748	* Status code VMCS reads.
749	*/
750	VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_EXIT_REASON);
751	VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_VM_INSTR_ERROR);
752	VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_EXIT_INSTR_LENGTH);
753	VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_EXIT_INTERRUPTION_ERRCODE);
754	VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO);
755	VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_EXIT_INSTR_INFO);
756	VMXSetupCachedReadVMCS(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
757	VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_IDT_INFO);
758	VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_IDT_ERRCODE);
759
760	if (pVM->hwaccm.s.fNestedPaging)
761	{
762	VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_CR3);
763	VMXSetupCachedReadVMCS(pCache, VMX_VMCS_EXIT_PHYS_ADDR_FULL);
764	pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
765	}
766	else
767	pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
768	} /* for each VMCPU */
769
770	/*
771	* Setup the right TLB function based on CPU capabilities.
772	*/
773	if (pVM->hwaccm.s.fNestedPaging && pVM->hwaccm.s.vmx.fVPID)
774	pVM->hwaccm.s.vmx.pfnSetupTaggedTLB = hmR0VmxSetupTLBBoth;
775	else if (pVM->hwaccm.s.fNestedPaging)
776	pVM->hwaccm.s.vmx.pfnSetupTaggedTLB = hmR0VmxSetupTLBEPT;
777	else if (pVM->hwaccm.s.vmx.fVPID)
778	pVM->hwaccm.s.vmx.pfnSetupTaggedTLB = hmR0VmxSetupTLBVPID;
779	else
780	pVM->hwaccm.s.vmx.pfnSetupTaggedTLB = hmR0VmxSetupTLBDummy;
781
782	vmx_end:
783	hmR0VmxCheckError(pVM, &pVM->aCpus[0], rc);
784	return rc;
785	}
786
787
788	/**
789	* Sets the permission bits for the specified MSR.
790	*
791	* @param pVCpu Pointer to the VMCPU.
792	* @param ulMSR The MSR value.
793	* @param fRead Whether reading is allowed.
794	* @param fWrite Whether writing is allowed.
795	*/
796	static void hmR0VmxSetMSRPermission(PVMCPU pVCpu, unsigned ulMSR, bool fRead, bool fWrite)
797	{
798	unsigned ulBit;
799	uint8_t pMSRBitmap = (uint8_t )pVCpu->hwaccm.s.vmx.pMSRBitmap;
800
801	/*
802	* Layout:
803	* 0x000 - 0x3ff - Low MSR read bits
804	* 0x400 - 0x7ff - High MSR read bits
805	* 0x800 - 0xbff - Low MSR write bits
806	* 0xc00 - 0xfff - High MSR write bits
807	*/
808	if (ulMSR <= 0x00001FFF)
809	{
810	/* Pentium-compatible MSRs */
811	ulBit = ulMSR;
812	}
813	else if ( ulMSR >= 0xC0000000
814	&& ulMSR <= 0xC0001FFF)
815	{
816	/* AMD Sixth Generation x86 Processor MSRs */
817	ulBit = (ulMSR - 0xC0000000);
818	pMSRBitmap += 0x400;
819	}
820	else
821	{
822	AssertFailed();
823	return;
824	}
825
826	Assert(ulBit <= 0x1fff);
827	if (fRead)
828	ASMBitClear(pMSRBitmap, ulBit);
829	else
830	ASMBitSet(pMSRBitmap, ulBit);
831
832	if (fWrite)
833	ASMBitClear(pMSRBitmap + 0x800, ulBit);
834	else
835	ASMBitSet(pMSRBitmap + 0x800, ulBit);
836	}
837
838
839	/**
840	* Injects an event (trap or external interrupt).
841	*
842	* @returns VBox status code. Note that it may return VINF_EM_RESET to
843	* indicate a triple fault when injecting X86_XCPT_DF.
844	*
845	* @param pVM Pointer to the VM.
846	* @param pVCpu Pointer to the VMCPU.
847	* @param pCtx Pointer to the guest CPU Context.
848	* @param intInfo VMX interrupt info.
849	* @param cbInstr Opcode length of faulting instruction.
850	* @param errCode Error code (optional).
851	*/
852	static int hmR0VmxInjectEvent(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, uint32_t intInfo, uint32_t cbInstr, uint32_t errCode)
853	{
854	int rc;
855	uint32_t iGate = VMX_EXIT_INTERRUPTION_INFO_VECTOR(intInfo);
856
857	#ifdef VBOX_WITH_STATISTICS
858	STAM_COUNTER_INC(&pVCpu->hwaccm.s.paStatInjectedIrqsR0[iGate & MASK_INJECT_IRQ_STAT]);
859	#endif
860
861	#ifdef VBOX_STRICT
862	if (iGate == 0xE)
863	{
864	LogFlow(("hmR0VmxInjectEvent: Injecting interrupt %d at %RGv error code=%08x CR2=%RGv intInfo=%08x\n", iGate,
865	(RTGCPTR)pCtx->rip, errCode, pCtx->cr2, intInfo));
866	}
867	else if (iGate < 0x20)
868	{
869	LogFlow(("hmR0VmxInjectEvent: Injecting interrupt %d at %RGv error code=%08x\n", iGate, (RTGCPTR)pCtx->rip,
870	errCode));
871	}
872	else
873	{
874	LogFlow(("INJ-EI: %x at %RGv\n", iGate, (RTGCPTR)pCtx->rip));
875	Assert( VMX_EXIT_INTERRUPTION_INFO_TYPE(intInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW
876	\|\| !VMCPU_FF_ISSET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
877	Assert( VMX_EXIT_INTERRUPTION_INFO_TYPE(intInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW
878	\|\| pCtx->eflags.u32 & X86_EFL_IF);
879	}
880	#endif
881
882	if ( CPUMIsGuestInRealModeEx(pCtx)
883	&& pVM->hwaccm.s.vmx.pRealModeTSS)
884	{
885	RTGCPHYS GCPhysHandler;
886	uint16_t offset, ip;
887	RTSEL sel;
888
889	/*
890	* Injecting events doesn't work right with real mode emulation.
891	* (#GP if we try to inject external hardware interrupts)
892	* Inject the interrupt or trap directly instead.
893	*
894	* ASSUMES no access handlers for the bits we read or write below (should be safe).
895	*/
896	Log(("Manual interrupt/trap '%x' inject (real mode)\n", iGate));
897
898	/*
899	* Check if the interrupt handler is present.
900	*/
901	if (iGate * 4 + 3 > pCtx->idtr.cbIdt)
902	{
903	Log(("IDT cbIdt violation\n"));
904	if (iGate != X86_XCPT_DF)
905	{
906	uint32_t intInfo2;
907
908	intInfo2 = (iGate == X86_XCPT_GP) ? (uint32_t)X86_XCPT_DF : iGate;
909	intInfo2 \|= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
910	intInfo2 \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
911	intInfo2 \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HWEXCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
912
913	return hmR0VmxInjectEvent(pVM, pVCpu, pCtx, intInfo2, 0, 0 /* no error code according to the Intel docs */);
914	}
915	Log(("Triple fault -> reset the VM!\n"));
916	return VINF_EM_RESET;
917	}
918	if ( VMX_EXIT_INTERRUPTION_INFO_TYPE(intInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW
919	\|\| iGate == 3 /* Both #BP and #OF point to the instruction after. */
920	\|\| iGate == 4)
921	{
922	ip = pCtx->ip + cbInstr;
923	}
924	else
925	ip = pCtx->ip;
926
927	/*
928	* Read the selector:offset pair of the interrupt handler.
929	*/
930	GCPhysHandler = (RTGCPHYS)pCtx->idtr.pIdt + iGate * 4;
931	rc = PGMPhysSimpleReadGCPhys(pVM, &offset, GCPhysHandler, sizeof(offset)); AssertRC(rc);
932	rc = PGMPhysSimpleReadGCPhys(pVM, &sel, GCPhysHandler + 2, sizeof(sel)); AssertRC(rc);
933
934	LogFlow(("IDT handler %04X:%04X\n", sel, offset));
935
936	/*
937	* Construct the stack frame.
938	*/
939	/** @todo Check stack limit. */
940	pCtx->sp -= 2;
941	LogFlow(("ss:sp %04X:%04X eflags=%x\n", pCtx->ss.Sel, pCtx->sp, pCtx->eflags.u));
942	rc = PGMPhysSimpleWriteGCPhys(pVM, pCtx->ss.u64Base + pCtx->sp, &pCtx->eflags, sizeof(uint16_t)); AssertRC(rc);
943	pCtx->sp -= 2;
944	LogFlow(("ss:sp %04X:%04X cs=%x\n", pCtx->ss.Sel, pCtx->sp, pCtx->cs.Sel));
945	rc = PGMPhysSimpleWriteGCPhys(pVM, pCtx->ss.u64Base + pCtx->sp, &pCtx->cs, sizeof(uint16_t)); AssertRC(rc);
946	pCtx->sp -= 2;
947	LogFlow(("ss:sp %04X:%04X ip=%x\n", pCtx->ss.Sel, pCtx->sp, ip));
948	rc = PGMPhysSimpleWriteGCPhys(pVM, pCtx->ss.u64Base + pCtx->sp, &ip, sizeof(ip)); AssertRC(rc);
949
950	/*
951	* Update the CPU state for executing the handler.
952	*/
953	pCtx->rip = offset;
954	pCtx->cs.Sel = sel;
955	pCtx->cs.u64Base = sel << 4;
956	pCtx->eflags.u &= ~(X86_EFL_IF \| X86_EFL_TF \| X86_EFL_RF \| X86_EFL_AC);
957
958	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_GUEST_SEGMENT_REGS;
959	return VINF_SUCCESS;
960	}
961
962	/*
963	* Set event injection state.
964	*/
965	rc = VMXWriteVMCS(VMX_VMCS_CTRL_ENTRY_IRQ_INFO, intInfo \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT));
966	rc \|= VMXWriteVMCS(VMX_VMCS_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
967	rc \|= VMXWriteVMCS(VMX_VMCS_CTRL_ENTRY_EXCEPTION_ERRCODE, errCode);
968
969	AssertRC(rc);
970	return rc;
971	}
972
973
974	/**
975	* Checks for pending guest interrupts and injects them.
976	*
977	* @returns VBox status code.
978	* @param pVM Pointer to the VM.
979	* @param pVCpu Pointer to the VMCPU.
980	* @param pCtx Pointer to the guest CPU context.
981	*/
982	static int hmR0VmxCheckPendingInterrupt(PVM pVM, PVMCPU pVCpu, CPUMCTX *pCtx)
983	{
984	int rc;
985
986	/*
987	* Dispatch any pending interrupts (injected before, but a VM exit occurred prematurely).
988	*/
989	if (pVCpu->hwaccm.s.Event.fPending)
990	{
991	Log(("CPU%d: Reinjecting event %RX64 %08x at %RGv cr2=%RX64\n", pVCpu->idCpu, pVCpu->hwaccm.s.Event.intInfo,
992	pVCpu->hwaccm.s.Event.errCode, (RTGCPTR)pCtx->rip, pCtx->cr2));
993	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatIntReinject);
994	rc = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, pVCpu->hwaccm.s.Event.intInfo, 0, pVCpu->hwaccm.s.Event.errCode);
995	AssertRC(rc);
996
997	pVCpu->hwaccm.s.Event.fPending = false;
998	return VINF_SUCCESS;
999	}
1000
1001	/*
1002	* If an active trap is already pending, we must forward it first!
1003	*/
1004	if (!TRPMHasTrap(pVCpu))
1005	{
1006	if (VMCPU_FF_TESTANDCLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI))
1007	{
1008	RTGCUINTPTR intInfo;
1009
1010	Log(("CPU%d: injecting #NMI\n", pVCpu->idCpu));
1011
1012	intInfo = X86_XCPT_NMI;
1013	intInfo \|= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
1014	intInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
1015
1016	rc = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, intInfo, 0, 0);
1017	AssertRC(rc);
1018
1019	return VINF_SUCCESS;
1020	}
1021
1022	/** @todo SMI interrupts. */
1023
1024	/*
1025	* When external interrupts are pending, we should exit the VM when IF is set.
1026	*/
1027	if (VMCPU_FF_ISPENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC\|VMCPU_FF_INTERRUPT_PIC)))
1028	{
1029	if (!(pCtx->eflags.u32 & X86_EFL_IF))
1030	{
1031	if (!(pVCpu->hwaccm.s.vmx.proc_ctls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_IRQ_WINDOW_EXIT))
1032	{
1033	LogFlow(("Enable irq window exit!\n"));
1034	pVCpu->hwaccm.s.vmx.proc_ctls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_IRQ_WINDOW_EXIT;
1035	rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
1036	AssertRC(rc);
1037	}
1038	/* else nothing to do but wait */
1039	}
1040	else if (!VMCPU_FF_ISSET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
1041	{
1042	uint8_t u8Interrupt;
1043
1044	rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
1045	Log(("CPU%d: Dispatch interrupt: u8Interrupt=%x (%d) rc=%Rrc cs:rip=%04X:%RGv\n", pVCpu->idCpu,
1046	u8Interrupt, u8Interrupt, rc, pCtx->cs.Sel, (RTGCPTR)pCtx->rip));
1047	if (RT_SUCCESS(rc))
1048	{
1049	rc = TRPMAssertTrap(pVCpu, u8Interrupt, TRPM_HARDWARE_INT);
1050	AssertRC(rc);
1051	}
1052	else
1053	{
1054	/* Can only happen in rare cases where a pending interrupt is cleared behind our back */
1055	Assert(!VMCPU_FF_ISPENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC\|VMCPU_FF_INTERRUPT_PIC)));
1056	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatSwitchGuestIrq);
1057	/* Just continue */
1058	}
1059	}
1060	else
1061	Log(("Pending interrupt blocked at %RGv by VM_FF_INHIBIT_INTERRUPTS!!\n", (RTGCPTR)pCtx->rip));
1062	}
1063	}
1064
1065	#ifdef VBOX_STRICT
1066	if (TRPMHasTrap(pVCpu))
1067	{
1068	uint8_t u8Vector;
1069	rc = TRPMQueryTrapAll(pVCpu, &u8Vector, 0, 0, 0);
1070	AssertRC(rc);
1071	}
1072	#endif
1073
1074	if ( (pCtx->eflags.u32 & X86_EFL_IF)
1075	&& (!VMCPU_FF_ISSET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
1076	&& TRPMHasTrap(pVCpu)
1077	)
1078	{
1079	uint8_t u8Vector;
1080	TRPMEVENT enmType;
1081	RTGCUINTPTR intInfo;
1082	RTGCUINT errCode;
1083
1084	/*
1085	* If a new event is pending, dispatch it now.
1086	*/
1087	rc = TRPMQueryTrapAll(pVCpu, &u8Vector, &enmType, &errCode, 0);
1088	AssertRC(rc);
1089	Assert(pCtx->eflags.Bits.u1IF == 1 \|\| enmType == TRPM_TRAP);
1090	Assert(enmType != TRPM_SOFTWARE_INT);
1091
1092	/*
1093	* Clear the pending trap.
1094	*/
1095	rc = TRPMResetTrap(pVCpu);
1096	AssertRC(rc);
1097
1098	intInfo = u8Vector;
1099	intInfo \|= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
1100
1101	if (enmType == TRPM_TRAP)
1102	{
1103	switch (u8Vector)
1104	{
1105	case X86_XCPT_DF:
1106	case X86_XCPT_TS:
1107	case X86_XCPT_NP:
1108	case X86_XCPT_SS:
1109	case X86_XCPT_GP:
1110	case X86_XCPT_PF:
1111	case X86_XCPT_AC:
1112	{
1113	/* Valid error codes. */
1114	intInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
1115	break;
1116	}
1117
1118	default:
1119	break;
1120	}
1121
1122	if ( u8Vector == X86_XCPT_BP
1123	\|\| u8Vector == X86_XCPT_OF)
1124	{
1125	intInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SWEXCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
1126	}
1127	else
1128	intInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HWEXCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
1129	}
1130	else
1131	intInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
1132
1133	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatIntInject);
1134	rc = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, intInfo, 0, errCode);
1135	AssertRC(rc);
1136	} /* if (interrupts can be dispatched) */
1137
1138	return VINF_SUCCESS;
1139	}
1140
1141
1142	/**
1143	* Save the host state into the VMCS.
1144	*
1145	* @returns VBox status code.
1146	* @param pVM Pointer to the VM.
1147	* @param pVCpu Pointer to the VMCPU.
1148	*/
1149	VMMR0DECL(int) VMXR0SaveHostState(PVM pVM, PVMCPU pVCpu)
1150	{
1151	int rc = VINF_SUCCESS;
1152	NOREF(pVM);
1153
1154	/*
1155	* Host CPU Context.
1156	*/
1157	if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_HOST_CONTEXT)
1158	{
1159	RTIDTR idtr;
1160	RTGDTR gdtr;
1161	RTSEL SelTR;
1162	PCX86DESCHC pDesc;
1163	uintptr_t trBase;
1164	RTSEL cs;
1165	RTSEL ss;
1166	uint64_t cr3;
1167
1168	/*
1169	* Control registers.
1170	*/
1171	rc = VMXWriteVMCS(VMX_VMCS_HOST_CR0, ASMGetCR0());
1172	Log2(("VMX_VMCS_HOST_CR0 %08x\n", ASMGetCR0()));
1173	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
1174	if (VMX_IS_64BIT_HOST_MODE())
1175	{
1176	cr3 = hwaccmR0Get64bitCR3();
1177	rc \|= VMXWriteVMCS64(VMX_VMCS_HOST_CR3, cr3);
1178	}
1179	else
1180	#endif
1181	{
1182	cr3 = ASMGetCR3();
1183	rc \|= VMXWriteVMCS(VMX_VMCS_HOST_CR3, cr3);
1184	}
1185	Log2(("VMX_VMCS_HOST_CR3 %08RX64\n", cr3));
1186	rc \|= VMXWriteVMCS(VMX_VMCS_HOST_CR4, ASMGetCR4());
1187	Log2(("VMX_VMCS_HOST_CR4 %08x\n", ASMGetCR4()));
1188	AssertRC(rc);
1189
1190	/*
1191	* Selector registers.
1192	*/
1193	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
1194	if (VMX_IS_64BIT_HOST_MODE())
1195	{
1196	cs = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelCS;
1197	ss = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelSS;
1198	}
1199	else
1200	{
1201	/* sysenter loads LDT cs & ss, VMX doesn't like this. Load the GDT ones (safe). */
1202	cs = (RTSEL)(uintptr_t)&SUPR0AbsKernelCS;
1203	ss = (RTSEL)(uintptr_t)&SUPR0AbsKernelSS;
1204	}
1205	#else
1206	cs = ASMGetCS();
1207	ss = ASMGetSS();
1208	#endif
1209	Assert(!(cs & X86_SEL_LDT)); Assert((cs & X86_SEL_RPL) == 0);
1210	Assert(!(ss & X86_SEL_LDT)); Assert((ss & X86_SEL_RPL) == 0);
1211	rc = VMXWriteVMCS(VMX_VMCS16_HOST_FIELD_CS, cs);
1212	/* Note: VMX is (again) very picky about the RPL of the selectors here; we'll restore them manually. */
1213	rc \|= VMXWriteVMCS(VMX_VMCS16_HOST_FIELD_DS, 0);
1214	rc \|= VMXWriteVMCS(VMX_VMCS16_HOST_FIELD_ES, 0);
1215	#if HC_ARCH_BITS == 32
1216	if (!VMX_IS_64BIT_HOST_MODE())
1217	{
1218	rc \|= VMXWriteVMCS(VMX_VMCS16_HOST_FIELD_FS, 0);
1219	rc \|= VMXWriteVMCS(VMX_VMCS16_HOST_FIELD_GS, 0);
1220	}
1221	#endif
1222	rc \|= VMXWriteVMCS(VMX_VMCS16_HOST_FIELD_SS, ss);
1223	SelTR = ASMGetTR();
1224	rc \|= VMXWriteVMCS(VMX_VMCS16_HOST_FIELD_TR, SelTR);
1225	AssertRC(rc);
1226	Log2(("VMX_VMCS_HOST_FIELD_CS %08x (%08x)\n", cs, ASMGetSS()));
1227	Log2(("VMX_VMCS_HOST_FIELD_DS 00000000 (%08x)\n", ASMGetDS()));
1228	Log2(("VMX_VMCS_HOST_FIELD_ES 00000000 (%08x)\n", ASMGetES()));
1229	Log2(("VMX_VMCS_HOST_FIELD_FS 00000000 (%08x)\n", ASMGetFS()));
1230	Log2(("VMX_VMCS_HOST_FIELD_GS 00000000 (%08x)\n", ASMGetGS()));
1231	Log2(("VMX_VMCS_HOST_FIELD_SS %08x (%08x)\n", ss, ASMGetSS()));
1232	Log2(("VMX_VMCS_HOST_FIELD_TR %08x\n", ASMGetTR()));
1233
1234	/*
1235	* GDTR & IDTR.
1236	*/
1237	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
1238	if (VMX_IS_64BIT_HOST_MODE())
1239	{
1240	X86XDTR64 gdtr64, idtr64;
1241	hwaccmR0Get64bitGDTRandIDTR(&gdtr64, &idtr64);
1242	rc = VMXWriteVMCS64(VMX_VMCS_HOST_GDTR_BASE, gdtr64.uAddr);
1243	rc \|= VMXWriteVMCS64(VMX_VMCS_HOST_IDTR_BASE, gdtr64.uAddr);
1244	AssertRC(rc);
1245	Log2(("VMX_VMCS_HOST_GDTR_BASE %RX64\n", gdtr64.uAddr));
1246	Log2(("VMX_VMCS_HOST_IDTR_BASE %RX64\n", idtr64.uAddr));
1247	gdtr.cbGdt = gdtr64.cb;
1248	gdtr.pGdt = (uintptr_t)gdtr64.uAddr;
1249	}
1250	else
1251	#endif
1252	{
1253	ASMGetGDTR(&gdtr);
1254	rc = VMXWriteVMCS(VMX_VMCS_HOST_GDTR_BASE, gdtr.pGdt);
1255	ASMGetIDTR(&idtr);
1256	rc \|= VMXWriteVMCS(VMX_VMCS_HOST_IDTR_BASE, idtr.pIdt);
1257	AssertRC(rc);
1258	Log2(("VMX_VMCS_HOST_GDTR_BASE %RHv\n", gdtr.pGdt));
1259	Log2(("VMX_VMCS_HOST_IDTR_BASE %RHv\n", idtr.pIdt));
1260	}
1261
1262	/*
1263	* Save the base address of the TR selector.
1264	*/
1265	if (SelTR > gdtr.cbGdt)
1266	{
1267	AssertMsgFailed(("Invalid TR selector %x. GDTR.cbGdt=%x\n", SelTR, gdtr.cbGdt));
1268	return VERR_VMX_INVALID_HOST_STATE;
1269	}
1270
1271	pDesc = (PCX86DESCHC)(gdtr.pGdt + (SelTR & X86_SEL_MASK));
1272	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
1273	if (VMX_IS_64BIT_HOST_MODE())
1274	{
1275	uint64_t trBase64 = X86DESC64_BASE((PX86DESC64)pDesc);
1276	rc = VMXWriteVMCS64(VMX_VMCS_HOST_TR_BASE, trBase64);
1277	Log2(("VMX_VMCS_HOST_TR_BASE %RX64\n", trBase64));
1278	AssertRC(rc);
1279	}
1280	else
1281	#endif
1282	{
1283	#if HC_ARCH_BITS == 64
1284	trBase = X86DESC64_BASE(pDesc);
1285	#else
1286	trBase = X86DESC_BASE(pDesc);
1287	#endif
1288	rc = VMXWriteVMCS(VMX_VMCS_HOST_TR_BASE, trBase);
1289	AssertRC(rc);
1290	Log2(("VMX_VMCS_HOST_TR_BASE %RHv\n", trBase));
1291	}
1292
1293	/*
1294	* FS base and GS base.
1295	*/
1296	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
1297	if (VMX_IS_64BIT_HOST_MODE())
1298	{
1299	Log2(("MSR_K8_FS_BASE = %RX64\n", ASMRdMsr(MSR_K8_FS_BASE)));
1300	Log2(("MSR_K8_GS_BASE = %RX64\n", ASMRdMsr(MSR_K8_GS_BASE)));
1301	rc = VMXWriteVMCS64(VMX_VMCS_HOST_FS_BASE, ASMRdMsr(MSR_K8_FS_BASE));
1302	rc \|= VMXWriteVMCS64(VMX_VMCS_HOST_GS_BASE, ASMRdMsr(MSR_K8_GS_BASE));
1303	}
1304	#endif
1305	AssertRC(rc);
1306
1307	/*
1308	* Sysenter MSRs.
1309	*/
1310	/** @todo expensive!! */
1311	rc = VMXWriteVMCS(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
1312	Log2(("VMX_VMCS_HOST_SYSENTER_CS %08x\n", ASMRdMsr_Low(MSR_IA32_SYSENTER_CS)));
1313	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
1314	if (VMX_IS_64BIT_HOST_MODE())
1315	{
1316	Log2(("VMX_VMCS_HOST_SYSENTER_EIP %RX64\n", ASMRdMsr(MSR_IA32_SYSENTER_EIP)));
1317	Log2(("VMX_VMCS_HOST_SYSENTER_ESP %RX64\n", ASMRdMsr(MSR_IA32_SYSENTER_ESP)));
1318	rc \|= VMXWriteVMCS64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
1319	rc \|= VMXWriteVMCS64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
1320	}
1321	else
1322	{
1323	rc \|= VMXWriteVMCS(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
1324	rc \|= VMXWriteVMCS(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
1325	Log2(("VMX_VMCS_HOST_SYSENTER_EIP %RX32\n", ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP)));
1326	Log2(("VMX_VMCS_HOST_SYSENTER_ESP %RX32\n", ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP)));
1327	}
1328	#elif HC_ARCH_BITS == 32
1329	rc \|= VMXWriteVMCS(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
1330	rc \|= VMXWriteVMCS(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
1331	Log2(("VMX_VMCS_HOST_SYSENTER_EIP %RX32\n", ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP)));
1332	Log2(("VMX_VMCS_HOST_SYSENTER_ESP %RX32\n", ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP)));
1333	#else
1334	Log2(("VMX_VMCS_HOST_SYSENTER_EIP %RX64\n", ASMRdMsr(MSR_IA32_SYSENTER_EIP)));
1335	Log2(("VMX_VMCS_HOST_SYSENTER_ESP %RX64\n", ASMRdMsr(MSR_IA32_SYSENTER_ESP)));
1336	rc \|= VMXWriteVMCS64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
1337	rc \|= VMXWriteVMCS64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
1338	#endif
1339	AssertRC(rc);
1340
1341
1342	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
1343	/*
1344	* Store all host MSRs in the VM-Exit load area, so they will be reloaded after
1345	* the world switch back to the host.
1346	*/
1347	PVMXMSR pMsr = (PVMXMSR)pVCpu->hwaccm.s.vmx.pHostMSR;
1348	unsigned idxMsr = 0;
1349
1350	uint32_t u32HostExtFeatures = ASMCpuId_EDX(0x80000001);
1351	if (u32HostExtFeatures & (X86_CPUID_EXT_FEATURE_EDX_NX \| X86_CPUID_EXT_FEATURE_EDX_LONG_MODE))
1352	{
1353	#if 0
1354	pMsr->u32IndexMSR = MSR_K6_EFER;
1355	pMsr->u32Reserved = 0;
1356	# if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
1357	if (CPUMIsGuestInLongMode(pVCpu))
1358	{
1359	/* Must match the EFER value in our 64 bits switcher. */
1360	pMsr->u64Value = ASMRdMsr(MSR_K6_EFER) \| MSR_K6_EFER_LME \| MSR_K6_EFER_SCE \| MSR_K6_EFER_NXE;
1361	}
1362	else
1363	# endif
1364	pMsr->u64Value = ASMRdMsr(MSR_K6_EFER);
1365	pMsr++; idxMsr++;
1366	#endif
1367	}
1368
1369	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
1370	if (VMX_IS_64BIT_HOST_MODE())
1371	{
1372	pMsr->u32IndexMSR = MSR_K6_STAR;
1373	pMsr->u32Reserved = 0;
1374	pMsr->u64Value = ASMRdMsr(MSR_K6_STAR); /* legacy syscall eip, cs & ss */
1375	pMsr++; idxMsr++;
1376	pMsr->u32IndexMSR = MSR_K8_LSTAR;
1377	pMsr->u32Reserved = 0;
1378	pMsr->u64Value = ASMRdMsr(MSR_K8_LSTAR); /* 64 bits mode syscall rip */
1379	pMsr++; idxMsr++;
1380	pMsr->u32IndexMSR = MSR_K8_SF_MASK;
1381	pMsr->u32Reserved = 0;
1382	pMsr->u64Value = ASMRdMsr(MSR_K8_SF_MASK); /* syscall flag mask */
1383	pMsr++; idxMsr++;
1384
1385	/* The KERNEL_GS_BASE MSR doesn't work reliably with auto load/store. See @bugref{6208} */
1386	#if 0
1387	pMsr->u32IndexMSR = MSR_K8_KERNEL_GS_BASE;
1388	pMsr->u32Reserved = 0;
1389	pMsr->u64Value = ASMRdMsr(MSR_K8_KERNEL_GS_BASE); /* swapgs exchange value */
1390	pMsr++; idxMsr++;
1391	#endif
1392	}
1393	# endif
1394
1395	if (pVCpu->hwaccm.s.vmx.proc_ctls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
1396	{
1397	pMsr->u32IndexMSR = MSR_K8_TSC_AUX;
1398	pMsr->u32Reserved = 0;
1399	pMsr->u64Value = ASMRdMsr(MSR_K8_TSC_AUX);
1400	pMsr++; idxMsr++;
1401	}
1402
1403	/** @todo r=ramshankar: check IA32_VMX_MISC bits 27:25 for valid idxMsr
1404	* range. */
1405	rc = VMXWriteVMCS(VMX_VMCS_CTRL_EXIT_MSR_LOAD_COUNT, idxMsr);
1406	AssertRC(rc);
1407	#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
1408
1409	pVCpu->hwaccm.s.fContextUseFlags &= ~HWACCM_CHANGED_HOST_CONTEXT;
1410	}
1411	return rc;
1412	}
1413
1414
1415	/**
1416	* Loads the 4 PDPEs into the guest state when nested paging is used and the
1417	* guest operates in PAE mode.
1418	*
1419	* @returns VBox status code.
1420	* @param pVCpu Pointer to the VMCPU.
1421	* @param pCtx Pointer to the guest CPU context.
1422	*/
1423	static int hmR0VmxLoadPaePdpes(PVMCPU pVCpu, PCPUMCTX pCtx)
1424	{
1425	if (CPUMIsGuestInPAEModeEx(pCtx))
1426	{
1427	X86PDPE aPdpes[4];
1428	int rc = PGMGstGetPaePdpes(pVCpu, &aPdpes[0]);
1429	AssertRCReturn(rc, rc);
1430
1431	rc = VMXWriteVMCS64(VMX_VMCS_GUEST_PDPTR0_FULL, aPdpes[0].u); AssertRCReturn(rc, rc);
1432	rc = VMXWriteVMCS64(VMX_VMCS_GUEST_PDPTR1_FULL, aPdpes[1].u); AssertRCReturn(rc, rc);
1433	rc = VMXWriteVMCS64(VMX_VMCS_GUEST_PDPTR2_FULL, aPdpes[2].u); AssertRCReturn(rc, rc);
1434	rc = VMXWriteVMCS64(VMX_VMCS_GUEST_PDPTR3_FULL, aPdpes[3].u); AssertRCReturn(rc, rc);
1435	}
1436	return VINF_SUCCESS;
1437	}
1438
1439
1440	/**
1441	* Saves the 4 PDPEs into the guest state when nested paging is used and the
1442	* guest operates in PAE mode.
1443	*
1444	* @returns VBox status code.
1445	* @param pVCpu Pointer to the VM CPU.
1446	* @param pCtx Pointer to the guest CPU context.
1447	*
1448	* @remarks Tell PGM about CR3 changes before calling this helper.
1449	*/
1450	static int hmR0VmxSavePaePdpes(PVMCPU pVCpu, PCPUMCTX pCtx)
1451	{
1452	if (CPUMIsGuestInPAEModeEx(pCtx))
1453	{
1454	int rc;
1455	X86PDPE aPdpes[4];
1456	rc = VMXReadVMCS64(VMX_VMCS_GUEST_PDPTR0_FULL, &aPdpes[0].u); AssertRCReturn(rc, rc);
1457	rc = VMXReadVMCS64(VMX_VMCS_GUEST_PDPTR1_FULL, &aPdpes[1].u); AssertRCReturn(rc, rc);
1458	rc = VMXReadVMCS64(VMX_VMCS_GUEST_PDPTR2_FULL, &aPdpes[2].u); AssertRCReturn(rc, rc);
1459	rc = VMXReadVMCS64(VMX_VMCS_GUEST_PDPTR3_FULL, &aPdpes[3].u); AssertRCReturn(rc, rc);
1460
1461	rc = PGMGstUpdatePaePdpes(pVCpu, &aPdpes[0]);
1462	AssertRCReturn(rc, rc);
1463	}
1464	return VINF_SUCCESS;
1465	}
1466
1467
1468	/**
1469	* Update the exception bitmap according to the current CPU state.
1470	*
1471	* @param pVM Pointer to the VM.
1472	* @param pVCpu Pointer to the VMCPU.
1473	* @param pCtx Pointer to the guest CPU context.
1474	*/
1475	static void hmR0VmxUpdateExceptionBitmap(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
1476	{
1477	uint32_t u32TrapMask;
1478	Assert(pCtx);
1479
1480	/*
1481	* Set up a mask for intercepting traps.
1482	*/
1483	/** @todo Do we really need to always intercept #DB? */
1484	u32TrapMask = RT_BIT(X86_XCPT_DB)
1485	\| RT_BIT(X86_XCPT_NM)
1486	#ifdef VBOX_ALWAYS_TRAP_PF
1487	\| RT_BIT(X86_XCPT_PF)
1488	#endif
1489	#ifdef VBOX_STRICT
1490	\| RT_BIT(X86_XCPT_BP)
1491	\| RT_BIT(X86_XCPT_DB)
1492	\| RT_BIT(X86_XCPT_DE)
1493	\| RT_BIT(X86_XCPT_NM)
1494	\| RT_BIT(X86_XCPT_UD)
1495	\| RT_BIT(X86_XCPT_NP)
1496	\| RT_BIT(X86_XCPT_SS)
1497	\| RT_BIT(X86_XCPT_GP)
1498	\| RT_BIT(X86_XCPT_MF)
1499	#endif
1500	;
1501
1502	/*
1503	* Without nested paging, #PF must be intercepted to implement shadow paging.
1504	*/
1505	/** @todo NP state won't change so maybe we should build the initial trap mask up front? */
1506	if (!pVM->hwaccm.s.fNestedPaging)
1507	u32TrapMask \|= RT_BIT(X86_XCPT_PF);
1508
1509	/* Catch floating point exceptions if we need to report them to the guest in a different way. */
1510	if (!(pCtx->cr0 & X86_CR0_NE))
1511	u32TrapMask \|= RT_BIT(X86_XCPT_MF);
1512
1513	#ifdef VBOX_STRICT
1514	Assert(u32TrapMask & RT_BIT(X86_XCPT_GP));
1515	#endif
1516
1517	/*
1518	* Intercept all exceptions in real mode as none of them can be injected directly (#GP otherwise).
1519	*/
1520	/** @todo Despite the claim to intercept everything, with NP we do not intercept #PF. Should we? */
1521	if ( CPUMIsGuestInRealModeEx(pCtx)
1522	&& pVM->hwaccm.s.vmx.pRealModeTSS)
1523	{
1524	u32TrapMask \|= RT_BIT(X86_XCPT_DE)
1525	\| RT_BIT(X86_XCPT_DB)
1526	\| RT_BIT(X86_XCPT_NMI)
1527	\| RT_BIT(X86_XCPT_BP)
1528	\| RT_BIT(X86_XCPT_OF)
1529	\| RT_BIT(X86_XCPT_BR)
1530	\| RT_BIT(X86_XCPT_UD)
1531	\| RT_BIT(X86_XCPT_DF)
1532	\| RT_BIT(X86_XCPT_CO_SEG_OVERRUN)
1533	\| RT_BIT(X86_XCPT_TS)
1534	\| RT_BIT(X86_XCPT_NP)
1535	\| RT_BIT(X86_XCPT_SS)
1536	\| RT_BIT(X86_XCPT_GP)
1537	\| RT_BIT(X86_XCPT_MF)
1538	\| RT_BIT(X86_XCPT_AC)
1539	\| RT_BIT(X86_XCPT_MC)
1540	\| RT_BIT(X86_XCPT_XF)
1541	;
1542	}
1543
1544	int rc = VMXWriteVMCS(VMX_VMCS_CTRL_EXCEPTION_BITMAP, u32TrapMask);
1545	AssertRC(rc);
1546	}
1547
1548
1549	/**
1550	* Loads a minimal guest state.
1551	*
1552	* NOTE: Don't do anything here that can cause a jump back to ring 3!!!!!
1553	*
1554	* @param pVM Pointer to the VM.
1555	* @param pVCpu Pointer to the VMCPU.
1556	* @param pCtx Pointer to the guest CPU context.
1557	*/
1558	VMMR0DECL(void) VMXR0LoadMinimalGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
1559	{
1560	int rc;
1561	X86EFLAGS eflags;
1562
1563	Assert(!(pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_ALL_GUEST));
1564
1565	/*
1566	* Load EIP, ESP and EFLAGS.
1567	*/
1568	rc = VMXWriteVMCS64(VMX_VMCS64_GUEST_RIP, pCtx->rip);
1569	rc \|= VMXWriteVMCS64(VMX_VMCS64_GUEST_RSP, pCtx->rsp);
1570	AssertRC(rc);
1571
1572	/*
1573	* Bits 22-31, 15, 5 & 3 must be zero. Bit 1 must be 1.
1574	*/
1575	eflags = pCtx->eflags;
1576	eflags.u32 &= VMX_EFLAGS_RESERVED_0;
1577	eflags.u32 \|= VMX_EFLAGS_RESERVED_1;
1578
1579	/*
1580	* Check if real mode emulation using v86 mode.
1581	*/
1582	if ( CPUMIsGuestInRealModeEx(pCtx)
1583	&& pVM->hwaccm.s.vmx.pRealModeTSS)
1584	{
1585	pVCpu->hwaccm.s.vmx.RealMode.eflags = eflags;
1586
1587	eflags.Bits.u1VM = 1;
1588	eflags.Bits.u2IOPL = 0; /* must always be 0 or else certain instructions won't cause faults. */
1589	}
1590	rc = VMXWriteVMCS(VMX_VMCS_GUEST_RFLAGS, eflags.u32);
1591	AssertRC(rc);
1592	}
1593
1594
1595	/**
1596	* Loads the guest state.
1597	*
1598	* NOTE: Don't do anything here that can cause a jump back to ring 3!!!!!
1599	*
1600	* @returns VBox status code.
1601	* @param pVM Pointer to the VM.
1602	* @param pVCpu Pointer to the VMCPU.
1603	* @param pCtx Pointer to the guest CPU context.
1604	*/
1605	VMMR0DECL(int) VMXR0LoadGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
1606	{
1607	int rc = VINF_SUCCESS;
1608	RTGCUINTPTR val;
1609
1610	/*
1611	* VMX_VMCS_CTRL_ENTRY_CONTROLS
1612	* Set required bits to one and zero according to the MSR capabilities.
1613	*/
1614	val = pVM->hwaccm.s.vmx.msr.vmx_entry.n.disallowed0;
1615
1616	/*
1617	* Load guest debug controls (DR7 & IA32_DEBUGCTL_MSR).
1618	* Forced to 1 on the 'first' VT-x capable CPUs; this actually includes the newest Nehalem CPUs
1619	*/
1620	val \|= VMX_VMCS_CTRL_ENTRY_CONTROLS_LOAD_DEBUG;
1621
1622	if (CPUMIsGuestInLongModeEx(pCtx))
1623	val \|= VMX_VMCS_CTRL_ENTRY_CONTROLS_IA64_MODE;
1624	/* else Must be zero when AMD64 is not available. */
1625
1626	/*
1627	* Mask away the bits that the CPU doesn't support.
1628	*/
1629	val &= pVM->hwaccm.s.vmx.msr.vmx_entry.n.allowed1;
1630	rc = VMXWriteVMCS(VMX_VMCS_CTRL_ENTRY_CONTROLS, val);
1631	AssertRC(rc);
1632
1633	/*
1634	* VMX_VMCS_CTRL_EXIT_CONTROLS
1635	* Set required bits to one and zero according to the MSR capabilities.
1636	*/
1637	val = pVM->hwaccm.s.vmx.msr.vmx_exit.n.disallowed0;
1638
1639	/*
1640	* Save debug controls (DR7 & IA32_DEBUGCTL_MSR)
1641	* Forced to 1 on the 'first' VT-x capable CPUs; this actually includes the newest Nehalem CPUs
1642	*/
1643	val \|= VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_DEBUG;
1644
1645	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
1646	if (VMX_IS_64BIT_HOST_MODE())
1647	val \|= VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_AMD64;
1648	/* else Must be zero when AMD64 is not available. */
1649	#elif HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
1650	if (CPUMIsGuestInLongModeEx(pCtx))
1651	val \|= VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_AMD64; /* our switcher goes to long mode */
1652	else
1653	Assert(!(val & VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_AMD64));
1654	#endif
1655	val &= pVM->hwaccm.s.vmx.msr.vmx_exit.n.allowed1;
1656
1657	/*
1658	* Don't acknowledge external interrupts on VM-exit.
1659	*/
1660	rc = VMXWriteVMCS(VMX_VMCS_CTRL_EXIT_CONTROLS, val);
1661	AssertRC(rc);
1662
1663	/*
1664	* Guest CPU context: ES, CS, SS, DS, FS, GS.
1665	*/
1666	if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_SEGMENT_REGS)
1667	{
1668	if (pVM->hwaccm.s.vmx.pRealModeTSS)
1669	{
1670	PGMMODE enmGuestMode = PGMGetGuestMode(pVCpu);
1671	if (pVCpu->hwaccm.s.vmx.enmLastSeenGuestMode != enmGuestMode)
1672	{
1673	/*
1674	* Correct weird requirements for switching to protected mode.
1675	*/
1676	if ( pVCpu->hwaccm.s.vmx.enmLastSeenGuestMode == PGMMODE_REAL
1677	&& enmGuestMode >= PGMMODE_PROTECTED)
1678	{
1679	#ifdef VBOX_WITH_REM
1680	/*
1681	* Flush the recompiler code cache as it's not unlikely the guest will rewrite code
1682	* it will later execute in real mode (OpenBSD 4.0 is one such example)
1683	*/
1684	REMFlushTBs(pVM);
1685	#endif
1686
1687	/*
1688	* DPL of all hidden selector registers must match the current CPL (0).
1689	*/
1690	pCtx->cs.Attr.n.u2Dpl = 0;
1691	pCtx->cs.Attr.n.u4Type = X86_SEL_TYPE_CODE \| X86_SEL_TYPE_RW_ACC;
1692
1693	pCtx->ds.Attr.n.u2Dpl = 0;
1694	pCtx->es.Attr.n.u2Dpl = 0;
1695	pCtx->fs.Attr.n.u2Dpl = 0;
1696	pCtx->gs.Attr.n.u2Dpl = 0;
1697	pCtx->ss.Attr.n.u2Dpl = 0;
1698	}
1699	pVCpu->hwaccm.s.vmx.enmLastSeenGuestMode = enmGuestMode;
1700	}
1701	else if ( CPUMIsGuestInRealModeEx(pCtx)
1702	&& pCtx->cs.u64Base == 0xffff0000)
1703	{
1704	/* VT-x will fail with a guest invalid state otherwise... (CPU state after a reset) */
1705	pCtx->cs.u64Base = 0xf0000;
1706	pCtx->cs.Sel = 0xf000;
1707	}
1708	}
1709
1710	VMX_WRITE_SELREG(ES, es);
1711	AssertRC(rc);
1712
1713	VMX_WRITE_SELREG(CS, cs);
1714	AssertRC(rc);
1715
1716	VMX_WRITE_SELREG(SS, ss);
1717	AssertRC(rc);
1718
1719	VMX_WRITE_SELREG(DS, ds);
1720	AssertRC(rc);
1721
1722	VMX_WRITE_SELREG(FS, fs);
1723	AssertRC(rc);
1724
1725	VMX_WRITE_SELREG(GS, gs);
1726	AssertRC(rc);
1727	}
1728
1729	/*
1730	* Guest CPU context: LDTR.
1731	*/
1732	if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_LDTR)
1733	{
1734	if (pCtx->ldtr.Sel == 0)
1735	{
1736	rc = VMXWriteVMCS(VMX_VMCS16_GUEST_FIELD_LDTR, 0);
1737	rc \|= VMXWriteVMCS(VMX_VMCS32_GUEST_LDTR_LIMIT, 0);
1738	rc \|= VMXWriteVMCS64(VMX_VMCS64_GUEST_LDTR_BASE, 0);
1739	/* Note: vmlaunch will fail with 0 or just 0x02. No idea why. */
1740	rc \|= VMXWriteVMCS(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, 0x82 /* present, LDT */);
1741	}
1742	else
1743	{
1744	rc = VMXWriteVMCS(VMX_VMCS16_GUEST_FIELD_LDTR, pCtx->ldtr.Sel);
1745	rc \|= VMXWriteVMCS(VMX_VMCS32_GUEST_LDTR_LIMIT, pCtx->ldtr.u32Limit);
1746	rc \|= VMXWriteVMCS64(VMX_VMCS64_GUEST_LDTR_BASE, pCtx->ldtr.u64Base);
1747	rc \|= VMXWriteVMCS(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, pCtx->ldtr.Attr.u);
1748	}
1749	AssertRC(rc);
1750	}
1751
1752	/*
1753	* Guest CPU context: TR.
1754	*/
1755	if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_TR)
1756	{
1757	/*
1758	* Real mode emulation using v86 mode with CR4.VME (interrupt redirection
1759	* using the int bitmap in the TSS).
1760	*/
1761	if ( CPUMIsGuestInRealModeEx(pCtx)
1762	&& pVM->hwaccm.s.vmx.pRealModeTSS)
1763	{
1764	RTGCPHYS GCPhys;
1765
1766	/* We convert it here every time as PCI regions could be reconfigured. */
1767	rc = PDMVMMDevHeapR3ToGCPhys(pVM, pVM->hwaccm.s.vmx.pRealModeTSS, &GCPhys);
1768	AssertRC(rc);
1769
1770	rc = VMXWriteVMCS(VMX_VMCS16_GUEST_FIELD_TR, 0);
1771	rc \|= VMXWriteVMCS(VMX_VMCS32_GUEST_TR_LIMIT, HWACCM_VTX_TSS_SIZE);
1772	rc \|= VMXWriteVMCS64(VMX_VMCS64_GUEST_TR_BASE, GCPhys /* phys = virt in this mode */);
1773
1774	X86DESCATTR attr;
1775
1776	attr.u = 0;
1777	attr.n.u1Present = 1;
1778	attr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
1779	val = attr.u;
1780	}
1781	else
1782	{
1783	rc = VMXWriteVMCS(VMX_VMCS16_GUEST_FIELD_TR, pCtx->tr.Sel);
1784	rc \|= VMXWriteVMCS(VMX_VMCS32_GUEST_TR_LIMIT, pCtx->tr.u32Limit);
1785	rc \|= VMXWriteVMCS64(VMX_VMCS64_GUEST_TR_BASE, pCtx->tr.u64Base);
1786
1787	val = pCtx->tr.Attr.u;
1788
1789	/* The TSS selector must be busy (REM bugs? see defect #XXXX). */
1790	if (!(val & X86_SEL_TYPE_SYS_TSS_BUSY_MASK))
1791	{
1792	if (val & 0xf)
1793	val \|= X86_SEL_TYPE_SYS_TSS_BUSY_MASK;
1794	else
1795	/* Default if no TR selector has been set (otherwise vmlaunch will fail!) */
1796	val = (val & ~0xF) \| X86_SEL_TYPE_SYS_386_TSS_BUSY;
1797	}
1798	AssertMsg((val & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY \|\| (val & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY,
1799	("%#x\n", val));
1800	}
1801	rc \|= VMXWriteVMCS(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, val);
1802	AssertRC(rc);
1803	}
1804
1805	/*
1806	* Guest CPU context: GDTR.
1807	*/
1808	if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_GDTR)
1809	{
1810	rc = VMXWriteVMCS(VMX_VMCS32_GUEST_GDTR_LIMIT, pCtx->gdtr.cbGdt);
1811	rc \|= VMXWriteVMCS64(VMX_VMCS64_GUEST_GDTR_BASE, pCtx->gdtr.pGdt);
1812	AssertRC(rc);
1813	}
1814
1815	/*
1816	* Guest CPU context: IDTR.
1817	*/
1818	if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_IDTR)
1819	{
1820	rc = VMXWriteVMCS(VMX_VMCS32_GUEST_IDTR_LIMIT, pCtx->idtr.cbIdt);
1821	rc \|= VMXWriteVMCS64(VMX_VMCS64_GUEST_IDTR_BASE, pCtx->idtr.pIdt);
1822	AssertRC(rc);
1823	}
1824
1825	/*
1826	* Sysenter MSRs.
1827	*/
1828	if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_MSR)
1829	{
1830	rc = VMXWriteVMCS(VMX_VMCS32_GUEST_SYSENTER_CS, pCtx->SysEnter.cs);
1831	rc \|= VMXWriteVMCS64(VMX_VMCS64_GUEST_SYSENTER_EIP, pCtx->SysEnter.eip);
1832	rc \|= VMXWriteVMCS64(VMX_VMCS64_GUEST_SYSENTER_ESP, pCtx->SysEnter.esp);
1833	AssertRC(rc);
1834	}
1835
1836	/*
1837	* Guest CPU context: Control registers.
1838	*/
1839	if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_CR0)
1840	{
1841	val = pCtx->cr0;
1842	rc = VMXWriteVMCS(VMX_VMCS_CTRL_CR0_READ_SHADOW, val);
1843	Log2(("Guest CR0-shadow %08x\n", val));
1844	if (CPUMIsGuestFPUStateActive(pVCpu) == false)
1845	{
1846	/* Always use #NM exceptions to load the FPU/XMM state on demand. */
1847	val \|= X86_CR0_TS \| X86_CR0_ET \| X86_CR0_NE \| X86_CR0_MP;
1848	}
1849	else
1850	{
1851	/** @todo check if we support the old style mess correctly. */
1852	if (!(val & X86_CR0_NE))
1853	Log(("Forcing X86_CR0_NE!!!\n"));
1854
1855	val \|= X86_CR0_NE; /* always turn on the native mechanism to report FPU errors (old style uses interrupts) */
1856	}
1857	/* Protected mode & paging are always enabled; we use them for emulating real and protected mode without paging too. */
1858	if (!pVM->hwaccm.s.vmx.fUnrestrictedGuest)
1859	val \|= X86_CR0_PE \| X86_CR0_PG;
1860
1861	if (pVM->hwaccm.s.fNestedPaging)
1862	{
1863	if (CPUMIsGuestInPagedProtectedModeEx(pCtx))
1864	{
1865	/* Disable CR3 read/write monitoring as we don't need it for EPT. */
1866	pVCpu->hwaccm.s.vmx.proc_ctls &= ~( VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
1867	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT);
1868	}
1869	else
1870	{
1871	/* Reenable CR3 read/write monitoring as our identity mapped page table is active. */
1872	pVCpu->hwaccm.s.vmx.proc_ctls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
1873	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT;
1874	}
1875	rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
1876	AssertRC(rc);
1877	}
1878	else
1879	{
1880	/* Note: We must also set this as we rely on protecting various pages for which supervisor writes must be caught. */
1881	val \|= X86_CR0_WP;
1882	}
1883
1884	/* Always enable caching. */
1885	val &= ~(X86_CR0_CD\|X86_CR0_NW);
1886
1887	rc \|= VMXWriteVMCS64(VMX_VMCS64_GUEST_CR0, val);
1888	Log2(("Guest CR0 %08x\n", val));
1889
1890	/*
1891	* CR0 flags owned by the host; if the guests attempts to change them, then the VM will exit.
1892	*/
1893	val = X86_CR0_PE /* Must monitor this bit (assumptions are made for real mode emulation) */
1894	\| X86_CR0_WP /* Must monitor this bit (it must always be enabled). */
1895	\| X86_CR0_PG /* Must monitor this bit (assumptions are made for real mode & protected mode without paging emulation) */
1896	\| X86_CR0_CD /* Bit not restored during VM-exit! */
1897	\| X86_CR0_NW /* Bit not restored during VM-exit! */
1898	\| X86_CR0_NE;
1899
1900	/*
1901	* When the guest's FPU state is active, then we no longer care about the FPU related bits.
1902	*/
1903	if (CPUMIsGuestFPUStateActive(pVCpu) == false)
1904	val \|= X86_CR0_TS \| X86_CR0_ET \| X86_CR0_MP;
1905
1906	pVCpu->hwaccm.s.vmx.cr0_mask = val;
1907
1908	rc \|= VMXWriteVMCS(VMX_VMCS_CTRL_CR0_MASK, val);
1909	Log2(("Guest CR0-mask %08x\n", val));
1910	AssertRC(rc);
1911	}
1912
1913	if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_CR4)
1914	{
1915	rc = VMXWriteVMCS(VMX_VMCS_CTRL_CR4_READ_SHADOW, pCtx->cr4);
1916	Log2(("Guest CR4-shadow %08x\n", pCtx->cr4));
1917	/* Set the required bits in cr4 too (currently X86_CR4_VMXE). */
1918	val = pCtx->cr4 \| (uint32_t)pVM->hwaccm.s.vmx.msr.vmx_cr4_fixed0;
1919
1920	if (!pVM->hwaccm.s.fNestedPaging)
1921	{
1922	switch (pVCpu->hwaccm.s.enmShadowMode)
1923	{
1924	case PGMMODE_REAL: /* Real mode -> emulated using v86 mode */
1925	case PGMMODE_PROTECTED: /* Protected mode, no paging -> emulated using identity mapping. */
1926	case PGMMODE_32_BIT: /* 32-bit paging. */
1927	val &= ~X86_CR4_PAE;
1928	break;
1929
1930	case PGMMODE_PAE: /* PAE paging. */
1931	case PGMMODE_PAE_NX: /* PAE paging with NX enabled. */
1932	/** Must use PAE paging as we could use physical memory > 4 GB */
1933	val \|= X86_CR4_PAE;
1934	break;
1935
1936	case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
1937	case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
1938	#ifdef VBOX_ENABLE_64_BITS_GUESTS
1939	break;
1940	#else
1941	AssertFailed();
1942	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
1943	#endif
1944	default: /* shut up gcc */
1945	AssertFailed();
1946	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
1947	}
1948	}
1949	else if ( !CPUMIsGuestInPagedProtectedModeEx(pCtx)
1950	&& !pVM->hwaccm.s.vmx.fUnrestrictedGuest)
1951	{
1952	/* We use 4 MB pages in our identity mapping page table for real and protected mode without paging. */
1953	val \|= X86_CR4_PSE;
1954	/* Our identity mapping is a 32 bits page directory. */
1955	val &= ~X86_CR4_PAE;
1956	}
1957
1958	/*
1959	* Turn off VME if we're in emulated real mode.
1960	*/
1961	if ( CPUMIsGuestInRealModeEx(pCtx)
1962	&& pVM->hwaccm.s.vmx.pRealModeTSS)
1963	{
1964	val &= ~X86_CR4_VME;
1965	}
1966
1967	rc \|= VMXWriteVMCS64(VMX_VMCS64_GUEST_CR4, val);
1968	Log2(("Guest CR4 %08x\n", val));
1969
1970	/*
1971	* CR4 flags owned by the host; if the guests attempts to change them, then the VM will exit.
1972	*/
1973	val = 0
1974	\| X86_CR4_VME
1975	\| X86_CR4_PAE
1976	\| X86_CR4_PGE
1977	\| X86_CR4_PSE
1978	\| X86_CR4_VMXE;
1979	pVCpu->hwaccm.s.vmx.cr4_mask = val;
1980
1981	rc \|= VMXWriteVMCS(VMX_VMCS_CTRL_CR4_MASK, val);
1982	Log2(("Guest CR4-mask %08x\n", val));
1983	AssertRC(rc);
1984	}
1985
1986	#if 0
1987	/* Enable single stepping if requested and CPU supports it. */
1988	if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_TRAP_FLAG)
1989	if (DBGFIsStepping(pVCpu))
1990	{
1991	pVCpu->hwaccm.s.vmx.proc_ctls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_TRAP_FLAG;
1992	rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
1993	AssertRC(rc);
1994	}
1995	#endif
1996
1997	if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_CR3)
1998	{
1999	if (pVM->hwaccm.s.fNestedPaging)
2000	{
2001	Assert(PGMGetHyperCR3(pVCpu));
2002	pVCpu->hwaccm.s.vmx.GCPhysEPTP = PGMGetHyperCR3(pVCpu);
2003
2004	Assert(!(pVCpu->hwaccm.s.vmx.GCPhysEPTP & 0xfff));
2005	/** @todo Check the IA32_VMX_EPT_VPID_CAP MSR for other supported memory types. */
2006	pVCpu->hwaccm.s.vmx.GCPhysEPTP \|= VMX_EPT_MEMTYPE_WB
2007	\| (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
2008
2009	rc = VMXWriteVMCS64(VMX_VMCS_CTRL_EPTP_FULL, pVCpu->hwaccm.s.vmx.GCPhysEPTP);
2010	AssertRC(rc);
2011
2012	if ( !CPUMIsGuestInPagedProtectedModeEx(pCtx)
2013	&& !pVM->hwaccm.s.vmx.fUnrestrictedGuest)
2014	{
2015	RTGCPHYS GCPhys;
2016
2017	/* We convert it here every time as PCI regions could be reconfigured. */
2018	rc = PDMVMMDevHeapR3ToGCPhys(pVM, pVM->hwaccm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
2019	AssertMsgRC(rc, ("pNonPagingModeEPTPageTable = %RGv\n", pVM->hwaccm.s.vmx.pNonPagingModeEPTPageTable));
2020
2021	/*
2022	* We use our identity mapping page table here as we need to map guest virtual to
2023	* guest physical addresses; EPT will take care of the translation to host physical addresses.
2024	*/
2025	val = GCPhys;
2026	}
2027	else
2028	{
2029	/* Save the real guest CR3 in VMX_VMCS_GUEST_CR3 */
2030	val = pCtx->cr3;
2031	rc = hmR0VmxLoadPaePdpes(pVCpu, pCtx);
2032	AssertRCReturn(rc, rc);
2033	}
2034	}
2035	else
2036	{
2037	val = PGMGetHyperCR3(pVCpu);
2038	Assert(val \|\| VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL));
2039	}
2040
2041	/* Save our shadow CR3 register. */
2042	rc = VMXWriteVMCS64(VMX_VMCS64_GUEST_CR3, val);
2043	AssertRC(rc);
2044	}
2045
2046	/*
2047	* Guest CPU context: Debug registers.
2048	*/
2049	if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_DEBUG)
2050	{
2051	pCtx->dr[6] \|= X86_DR6_INIT_VAL; /* set all reserved bits to 1. */
2052	pCtx->dr[6] &= ~RT_BIT(12); /* must be zero. */
2053
2054	pCtx->dr[7] &= 0xffffffff; /* upper 32 bits reserved */
2055	pCtx->dr[7] &= ~(RT_BIT(11) \| RT_BIT(12) \| RT_BIT(14) \| RT_BIT(15)); /* must be zero */
2056	pCtx->dr[7] \|= 0x400; /* must be one */
2057
2058	/* Resync DR7 */
2059	rc = VMXWriteVMCS64(VMX_VMCS64_GUEST_DR7, pCtx->dr[7]);
2060	AssertRC(rc);
2061
2062	#ifdef DEBUG
2063	/* Sync the hypervisor debug state now if any breakpoint is armed. */
2064	if ( CPUMGetHyperDR7(pVCpu) & (X86_DR7_ENABLED_MASK\|X86_DR7_GD)
2065	&& !CPUMIsHyperDebugStateActive(pVCpu)
2066	&& !DBGFIsStepping(pVCpu))
2067	{
2068	/* Save the host and load the hypervisor debug state. */
2069	rc = CPUMR0LoadHyperDebugState(pVM, pVCpu, pCtx, true /* include DR6 */);
2070	AssertRC(rc);
2071
2072	/* DRx intercepts remain enabled. */
2073
2074	/* Override dr7 with the hypervisor value. */
2075	rc = VMXWriteVMCS64(VMX_VMCS64_GUEST_DR7, CPUMGetHyperDR7(pVCpu));
2076	AssertRC(rc);
2077	}
2078	else
2079	#endif
2080	/* Sync the debug state now if any breakpoint is armed. */
2081	if ( (pCtx->dr[7] & (X86_DR7_ENABLED_MASK\|X86_DR7_GD))
2082	&& !CPUMIsGuestDebugStateActive(pVCpu)
2083	&& !DBGFIsStepping(pVCpu))
2084	{
2085	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatDRxArmed);
2086
2087	/* Disable DRx move intercepts. */
2088	pVCpu->hwaccm.s.vmx.proc_ctls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT;
2089	rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
2090	AssertRC(rc);
2091
2092	/* Save the host and load the guest debug state. */
2093	rc = CPUMR0LoadGuestDebugState(pVM, pVCpu, pCtx, true /* include DR6 */);
2094	AssertRC(rc);
2095	}
2096
2097	/* IA32_DEBUGCTL MSR. */
2098	rc = VMXWriteVMCS64(VMX_VMCS_GUEST_DEBUGCTL_FULL, 0);
2099	AssertRC(rc);
2100
2101	/** @todo do we really ever need this? */
2102	rc \|= VMXWriteVMCS(VMX_VMCS_GUEST_DEBUG_EXCEPTIONS, 0);
2103	AssertRC(rc);
2104	}
2105
2106	/*
2107	* 64-bit guest mode.
2108	*/
2109	if (CPUMIsGuestInLongModeEx(pCtx))
2110	{
2111	#if !defined(VBOX_ENABLE_64_BITS_GUESTS)
2112	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
2113	#elif HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2114	pVCpu->hwaccm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
2115	#else
2116	# ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2117	if (!pVM->hwaccm.s.fAllow64BitGuests)
2118	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
2119	# endif
2120	pVCpu->hwaccm.s.vmx.pfnStartVM = VMXR0StartVM64;
2121	#endif
2122	if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_MSR)
2123	{
2124	/* Update these as wrmsr might have changed them. */
2125	rc = VMXWriteVMCS64(VMX_VMCS64_GUEST_FS_BASE, pCtx->fs.u64Base);
2126	AssertRC(rc);
2127	rc = VMXWriteVMCS64(VMX_VMCS64_GUEST_GS_BASE, pCtx->gs.u64Base);
2128	AssertRC(rc);
2129	}
2130	}
2131	else
2132	{
2133	pVCpu->hwaccm.s.vmx.pfnStartVM = VMXR0StartVM32;
2134	}
2135
2136	hmR0VmxUpdateExceptionBitmap(pVM, pVCpu, pCtx);
2137
2138	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
2139	/*
2140	* Store all guest MSRs in the VM-entry load area, so they will be loaded
2141	* during VM-entry and restored into the VM-exit store area during VM-exit.
2142	*/
2143	PVMXMSR pMsr = (PVMXMSR)pVCpu->hwaccm.s.vmx.pGuestMSR;
2144	unsigned idxMsr = 0;
2145
2146	uint32_t u32GstExtFeatures;
2147	uint32_t u32Temp;
2148	CPUMGetGuestCpuId(pVCpu, 0x80000001, &u32Temp, &u32Temp, &u32Temp, &u32GstExtFeatures);
2149
2150	if (u32GstExtFeatures & (X86_CPUID_EXT_FEATURE_EDX_NX \| X86_CPUID_EXT_FEATURE_EDX_LONG_MODE))
2151	{
2152	#if 0
2153	pMsr->u32IndexMSR = MSR_K6_EFER;
2154	pMsr->u32Reserved = 0;
2155	pMsr->u64Value = pCtx->msrEFER;
2156	/* VT-x will complain if only MSR_K6_EFER_LME is set. */
2157	if (!CPUMIsGuestInLongModeEx(pCtx))
2158	pMsr->u64Value &= ~(MSR_K6_EFER_LMA \| MSR_K6_EFER_LME);
2159	pMsr++; idxMsr++;
2160	#endif
2161
2162	if (u32GstExtFeatures & X86_CPUID_EXT_FEATURE_EDX_LONG_MODE)
2163	{
2164	pMsr->u32IndexMSR = MSR_K8_LSTAR;
2165	pMsr->u32Reserved = 0;
2166	pMsr->u64Value = pCtx->msrLSTAR; /* 64 bits mode syscall rip */
2167	pMsr++; idxMsr++;
2168	pMsr->u32IndexMSR = MSR_K6_STAR;
2169	pMsr->u32Reserved = 0;
2170	pMsr->u64Value = pCtx->msrSTAR; /* legacy syscall eip, cs & ss */
2171	pMsr++; idxMsr++;
2172	pMsr->u32IndexMSR = MSR_K8_SF_MASK;
2173	pMsr->u32Reserved = 0;
2174	pMsr->u64Value = pCtx->msrSFMASK; /* syscall flag mask */
2175	pMsr++; idxMsr++;
2176
2177	/* The KERNEL_GS_BASE MSR doesn't work reliably with auto load/store. See @bugref{6208} */
2178	#if 0
2179	pMsr->u32IndexMSR = MSR_K8_KERNEL_GS_BASE;
2180	pMsr->u32Reserved = 0;
2181	pMsr->u64Value = pCtx->msrKERNELGSBASE; /* swapgs exchange value */
2182	pMsr++; idxMsr++;
2183	#endif
2184	}
2185	}
2186
2187	if ( pVCpu->hwaccm.s.vmx.proc_ctls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP
2188	&& (u32GstExtFeatures & X86_CPUID_EXT_FEATURE_EDX_RDTSCP))
2189	{
2190	pMsr->u32IndexMSR = MSR_K8_TSC_AUX;
2191	pMsr->u32Reserved = 0;
2192	rc = CPUMQueryGuestMsr(pVCpu, MSR_K8_TSC_AUX, &pMsr->u64Value);
2193	AssertRC(rc);
2194	pMsr++; idxMsr++;
2195	}
2196
2197	pVCpu->hwaccm.s.vmx.cCachedMSRs = idxMsr;
2198
2199	rc = VMXWriteVMCS(VMX_VMCS_CTRL_ENTRY_MSR_LOAD_COUNT, idxMsr);
2200	AssertRC(rc);
2201
2202	rc = VMXWriteVMCS(VMX_VMCS_CTRL_EXIT_MSR_STORE_COUNT, idxMsr);
2203	AssertRC(rc);
2204	#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
2205
2206	bool fOffsettedTsc;
2207	if (pVM->hwaccm.s.vmx.fUsePreemptTimer)
2208	{
2209	uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVCpu, &fOffsettedTsc, &pVCpu->hwaccm.s.vmx.u64TSCOffset);
2210
2211	/* Make sure the returned values have sane upper and lower boundaries. */
2212	uint64_t u64CpuHz = SUPGetCpuHzFromGIP(g_pSUPGlobalInfoPage);
2213
2214	cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64 of a second */
2215	cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
2216
2217	cTicksToDeadline >>= pVM->hwaccm.s.vmx.cPreemptTimerShift;
2218	uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
2219	rc = VMXWriteVMCS(VMX_VMCS32_GUEST_PREEMPTION_TIMER_VALUE, cPreemptionTickCount);
2220	AssertRC(rc);
2221	}
2222	else
2223	fOffsettedTsc = TMCpuTickCanUseRealTSC(pVCpu, &pVCpu->hwaccm.s.vmx.u64TSCOffset);
2224
2225	if (fOffsettedTsc)
2226	{
2227	uint64_t u64CurTSC = ASMReadTSC();
2228	if (u64CurTSC + pVCpu->hwaccm.s.vmx.u64TSCOffset >= TMCpuTickGetLastSeen(pVCpu))
2229	{
2230	/* Note: VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT takes precedence over TSC_OFFSET, applies to RDTSCP too. */
2231	rc = VMXWriteVMCS64(VMX_VMCS_CTRL_TSC_OFFSET_FULL, pVCpu->hwaccm.s.vmx.u64TSCOffset);
2232	AssertRC(rc);
2233
2234	pVCpu->hwaccm.s.vmx.proc_ctls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT;
2235	rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
2236	AssertRC(rc);
2237	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatTSCOffset);
2238	}
2239	else
2240	{
2241	/* Fall back to rdtsc, rdtscp emulation as we would otherwise pass decreasing tsc values to the guest. */
2242	LogFlow(("TSC %RX64 offset %RX64 time=%RX64 last=%RX64 (diff=%RX64, virt_tsc=%RX64)\n", u64CurTSC,
2243	pVCpu->hwaccm.s.vmx.u64TSCOffset, u64CurTSC + pVCpu->hwaccm.s.vmx.u64TSCOffset,
2244	TMCpuTickGetLastSeen(pVCpu), TMCpuTickGetLastSeen(pVCpu) - u64CurTSC - pVCpu->hwaccm.s.vmx.u64TSCOffset,
2245	TMCpuTickGet(pVCpu)));
2246	pVCpu->hwaccm.s.vmx.proc_ctls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT;
2247	rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
2248	AssertRC(rc);
2249	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatTSCInterceptOverFlow);
2250	}
2251	}
2252	else
2253	{
2254	pVCpu->hwaccm.s.vmx.proc_ctls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT;
2255	rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
2256	AssertRC(rc);
2257	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatTSCIntercept);
2258	}
2259
2260	/* Done with the major changes */
2261	pVCpu->hwaccm.s.fContextUseFlags &= ~HWACCM_CHANGED_ALL_GUEST;
2262
2263	/* Minimal guest state update (ESP, EIP, EFLAGS mostly) */
2264	VMXR0LoadMinimalGuestState(pVM, pVCpu, pCtx);
2265	return rc;
2266	}
2267
2268
2269	/**
2270	* Syncs back the guest state from VMCS.
2271	*
2272	* @returns VBox status code.
2273	* @param pVM Pointer to the VM.
2274	* @param pVCpu Pointer to the VMCPU.
2275	* @param pCtx Pointer to the guest CPU context.
2276	*/
2277	DECLINLINE(int) VMXR0SaveGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
2278	{
2279	RTGCUINTREG val, valShadow;
2280	RTGCUINTPTR uInterruptState;
2281	int rc;
2282
2283	/* First sync back EIP, ESP, and EFLAGS. */
2284	rc = VMXReadCachedVMCS(VMX_VMCS64_GUEST_RIP, &val);
2285	AssertRC(rc);
2286	pCtx->rip = val;
2287	rc = VMXReadCachedVMCS(VMX_VMCS64_GUEST_RSP, &val);
2288	AssertRC(rc);
2289	pCtx->rsp = val;
2290	rc = VMXReadCachedVMCS(VMX_VMCS_GUEST_RFLAGS, &val);
2291	AssertRC(rc);
2292	pCtx->eflags.u32 = val;
2293
2294	/* Take care of instruction fusing (sti, mov ss) */
2295	rc \|= VMXReadCachedVMCS(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &val);
2296	uInterruptState = val;
2297	if (uInterruptState != 0)
2298	{
2299	Assert(uInterruptState <= 2); /* only sti & mov ss */
2300	Log(("uInterruptState %x eip=%RGv\n", (uint32_t)uInterruptState, pCtx->rip));
2301	EMSetInhibitInterruptsPC(pVCpu, pCtx->rip);
2302	}
2303	else
2304	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
2305
2306	/* Control registers. */
2307	VMXReadCachedVMCS(VMX_VMCS_CTRL_CR0_READ_SHADOW, &valShadow);
2308	VMXReadCachedVMCS(VMX_VMCS64_GUEST_CR0, &val);
2309	val = (valShadow & pVCpu->hwaccm.s.vmx.cr0_mask) \| (val & ~pVCpu->hwaccm.s.vmx.cr0_mask);
2310	CPUMSetGuestCR0(pVCpu, val);
2311
2312	VMXReadCachedVMCS(VMX_VMCS_CTRL_CR4_READ_SHADOW, &valShadow);
2313	VMXReadCachedVMCS(VMX_VMCS64_GUEST_CR4, &val);
2314	val = (valShadow & pVCpu->hwaccm.s.vmx.cr4_mask) \| (val & ~pVCpu->hwaccm.s.vmx.cr4_mask);
2315	CPUMSetGuestCR4(pVCpu, val);
2316
2317	/*
2318	* No reason to sync back the CRx registers. They can't be changed by the guest unless in
2319	* the nested paging case where CR3 & CR4 can be changed by the guest.
2320	*/
2321	if ( pVM->hwaccm.s.fNestedPaging
2322	&& CPUMIsGuestInPagedProtectedModeEx(pCtx)) /** @todo check if we will always catch mode switches and such... */
2323	{
2324	PVMCSCACHE pCache = &pVCpu->hwaccm.s.vmx.VMCSCache;
2325
2326	/* Can be updated behind our back in the nested paging case. */
2327	CPUMSetGuestCR2(pVCpu, pCache->cr2);
2328
2329	VMXReadCachedVMCS(VMX_VMCS64_GUEST_CR3, &val);
2330
2331	if (val != pCtx->cr3)
2332	{
2333	CPUMSetGuestCR3(pVCpu, val);
2334	PGMUpdateCR3(pVCpu, val);
2335	}
2336	rc = hmR0VmxSavePaePdpes(pVCpu, pCtx);
2337	AssertRCReturn(rc, rc);
2338	}
2339
2340	/* Sync back DR7. */
2341	VMXReadCachedVMCS(VMX_VMCS64_GUEST_DR7, &val);
2342	pCtx->dr[7] = val;
2343
2344	/* Guest CPU context: ES, CS, SS, DS, FS, GS. */
2345	VMX_READ_SELREG(ES, es);
2346	VMX_READ_SELREG(SS, ss);
2347	VMX_READ_SELREG(CS, cs);
2348	VMX_READ_SELREG(DS, ds);
2349	VMX_READ_SELREG(FS, fs);
2350	VMX_READ_SELREG(GS, gs);
2351
2352	/* System MSRs */
2353	VMXReadCachedVMCS(VMX_VMCS32_GUEST_SYSENTER_CS, &val);
2354	pCtx->SysEnter.cs = val;
2355	VMXReadCachedVMCS(VMX_VMCS64_GUEST_SYSENTER_EIP, &val);
2356	pCtx->SysEnter.eip = val;
2357	VMXReadCachedVMCS(VMX_VMCS64_GUEST_SYSENTER_ESP, &val);
2358	pCtx->SysEnter.esp = val;
2359
2360	/* Misc. registers; must sync everything otherwise we can get out of sync when jumping to ring 3. */
2361	VMX_READ_SELREG(LDTR, ldtr);
2362
2363	VMXReadCachedVMCS(VMX_VMCS32_GUEST_GDTR_LIMIT, &val);
2364	pCtx->gdtr.cbGdt = val;
2365	VMXReadCachedVMCS(VMX_VMCS64_GUEST_GDTR_BASE, &val);
2366	pCtx->gdtr.pGdt = val;
2367
2368	VMXReadCachedVMCS(VMX_VMCS32_GUEST_IDTR_LIMIT, &val);
2369	pCtx->idtr.cbIdt = val;
2370	VMXReadCachedVMCS(VMX_VMCS64_GUEST_IDTR_BASE, &val);
2371	pCtx->idtr.pIdt = val;
2372
2373	/* Real mode emulation using v86 mode. */
2374	if ( CPUMIsGuestInRealModeEx(pCtx)
2375	&& pVM->hwaccm.s.vmx.pRealModeTSS)
2376	{
2377	/* Hide our emulation flags */
2378	pCtx->eflags.Bits.u1VM = 0;
2379
2380	/* Restore original IOPL setting as we always use 0. */
2381	pCtx->eflags.Bits.u2IOPL = pVCpu->hwaccm.s.vmx.RealMode.eflags.Bits.u2IOPL;
2382
2383	/* Force a TR resync every time in case we switch modes. */
2384	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_GUEST_TR;
2385	}
2386	else
2387	{
2388	/* In real mode we have a fake TSS, so only sync it back when it's supposed to be valid. */
2389	VMX_READ_SELREG(TR, tr);
2390	}
2391
2392	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
2393	/*
2394	* Save the possibly changed MSRs that we automatically restore and save during a world switch.
2395	*/
2396	for (unsigned i = 0; i < pVCpu->hwaccm.s.vmx.cCachedMSRs; i++)
2397	{
2398	PVMXMSR pMsr = (PVMXMSR)pVCpu->hwaccm.s.vmx.pGuestMSR;
2399	pMsr += i;
2400
2401	switch (pMsr->u32IndexMSR)
2402	{
2403	case MSR_K8_LSTAR:
2404	pCtx->msrLSTAR = pMsr->u64Value;
2405	break;
2406	case MSR_K6_STAR:
2407	pCtx->msrSTAR = pMsr->u64Value;
2408	break;
2409	case MSR_K8_SF_MASK:
2410	pCtx->msrSFMASK = pMsr->u64Value;
2411	break;
2412	/* The KERNEL_GS_BASE MSR doesn't work reliably with auto load/store. See @bugref{6208} */
2413	#if 0
2414	case MSR_K8_KERNEL_GS_BASE:
2415	pCtx->msrKERNELGSBASE = pMsr->u64Value;
2416	break;
2417	#endif
2418	case MSR_K8_TSC_AUX:
2419	CPUMSetGuestMsr(pVCpu, MSR_K8_TSC_AUX, pMsr->u64Value);
2420	break;
2421	#if 0
2422	case MSR_K6_EFER:
2423	/* EFER can't be changed without causing a VM-exit. */
2424	/* Assert(pCtx->msrEFER == pMsr->u64Value); */
2425	break;
2426	#endif
2427	default:
2428	AssertFailed();
2429	return VERR_HM_UNEXPECTED_LD_ST_MSR;
2430	}
2431	}
2432	#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
2433	return VINF_SUCCESS;
2434	}
2435
2436
2437	/**
2438	* Dummy placeholder for TLB flush handling before VM-entry. Used in the case
2439	* where neither EPT nor VPID is supported by the CPU.
2440	*
2441	* @param pVM Pointer to the VM.
2442	* @param pVCpu Pointer to the VMCPU.
2443	*/
2444	static DECLCALLBACK(void) hmR0VmxSetupTLBDummy(PVM pVM, PVMCPU pVCpu)
2445	{
2446	NOREF(pVM);
2447	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
2448	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
2449	pVCpu->hwaccm.s.TlbShootdown.cPages = 0;
2450	return;
2451	}
2452
2453
2454	/**
2455	* Setup the tagged TLB for EPT+VPID.
2456	*
2457	* @param pVM Pointer to the VM.
2458	* @param pVCpu Pointer to the VMCPU.
2459	*/
2460	static DECLCALLBACK(void) hmR0VmxSetupTLBBoth(PVM pVM, PVMCPU pVCpu)
2461	{
2462	PHMGLOBLCPUINFO pCpu;
2463
2464	Assert(pVM->hwaccm.s.fNestedPaging && pVM->hwaccm.s.vmx.fVPID);
2465
2466	pCpu = HWACCMR0GetCurrentCpu();
2467
2468	/*
2469	* Force a TLB flush for the first world switch if the current CPU differs from the one we ran on last
2470	* This can happen both for start & resume due to long jumps back to ring-3.
2471	* If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
2472	* or the host Cpu is online after a suspend/resume, so we cannot reuse the current ASID anymore.
2473	*/
2474	bool fNewASID = false;
2475	if ( pVCpu->hwaccm.s.idLastCpu != pCpu->idCpu
2476	\|\| pVCpu->hwaccm.s.cTLBFlushes != pCpu->cTLBFlushes)
2477	{
2478	pVCpu->hwaccm.s.fForceTLBFlush = true;
2479	fNewASID = true;
2480	}
2481
2482	/*
2483	* Check for explicit TLB shootdowns.
2484	*/
2485	if (VMCPU_FF_TESTANDCLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2486	pVCpu->hwaccm.s.fForceTLBFlush = true;
2487
2488	pVCpu->hwaccm.s.idLastCpu = pCpu->idCpu;
2489
2490	if (pVCpu->hwaccm.s.fForceTLBFlush)
2491	{
2492	if (fNewASID)
2493	{
2494	++pCpu->uCurrentASID;
2495	if (pCpu->uCurrentASID >= pVM->hwaccm.s.uMaxASID)
2496	{
2497	pCpu->uCurrentASID = 1; /* start at 1; host uses 0 */
2498	pCpu->cTLBFlushes++;
2499	pCpu->fFlushASIDBeforeUse = true;
2500	}
2501
2502	pVCpu->hwaccm.s.uCurrentASID = pCpu->uCurrentASID;
2503	if (pCpu->fFlushASIDBeforeUse)
2504	{
2505	hmR0VmxFlushVPID(pVM, pVCpu, pVM->hwaccm.s.vmx.enmFlushVPID, 0 /* GCPtr */);
2506	#ifdef VBOX_WITH_STATISTICS
2507	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatFlushASID);
2508	#endif
2509	}
2510	}
2511	else
2512	{
2513	if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVVPID_CAPS_SINGLE_CONTEXT)
2514	hmR0VmxFlushVPID(pVM, pVCpu, VMX_FLUSH_VPID_SINGLE_CONTEXT, 0 /* GCPtr */);
2515	else
2516	hmR0VmxFlushEPT(pVM, pVCpu, pVM->hwaccm.s.vmx.enmFlushEPT);
2517
2518	#ifdef VBOX_WITH_STATISTICS
2519	/*
2520	* This is not terribly accurate (i.e. we don't have any StatFlushEPT counter). We currently count these
2521	* as ASID flushes too, better than including them under StatFlushTLBWorldSwitch.
2522	*/
2523	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatFlushASID);
2524	#endif
2525	}
2526
2527	pVCpu->hwaccm.s.cTLBFlushes = pCpu->cTLBFlushes;
2528	pVCpu->hwaccm.s.fForceTLBFlush = false;
2529	}
2530	else
2531	{
2532	AssertMsg(pVCpu->hwaccm.s.uCurrentASID && pCpu->uCurrentASID,
2533	("hwaccm->uCurrentASID=%lu hwaccm->cTLBFlushes=%lu cpu->uCurrentASID=%lu cpu->cTLBFlushes=%lu\n",
2534	pVCpu->hwaccm.s.uCurrentASID, pVCpu->hwaccm.s.cTLBFlushes,
2535	pCpu->uCurrentASID, pCpu->cTLBFlushes));
2536
2537	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
2538	* not be executed. See hwaccmQueueInvlPage() where it is commented
2539	* out. Support individual entry flushing someday. */
2540	if (VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
2541	{
2542	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatTlbShootdown);
2543
2544	/*
2545	* Flush individual guest entries using VPID from the TLB or as little as possible with EPT
2546	* as supported by the CPU.
2547	*/
2548	if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVVPID_CAPS_INDIV_ADDR)
2549	{
2550	for (unsigned i = 0; i < pVCpu->hwaccm.s.TlbShootdown.cPages; i++)
2551	hmR0VmxFlushVPID(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, pVCpu->hwaccm.s.TlbShootdown.aPages[i]);
2552	}
2553	else
2554	hmR0VmxFlushEPT(pVM, pVCpu, pVM->hwaccm.s.vmx.enmFlushEPT);
2555	}
2556	else
2557	{
2558	#ifdef VBOX_WITH_STATISTICS
2559	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatNoFlushTLBWorldSwitch);
2560	#endif
2561	}
2562	}
2563	pVCpu->hwaccm.s.TlbShootdown.cPages = 0;
2564	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
2565
2566	AssertMsg(pVCpu->hwaccm.s.cTLBFlushes == pCpu->cTLBFlushes,
2567	("Flush count mismatch for cpu %d (%x vs %x)\n", pCpu->idCpu, pVCpu->hwaccm.s.cTLBFlushes, pCpu->cTLBFlushes));
2568	AssertMsg(pCpu->uCurrentASID >= 1 && pCpu->uCurrentASID < pVM->hwaccm.s.uMaxASID,
2569	("cpu%d uCurrentASID = %x\n", pCpu->idCpu, pCpu->uCurrentASID));
2570	AssertMsg(pVCpu->hwaccm.s.uCurrentASID >= 1 && pVCpu->hwaccm.s.uCurrentASID < pVM->hwaccm.s.uMaxASID,
2571	("cpu%d VM uCurrentASID = %x\n", pCpu->idCpu, pVCpu->hwaccm.s.uCurrentASID));
2572
2573	/* Update VMCS with the VPID. */
2574	int rc = VMXWriteVMCS(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hwaccm.s.uCurrentASID);
2575	AssertRC(rc);
2576	}
2577
2578
2579	/**
2580	* Setup the tagged TLB for EPT only.
2581	*
2582	* @returns VBox status code.
2583	* @param pVM Pointer to the VM.
2584	* @param pVCpu Pointer to the VMCPU.
2585	*/
2586	static DECLCALLBACK(void) hmR0VmxSetupTLBEPT(PVM pVM, PVMCPU pVCpu)
2587	{
2588	PHMGLOBLCPUINFO pCpu;
2589
2590	Assert(pVM->hwaccm.s.fNestedPaging);
2591	Assert(!pVM->hwaccm.s.vmx.fVPID);
2592
2593	pCpu = HWACCMR0GetCurrentCpu();
2594
2595	/*
2596	* Force a TLB flush for the first world switch if the current CPU differs from the one we ran on last
2597	* This can happen both for start & resume due to long jumps back to ring-3.
2598	* A change in the TLB flush count implies the host Cpu is online after a suspend/resume.
2599	*/
2600	if ( pVCpu->hwaccm.s.idLastCpu != pCpu->idCpu
2601	\|\| pVCpu->hwaccm.s.cTLBFlushes != pCpu->cTLBFlushes)
2602	{
2603	pVCpu->hwaccm.s.fForceTLBFlush = true;
2604	}
2605
2606	/*
2607	* Check for explicit TLB shootdown flushes.
2608	*/
2609	if (VMCPU_FF_TESTANDCLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2610	pVCpu->hwaccm.s.fForceTLBFlush = true;
2611
2612	pVCpu->hwaccm.s.idLastCpu = pCpu->idCpu;
2613	pVCpu->hwaccm.s.cTLBFlushes = pCpu->cTLBFlushes;
2614
2615	if (pVCpu->hwaccm.s.fForceTLBFlush)
2616	hmR0VmxFlushEPT(pVM, pVCpu, pVM->hwaccm.s.vmx.enmFlushEPT);
2617	else
2618	{
2619	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
2620	* not be executed. See hwaccmQueueInvlPage() where it is commented
2621	* out. Support individual entry flushing someday. */
2622	if (VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
2623	{
2624	/*
2625	* We cannot flush individual entries without VPID support. Flush using EPT.
2626	*/
2627	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatTlbShootdown);
2628	hmR0VmxFlushEPT(pVM, pVCpu, pVM->hwaccm.s.vmx.enmFlushEPT);
2629	}
2630	}
2631	pVCpu->hwaccm.s.TlbShootdown.cPages= 0;
2632	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
2633
2634	#ifdef VBOX_WITH_STATISTICS
2635	if (pVCpu->hwaccm.s.fForceTLBFlush)
2636	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatFlushTLBWorldSwitch);
2637	else
2638	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatNoFlushTLBWorldSwitch);
2639	#endif
2640	}
2641
2642
2643	/**
2644	* Setup the tagged TLB for VPID.
2645	*
2646	* @returns VBox status code.
2647	* @param pVM Pointer to the VM.
2648	* @param pVCpu Pointer to the VMCPU.
2649	*/
2650	static DECLCALLBACK(void) hmR0VmxSetupTLBVPID(PVM pVM, PVMCPU pVCpu)
2651	{
2652	PHMGLOBLCPUINFO pCpu;
2653
2654	Assert(pVM->hwaccm.s.vmx.fVPID);
2655	Assert(!pVM->hwaccm.s.fNestedPaging);
2656
2657	pCpu = HWACCMR0GetCurrentCpu();
2658
2659	/*
2660	* Force a TLB flush for the first world switch if the current CPU differs from the one we ran on last
2661	* This can happen both for start & resume due to long jumps back to ring-3.
2662	* If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
2663	* or the host Cpu is online after a suspend/resume, so we cannot reuse the current ASID anymore.
2664	*/
2665	if ( pVCpu->hwaccm.s.idLastCpu != pCpu->idCpu
2666	\|\| pVCpu->hwaccm.s.cTLBFlushes != pCpu->cTLBFlushes)
2667	{
2668	/* Force a TLB flush on VM entry. */
2669	pVCpu->hwaccm.s.fForceTLBFlush = true;
2670	}
2671
2672	/*
2673	* Check for explicit TLB shootdown flushes.
2674	*/
2675	if (VMCPU_FF_TESTANDCLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2676	pVCpu->hwaccm.s.fForceTLBFlush = true;
2677
2678	pVCpu->hwaccm.s.idLastCpu = pCpu->idCpu;
2679
2680	if (pVCpu->hwaccm.s.fForceTLBFlush)
2681	{
2682	++pCpu->uCurrentASID;
2683	if (pCpu->uCurrentASID >= pVM->hwaccm.s.uMaxASID)
2684	{
2685	pCpu->uCurrentASID = 1; /* start at 1; host uses 0 */
2686	pCpu->cTLBFlushes++;
2687	pCpu->fFlushASIDBeforeUse = true;
2688	}
2689	else
2690	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatFlushASID);
2691
2692	pVCpu->hwaccm.s.fForceTLBFlush = false;
2693	pVCpu->hwaccm.s.cTLBFlushes = pCpu->cTLBFlushes;
2694	pVCpu->hwaccm.s.uCurrentASID = pCpu->uCurrentASID;
2695	if (pCpu->fFlushASIDBeforeUse)
2696	hmR0VmxFlushVPID(pVM, pVCpu, pVM->hwaccm.s.vmx.enmFlushVPID, 0 /* GCPtr */);
2697	}
2698	else
2699	{
2700	AssertMsg(pVCpu->hwaccm.s.uCurrentASID && pCpu->uCurrentASID,
2701	("hwaccm->uCurrentASID=%lu hwaccm->cTLBFlushes=%lu cpu->uCurrentASID=%lu cpu->cTLBFlushes=%lu\n",
2702	pVCpu->hwaccm.s.uCurrentASID, pVCpu->hwaccm.s.cTLBFlushes,
2703	pCpu->uCurrentASID, pCpu->cTLBFlushes));
2704
2705	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
2706	* not be executed. See hwaccmQueueInvlPage() where it is commented
2707	* out. Support individual entry flushing someday. */
2708	if (VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
2709	{
2710	/*
2711	* Flush individual guest entries using VPID from the TLB or as little as possible with EPT
2712	* as supported by the CPU.
2713	*/
2714	if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVVPID_CAPS_INDIV_ADDR)
2715	{
2716	for (unsigned i = 0; i < pVCpu->hwaccm.s.TlbShootdown.cPages; i++)
2717	hmR0VmxFlushVPID(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, pVCpu->hwaccm.s.TlbShootdown.aPages[i]);
2718	}
2719	else
2720	hmR0VmxFlushVPID(pVM, pVCpu, pVM->hwaccm.s.vmx.enmFlushVPID, 0 /* GCPtr */);
2721	}
2722	}
2723	pVCpu->hwaccm.s.TlbShootdown.cPages = 0;
2724	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
2725
2726	AssertMsg(pVCpu->hwaccm.s.cTLBFlushes == pCpu->cTLBFlushes,
2727	("Flush count mismatch for cpu %d (%x vs %x)\n", pCpu->idCpu, pVCpu->hwaccm.s.cTLBFlushes, pCpu->cTLBFlushes));
2728	AssertMsg(pCpu->uCurrentASID >= 1 && pCpu->uCurrentASID < pVM->hwaccm.s.uMaxASID,
2729	("cpu%d uCurrentASID = %x\n", pCpu->idCpu, pCpu->uCurrentASID));
2730	AssertMsg(pVCpu->hwaccm.s.uCurrentASID >= 1 && pVCpu->hwaccm.s.uCurrentASID < pVM->hwaccm.s.uMaxASID,
2731	("cpu%d VM uCurrentASID = %x\n", pCpu->idCpu, pVCpu->hwaccm.s.uCurrentASID));
2732
2733	int rc = VMXWriteVMCS(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hwaccm.s.uCurrentASID);
2734	AssertRC(rc);
2735
2736	# ifdef VBOX_WITH_STATISTICS
2737	if (pVCpu->hwaccm.s.fForceTLBFlush)
2738	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatFlushTLBWorldSwitch);
2739	else
2740	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatNoFlushTLBWorldSwitch);
2741	# endif
2742	}
2743
2744
2745	/**
2746	* Runs guest code in a VT-x VM.
2747	*
2748	* @returns VBox status code.
2749	* @param pVM Pointer to the VM.
2750	* @param pVCpu Pointer to the VMCPU.
2751	* @param pCtx Pointer to the guest CPU context.
2752	*/
2753	VMMR0DECL(int) VMXR0RunGuestCode(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
2754	{
2755	STAM_PROFILE_ADV_START(&pVCpu->hwaccm.s.StatEntry, x);
2756	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hwaccm.s.StatExit1);
2757	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hwaccm.s.StatExit2);
2758
2759	VBOXSTRICTRC rc = VINF_SUCCESS;
2760	int rc2;
2761	RTGCUINTREG val;
2762	RTGCUINTREG exitReason = (RTGCUINTREG)VMX_EXIT_INVALID;
2763	RTGCUINTREG instrError, cbInstr;
2764	RTGCUINTPTR exitQualification = 0;
2765	RTGCUINTPTR intInfo = 0; /* shut up buggy gcc 4 */
2766	RTGCUINTPTR errCode, instrInfo;
2767	bool fSetupTPRCaching = false;
2768	uint64_t u64OldLSTAR = 0;
2769	uint8_t u8LastTPR = 0;
2770	RTCCUINTREG uOldEFlags = ~(RTCCUINTREG)0;
2771	unsigned cResume = 0;
2772	#ifdef VBOX_STRICT
2773	RTCPUID idCpuCheck;
2774	bool fWasInLongMode = false;
2775	#endif
2776	#ifdef VBOX_HIGH_RES_TIMERS_HACK_IN_RING0
2777	uint64_t u64LastTime = RTTimeMilliTS();
2778	#endif
2779
2780	Assert(!(pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
2781	\|\| (pVCpu->hwaccm.s.vmx.pbVAPIC && pVM->hwaccm.s.vmx.pAPIC));
2782
2783	/*
2784	* Check if we need to use TPR shadowing.
2785	*/
2786	if ( CPUMIsGuestInLongModeEx(pCtx)
2787	\|\| ( (( pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
2788	\|\| pVM->hwaccm.s.fTRPPatchingAllowed)
2789	&& pVM->hwaccm.s.fHasIoApic)
2790	)
2791	{
2792	fSetupTPRCaching = true;
2793	}
2794
2795	Log2(("\nE"));
2796
2797	#ifdef VBOX_STRICT
2798	{
2799	RTCCUINTREG val2;
2800
2801	rc2 = VMXReadVMCS(VMX_VMCS_CTRL_PIN_EXEC_CONTROLS, &val2);
2802	AssertRC(rc2);
2803	Log2(("VMX_VMCS_CTRL_PIN_EXEC_CONTROLS = %08x\n", val2));
2804
2805	/* allowed zero */
2806	if ((val2 & pVM->hwaccm.s.vmx.msr.vmx_pin_ctls.n.disallowed0) != pVM->hwaccm.s.vmx.msr.vmx_pin_ctls.n.disallowed0)
2807	Log(("Invalid VMX_VMCS_CTRL_PIN_EXEC_CONTROLS: zero\n"));
2808
2809	/* allowed one */
2810	if ((val2 & ~pVM->hwaccm.s.vmx.msr.vmx_pin_ctls.n.allowed1) != 0)
2811	Log(("Invalid VMX_VMCS_CTRL_PIN_EXEC_CONTROLS: one\n"));
2812
2813	rc2 = VMXReadVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, &val2);
2814	AssertRC(rc2);
2815	Log2(("VMX_VMCS_CTRL_PROC_EXEC_CONTROLS = %08x\n", val2));
2816
2817	/*
2818	* Must be set according to the MSR, but can be cleared if nested paging is used.
2819	*/
2820	if (pVM->hwaccm.s.fNestedPaging)
2821	{
2822	val2 \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INVLPG_EXIT
2823	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
2824	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT;
2825	}
2826
2827	/* allowed zero */
2828	if ((val2 & pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.disallowed0) != pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.disallowed0)
2829	Log(("Invalid VMX_VMCS_CTRL_PROC_EXEC_CONTROLS: zero\n"));
2830
2831	/* allowed one */
2832	if ((val2 & ~pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1) != 0)
2833	Log(("Invalid VMX_VMCS_CTRL_PROC_EXEC_CONTROLS: one\n"));
2834
2835	rc2 = VMXReadVMCS(VMX_VMCS_CTRL_ENTRY_CONTROLS, &val2);
2836	AssertRC(rc2);
2837	Log2(("VMX_VMCS_CTRL_ENTRY_CONTROLS = %08x\n", val2));
2838
2839	/* allowed zero */
2840	if ((val2 & pVM->hwaccm.s.vmx.msr.vmx_entry.n.disallowed0) != pVM->hwaccm.s.vmx.msr.vmx_entry.n.disallowed0)
2841	Log(("Invalid VMX_VMCS_CTRL_ENTRY_CONTROLS: zero\n"));
2842
2843	/* allowed one */
2844	if ((val2 & ~pVM->hwaccm.s.vmx.msr.vmx_entry.n.allowed1) != 0)
2845	Log(("Invalid VMX_VMCS_CTRL_ENTRY_CONTROLS: one\n"));
2846
2847	rc2 = VMXReadVMCS(VMX_VMCS_CTRL_EXIT_CONTROLS, &val2);
2848	AssertRC(rc2);
2849	Log2(("VMX_VMCS_CTRL_EXIT_CONTROLS = %08x\n", val2));
2850
2851	/* allowed zero */
2852	if ((val2 & pVM->hwaccm.s.vmx.msr.vmx_exit.n.disallowed0) != pVM->hwaccm.s.vmx.msr.vmx_exit.n.disallowed0)
2853	Log(("Invalid VMX_VMCS_CTRL_EXIT_CONTROLS: zero\n"));
2854
2855	/* allowed one */
2856	if ((val2 & ~pVM->hwaccm.s.vmx.msr.vmx_exit.n.allowed1) != 0)
2857	Log(("Invalid VMX_VMCS_CTRL_EXIT_CONTROLS: one\n"));
2858	}
2859	fWasInLongMode = CPUMIsGuestInLongModeEx(pCtx);
2860	#endif /* VBOX_STRICT */
2861
2862	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
2863	pVCpu->hwaccm.s.vmx.VMCSCache.u64TimeEntry = RTTimeNanoTS();
2864	#endif
2865
2866	/*
2867	* We can jump to this point to resume execution after determining that a VM-exit is innocent.
2868	*/
2869	ResumeExecution:
2870	if (!STAM_REL_PROFILE_ADV_IS_RUNNING(&pVCpu->hwaccm.s.StatEntry))
2871	STAM_REL_PROFILE_ADV_STOP_START(&pVCpu->hwaccm.s.StatExit2, &pVCpu->hwaccm.s.StatEntry, x);
2872	AssertMsg(pVCpu->hwaccm.s.idEnteredCpu == RTMpCpuId(),
2873	("Expected %d, I'm %d; cResume=%d exitReason=%RGv exitQualification=%RGv\n",
2874	(int)pVCpu->hwaccm.s.idEnteredCpu, (int)RTMpCpuId(), cResume, exitReason, exitQualification));
2875	Assert(!HWACCMR0SuspendPending());
2876	/* Not allowed to switch modes without reloading the host state (32->64 switcher)!! */
2877	Assert(fWasInLongMode == CPUMIsGuestInLongModeEx(pCtx));
2878
2879	/*
2880	* Safety precaution; looping for too long here can have a very bad effect on the host.
2881	*/
2882	if (RT_UNLIKELY(++cResume > pVM->hwaccm.s.cMaxResumeLoops))
2883	{
2884	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitMaxResume);
2885	rc = VINF_EM_RAW_INTERRUPT;
2886	goto end;
2887	}
2888
2889	/*
2890	* Check for IRQ inhibition due to instruction fusing (sti, mov ss).
2891	*/
2892	if (VMCPU_FF_ISSET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
2893	{
2894	Log(("VM_FF_INHIBIT_INTERRUPTS at %RGv successor %RGv\n", (RTGCPTR)pCtx->rip, EMGetInhibitInterruptsPC(pVCpu)));
2895	if (pCtx->rip != EMGetInhibitInterruptsPC(pVCpu))
2896	{
2897	/*
2898	* Note: we intentionally don't clear VM_FF_INHIBIT_INTERRUPTS here.
2899	* Before we are able to execute this instruction in raw mode (iret to guest code) an external interrupt might
2900	* force a world switch again. Possibly allowing a guest interrupt to be dispatched in the process. This could
2901	* break the guest. Sounds very unlikely, but such timing sensitive problems are not as rare as you might think.
2902	*/
2903	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
2904	/* Irq inhibition is no longer active; clear the corresponding VMX state. */
2905	rc2 = VMXWriteVMCS(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, 0);
2906	AssertRC(rc2);
2907	}
2908	}
2909	else
2910	{
2911	/* Irq inhibition is no longer active; clear the corresponding VMX state. */
2912	rc2 = VMXWriteVMCS(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, 0);
2913	AssertRC(rc2);
2914	}
2915
2916	#ifdef VBOX_HIGH_RES_TIMERS_HACK_IN_RING0
2917	if (RT_UNLIKELY((cResume & 0xf) == 0))
2918	{
2919	uint64_t u64CurTime = RTTimeMilliTS();
2920
2921	if (RT_UNLIKELY(u64CurTime > u64LastTime))
2922	{
2923	u64LastTime = u64CurTime;
2924	TMTimerPollVoid(pVM, pVCpu);
2925	}
2926	}
2927	#endif
2928
2929	/*
2930	* Check for pending actions that force us to go back to ring-3.
2931	*/
2932	if ( VM_FF_ISPENDING(pVM, VM_FF_HWACCM_TO_R3_MASK \| VM_FF_REQUEST \| VM_FF_PGM_POOL_FLUSH_PENDING \| VM_FF_PDM_DMA)
2933	\|\| VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_HWACCM_TO_R3_MASK \| VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL \| VMCPU_FF_REQUEST))
2934	{
2935	/* Check if a sync operation is pending. */
2936	if (VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
2937	{
2938	rc = PGMSyncCR3(pVCpu, pCtx->cr0, pCtx->cr3, pCtx->cr4, VMCPU_FF_ISSET(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
2939	if (rc != VINF_SUCCESS)
2940	{
2941	AssertRC(VBOXSTRICTRC_VAL(rc));
2942	Log(("Pending pool sync is forcing us back to ring 3; rc=%d\n", VBOXSTRICTRC_VAL(rc)));
2943	goto end;
2944	}
2945	}
2946
2947	#ifdef DEBUG
2948	/* Intercept X86_XCPT_DB if stepping is enabled */
2949	if (!DBGFIsStepping(pVCpu))
2950	#endif
2951	{
2952	if ( VM_FF_ISPENDING(pVM, VM_FF_HWACCM_TO_R3_MASK)
2953	\|\| VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_HWACCM_TO_R3_MASK))
2954	{
2955	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatSwitchToR3);
2956	rc = RT_UNLIKELY(VM_FF_ISPENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
2957	goto end;
2958	}
2959	}
2960
2961	/* Pending request packets might contain actions that need immediate attention, such as pending hardware interrupts. */
2962	if ( VM_FF_ISPENDING(pVM, VM_FF_REQUEST)
2963	\|\| VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_REQUEST))
2964	{
2965	rc = VINF_EM_PENDING_REQUEST;
2966	goto end;
2967	}
2968
2969	/* Check if a pgm pool flush is in progress. */
2970	if (VM_FF_ISPENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
2971	{
2972	rc = VINF_PGM_POOL_FLUSH_PENDING;
2973	goto end;
2974	}
2975
2976	/* Check if DMA work is pending (2nd+ run). */
2977	if (VM_FF_ISPENDING(pVM, VM_FF_PDM_DMA) && cResume > 1)
2978	{
2979	rc = VINF_EM_RAW_TO_R3;
2980	goto end;
2981	}
2982	}
2983
2984	#ifdef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
2985	/*
2986	* Exit to ring-3 preemption/work is pending.
2987	*
2988	* Interrupts are disabled before the call to make sure we don't miss any interrupt
2989	* that would flag preemption (IPI, timer tick, ++). (Would've been nice to do this
2990	* further down, but hmR0VmxCheckPendingInterrupt makes that impossible.)
2991	*
2992	* Note! Interrupts must be disabled done before we check for TLB flushes; TLB
2993	* shootdowns rely on this.
2994	*/
2995	uOldEFlags = ASMIntDisableFlags();
2996	if (RTThreadPreemptIsPending(NIL_RTTHREAD))
2997	{
2998	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitPreemptPending);
2999	rc = VINF_EM_RAW_INTERRUPT;
3000	goto end;
3001	}
3002	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
3003	#endif
3004
3005	/*
3006	* When external interrupts are pending, we should exit the VM when IF is et.
3007	* Note: After VM_FF_INHIBIT_INTERRUPTS check!
3008	*/
3009	rc = hmR0VmxCheckPendingInterrupt(pVM, pVCpu, pCtx);
3010	if (RT_FAILURE(rc))
3011	goto end;
3012
3013	/** @todo check timers?? */
3014
3015	/*
3016	* TPR caching using CR8 is only available in 64-bit mode.
3017	* Note: The 32-bit exception for AMD (X86_CPUID_AMD_FEATURE_ECX_CR8L), but this appears missing in Intel CPUs.
3018	* Note: We can't do this in LoadGuestState() as PDMApicGetTPR can jump back to ring-3 (lock)!! (no longer true) .
3019	*/
3020	/** @todo query and update the TPR only when it could have been changed (mmio
3021	* access & wrsmr (x2apic) */
3022	if (fSetupTPRCaching)
3023	{
3024	/* TPR caching in CR8 */
3025	bool fPending;
3026
3027	rc2 = PDMApicGetTPR(pVCpu, &u8LastTPR, &fPending);
3028	AssertRC(rc2);
3029	/* The TPR can be found at offset 0x80 in the APIC mmio page. */
3030	pVCpu->hwaccm.s.vmx.pbVAPIC[0x80] = u8LastTPR;
3031
3032	/*
3033	* Two options here:
3034	* - external interrupt pending, but masked by the TPR value.
3035	* -> a CR8 update that lower the current TPR value should cause an exit
3036	* - no pending interrupts
3037	* -> We don't need to be explicitely notified. There are enough world switches for detecting pending interrupts.
3038	*/
3039
3040	/* cr8 bits 3-0 correspond to bits 7-4 of the task priority mmio register. */
3041	rc = VMXWriteVMCS(VMX_VMCS_CTRL_TPR_THRESHOLD, (fPending) ? (u8LastTPR >> 4) : 0);
3042	AssertRC(VBOXSTRICTRC_VAL(rc));
3043
3044	if (pVM->hwaccm.s.fTPRPatchingActive)
3045	{
3046	Assert(!CPUMIsGuestInLongModeEx(pCtx));
3047	/* Our patch code uses LSTAR for TPR caching. */
3048	pCtx->msrLSTAR = u8LastTPR;
3049
3050	/** @todo r=ramshankar: we should check for MSR-bitmap support here. */
3051	if (fPending)
3052	{
3053	/* A TPR change could activate a pending interrupt, so catch lstar writes. */
3054	hmR0VmxSetMSRPermission(pVCpu, MSR_K8_LSTAR, true, false);
3055	}
3056	else
3057	{
3058	/*
3059	* No interrupts are pending, so we don't need to be explicitely notified.
3060	* There are enough world switches for detecting pending interrupts.
3061	*/
3062	hmR0VmxSetMSRPermission(pVCpu, MSR_K8_LSTAR, true, true);
3063	}
3064	}
3065	}
3066
3067	#ifdef LOG_ENABLED
3068	if ( pVM->hwaccm.s.fNestedPaging
3069	\|\| pVM->hwaccm.s.vmx.fVPID)
3070	{
3071	PHMGLOBLCPUINFO pCpu = HWACCMR0GetCurrentCpu();
3072	if (pVCpu->hwaccm.s.idLastCpu != pCpu->idCpu)
3073	{
3074	LogFlow(("Force TLB flush due to rescheduling to a different cpu (%d vs %d)\n", pVCpu->hwaccm.s.idLastCpu,
3075	pCpu->idCpu));
3076	}
3077	else if (pVCpu->hwaccm.s.cTLBFlushes != pCpu->cTLBFlushes)
3078	{
3079	LogFlow(("Force TLB flush due to changed TLB flush count (%x vs %x)\n", pVCpu->hwaccm.s.cTLBFlushes,
3080	pCpu->cTLBFlushes));
3081	}
3082	else if (VMCPU_FF_ISSET(pVCpu, VMCPU_FF_TLB_FLUSH))
3083	LogFlow(("Manual TLB flush\n"));
3084	}
3085	#endif
3086	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
3087	PGMRZDynMapFlushAutoSet(pVCpu);
3088	#endif
3089
3090	/*
3091	* NOTE: DO NOT DO ANYTHING AFTER THIS POINT THAT MIGHT JUMP BACK TO RING-3!
3092	* (until the actual world switch)
3093	*/
3094	#ifdef VBOX_STRICT
3095	idCpuCheck = RTMpCpuId();
3096	#endif
3097	#ifdef LOG_ENABLED
3098	VMMR0LogFlushDisable(pVCpu);
3099	#endif
3100
3101	/*
3102	* Save the host state first.
3103	*/
3104	if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_HOST_CONTEXT)
3105	{
3106	rc = VMXR0SaveHostState(pVM, pVCpu);
3107	if (RT_UNLIKELY(rc != VINF_SUCCESS))
3108	{
3109	VMMR0LogFlushEnable(pVCpu);
3110	goto end;
3111	}
3112	}
3113
3114	/*
3115	* Load the guest state.
3116	*/
3117	if (!pVCpu->hwaccm.s.fContextUseFlags)
3118	{
3119	VMXR0LoadMinimalGuestState(pVM, pVCpu, pCtx);
3120	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatLoadMinimal);
3121	}
3122	else
3123	{
3124	rc = VMXR0LoadGuestState(pVM, pVCpu, pCtx);
3125	if (RT_UNLIKELY(rc != VINF_SUCCESS))
3126	{
3127	VMMR0LogFlushEnable(pVCpu);
3128	goto end;
3129	}
3130	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatLoadFull);
3131	}
3132
3133	#ifndef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
3134	/*
3135	* Disable interrupts to make sure a poke will interrupt execution.
3136	* This must be done before we check for TLB flushes; TLB shootdowns rely on this.
3137	*/
3138	uOldEFlags = ASMIntDisableFlags();
3139	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
3140	#endif
3141
3142	/* Non-register state Guest Context */
3143	/** @todo change me according to cpu state */
3144	rc2 = VMXWriteVMCS(VMX_VMCS32_GUEST_ACTIVITY_STATE, VMX_CMS_GUEST_ACTIVITY_ACTIVE);
3145	AssertRC(rc2);
3146
3147	/* Set TLB flush state as checked until we return from the world switch. */
3148	ASMAtomicWriteBool(&pVCpu->hwaccm.s.fCheckedTLBFlush, true);
3149	/* Deal with tagged TLB setup and invalidation. */
3150	pVM->hwaccm.s.vmx.pfnSetupTaggedTLB(pVM, pVCpu);
3151
3152	/*
3153	* Manual save and restore:
3154	* - General purpose registers except RIP, RSP
3155	*
3156	* Trashed:
3157	* - CR2 (we don't care)
3158	* - LDTR (reset to 0)
3159	* - DRx (presumably not changed at all)
3160	* - DR7 (reset to 0x400)
3161	* - EFLAGS (reset to RT_BIT(1); not relevant)
3162	*/
3163
3164	/* All done! Let's start VM execution. */
3165	STAM_PROFILE_ADV_STOP_START(&pVCpu->hwaccm.s.StatEntry, &pVCpu->hwaccm.s.StatInGC, x);
3166	Assert(idCpuCheck == RTMpCpuId());
3167
3168	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
3169	pVCpu->hwaccm.s.vmx.VMCSCache.cResume = cResume;
3170	pVCpu->hwaccm.s.vmx.VMCSCache.u64TimeSwitch = RTTimeNanoTS();
3171	#endif
3172
3173	/*
3174	* Save the current TPR value in the LSTAR MSR so our patches can access it.
3175	*/
3176	if (pVM->hwaccm.s.fTPRPatchingActive)
3177	{
3178	Assert(pVM->hwaccm.s.fTPRPatchingActive);
3179	u64OldLSTAR = ASMRdMsr(MSR_K8_LSTAR);
3180	ASMWrMsr(MSR_K8_LSTAR, u8LastTPR);
3181	}
3182
3183	TMNotifyStartOfExecution(pVCpu);
3184
3185	#ifndef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
3186	/*
3187	* Save the current Host TSC_AUX and write the guest TSC_AUX to the host, so that
3188	* RDTSCPs (that don't cause exits) reads the guest MSR. See @bugref{3324}.
3189	*/
3190	if ( (pVCpu->hwaccm.s.vmx.proc_ctls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
3191	&& !(pVCpu->hwaccm.s.vmx.proc_ctls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT))
3192	{
3193	pVCpu->hwaccm.s.u64HostTSCAux = ASMRdMsr(MSR_K8_TSC_AUX);
3194	uint64_t u64GuestTSCAux = 0;
3195	rc2 = CPUMQueryGuestMsr(pVCpu, MSR_K8_TSC_AUX, &u64GuestTSCAux);
3196	AssertRC(rc2);
3197	ASMWrMsr(MSR_K8_TSC_AUX, u64GuestTSCAux);
3198	}
3199	#endif
3200
3201	#ifdef VBOX_WITH_KERNEL_USING_XMM
3202	rc = hwaccmR0VMXStartVMWrapXMM(pVCpu->hwaccm.s.fResumeVM, pCtx, &pVCpu->hwaccm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hwaccm.s.vmx.pfnStartVM);
3203	#else
3204	rc = pVCpu->hwaccm.s.vmx.pfnStartVM(pVCpu->hwaccm.s.fResumeVM, pCtx, &pVCpu->hwaccm.s.vmx.VMCSCache, pVM, pVCpu);
3205	#endif
3206	ASMAtomicWriteBool(&pVCpu->hwaccm.s.fCheckedTLBFlush, false);
3207	ASMAtomicIncU32(&pVCpu->hwaccm.s.cWorldSwitchExits);
3208
3209	/* Possibly the last TSC value seen by the guest (too high) (only when we're in TSC offset mode). */
3210	if (!(pVCpu->hwaccm.s.vmx.proc_ctls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT))
3211	{
3212	#ifndef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
3213	/* Restore host's TSC_AUX. */
3214	if (pVCpu->hwaccm.s.vmx.proc_ctls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
3215	ASMWrMsr(MSR_K8_TSC_AUX, pVCpu->hwaccm.s.u64HostTSCAux);
3216	#endif
3217
3218	TMCpuTickSetLastSeen(pVCpu,
3219	ASMReadTSC() + pVCpu->hwaccm.s.vmx.u64TSCOffset - 0x400 /* guestimate of world switch overhead in clock ticks */);
3220	}
3221
3222	TMNotifyEndOfExecution(pVCpu);
3223	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED);
3224	Assert(!(ASMGetFlags() & X86_EFL_IF));
3225
3226	/*
3227	* Restore the host LSTAR MSR if the guest could have changed it.
3228	*/
3229	if (pVM->hwaccm.s.fTPRPatchingActive)
3230	{
3231	Assert(pVM->hwaccm.s.fTPRPatchingActive);
3232	pVCpu->hwaccm.s.vmx.pbVAPIC[0x80] = pCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR);
3233	ASMWrMsr(MSR_K8_LSTAR, u64OldLSTAR);
3234	}
3235
3236	STAM_PROFILE_ADV_STOP_START(&pVCpu->hwaccm.s.StatInGC, &pVCpu->hwaccm.s.StatExit1, x);
3237	ASMSetFlags(uOldEFlags);
3238	#ifdef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
3239	uOldEFlags = ~(RTCCUINTREG)0;
3240	#endif
3241
3242	AssertMsg(!pVCpu->hwaccm.s.vmx.VMCSCache.Write.cValidEntries, ("pVCpu->hwaccm.s.vmx.VMCSCache.Write.cValidEntries=%d\n",
3243	pVCpu->hwaccm.s.vmx.VMCSCache.Write.cValidEntries));
3244
3245	/* In case we execute a goto ResumeExecution later on. */
3246	pVCpu->hwaccm.s.fResumeVM = true;
3247	pVCpu->hwaccm.s.fForceTLBFlush = false;
3248
3249	/*
3250	* !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
3251	* IMPORTANT: WE CAN'T DO ANY LOGGING OR OPERATIONS THAT CAN DO A LONGJMP BACK TO RING 3 BEFORE WE'VE SYNCED BACK (MOST OF) THE GUEST STATE
3252	* !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
3253	*/
3254
3255	if (RT_UNLIKELY(rc != VINF_SUCCESS))
3256	{
3257	hmR0VmxReportWorldSwitchError(pVM, pVCpu, rc, pCtx);
3258	VMMR0LogFlushEnable(pVCpu);
3259	goto end;
3260	}
3261
3262	/* Success. Query the guest state and figure out what has happened. */
3263
3264	/* Investigate why there was a VM-exit. */
3265	rc2 = VMXReadCachedVMCS(VMX_VMCS32_RO_EXIT_REASON, &exitReason);
3266	STAM_COUNTER_INC(&pVCpu->hwaccm.s.paStatExitReasonR0[exitReason & MASK_EXITREASON_STAT]);
3267
3268	exitReason &= 0xffff; /* bit 0-15 contain the exit code. */
3269	rc2 \|= VMXReadCachedVMCS(VMX_VMCS32_RO_VM_INSTR_ERROR, &instrError);
3270	rc2 \|= VMXReadCachedVMCS(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &cbInstr);
3271	rc2 \|= VMXReadCachedVMCS(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &intInfo);
3272	/* might not be valid; depends on VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID. */
3273	rc2 \|= VMXReadCachedVMCS(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERRCODE, &errCode);
3274	rc2 \|= VMXReadCachedVMCS(VMX_VMCS32_RO_EXIT_INSTR_INFO, &instrInfo);
3275	rc2 \|= VMXReadCachedVMCS(VMX_VMCS_RO_EXIT_QUALIFICATION, &exitQualification);
3276	AssertRC(rc2);
3277
3278	/*
3279	* Sync back the guest state.
3280	*/
3281	rc2 = VMXR0SaveGuestState(pVM, pVCpu, pCtx);
3282	AssertRC(rc2);
3283
3284	/* Note! NOW IT'S SAFE FOR LOGGING! */
3285	VMMR0LogFlushEnable(pVCpu);
3286	Log2(("Raw exit reason %08x\n", exitReason));
3287	#if ARCH_BITS == 64 /* for the time being */
3288	VBOXVMM_R0_HMVMX_VMEXIT(pVCpu, pCtx, exitReason);
3289	#endif
3290
3291	/*
3292	* Check if an injected event was interrupted prematurely.
3293	*/
3294	rc2 = VMXReadCachedVMCS(VMX_VMCS32_RO_IDT_INFO, &val);
3295	AssertRC(rc2);
3296	pVCpu->hwaccm.s.Event.intInfo = VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(val);
3297	if ( VMX_EXIT_INTERRUPTION_INFO_VALID(pVCpu->hwaccm.s.Event.intInfo)
3298	/* Ignore 'int xx' as they'll be restarted anyway. */
3299	&& VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hwaccm.s.Event.intInfo) != VMX_EXIT_INTERRUPTION_INFO_TYPE_SW
3300	/* Ignore software exceptions (such as int3) as they'll reoccur when we restart the instruction anyway. */
3301	&& VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hwaccm.s.Event.intInfo) != VMX_EXIT_INTERRUPTION_INFO_TYPE_SWEXCPT)
3302	{
3303	Assert(!pVCpu->hwaccm.s.Event.fPending);
3304	pVCpu->hwaccm.s.Event.fPending = true;
3305	/* Error code present? */
3306	if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(pVCpu->hwaccm.s.Event.intInfo))
3307	{
3308	rc2 = VMXReadCachedVMCS(VMX_VMCS32_RO_IDT_ERRCODE, &val);
3309	AssertRC(rc2);
3310	pVCpu->hwaccm.s.Event.errCode = val;
3311	Log(("Pending inject %RX64 at %RGv exit=%08x intInfo=%08x exitQualification=%RGv pending error=%RX64\n",
3312	pVCpu->hwaccm.s.Event.intInfo, (RTGCPTR)pCtx->rip, exitReason, intInfo, exitQualification, val));
3313	}
3314	else
3315	{
3316	Log(("Pending inject %RX64 at %RGv exit=%08x intInfo=%08x exitQualification=%RGv\n", pVCpu->hwaccm.s.Event.intInfo,
3317	(RTGCPTR)pCtx->rip, exitReason, intInfo, exitQualification));
3318	pVCpu->hwaccm.s.Event.errCode = 0;
3319	}
3320	}
3321	#ifdef VBOX_STRICT
3322	else if ( VMX_EXIT_INTERRUPTION_INFO_VALID(pVCpu->hwaccm.s.Event.intInfo)
3323	/* Ignore software exceptions (such as int3) as they're reoccur when we restart the instruction anyway. */
3324	&& VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hwaccm.s.Event.intInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_SWEXCPT)
3325	{
3326	Log(("Ignore pending inject %RX64 at %RGv exit=%08x intInfo=%08x exitQualification=%RGv\n",
3327	pVCpu->hwaccm.s.Event.intInfo, (RTGCPTR)pCtx->rip, exitReason, intInfo, exitQualification));
3328	}
3329
3330	if (exitReason == VMX_EXIT_ERR_INVALID_GUEST_STATE)
3331	HWACCMDumpRegs(pVM, pVCpu, pCtx);
3332	#endif
3333
3334	Log2(("E%d: New EIP=%x:%RGv\n", (uint32_t)exitReason, pCtx->cs.Sel, (RTGCPTR)pCtx->rip));
3335	Log2(("Exit reason %d, exitQualification %RGv\n", (uint32_t)exitReason, exitQualification));
3336	Log2(("instrInfo=%d instrError=%d instr length=%d\n", (uint32_t)instrInfo, (uint32_t)instrError, (uint32_t)cbInstr));
3337	Log2(("Interruption error code %d\n", (uint32_t)errCode));
3338	Log2(("IntInfo = %08x\n", (uint32_t)intInfo));
3339
3340	/*
3341	* Sync back the TPR if it was changed.
3342	*/
3343	if ( fSetupTPRCaching
3344	&& u8LastTPR != pVCpu->hwaccm.s.vmx.pbVAPIC[0x80])
3345	{
3346	rc2 = PDMApicSetTPR(pVCpu, pVCpu->hwaccm.s.vmx.pbVAPIC[0x80]);
3347	AssertRC(rc2);
3348	}
3349
3350	#ifdef DBGFTRACE_ENABLED /** @todo DTrace later. */
3351	RTTraceBufAddMsgF(pVM->CTX_SUFF(hTraceBuf), "vmexit %08x %016RX64 at %04:%08RX64 %RX64",
3352	exitReason, (uint64_t)exitQualification, pCtx->cs.Sel, pCtx->rip, (uint64_t)intInfo);
3353	#endif
3354	STAM_PROFILE_ADV_STOP_START(&pVCpu->hwaccm.s.StatExit1, &pVCpu->hwaccm.s.StatExit2, x);
3355
3356	/* Some cases don't need a complete resync of the guest CPU state; handle them here. */
3357	Assert(rc == VINF_SUCCESS); /* might consider VERR_IPE_UNINITIALIZED_STATUS here later... */
3358	switch (exitReason)
3359	{
3360	case VMX_EXIT_EXCEPTION: /* 0 Exception or non-maskable interrupt (NMI). */
3361	case VMX_EXIT_EXTERNAL_IRQ: /* 1 External interrupt. */
3362	{
3363	uint32_t vector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(intInfo);
3364
3365	if (!VMX_EXIT_INTERRUPTION_INFO_VALID(intInfo))
3366	{
3367	Assert(exitReason == VMX_EXIT_EXTERNAL_IRQ);
3368	#if 0 //def VBOX_WITH_VMMR0_DISABLE_PREEMPTION
3369	if ( RTThreadPreemptIsPendingTrusty()
3370	&& !RTThreadPreemptIsPending(NIL_RTTHREAD))
3371	goto ResumeExecution;
3372	#endif
3373	/* External interrupt; leave to allow it to be dispatched again. */
3374	rc = VINF_EM_RAW_INTERRUPT;
3375	break;
3376	}
3377	STAM_PROFILE_ADV_START(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3378	switch (VMX_EXIT_INTERRUPTION_INFO_TYPE(intInfo))
3379	{
3380	case VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI: /* Non-maskable interrupt. */
3381	/* External interrupt; leave to allow it to be dispatched again. */
3382	rc = VINF_EM_RAW_INTERRUPT;
3383	break;
3384
3385	case VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT: /* External hardware interrupt. */
3386	AssertFailed(); /* can't come here; fails the first check. */
3387	break;
3388
3389	case VMX_EXIT_INTERRUPTION_INFO_TYPE_DBEXCPT: /* Unknown why we get this type for #DB */
3390	case VMX_EXIT_INTERRUPTION_INFO_TYPE_SWEXCPT: /* Software exception. (#BP or #OF) */
3391	Assert(vector == 1 \|\| vector == 3 \|\| vector == 4);
3392	/* no break */
3393	case VMX_EXIT_INTERRUPTION_INFO_TYPE_HWEXCPT: /* Hardware exception. */
3394	Log2(("Hardware/software interrupt %d\n", vector));
3395	switch (vector)
3396	{
3397	case X86_XCPT_NM:
3398	{
3399	Log(("#NM fault at %RGv error code %x\n", (RTGCPTR)pCtx->rip, errCode));
3400
3401	/** @todo don't intercept #NM exceptions anymore when we've activated the guest FPU state. */
3402	/* If we sync the FPU/XMM state on-demand, then we can continue execution as if nothing has happened. */
3403	rc = CPUMR0LoadGuestFPU(pVM, pVCpu, pCtx);
3404	if (rc == VINF_SUCCESS)
3405	{
3406	Assert(CPUMIsGuestFPUStateActive(pVCpu));
3407
3408	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitShadowNM);
3409
3410	/* Continue execution. */
3411	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_GUEST_CR0;
3412
3413	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3414	goto ResumeExecution;
3415	}
3416
3417	Log(("Forward #NM fault to the guest\n"));
3418	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestNM);
3419	rc2 = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo),
3420	cbInstr, 0);
3421	AssertRC(rc2);
3422	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3423	goto ResumeExecution;
3424	}
3425
3426	case X86_XCPT_PF: /* Page fault */
3427	{
3428	#ifdef VBOX_ALWAYS_TRAP_PF
3429	if (pVM->hwaccm.s.fNestedPaging)
3430	{
3431	/*
3432	* A genuine pagefault. Forward the trap to the guest by injecting the exception and resuming execution.
3433	*/
3434	Log(("Guest page fault at %RGv cr2=%RGv error code %RGv rsp=%RGv\n", (RTGCPTR)pCtx->rip, exitQualification,
3435	errCode, (RTGCPTR)pCtx->rsp));
3436
3437	Assert(CPUMIsGuestInPagedProtectedModeEx(pCtx));
3438
3439	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestPF);
3440
3441	/* Now we must update CR2. */
3442	pCtx->cr2 = exitQualification;
3443	rc2 = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo),
3444	cbInstr, errCode);
3445	AssertRC(rc2);
3446
3447	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3448	goto ResumeExecution;
3449	}
3450	#else
3451	Assert(!pVM->hwaccm.s.fNestedPaging);
3452	#endif
3453
3454	#ifdef VBOX_HWACCM_WITH_GUEST_PATCHING
3455	/* Shortcut for APIC TPR reads and writes; 32 bits guests only */
3456	if ( pVM->hwaccm.s.fTRPPatchingAllowed
3457	&& pVM->hwaccm.s.pGuestPatchMem
3458	&& (exitQualification & 0xfff) == 0x080
3459	&& !(errCode & X86_TRAP_PF_P) /* not present */
3460	&& CPUMGetGuestCPL(pVCpu) == 0
3461	&& !CPUMIsGuestInLongModeEx(pCtx)
3462	&& pVM->hwaccm.s.cPatches < RT_ELEMENTS(pVM->hwaccm.s.aPatches))
3463	{
3464	RTGCPHYS GCPhysApicBase, GCPhys;
3465	PDMApicGetBase(pVM, &GCPhysApicBase); /** @todo cache this */
3466	GCPhysApicBase &= PAGE_BASE_GC_MASK;
3467
3468	rc = PGMGstGetPage(pVCpu, (RTGCPTR)exitQualification, NULL, &GCPhys);
3469	if ( rc == VINF_SUCCESS
3470	&& GCPhys == GCPhysApicBase)
3471	{
3472	/* Only attempt to patch the instruction once. */
3473	PHWACCMTPRPATCH pPatch = (PHWACCMTPRPATCH)RTAvloU32Get(&pVM->hwaccm.s.PatchTree, (AVLOU32KEY)pCtx->eip);
3474	if (!pPatch)
3475	{
3476	rc = VINF_EM_HWACCM_PATCH_TPR_INSTR;
3477	break;
3478	}
3479	}
3480	}
3481	#endif
3482
3483	Log2(("Page fault at %RGv error code %x\n", exitQualification, errCode));
3484	/* Exit qualification contains the linear address of the page fault. */
3485	TRPMAssertTrap(pVCpu, X86_XCPT_PF, TRPM_TRAP);
3486	TRPMSetErrorCode(pVCpu, errCode);
3487	TRPMSetFaultAddress(pVCpu, exitQualification);
3488
3489	/* Shortcut for APIC TPR reads and writes. */
3490	if ( (exitQualification & 0xfff) == 0x080
3491	&& !(errCode & X86_TRAP_PF_P) /* not present */
3492	&& fSetupTPRCaching
3493	&& (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC))
3494	{
3495	RTGCPHYS GCPhysApicBase, GCPhys;
3496	PDMApicGetBase(pVM, &GCPhysApicBase); /* @todo cache this */
3497	GCPhysApicBase &= PAGE_BASE_GC_MASK;
3498
3499	rc = PGMGstGetPage(pVCpu, (RTGCPTR)exitQualification, NULL, &GCPhys);
3500	if ( rc == VINF_SUCCESS
3501	&& GCPhys == GCPhysApicBase)
3502	{
3503	Log(("Enable VT-x virtual APIC access filtering\n"));
3504	rc2 = IOMMMIOMapMMIOHCPage(pVM, GCPhysApicBase, pVM->hwaccm.s.vmx.pAPICPhys, X86_PTE_RW \| X86_PTE_P);
3505	AssertRC(rc2);
3506	}
3507	}
3508
3509	/* Forward it to our trap handler first, in case our shadow pages are out of sync. */
3510	rc = PGMTrap0eHandler(pVCpu, errCode, CPUMCTX2CORE(pCtx), (RTGCPTR)exitQualification);
3511	Log2(("PGMTrap0eHandler %RGv returned %Rrc\n", (RTGCPTR)pCtx->rip, VBOXSTRICTRC_VAL(rc)));
3512
3513	if (rc == VINF_SUCCESS)
3514	{ /* We've successfully synced our shadow pages, so let's just continue execution. */
3515	Log2(("Shadow page fault at %RGv cr2=%RGv error code %x\n", (RTGCPTR)pCtx->rip, exitQualification ,errCode));
3516	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitShadowPF);
3517
3518	TRPMResetTrap(pVCpu);
3519	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3520	goto ResumeExecution;
3521	}
3522	else if (rc == VINF_EM_RAW_GUEST_TRAP)
3523	{
3524	/*
3525	* A genuine pagefault. Forward the trap to the guest by injecting the exception and resuming execution.
3526	*/
3527	Log2(("Forward page fault to the guest\n"));
3528
3529	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestPF);
3530	/* The error code might have been changed. */
3531	errCode = TRPMGetErrorCode(pVCpu);
3532
3533	TRPMResetTrap(pVCpu);
3534
3535	/* Now we must update CR2. */
3536	pCtx->cr2 = exitQualification;
3537	rc2 = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo),
3538	cbInstr, errCode);
3539	AssertRC(rc2);
3540
3541	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3542	goto ResumeExecution;
3543	}
3544	#ifdef VBOX_STRICT
3545	if (rc != VINF_EM_RAW_EMULATE_INSTR && rc != VINF_EM_RAW_EMULATE_IO_BLOCK)
3546	Log2(("PGMTrap0eHandler failed with %d\n", VBOXSTRICTRC_VAL(rc)));
3547	#endif
3548	/* Need to go back to the recompiler to emulate the instruction. */
3549	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitShadowPFEM);
3550	TRPMResetTrap(pVCpu);
3551	break;
3552	}
3553
3554	case X86_XCPT_MF: /* Floating point exception. */
3555	{
3556	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestMF);
3557	if (!(pCtx->cr0 & X86_CR0_NE))
3558	{
3559	/* old style FPU error reporting needs some extra work. */
3560	/** @todo don't fall back to the recompiler, but do it manually. */
3561	rc = VINF_EM_RAW_EMULATE_INSTR;
3562	break;
3563	}
3564	Log(("Trap %x at %04X:%RGv\n", vector, pCtx->cs.Sel, (RTGCPTR)pCtx->rip));
3565	rc2 = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo),
3566	cbInstr, errCode);
3567	AssertRC(rc2);
3568
3569	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3570	goto ResumeExecution;
3571	}
3572
3573	case X86_XCPT_DB: /* Debug exception. */
3574	{
3575	uint64_t uDR6;
3576
3577	/*
3578	* DR6, DR7.GD and IA32_DEBUGCTL.LBR are not updated yet.
3579	*
3580	* Exit qualification bits:
3581	* 3:0 B0-B3 which breakpoint condition was met
3582	* 12:4 Reserved (0)
3583	* 13 BD - debug register access detected
3584	* 14 BS - single step execution or branch taken
3585	* 63:15 Reserved (0)
3586	*/
3587	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestDB);
3588
3589	/* Note that we don't support guest and host-initiated debugging at the same time. */
3590
3591	uDR6 = X86_DR6_INIT_VAL;
3592	uDR6 \|= (exitQualification & (X86_DR6_B0\|X86_DR6_B1\|X86_DR6_B2\|X86_DR6_B3\|X86_DR6_BD\|X86_DR6_BS));
3593	rc = DBGFRZTrap01Handler(pVM, pVCpu, CPUMCTX2CORE(pCtx), uDR6);
3594	if (rc == VINF_EM_RAW_GUEST_TRAP)
3595	{
3596	/* Update DR6 here. */
3597	pCtx->dr[6] = uDR6;
3598
3599	/* Resync DR6 if the debug state is active. */
3600	if (CPUMIsGuestDebugStateActive(pVCpu))
3601	ASMSetDR6(pCtx->dr[6]);
3602
3603	/* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
3604	pCtx->dr[7] &= ~X86_DR7_GD;
3605
3606	/* Paranoia. */
3607	pCtx->dr[7] &= 0xffffffff; /* upper 32 bits reserved */
3608	pCtx->dr[7] &= ~(RT_BIT(11) \| RT_BIT(12) \| RT_BIT(14) \| RT_BIT(15)); /* must be zero */
3609	pCtx->dr[7] \|= 0x400; /* must be one */
3610
3611	/* Resync DR7 */
3612	rc2 = VMXWriteVMCS64(VMX_VMCS64_GUEST_DR7, pCtx->dr[7]);
3613	AssertRC(rc2);
3614
3615	Log(("Trap %x (debug) at %RGv exit qualification %RX64 dr6=%x dr7=%x\n", vector, (RTGCPTR)pCtx->rip,
3616	exitQualification, (uint32_t)pCtx->dr[6], (uint32_t)pCtx->dr[7]));
3617	rc2 = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo),
3618	cbInstr, errCode);
3619	AssertRC(rc2);
3620
3621	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3622	goto ResumeExecution;
3623	}
3624	/* Return to ring 3 to deal with the debug exit code. */
3625	Log(("Debugger hardware BP at %04x:%RGv (rc=%Rrc)\n", pCtx->cs.Sel, pCtx->rip, VBOXSTRICTRC_VAL(rc)));
3626	break;
3627	}
3628
3629	case X86_XCPT_BP: /* Breakpoint. */
3630	{
3631	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestBP);
3632	rc = DBGFRZTrap03Handler(pVM, pVCpu, CPUMCTX2CORE(pCtx));
3633	if (rc == VINF_EM_RAW_GUEST_TRAP)
3634	{
3635	Log(("Guest #BP at %04x:%RGv\n", pCtx->cs.Sel, pCtx->rip));
3636	rc2 = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo),
3637	cbInstr, errCode);
3638	AssertRC(rc2);
3639	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3640	goto ResumeExecution;
3641	}
3642	if (rc == VINF_SUCCESS)
3643	{
3644	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3645	goto ResumeExecution;
3646	}
3647	Log(("Debugger BP at %04x:%RGv (rc=%Rrc)\n", pCtx->cs.Sel, pCtx->rip, VBOXSTRICTRC_VAL(rc)));
3648	break;
3649	}
3650
3651	case X86_XCPT_GP: /* General protection failure exception. */
3652	{
3653	uint32_t cbOp;
3654	PDISCPUSTATE pDis = &pVCpu->hwaccm.s.DisState;
3655
3656	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestGP);
3657	#ifdef VBOX_STRICT
3658	if ( !CPUMIsGuestInRealModeEx(pCtx)
3659	\|\| !pVM->hwaccm.s.vmx.pRealModeTSS)
3660	{
3661	Log(("Trap %x at %04X:%RGv errorCode=%RGv\n", vector, pCtx->cs.Sel, (RTGCPTR)pCtx->rip, errCode));
3662	rc2 = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo),
3663	cbInstr, errCode);
3664	AssertRC(rc2);
3665	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3666	goto ResumeExecution;
3667	}
3668	#endif
3669	Assert(CPUMIsGuestInRealModeEx(pCtx));
3670
3671	LogFlow(("Real mode X86_XCPT_GP instruction emulation at %x:%RGv\n", pCtx->cs.Sel, (RTGCPTR)pCtx->rip));
3672
3673	rc2 = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbOp);
3674	if (RT_SUCCESS(rc2))
3675	{
3676	bool fUpdateRIP = true;
3677
3678	rc = VINF_SUCCESS;
3679	Assert(cbOp == pDis->cbInstr);
3680	switch (pDis->pCurInstr->uOpcode)
3681	{
3682	case OP_CLI:
3683	pCtx->eflags.Bits.u1IF = 0;
3684	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitCli);
3685	break;
3686
3687	case OP_STI:
3688	pCtx->eflags.Bits.u1IF = 1;
3689	EMSetInhibitInterruptsPC(pVCpu, pCtx->rip + pDis->cbInstr);
3690	Assert(VMCPU_FF_ISSET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
3691	rc2 = VMXWriteVMCS(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE,
3692	VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
3693	AssertRC(rc2);
3694	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitSti);
3695	break;
3696
3697	case OP_HLT:
3698	fUpdateRIP = false;
3699	rc = VINF_EM_HALT;
3700	pCtx->rip += pDis->cbInstr;
3701	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitHlt);
3702	break;
3703
3704	case OP_POPF:
3705	{
3706	RTGCPTR GCPtrStack;
3707	uint32_t cbParm;
3708	uint32_t uMask;
3709	X86EFLAGS eflags;
3710
3711	if (pDis->fPrefix & DISPREFIX_OPSIZE)
3712	{
3713	cbParm = 4;
3714	uMask = 0xffffffff;
3715	}
3716	else
3717	{
3718	cbParm = 2;
3719	uMask = 0xffff;
3720	}
3721
3722	rc2 = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pCtx), pCtx->esp & uMask, 0, &GCPtrStack);
3723	if (RT_FAILURE(rc2))
3724	{
3725	rc = VERR_EM_INTERPRETER;
3726	break;
3727	}
3728	eflags.u = 0;
3729	rc2 = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &eflags.u, cbParm);
3730	if (RT_FAILURE(rc2))
3731	{
3732	rc = VERR_EM_INTERPRETER;
3733	break;
3734	}
3735	LogFlow(("POPF %x -> %RGv mask=%x\n", eflags.u, pCtx->rsp, uMask));
3736	pCtx->eflags.u = (pCtx->eflags.u & ~(X86_EFL_POPF_BITS & uMask))
3737	\| (eflags.u & X86_EFL_POPF_BITS & uMask);
3738	/* RF cleared when popped in real mode; see pushf description in AMD manual. */
3739	pCtx->eflags.Bits.u1RF = 0;
3740	pCtx->esp += cbParm;
3741	pCtx->esp &= uMask;
3742
3743	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitPopf);
3744	break;
3745	}
3746
3747	case OP_PUSHF:
3748	{
3749	RTGCPTR GCPtrStack;
3750	uint32_t cbParm;
3751	uint32_t uMask;
3752	X86EFLAGS eflags;
3753
3754	if (pDis->fPrefix & DISPREFIX_OPSIZE)
3755	{
3756	cbParm = 4;
3757	uMask = 0xffffffff;
3758	}
3759	else
3760	{
3761	cbParm = 2;
3762	uMask = 0xffff;
3763	}
3764
3765	rc2 = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pCtx), (pCtx->esp - cbParm) & uMask, 0,
3766	&GCPtrStack);
3767	if (RT_FAILURE(rc2))
3768	{
3769	rc = VERR_EM_INTERPRETER;
3770	break;
3771	}
3772	eflags = pCtx->eflags;
3773	/* RF & VM cleared when pushed in real mode; see pushf description in AMD manual. */
3774	eflags.Bits.u1RF = 0;
3775	eflags.Bits.u1VM = 0;
3776
3777	rc2 = PGMPhysWrite(pVM, (RTGCPHYS)GCPtrStack, &eflags.u, cbParm);
3778	if (RT_FAILURE(rc2))
3779	{
3780	rc = VERR_EM_INTERPRETER;
3781	break;
3782	}
3783	LogFlow(("PUSHF %x -> %RGv\n", eflags.u, GCPtrStack));
3784	pCtx->esp -= cbParm;
3785	pCtx->esp &= uMask;
3786	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitPushf);
3787	break;
3788	}
3789
3790	case OP_IRET:
3791	{
3792	RTGCPTR GCPtrStack;
3793	uint32_t uMask = 0xffff;
3794	uint16_t aIretFrame[3];
3795
3796	if (pDis->fPrefix & (DISPREFIX_OPSIZE \| DISPREFIX_ADDRSIZE))
3797	{
3798	rc = VERR_EM_INTERPRETER;
3799	break;
3800	}
3801
3802	rc2 = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pCtx), pCtx->esp & uMask, 0, &GCPtrStack);
3803	if (RT_FAILURE(rc2))
3804	{
3805	rc = VERR_EM_INTERPRETER;
3806	break;
3807	}
3808	rc2 = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &aIretFrame[0], sizeof(aIretFrame));
3809	if (RT_FAILURE(rc2))
3810	{
3811	rc = VERR_EM_INTERPRETER;
3812	break;
3813	}
3814	pCtx->ip = aIretFrame[0];
3815	pCtx->cs.Sel = aIretFrame[1];
3816	pCtx->cs.ValidSel = aIretFrame[1];
3817	pCtx->cs.u64Base = (uint32_t)pCtx->cs.Sel << 4;
3818	pCtx->eflags.u = (pCtx->eflags.u & ~(X86_EFL_POPF_BITS & uMask))
3819	\| (aIretFrame[2] & X86_EFL_POPF_BITS & uMask);
3820	pCtx->sp += sizeof(aIretFrame);
3821
3822	LogFlow(("iret to %04x:%x\n", pCtx->cs.Sel, pCtx->ip));
3823	fUpdateRIP = false;
3824	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitIret);
3825	break;
3826	}
3827
3828	case OP_INT:
3829	{
3830	uint32_t intInfo2;
3831
3832	LogFlow(("Realmode: INT %x\n", pDis->Param1.uValue & 0xff));
3833	intInfo2 = pDis->Param1.uValue & 0xff;
3834	intInfo2 \|= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
3835	intInfo2 \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
3836
3837	rc = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, intInfo2, cbOp, 0);
3838	AssertRC(VBOXSTRICTRC_VAL(rc));
3839	fUpdateRIP = false;
3840	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitInt);
3841	break;
3842	}
3843
3844	case OP_INTO:
3845	{
3846	if (pCtx->eflags.Bits.u1OF)
3847	{
3848	uint32_t intInfo2;
3849
3850	LogFlow(("Realmode: INTO\n"));
3851	intInfo2 = X86_XCPT_OF;
3852	intInfo2 \|= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
3853	intInfo2 \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
3854
3855	rc = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, intInfo2, cbOp, 0);
3856	AssertRC(VBOXSTRICTRC_VAL(rc));
3857	fUpdateRIP = false;
3858	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitInt);
3859	}
3860	break;
3861	}
3862
3863	case OP_INT3:
3864	{
3865	uint32_t intInfo2;
3866
3867	LogFlow(("Realmode: INT 3\n"));
3868	intInfo2 = 3;
3869	intInfo2 \|= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
3870	intInfo2 \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
3871
3872	rc = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, intInfo2, cbOp, 0);
3873	AssertRC(VBOXSTRICTRC_VAL(rc));
3874	fUpdateRIP = false;
3875	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitInt);
3876	break;
3877	}
3878
3879	default:
3880	rc = EMInterpretInstructionDisasState(pVCpu, pDis, CPUMCTX2CORE(pCtx), 0, EMCODETYPE_SUPERVISOR);
3881	fUpdateRIP = false;
3882	break;
3883	}
3884
3885	if (rc == VINF_SUCCESS)
3886	{
3887	if (fUpdateRIP)
3888	pCtx->rip += cbOp; /* Move on to the next instruction. */
3889
3890	/*
3891	* LIDT, LGDT can end up here. In the future CRx changes as well. Just reload the
3892	* whole context to be done with it.
3893	*/
3894	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_ALL;
3895
3896	/* Only resume if successful. */
3897	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3898	goto ResumeExecution;
3899	}
3900	}
3901	else
3902	rc = VERR_EM_INTERPRETER;
3903
3904	AssertMsg(rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_EM_HALT,
3905	("Unexpected rc=%Rrc\n", VBOXSTRICTRC_VAL(rc)));
3906	break;
3907	}
3908
3909	#ifdef VBOX_STRICT
3910	case X86_XCPT_XF: /* SIMD exception. */
3911	case X86_XCPT_DE: /* Divide error. */
3912	case X86_XCPT_UD: /* Unknown opcode exception. */
3913	case X86_XCPT_SS: /* Stack segment exception. */
3914	case X86_XCPT_NP: /* Segment not present exception. */
3915	{
3916	switch (vector)
3917	{
3918	case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestDE); break;
3919	case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestUD); break;
3920	case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestSS); break;
3921	case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestNP); break;
3922	case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestXF); break;
3923	}
3924
3925	Log(("Trap %x at %04X:%RGv\n", vector, pCtx->cs.Sel, (RTGCPTR)pCtx->rip));
3926	rc2 = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo),
3927	cbInstr, errCode);
3928	AssertRC(rc2);
3929
3930	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3931	goto ResumeExecution;
3932	}
3933	#endif
3934	default:
3935	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestXcpUnk);
3936	if ( CPUMIsGuestInRealModeEx(pCtx)
3937	&& pVM->hwaccm.s.vmx.pRealModeTSS)
3938	{
3939	Log(("Real Mode Trap %x at %04x:%04X error code %x\n", vector, pCtx->cs.Sel, pCtx->eip, errCode));
3940	rc = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo),
3941	cbInstr, errCode);
3942	AssertRC(VBOXSTRICTRC_VAL(rc)); /* Strict RC check below. */
3943
3944	/* Go back to ring-3 in case of a triple fault. */
3945	if ( vector == X86_XCPT_DF
3946	&& rc == VINF_EM_RESET)
3947	{
3948	break;
3949	}
3950
3951	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3952	goto ResumeExecution;
3953	}
3954	AssertMsgFailed(("Unexpected vm-exit caused by exception %x\n", vector));
3955	rc = VERR_VMX_UNEXPECTED_EXCEPTION;
3956	break;
3957	} /* switch (vector) */
3958
3959	break;
3960
3961	default:
3962	rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_CODE;
3963	AssertMsgFailed(("Unexpected interruption code %x\n", intInfo));
3964	break;
3965	}
3966
3967	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3968	break;
3969	}
3970
3971	/*
3972	* 48 EPT violation. An attempt to access memory with a guest-physical address was disallowed
3973	* by the configuration of the EPT paging structures.
3974	*/
3975	case VMX_EXIT_EPT_VIOLATION:
3976	{
3977	RTGCPHYS GCPhys;
3978
3979	Assert(pVM->hwaccm.s.fNestedPaging);
3980
3981	rc2 = VMXReadVMCS64(VMX_VMCS_EXIT_PHYS_ADDR_FULL, &GCPhys);
3982	AssertRC(rc2);
3983	Assert(((exitQualification >> 7) & 3) != 2);
3984
3985	/* Determine the kind of violation. */
3986	errCode = 0;
3987	if (exitQualification & VMX_EXIT_QUALIFICATION_EPT_INSTR_FETCH)
3988	errCode \|= X86_TRAP_PF_ID;
3989
3990	if (exitQualification & VMX_EXIT_QUALIFICATION_EPT_DATA_WRITE)
3991	errCode \|= X86_TRAP_PF_RW;
3992
3993	/* If the page is present, then it's a page level protection fault. */
3994	if (exitQualification & VMX_EXIT_QUALIFICATION_EPT_ENTRY_PRESENT)
3995	errCode \|= X86_TRAP_PF_P;
3996	else
3997	{
3998	/* Shortcut for APIC TPR reads and writes. */
3999	if ( (GCPhys & 0xfff) == 0x080
4000	&& GCPhys > 0x1000000 /* to skip VGA frame buffer accesses */
4001	&& fSetupTPRCaching
4002	&& (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC))
4003	{
4004	RTGCPHYS GCPhysApicBase;
4005	PDMApicGetBase(pVM, &GCPhysApicBase); /* @todo cache this */
4006	GCPhysApicBase &= PAGE_BASE_GC_MASK;
4007	if (GCPhys == GCPhysApicBase + 0x80)
4008	{
4009	Log(("Enable VT-x virtual APIC access filtering\n"));
4010	rc2 = IOMMMIOMapMMIOHCPage(pVM, GCPhysApicBase, pVM->hwaccm.s.vmx.pAPICPhys, X86_PTE_RW \| X86_PTE_P);
4011	AssertRC(rc2);
4012	}
4013	}
4014	}
4015	Log(("EPT Page fault %x at %RGp error code %x\n", (uint32_t)exitQualification, GCPhys, errCode));
4016
4017	/* GCPhys contains the guest physical address of the page fault. */
4018	TRPMAssertTrap(pVCpu, X86_XCPT_PF, TRPM_TRAP);
4019	TRPMSetErrorCode(pVCpu, errCode);
4020	TRPMSetFaultAddress(pVCpu, GCPhys);
4021
4022	/* Handle the pagefault trap for the nested shadow table. */
4023	rc = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, errCode, CPUMCTX2CORE(pCtx), GCPhys);
4024
4025	/*
4026	* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment below, @bugref{6043}.
4027	*/
4028	if ( rc == VINF_SUCCESS
4029	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
4030	\|\| rc == VERR_PAGE_NOT_PRESENT)
4031	{
4032	/* We've successfully synced our shadow pages, so let's just continue execution. */
4033	Log2(("Shadow page fault at %RGv cr2=%RGp error code %x\n", (RTGCPTR)pCtx->rip, exitQualification , errCode));
4034	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitReasonNPF);
4035
4036	TRPMResetTrap(pVCpu);
4037	goto ResumeExecution;
4038	}
4039
4040	#ifdef VBOX_STRICT
4041	if (rc != VINF_EM_RAW_EMULATE_INSTR)
4042	LogFlow(("PGMTrap0eHandlerNestedPaging at %RGv failed with %Rrc\n", (RTGCPTR)pCtx->rip, VBOXSTRICTRC_VAL(rc)));
4043	#endif
4044	/* Need to go back to the recompiler to emulate the instruction. */
4045	TRPMResetTrap(pVCpu);
4046	break;
4047	}
4048
4049	case VMX_EXIT_EPT_MISCONFIG:
4050	{
4051	RTGCPHYS GCPhys;
4052
4053	Assert(pVM->hwaccm.s.fNestedPaging);
4054
4055	rc2 = VMXReadVMCS64(VMX_VMCS_EXIT_PHYS_ADDR_FULL, &GCPhys);
4056	AssertRC(rc2);
4057	Log(("VMX_EXIT_EPT_MISCONFIG for %RGp\n", GCPhys));
4058
4059	/* Shortcut for APIC TPR reads and writes. */
4060	if ( (GCPhys & 0xfff) == 0x080
4061	&& GCPhys > 0x1000000 /* to skip VGA frame buffer accesses */
4062	&& fSetupTPRCaching
4063	&& (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC))
4064	{
4065	RTGCPHYS GCPhysApicBase;
4066	PDMApicGetBase(pVM, &GCPhysApicBase); /* @todo cache this */
4067	GCPhysApicBase &= PAGE_BASE_GC_MASK;
4068	if (GCPhys == GCPhysApicBase + 0x80)
4069	{
4070	Log(("Enable VT-x virtual APIC access filtering\n"));
4071	rc2 = IOMMMIOMapMMIOHCPage(pVM, GCPhysApicBase, pVM->hwaccm.s.vmx.pAPICPhys, X86_PTE_RW \| X86_PTE_P);
4072	AssertRC(rc2);
4073	}
4074	}
4075
4076	rc = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pCtx), GCPhys, UINT32_MAX);
4077
4078	/*
4079	* If we succeed, resume execution.
4080	* Or, if fail in interpreting the instruction because we couldn't get the guest physical address
4081	* of the page containing the instruction via the guest's page tables (we would invalidate the guest page
4082	* in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
4083	* weird case. See @bugref{6043}.
4084	*/
4085	if ( rc == VINF_SUCCESS
4086	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
4087	\|\| rc == VERR_PAGE_NOT_PRESENT)
4088	{
4089	Log2(("PGMR0Trap0eHandlerNPMisconfig(,,,%RGp) at %RGv -> resume\n", GCPhys, (RTGCPTR)pCtx->rip));
4090	goto ResumeExecution;
4091	}
4092
4093	Log2(("PGMR0Trap0eHandlerNPMisconfig(,,,%RGp) at %RGv -> %Rrc\n", GCPhys, (RTGCPTR)pCtx->rip, VBOXSTRICTRC_VAL(rc)));
4094	break;
4095	}
4096
4097	case VMX_EXIT_IRQ_WINDOW: /* 7 Interrupt window. */
4098	/* Clear VM-exit on IF=1 change. */
4099	LogFlow(("VMX_EXIT_IRQ_WINDOW %RGv pending=%d IF=%d\n", (RTGCPTR)pCtx->rip,
4100	VMCPU_FF_ISPENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC\|VMCPU_FF_INTERRUPT_PIC)), pCtx->eflags.Bits.u1IF));
4101	pVCpu->hwaccm.s.vmx.proc_ctls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_IRQ_WINDOW_EXIT;
4102	rc2 = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
4103	AssertRC(rc2);
4104	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitIrqWindow);
4105	goto ResumeExecution; /* we check for pending guest interrupts there */
4106
4107	case VMX_EXIT_WBINVD: /* 54 Guest software attempted to execute WBINVD. (conditional) */
4108	case VMX_EXIT_INVD: /* 13 Guest software attempted to execute INVD. (unconditional) */
4109	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitInvd);
4110	/* Skip instruction and continue directly. */
4111	pCtx->rip += cbInstr;
4112	/* Continue execution.*/
4113	goto ResumeExecution;
4114
4115	case VMX_EXIT_CPUID: /* 10 Guest software attempted to execute CPUID. */
4116	{
4117	Log2(("VMX: Cpuid %x\n", pCtx->eax));
4118	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitCpuid);
4119	rc = EMInterpretCpuId(pVM, pVCpu, CPUMCTX2CORE(pCtx));
4120	if (rc == VINF_SUCCESS)
4121	{
4122	/* Update EIP and continue execution. */
4123	Assert(cbInstr == 2);
4124	pCtx->rip += cbInstr;
4125	goto ResumeExecution;
4126	}
4127	AssertMsgFailed(("EMU: cpuid failed with %Rrc\n", VBOXSTRICTRC_VAL(rc)));
4128	rc = VINF_EM_RAW_EMULATE_INSTR;
4129	break;
4130	}
4131
4132	case VMX_EXIT_RDPMC: /* 15 Guest software attempted to execute RDPMC. */
4133	{
4134	Log2(("VMX: Rdpmc %x\n", pCtx->ecx));
4135	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitRdpmc);
4136	rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pCtx));
4137	if (rc == VINF_SUCCESS)
4138	{
4139	/* Update EIP and continue execution. */
4140	Assert(cbInstr == 2);
4141	pCtx->rip += cbInstr;
4142	goto ResumeExecution;
4143	}
4144	rc = VINF_EM_RAW_EMULATE_INSTR;
4145	break;
4146	}
4147
4148	case VMX_EXIT_RDTSC: /* 16 Guest software attempted to execute RDTSC. */
4149	{
4150	Log2(("VMX: Rdtsc\n"));
4151	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitRdtsc);
4152	rc = EMInterpretRdtsc(pVM, pVCpu, CPUMCTX2CORE(pCtx));
4153	if (rc == VINF_SUCCESS)
4154	{
4155	/* Update EIP and continue execution. */
4156	Assert(cbInstr == 2);
4157	pCtx->rip += cbInstr;
4158	goto ResumeExecution;
4159	}
4160	rc = VINF_EM_RAW_EMULATE_INSTR;
4161	break;
4162	}
4163
4164	case VMX_EXIT_RDTSCP: /* 51 Guest software attempted to execute RDTSCP. */
4165	{
4166	Log2(("VMX: Rdtscp\n"));
4167	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitRdtscp);
4168	rc = EMInterpretRdtscp(pVM, pVCpu, pCtx);
4169	if (rc == VINF_SUCCESS)
4170	{
4171	/* Update EIP and continue execution. */
4172	Assert(cbInstr == 3);
4173	pCtx->rip += cbInstr;
4174	goto ResumeExecution;
4175	}
4176	rc = VINF_EM_RAW_EMULATE_INSTR;
4177	break;
4178	}
4179
4180	case VMX_EXIT_INVLPG: /* 14 Guest software attempted to execute INVLPG. */
4181	{
4182	Log2(("VMX: invlpg\n"));
4183	Assert(!pVM->hwaccm.s.fNestedPaging);
4184
4185	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitInvlpg);
4186	rc = EMInterpretInvlpg(pVM, pVCpu, CPUMCTX2CORE(pCtx), exitQualification);
4187	if (rc == VINF_SUCCESS)
4188	{
4189	/* Update EIP and continue execution. */
4190	pCtx->rip += cbInstr;
4191	goto ResumeExecution;
4192	}
4193	AssertMsg(rc == VERR_EM_INTERPRETER, ("EMU: invlpg %RGv failed with %Rrc\n", exitQualification, VBOXSTRICTRC_VAL(rc)));
4194	break;
4195	}
4196
4197	case VMX_EXIT_MONITOR: /* 39 Guest software attempted to execute MONITOR. */
4198	{
4199	Log2(("VMX: monitor\n"));
4200
4201	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitMonitor);
4202	rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pCtx));
4203	if (rc == VINF_SUCCESS)
4204	{
4205	/* Update EIP and continue execution. */
4206	pCtx->rip += cbInstr;
4207	goto ResumeExecution;
4208	}
4209	AssertMsg(rc == VERR_EM_INTERPRETER, ("EMU: monitor failed with %Rrc\n", VBOXSTRICTRC_VAL(rc)));
4210	break;
4211	}
4212
4213	case VMX_EXIT_WRMSR: /* 32 WRMSR. Guest software attempted to execute WRMSR. */
4214	/* When an interrupt is pending, we'll let MSR_K8_LSTAR writes fault in our TPR patch code. */
4215	if ( pVM->hwaccm.s.fTPRPatchingActive
4216	&& pCtx->ecx == MSR_K8_LSTAR)
4217	{
4218	Assert(!CPUMIsGuestInLongModeEx(pCtx));
4219	if ((pCtx->eax & 0xff) != u8LastTPR)
4220	{
4221	Log(("VMX: Faulting MSR_K8_LSTAR write with new TPR value %x\n", pCtx->eax & 0xff));
4222
4223	/* Our patch code uses LSTAR for TPR caching. */
4224	rc2 = PDMApicSetTPR(pVCpu, pCtx->eax & 0xff);
4225	AssertRC(rc2);
4226	}
4227
4228	/* Skip the instruction and continue. */
4229	pCtx->rip += cbInstr; /* wrmsr = [0F 30] */
4230
4231	/* Only resume if successful. */
4232	goto ResumeExecution;
4233	}
4234	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_GUEST_MSR;
4235	/* no break */
4236	case VMX_EXIT_RDMSR: /* 31 RDMSR. Guest software attempted to execute RDMSR. */
4237	{
4238	STAM_COUNTER_INC((exitReason == VMX_EXIT_RDMSR) ? &pVCpu->hwaccm.s.StatExitRdmsr : &pVCpu->hwaccm.s.StatExitWrmsr);
4239
4240	/*
4241	* Note: The Intel spec. claims there's an REX version of RDMSR that's slightly different,
4242	* so we play safe by completely disassembling the instruction.
4243	*/
4244	Log2(("VMX: %s\n", (exitReason == VMX_EXIT_RDMSR) ? "rdmsr" : "wrmsr"));
4245	rc = EMInterpretInstruction(pVCpu, CPUMCTX2CORE(pCtx), 0);
4246	if (rc == VINF_SUCCESS)
4247	{
4248	/* EIP has been updated already. */
4249	/* Only resume if successful. */
4250	goto ResumeExecution;
4251	}
4252	AssertMsg(rc == VERR_EM_INTERPRETER, ("EMU: %s failed with %Rrc\n",
4253	(exitReason == VMX_EXIT_RDMSR) ? "rdmsr" : "wrmsr", VBOXSTRICTRC_VAL(rc)));
4254	break;
4255	}
4256
4257	case VMX_EXIT_CRX_MOVE: /* 28 Control-register accesses. */
4258	{
4259	STAM_PROFILE_ADV_START(&pVCpu->hwaccm.s.StatExit2Sub2, y2);
4260
4261	switch (VMX_EXIT_QUALIFICATION_CRX_ACCESS(exitQualification))
4262	{
4263	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_WRITE:
4264	{
4265	Log2(("VMX: %RGv mov cr%d, x\n", (RTGCPTR)pCtx->rip, VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification)));
4266	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitCRxWrite[VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification)]);
4267	rc = EMInterpretCRxWrite(pVM, pVCpu, CPUMCTX2CORE(pCtx),
4268	VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification),
4269	VMX_EXIT_QUALIFICATION_CRX_GENREG(exitQualification));
4270	switch (VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification))
4271	{
4272	case 0:
4273	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_GUEST_CR0 \| HWACCM_CHANGED_GUEST_CR3;
4274	break;
4275	case 2:
4276	break;
4277	case 3:
4278	Assert(!pVM->hwaccm.s.fNestedPaging \|\| !CPUMIsGuestInPagedProtectedModeEx(pCtx));
4279	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_GUEST_CR3;
4280	break;
4281	case 4:
4282	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_GUEST_CR4;
4283	break;
4284	case 8:
4285	/* CR8 contains the APIC TPR */
4286	Assert(!(pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1
4287	& VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW));
4288	break;
4289
4290	default:
4291	AssertFailed();
4292	break;
4293	}
4294	break;
4295	}
4296
4297	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ:
4298	{
4299	Log2(("VMX: mov x, crx\n"));
4300	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitCRxRead[VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification)]);
4301
4302	Assert( !pVM->hwaccm.s.fNestedPaging
4303	\|\| !CPUMIsGuestInPagedProtectedModeEx(pCtx)
4304	\|\| VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification) != DISCREG_CR3);
4305
4306	/* CR8 reads only cause an exit when the TPR shadow feature isn't present. */
4307	Assert( VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification) != 8
4308	\|\| !(pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW));
4309
4310	rc = EMInterpretCRxRead(pVM, pVCpu, CPUMCTX2CORE(pCtx),
4311	VMX_EXIT_QUALIFICATION_CRX_GENREG(exitQualification),
4312	VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification));
4313	break;
4314	}
4315
4316	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_CLTS:
4317	{
4318	Log2(("VMX: clts\n"));
4319	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitCLTS);
4320	rc = EMInterpretCLTS(pVM, pVCpu);
4321	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_GUEST_CR0;
4322	break;
4323	}
4324
4325	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_LMSW:
4326	{
4327	Log2(("VMX: lmsw %x\n", VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(exitQualification)));
4328	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitLMSW);
4329	rc = EMInterpretLMSW(pVM, pVCpu, CPUMCTX2CORE(pCtx), VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(exitQualification));
4330	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_GUEST_CR0;
4331	break;
4332	}
4333	}
4334
4335	/* Update EIP if no error occurred. */
4336	if (RT_SUCCESS(rc))
4337	pCtx->rip += cbInstr;
4338
4339	if (rc == VINF_SUCCESS)
4340	{
4341	/* Only resume if successful. */
4342	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub2, y2);
4343	goto ResumeExecution;
4344	}
4345	Assert(rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_PGM_SYNC_CR3);
4346	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub2, y2);
4347	break;
4348	}
4349
4350	case VMX_EXIT_DRX_MOVE: /* 29 Debug-register accesses. */
4351	{
4352	if ( !DBGFIsStepping(pVCpu)
4353	&& !CPUMIsHyperDebugStateActive(pVCpu))
4354	{
4355	/* Disable DRx move intercepts. */
4356	pVCpu->hwaccm.s.vmx.proc_ctls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT;
4357	rc2 = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
4358	AssertRC(rc2);
4359
4360	/* Save the host and load the guest debug state. */
4361	rc2 = CPUMR0LoadGuestDebugState(pVM, pVCpu, pCtx, true /* include DR6 */);
4362	AssertRC(rc2);
4363
4364	#ifdef LOG_ENABLED
4365	if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(exitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
4366	{
4367	Log(("VMX_EXIT_DRX_MOVE: write DR%d genreg %d\n", VMX_EXIT_QUALIFICATION_DRX_REGISTER(exitQualification),
4368	VMX_EXIT_QUALIFICATION_DRX_GENREG(exitQualification)));
4369	}
4370	else
4371	Log(("VMX_EXIT_DRX_MOVE: read DR%d\n", VMX_EXIT_QUALIFICATION_DRX_REGISTER(exitQualification)));
4372	#endif
4373
4374	#ifdef VBOX_WITH_STATISTICS
4375	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatDRxContextSwitch);
4376	if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(exitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
4377	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitDRxWrite);
4378	else
4379	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitDRxRead);
4380	#endif
4381
4382	goto ResumeExecution;
4383	}
4384
4385	/** @todo clear VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT after the first
4386	* time and restore DRx registers afterwards */
4387	if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(exitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
4388	{
4389	Log2(("VMX: mov DRx%d, genreg%d\n", VMX_EXIT_QUALIFICATION_DRX_REGISTER(exitQualification),
4390	VMX_EXIT_QUALIFICATION_DRX_GENREG(exitQualification)));
4391	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitDRxWrite);
4392	rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pCtx),
4393	VMX_EXIT_QUALIFICATION_DRX_REGISTER(exitQualification),
4394	VMX_EXIT_QUALIFICATION_DRX_GENREG(exitQualification));
4395	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_GUEST_DEBUG;
4396	Log2(("DR7=%08x\n", pCtx->dr[7]));
4397	}
4398	else
4399	{
4400	Log2(("VMX: mov x, DRx\n"));
4401	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitDRxRead);
4402	rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pCtx),
4403	VMX_EXIT_QUALIFICATION_DRX_GENREG(exitQualification),
4404	VMX_EXIT_QUALIFICATION_DRX_REGISTER(exitQualification));
4405	}
4406	/* Update EIP if no error occurred. */
4407	if (RT_SUCCESS(rc))
4408	pCtx->rip += cbInstr;
4409
4410	if (rc == VINF_SUCCESS)
4411	{
4412	/* Only resume if successful. */
4413	goto ResumeExecution;
4414	}
4415	Assert(rc == VERR_EM_INTERPRETER);
4416	break;
4417	}
4418
4419	/* Note: We'll get a #GP if the IO instruction isn't allowed (IOPL or TSS bitmap); no need to double check. */
4420	case VMX_EXIT_PORT_IO: /* 30 I/O instruction. */
4421	{
4422	STAM_PROFILE_ADV_START(&pVCpu->hwaccm.s.StatExit2Sub1, y1);
4423	uint32_t uPort;
4424	uint32_t uIOWidth = VMX_EXIT_QUALIFICATION_IO_WIDTH(exitQualification);
4425	bool fIOWrite = (VMX_EXIT_QUALIFICATION_IO_DIRECTION(exitQualification) == VMX_EXIT_QUALIFICATION_IO_DIRECTION_OUT);
4426
4427	/** @todo necessary to make the distinction? */
4428	if (VMX_EXIT_QUALIFICATION_IO_ENCODING(exitQualification) == VMX_EXIT_QUALIFICATION_IO_ENCODING_DX)
4429	uPort = pCtx->edx & 0xffff;
4430	else
4431	uPort = VMX_EXIT_QUALIFICATION_IO_PORT(exitQualification); /* Immediate encoding. */
4432
4433	if (RT_UNLIKELY(uIOWidth == 2 \|\| uIOWidth >= 4)) /* paranoia */
4434	{
4435	rc = fIOWrite ? VINF_IOM_R3_IOPORT_WRITE : VINF_IOM_R3_IOPORT_READ;
4436	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub1, y1);
4437	break;
4438	}
4439
4440	uint32_t cbSize = g_aIOSize[uIOWidth];
4441	if (VMX_EXIT_QUALIFICATION_IO_STRING(exitQualification))
4442	{
4443	/* ins/outs */
4444	PDISCPUSTATE pDis = &pVCpu->hwaccm.s.DisState;
4445
4446	/* Disassemble manually to deal with segment prefixes. */
4447	/** @todo VMX_VMCS_EXIT_GUEST_LINEAR_ADDR contains the flat pointer operand of the instruction. */
4448	/** @todo VMX_VMCS32_RO_EXIT_INSTR_INFO also contains segment prefix info. */
4449	rc2 = EMInterpretDisasCurrent(pVM, pVCpu, pDis, NULL);
4450	if (RT_SUCCESS(rc))
4451	{
4452	if (fIOWrite)
4453	{
4454	Log2(("IOMInterpretOUTSEx %RGv %x size=%d\n", (RTGCPTR)pCtx->rip, uPort, cbSize));
4455	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitIOStringWrite);
4456	rc = IOMInterpretOUTSEx(pVM, CPUMCTX2CORE(pCtx), uPort, pDis->fPrefix, (DISCPUMODE)pDis->uAddrMode, cbSize);
4457	}
4458	else
4459	{
4460	Log2(("IOMInterpretINSEx %RGv %x size=%d\n", (RTGCPTR)pCtx->rip, uPort, cbSize));
4461	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitIOStringRead);
4462	rc = IOMInterpretINSEx(pVM, CPUMCTX2CORE(pCtx), uPort, pDis->fPrefix, (DISCPUMODE)pDis->uAddrMode, cbSize);
4463	}
4464	}
4465	else
4466	rc = VINF_EM_RAW_EMULATE_INSTR;
4467	}
4468	else
4469	{
4470	/* Normal in/out */
4471	uint32_t uAndVal = g_aIOOpAnd[uIOWidth];
4472
4473	Assert(!VMX_EXIT_QUALIFICATION_IO_REP(exitQualification));
4474
4475	if (fIOWrite)
4476	{
4477	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitIOWrite);
4478	rc = IOMIOPortWrite(pVM, uPort, pCtx->eax & uAndVal, cbSize);
4479	if (rc == VINF_IOM_R3_IOPORT_WRITE)
4480	HWACCMR0SavePendingIOPortWrite(pVCpu, pCtx->rip, pCtx->rip + cbInstr, uPort, uAndVal, cbSize);
4481	}
4482	else
4483	{
4484	uint32_t u32Val = 0;
4485
4486	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitIORead);
4487	rc = IOMIOPortRead(pVM, uPort, &u32Val, cbSize);
4488	if (IOM_SUCCESS(rc))
4489	{
4490	/* Write back to the EAX register. */
4491	pCtx->eax = (pCtx->eax & ~uAndVal) \| (u32Val & uAndVal);
4492	}
4493	else
4494	if (rc == VINF_IOM_R3_IOPORT_READ)
4495	HWACCMR0SavePendingIOPortRead(pVCpu, pCtx->rip, pCtx->rip + cbInstr, uPort, uAndVal, cbSize);
4496	}
4497	}
4498
4499	/*
4500	* Handled the I/O return codes.
4501	* (The unhandled cases end up with rc == VINF_EM_RAW_EMULATE_INSTR.)
4502	*/
4503	if (IOM_SUCCESS(rc))
4504	{
4505	/* Update EIP and continue execution. */
4506	pCtx->rip += cbInstr;
4507	if (RT_LIKELY(rc == VINF_SUCCESS))
4508	{
4509	/* If any IO breakpoints are armed, then we should check if a debug trap needs to be generated. */
4510	if (pCtx->dr[7] & X86_DR7_ENABLED_MASK)
4511	{
4512	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatDRxIOCheck);
4513	for (unsigned i = 0; i < 4; i++)
4514	{
4515	unsigned uBPLen = g_aIOSize[X86_DR7_GET_LEN(pCtx->dr[7], i)];
4516
4517	if ( (uPort >= pCtx->dr[i] && uPort < pCtx->dr[i] + uBPLen)
4518	&& (pCtx->dr[7] & (X86_DR7_L(i) \| X86_DR7_G(i)))
4519	&& (pCtx->dr[7] & X86_DR7_RW(i, X86_DR7_RW_IO)) == X86_DR7_RW(i, X86_DR7_RW_IO))
4520	{
4521	uint64_t uDR6;
4522
4523	Assert(CPUMIsGuestDebugStateActive(pVCpu));
4524
4525	uDR6 = ASMGetDR6();
4526
4527	/* Clear all breakpoint status flags and set the one we just hit. */
4528	uDR6 &= ~(X86_DR6_B0\|X86_DR6_B1\|X86_DR6_B2\|X86_DR6_B3);
4529	uDR6 \|= (uint64_t)RT_BIT(i);
4530
4531	/*
4532	* Note: AMD64 Architecture Programmer's Manual 13.1:
4533	* Bits 15:13 of the DR6 register is never cleared by the processor and must
4534	* be cleared by software after the contents have been read.
4535	*/
4536	ASMSetDR6(uDR6);
4537
4538	/* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
4539	pCtx->dr[7] &= ~X86_DR7_GD;
4540
4541	/* Paranoia. */
4542	pCtx->dr[7] &= 0xffffffff; /* upper 32 bits reserved */
4543	pCtx->dr[7] &= ~(RT_BIT(11) \| RT_BIT(12) \| RT_BIT(14) \| RT_BIT(15)); /* must be zero */
4544	pCtx->dr[7] \|= 0x400; /* must be one */
4545
4546	/* Resync DR7 */
4547	rc2 = VMXWriteVMCS64(VMX_VMCS64_GUEST_DR7, pCtx->dr[7]);
4548	AssertRC(rc2);
4549
4550	/* Construct inject info. */
4551	intInfo = X86_XCPT_DB;
4552	intInfo \|= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
4553	intInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HWEXCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
4554
4555	Log(("Inject IO debug trap at %RGv\n", (RTGCPTR)pCtx->rip));
4556	rc2 = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo),
4557	0 /* cbInstr /, 0 / errCode */);
4558	AssertRC(rc2);
4559
4560	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub1, y1);
4561	goto ResumeExecution;
4562	}
4563	}
4564	}
4565	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub1, y1);
4566	goto ResumeExecution;
4567	}
4568	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub1, y1);
4569	break;
4570	}
4571
4572	#ifdef VBOX_STRICT
4573	if (rc == VINF_IOM_R3_IOPORT_READ)
4574	Assert(!fIOWrite);
4575	else if (rc == VINF_IOM_R3_IOPORT_WRITE)
4576	Assert(fIOWrite);
4577	else
4578	{
4579	AssertMsg( RT_FAILURE(rc)
4580	\|\| rc == VINF_EM_RAW_EMULATE_INSTR
4581	\|\| rc == VINF_EM_RAW_GUEST_TRAP
4582	\|\| rc == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", VBOXSTRICTRC_VAL(rc)));
4583	}
4584	#endif
4585	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub1, y1);
4586	break;
4587	}
4588
4589	case VMX_EXIT_TPR: /* 43 TPR below threshold. Guest software executed MOV to CR8. */
4590	LogFlow(("VMX_EXIT_TPR\n"));
4591	/* RIP is already set to the next instruction and the TPR has been synced back. Just resume. */
4592	goto ResumeExecution;
4593
4594	case VMX_EXIT_APIC_ACCESS: /* 44 APIC access. Guest software attempted to access memory at a physical address
4595	on the APIC-access page. */
4596	{
4597	LogFlow(("VMX_EXIT_APIC_ACCESS\n"));
4598	unsigned uAccessType = VMX_EXIT_QUALIFICATION_APIC_ACCESS_TYPE(exitQualification);
4599
4600	switch (uAccessType)
4601	{
4602	case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
4603	case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
4604	{
4605	RTGCPHYS GCPhys;
4606	PDMApicGetBase(pVM, &GCPhys);
4607	GCPhys &= PAGE_BASE_GC_MASK;
4608	GCPhys += VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(exitQualification);
4609
4610	LogFlow(("Apic access at %RGp\n", GCPhys));
4611	rc = IOMMMIOPhysHandler(pVM, (uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ) ? 0 : X86_TRAP_PF_RW,
4612	CPUMCTX2CORE(pCtx), GCPhys);
4613	if (rc == VINF_SUCCESS)
4614	goto ResumeExecution; /* rip already updated */
4615	break;
4616	}
4617
4618	default:
4619	rc = VINF_EM_RAW_EMULATE_INSTR;
4620	break;
4621	}
4622	break;
4623	}
4624
4625	case VMX_EXIT_PREEMPTION_TIMER: /* 52 VMX-preemption timer expired. The preemption timer counted down to zero. */
4626	if (!TMTimerPollBool(pVM, pVCpu))
4627	goto ResumeExecution;
4628	rc = VINF_EM_RAW_TIMER_PENDING;
4629	break;
4630
4631	default:
4632	/* The rest is handled after syncing the entire CPU state. */
4633	break;
4634	}
4635
4636
4637	/*
4638	* Note: The guest state is not entirely synced back at this stage!
4639	*/
4640
4641	/* Investigate why there was a VM-exit. (part 2) */
4642	switch (exitReason)
4643	{
4644	case VMX_EXIT_EXCEPTION: /* 0 Exception or non-maskable interrupt (NMI). */
4645	case VMX_EXIT_EXTERNAL_IRQ: /* 1 External interrupt. */
4646	case VMX_EXIT_EPT_VIOLATION:
4647	case VMX_EXIT_EPT_MISCONFIG: /* 49 EPT misconfig is used by the PGM/MMIO optimizations. */
4648	case VMX_EXIT_PREEMPTION_TIMER: /* 52 VMX-preemption timer expired. The preemption timer counted down to zero. */
4649	/* Already handled above. */
4650	break;
4651
4652	case VMX_EXIT_TRIPLE_FAULT: /* 2 Triple fault. */
4653	rc = VINF_EM_RESET; /* Triple fault equals a reset. */
4654	break;
4655
4656	case VMX_EXIT_INIT_SIGNAL: /* 3 INIT signal. */
4657	case VMX_EXIT_SIPI: /* 4 Start-up IPI (SIPI). */
4658	rc = VINF_EM_RAW_INTERRUPT;
4659	AssertFailed(); /* Can't happen. Yet. */
4660	break;
4661
4662	case VMX_EXIT_IO_SMI_IRQ: /* 5 I/O system-management interrupt (SMI). */
4663	case VMX_EXIT_SMI_IRQ: /* 6 Other SMI. */
4664	rc = VINF_EM_RAW_INTERRUPT;
4665	AssertFailed(); /* Can't happen afaik. */
4666	break;
4667
4668	case VMX_EXIT_TASK_SWITCH: /* 9 Task switch: too complicated to emulate, so fall back to the recompiler */
4669	Log(("VMX_EXIT_TASK_SWITCH: exit=%RX64\n", exitQualification));
4670	if ( (VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE(exitQualification) == VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE_IDT)
4671	&& pVCpu->hwaccm.s.Event.fPending)
4672	{
4673	/* Caused by an injected interrupt. */
4674	pVCpu->hwaccm.s.Event.fPending = false;
4675
4676	Log(("VMX_EXIT_TASK_SWITCH: reassert trap %d\n", VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVCpu->hwaccm.s.Event.intInfo)));
4677	Assert(!VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(pVCpu->hwaccm.s.Event.intInfo));
4678	rc2 = TRPMAssertTrap(pVCpu, VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVCpu->hwaccm.s.Event.intInfo), TRPM_HARDWARE_INT);
4679	AssertRC(rc2);
4680	}
4681	/* else Exceptions and software interrupts can just be restarted. */
4682	rc = VERR_EM_INTERPRETER;
4683	break;
4684
4685	case VMX_EXIT_HLT: /* 12 Guest software attempted to execute HLT. */
4686	/* Check if external interrupts are pending; if so, don't switch back. */
4687	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitHlt);
4688	pCtx->rip++; /* skip hlt */
4689	if (EMShouldContinueAfterHalt(pVCpu, pCtx))
4690	goto ResumeExecution;
4691
4692	rc = VINF_EM_HALT;
4693	break;
4694
4695	case VMX_EXIT_MWAIT: /* 36 Guest software executed MWAIT. */
4696	Log2(("VMX: mwait\n"));
4697	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitMwait);
4698	rc = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pCtx));
4699	if ( rc == VINF_EM_HALT
4700	\|\| rc == VINF_SUCCESS)
4701	{
4702	/* Update EIP and continue execution. */
4703	pCtx->rip += cbInstr;
4704
4705	/* Check if external interrupts are pending; if so, don't switch back. */
4706	if ( rc == VINF_SUCCESS
4707	\|\| ( rc == VINF_EM_HALT
4708	&& EMShouldContinueAfterHalt(pVCpu, pCtx))
4709	)
4710	goto ResumeExecution;
4711	}
4712	AssertMsg(rc == VERR_EM_INTERPRETER \|\| rc == VINF_EM_HALT, ("EMU: mwait failed with %Rrc\n", VBOXSTRICTRC_VAL(rc)));
4713	break;
4714
4715	case VMX_EXIT_RSM: /* 17 Guest software attempted to execute RSM in SMM. */
4716	AssertFailed(); /* can't happen. */
4717	rc = VERR_EM_INTERPRETER;
4718	break;
4719
4720	case VMX_EXIT_MTF: /* 37 Exit due to Monitor Trap Flag. */
4721	LogFlow(("VMX_EXIT_MTF at %RGv\n", (RTGCPTR)pCtx->rip));
4722	pVCpu->hwaccm.s.vmx.proc_ctls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_TRAP_FLAG;
4723	rc2 = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
4724	AssertRC(rc2);
4725	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitMTF);
4726	#if 0
4727	DBGFDoneStepping(pVCpu);
4728	#endif
4729	rc = VINF_EM_DBG_STOP;
4730	break;
4731
4732	case VMX_EXIT_VMCALL: /* 18 Guest software executed VMCALL. */
4733	case VMX_EXIT_VMCLEAR: /* 19 Guest software executed VMCLEAR. */
4734	case VMX_EXIT_VMLAUNCH: /* 20 Guest software executed VMLAUNCH. */
4735	case VMX_EXIT_VMPTRLD: /* 21 Guest software executed VMPTRLD. */
4736	case VMX_EXIT_VMPTRST: /* 22 Guest software executed VMPTRST. */
4737	case VMX_EXIT_VMREAD: /* 23 Guest software executed VMREAD. */
4738	case VMX_EXIT_VMRESUME: /* 24 Guest software executed VMRESUME. */
4739	case VMX_EXIT_VMWRITE: /* 25 Guest software executed VMWRITE. */
4740	case VMX_EXIT_VMXOFF: /* 26 Guest software executed VMXOFF. */
4741	case VMX_EXIT_VMXON: /* 27 Guest software executed VMXON. */
4742	/** @todo inject #UD immediately */
4743	rc = VERR_EM_INTERPRETER;
4744	break;
4745
4746	case VMX_EXIT_CPUID: /* 10 Guest software attempted to execute CPUID. */
4747	case VMX_EXIT_RDTSC: /* 16 Guest software attempted to execute RDTSC. */
4748	case VMX_EXIT_INVLPG: /* 14 Guest software attempted to execute INVLPG. */
4749	case VMX_EXIT_CRX_MOVE: /* 28 Control-register accesses. */
4750	case VMX_EXIT_DRX_MOVE: /* 29 Debug-register accesses. */
4751	case VMX_EXIT_PORT_IO: /* 30 I/O instruction. */
4752	case VMX_EXIT_RDPMC: /* 15 Guest software attempted to execute RDPMC. */
4753	case VMX_EXIT_RDTSCP: /* 51 Guest software attempted to execute RDTSCP. */
4754	/* already handled above */
4755	AssertMsg( rc == VINF_PGM_CHANGE_MODE
4756	\|\| rc == VINF_EM_RAW_INTERRUPT
4757	\|\| rc == VERR_EM_INTERPRETER
4758	\|\| rc == VINF_EM_RAW_EMULATE_INSTR
4759	\|\| rc == VINF_PGM_SYNC_CR3
4760	\|\| rc == VINF_IOM_R3_IOPORT_READ
4761	\|\| rc == VINF_IOM_R3_IOPORT_WRITE
4762	\|\| rc == VINF_EM_RAW_GUEST_TRAP
4763	\|\| rc == VINF_TRPM_XCPT_DISPATCHED
4764	\|\| rc == VINF_EM_RESCHEDULE_REM,
4765	("rc = %d\n", VBOXSTRICTRC_VAL(rc)));
4766	break;
4767
4768	case VMX_EXIT_TPR: /* 43 TPR below threshold. Guest software executed MOV to CR8. */
4769	case VMX_EXIT_RDMSR: /* 31 RDMSR. Guest software attempted to execute RDMSR. */
4770	case VMX_EXIT_WRMSR: /* 32 WRMSR. Guest software attempted to execute WRMSR. */
4771	case VMX_EXIT_PAUSE: /* 40 Guest software attempted to execute PAUSE. */
4772	case VMX_EXIT_MONITOR: /* 39 Guest software attempted to execute MONITOR. */
4773	case VMX_EXIT_APIC_ACCESS: /* 44 APIC access. Guest software attempted to access memory at a physical address
4774	on the APIC-access page. */
4775	{
4776	/*
4777	* If we decided to emulate them here, then we must sync the MSRs that could have been changed (sysenter, FS/GS base)
4778	*/
4779	rc = VERR_EM_INTERPRETER;
4780	break;
4781	}
4782
4783	case VMX_EXIT_IRQ_WINDOW: /* 7 Interrupt window. */
4784	Assert(rc == VINF_EM_RAW_INTERRUPT);
4785	break;
4786
4787	case VMX_EXIT_ERR_INVALID_GUEST_STATE: /* 33 VM-entry failure due to invalid guest state. */
4788	{
4789	#ifdef VBOX_STRICT
4790	RTCCUINTREG val2 = 0;
4791
4792	Log(("VMX_EXIT_ERR_INVALID_GUEST_STATE\n"));
4793
4794	VMXReadVMCS(VMX_VMCS64_GUEST_RIP, &val2);
4795	Log(("Old eip %RGv new %RGv\n", (RTGCPTR)pCtx->rip, (RTGCPTR)val2));
4796
4797	VMXReadVMCS(VMX_VMCS64_GUEST_CR0, &val2);
4798	Log(("VMX_VMCS_GUEST_CR0 %RX64\n", (uint64_t)val2));
4799
4800	VMXReadVMCS(VMX_VMCS64_GUEST_CR3, &val2);
4801	Log(("VMX_VMCS_GUEST_CR3 %RX64\n", (uint64_t)val2));
4802
4803	VMXReadVMCS(VMX_VMCS64_GUEST_CR4, &val2);
4804	Log(("VMX_VMCS_GUEST_CR4 %RX64\n", (uint64_t)val2));
4805
4806	VMXReadVMCS(VMX_VMCS_GUEST_RFLAGS, &val2);
4807	Log(("VMX_VMCS_GUEST_RFLAGS %08x\n", val2));
4808
4809	VMX_LOG_SELREG(CS, "CS", val2);
4810	VMX_LOG_SELREG(DS, "DS", val2);
4811	VMX_LOG_SELREG(ES, "ES", val2);
4812	VMX_LOG_SELREG(FS, "FS", val2);
4813	VMX_LOG_SELREG(GS, "GS", val2);
4814	VMX_LOG_SELREG(SS, "SS", val2);
4815	VMX_LOG_SELREG(TR, "TR", val2);
4816	VMX_LOG_SELREG(LDTR, "LDTR", val2);
4817
4818	VMXReadVMCS(VMX_VMCS64_GUEST_GDTR_BASE, &val2);
4819	Log(("VMX_VMCS_GUEST_GDTR_BASE %RX64\n", (uint64_t)val2));
4820	VMXReadVMCS(VMX_VMCS64_GUEST_IDTR_BASE, &val2);
4821	Log(("VMX_VMCS_GUEST_IDTR_BASE %RX64\n", (uint64_t)val2));
4822	#endif /* VBOX_STRICT */
4823	rc = VERR_VMX_INVALID_GUEST_STATE;
4824	break;
4825	}
4826
4827	case VMX_EXIT_ERR_MSR_LOAD: /* 34 VM-entry failure due to MSR loading. */
4828	case VMX_EXIT_ERR_MACHINE_CHECK: /* 41 VM-entry failure due to machine-check. */
4829	default:
4830	rc = VERR_VMX_UNEXPECTED_EXIT_CODE;
4831	AssertMsgFailed(("Unexpected exit code %d\n", exitReason)); /* Can't happen. */
4832	break;
4833
4834	}
4835
4836	end:
4837	/* We now going back to ring-3, so clear the action flag. */
4838	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
4839
4840	/*
4841	* Signal changes for the recompiler.
4842	*/
4843	CPUMSetChangedFlags(pVCpu,
4844	CPUM_CHANGED_SYSENTER_MSR
4845	\| CPUM_CHANGED_LDTR
4846	\| CPUM_CHANGED_GDTR
4847	\| CPUM_CHANGED_IDTR
4848	\| CPUM_CHANGED_TR
4849	\| CPUM_CHANGED_HIDDEN_SEL_REGS);
4850
4851	/*
4852	* If we executed vmlaunch/vmresume and an external IRQ was pending, then we don't have to do a full sync the next time.
4853	*/
4854	if ( exitReason == VMX_EXIT_EXTERNAL_IRQ
4855	&& !VMX_EXIT_INTERRUPTION_INFO_VALID(intInfo))
4856	{
4857	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatPendingHostIrq);
4858	/* On the next entry we'll only sync the host context. */
4859	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_HOST_CONTEXT;
4860	}
4861	else
4862	{
4863	/* On the next entry we'll sync everything. */
4864	/** @todo we can do better than this */
4865	/* Not in the VINF_PGM_CHANGE_MODE though! */
4866	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_ALL;
4867	}
4868
4869	/* Translate into a less severe return code */
4870	if (rc == VERR_EM_INTERPRETER)
4871	rc = VINF_EM_RAW_EMULATE_INSTR;
4872	else if (rc == VERR_VMX_INVALID_VMCS_PTR)
4873	{
4874	/* Try to extract more information about what might have gone wrong here. */
4875	VMXGetActivateVMCS(&pVCpu->hwaccm.s.vmx.lasterror.u64VMCSPhys);
4876	pVCpu->hwaccm.s.vmx.lasterror.ulVMCSRevision = (uint32_t )pVCpu->hwaccm.s.vmx.pvVMCS;
4877	pVCpu->hwaccm.s.vmx.lasterror.idEnteredCpu = pVCpu->hwaccm.s.idEnteredCpu;
4878	pVCpu->hwaccm.s.vmx.lasterror.idCurrentCpu = RTMpCpuId();
4879	}
4880
4881	/* Just set the correct state here instead of trying to catch every goto above. */
4882	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED, VMCPUSTATE_STARTED_EXEC);
4883
4884	#ifdef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
4885	/* Restore interrupts if we exited after disabling them. */
4886	if (uOldEFlags != ~(RTCCUINTREG)0)
4887	ASMSetFlags(uOldEFlags);
4888	#endif
4889
4890	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2, x);
4891	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit1, x);
4892	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatEntry, x);
4893	Log2(("X"));
4894	return VBOXSTRICTRC_TODO(rc);
4895	}
4896
4897
4898	/**
4899	* Enters the VT-x session.
4900	*
4901	* @returns VBox status code.
4902	* @param pVM Pointer to the VM.
4903	* @param pVCpu Pointer to the VMCPU.
4904	* @param pCpu Pointer to the CPU info struct.
4905	*/
4906	VMMR0DECL(int) VMXR0Enter(PVM pVM, PVMCPU pVCpu, PHMGLOBLCPUINFO pCpu)
4907	{
4908	Assert(pVM->hwaccm.s.vmx.fSupported);
4909	NOREF(pCpu);
4910
4911	unsigned cr4 = ASMGetCR4();
4912	if (!(cr4 & X86_CR4_VMXE))
4913	{
4914	AssertMsgFailed(("X86_CR4_VMXE should be set!\n"));
4915	return VERR_VMX_X86_CR4_VMXE_CLEARED;
4916	}
4917
4918	/* Activate the VMCS. */
4919	int rc = VMXActivateVMCS(pVCpu->hwaccm.s.vmx.HCPhysVMCS);
4920	if (RT_FAILURE(rc))
4921	return rc;
4922
4923	pVCpu->hwaccm.s.fResumeVM = false;
4924	return VINF_SUCCESS;
4925	}
4926
4927
4928	/**
4929	* Leaves the VT-x session.
4930	*
4931	* @returns VBox status code.
4932	* @param pVM Pointer to the VM.
4933	* @param pVCpu Pointer to the VMCPU.
4934	* @param pCtx Pointer to the guests CPU context.
4935	*/
4936	VMMR0DECL(int) VMXR0Leave(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
4937	{
4938	Assert(pVM->hwaccm.s.vmx.fSupported);
4939
4940	#ifdef DEBUG
4941	if (CPUMIsHyperDebugStateActive(pVCpu))
4942	{
4943	CPUMR0LoadHostDebugState(pVM, pVCpu);
4944	Assert(pVCpu->hwaccm.s.vmx.proc_ctls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT);
4945	}
4946	else
4947	#endif
4948
4949	/*
4950	* Save the guest debug state if necessary.
4951	*/
4952	if (CPUMIsGuestDebugStateActive(pVCpu))
4953	{
4954	CPUMR0SaveGuestDebugState(pVM, pVCpu, pCtx, true /* save DR6 */);
4955
4956	/* Enable DRx move intercepts again. */
4957	pVCpu->hwaccm.s.vmx.proc_ctls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT;
4958	int rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
4959	AssertRC(rc);
4960
4961	/* Resync the debug registers the next time. */
4962	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_GUEST_DEBUG;
4963	}
4964	else
4965	Assert(pVCpu->hwaccm.s.vmx.proc_ctls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT);
4966
4967	/*
4968	* Clear VMCS, marking it inactive, clearing implementation-specific data and writing
4969	* VMCS data back to memory.
4970	*/
4971	int rc = VMXClearVMCS(pVCpu->hwaccm.s.vmx.HCPhysVMCS);
4972	AssertRC(rc);
4973
4974	return VINF_SUCCESS;
4975	}
4976
4977
4978	/**
4979	* Flush the TLB using EPT.
4980	*
4981	* @returns VBox status code.
4982	* @param pVM Pointer to the VM.
4983	* @param pVCpu Pointer to the VMCPU.
4984	* @param enmFlush Type of flush.
4985	*/
4986	static void hmR0VmxFlushEPT(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_EPT enmFlush)
4987	{
4988	uint64_t descriptor[2];
4989
4990	LogFlow(("hmR0VmxFlushEPT %d\n", enmFlush));
4991	Assert(pVM->hwaccm.s.fNestedPaging);
4992	descriptor[0] = pVCpu->hwaccm.s.vmx.GCPhysEPTP;
4993	descriptor[1] = 0; /* MBZ. Intel spec. 33.3 VMX Instructions */
4994	int rc = VMXR0InvEPT(enmFlush, &descriptor[0]);
4995	AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %x %RGv failed with %d\n", enmFlush, pVCpu->hwaccm.s.vmx.GCPhysEPTP, rc));
4996	}
4997
4998
4999	/**
5000	* Flush the TLB using VPID.
5001	*
5002	* @returns VBox status code.
5003	* @param pVM Pointer to the VM.
5004	* @param pVCpu Pointer to the VMCPU (can be NULL depending on @a
5005	* enmFlush).
5006	* @param enmFlush Type of flush.
5007	* @param GCPtr Virtual address of the page to flush (can be 0 depending
5008	* on @a enmFlush).
5009	*/
5010	static void hmR0VmxFlushVPID(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_VPID enmFlush, RTGCPTR GCPtr)
5011	{
5012	uint64_t descriptor[2];
5013
5014	Assert(pVM->hwaccm.s.vmx.fVPID);
5015	if (enmFlush == VMX_FLUSH_VPID_ALL_CONTEXTS)
5016	{
5017	descriptor[0] = 0;
5018	descriptor[1] = 0;
5019	}
5020	else
5021	{
5022	AssertPtr(pVCpu);
5023	AssertMsg(pVCpu->hwaccm.s.uCurrentASID != 0, ("VMXR0InvVPID invalid ASID %lu\n", pVCpu->hwaccm.s.uCurrentASID));
5024	AssertMsg(pVCpu->hwaccm.s.uCurrentASID <= UINT16_MAX, ("VMXR0InvVPID invalid ASID %lu\n", pVCpu->hwaccm.s.uCurrentASID));
5025	descriptor[0] = pVCpu->hwaccm.s.uCurrentASID;
5026	descriptor[1] = GCPtr;
5027	}
5028	int rc = VMXR0InvVPID(enmFlush, &descriptor[0]); NOREF(rc);
5029	AssertMsg(rc == VINF_SUCCESS,
5030	("VMXR0InvVPID %x %x %RGv failed with %d\n", enmFlush, pVCpu ? pVCpu->hwaccm.s.uCurrentASID : 0, GCPtr, rc));
5031	}
5032
5033
5034	/**
5035	* Invalidates a guest page by guest virtual address. Only relevant for
5036	* EPT/VPID, otherwise there is nothing really to invalidate.
5037	*
5038	* @returns VBox status code.
5039	* @param pVM Pointer to the VM.
5040	* @param pVCpu Pointer to the VMCPU.
5041	* @param GCVirt Guest virtual address of the page to invalidate.
5042	*/
5043	VMMR0DECL(int) VMXR0InvalidatePage(PVM pVM, PVMCPU pVCpu, RTGCPTR GCVirt)
5044	{
5045	bool fFlushPending = VMCPU_FF_ISSET(pVCpu, VMCPU_FF_TLB_FLUSH);
5046
5047	Log2(("VMXR0InvalidatePage %RGv\n", GCVirt));
5048
5049	if (!fFlushPending)
5050	{
5051	/*
5052	* We must invalidate the guest TLB entry in either case, we cannot ignore it even for the EPT case
5053	* See @bugref{6043} and @bugref{6177}
5054	*
5055	* Set the VMCPU_FF_TLB_FLUSH force flag and flush before VMENTRY in hmR0VmxSetupTLB*() as this
5056	* function maybe called in a loop with individual addresses.
5057	*/
5058	if (pVM->hwaccm.s.vmx.fVPID)
5059	{
5060	/* If we can flush just this page do it, otherwise flush as little as possible. */
5061	if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVVPID_CAPS_INDIV_ADDR)
5062	hmR0VmxFlushVPID(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, GCVirt);
5063	else
5064	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
5065	}
5066	else if (pVM->hwaccm.s.fNestedPaging)
5067	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
5068	}
5069
5070	return VINF_SUCCESS;
5071	}
5072
5073
5074	/**
5075	* Invalidates a guest page by physical address. Only relevant for EPT/VPID,
5076	* otherwise there is nothing really to invalidate.
5077	*
5078	* NOTE: Assumes the current instruction references this physical page though a virtual address!!
5079	*
5080	* @returns VBox status code.
5081	* @param pVM Pointer to the VM.
5082	* @param pVCpu Pointer to the VMCPU.
5083	* @param GCPhys Guest physical address of the page to invalidate.
5084	*/
5085	VMMR0DECL(int) VMXR0InvalidatePhysPage(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys)
5086	{
5087	LogFlow(("VMXR0InvalidatePhysPage %RGp\n", GCPhys));
5088
5089	/*
5090	* We cannot flush a page by guest-physical address. invvpid takes only a linear address
5091	* while invept only flushes by EPT not individual addresses. We update the force flag here
5092	* and flush before VMENTRY in hmR0VmxSetupTLB*(). This function might be called in a loop.
5093	*/
5094	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
5095	return VINF_SUCCESS;
5096	}
5097
5098
5099	/**
5100	* Report world switch error and dump some useful debug info.
5101	*
5102	* @param pVM Pointer to the VM.
5103	* @param pVCpu Pointer to the VMCPU.
5104	* @param rc Return code.
5105	* @param pCtx Pointer to the current guest CPU context (not updated).
5106	*/
5107	static void hmR0VmxReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, VBOXSTRICTRC rc, PCPUMCTX pCtx)
5108	{
5109	NOREF(pVM);
5110
5111	switch (VBOXSTRICTRC_VAL(rc))
5112	{
5113	case VERR_VMX_INVALID_VMXON_PTR:
5114	AssertFailed();
5115	break;
5116
5117	case VERR_VMX_UNABLE_TO_START_VM:
5118	case VERR_VMX_UNABLE_TO_RESUME_VM:
5119	{
5120	int rc2;
5121	RTCCUINTREG exitReason, instrError;
5122
5123	rc2 = VMXReadVMCS(VMX_VMCS32_RO_EXIT_REASON, &exitReason);
5124	rc2 \|= VMXReadVMCS(VMX_VMCS32_RO_VM_INSTR_ERROR, &instrError);
5125	AssertRC(rc2);
5126	if (rc2 == VINF_SUCCESS)
5127	{
5128	Log(("Unable to start/resume VM for reason: %x. Instruction error %x\n", (uint32_t)exitReason,
5129	(uint32_t)instrError));
5130	Log(("Current stack %08x\n", &rc2));
5131
5132	pVCpu->hwaccm.s.vmx.lasterror.ulInstrError = instrError;
5133	pVCpu->hwaccm.s.vmx.lasterror.ulExitReason = exitReason;
5134
5135	#ifdef VBOX_STRICT
5136	RTGDTR gdtr;
5137	PCX86DESCHC pDesc;
5138	RTCCUINTREG val;
5139
5140	ASMGetGDTR(&gdtr);
5141
5142	VMXReadVMCS(VMX_VMCS64_GUEST_RIP, &val);
5143	Log(("Old eip %RGv new %RGv\n", (RTGCPTR)pCtx->rip, (RTGCPTR)val));
5144	VMXReadVMCS(VMX_VMCS_CTRL_PIN_EXEC_CONTROLS, &val);
5145	Log(("VMX_VMCS_CTRL_PIN_EXEC_CONTROLS %08x\n", val));
5146	VMXReadVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, &val);
5147	Log(("VMX_VMCS_CTRL_PROC_EXEC_CONTROLS %08x\n", val));
5148	VMXReadVMCS(VMX_VMCS_CTRL_ENTRY_CONTROLS, &val);
5149	Log(("VMX_VMCS_CTRL_ENTRY_CONTROLS %08x\n", val));
5150	VMXReadVMCS(VMX_VMCS_CTRL_EXIT_CONTROLS, &val);
5151	Log(("VMX_VMCS_CTRL_EXIT_CONTROLS %08x\n", val));
5152
5153	VMXReadVMCS(VMX_VMCS_HOST_CR0, &val);
5154	Log(("VMX_VMCS_HOST_CR0 %08x\n", val));
5155	VMXReadVMCS(VMX_VMCS_HOST_CR3, &val);
5156	Log(("VMX_VMCS_HOST_CR3 %08x\n", val));
5157	VMXReadVMCS(VMX_VMCS_HOST_CR4, &val);
5158	Log(("VMX_VMCS_HOST_CR4 %08x\n", val));
5159
5160	VMXReadVMCS(VMX_VMCS16_HOST_FIELD_CS, &val);
5161	Log(("VMX_VMCS_HOST_FIELD_CS %08x\n", val));
5162	VMXReadVMCS(VMX_VMCS_GUEST_RFLAGS, &val);
5163	Log(("VMX_VMCS_GUEST_RFLAGS %08x\n", val));
5164
5165	if (val < gdtr.cbGdt)
5166	{
5167	pDesc = (PCX86DESCHC)(gdtr.pGdt + (val & X86_SEL_MASK));
5168	HWACCMR0DumpDescriptor(pDesc, val, "CS: ");
5169	}
5170
5171	VMXReadVMCS(VMX_VMCS16_HOST_FIELD_DS, &val);
5172	Log(("VMX_VMCS_HOST_FIELD_DS %08x\n", val));
5173	if (val < gdtr.cbGdt)
5174	{
5175	pDesc = (PCX86DESCHC)(gdtr.pGdt + (val & X86_SEL_MASK));
5176	HWACCMR0DumpDescriptor(pDesc, val, "DS: ");
5177	}
5178
5179	VMXReadVMCS(VMX_VMCS16_HOST_FIELD_ES, &val);
5180	Log(("VMX_VMCS_HOST_FIELD_ES %08x\n", val));
5181	if (val < gdtr.cbGdt)
5182	{
5183	pDesc = (PCX86DESCHC)(gdtr.pGdt + (val & X86_SEL_MASK));
5184	HWACCMR0DumpDescriptor(pDesc, val, "ES: ");
5185	}
5186
5187	VMXReadVMCS(VMX_VMCS16_HOST_FIELD_FS, &val);
5188	Log(("VMX_VMCS16_HOST_FIELD_FS %08x\n", val));
5189	if (val < gdtr.cbGdt)
5190	{
5191	pDesc = (PCX86DESCHC)(gdtr.pGdt + (val & X86_SEL_MASK));
5192	HWACCMR0DumpDescriptor(pDesc, val, "FS: ");
5193	}
5194
5195	VMXReadVMCS(VMX_VMCS16_HOST_FIELD_GS, &val);
5196	Log(("VMX_VMCS16_HOST_FIELD_GS %08x\n", val));
5197	if (val < gdtr.cbGdt)
5198	{
5199	pDesc = (PCX86DESCHC)(gdtr.pGdt + (val & X86_SEL_MASK));
5200	HWACCMR0DumpDescriptor(pDesc, val, "GS: ");
5201	}
5202
5203	VMXReadVMCS(VMX_VMCS16_HOST_FIELD_SS, &val);
5204	Log(("VMX_VMCS16_HOST_FIELD_SS %08x\n", val));
5205	if (val < gdtr.cbGdt)
5206	{
5207	pDesc = (PCX86DESCHC)(gdtr.pGdt + (val & X86_SEL_MASK));
5208	HWACCMR0DumpDescriptor(pDesc, val, "SS: ");
5209	}
5210
5211	VMXReadVMCS(VMX_VMCS16_HOST_FIELD_TR, &val);
5212	Log(("VMX_VMCS16_HOST_FIELD_TR %08x\n", val));
5213	if (val < gdtr.cbGdt)
5214	{
5215	pDesc = (PCX86DESCHC)(gdtr.pGdt + (val & X86_SEL_MASK));
5216	HWACCMR0DumpDescriptor(pDesc, val, "TR: ");
5217	}
5218
5219	VMXReadVMCS(VMX_VMCS_HOST_TR_BASE, &val);
5220	Log(("VMX_VMCS_HOST_TR_BASE %RHv\n", val));
5221	VMXReadVMCS(VMX_VMCS_HOST_GDTR_BASE, &val);
5222	Log(("VMX_VMCS_HOST_GDTR_BASE %RHv\n", val));
5223	VMXReadVMCS(VMX_VMCS_HOST_IDTR_BASE, &val);
5224	Log(("VMX_VMCS_HOST_IDTR_BASE %RHv\n", val));
5225	VMXReadVMCS(VMX_VMCS32_HOST_SYSENTER_CS, &val);
5226	Log(("VMX_VMCS_HOST_SYSENTER_CS %08x\n", val));
5227	VMXReadVMCS(VMX_VMCS_HOST_SYSENTER_EIP, &val);
5228	Log(("VMX_VMCS_HOST_SYSENTER_EIP %RHv\n", val));
5229	VMXReadVMCS(VMX_VMCS_HOST_SYSENTER_ESP, &val);
5230	Log(("VMX_VMCS_HOST_SYSENTER_ESP %RHv\n", val));
5231	VMXReadVMCS(VMX_VMCS_HOST_RSP, &val);
5232	Log(("VMX_VMCS_HOST_RSP %RHv\n", val));
5233	VMXReadVMCS(VMX_VMCS_HOST_RIP, &val);
5234	Log(("VMX_VMCS_HOST_RIP %RHv\n", val));
5235	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
5236	if (VMX_IS_64BIT_HOST_MODE())
5237	{
5238	Log(("MSR_K6_EFER = %RX64\n", ASMRdMsr(MSR_K6_EFER)));
5239	Log(("MSR_K6_STAR = %RX64\n", ASMRdMsr(MSR_K6_STAR)));
5240	Log(("MSR_K8_LSTAR = %RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
5241	Log(("MSR_K8_CSTAR = %RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
5242	Log(("MSR_K8_SF_MASK = %RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
5243	Log(("MSR_K8_KERNEL_GS_BASE = %RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
5244	}
5245	# endif
5246	#endif /* VBOX_STRICT */
5247	}
5248	break;
5249	}
5250
5251	default:
5252	/* impossible */
5253	AssertMsgFailed(("%Rrc (%#x)\n", VBOXSTRICTRC_VAL(rc), VBOXSTRICTRC_VAL(rc)));
5254	break;
5255	}
5256	}
5257
5258
5259	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
5260	/**
5261	* Prepares for and executes VMLAUNCH (64 bits guest mode).
5262	*
5263	* @returns VBox status code.
5264	* @param fResume Whether to vmlauch/vmresume.
5265	* @param pCtx Pointer to the guest CPU context.
5266	* @param pCache Pointer to the VMCS cache.
5267	* @param pVM Pointer to the VM.
5268	* @param pVCpu Pointer to the VMCPU.
5269	*/
5270	DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
5271	{
5272	uint32_t aParam[6];
5273	PHMGLOBLCPUINFO pCpu;
5274	RTHCPHYS HCPhysCpuPage;
5275	int rc;
5276
5277	pCpu = HWACCMR0GetCurrentCpu();
5278	HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
5279
5280	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5281	pCache->uPos = 1;
5282	pCache->interPD = PGMGetInterPaeCR3(pVM);
5283	pCache->pSwitcher = (uint64_t)pVM->hwaccm.s.pfnHost32ToGuest64R0;
5284	#endif
5285
5286	#ifdef DEBUG
5287	pCache->TestIn.HCPhysCpuPage= 0;
5288	pCache->TestIn.HCPhysVMCS = 0;
5289	pCache->TestIn.pCache = 0;
5290	pCache->TestOut.HCPhysVMCS = 0;
5291	pCache->TestOut.pCache = 0;
5292	pCache->TestOut.pCtx = 0;
5293	pCache->TestOut.eflags = 0;
5294	#endif
5295
5296	aParam[0] = (uint32_t)(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
5297	aParam[1] = (uint32_t)(HCPhysCpuPage >> 32); /* Param 1: VMXON physical address - Hi. */
5298	aParam[2] = (uint32_t)(pVCpu->hwaccm.s.vmx.HCPhysVMCS); /* Param 2: VMCS physical address - Lo. */
5299	aParam[3] = (uint32_t)(pVCpu->hwaccm.s.vmx.HCPhysVMCS >> 32); /* Param 2: VMCS physical address - Hi. */
5300	aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hwaccm.s.vmx.VMCSCache);
5301	aParam[5] = 0;
5302
5303	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5304	pCtx->dr[4] = pVM->hwaccm.s.vmx.pScratchPhys + 16 + 8;
5305	(uint32_t )(pVM->hwaccm.s.vmx.pScratch + 16 + 8) = 1;
5306	#endif
5307	rc = VMXR0Execute64BitsHandler(pVM, pVCpu, pCtx, pVM->hwaccm.s.pfnVMXGCStartVM64, 6, &aParam[0]);
5308
5309	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5310	Assert((uint32_t )(pVM->hwaccm.s.vmx.pScratch + 16 + 8) == 5);
5311	Assert(pCtx->dr[4] == 10);
5312	(uint32_t )(pVM->hwaccm.s.vmx.pScratch + 16 + 8) = 0xff;
5313	#endif
5314
5315	#ifdef DEBUG
5316	AssertMsg(pCache->TestIn.HCPhysCpuPage== HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
5317	AssertMsg(pCache->TestIn.HCPhysVMCS == pVCpu->hwaccm.s.vmx.HCPhysVMCS, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVMCS,
5318	pVCpu->hwaccm.s.vmx.HCPhysVMCS));
5319	AssertMsg(pCache->TestIn.HCPhysVMCS == pCache->TestOut.HCPhysVMCS, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVMCS,
5320	pCache->TestOut.HCPhysVMCS));
5321	AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
5322	pCache->TestOut.pCache));
5323	AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hwaccm.s.vmx.VMCSCache),
5324	("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hwaccm.s.vmx.VMCSCache)));
5325	AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
5326	pCache->TestOut.pCtx));
5327	Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
5328	#endif
5329	return rc;
5330	}
5331
5332
5333	# ifdef VBOX_STRICT
5334	static bool hmR0VmxIsValidReadField(uint32_t idxField)
5335	{
5336	switch (idxField)
5337	{
5338	case VMX_VMCS64_GUEST_RIP:
5339	case VMX_VMCS64_GUEST_RSP:
5340	case VMX_VMCS_GUEST_RFLAGS:
5341	case VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE:
5342	case VMX_VMCS_CTRL_CR0_READ_SHADOW:
5343	case VMX_VMCS64_GUEST_CR0:
5344	case VMX_VMCS_CTRL_CR4_READ_SHADOW:
5345	case VMX_VMCS64_GUEST_CR4:
5346	case VMX_VMCS64_GUEST_DR7:
5347	case VMX_VMCS32_GUEST_SYSENTER_CS:
5348	case VMX_VMCS64_GUEST_SYSENTER_EIP:
5349	case VMX_VMCS64_GUEST_SYSENTER_ESP:
5350	case VMX_VMCS32_GUEST_GDTR_LIMIT:
5351	case VMX_VMCS64_GUEST_GDTR_BASE:
5352	case VMX_VMCS32_GUEST_IDTR_LIMIT:
5353	case VMX_VMCS64_GUEST_IDTR_BASE:
5354	case VMX_VMCS16_GUEST_FIELD_CS:
5355	case VMX_VMCS32_GUEST_CS_LIMIT:
5356	case VMX_VMCS64_GUEST_CS_BASE:
5357	case VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS:
5358	case VMX_VMCS16_GUEST_FIELD_DS:
5359	case VMX_VMCS32_GUEST_DS_LIMIT:
5360	case VMX_VMCS64_GUEST_DS_BASE:
5361	case VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS:
5362	case VMX_VMCS16_GUEST_FIELD_ES:
5363	case VMX_VMCS32_GUEST_ES_LIMIT:
5364	case VMX_VMCS64_GUEST_ES_BASE:
5365	case VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS:
5366	case VMX_VMCS16_GUEST_FIELD_FS:
5367	case VMX_VMCS32_GUEST_FS_LIMIT:
5368	case VMX_VMCS64_GUEST_FS_BASE:
5369	case VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS:
5370	case VMX_VMCS16_GUEST_FIELD_GS:
5371	case VMX_VMCS32_GUEST_GS_LIMIT:
5372	case VMX_VMCS64_GUEST_GS_BASE:
5373	case VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS:
5374	case VMX_VMCS16_GUEST_FIELD_SS:
5375	case VMX_VMCS32_GUEST_SS_LIMIT:
5376	case VMX_VMCS64_GUEST_SS_BASE:
5377	case VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS:
5378	case VMX_VMCS16_GUEST_FIELD_LDTR:
5379	case VMX_VMCS32_GUEST_LDTR_LIMIT:
5380	case VMX_VMCS64_GUEST_LDTR_BASE:
5381	case VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS:
5382	case VMX_VMCS16_GUEST_FIELD_TR:
5383	case VMX_VMCS32_GUEST_TR_LIMIT:
5384	case VMX_VMCS64_GUEST_TR_BASE:
5385	case VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS:
5386	case VMX_VMCS32_RO_EXIT_REASON:
5387	case VMX_VMCS32_RO_VM_INSTR_ERROR:
5388	case VMX_VMCS32_RO_EXIT_INSTR_LENGTH:
5389	case VMX_VMCS32_RO_EXIT_INTERRUPTION_ERRCODE:
5390	case VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO:
5391	case VMX_VMCS32_RO_EXIT_INSTR_INFO:
5392	case VMX_VMCS_RO_EXIT_QUALIFICATION:
5393	case VMX_VMCS32_RO_IDT_INFO:
5394	case VMX_VMCS32_RO_IDT_ERRCODE:
5395	case VMX_VMCS64_GUEST_CR3:
5396	case VMX_VMCS_EXIT_PHYS_ADDR_FULL:
5397	return true;
5398	}
5399	return false;
5400	}
5401
5402
5403	static bool hmR0VmxIsValidWriteField(uint32_t idxField)
5404	{
5405	switch (idxField)
5406	{
5407	case VMX_VMCS64_GUEST_LDTR_BASE:
5408	case VMX_VMCS64_GUEST_TR_BASE:
5409	case VMX_VMCS64_GUEST_GDTR_BASE:
5410	case VMX_VMCS64_GUEST_IDTR_BASE:
5411	case VMX_VMCS64_GUEST_SYSENTER_EIP:
5412	case VMX_VMCS64_GUEST_SYSENTER_ESP:
5413	case VMX_VMCS64_GUEST_CR0:
5414	case VMX_VMCS64_GUEST_CR4:
5415	case VMX_VMCS64_GUEST_CR3:
5416	case VMX_VMCS64_GUEST_DR7:
5417	case VMX_VMCS64_GUEST_RIP:
5418	case VMX_VMCS64_GUEST_RSP:
5419	case VMX_VMCS64_GUEST_CS_BASE:
5420	case VMX_VMCS64_GUEST_DS_BASE:
5421	case VMX_VMCS64_GUEST_ES_BASE:
5422	case VMX_VMCS64_GUEST_FS_BASE:
5423	case VMX_VMCS64_GUEST_GS_BASE:
5424	case VMX_VMCS64_GUEST_SS_BASE:
5425	return true;
5426	}
5427	return false;
5428	}
5429	# endif /* VBOX_STRICT */
5430
5431
5432	/**
5433	* Executes the specified handler in 64-bit mode.
5434	*
5435	* @returns VBox status code.
5436	* @param pVM Pointer to the VM.
5437	* @param pVCpu Pointer to the VMCPU.
5438	* @param pCtx Pointer to the guest CPU context.
5439	* @param pfnHandler Pointer to the RC handler function.
5440	* @param cbParam Number of parameters.
5441	* @param paParam Array of 32-bit parameters.
5442	*/
5443	VMMR0DECL(int) VMXR0Execute64BitsHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, RTRCPTR pfnHandler, uint32_t cbParam,
5444	uint32_t *paParam)
5445	{
5446	int rc, rc2;
5447	PHMGLOBLCPUINFO pCpu;
5448	RTHCPHYS HCPhysCpuPage;
5449	RTHCUINTREG uOldEFlags;
5450
5451	AssertReturn(pVM->hwaccm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
5452	Assert(pfnHandler);
5453	Assert(pVCpu->hwaccm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hwaccm.s.vmx.VMCSCache.Write.aField));
5454	Assert(pVCpu->hwaccm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hwaccm.s.vmx.VMCSCache.Read.aField));
5455
5456	#ifdef VBOX_STRICT
5457	for (unsigned i=0;i<pVCpu->hwaccm.s.vmx.VMCSCache.Write.cValidEntries;i++)
5458	Assert(hmR0VmxIsValidWriteField(pVCpu->hwaccm.s.vmx.VMCSCache.Write.aField[i]));
5459
5460	for (unsigned i=0;i<pVCpu->hwaccm.s.vmx.VMCSCache.Read.cValidEntries;i++)
5461	Assert(hmR0VmxIsValidReadField(pVCpu->hwaccm.s.vmx.VMCSCache.Read.aField[i]));
5462	#endif
5463
5464	/* Disable interrupts. */
5465	uOldEFlags = ASMIntDisableFlags();
5466
5467	#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
5468	RTCPUID idHostCpu = RTMpCpuId();
5469	CPUMR0SetLApic(pVM, idHostCpu);
5470	#endif
5471
5472	pCpu = HWACCMR0GetCurrentCpu();
5473	HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
5474
5475	/* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
5476	VMXClearVMCS(pVCpu->hwaccm.s.vmx.HCPhysVMCS);
5477
5478	/* Leave VMX Root Mode. */
5479	VMXDisable();
5480
5481	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
5482
5483	CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
5484	CPUMSetHyperEIP(pVCpu, pfnHandler);
5485	for (int i=(int)cbParam-1;i>=0;i--)
5486	CPUMPushHyper(pVCpu, paParam[i]);
5487
5488	STAM_PROFILE_ADV_START(&pVCpu->hwaccm.s.StatWorldSwitch3264, z);
5489
5490	/* Call switcher. */
5491	rc = pVM->hwaccm.s.pfnHost32ToGuest64R0(pVM, RT_OFFSETOF(VM, aCpus[pVCpu->idCpu].cpum) - RT_OFFSETOF(VM, cpum));
5492	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatWorldSwitch3264, z);
5493
5494	/* Make sure the VMX instructions don't cause #UD faults. */
5495	ASMSetCR4(ASMGetCR4() \| X86_CR4_VMXE);
5496
5497	/* Enter VMX Root Mode */
5498	rc2 = VMXEnable(HCPhysCpuPage);
5499	if (RT_FAILURE(rc2))
5500	{
5501	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
5502	ASMSetFlags(uOldEFlags);
5503	return VERR_VMX_VMXON_FAILED;
5504	}
5505
5506	rc2 = VMXActivateVMCS(pVCpu->hwaccm.s.vmx.HCPhysVMCS);
5507	AssertRC(rc2);
5508	Assert(!(ASMGetFlags() & X86_EFL_IF));
5509	ASMSetFlags(uOldEFlags);
5510	return rc;
5511	}
5512	#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL) */
5513
5514
5515	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
5516	/**
5517	* Executes VMWRITE.
5518	*
5519	* @returns VBox status code
5520	* @param pVCpu Pointer to the VMCPU.
5521	* @param idxField VMCS field index.
5522	* @param u64Val 16, 32 or 64 bits value.
5523	*/
5524	VMMR0DECL(int) VMXWriteVMCS64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5525	{
5526	int rc;
5527	switch (idxField)
5528	{
5529	case VMX_VMCS_CTRL_TSC_OFFSET_FULL:
5530	case VMX_VMCS_CTRL_IO_BITMAP_A_FULL:
5531	case VMX_VMCS_CTRL_IO_BITMAP_B_FULL:
5532	case VMX_VMCS_CTRL_MSR_BITMAP_FULL:
5533	case VMX_VMCS_CTRL_VMEXIT_MSR_STORE_FULL:
5534	case VMX_VMCS_CTRL_VMEXIT_MSR_LOAD_FULL:
5535	case VMX_VMCS_CTRL_VMENTRY_MSR_LOAD_FULL:
5536	case VMX_VMCS_CTRL_VAPIC_PAGEADDR_FULL:
5537	case VMX_VMCS_CTRL_APIC_ACCESSADDR_FULL:
5538	case VMX_VMCS_GUEST_LINK_PTR_FULL:
5539	case VMX_VMCS_GUEST_PDPTR0_FULL:
5540	case VMX_VMCS_GUEST_PDPTR1_FULL:
5541	case VMX_VMCS_GUEST_PDPTR2_FULL:
5542	case VMX_VMCS_GUEST_PDPTR3_FULL:
5543	case VMX_VMCS_GUEST_DEBUGCTL_FULL:
5544	case VMX_VMCS_GUEST_EFER_FULL:
5545	case VMX_VMCS_CTRL_EPTP_FULL:
5546	/* These fields consist of two parts, which are both writable in 32 bits mode. */
5547	rc = VMXWriteVMCS32(idxField, u64Val);
5548	rc \|= VMXWriteVMCS32(idxField + 1, (uint32_t)(u64Val >> 32ULL));
5549	AssertRC(rc);
5550	return rc;
5551
5552	case VMX_VMCS64_GUEST_LDTR_BASE:
5553	case VMX_VMCS64_GUEST_TR_BASE:
5554	case VMX_VMCS64_GUEST_GDTR_BASE:
5555	case VMX_VMCS64_GUEST_IDTR_BASE:
5556	case VMX_VMCS64_GUEST_SYSENTER_EIP:
5557	case VMX_VMCS64_GUEST_SYSENTER_ESP:
5558	case VMX_VMCS64_GUEST_CR0:
5559	case VMX_VMCS64_GUEST_CR4:
5560	case VMX_VMCS64_GUEST_CR3:
5561	case VMX_VMCS64_GUEST_DR7:
5562	case VMX_VMCS64_GUEST_RIP:
5563	case VMX_VMCS64_GUEST_RSP:
5564	case VMX_VMCS64_GUEST_CS_BASE:
5565	case VMX_VMCS64_GUEST_DS_BASE:
5566	case VMX_VMCS64_GUEST_ES_BASE:
5567	case VMX_VMCS64_GUEST_FS_BASE:
5568	case VMX_VMCS64_GUEST_GS_BASE:
5569	case VMX_VMCS64_GUEST_SS_BASE:
5570	/* Queue a 64 bits value as we can't set it in 32 bits host mode. */
5571	if (u64Val >> 32ULL)
5572	rc = VMXWriteCachedVMCSEx(pVCpu, idxField, u64Val);
5573	else
5574	rc = VMXWriteVMCS32(idxField, (uint32_t)u64Val);
5575
5576	return rc;
5577
5578	default:
5579	AssertMsgFailed(("Unexpected field %x\n", idxField));
5580	return VERR_INVALID_PARAMETER;
5581	}
5582	}
5583
5584
5585	/**
5586	* Cache VMCS writes for performance reasons (Darwin) and for running 64 bits guests on 32 bits hosts.
5587	*
5588	* @param pVCpu Pointer to the VMCPU.
5589	* @param idxField VMCS field index.
5590	* @param u64Val 16, 32 or 64 bits value.
5591	*/
5592	VMMR0DECL(int) VMXWriteCachedVMCSEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5593	{
5594	PVMCSCACHE pCache = &pVCpu->hwaccm.s.vmx.VMCSCache;
5595
5596	AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1,
5597	("entries=%x\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
5598
5599	/* Make sure there are no duplicates. */
5600	for (unsigned i = 0; i < pCache->Write.cValidEntries; i++)
5601	{
5602	if (pCache->Write.aField[i] == idxField)
5603	{
5604	pCache->Write.aFieldVal[i] = u64Val;
5605	return VINF_SUCCESS;
5606	}
5607	}
5608
5609	pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
5610	pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
5611	pCache->Write.cValidEntries++;
5612	return VINF_SUCCESS;
5613	}
5614
5615	#endif /* HC_ARCH_BITS == 32 && !VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0 */
5616

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMR0/HWVMXR0.cpp@ 43303

Download in other formats: