HWVMXR0.cpp@ 43354

Last change on this file since 43354 was 43354, checked in by vboxsync, 12 years ago
VMM/HWVMXR0: Don't create executable mappings for VMCS and friends.
Property svn:eol-style set to `native` Property svn:keywords set to `Id Revision`
File size: 221.8 KB

Line
1	/* $Id: HWVMXR0.cpp 43354 2012-09-18 15:25:32Z vboxsync $ */
2	/** @file
3	* HM VMX (VT-x) - Host Context Ring-0.
4	*/
5
6	/*
7	* Copyright (C) 2006-2012 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18
19	/*******************************************************************************
20	* Header Files *
21	*******************************************************************************/
22	#define LOG_GROUP LOG_GROUP_HWACCM
23	#include <iprt/asm-amd64-x86.h>
24	#include <VBox/vmm/hwaccm.h>
25	#include <VBox/vmm/pgm.h>
26	#include <VBox/vmm/dbgf.h>
27	#include <VBox/vmm/dbgftrace.h>
28	#include <VBox/vmm/selm.h>
29	#include <VBox/vmm/iom.h>
30	#ifdef VBOX_WITH_REM
31	# include <VBox/vmm/rem.h>
32	#endif
33	#include <VBox/vmm/tm.h>
34	#include "HWACCMInternal.h"
35	#include <VBox/vmm/vm.h>
36	#include <VBox/vmm/pdmapi.h>
37	#include <VBox/err.h>
38	#include <VBox/log.h>
39	#include <iprt/assert.h>
40	#include <iprt/param.h>
41	#include <iprt/string.h>
42	#include <iprt/time.h>
43	#ifdef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
44	# include <iprt/thread.h>
45	#endif
46	#include <iprt/x86.h>
47	#include "HWVMXR0.h"
48
49	#include "dtrace/VBoxVMM.h"
50
51
52	/*******************************************************************************
53	* Defined Constants And Macros *
54	*******************************************************************************/
55	#if defined(RT_ARCH_AMD64)
56	# define VMX_IS_64BIT_HOST_MODE() (true)
57	#elif defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
58	# define VMX_IS_64BIT_HOST_MODE() (g_fVMXIs64bitHost != 0)
59	#else
60	# define VMX_IS_64BIT_HOST_MODE() (false)
61	#endif
62
63
64	/*******************************************************************************
65	* Global Variables *
66	*******************************************************************************/
67	/* IO operation lookup arrays. */
68	static uint32_t const g_aIOSize[4] = {1, 2, 0, 4};
69	static uint32_t const g_aIOOpAnd[4] = {0xff, 0xffff, 0, 0xffffffff};
70
71	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
72	/** See HWACCMR0A.asm. */
73	extern "C" uint32_t g_fVMXIs64bitHost;
74	#endif
75
76
77	/*******************************************************************************
78	* Local Functions *
79	*******************************************************************************/
80	static DECLCALLBACK(void) hmR0VmxSetupTLBEPT(PVM pVM, PVMCPU pVCpu);
81	static DECLCALLBACK(void) hmR0VmxSetupTLBVPID(PVM pVM, PVMCPU pVCpu);
82	static DECLCALLBACK(void) hmR0VmxSetupTLBBoth(PVM pVM, PVMCPU pVCpu);
83	static DECLCALLBACK(void) hmR0VmxSetupTLBDummy(PVM pVM, PVMCPU pVCpu);
84	static void hmR0VmxFlushEPT(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_EPT enmFlush);
85	static void hmR0VmxFlushVPID(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_VPID enmFlush, RTGCPTR GCPtr);
86	static void hmR0VmxUpdateExceptionBitmap(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx);
87	static void hmR0VmxSetMSRPermission(PVMCPU pVCpu, unsigned ulMSR, bool fRead, bool fWrite);
88	static void hmR0VmxReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, VBOXSTRICTRC rc, PCPUMCTX pCtx);
89
90
91	/**
92	* Updates error from VMCS to HWACCMCPU's lasterror record.
93	*
94	* @param pVM Pointer to the VM.
95	* @param pVCpu Pointer to the VMCPU.
96	* @param rc The error code.
97	*/
98	static void hmR0VmxCheckError(PVM pVM, PVMCPU pVCpu, int rc)
99	{
100	if (rc == VERR_VMX_GENERIC)
101	{
102	RTCCUINTREG instrError;
103
104	VMXReadVMCS(VMX_VMCS32_RO_VM_INSTR_ERROR, &instrError);
105	pVCpu->hwaccm.s.vmx.lasterror.ulInstrError = instrError;
106	}
107	pVM->hwaccm.s.lLastError = rc;
108	}
109
110
111	/**
112	* Sets up and activates VT-x on the current CPU.
113	*
114	* @returns VBox status code.
115	* @param pCpu Pointer to the CPU info struct.
116	* @param pVM Pointer to the VM. (can be NULL after a resume!!)
117	* @param pvCpuPage Pointer to the global CPU page.
118	* @param HCPhysCpuPage Physical address of the global CPU page.
119	* @param fEnabledByHost Set if SUPR0EnableVTx or similar was used to enable
120	* VT-x/AMD-V on the host.
121	*/
122	VMMR0DECL(int) VMXR0EnableCpu(PHMGLOBLCPUINFO pCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost)
123	{
124	if (!fEnabledByHost)
125	{
126	AssertReturn(HCPhysCpuPage != 0 && HCPhysCpuPage != NIL_RTHCPHYS, VERR_INVALID_PARAMETER);
127	AssertReturn(pvCpuPage, VERR_INVALID_PARAMETER);
128
129	if (pVM)
130	{
131	/* Set revision dword at the beginning of the VMXON structure. */
132	(uint32_t )pvCpuPage = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hwaccm.s.vmx.msr.vmx_basic_info);
133	}
134
135	/** @todo we should unmap the two pages from the virtual address space in order to prevent accidental corruption.
136	* (which can have very bad consequences!!!)
137	*/
138
139	if (ASMGetCR4() & X86_CR4_VMXE)
140	return VERR_VMX_IN_VMX_ROOT_MODE;
141
142	ASMSetCR4(ASMGetCR4() \| X86_CR4_VMXE); /* Make sure the VMX instructions don't cause #UD faults. */
143
144	/*
145	* Enter VM root mode.
146	*/
147	int rc = VMXEnable(HCPhysCpuPage);
148	if (RT_FAILURE(rc))
149	{
150	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
151	return VERR_VMX_VMXON_FAILED;
152	}
153	}
154
155	/*
156	* Flush all VPIDs (in case we or any other hypervisor have been using VPIDs) so that
157	* we can avoid an explicit flush while using new VPIDs. We would still need to flush
158	* each time while reusing a VPID after hitting the MaxASID limit once.
159	*/
160	if ( pVM
161	&& pVM->hwaccm.s.vmx.fVPID
162	&& (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVVPID_CAPS_ALL_CONTEXTS))
163	{
164	hmR0VmxFlushVPID(pVM, NULL /* pvCpu /, VMX_FLUSH_VPID_ALL_CONTEXTS, 0 / GCPtr */);
165	pCpu->fFlushASIDBeforeUse = false;
166	}
167	else
168	pCpu->fFlushASIDBeforeUse = true;
169
170	/*
171	* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}.
172	*/
173	++pCpu->cTLBFlushes;
174
175	return VINF_SUCCESS;
176	}
177
178
179	/**
180	* Deactivates VT-x on the current CPU.
181	*
182	* @returns VBox status code.
183	* @param pCpu Pointer to the CPU info struct.
184	* @param pvCpuPage Pointer to the global CPU page.
185	* @param HCPhysCpuPage Physical address of the global CPU page.
186	*/
187	VMMR0DECL(int) VMXR0DisableCpu(PHMGLOBLCPUINFO pCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
188	{
189	AssertReturn(HCPhysCpuPage != 0 && HCPhysCpuPage != NIL_RTHCPHYS, VERR_INVALID_PARAMETER);
190	AssertReturn(pvCpuPage, VERR_INVALID_PARAMETER);
191	NOREF(pCpu);
192
193	/* If we're somehow not in VMX root mode, then we shouldn't dare leaving it. */
194	if (!(ASMGetCR4() & X86_CR4_VMXE))
195	return VERR_VMX_NOT_IN_VMX_ROOT_MODE;
196
197	/* Leave VMX Root Mode. */
198	VMXDisable();
199
200	/* And clear the X86_CR4_VMXE bit. */
201	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
202	return VINF_SUCCESS;
203	}
204
205
206	/**
207	* Does Ring-0 per VM VT-x initialization.
208	*
209	* @returns VBox status code.
210	* @param pVM Pointer to the VM.
211	*/
212	VMMR0DECL(int) VMXR0InitVM(PVM pVM)
213	{
214	int rc;
215
216	#ifdef LOG_ENABLED
217	SUPR0Printf("VMXR0InitVM %p\n", pVM);
218	#endif
219
220	pVM->hwaccm.s.vmx.pMemObjAPIC = NIL_RTR0MEMOBJ;
221
222	if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
223	{
224	/* Allocate one page for the APIC physical page (serves for filtering accesses). */
225	rc = RTR0MemObjAllocCont(&pVM->hwaccm.s.vmx.pMemObjAPIC, PAGE_SIZE, false /* executable R0 mapping */);
226	AssertRC(rc);
227	if (RT_FAILURE(rc))
228	return rc;
229
230	pVM->hwaccm.s.vmx.pAPIC = (uint8_t *)RTR0MemObjAddress(pVM->hwaccm.s.vmx.pMemObjAPIC);
231	pVM->hwaccm.s.vmx.pAPICPhys = RTR0MemObjGetPagePhysAddr(pVM->hwaccm.s.vmx.pMemObjAPIC, 0);
232	ASMMemZero32(pVM->hwaccm.s.vmx.pAPIC, PAGE_SIZE);
233	}
234	else
235	{
236	pVM->hwaccm.s.vmx.pMemObjAPIC = 0;
237	pVM->hwaccm.s.vmx.pAPIC = 0;
238	pVM->hwaccm.s.vmx.pAPICPhys = 0;
239	}
240
241	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
242	{
243	rc = RTR0MemObjAllocCont(&pVM->hwaccm.s.vmx.pMemObjScratch, PAGE_SIZE, false /* executable R0 mapping */);
244	AssertRC(rc);
245	if (RT_FAILURE(rc))
246	return rc;
247
248	pVM->hwaccm.s.vmx.pScratch = (uint8_t *)RTR0MemObjAddress(pVM->hwaccm.s.vmx.pMemObjScratch);
249	pVM->hwaccm.s.vmx.pScratchPhys = RTR0MemObjGetPagePhysAddr(pVM->hwaccm.s.vmx.pMemObjScratch, 0);
250
251	ASMMemZero32(pVM->hwaccm.s.vmx.pScratch, PAGE_SIZE);
252	strcpy((char *)pVM->hwaccm.s.vmx.pScratch, "SCRATCH Magic");
253	(uint64_t )(pVM->hwaccm.s.vmx.pScratch + 16) = UINT64_C(0xDEADBEEFDEADBEEF);
254	}
255	#endif
256
257	/* Allocate VMCSs for all guest CPUs. */
258	for (VMCPUID i = 0; i < pVM->cCpus; i++)
259	{
260	PVMCPU pVCpu = &pVM->aCpus[i];
261
262	pVCpu->hwaccm.s.vmx.hMemObjVMCS = NIL_RTR0MEMOBJ;
263
264	/* Allocate one page for the VM control structure (VMCS). */
265	rc = RTR0MemObjAllocCont(&pVCpu->hwaccm.s.vmx.hMemObjVMCS, PAGE_SIZE, false /* executable R0 mapping */);
266	AssertRC(rc);
267	if (RT_FAILURE(rc))
268	return rc;
269
270	pVCpu->hwaccm.s.vmx.pvVMCS = RTR0MemObjAddress(pVCpu->hwaccm.s.vmx.hMemObjVMCS);
271	pVCpu->hwaccm.s.vmx.HCPhysVMCS = RTR0MemObjGetPagePhysAddr(pVCpu->hwaccm.s.vmx.hMemObjVMCS, 0);
272	ASMMemZeroPage(pVCpu->hwaccm.s.vmx.pvVMCS);
273
274	pVCpu->hwaccm.s.vmx.cr0_mask = 0;
275	pVCpu->hwaccm.s.vmx.cr4_mask = 0;
276
277	/* Allocate one page for the virtual APIC page for TPR caching. */
278	rc = RTR0MemObjAllocCont(&pVCpu->hwaccm.s.vmx.hMemObjVAPIC, PAGE_SIZE, false /* executable R0 mapping */);
279	AssertRC(rc);
280	if (RT_FAILURE(rc))
281	return rc;
282
283	pVCpu->hwaccm.s.vmx.pbVAPIC = (uint8_t *)RTR0MemObjAddress(pVCpu->hwaccm.s.vmx.hMemObjVAPIC);
284	pVCpu->hwaccm.s.vmx.HCPhysVAPIC = RTR0MemObjGetPagePhysAddr(pVCpu->hwaccm.s.vmx.hMemObjVAPIC, 0);
285	ASMMemZeroPage(pVCpu->hwaccm.s.vmx.pbVAPIC);
286
287	/* Allocate the MSR bitmap if this feature is supported. */
288	if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
289	{
290	rc = RTR0MemObjAllocCont(&pVCpu->hwaccm.s.vmx.pMemObjMSRBitmap, PAGE_SIZE, false /* executable R0 mapping */);
291	AssertRC(rc);
292	if (RT_FAILURE(rc))
293	return rc;
294
295	pVCpu->hwaccm.s.vmx.pMSRBitmap = (uint8_t *)RTR0MemObjAddress(pVCpu->hwaccm.s.vmx.pMemObjMSRBitmap);
296	pVCpu->hwaccm.s.vmx.pMSRBitmapPhys = RTR0MemObjGetPagePhysAddr(pVCpu->hwaccm.s.vmx.pMemObjMSRBitmap, 0);
297	memset(pVCpu->hwaccm.s.vmx.pMSRBitmap, 0xff, PAGE_SIZE);
298	}
299
300	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
301	/* Allocate one page for the guest MSR load area (for preloading guest MSRs during the world switch). */
302	rc = RTR0MemObjAllocCont(&pVCpu->hwaccm.s.vmx.pMemObjGuestMSR, PAGE_SIZE, false /* executable R0 mapping */);
303	AssertRC(rc);
304	if (RT_FAILURE(rc))
305	return rc;
306
307	pVCpu->hwaccm.s.vmx.pGuestMSR = (uint8_t *)RTR0MemObjAddress(pVCpu->hwaccm.s.vmx.pMemObjGuestMSR);
308	pVCpu->hwaccm.s.vmx.pGuestMSRPhys = RTR0MemObjGetPagePhysAddr(pVCpu->hwaccm.s.vmx.pMemObjGuestMSR, 0);
309	Assert(!(pVCpu->hwaccm.s.vmx.pGuestMSRPhys & 0xf));
310	memset(pVCpu->hwaccm.s.vmx.pGuestMSR, 0, PAGE_SIZE);
311
312	/* Allocate one page for the host MSR load area (for restoring host MSRs after the world switch back). */
313	rc = RTR0MemObjAllocCont(&pVCpu->hwaccm.s.vmx.pMemObjHostMSR, PAGE_SIZE, false /* executable R0 mapping */);
314	AssertRC(rc);
315	if (RT_FAILURE(rc))
316	return rc;
317
318	pVCpu->hwaccm.s.vmx.pHostMSR = (uint8_t *)RTR0MemObjAddress(pVCpu->hwaccm.s.vmx.pMemObjHostMSR);
319	pVCpu->hwaccm.s.vmx.pHostMSRPhys = RTR0MemObjGetPagePhysAddr(pVCpu->hwaccm.s.vmx.pMemObjHostMSR, 0);
320	Assert(!(pVCpu->hwaccm.s.vmx.pHostMSRPhys & 0xf));
321	memset(pVCpu->hwaccm.s.vmx.pHostMSR, 0, PAGE_SIZE);
322	#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
323
324	/* Current guest paging mode. */
325	pVCpu->hwaccm.s.vmx.enmLastSeenGuestMode = PGMMODE_REAL;
326
327	#ifdef LOG_ENABLED
328	SUPR0Printf("VMXR0InitVM %x VMCS=%x (%x)\n", pVM, pVCpu->hwaccm.s.vmx.pvVMCS, (uint32_t)pVCpu->hwaccm.s.vmx.HCPhysVMCS);
329	#endif
330	}
331
332	return VINF_SUCCESS;
333	}
334
335
336	/**
337	* Does Ring-0 per VM VT-x termination.
338	*
339	* @returns VBox status code.
340	* @param pVM Pointer to the VM.
341	*/
342	VMMR0DECL(int) VMXR0TermVM(PVM pVM)
343	{
344	for (VMCPUID i = 0; i < pVM->cCpus; i++)
345	{
346	PVMCPU pVCpu = &pVM->aCpus[i];
347
348	if (pVCpu->hwaccm.s.vmx.hMemObjVMCS != NIL_RTR0MEMOBJ)
349	{
350	RTR0MemObjFree(pVCpu->hwaccm.s.vmx.hMemObjVMCS, false);
351	pVCpu->hwaccm.s.vmx.hMemObjVMCS = NIL_RTR0MEMOBJ;
352	pVCpu->hwaccm.s.vmx.pvVMCS = 0;
353	pVCpu->hwaccm.s.vmx.HCPhysVMCS = 0;
354	}
355	if (pVCpu->hwaccm.s.vmx.hMemObjVAPIC != NIL_RTR0MEMOBJ)
356	{
357	RTR0MemObjFree(pVCpu->hwaccm.s.vmx.hMemObjVAPIC, false);
358	pVCpu->hwaccm.s.vmx.hMemObjVAPIC = NIL_RTR0MEMOBJ;
359	pVCpu->hwaccm.s.vmx.pbVAPIC = 0;
360	pVCpu->hwaccm.s.vmx.HCPhysVAPIC = 0;
361	}
362	if (pVCpu->hwaccm.s.vmx.pMemObjMSRBitmap != NIL_RTR0MEMOBJ)
363	{
364	RTR0MemObjFree(pVCpu->hwaccm.s.vmx.pMemObjMSRBitmap, false);
365	pVCpu->hwaccm.s.vmx.pMemObjMSRBitmap = NIL_RTR0MEMOBJ;
366	pVCpu->hwaccm.s.vmx.pMSRBitmap = 0;
367	pVCpu->hwaccm.s.vmx.pMSRBitmapPhys = 0;
368	}
369	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
370	if (pVCpu->hwaccm.s.vmx.pMemObjHostMSR != NIL_RTR0MEMOBJ)
371	{
372	RTR0MemObjFree(pVCpu->hwaccm.s.vmx.pMemObjHostMSR, false);
373	pVCpu->hwaccm.s.vmx.pMemObjHostMSR = NIL_RTR0MEMOBJ;
374	pVCpu->hwaccm.s.vmx.pHostMSR = 0;
375	pVCpu->hwaccm.s.vmx.pHostMSRPhys = 0;
376	}
377	if (pVCpu->hwaccm.s.vmx.pMemObjGuestMSR != NIL_RTR0MEMOBJ)
378	{
379	RTR0MemObjFree(pVCpu->hwaccm.s.vmx.pMemObjGuestMSR, false);
380	pVCpu->hwaccm.s.vmx.pMemObjGuestMSR = NIL_RTR0MEMOBJ;
381	pVCpu->hwaccm.s.vmx.pGuestMSR = 0;
382	pVCpu->hwaccm.s.vmx.pGuestMSRPhys = 0;
383	}
384	#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
385	}
386	if (pVM->hwaccm.s.vmx.pMemObjAPIC != NIL_RTR0MEMOBJ)
387	{
388	RTR0MemObjFree(pVM->hwaccm.s.vmx.pMemObjAPIC, false);
389	pVM->hwaccm.s.vmx.pMemObjAPIC = NIL_RTR0MEMOBJ;
390	pVM->hwaccm.s.vmx.pAPIC = 0;
391	pVM->hwaccm.s.vmx.pAPICPhys = 0;
392	}
393	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
394	if (pVM->hwaccm.s.vmx.pMemObjScratch != NIL_RTR0MEMOBJ)
395	{
396	ASMMemZero32(pVM->hwaccm.s.vmx.pScratch, PAGE_SIZE);
397	RTR0MemObjFree(pVM->hwaccm.s.vmx.pMemObjScratch, false);
398	pVM->hwaccm.s.vmx.pMemObjScratch = NIL_RTR0MEMOBJ;
399	pVM->hwaccm.s.vmx.pScratch = 0;
400	pVM->hwaccm.s.vmx.pScratchPhys = 0;
401	}
402	#endif
403	return VINF_SUCCESS;
404	}
405
406
407	/**
408	* Sets up VT-x for the specified VM.
409	*
410	* @returns VBox status code.
411	* @param pVM Pointer to the VM.
412	*/
413	VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
414	{
415	int rc = VINF_SUCCESS;
416	uint32_t val;
417
418	AssertReturn(pVM, VERR_INVALID_PARAMETER);
419
420	/* Initialize these always, see hwaccmR3InitFinalizeR0().*/
421	pVM->hwaccm.s.vmx.enmFlushEPT = VMX_FLUSH_EPT_NONE;
422	pVM->hwaccm.s.vmx.enmFlushVPID = VMX_FLUSH_VPID_NONE;
423
424	/* Determine optimal flush type for EPT. */
425	if (pVM->hwaccm.s.fNestedPaging)
426	{
427	if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVEPT)
428	{
429	if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVEPT_CAPS_SINGLE_CONTEXT)
430	pVM->hwaccm.s.vmx.enmFlushEPT = VMX_FLUSH_EPT_SINGLE_CONTEXT;
431	else if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVEPT_CAPS_ALL_CONTEXTS)
432	pVM->hwaccm.s.vmx.enmFlushEPT = VMX_FLUSH_EPT_ALL_CONTEXTS;
433	else
434	{
435	/*
436	* Should never really happen. EPT is supported but no suitable flush types supported.
437	* We cannot ignore EPT at this point as we've already setup Unrestricted Guest execution.
438	*/
439	pVM->hwaccm.s.vmx.enmFlushEPT = VMX_FLUSH_EPT_NOT_SUPPORTED;
440	return VERR_VMX_GENERIC;
441	}
442	}
443	else
444	{
445	/*
446	* Should never really happen. EPT is supported but INVEPT instruction is not supported.
447	*/
448	pVM->hwaccm.s.vmx.enmFlushEPT = VMX_FLUSH_EPT_NOT_SUPPORTED;
449	return VERR_VMX_GENERIC;
450	}
451	}
452
453	/* Determine optimal flush type for VPID. */
454	if (pVM->hwaccm.s.vmx.fVPID)
455	{
456	if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVVPID)
457	{
458	if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVVPID_CAPS_SINGLE_CONTEXT)
459	pVM->hwaccm.s.vmx.enmFlushVPID = VMX_FLUSH_VPID_SINGLE_CONTEXT;
460	else if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVVPID_CAPS_ALL_CONTEXTS)
461	pVM->hwaccm.s.vmx.enmFlushVPID = VMX_FLUSH_VPID_ALL_CONTEXTS;
462	else
463	{
464	/*
465	* Neither SINGLE nor ALL context flush types for VPID supported by the CPU.
466	* We do not handle other flush type combinations, ignore VPID capabilities.
467	*/
468	if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVVPID_CAPS_INDIV_ADDR)
469	Log(("VMXR0SetupVM: Only VMX_FLUSH_VPID_INDIV_ADDR supported. Ignoring VPID.\n"));
470	if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVVPID_CAPS_SINGLE_CONTEXT_RETAIN_GLOBALS)
471	Log(("VMXR0SetupVM: Only VMX_FLUSH_VPID_SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
472	pVM->hwaccm.s.vmx.enmFlushVPID = VMX_FLUSH_VPID_NOT_SUPPORTED;
473	pVM->hwaccm.s.vmx.fVPID = false;
474	}
475	}
476	else
477	{
478	/*
479	* Should not really happen. EPT is supported but INVEPT is not supported.
480	* Ignore VPID capabilities as our code relies on using INVEPT for selective flushing.
481	*/
482	Log(("VMXR0SetupVM: VPID supported without INVEPT support. Ignoring VPID.\n"));
483	pVM->hwaccm.s.vmx.enmFlushVPID = VMX_FLUSH_VPID_NOT_SUPPORTED;
484	pVM->hwaccm.s.vmx.fVPID = false;
485	}
486	}
487
488	for (VMCPUID i = 0; i < pVM->cCpus; i++)
489	{
490	PVMCPU pVCpu = &pVM->aCpus[i];
491
492	AssertPtr(pVCpu->hwaccm.s.vmx.pvVMCS);
493
494	/* Set revision dword at the beginning of the VMCS structure. */
495	(uint32_t )pVCpu->hwaccm.s.vmx.pvVMCS = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hwaccm.s.vmx.msr.vmx_basic_info);
496
497	/*
498	* Clear and activate the VMCS.
499	*/
500	Log(("HCPhysVMCS = %RHp\n", pVCpu->hwaccm.s.vmx.HCPhysVMCS));
501	rc = VMXClearVMCS(pVCpu->hwaccm.s.vmx.HCPhysVMCS);
502	if (RT_FAILURE(rc))
503	goto vmx_end;
504
505	rc = VMXActivateVMCS(pVCpu->hwaccm.s.vmx.HCPhysVMCS);
506	if (RT_FAILURE(rc))
507	goto vmx_end;
508
509	/*
510	* VMX_VMCS_CTRL_PIN_EXEC_CONTROLS
511	* Set required bits to one and zero according to the MSR capabilities.
512	*/
513	val = pVM->hwaccm.s.vmx.msr.vmx_pin_ctls.n.disallowed0;
514	val \|= VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_EXT_INT_EXIT /* External interrupts */
515	\| VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_NMI_EXIT; /* Non-maskable interrupts */
516
517	/*
518	* Enable the VMX preemption timer.
519	*/
520	if (pVM->hwaccm.s.vmx.fUsePreemptTimer)
521	val \|= VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_PREEMPT_TIMER;
522	val &= pVM->hwaccm.s.vmx.msr.vmx_pin_ctls.n.allowed1;
523
524	rc = VMXWriteVMCS(VMX_VMCS_CTRL_PIN_EXEC_CONTROLS, val);
525	AssertRC(rc);
526
527	/*
528	* VMX_VMCS_CTRL_PROC_EXEC_CONTROLS
529	* Set required bits to one and zero according to the MSR capabilities.
530	*/
531	val = pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.disallowed0;
532	/* Program which event cause VM-exits and which features we want to use. */
533	val \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_HLT_EXIT
534	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_TSC_OFFSET
535	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT
536	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_UNCOND_IO_EXIT
537	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDPMC_EXIT
538	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_EXIT
539	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MWAIT_EXIT; /* don't execute mwait or else we'll idle inside
540	the guest (host thinks the cpu load is high) */
541
542	/* Without nested paging we should intercept invlpg and cr3 mov instructions. */
543	if (!pVM->hwaccm.s.fNestedPaging)
544	{
545	val \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INVLPG_EXIT
546	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
547	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT;
548	}
549
550	/*
551	* VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MWAIT_EXIT might cause a vmlaunch
552	* failure with an invalid control fields error. (combined with some other exit reasons)
553	*/
554	if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
555	{
556	/* CR8 reads from the APIC shadow page; writes cause an exit is they lower the TPR below the threshold */
557	val \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW;
558	Assert(pVM->hwaccm.s.vmx.pAPIC);
559	}
560	else
561	/* Exit on CR8 reads & writes in case the TPR shadow feature isn't present. */
562	val \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR8_STORE_EXIT \| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR8_LOAD_EXIT;
563
564	if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
565	{
566	Assert(pVCpu->hwaccm.s.vmx.pMSRBitmapPhys);
567	val \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS;
568	}
569
570	/* We will use the secondary control if it's present. */
571	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
572
573	/* Mask away the bits that the CPU doesn't support */
574	/** @todo make sure they don't conflict with the above requirements. */
575	val &= pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1;
576	pVCpu->hwaccm.s.vmx.proc_ctls = val;
577
578	rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, val);
579	AssertRC(rc);
580
581	if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
582	{
583	/*
584	* VMX_VMCS_CTRL_PROC_EXEC_CONTROLS2
585	* Set required bits to one and zero according to the MSR capabilities.
586	*/
587	val = pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2.n.disallowed0;
588	val \|= VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT;
589
590	if (pVM->hwaccm.s.fNestedPaging)
591	val \|= VMX_VMCS_CTRL_PROC_EXEC2_EPT;
592
593	if (pVM->hwaccm.s.vmx.fVPID)
594	val \|= VMX_VMCS_CTRL_PROC_EXEC2_VPID;
595
596	if (pVM->hwaccm.s.fHasIoApic)
597	val \|= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC;
598
599	if (pVM->hwaccm.s.vmx.fUnrestrictedGuest)
600	val \|= VMX_VMCS_CTRL_PROC_EXEC2_REAL_MODE;
601
602	if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
603	val \|= VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP;
604
605	/* Mask away the bits that the CPU doesn't support */
606	/** @todo make sure they don't conflict with the above requirements. */
607	val &= pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2.n.allowed1;
608	pVCpu->hwaccm.s.vmx.proc_ctls2 = val;
609	rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS2, val);
610	AssertRC(rc);
611	}
612
613	/*
614	* VMX_VMCS_CTRL_CR3_TARGET_COUNT
615	* Set required bits to one and zero according to the MSR capabilities.
616	*/
617	rc = VMXWriteVMCS(VMX_VMCS_CTRL_CR3_TARGET_COUNT, 0);
618	AssertRC(rc);
619
620	/*
621	* Forward all exception except #NM & #PF to the guest.
622	* We always need to check pagefaults since our shadow page table can be out of sync.
623	* And we always lazily sync the FPU & XMM state. .
624	*/
625
626	/** @todo Possible optimization:
627	* Keep the FPU and XMM state current in the EM thread. That way there's no need to
628	* lazily sync anything, but the downside is that we can't use the FPU stack or XMM
629	* registers ourselves of course.
630	*
631	* Note: only possible if the current state is actually ours (X86_CR0_TS flag)
632	*/
633
634	/*
635	* Don't filter page faults, all of them should cause a world switch.
636	*/
637	rc = VMXWriteVMCS(VMX_VMCS_CTRL_PAGEFAULT_ERROR_MASK, 0);
638	AssertRC(rc);
639	rc = VMXWriteVMCS(VMX_VMCS_CTRL_PAGEFAULT_ERROR_MATCH, 0);
640	AssertRC(rc);
641
642	rc = VMXWriteVMCS64(VMX_VMCS_CTRL_TSC_OFFSET_FULL, 0);
643	AssertRC(rc);
644	rc = VMXWriteVMCS64(VMX_VMCS_CTRL_IO_BITMAP_A_FULL, 0);
645	AssertRC(rc);
646	rc = VMXWriteVMCS64(VMX_VMCS_CTRL_IO_BITMAP_B_FULL, 0);
647	AssertRC(rc);
648
649	/*
650	* Set the MSR bitmap address.
651	*/
652	if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
653	{
654	Assert(pVCpu->hwaccm.s.vmx.pMSRBitmapPhys);
655
656	rc = VMXWriteVMCS64(VMX_VMCS_CTRL_MSR_BITMAP_FULL, pVCpu->hwaccm.s.vmx.pMSRBitmapPhys);
657	AssertRC(rc);
658
659	/*
660	* Allow the guest to directly modify these MSRs; they are loaded/stored automatically
661	* using MSR-load/store areas in the VMCS.
662	*/
663	hmR0VmxSetMSRPermission(pVCpu, MSR_IA32_SYSENTER_CS, true, true);
664	hmR0VmxSetMSRPermission(pVCpu, MSR_IA32_SYSENTER_ESP, true, true);
665	hmR0VmxSetMSRPermission(pVCpu, MSR_IA32_SYSENTER_EIP, true, true);
666	hmR0VmxSetMSRPermission(pVCpu, MSR_K8_LSTAR, true, true);
667	hmR0VmxSetMSRPermission(pVCpu, MSR_K6_STAR, true, true);
668	hmR0VmxSetMSRPermission(pVCpu, MSR_K8_SF_MASK, true, true);
669	hmR0VmxSetMSRPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, true, true);
670	hmR0VmxSetMSRPermission(pVCpu, MSR_K8_GS_BASE, true, true);
671	hmR0VmxSetMSRPermission(pVCpu, MSR_K8_FS_BASE, true, true);
672	if (pVCpu->hwaccm.s.vmx.proc_ctls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
673	hmR0VmxSetMSRPermission(pVCpu, MSR_K8_TSC_AUX, true, true);
674	}
675
676	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
677	/*
678	* Set the guest & host MSR load/store physical addresses.
679	*/
680	Assert(pVCpu->hwaccm.s.vmx.pGuestMSRPhys);
681	rc = VMXWriteVMCS64(VMX_VMCS_CTRL_VMENTRY_MSR_LOAD_FULL, pVCpu->hwaccm.s.vmx.pGuestMSRPhys);
682	AssertRC(rc);
683	rc = VMXWriteVMCS64(VMX_VMCS_CTRL_VMEXIT_MSR_STORE_FULL, pVCpu->hwaccm.s.vmx.pGuestMSRPhys);
684	AssertRC(rc);
685	Assert(pVCpu->hwaccm.s.vmx.pHostMSRPhys);
686	rc = VMXWriteVMCS64(VMX_VMCS_CTRL_VMEXIT_MSR_LOAD_FULL, pVCpu->hwaccm.s.vmx.pHostMSRPhys);
687	AssertRC(rc);
688	#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
689
690	rc = VMXWriteVMCS(VMX_VMCS_CTRL_ENTRY_MSR_LOAD_COUNT, 0);
691	AssertRC(rc);
692	rc = VMXWriteVMCS(VMX_VMCS_CTRL_EXIT_MSR_STORE_COUNT, 0);
693	AssertRC(rc);
694	rc = VMXWriteVMCS(VMX_VMCS_CTRL_EXIT_MSR_LOAD_COUNT, 0);
695	AssertRC(rc);
696
697	if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
698	{
699	Assert(pVM->hwaccm.s.vmx.pMemObjAPIC);
700	/* Optional */
701	rc = VMXWriteVMCS(VMX_VMCS_CTRL_TPR_THRESHOLD, 0);
702	rc \|= VMXWriteVMCS64(VMX_VMCS_CTRL_VAPIC_PAGEADDR_FULL, pVCpu->hwaccm.s.vmx.HCPhysVAPIC);
703
704	if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
705	rc \|= VMXWriteVMCS64(VMX_VMCS_CTRL_APIC_ACCESSADDR_FULL, pVM->hwaccm.s.vmx.pAPICPhys);
706
707	AssertRC(rc);
708	}
709
710	/* Set link pointer to -1. Not currently used. */
711	rc = VMXWriteVMCS64(VMX_VMCS_GUEST_LINK_PTR_FULL, 0xFFFFFFFFFFFFFFFFULL);
712	AssertRC(rc);
713
714	/*
715	* Clear VMCS, marking it inactive. Clear implementation specific data and writing back
716	* VMCS data back to memory.
717	*/
718	rc = VMXClearVMCS(pVCpu->hwaccm.s.vmx.HCPhysVMCS);
719	AssertRC(rc);
720
721	/*
722	* Configure the VMCS read cache.
723	*/
724	PVMCSCACHE pCache = &pVCpu->hwaccm.s.vmx.VMCSCache;
725
726	VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_RIP);
727	VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_RSP);
728	VMXSetupCachedReadVMCS(pCache, VMX_VMCS_GUEST_RFLAGS);
729	VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE);
730	VMXSetupCachedReadVMCS(pCache, VMX_VMCS_CTRL_CR0_READ_SHADOW);
731	VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_CR0);
732	VMXSetupCachedReadVMCS(pCache, VMX_VMCS_CTRL_CR4_READ_SHADOW);
733	VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_CR4);
734	VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_DR7);
735	VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_GUEST_SYSENTER_CS);
736	VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_SYSENTER_EIP);
737	VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_SYSENTER_ESP);
738	VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_GUEST_GDTR_LIMIT);
739	VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_GDTR_BASE);
740	VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_GUEST_IDTR_LIMIT);
741	VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_IDTR_BASE);
742
743	VMX_SETUP_SELREG(ES, pCache);
744	VMX_SETUP_SELREG(SS, pCache);
745	VMX_SETUP_SELREG(CS, pCache);
746	VMX_SETUP_SELREG(DS, pCache);
747	VMX_SETUP_SELREG(FS, pCache);
748	VMX_SETUP_SELREG(GS, pCache);
749	VMX_SETUP_SELREG(LDTR, pCache);
750	VMX_SETUP_SELREG(TR, pCache);
751
752	/*
753	* Status code VMCS reads.
754	*/
755	VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_EXIT_REASON);
756	VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_VM_INSTR_ERROR);
757	VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_EXIT_INSTR_LENGTH);
758	VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_EXIT_INTERRUPTION_ERRCODE);
759	VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO);
760	VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_EXIT_INSTR_INFO);
761	VMXSetupCachedReadVMCS(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
762	VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_IDT_INFO);
763	VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_IDT_ERRCODE);
764
765	if (pVM->hwaccm.s.fNestedPaging)
766	{
767	VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_CR3);
768	VMXSetupCachedReadVMCS(pCache, VMX_VMCS_EXIT_PHYS_ADDR_FULL);
769	pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
770	}
771	else
772	pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
773	} /* for each VMCPU */
774
775	/*
776	* Setup the right TLB function based on CPU capabilities.
777	*/
778	if (pVM->hwaccm.s.fNestedPaging && pVM->hwaccm.s.vmx.fVPID)
779	pVM->hwaccm.s.vmx.pfnSetupTaggedTLB = hmR0VmxSetupTLBBoth;
780	else if (pVM->hwaccm.s.fNestedPaging)
781	pVM->hwaccm.s.vmx.pfnSetupTaggedTLB = hmR0VmxSetupTLBEPT;
782	else if (pVM->hwaccm.s.vmx.fVPID)
783	pVM->hwaccm.s.vmx.pfnSetupTaggedTLB = hmR0VmxSetupTLBVPID;
784	else
785	pVM->hwaccm.s.vmx.pfnSetupTaggedTLB = hmR0VmxSetupTLBDummy;
786
787	vmx_end:
788	hmR0VmxCheckError(pVM, &pVM->aCpus[0], rc);
789	return rc;
790	}
791
792
793	/**
794	* Sets the permission bits for the specified MSR.
795	*
796	* @param pVCpu Pointer to the VMCPU.
797	* @param ulMSR The MSR value.
798	* @param fRead Whether reading is allowed.
799	* @param fWrite Whether writing is allowed.
800	*/
801	static void hmR0VmxSetMSRPermission(PVMCPU pVCpu, unsigned ulMSR, bool fRead, bool fWrite)
802	{
803	unsigned ulBit;
804	uint8_t pMSRBitmap = (uint8_t )pVCpu->hwaccm.s.vmx.pMSRBitmap;
805
806	/*
807	* Layout:
808	* 0x000 - 0x3ff - Low MSR read bits
809	* 0x400 - 0x7ff - High MSR read bits
810	* 0x800 - 0xbff - Low MSR write bits
811	* 0xc00 - 0xfff - High MSR write bits
812	*/
813	if (ulMSR <= 0x00001FFF)
814	{
815	/* Pentium-compatible MSRs */
816	ulBit = ulMSR;
817	}
818	else if ( ulMSR >= 0xC0000000
819	&& ulMSR <= 0xC0001FFF)
820	{
821	/* AMD Sixth Generation x86 Processor MSRs */
822	ulBit = (ulMSR - 0xC0000000);
823	pMSRBitmap += 0x400;
824	}
825	else
826	{
827	AssertFailed();
828	return;
829	}
830
831	Assert(ulBit <= 0x1fff);
832	if (fRead)
833	ASMBitClear(pMSRBitmap, ulBit);
834	else
835	ASMBitSet(pMSRBitmap, ulBit);
836
837	if (fWrite)
838	ASMBitClear(pMSRBitmap + 0x800, ulBit);
839	else
840	ASMBitSet(pMSRBitmap + 0x800, ulBit);
841	}
842
843
844	/**
845	* Injects an event (trap or external interrupt).
846	*
847	* @returns VBox status code. Note that it may return VINF_EM_RESET to
848	* indicate a triple fault when injecting X86_XCPT_DF.
849	*
850	* @param pVM Pointer to the VM.
851	* @param pVCpu Pointer to the VMCPU.
852	* @param pCtx Pointer to the guest CPU Context.
853	* @param intInfo VMX interrupt info.
854	* @param cbInstr Opcode length of faulting instruction.
855	* @param errCode Error code (optional).
856	*/
857	static int hmR0VmxInjectEvent(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, uint32_t intInfo, uint32_t cbInstr, uint32_t errCode)
858	{
859	int rc;
860	uint32_t iGate = VMX_EXIT_INTERRUPTION_INFO_VECTOR(intInfo);
861
862	#ifdef VBOX_WITH_STATISTICS
863	STAM_COUNTER_INC(&pVCpu->hwaccm.s.paStatInjectedIrqsR0[iGate & MASK_INJECT_IRQ_STAT]);
864	#endif
865
866	#ifdef VBOX_STRICT
867	if (iGate == 0xE)
868	{
869	LogFlow(("hmR0VmxInjectEvent: Injecting interrupt %d at %RGv error code=%08x CR2=%RGv intInfo=%08x\n", iGate,
870	(RTGCPTR)pCtx->rip, errCode, pCtx->cr2, intInfo));
871	}
872	else if (iGate < 0x20)
873	{
874	LogFlow(("hmR0VmxInjectEvent: Injecting interrupt %d at %RGv error code=%08x\n", iGate, (RTGCPTR)pCtx->rip,
875	errCode));
876	}
877	else
878	{
879	LogFlow(("INJ-EI: %x at %RGv\n", iGate, (RTGCPTR)pCtx->rip));
880	Assert( VMX_EXIT_INTERRUPTION_INFO_TYPE(intInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW
881	\|\| !VMCPU_FF_ISSET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
882	Assert( VMX_EXIT_INTERRUPTION_INFO_TYPE(intInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW
883	\|\| pCtx->eflags.u32 & X86_EFL_IF);
884	}
885	#endif
886
887	if ( CPUMIsGuestInRealModeEx(pCtx)
888	&& pVM->hwaccm.s.vmx.pRealModeTSS)
889	{
890	RTGCPHYS GCPhysHandler;
891	uint16_t offset, ip;
892	RTSEL sel;
893
894	/*
895	* Injecting events doesn't work right with real mode emulation.
896	* (#GP if we try to inject external hardware interrupts)
897	* Inject the interrupt or trap directly instead.
898	*
899	* ASSUMES no access handlers for the bits we read or write below (should be safe).
900	*/
901	Log(("Manual interrupt/trap '%x' inject (real mode)\n", iGate));
902
903	/*
904	* Check if the interrupt handler is present.
905	*/
906	if (iGate * 4 + 3 > pCtx->idtr.cbIdt)
907	{
908	Log(("IDT cbIdt violation\n"));
909	if (iGate != X86_XCPT_DF)
910	{
911	uint32_t intInfo2;
912
913	intInfo2 = (iGate == X86_XCPT_GP) ? (uint32_t)X86_XCPT_DF : iGate;
914	intInfo2 \|= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
915	intInfo2 \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
916	intInfo2 \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HWEXCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
917
918	return hmR0VmxInjectEvent(pVM, pVCpu, pCtx, intInfo2, 0, 0 /* no error code according to the Intel docs */);
919	}
920	Log(("Triple fault -> reset the VM!\n"));
921	return VINF_EM_RESET;
922	}
923	if ( VMX_EXIT_INTERRUPTION_INFO_TYPE(intInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW
924	\|\| iGate == 3 /* Both #BP and #OF point to the instruction after. */
925	\|\| iGate == 4)
926	{
927	ip = pCtx->ip + cbInstr;
928	}
929	else
930	ip = pCtx->ip;
931
932	/*
933	* Read the selector:offset pair of the interrupt handler.
934	*/
935	GCPhysHandler = (RTGCPHYS)pCtx->idtr.pIdt + iGate * 4;
936	rc = PGMPhysSimpleReadGCPhys(pVM, &offset, GCPhysHandler, sizeof(offset)); AssertRC(rc);
937	rc = PGMPhysSimpleReadGCPhys(pVM, &sel, GCPhysHandler + 2, sizeof(sel)); AssertRC(rc);
938
939	LogFlow(("IDT handler %04X:%04X\n", sel, offset));
940
941	/*
942	* Construct the stack frame.
943	*/
944	/** @todo Check stack limit. */
945	pCtx->sp -= 2;
946	LogFlow(("ss:sp %04X:%04X eflags=%x\n", pCtx->ss.Sel, pCtx->sp, pCtx->eflags.u));
947	rc = PGMPhysSimpleWriteGCPhys(pVM, pCtx->ss.u64Base + pCtx->sp, &pCtx->eflags, sizeof(uint16_t)); AssertRC(rc);
948	pCtx->sp -= 2;
949	LogFlow(("ss:sp %04X:%04X cs=%x\n", pCtx->ss.Sel, pCtx->sp, pCtx->cs.Sel));
950	rc = PGMPhysSimpleWriteGCPhys(pVM, pCtx->ss.u64Base + pCtx->sp, &pCtx->cs, sizeof(uint16_t)); AssertRC(rc);
951	pCtx->sp -= 2;
952	LogFlow(("ss:sp %04X:%04X ip=%x\n", pCtx->ss.Sel, pCtx->sp, ip));
953	rc = PGMPhysSimpleWriteGCPhys(pVM, pCtx->ss.u64Base + pCtx->sp, &ip, sizeof(ip)); AssertRC(rc);
954
955	/*
956	* Update the CPU state for executing the handler.
957	*/
958	pCtx->rip = offset;
959	pCtx->cs.Sel = sel;
960	pCtx->cs.u64Base = sel << 4;
961	pCtx->eflags.u &= ~(X86_EFL_IF \| X86_EFL_TF \| X86_EFL_RF \| X86_EFL_AC);
962
963	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_GUEST_SEGMENT_REGS;
964	return VINF_SUCCESS;
965	}
966
967	/*
968	* Set event injection state.
969	*/
970	rc = VMXWriteVMCS(VMX_VMCS_CTRL_ENTRY_IRQ_INFO, intInfo \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT));
971	rc \|= VMXWriteVMCS(VMX_VMCS_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
972	rc \|= VMXWriteVMCS(VMX_VMCS_CTRL_ENTRY_EXCEPTION_ERRCODE, errCode);
973
974	AssertRC(rc);
975	return rc;
976	}
977
978
979	/**
980	* Checks for pending guest interrupts and injects them.
981	*
982	* @returns VBox status code.
983	* @param pVM Pointer to the VM.
984	* @param pVCpu Pointer to the VMCPU.
985	* @param pCtx Pointer to the guest CPU context.
986	*/
987	static int hmR0VmxCheckPendingInterrupt(PVM pVM, PVMCPU pVCpu, CPUMCTX *pCtx)
988	{
989	int rc;
990
991	/*
992	* Dispatch any pending interrupts (injected before, but a VM exit occurred prematurely).
993	*/
994	if (pVCpu->hwaccm.s.Event.fPending)
995	{
996	Log(("CPU%d: Reinjecting event %RX64 %08x at %RGv cr2=%RX64\n", pVCpu->idCpu, pVCpu->hwaccm.s.Event.intInfo,
997	pVCpu->hwaccm.s.Event.errCode, (RTGCPTR)pCtx->rip, pCtx->cr2));
998	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatIntReinject);
999	rc = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, pVCpu->hwaccm.s.Event.intInfo, 0, pVCpu->hwaccm.s.Event.errCode);
1000	AssertRC(rc);
1001
1002	pVCpu->hwaccm.s.Event.fPending = false;
1003	return VINF_SUCCESS;
1004	}
1005
1006	/*
1007	* If an active trap is already pending, we must forward it first!
1008	*/
1009	if (!TRPMHasTrap(pVCpu))
1010	{
1011	if (VMCPU_FF_TESTANDCLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI))
1012	{
1013	RTGCUINTPTR intInfo;
1014
1015	Log(("CPU%d: injecting #NMI\n", pVCpu->idCpu));
1016
1017	intInfo = X86_XCPT_NMI;
1018	intInfo \|= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
1019	intInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
1020
1021	rc = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, intInfo, 0, 0);
1022	AssertRC(rc);
1023
1024	return VINF_SUCCESS;
1025	}
1026
1027	/** @todo SMI interrupts. */
1028
1029	/*
1030	* When external interrupts are pending, we should exit the VM when IF is set.
1031	*/
1032	if (VMCPU_FF_ISPENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC\|VMCPU_FF_INTERRUPT_PIC)))
1033	{
1034	if (!(pCtx->eflags.u32 & X86_EFL_IF))
1035	{
1036	if (!(pVCpu->hwaccm.s.vmx.proc_ctls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_IRQ_WINDOW_EXIT))
1037	{
1038	LogFlow(("Enable irq window exit!\n"));
1039	pVCpu->hwaccm.s.vmx.proc_ctls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_IRQ_WINDOW_EXIT;
1040	rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
1041	AssertRC(rc);
1042	}
1043	/* else nothing to do but wait */
1044	}
1045	else if (!VMCPU_FF_ISSET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
1046	{
1047	uint8_t u8Interrupt;
1048
1049	rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
1050	Log(("CPU%d: Dispatch interrupt: u8Interrupt=%x (%d) rc=%Rrc cs:rip=%04X:%RGv\n", pVCpu->idCpu,
1051	u8Interrupt, u8Interrupt, rc, pCtx->cs.Sel, (RTGCPTR)pCtx->rip));
1052	if (RT_SUCCESS(rc))
1053	{
1054	rc = TRPMAssertTrap(pVCpu, u8Interrupt, TRPM_HARDWARE_INT);
1055	AssertRC(rc);
1056	}
1057	else
1058	{
1059	/* Can only happen in rare cases where a pending interrupt is cleared behind our back */
1060	Assert(!VMCPU_FF_ISPENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC\|VMCPU_FF_INTERRUPT_PIC)));
1061	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatSwitchGuestIrq);
1062	/* Just continue */
1063	}
1064	}
1065	else
1066	Log(("Pending interrupt blocked at %RGv by VM_FF_INHIBIT_INTERRUPTS!!\n", (RTGCPTR)pCtx->rip));
1067	}
1068	}
1069
1070	#ifdef VBOX_STRICT
1071	if (TRPMHasTrap(pVCpu))
1072	{
1073	uint8_t u8Vector;
1074	rc = TRPMQueryTrapAll(pVCpu, &u8Vector, 0, 0, 0);
1075	AssertRC(rc);
1076	}
1077	#endif
1078
1079	if ( (pCtx->eflags.u32 & X86_EFL_IF)
1080	&& (!VMCPU_FF_ISSET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
1081	&& TRPMHasTrap(pVCpu)
1082	)
1083	{
1084	uint8_t u8Vector;
1085	TRPMEVENT enmType;
1086	RTGCUINTPTR intInfo;
1087	RTGCUINT errCode;
1088
1089	/*
1090	* If a new event is pending, dispatch it now.
1091	*/
1092	rc = TRPMQueryTrapAll(pVCpu, &u8Vector, &enmType, &errCode, 0);
1093	AssertRC(rc);
1094	Assert(pCtx->eflags.Bits.u1IF == 1 \|\| enmType == TRPM_TRAP);
1095	Assert(enmType != TRPM_SOFTWARE_INT);
1096
1097	/*
1098	* Clear the pending trap.
1099	*/
1100	rc = TRPMResetTrap(pVCpu);
1101	AssertRC(rc);
1102
1103	intInfo = u8Vector;
1104	intInfo \|= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
1105
1106	if (enmType == TRPM_TRAP)
1107	{
1108	switch (u8Vector)
1109	{
1110	case X86_XCPT_DF:
1111	case X86_XCPT_TS:
1112	case X86_XCPT_NP:
1113	case X86_XCPT_SS:
1114	case X86_XCPT_GP:
1115	case X86_XCPT_PF:
1116	case X86_XCPT_AC:
1117	{
1118	/* Valid error codes. */
1119	intInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
1120	break;
1121	}
1122
1123	default:
1124	break;
1125	}
1126
1127	if ( u8Vector == X86_XCPT_BP
1128	\|\| u8Vector == X86_XCPT_OF)
1129	{
1130	intInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SWEXCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
1131	}
1132	else
1133	intInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HWEXCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
1134	}
1135	else
1136	intInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
1137
1138	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatIntInject);
1139	rc = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, intInfo, 0, errCode);
1140	AssertRC(rc);
1141	} /* if (interrupts can be dispatched) */
1142
1143	return VINF_SUCCESS;
1144	}
1145
1146
1147	/**
1148	* Save the host state into the VMCS.
1149	*
1150	* @returns VBox status code.
1151	* @param pVM Pointer to the VM.
1152	* @param pVCpu Pointer to the VMCPU.
1153	*/
1154	VMMR0DECL(int) VMXR0SaveHostState(PVM pVM, PVMCPU pVCpu)
1155	{
1156	int rc = VINF_SUCCESS;
1157	NOREF(pVM);
1158
1159	/*
1160	* Host CPU Context.
1161	*/
1162	if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_HOST_CONTEXT)
1163	{
1164	RTIDTR idtr;
1165	RTGDTR gdtr;
1166	RTSEL SelTR;
1167	PCX86DESCHC pDesc;
1168	uintptr_t trBase;
1169	RTSEL cs;
1170	RTSEL ss;
1171	uint64_t cr3;
1172
1173	/*
1174	* Control registers.
1175	*/
1176	rc = VMXWriteVMCS(VMX_VMCS_HOST_CR0, ASMGetCR0());
1177	Log2(("VMX_VMCS_HOST_CR0 %08x\n", ASMGetCR0()));
1178	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
1179	if (VMX_IS_64BIT_HOST_MODE())
1180	{
1181	cr3 = hwaccmR0Get64bitCR3();
1182	rc \|= VMXWriteVMCS64(VMX_VMCS_HOST_CR3, cr3);
1183	}
1184	else
1185	#endif
1186	{
1187	cr3 = ASMGetCR3();
1188	rc \|= VMXWriteVMCS(VMX_VMCS_HOST_CR3, cr3);
1189	}
1190	Log2(("VMX_VMCS_HOST_CR3 %08RX64\n", cr3));
1191	rc \|= VMXWriteVMCS(VMX_VMCS_HOST_CR4, ASMGetCR4());
1192	Log2(("VMX_VMCS_HOST_CR4 %08x\n", ASMGetCR4()));
1193	AssertRC(rc);
1194
1195	/*
1196	* Selector registers.
1197	*/
1198	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
1199	if (VMX_IS_64BIT_HOST_MODE())
1200	{
1201	cs = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelCS;
1202	ss = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelSS;
1203	}
1204	else
1205	{
1206	/* sysenter loads LDT cs & ss, VMX doesn't like this. Load the GDT ones (safe). */
1207	cs = (RTSEL)(uintptr_t)&SUPR0AbsKernelCS;
1208	ss = (RTSEL)(uintptr_t)&SUPR0AbsKernelSS;
1209	}
1210	#else
1211	cs = ASMGetCS();
1212	ss = ASMGetSS();
1213	#endif
1214	Assert(!(cs & X86_SEL_LDT)); Assert((cs & X86_SEL_RPL) == 0);
1215	Assert(!(ss & X86_SEL_LDT)); Assert((ss & X86_SEL_RPL) == 0);
1216	rc = VMXWriteVMCS(VMX_VMCS16_HOST_FIELD_CS, cs);
1217	/* Note: VMX is (again) very picky about the RPL of the selectors here; we'll restore them manually. */
1218	rc \|= VMXWriteVMCS(VMX_VMCS16_HOST_FIELD_DS, 0);
1219	rc \|= VMXWriteVMCS(VMX_VMCS16_HOST_FIELD_ES, 0);
1220	#if HC_ARCH_BITS == 32
1221	if (!VMX_IS_64BIT_HOST_MODE())
1222	{
1223	rc \|= VMXWriteVMCS(VMX_VMCS16_HOST_FIELD_FS, 0);
1224	rc \|= VMXWriteVMCS(VMX_VMCS16_HOST_FIELD_GS, 0);
1225	}
1226	#endif
1227	rc \|= VMXWriteVMCS(VMX_VMCS16_HOST_FIELD_SS, ss);
1228	SelTR = ASMGetTR();
1229	rc \|= VMXWriteVMCS(VMX_VMCS16_HOST_FIELD_TR, SelTR);
1230	AssertRC(rc);
1231	Log2(("VMX_VMCS_HOST_FIELD_CS %08x (%08x)\n", cs, ASMGetSS()));
1232	Log2(("VMX_VMCS_HOST_FIELD_DS 00000000 (%08x)\n", ASMGetDS()));
1233	Log2(("VMX_VMCS_HOST_FIELD_ES 00000000 (%08x)\n", ASMGetES()));
1234	Log2(("VMX_VMCS_HOST_FIELD_FS 00000000 (%08x)\n", ASMGetFS()));
1235	Log2(("VMX_VMCS_HOST_FIELD_GS 00000000 (%08x)\n", ASMGetGS()));
1236	Log2(("VMX_VMCS_HOST_FIELD_SS %08x (%08x)\n", ss, ASMGetSS()));
1237	Log2(("VMX_VMCS_HOST_FIELD_TR %08x\n", ASMGetTR()));
1238
1239	/*
1240	* GDTR & IDTR.
1241	*/
1242	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
1243	if (VMX_IS_64BIT_HOST_MODE())
1244	{
1245	X86XDTR64 gdtr64, idtr64;
1246	hwaccmR0Get64bitGDTRandIDTR(&gdtr64, &idtr64);
1247	rc = VMXWriteVMCS64(VMX_VMCS_HOST_GDTR_BASE, gdtr64.uAddr);
1248	rc \|= VMXWriteVMCS64(VMX_VMCS_HOST_IDTR_BASE, gdtr64.uAddr);
1249	AssertRC(rc);
1250	Log2(("VMX_VMCS_HOST_GDTR_BASE %RX64\n", gdtr64.uAddr));
1251	Log2(("VMX_VMCS_HOST_IDTR_BASE %RX64\n", idtr64.uAddr));
1252	gdtr.cbGdt = gdtr64.cb;
1253	gdtr.pGdt = (uintptr_t)gdtr64.uAddr;
1254	}
1255	else
1256	#endif
1257	{
1258	ASMGetGDTR(&gdtr);
1259	rc = VMXWriteVMCS(VMX_VMCS_HOST_GDTR_BASE, gdtr.pGdt);
1260	ASMGetIDTR(&idtr);
1261	rc \|= VMXWriteVMCS(VMX_VMCS_HOST_IDTR_BASE, idtr.pIdt);
1262	AssertRC(rc);
1263	Log2(("VMX_VMCS_HOST_GDTR_BASE %RHv\n", gdtr.pGdt));
1264	Log2(("VMX_VMCS_HOST_IDTR_BASE %RHv\n", idtr.pIdt));
1265	}
1266
1267	/*
1268	* Save the base address of the TR selector.
1269	*/
1270	if (SelTR > gdtr.cbGdt)
1271	{
1272	AssertMsgFailed(("Invalid TR selector %x. GDTR.cbGdt=%x\n", SelTR, gdtr.cbGdt));
1273	return VERR_VMX_INVALID_HOST_STATE;
1274	}
1275
1276	pDesc = (PCX86DESCHC)(gdtr.pGdt + (SelTR & X86_SEL_MASK));
1277	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
1278	if (VMX_IS_64BIT_HOST_MODE())
1279	{
1280	uint64_t trBase64 = X86DESC64_BASE((PX86DESC64)pDesc);
1281	rc = VMXWriteVMCS64(VMX_VMCS_HOST_TR_BASE, trBase64);
1282	Log2(("VMX_VMCS_HOST_TR_BASE %RX64\n", trBase64));
1283	AssertRC(rc);
1284	}
1285	else
1286	#endif
1287	{
1288	#if HC_ARCH_BITS == 64
1289	trBase = X86DESC64_BASE(pDesc);
1290	#else
1291	trBase = X86DESC_BASE(pDesc);
1292	#endif
1293	rc = VMXWriteVMCS(VMX_VMCS_HOST_TR_BASE, trBase);
1294	AssertRC(rc);
1295	Log2(("VMX_VMCS_HOST_TR_BASE %RHv\n", trBase));
1296	}
1297
1298	/*
1299	* FS base and GS base.
1300	*/
1301	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
1302	if (VMX_IS_64BIT_HOST_MODE())
1303	{
1304	Log2(("MSR_K8_FS_BASE = %RX64\n", ASMRdMsr(MSR_K8_FS_BASE)));
1305	Log2(("MSR_K8_GS_BASE = %RX64\n", ASMRdMsr(MSR_K8_GS_BASE)));
1306	rc = VMXWriteVMCS64(VMX_VMCS_HOST_FS_BASE, ASMRdMsr(MSR_K8_FS_BASE));
1307	rc \|= VMXWriteVMCS64(VMX_VMCS_HOST_GS_BASE, ASMRdMsr(MSR_K8_GS_BASE));
1308	}
1309	#endif
1310	AssertRC(rc);
1311
1312	/*
1313	* Sysenter MSRs.
1314	*/
1315	/** @todo expensive!! */
1316	rc = VMXWriteVMCS(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
1317	Log2(("VMX_VMCS_HOST_SYSENTER_CS %08x\n", ASMRdMsr_Low(MSR_IA32_SYSENTER_CS)));
1318	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
1319	if (VMX_IS_64BIT_HOST_MODE())
1320	{
1321	Log2(("VMX_VMCS_HOST_SYSENTER_EIP %RX64\n", ASMRdMsr(MSR_IA32_SYSENTER_EIP)));
1322	Log2(("VMX_VMCS_HOST_SYSENTER_ESP %RX64\n", ASMRdMsr(MSR_IA32_SYSENTER_ESP)));
1323	rc \|= VMXWriteVMCS64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
1324	rc \|= VMXWriteVMCS64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
1325	}
1326	else
1327	{
1328	rc \|= VMXWriteVMCS(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
1329	rc \|= VMXWriteVMCS(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
1330	Log2(("VMX_VMCS_HOST_SYSENTER_EIP %RX32\n", ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP)));
1331	Log2(("VMX_VMCS_HOST_SYSENTER_ESP %RX32\n", ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP)));
1332	}
1333	#elif HC_ARCH_BITS == 32
1334	rc \|= VMXWriteVMCS(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
1335	rc \|= VMXWriteVMCS(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
1336	Log2(("VMX_VMCS_HOST_SYSENTER_EIP %RX32\n", ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP)));
1337	Log2(("VMX_VMCS_HOST_SYSENTER_ESP %RX32\n", ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP)));
1338	#else
1339	Log2(("VMX_VMCS_HOST_SYSENTER_EIP %RX64\n", ASMRdMsr(MSR_IA32_SYSENTER_EIP)));
1340	Log2(("VMX_VMCS_HOST_SYSENTER_ESP %RX64\n", ASMRdMsr(MSR_IA32_SYSENTER_ESP)));
1341	rc \|= VMXWriteVMCS64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
1342	rc \|= VMXWriteVMCS64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
1343	#endif
1344	AssertRC(rc);
1345
1346
1347	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
1348	/*
1349	* Store all host MSRs in the VM-Exit load area, so they will be reloaded after
1350	* the world switch back to the host.
1351	*/
1352	PVMXMSR pMsr = (PVMXMSR)pVCpu->hwaccm.s.vmx.pHostMSR;
1353	unsigned idxMsr = 0;
1354
1355	uint32_t u32HostExtFeatures = ASMCpuId_EDX(0x80000001);
1356	if (u32HostExtFeatures & (X86_CPUID_EXT_FEATURE_EDX_NX \| X86_CPUID_EXT_FEATURE_EDX_LONG_MODE))
1357	{
1358	#if 0
1359	pMsr->u32IndexMSR = MSR_K6_EFER;
1360	pMsr->u32Reserved = 0;
1361	# if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
1362	if (CPUMIsGuestInLongMode(pVCpu))
1363	{
1364	/* Must match the EFER value in our 64 bits switcher. */
1365	pMsr->u64Value = ASMRdMsr(MSR_K6_EFER) \| MSR_K6_EFER_LME \| MSR_K6_EFER_SCE \| MSR_K6_EFER_NXE;
1366	}
1367	else
1368	# endif
1369	pMsr->u64Value = ASMRdMsr(MSR_K6_EFER);
1370	pMsr++; idxMsr++;
1371	#endif
1372	}
1373
1374	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
1375	if (VMX_IS_64BIT_HOST_MODE())
1376	{
1377	pMsr->u32IndexMSR = MSR_K6_STAR;
1378	pMsr->u32Reserved = 0;
1379	pMsr->u64Value = ASMRdMsr(MSR_K6_STAR); /* legacy syscall eip, cs & ss */
1380	pMsr++; idxMsr++;
1381	pMsr->u32IndexMSR = MSR_K8_LSTAR;
1382	pMsr->u32Reserved = 0;
1383	pMsr->u64Value = ASMRdMsr(MSR_K8_LSTAR); /* 64 bits mode syscall rip */
1384	pMsr++; idxMsr++;
1385	pMsr->u32IndexMSR = MSR_K8_SF_MASK;
1386	pMsr->u32Reserved = 0;
1387	pMsr->u64Value = ASMRdMsr(MSR_K8_SF_MASK); /* syscall flag mask */
1388	pMsr++; idxMsr++;
1389
1390	/* The KERNEL_GS_BASE MSR doesn't work reliably with auto load/store. See @bugref{6208} */
1391	#if 0
1392	pMsr->u32IndexMSR = MSR_K8_KERNEL_GS_BASE;
1393	pMsr->u32Reserved = 0;
1394	pMsr->u64Value = ASMRdMsr(MSR_K8_KERNEL_GS_BASE); /* swapgs exchange value */
1395	pMsr++; idxMsr++;
1396	#endif
1397	}
1398	# endif
1399
1400	if (pVCpu->hwaccm.s.vmx.proc_ctls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
1401	{
1402	pMsr->u32IndexMSR = MSR_K8_TSC_AUX;
1403	pMsr->u32Reserved = 0;
1404	pMsr->u64Value = ASMRdMsr(MSR_K8_TSC_AUX);
1405	pMsr++; idxMsr++;
1406	}
1407
1408	/** @todo r=ramshankar: check IA32_VMX_MISC bits 27:25 for valid idxMsr
1409	* range. */
1410	rc = VMXWriteVMCS(VMX_VMCS_CTRL_EXIT_MSR_LOAD_COUNT, idxMsr);
1411	AssertRC(rc);
1412	#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
1413
1414	pVCpu->hwaccm.s.fContextUseFlags &= ~HWACCM_CHANGED_HOST_CONTEXT;
1415	}
1416	return rc;
1417	}
1418
1419
1420	/**
1421	* Loads the 4 PDPEs into the guest state when nested paging is used and the
1422	* guest operates in PAE mode.
1423	*
1424	* @returns VBox status code.
1425	* @param pVCpu Pointer to the VMCPU.
1426	* @param pCtx Pointer to the guest CPU context.
1427	*/
1428	static int hmR0VmxLoadPaePdpes(PVMCPU pVCpu, PCPUMCTX pCtx)
1429	{
1430	if (CPUMIsGuestInPAEModeEx(pCtx))
1431	{
1432	X86PDPE aPdpes[4];
1433	int rc = PGMGstGetPaePdpes(pVCpu, &aPdpes[0]);
1434	AssertRCReturn(rc, rc);
1435
1436	rc = VMXWriteVMCS64(VMX_VMCS_GUEST_PDPTR0_FULL, aPdpes[0].u); AssertRCReturn(rc, rc);
1437	rc = VMXWriteVMCS64(VMX_VMCS_GUEST_PDPTR1_FULL, aPdpes[1].u); AssertRCReturn(rc, rc);
1438	rc = VMXWriteVMCS64(VMX_VMCS_GUEST_PDPTR2_FULL, aPdpes[2].u); AssertRCReturn(rc, rc);
1439	rc = VMXWriteVMCS64(VMX_VMCS_GUEST_PDPTR3_FULL, aPdpes[3].u); AssertRCReturn(rc, rc);
1440	}
1441	return VINF_SUCCESS;
1442	}
1443
1444
1445	/**
1446	* Saves the 4 PDPEs into the guest state when nested paging is used and the
1447	* guest operates in PAE mode.
1448	*
1449	* @returns VBox status code.
1450	* @param pVCpu Pointer to the VM CPU.
1451	* @param pCtx Pointer to the guest CPU context.
1452	*
1453	* @remarks Tell PGM about CR3 changes before calling this helper.
1454	*/
1455	static int hmR0VmxSavePaePdpes(PVMCPU pVCpu, PCPUMCTX pCtx)
1456	{
1457	if (CPUMIsGuestInPAEModeEx(pCtx))
1458	{
1459	int rc;
1460	X86PDPE aPdpes[4];
1461	rc = VMXReadVMCS64(VMX_VMCS_GUEST_PDPTR0_FULL, &aPdpes[0].u); AssertRCReturn(rc, rc);
1462	rc = VMXReadVMCS64(VMX_VMCS_GUEST_PDPTR1_FULL, &aPdpes[1].u); AssertRCReturn(rc, rc);
1463	rc = VMXReadVMCS64(VMX_VMCS_GUEST_PDPTR2_FULL, &aPdpes[2].u); AssertRCReturn(rc, rc);
1464	rc = VMXReadVMCS64(VMX_VMCS_GUEST_PDPTR3_FULL, &aPdpes[3].u); AssertRCReturn(rc, rc);
1465
1466	rc = PGMGstUpdatePaePdpes(pVCpu, &aPdpes[0]);
1467	AssertRCReturn(rc, rc);
1468	}
1469	return VINF_SUCCESS;
1470	}
1471
1472
1473	/**
1474	* Update the exception bitmap according to the current CPU state.
1475	*
1476	* @param pVM Pointer to the VM.
1477	* @param pVCpu Pointer to the VMCPU.
1478	* @param pCtx Pointer to the guest CPU context.
1479	*/
1480	static void hmR0VmxUpdateExceptionBitmap(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
1481	{
1482	uint32_t u32TrapMask;
1483	Assert(pCtx);
1484
1485	/*
1486	* Set up a mask for intercepting traps.
1487	*/
1488	/** @todo Do we really need to always intercept #DB? */
1489	u32TrapMask = RT_BIT(X86_XCPT_DB)
1490	\| RT_BIT(X86_XCPT_NM)
1491	#ifdef VBOX_ALWAYS_TRAP_PF
1492	\| RT_BIT(X86_XCPT_PF)
1493	#endif
1494	#ifdef VBOX_STRICT
1495	\| RT_BIT(X86_XCPT_BP)
1496	\| RT_BIT(X86_XCPT_DB)
1497	\| RT_BIT(X86_XCPT_DE)
1498	\| RT_BIT(X86_XCPT_NM)
1499	\| RT_BIT(X86_XCPT_UD)
1500	\| RT_BIT(X86_XCPT_NP)
1501	\| RT_BIT(X86_XCPT_SS)
1502	\| RT_BIT(X86_XCPT_GP)
1503	\| RT_BIT(X86_XCPT_MF)
1504	#endif
1505	;
1506
1507	/*
1508	* Without nested paging, #PF must be intercepted to implement shadow paging.
1509	*/
1510	/** @todo NP state won't change so maybe we should build the initial trap mask up front? */
1511	if (!pVM->hwaccm.s.fNestedPaging)
1512	u32TrapMask \|= RT_BIT(X86_XCPT_PF);
1513
1514	/* Catch floating point exceptions if we need to report them to the guest in a different way. */
1515	if (!(pCtx->cr0 & X86_CR0_NE))
1516	u32TrapMask \|= RT_BIT(X86_XCPT_MF);
1517
1518	#ifdef VBOX_STRICT
1519	Assert(u32TrapMask & RT_BIT(X86_XCPT_GP));
1520	#endif
1521
1522	/*
1523	* Intercept all exceptions in real mode as none of them can be injected directly (#GP otherwise).
1524	*/
1525	/** @todo Despite the claim to intercept everything, with NP we do not intercept #PF. Should we? */
1526	if ( CPUMIsGuestInRealModeEx(pCtx)
1527	&& pVM->hwaccm.s.vmx.pRealModeTSS)
1528	{
1529	u32TrapMask \|= RT_BIT(X86_XCPT_DE)
1530	\| RT_BIT(X86_XCPT_DB)
1531	\| RT_BIT(X86_XCPT_NMI)
1532	\| RT_BIT(X86_XCPT_BP)
1533	\| RT_BIT(X86_XCPT_OF)
1534	\| RT_BIT(X86_XCPT_BR)
1535	\| RT_BIT(X86_XCPT_UD)
1536	\| RT_BIT(X86_XCPT_DF)
1537	\| RT_BIT(X86_XCPT_CO_SEG_OVERRUN)
1538	\| RT_BIT(X86_XCPT_TS)
1539	\| RT_BIT(X86_XCPT_NP)
1540	\| RT_BIT(X86_XCPT_SS)
1541	\| RT_BIT(X86_XCPT_GP)
1542	\| RT_BIT(X86_XCPT_MF)
1543	\| RT_BIT(X86_XCPT_AC)
1544	\| RT_BIT(X86_XCPT_MC)
1545	\| RT_BIT(X86_XCPT_XF)
1546	;
1547	}
1548
1549	int rc = VMXWriteVMCS(VMX_VMCS_CTRL_EXCEPTION_BITMAP, u32TrapMask);
1550	AssertRC(rc);
1551	}
1552
1553
1554	/**
1555	* Loads a minimal guest state.
1556	*
1557	* NOTE: Don't do anything here that can cause a jump back to ring 3!!!!!
1558	*
1559	* @param pVM Pointer to the VM.
1560	* @param pVCpu Pointer to the VMCPU.
1561	* @param pCtx Pointer to the guest CPU context.
1562	*/
1563	VMMR0DECL(void) VMXR0LoadMinimalGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
1564	{
1565	int rc;
1566	X86EFLAGS eflags;
1567
1568	Assert(!(pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_ALL_GUEST));
1569
1570	/*
1571	* Load EIP, ESP and EFLAGS.
1572	*/
1573	rc = VMXWriteVMCS64(VMX_VMCS64_GUEST_RIP, pCtx->rip);
1574	rc \|= VMXWriteVMCS64(VMX_VMCS64_GUEST_RSP, pCtx->rsp);
1575	AssertRC(rc);
1576
1577	/*
1578	* Bits 22-31, 15, 5 & 3 must be zero. Bit 1 must be 1.
1579	*/
1580	eflags = pCtx->eflags;
1581	eflags.u32 &= VMX_EFLAGS_RESERVED_0;
1582	eflags.u32 \|= VMX_EFLAGS_RESERVED_1;
1583
1584	/*
1585	* Check if real mode emulation using v86 mode.
1586	*/
1587	if ( CPUMIsGuestInRealModeEx(pCtx)
1588	&& pVM->hwaccm.s.vmx.pRealModeTSS)
1589	{
1590	pVCpu->hwaccm.s.vmx.RealMode.eflags = eflags;
1591
1592	eflags.Bits.u1VM = 1;
1593	eflags.Bits.u2IOPL = 0; /* must always be 0 or else certain instructions won't cause faults. */
1594	}
1595	rc = VMXWriteVMCS(VMX_VMCS_GUEST_RFLAGS, eflags.u32);
1596	AssertRC(rc);
1597	}
1598
1599
1600	/**
1601	* Loads the guest state.
1602	*
1603	* NOTE: Don't do anything here that can cause a jump back to ring 3!!!!!
1604	*
1605	* @returns VBox status code.
1606	* @param pVM Pointer to the VM.
1607	* @param pVCpu Pointer to the VMCPU.
1608	* @param pCtx Pointer to the guest CPU context.
1609	*/
1610	VMMR0DECL(int) VMXR0LoadGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
1611	{
1612	int rc = VINF_SUCCESS;
1613	RTGCUINTPTR val;
1614
1615	/*
1616	* VMX_VMCS_CTRL_ENTRY_CONTROLS
1617	* Set required bits to one and zero according to the MSR capabilities.
1618	*/
1619	val = pVM->hwaccm.s.vmx.msr.vmx_entry.n.disallowed0;
1620
1621	/*
1622	* Load guest debug controls (DR7 & IA32_DEBUGCTL_MSR).
1623	* Forced to 1 on the 'first' VT-x capable CPUs; this actually includes the newest Nehalem CPUs
1624	*/
1625	val \|= VMX_VMCS_CTRL_ENTRY_CONTROLS_LOAD_DEBUG;
1626
1627	if (CPUMIsGuestInLongModeEx(pCtx))
1628	val \|= VMX_VMCS_CTRL_ENTRY_CONTROLS_IA64_MODE;
1629	/* else Must be zero when AMD64 is not available. */
1630
1631	/*
1632	* Mask away the bits that the CPU doesn't support.
1633	*/
1634	val &= pVM->hwaccm.s.vmx.msr.vmx_entry.n.allowed1;
1635	rc = VMXWriteVMCS(VMX_VMCS_CTRL_ENTRY_CONTROLS, val);
1636	AssertRC(rc);
1637
1638	/*
1639	* VMX_VMCS_CTRL_EXIT_CONTROLS
1640	* Set required bits to one and zero according to the MSR capabilities.
1641	*/
1642	val = pVM->hwaccm.s.vmx.msr.vmx_exit.n.disallowed0;
1643
1644	/*
1645	* Save debug controls (DR7 & IA32_DEBUGCTL_MSR)
1646	* Forced to 1 on the 'first' VT-x capable CPUs; this actually includes the newest Nehalem CPUs
1647	*/
1648	val \|= VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_DEBUG;
1649
1650	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
1651	if (VMX_IS_64BIT_HOST_MODE())
1652	val \|= VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_AMD64;
1653	/* else Must be zero when AMD64 is not available. */
1654	#elif HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
1655	if (CPUMIsGuestInLongModeEx(pCtx))
1656	val \|= VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_AMD64; /* our switcher goes to long mode */
1657	else
1658	Assert(!(val & VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_AMD64));
1659	#endif
1660	val &= pVM->hwaccm.s.vmx.msr.vmx_exit.n.allowed1;
1661
1662	/*
1663	* Don't acknowledge external interrupts on VM-exit.
1664	*/
1665	rc = VMXWriteVMCS(VMX_VMCS_CTRL_EXIT_CONTROLS, val);
1666	AssertRC(rc);
1667
1668	/*
1669	* Guest CPU context: ES, CS, SS, DS, FS, GS.
1670	*/
1671	if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_SEGMENT_REGS)
1672	{
1673	if (pVM->hwaccm.s.vmx.pRealModeTSS)
1674	{
1675	PGMMODE enmGuestMode = PGMGetGuestMode(pVCpu);
1676	if (pVCpu->hwaccm.s.vmx.enmLastSeenGuestMode != enmGuestMode)
1677	{
1678	/*
1679	* Correct weird requirements for switching to protected mode.
1680	*/
1681	if ( pVCpu->hwaccm.s.vmx.enmLastSeenGuestMode == PGMMODE_REAL
1682	&& enmGuestMode >= PGMMODE_PROTECTED)
1683	{
1684	#ifdef VBOX_WITH_REM
1685	/*
1686	* Flush the recompiler code cache as it's not unlikely the guest will rewrite code
1687	* it will later execute in real mode (OpenBSD 4.0 is one such example)
1688	*/
1689	REMFlushTBs(pVM);
1690	#endif
1691
1692	/*
1693	* DPL of all hidden selector registers must match the current CPL (0).
1694	*/
1695	pCtx->cs.Attr.n.u2Dpl = 0;
1696	pCtx->cs.Attr.n.u4Type = X86_SEL_TYPE_CODE \| X86_SEL_TYPE_RW_ACC;
1697
1698	pCtx->ds.Attr.n.u2Dpl = 0;
1699	pCtx->es.Attr.n.u2Dpl = 0;
1700	pCtx->fs.Attr.n.u2Dpl = 0;
1701	pCtx->gs.Attr.n.u2Dpl = 0;
1702	pCtx->ss.Attr.n.u2Dpl = 0;
1703	}
1704	pVCpu->hwaccm.s.vmx.enmLastSeenGuestMode = enmGuestMode;
1705	}
1706	else if ( CPUMIsGuestInRealModeEx(pCtx)
1707	&& pCtx->cs.u64Base == 0xffff0000)
1708	{
1709	/* VT-x will fail with a guest invalid state otherwise... (CPU state after a reset) */
1710	pCtx->cs.u64Base = 0xf0000;
1711	pCtx->cs.Sel = 0xf000;
1712	}
1713	}
1714
1715	VMX_WRITE_SELREG(ES, es);
1716	AssertRC(rc);
1717
1718	VMX_WRITE_SELREG(CS, cs);
1719	AssertRC(rc);
1720
1721	VMX_WRITE_SELREG(SS, ss);
1722	AssertRC(rc);
1723
1724	VMX_WRITE_SELREG(DS, ds);
1725	AssertRC(rc);
1726
1727	VMX_WRITE_SELREG(FS, fs);
1728	AssertRC(rc);
1729
1730	VMX_WRITE_SELREG(GS, gs);
1731	AssertRC(rc);
1732	}
1733
1734	/*
1735	* Guest CPU context: LDTR.
1736	*/
1737	if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_LDTR)
1738	{
1739	if (pCtx->ldtr.Sel == 0)
1740	{
1741	rc = VMXWriteVMCS(VMX_VMCS16_GUEST_FIELD_LDTR, 0);
1742	rc \|= VMXWriteVMCS(VMX_VMCS32_GUEST_LDTR_LIMIT, 0);
1743	rc \|= VMXWriteVMCS64(VMX_VMCS64_GUEST_LDTR_BASE, 0);
1744	/* Note: vmlaunch will fail with 0 or just 0x02. No idea why. */
1745	rc \|= VMXWriteVMCS(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, 0x82 /* present, LDT */);
1746	}
1747	else
1748	{
1749	rc = VMXWriteVMCS(VMX_VMCS16_GUEST_FIELD_LDTR, pCtx->ldtr.Sel);
1750	rc \|= VMXWriteVMCS(VMX_VMCS32_GUEST_LDTR_LIMIT, pCtx->ldtr.u32Limit);
1751	rc \|= VMXWriteVMCS64(VMX_VMCS64_GUEST_LDTR_BASE, pCtx->ldtr.u64Base);
1752	rc \|= VMXWriteVMCS(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, pCtx->ldtr.Attr.u);
1753	}
1754	AssertRC(rc);
1755	}
1756
1757	/*
1758	* Guest CPU context: TR.
1759	*/
1760	if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_TR)
1761	{
1762	/*
1763	* Real mode emulation using v86 mode with CR4.VME (interrupt redirection
1764	* using the int bitmap in the TSS).
1765	*/
1766	if ( CPUMIsGuestInRealModeEx(pCtx)
1767	&& pVM->hwaccm.s.vmx.pRealModeTSS)
1768	{
1769	RTGCPHYS GCPhys;
1770
1771	/* We convert it here every time as PCI regions could be reconfigured. */
1772	rc = PDMVMMDevHeapR3ToGCPhys(pVM, pVM->hwaccm.s.vmx.pRealModeTSS, &GCPhys);
1773	AssertRC(rc);
1774
1775	rc = VMXWriteVMCS(VMX_VMCS16_GUEST_FIELD_TR, 0);
1776	rc \|= VMXWriteVMCS(VMX_VMCS32_GUEST_TR_LIMIT, HWACCM_VTX_TSS_SIZE);
1777	rc \|= VMXWriteVMCS64(VMX_VMCS64_GUEST_TR_BASE, GCPhys /* phys = virt in this mode */);
1778
1779	X86DESCATTR attr;
1780
1781	attr.u = 0;
1782	attr.n.u1Present = 1;
1783	attr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
1784	val = attr.u;
1785	}
1786	else
1787	{
1788	rc = VMXWriteVMCS(VMX_VMCS16_GUEST_FIELD_TR, pCtx->tr.Sel);
1789	rc \|= VMXWriteVMCS(VMX_VMCS32_GUEST_TR_LIMIT, pCtx->tr.u32Limit);
1790	rc \|= VMXWriteVMCS64(VMX_VMCS64_GUEST_TR_BASE, pCtx->tr.u64Base);
1791
1792	val = pCtx->tr.Attr.u;
1793
1794	/* The TSS selector must be busy (REM bugs? see defect #XXXX). */
1795	if (!(val & X86_SEL_TYPE_SYS_TSS_BUSY_MASK))
1796	{
1797	if (val & 0xf)
1798	val \|= X86_SEL_TYPE_SYS_TSS_BUSY_MASK;
1799	else
1800	/* Default if no TR selector has been set (otherwise vmlaunch will fail!) */
1801	val = (val & ~0xF) \| X86_SEL_TYPE_SYS_386_TSS_BUSY;
1802	}
1803	AssertMsg((val & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY \|\| (val & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY,
1804	("%#x\n", val));
1805	}
1806	rc \|= VMXWriteVMCS(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, val);
1807	AssertRC(rc);
1808	}
1809
1810	/*
1811	* Guest CPU context: GDTR.
1812	*/
1813	if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_GDTR)
1814	{
1815	rc = VMXWriteVMCS(VMX_VMCS32_GUEST_GDTR_LIMIT, pCtx->gdtr.cbGdt);
1816	rc \|= VMXWriteVMCS64(VMX_VMCS64_GUEST_GDTR_BASE, pCtx->gdtr.pGdt);
1817	AssertRC(rc);
1818	}
1819
1820	/*
1821	* Guest CPU context: IDTR.
1822	*/
1823	if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_IDTR)
1824	{
1825	rc = VMXWriteVMCS(VMX_VMCS32_GUEST_IDTR_LIMIT, pCtx->idtr.cbIdt);
1826	rc \|= VMXWriteVMCS64(VMX_VMCS64_GUEST_IDTR_BASE, pCtx->idtr.pIdt);
1827	AssertRC(rc);
1828	}
1829
1830	/*
1831	* Sysenter MSRs.
1832	*/
1833	if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_MSR)
1834	{
1835	rc = VMXWriteVMCS(VMX_VMCS32_GUEST_SYSENTER_CS, pCtx->SysEnter.cs);
1836	rc \|= VMXWriteVMCS64(VMX_VMCS64_GUEST_SYSENTER_EIP, pCtx->SysEnter.eip);
1837	rc \|= VMXWriteVMCS64(VMX_VMCS64_GUEST_SYSENTER_ESP, pCtx->SysEnter.esp);
1838	AssertRC(rc);
1839	}
1840
1841	/*
1842	* Guest CPU context: Control registers.
1843	*/
1844	if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_CR0)
1845	{
1846	val = pCtx->cr0;
1847	rc = VMXWriteVMCS(VMX_VMCS_CTRL_CR0_READ_SHADOW, val);
1848	Log2(("Guest CR0-shadow %08x\n", val));
1849	if (CPUMIsGuestFPUStateActive(pVCpu) == false)
1850	{
1851	/* Always use #NM exceptions to load the FPU/XMM state on demand. */
1852	val \|= X86_CR0_TS \| X86_CR0_ET \| X86_CR0_NE \| X86_CR0_MP;
1853	}
1854	else
1855	{
1856	/** @todo check if we support the old style mess correctly. */
1857	if (!(val & X86_CR0_NE))
1858	Log(("Forcing X86_CR0_NE!!!\n"));
1859
1860	val \|= X86_CR0_NE; /* always turn on the native mechanism to report FPU errors (old style uses interrupts) */
1861	}
1862	/* Protected mode & paging are always enabled; we use them for emulating real and protected mode without paging too. */
1863	if (!pVM->hwaccm.s.vmx.fUnrestrictedGuest)
1864	val \|= X86_CR0_PE \| X86_CR0_PG;
1865
1866	if (pVM->hwaccm.s.fNestedPaging)
1867	{
1868	if (CPUMIsGuestInPagedProtectedModeEx(pCtx))
1869	{
1870	/* Disable CR3 read/write monitoring as we don't need it for EPT. */
1871	pVCpu->hwaccm.s.vmx.proc_ctls &= ~( VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
1872	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT);
1873	}
1874	else
1875	{
1876	/* Reenable CR3 read/write monitoring as our identity mapped page table is active. */
1877	pVCpu->hwaccm.s.vmx.proc_ctls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
1878	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT;
1879	}
1880	rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
1881	AssertRC(rc);
1882	}
1883	else
1884	{
1885	/* Note: We must also set this as we rely on protecting various pages for which supervisor writes must be caught. */
1886	val \|= X86_CR0_WP;
1887	}
1888
1889	/* Always enable caching. */
1890	val &= ~(X86_CR0_CD\|X86_CR0_NW);
1891
1892	rc \|= VMXWriteVMCS64(VMX_VMCS64_GUEST_CR0, val);
1893	Log2(("Guest CR0 %08x\n", val));
1894
1895	/*
1896	* CR0 flags owned by the host; if the guests attempts to change them, then the VM will exit.
1897	*/
1898	val = X86_CR0_PE /* Must monitor this bit (assumptions are made for real mode emulation) */
1899	\| X86_CR0_WP /* Must monitor this bit (it must always be enabled). */
1900	\| X86_CR0_PG /* Must monitor this bit (assumptions are made for real mode & protected mode without paging emulation) */
1901	\| X86_CR0_CD /* Bit not restored during VM-exit! */
1902	\| X86_CR0_NW /* Bit not restored during VM-exit! */
1903	\| X86_CR0_NE;
1904
1905	/*
1906	* When the guest's FPU state is active, then we no longer care about the FPU related bits.
1907	*/
1908	if (CPUMIsGuestFPUStateActive(pVCpu) == false)
1909	val \|= X86_CR0_TS \| X86_CR0_ET \| X86_CR0_MP;
1910
1911	pVCpu->hwaccm.s.vmx.cr0_mask = val;
1912
1913	rc \|= VMXWriteVMCS(VMX_VMCS_CTRL_CR0_MASK, val);
1914	Log2(("Guest CR0-mask %08x\n", val));
1915	AssertRC(rc);
1916	}
1917
1918	if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_CR4)
1919	{
1920	rc = VMXWriteVMCS(VMX_VMCS_CTRL_CR4_READ_SHADOW, pCtx->cr4);
1921	Log2(("Guest CR4-shadow %08x\n", pCtx->cr4));
1922	/* Set the required bits in cr4 too (currently X86_CR4_VMXE). */
1923	val = pCtx->cr4 \| (uint32_t)pVM->hwaccm.s.vmx.msr.vmx_cr4_fixed0;
1924
1925	if (!pVM->hwaccm.s.fNestedPaging)
1926	{
1927	switch (pVCpu->hwaccm.s.enmShadowMode)
1928	{
1929	case PGMMODE_REAL: /* Real mode -> emulated using v86 mode */
1930	case PGMMODE_PROTECTED: /* Protected mode, no paging -> emulated using identity mapping. */
1931	case PGMMODE_32_BIT: /* 32-bit paging. */
1932	val &= ~X86_CR4_PAE;
1933	break;
1934
1935	case PGMMODE_PAE: /* PAE paging. */
1936	case PGMMODE_PAE_NX: /* PAE paging with NX enabled. */
1937	/** Must use PAE paging as we could use physical memory > 4 GB */
1938	val \|= X86_CR4_PAE;
1939	break;
1940
1941	case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
1942	case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
1943	#ifdef VBOX_ENABLE_64_BITS_GUESTS
1944	break;
1945	#else
1946	AssertFailed();
1947	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
1948	#endif
1949	default: /* shut up gcc */
1950	AssertFailed();
1951	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
1952	}
1953	}
1954	else if ( !CPUMIsGuestInPagedProtectedModeEx(pCtx)
1955	&& !pVM->hwaccm.s.vmx.fUnrestrictedGuest)
1956	{
1957	/* We use 4 MB pages in our identity mapping page table for real and protected mode without paging. */
1958	val \|= X86_CR4_PSE;
1959	/* Our identity mapping is a 32 bits page directory. */
1960	val &= ~X86_CR4_PAE;
1961	}
1962
1963	/*
1964	* Turn off VME if we're in emulated real mode.
1965	*/
1966	if ( CPUMIsGuestInRealModeEx(pCtx)
1967	&& pVM->hwaccm.s.vmx.pRealModeTSS)
1968	{
1969	val &= ~X86_CR4_VME;
1970	}
1971
1972	rc \|= VMXWriteVMCS64(VMX_VMCS64_GUEST_CR4, val);
1973	Log2(("Guest CR4 %08x\n", val));
1974
1975	/*
1976	* CR4 flags owned by the host; if the guests attempts to change them, then the VM will exit.
1977	*/
1978	val = 0
1979	\| X86_CR4_VME
1980	\| X86_CR4_PAE
1981	\| X86_CR4_PGE
1982	\| X86_CR4_PSE
1983	\| X86_CR4_VMXE;
1984	pVCpu->hwaccm.s.vmx.cr4_mask = val;
1985
1986	rc \|= VMXWriteVMCS(VMX_VMCS_CTRL_CR4_MASK, val);
1987	Log2(("Guest CR4-mask %08x\n", val));
1988	AssertRC(rc);
1989	}
1990
1991	#if 0
1992	/* Enable single stepping if requested and CPU supports it. */
1993	if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_TRAP_FLAG)
1994	if (DBGFIsStepping(pVCpu))
1995	{
1996	pVCpu->hwaccm.s.vmx.proc_ctls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_TRAP_FLAG;
1997	rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
1998	AssertRC(rc);
1999	}
2000	#endif
2001
2002	if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_CR3)
2003	{
2004	if (pVM->hwaccm.s.fNestedPaging)
2005	{
2006	Assert(PGMGetHyperCR3(pVCpu));
2007	pVCpu->hwaccm.s.vmx.GCPhysEPTP = PGMGetHyperCR3(pVCpu);
2008
2009	Assert(!(pVCpu->hwaccm.s.vmx.GCPhysEPTP & 0xfff));
2010	/** @todo Check the IA32_VMX_EPT_VPID_CAP MSR for other supported memory types. */
2011	pVCpu->hwaccm.s.vmx.GCPhysEPTP \|= VMX_EPT_MEMTYPE_WB
2012	\| (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
2013
2014	rc = VMXWriteVMCS64(VMX_VMCS_CTRL_EPTP_FULL, pVCpu->hwaccm.s.vmx.GCPhysEPTP);
2015	AssertRC(rc);
2016
2017	if ( !CPUMIsGuestInPagedProtectedModeEx(pCtx)
2018	&& !pVM->hwaccm.s.vmx.fUnrestrictedGuest)
2019	{
2020	RTGCPHYS GCPhys;
2021
2022	/* We convert it here every time as PCI regions could be reconfigured. */
2023	rc = PDMVMMDevHeapR3ToGCPhys(pVM, pVM->hwaccm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
2024	AssertMsgRC(rc, ("pNonPagingModeEPTPageTable = %RGv\n", pVM->hwaccm.s.vmx.pNonPagingModeEPTPageTable));
2025
2026	/*
2027	* We use our identity mapping page table here as we need to map guest virtual to
2028	* guest physical addresses; EPT will take care of the translation to host physical addresses.
2029	*/
2030	val = GCPhys;
2031	}
2032	else
2033	{
2034	/* Save the real guest CR3 in VMX_VMCS_GUEST_CR3 */
2035	val = pCtx->cr3;
2036	rc = hmR0VmxLoadPaePdpes(pVCpu, pCtx);
2037	AssertRCReturn(rc, rc);
2038	}
2039	}
2040	else
2041	{
2042	val = PGMGetHyperCR3(pVCpu);
2043	Assert(val \|\| VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL));
2044	}
2045
2046	/* Save our shadow CR3 register. */
2047	rc = VMXWriteVMCS64(VMX_VMCS64_GUEST_CR3, val);
2048	AssertRC(rc);
2049	}
2050
2051	/*
2052	* Guest CPU context: Debug registers.
2053	*/
2054	if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_DEBUG)
2055	{
2056	pCtx->dr[6] \|= X86_DR6_INIT_VAL; /* set all reserved bits to 1. */
2057	pCtx->dr[6] &= ~RT_BIT(12); /* must be zero. */
2058
2059	pCtx->dr[7] &= 0xffffffff; /* upper 32 bits reserved */
2060	pCtx->dr[7] &= ~(RT_BIT(11) \| RT_BIT(12) \| RT_BIT(14) \| RT_BIT(15)); /* must be zero */
2061	pCtx->dr[7] \|= 0x400; /* must be one */
2062
2063	/* Resync DR7 */
2064	rc = VMXWriteVMCS64(VMX_VMCS64_GUEST_DR7, pCtx->dr[7]);
2065	AssertRC(rc);
2066
2067	#ifdef DEBUG
2068	/* Sync the hypervisor debug state now if any breakpoint is armed. */
2069	if ( CPUMGetHyperDR7(pVCpu) & (X86_DR7_ENABLED_MASK\|X86_DR7_GD)
2070	&& !CPUMIsHyperDebugStateActive(pVCpu)
2071	&& !DBGFIsStepping(pVCpu))
2072	{
2073	/* Save the host and load the hypervisor debug state. */
2074	rc = CPUMR0LoadHyperDebugState(pVM, pVCpu, pCtx, true /* include DR6 */);
2075	AssertRC(rc);
2076
2077	/* DRx intercepts remain enabled. */
2078
2079	/* Override dr7 with the hypervisor value. */
2080	rc = VMXWriteVMCS64(VMX_VMCS64_GUEST_DR7, CPUMGetHyperDR7(pVCpu));
2081	AssertRC(rc);
2082	}
2083	else
2084	#endif
2085	/* Sync the debug state now if any breakpoint is armed. */
2086	if ( (pCtx->dr[7] & (X86_DR7_ENABLED_MASK\|X86_DR7_GD))
2087	&& !CPUMIsGuestDebugStateActive(pVCpu)
2088	&& !DBGFIsStepping(pVCpu))
2089	{
2090	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatDRxArmed);
2091
2092	/* Disable DRx move intercepts. */
2093	pVCpu->hwaccm.s.vmx.proc_ctls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT;
2094	rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
2095	AssertRC(rc);
2096
2097	/* Save the host and load the guest debug state. */
2098	rc = CPUMR0LoadGuestDebugState(pVM, pVCpu, pCtx, true /* include DR6 */);
2099	AssertRC(rc);
2100	}
2101
2102	/* IA32_DEBUGCTL MSR. */
2103	rc = VMXWriteVMCS64(VMX_VMCS_GUEST_DEBUGCTL_FULL, 0);
2104	AssertRC(rc);
2105
2106	/** @todo do we really ever need this? */
2107	rc \|= VMXWriteVMCS(VMX_VMCS_GUEST_DEBUG_EXCEPTIONS, 0);
2108	AssertRC(rc);
2109	}
2110
2111	/*
2112	* 64-bit guest mode.
2113	*/
2114	if (CPUMIsGuestInLongModeEx(pCtx))
2115	{
2116	#if !defined(VBOX_ENABLE_64_BITS_GUESTS)
2117	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
2118	#elif HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2119	pVCpu->hwaccm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
2120	#else
2121	# ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2122	if (!pVM->hwaccm.s.fAllow64BitGuests)
2123	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
2124	# endif
2125	pVCpu->hwaccm.s.vmx.pfnStartVM = VMXR0StartVM64;
2126	#endif
2127	if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_MSR)
2128	{
2129	/* Update these as wrmsr might have changed them. */
2130	rc = VMXWriteVMCS64(VMX_VMCS64_GUEST_FS_BASE, pCtx->fs.u64Base);
2131	AssertRC(rc);
2132	rc = VMXWriteVMCS64(VMX_VMCS64_GUEST_GS_BASE, pCtx->gs.u64Base);
2133	AssertRC(rc);
2134	}
2135	}
2136	else
2137	{
2138	pVCpu->hwaccm.s.vmx.pfnStartVM = VMXR0StartVM32;
2139	}
2140
2141	hmR0VmxUpdateExceptionBitmap(pVM, pVCpu, pCtx);
2142
2143	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
2144	/*
2145	* Store all guest MSRs in the VM-entry load area, so they will be loaded
2146	* during VM-entry and restored into the VM-exit store area during VM-exit.
2147	*/
2148	PVMXMSR pMsr = (PVMXMSR)pVCpu->hwaccm.s.vmx.pGuestMSR;
2149	unsigned idxMsr = 0;
2150
2151	uint32_t u32GstExtFeatures;
2152	uint32_t u32Temp;
2153	CPUMGetGuestCpuId(pVCpu, 0x80000001, &u32Temp, &u32Temp, &u32Temp, &u32GstExtFeatures);
2154
2155	if (u32GstExtFeatures & (X86_CPUID_EXT_FEATURE_EDX_NX \| X86_CPUID_EXT_FEATURE_EDX_LONG_MODE))
2156	{
2157	#if 0
2158	pMsr->u32IndexMSR = MSR_K6_EFER;
2159	pMsr->u32Reserved = 0;
2160	pMsr->u64Value = pCtx->msrEFER;
2161	/* VT-x will complain if only MSR_K6_EFER_LME is set. */
2162	if (!CPUMIsGuestInLongModeEx(pCtx))
2163	pMsr->u64Value &= ~(MSR_K6_EFER_LMA \| MSR_K6_EFER_LME);
2164	pMsr++; idxMsr++;
2165	#endif
2166
2167	if (u32GstExtFeatures & X86_CPUID_EXT_FEATURE_EDX_LONG_MODE)
2168	{
2169	pMsr->u32IndexMSR = MSR_K8_LSTAR;
2170	pMsr->u32Reserved = 0;
2171	pMsr->u64Value = pCtx->msrLSTAR; /* 64 bits mode syscall rip */
2172	pMsr++; idxMsr++;
2173	pMsr->u32IndexMSR = MSR_K6_STAR;
2174	pMsr->u32Reserved = 0;
2175	pMsr->u64Value = pCtx->msrSTAR; /* legacy syscall eip, cs & ss */
2176	pMsr++; idxMsr++;
2177	pMsr->u32IndexMSR = MSR_K8_SF_MASK;
2178	pMsr->u32Reserved = 0;
2179	pMsr->u64Value = pCtx->msrSFMASK; /* syscall flag mask */
2180	pMsr++; idxMsr++;
2181
2182	/* The KERNEL_GS_BASE MSR doesn't work reliably with auto load/store. See @bugref{6208} */
2183	#if 0
2184	pMsr->u32IndexMSR = MSR_K8_KERNEL_GS_BASE;
2185	pMsr->u32Reserved = 0;
2186	pMsr->u64Value = pCtx->msrKERNELGSBASE; /* swapgs exchange value */
2187	pMsr++; idxMsr++;
2188	#endif
2189	}
2190	}
2191
2192	if ( pVCpu->hwaccm.s.vmx.proc_ctls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP
2193	&& (u32GstExtFeatures & X86_CPUID_EXT_FEATURE_EDX_RDTSCP))
2194	{
2195	pMsr->u32IndexMSR = MSR_K8_TSC_AUX;
2196	pMsr->u32Reserved = 0;
2197	rc = CPUMQueryGuestMsr(pVCpu, MSR_K8_TSC_AUX, &pMsr->u64Value);
2198	AssertRC(rc);
2199	pMsr++; idxMsr++;
2200	}
2201
2202	pVCpu->hwaccm.s.vmx.cCachedMSRs = idxMsr;
2203
2204	rc = VMXWriteVMCS(VMX_VMCS_CTRL_ENTRY_MSR_LOAD_COUNT, idxMsr);
2205	AssertRC(rc);
2206
2207	rc = VMXWriteVMCS(VMX_VMCS_CTRL_EXIT_MSR_STORE_COUNT, idxMsr);
2208	AssertRC(rc);
2209	#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
2210
2211	bool fOffsettedTsc;
2212	if (pVM->hwaccm.s.vmx.fUsePreemptTimer)
2213	{
2214	uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVCpu, &fOffsettedTsc, &pVCpu->hwaccm.s.vmx.u64TSCOffset);
2215
2216	/* Make sure the returned values have sane upper and lower boundaries. */
2217	uint64_t u64CpuHz = SUPGetCpuHzFromGIP(g_pSUPGlobalInfoPage);
2218
2219	cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64 of a second */
2220	cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
2221
2222	cTicksToDeadline >>= pVM->hwaccm.s.vmx.cPreemptTimerShift;
2223	uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
2224	rc = VMXWriteVMCS(VMX_VMCS32_GUEST_PREEMPTION_TIMER_VALUE, cPreemptionTickCount);
2225	AssertRC(rc);
2226	}
2227	else
2228	fOffsettedTsc = TMCpuTickCanUseRealTSC(pVCpu, &pVCpu->hwaccm.s.vmx.u64TSCOffset);
2229
2230	if (fOffsettedTsc)
2231	{
2232	uint64_t u64CurTSC = ASMReadTSC();
2233	if (u64CurTSC + pVCpu->hwaccm.s.vmx.u64TSCOffset >= TMCpuTickGetLastSeen(pVCpu))
2234	{
2235	/* Note: VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT takes precedence over TSC_OFFSET, applies to RDTSCP too. */
2236	rc = VMXWriteVMCS64(VMX_VMCS_CTRL_TSC_OFFSET_FULL, pVCpu->hwaccm.s.vmx.u64TSCOffset);
2237	AssertRC(rc);
2238
2239	pVCpu->hwaccm.s.vmx.proc_ctls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT;
2240	rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
2241	AssertRC(rc);
2242	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatTSCOffset);
2243	}
2244	else
2245	{
2246	/* Fall back to rdtsc, rdtscp emulation as we would otherwise pass decreasing tsc values to the guest. */
2247	LogFlow(("TSC %RX64 offset %RX64 time=%RX64 last=%RX64 (diff=%RX64, virt_tsc=%RX64)\n", u64CurTSC,
2248	pVCpu->hwaccm.s.vmx.u64TSCOffset, u64CurTSC + pVCpu->hwaccm.s.vmx.u64TSCOffset,
2249	TMCpuTickGetLastSeen(pVCpu), TMCpuTickGetLastSeen(pVCpu) - u64CurTSC - pVCpu->hwaccm.s.vmx.u64TSCOffset,
2250	TMCpuTickGet(pVCpu)));
2251	pVCpu->hwaccm.s.vmx.proc_ctls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT;
2252	rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
2253	AssertRC(rc);
2254	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatTSCInterceptOverFlow);
2255	}
2256	}
2257	else
2258	{
2259	pVCpu->hwaccm.s.vmx.proc_ctls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT;
2260	rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
2261	AssertRC(rc);
2262	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatTSCIntercept);
2263	}
2264
2265	/* Done with the major changes */
2266	pVCpu->hwaccm.s.fContextUseFlags &= ~HWACCM_CHANGED_ALL_GUEST;
2267
2268	/* Minimal guest state update (ESP, EIP, EFLAGS mostly) */
2269	VMXR0LoadMinimalGuestState(pVM, pVCpu, pCtx);
2270	return rc;
2271	}
2272
2273
2274	/**
2275	* Syncs back the guest state from VMCS.
2276	*
2277	* @returns VBox status code.
2278	* @param pVM Pointer to the VM.
2279	* @param pVCpu Pointer to the VMCPU.
2280	* @param pCtx Pointer to the guest CPU context.
2281	*/
2282	DECLINLINE(int) VMXR0SaveGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
2283	{
2284	RTGCUINTREG val, valShadow;
2285	RTGCUINTPTR uInterruptState;
2286	int rc;
2287
2288	/* First sync back EIP, ESP, and EFLAGS. */
2289	rc = VMXReadCachedVMCS(VMX_VMCS64_GUEST_RIP, &val);
2290	AssertRC(rc);
2291	pCtx->rip = val;
2292	rc = VMXReadCachedVMCS(VMX_VMCS64_GUEST_RSP, &val);
2293	AssertRC(rc);
2294	pCtx->rsp = val;
2295	rc = VMXReadCachedVMCS(VMX_VMCS_GUEST_RFLAGS, &val);
2296	AssertRC(rc);
2297	pCtx->eflags.u32 = val;
2298
2299	/* Take care of instruction fusing (sti, mov ss) */
2300	rc \|= VMXReadCachedVMCS(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &val);
2301	uInterruptState = val;
2302	if (uInterruptState != 0)
2303	{
2304	Assert(uInterruptState <= 2); /* only sti & mov ss */
2305	Log(("uInterruptState %x eip=%RGv\n", (uint32_t)uInterruptState, pCtx->rip));
2306	EMSetInhibitInterruptsPC(pVCpu, pCtx->rip);
2307	}
2308	else
2309	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
2310
2311	/* Control registers. */
2312	VMXReadCachedVMCS(VMX_VMCS_CTRL_CR0_READ_SHADOW, &valShadow);
2313	VMXReadCachedVMCS(VMX_VMCS64_GUEST_CR0, &val);
2314	val = (valShadow & pVCpu->hwaccm.s.vmx.cr0_mask) \| (val & ~pVCpu->hwaccm.s.vmx.cr0_mask);
2315	CPUMSetGuestCR0(pVCpu, val);
2316
2317	VMXReadCachedVMCS(VMX_VMCS_CTRL_CR4_READ_SHADOW, &valShadow);
2318	VMXReadCachedVMCS(VMX_VMCS64_GUEST_CR4, &val);
2319	val = (valShadow & pVCpu->hwaccm.s.vmx.cr4_mask) \| (val & ~pVCpu->hwaccm.s.vmx.cr4_mask);
2320	CPUMSetGuestCR4(pVCpu, val);
2321
2322	/*
2323	* No reason to sync back the CRx registers. They can't be changed by the guest unless in
2324	* the nested paging case where CR3 & CR4 can be changed by the guest.
2325	*/
2326	if ( pVM->hwaccm.s.fNestedPaging
2327	&& CPUMIsGuestInPagedProtectedModeEx(pCtx)) /** @todo check if we will always catch mode switches and such... */
2328	{
2329	PVMCSCACHE pCache = &pVCpu->hwaccm.s.vmx.VMCSCache;
2330
2331	/* Can be updated behind our back in the nested paging case. */
2332	CPUMSetGuestCR2(pVCpu, pCache->cr2);
2333
2334	VMXReadCachedVMCS(VMX_VMCS64_GUEST_CR3, &val);
2335
2336	if (val != pCtx->cr3)
2337	{
2338	CPUMSetGuestCR3(pVCpu, val);
2339	PGMUpdateCR3(pVCpu, val);
2340	}
2341	rc = hmR0VmxSavePaePdpes(pVCpu, pCtx);
2342	AssertRCReturn(rc, rc);
2343	}
2344
2345	/* Sync back DR7. */
2346	VMXReadCachedVMCS(VMX_VMCS64_GUEST_DR7, &val);
2347	pCtx->dr[7] = val;
2348
2349	/* Guest CPU context: ES, CS, SS, DS, FS, GS. */
2350	VMX_READ_SELREG(ES, es);
2351	VMX_READ_SELREG(SS, ss);
2352	VMX_READ_SELREG(CS, cs);
2353	VMX_READ_SELREG(DS, ds);
2354	VMX_READ_SELREG(FS, fs);
2355	VMX_READ_SELREG(GS, gs);
2356
2357	/* System MSRs */
2358	VMXReadCachedVMCS(VMX_VMCS32_GUEST_SYSENTER_CS, &val);
2359	pCtx->SysEnter.cs = val;
2360	VMXReadCachedVMCS(VMX_VMCS64_GUEST_SYSENTER_EIP, &val);
2361	pCtx->SysEnter.eip = val;
2362	VMXReadCachedVMCS(VMX_VMCS64_GUEST_SYSENTER_ESP, &val);
2363	pCtx->SysEnter.esp = val;
2364
2365	/* Misc. registers; must sync everything otherwise we can get out of sync when jumping to ring 3. */
2366	VMX_READ_SELREG(LDTR, ldtr);
2367
2368	VMXReadCachedVMCS(VMX_VMCS32_GUEST_GDTR_LIMIT, &val);
2369	pCtx->gdtr.cbGdt = val;
2370	VMXReadCachedVMCS(VMX_VMCS64_GUEST_GDTR_BASE, &val);
2371	pCtx->gdtr.pGdt = val;
2372
2373	VMXReadCachedVMCS(VMX_VMCS32_GUEST_IDTR_LIMIT, &val);
2374	pCtx->idtr.cbIdt = val;
2375	VMXReadCachedVMCS(VMX_VMCS64_GUEST_IDTR_BASE, &val);
2376	pCtx->idtr.pIdt = val;
2377
2378	/* Real mode emulation using v86 mode. */
2379	if ( CPUMIsGuestInRealModeEx(pCtx)
2380	&& pVM->hwaccm.s.vmx.pRealModeTSS)
2381	{
2382	/* Hide our emulation flags */
2383	pCtx->eflags.Bits.u1VM = 0;
2384
2385	/* Restore original IOPL setting as we always use 0. */
2386	pCtx->eflags.Bits.u2IOPL = pVCpu->hwaccm.s.vmx.RealMode.eflags.Bits.u2IOPL;
2387
2388	/* Force a TR resync every time in case we switch modes. */
2389	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_GUEST_TR;
2390	}
2391	else
2392	{
2393	/* In real mode we have a fake TSS, so only sync it back when it's supposed to be valid. */
2394	VMX_READ_SELREG(TR, tr);
2395	}
2396
2397	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
2398	/*
2399	* Save the possibly changed MSRs that we automatically restore and save during a world switch.
2400	*/
2401	for (unsigned i = 0; i < pVCpu->hwaccm.s.vmx.cCachedMSRs; i++)
2402	{
2403	PVMXMSR pMsr = (PVMXMSR)pVCpu->hwaccm.s.vmx.pGuestMSR;
2404	pMsr += i;
2405
2406	switch (pMsr->u32IndexMSR)
2407	{
2408	case MSR_K8_LSTAR:
2409	pCtx->msrLSTAR = pMsr->u64Value;
2410	break;
2411	case MSR_K6_STAR:
2412	pCtx->msrSTAR = pMsr->u64Value;
2413	break;
2414	case MSR_K8_SF_MASK:
2415	pCtx->msrSFMASK = pMsr->u64Value;
2416	break;
2417	/* The KERNEL_GS_BASE MSR doesn't work reliably with auto load/store. See @bugref{6208} */
2418	#if 0
2419	case MSR_K8_KERNEL_GS_BASE:
2420	pCtx->msrKERNELGSBASE = pMsr->u64Value;
2421	break;
2422	#endif
2423	case MSR_K8_TSC_AUX:
2424	CPUMSetGuestMsr(pVCpu, MSR_K8_TSC_AUX, pMsr->u64Value);
2425	break;
2426	#if 0
2427	case MSR_K6_EFER:
2428	/* EFER can't be changed without causing a VM-exit. */
2429	/* Assert(pCtx->msrEFER == pMsr->u64Value); */
2430	break;
2431	#endif
2432	default:
2433	AssertFailed();
2434	return VERR_HM_UNEXPECTED_LD_ST_MSR;
2435	}
2436	}
2437	#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
2438	return VINF_SUCCESS;
2439	}
2440
2441
2442	/**
2443	* Dummy placeholder for TLB flush handling before VM-entry. Used in the case
2444	* where neither EPT nor VPID is supported by the CPU.
2445	*
2446	* @param pVM Pointer to the VM.
2447	* @param pVCpu Pointer to the VMCPU.
2448	*/
2449	static DECLCALLBACK(void) hmR0VmxSetupTLBDummy(PVM pVM, PVMCPU pVCpu)
2450	{
2451	NOREF(pVM);
2452	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
2453	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
2454	pVCpu->hwaccm.s.TlbShootdown.cPages = 0;
2455	return;
2456	}
2457
2458
2459	/**
2460	* Setup the tagged TLB for EPT+VPID.
2461	*
2462	* @param pVM Pointer to the VM.
2463	* @param pVCpu Pointer to the VMCPU.
2464	*/
2465	static DECLCALLBACK(void) hmR0VmxSetupTLBBoth(PVM pVM, PVMCPU pVCpu)
2466	{
2467	PHMGLOBLCPUINFO pCpu;
2468
2469	Assert(pVM->hwaccm.s.fNestedPaging && pVM->hwaccm.s.vmx.fVPID);
2470
2471	pCpu = HWACCMR0GetCurrentCpu();
2472
2473	/*
2474	* Force a TLB flush for the first world switch if the current CPU differs from the one we ran on last
2475	* This can happen both for start & resume due to long jumps back to ring-3.
2476	* If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
2477	* or the host Cpu is online after a suspend/resume, so we cannot reuse the current ASID anymore.
2478	*/
2479	bool fNewASID = false;
2480	if ( pVCpu->hwaccm.s.idLastCpu != pCpu->idCpu
2481	\|\| pVCpu->hwaccm.s.cTLBFlushes != pCpu->cTLBFlushes)
2482	{
2483	pVCpu->hwaccm.s.fForceTLBFlush = true;
2484	fNewASID = true;
2485	}
2486
2487	/*
2488	* Check for explicit TLB shootdowns.
2489	*/
2490	if (VMCPU_FF_TESTANDCLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2491	pVCpu->hwaccm.s.fForceTLBFlush = true;
2492
2493	pVCpu->hwaccm.s.idLastCpu = pCpu->idCpu;
2494
2495	if (pVCpu->hwaccm.s.fForceTLBFlush)
2496	{
2497	if (fNewASID)
2498	{
2499	++pCpu->uCurrentASID;
2500	if (pCpu->uCurrentASID >= pVM->hwaccm.s.uMaxASID)
2501	{
2502	pCpu->uCurrentASID = 1; /* start at 1; host uses 0 */
2503	pCpu->cTLBFlushes++;
2504	pCpu->fFlushASIDBeforeUse = true;
2505	}
2506
2507	pVCpu->hwaccm.s.uCurrentASID = pCpu->uCurrentASID;
2508	if (pCpu->fFlushASIDBeforeUse)
2509	{
2510	hmR0VmxFlushVPID(pVM, pVCpu, pVM->hwaccm.s.vmx.enmFlushVPID, 0 /* GCPtr */);
2511	#ifdef VBOX_WITH_STATISTICS
2512	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatFlushASID);
2513	#endif
2514	}
2515	}
2516	else
2517	{
2518	if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVVPID_CAPS_SINGLE_CONTEXT)
2519	hmR0VmxFlushVPID(pVM, pVCpu, VMX_FLUSH_VPID_SINGLE_CONTEXT, 0 /* GCPtr */);
2520	else
2521	hmR0VmxFlushEPT(pVM, pVCpu, pVM->hwaccm.s.vmx.enmFlushEPT);
2522
2523	#ifdef VBOX_WITH_STATISTICS
2524	/*
2525	* This is not terribly accurate (i.e. we don't have any StatFlushEPT counter). We currently count these
2526	* as ASID flushes too, better than including them under StatFlushTLBWorldSwitch.
2527	*/
2528	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatFlushASID);
2529	#endif
2530	}
2531
2532	pVCpu->hwaccm.s.cTLBFlushes = pCpu->cTLBFlushes;
2533	pVCpu->hwaccm.s.fForceTLBFlush = false;
2534	}
2535	else
2536	{
2537	AssertMsg(pVCpu->hwaccm.s.uCurrentASID && pCpu->uCurrentASID,
2538	("hwaccm->uCurrentASID=%lu hwaccm->cTLBFlushes=%lu cpu->uCurrentASID=%lu cpu->cTLBFlushes=%lu\n",
2539	pVCpu->hwaccm.s.uCurrentASID, pVCpu->hwaccm.s.cTLBFlushes,
2540	pCpu->uCurrentASID, pCpu->cTLBFlushes));
2541
2542	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
2543	* not be executed. See hwaccmQueueInvlPage() where it is commented
2544	* out. Support individual entry flushing someday. */
2545	if (VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
2546	{
2547	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatTlbShootdown);
2548
2549	/*
2550	* Flush individual guest entries using VPID from the TLB or as little as possible with EPT
2551	* as supported by the CPU.
2552	*/
2553	if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVVPID_CAPS_INDIV_ADDR)
2554	{
2555	for (unsigned i = 0; i < pVCpu->hwaccm.s.TlbShootdown.cPages; i++)
2556	hmR0VmxFlushVPID(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, pVCpu->hwaccm.s.TlbShootdown.aPages[i]);
2557	}
2558	else
2559	hmR0VmxFlushEPT(pVM, pVCpu, pVM->hwaccm.s.vmx.enmFlushEPT);
2560	}
2561	else
2562	{
2563	#ifdef VBOX_WITH_STATISTICS
2564	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatNoFlushTLBWorldSwitch);
2565	#endif
2566	}
2567	}
2568	pVCpu->hwaccm.s.TlbShootdown.cPages = 0;
2569	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
2570
2571	AssertMsg(pVCpu->hwaccm.s.cTLBFlushes == pCpu->cTLBFlushes,
2572	("Flush count mismatch for cpu %d (%x vs %x)\n", pCpu->idCpu, pVCpu->hwaccm.s.cTLBFlushes, pCpu->cTLBFlushes));
2573	AssertMsg(pCpu->uCurrentASID >= 1 && pCpu->uCurrentASID < pVM->hwaccm.s.uMaxASID,
2574	("cpu%d uCurrentASID = %x\n", pCpu->idCpu, pCpu->uCurrentASID));
2575	AssertMsg(pVCpu->hwaccm.s.uCurrentASID >= 1 && pVCpu->hwaccm.s.uCurrentASID < pVM->hwaccm.s.uMaxASID,
2576	("cpu%d VM uCurrentASID = %x\n", pCpu->idCpu, pVCpu->hwaccm.s.uCurrentASID));
2577
2578	/* Update VMCS with the VPID. */
2579	int rc = VMXWriteVMCS(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hwaccm.s.uCurrentASID);
2580	AssertRC(rc);
2581	}
2582
2583
2584	/**
2585	* Setup the tagged TLB for EPT only.
2586	*
2587	* @returns VBox status code.
2588	* @param pVM Pointer to the VM.
2589	* @param pVCpu Pointer to the VMCPU.
2590	*/
2591	static DECLCALLBACK(void) hmR0VmxSetupTLBEPT(PVM pVM, PVMCPU pVCpu)
2592	{
2593	PHMGLOBLCPUINFO pCpu;
2594
2595	Assert(pVM->hwaccm.s.fNestedPaging);
2596	Assert(!pVM->hwaccm.s.vmx.fVPID);
2597
2598	pCpu = HWACCMR0GetCurrentCpu();
2599
2600	/*
2601	* Force a TLB flush for the first world switch if the current CPU differs from the one we ran on last
2602	* This can happen both for start & resume due to long jumps back to ring-3.
2603	* A change in the TLB flush count implies the host Cpu is online after a suspend/resume.
2604	*/
2605	if ( pVCpu->hwaccm.s.idLastCpu != pCpu->idCpu
2606	\|\| pVCpu->hwaccm.s.cTLBFlushes != pCpu->cTLBFlushes)
2607	{
2608	pVCpu->hwaccm.s.fForceTLBFlush = true;
2609	}
2610
2611	/*
2612	* Check for explicit TLB shootdown flushes.
2613	*/
2614	if (VMCPU_FF_TESTANDCLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2615	pVCpu->hwaccm.s.fForceTLBFlush = true;
2616
2617	pVCpu->hwaccm.s.idLastCpu = pCpu->idCpu;
2618	pVCpu->hwaccm.s.cTLBFlushes = pCpu->cTLBFlushes;
2619
2620	if (pVCpu->hwaccm.s.fForceTLBFlush)
2621	hmR0VmxFlushEPT(pVM, pVCpu, pVM->hwaccm.s.vmx.enmFlushEPT);
2622	else
2623	{
2624	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
2625	* not be executed. See hwaccmQueueInvlPage() where it is commented
2626	* out. Support individual entry flushing someday. */
2627	if (VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
2628	{
2629	/*
2630	* We cannot flush individual entries without VPID support. Flush using EPT.
2631	*/
2632	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatTlbShootdown);
2633	hmR0VmxFlushEPT(pVM, pVCpu, pVM->hwaccm.s.vmx.enmFlushEPT);
2634	}
2635	}
2636	pVCpu->hwaccm.s.TlbShootdown.cPages= 0;
2637	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
2638
2639	#ifdef VBOX_WITH_STATISTICS
2640	if (pVCpu->hwaccm.s.fForceTLBFlush)
2641	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatFlushTLBWorldSwitch);
2642	else
2643	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatNoFlushTLBWorldSwitch);
2644	#endif
2645	}
2646
2647
2648	/**
2649	* Setup the tagged TLB for VPID.
2650	*
2651	* @returns VBox status code.
2652	* @param pVM Pointer to the VM.
2653	* @param pVCpu Pointer to the VMCPU.
2654	*/
2655	static DECLCALLBACK(void) hmR0VmxSetupTLBVPID(PVM pVM, PVMCPU pVCpu)
2656	{
2657	PHMGLOBLCPUINFO pCpu;
2658
2659	Assert(pVM->hwaccm.s.vmx.fVPID);
2660	Assert(!pVM->hwaccm.s.fNestedPaging);
2661
2662	pCpu = HWACCMR0GetCurrentCpu();
2663
2664	/*
2665	* Force a TLB flush for the first world switch if the current CPU differs from the one we ran on last
2666	* This can happen both for start & resume due to long jumps back to ring-3.
2667	* If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
2668	* or the host Cpu is online after a suspend/resume, so we cannot reuse the current ASID anymore.
2669	*/
2670	if ( pVCpu->hwaccm.s.idLastCpu != pCpu->idCpu
2671	\|\| pVCpu->hwaccm.s.cTLBFlushes != pCpu->cTLBFlushes)
2672	{
2673	/* Force a TLB flush on VM entry. */
2674	pVCpu->hwaccm.s.fForceTLBFlush = true;
2675	}
2676
2677	/*
2678	* Check for explicit TLB shootdown flushes.
2679	*/
2680	if (VMCPU_FF_TESTANDCLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2681	pVCpu->hwaccm.s.fForceTLBFlush = true;
2682
2683	pVCpu->hwaccm.s.idLastCpu = pCpu->idCpu;
2684
2685	if (pVCpu->hwaccm.s.fForceTLBFlush)
2686	{
2687	++pCpu->uCurrentASID;
2688	if (pCpu->uCurrentASID >= pVM->hwaccm.s.uMaxASID)
2689	{
2690	pCpu->uCurrentASID = 1; /* start at 1; host uses 0 */
2691	pCpu->cTLBFlushes++;
2692	pCpu->fFlushASIDBeforeUse = true;
2693	}
2694	else
2695	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatFlushASID);
2696
2697	pVCpu->hwaccm.s.fForceTLBFlush = false;
2698	pVCpu->hwaccm.s.cTLBFlushes = pCpu->cTLBFlushes;
2699	pVCpu->hwaccm.s.uCurrentASID = pCpu->uCurrentASID;
2700	if (pCpu->fFlushASIDBeforeUse)
2701	hmR0VmxFlushVPID(pVM, pVCpu, pVM->hwaccm.s.vmx.enmFlushVPID, 0 /* GCPtr */);
2702	}
2703	else
2704	{
2705	AssertMsg(pVCpu->hwaccm.s.uCurrentASID && pCpu->uCurrentASID,
2706	("hwaccm->uCurrentASID=%lu hwaccm->cTLBFlushes=%lu cpu->uCurrentASID=%lu cpu->cTLBFlushes=%lu\n",
2707	pVCpu->hwaccm.s.uCurrentASID, pVCpu->hwaccm.s.cTLBFlushes,
2708	pCpu->uCurrentASID, pCpu->cTLBFlushes));
2709
2710	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
2711	* not be executed. See hwaccmQueueInvlPage() where it is commented
2712	* out. Support individual entry flushing someday. */
2713	if (VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
2714	{
2715	/*
2716	* Flush individual guest entries using VPID from the TLB or as little as possible with EPT
2717	* as supported by the CPU.
2718	*/
2719	if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVVPID_CAPS_INDIV_ADDR)
2720	{
2721	for (unsigned i = 0; i < pVCpu->hwaccm.s.TlbShootdown.cPages; i++)
2722	hmR0VmxFlushVPID(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, pVCpu->hwaccm.s.TlbShootdown.aPages[i]);
2723	}
2724	else
2725	hmR0VmxFlushVPID(pVM, pVCpu, pVM->hwaccm.s.vmx.enmFlushVPID, 0 /* GCPtr */);
2726	}
2727	}
2728	pVCpu->hwaccm.s.TlbShootdown.cPages = 0;
2729	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
2730
2731	AssertMsg(pVCpu->hwaccm.s.cTLBFlushes == pCpu->cTLBFlushes,
2732	("Flush count mismatch for cpu %d (%x vs %x)\n", pCpu->idCpu, pVCpu->hwaccm.s.cTLBFlushes, pCpu->cTLBFlushes));
2733	AssertMsg(pCpu->uCurrentASID >= 1 && pCpu->uCurrentASID < pVM->hwaccm.s.uMaxASID,
2734	("cpu%d uCurrentASID = %x\n", pCpu->idCpu, pCpu->uCurrentASID));
2735	AssertMsg(pVCpu->hwaccm.s.uCurrentASID >= 1 && pVCpu->hwaccm.s.uCurrentASID < pVM->hwaccm.s.uMaxASID,
2736	("cpu%d VM uCurrentASID = %x\n", pCpu->idCpu, pVCpu->hwaccm.s.uCurrentASID));
2737
2738	int rc = VMXWriteVMCS(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hwaccm.s.uCurrentASID);
2739	AssertRC(rc);
2740
2741	# ifdef VBOX_WITH_STATISTICS
2742	if (pVCpu->hwaccm.s.fForceTLBFlush)
2743	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatFlushTLBWorldSwitch);
2744	else
2745	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatNoFlushTLBWorldSwitch);
2746	# endif
2747	}
2748
2749
2750	/**
2751	* Runs guest code in a VT-x VM.
2752	*
2753	* @returns VBox status code.
2754	* @param pVM Pointer to the VM.
2755	* @param pVCpu Pointer to the VMCPU.
2756	* @param pCtx Pointer to the guest CPU context.
2757	*/
2758	VMMR0DECL(int) VMXR0RunGuestCode(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
2759	{
2760	STAM_PROFILE_ADV_START(&pVCpu->hwaccm.s.StatEntry, x);
2761	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hwaccm.s.StatExit1);
2762	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hwaccm.s.StatExit2);
2763
2764	VBOXSTRICTRC rc = VINF_SUCCESS;
2765	int rc2;
2766	RTGCUINTREG val;
2767	RTGCUINTREG exitReason = (RTGCUINTREG)VMX_EXIT_INVALID;
2768	RTGCUINTREG instrError, cbInstr;
2769	RTGCUINTPTR exitQualification = 0;
2770	RTGCUINTPTR intInfo = 0; /* shut up buggy gcc 4 */
2771	RTGCUINTPTR errCode, instrInfo;
2772	bool fSetupTPRCaching = false;
2773	uint64_t u64OldLSTAR = 0;
2774	uint8_t u8LastTPR = 0;
2775	RTCCUINTREG uOldEFlags = ~(RTCCUINTREG)0;
2776	unsigned cResume = 0;
2777	#ifdef VBOX_STRICT
2778	RTCPUID idCpuCheck;
2779	bool fWasInLongMode = false;
2780	#endif
2781	#ifdef VBOX_HIGH_RES_TIMERS_HACK_IN_RING0
2782	uint64_t u64LastTime = RTTimeMilliTS();
2783	#endif
2784
2785	Assert(!(pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
2786	\|\| (pVCpu->hwaccm.s.vmx.pbVAPIC && pVM->hwaccm.s.vmx.pAPIC));
2787
2788	/*
2789	* Check if we need to use TPR shadowing.
2790	*/
2791	if ( CPUMIsGuestInLongModeEx(pCtx)
2792	\|\| ( (( pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
2793	\|\| pVM->hwaccm.s.fTRPPatchingAllowed)
2794	&& pVM->hwaccm.s.fHasIoApic)
2795	)
2796	{
2797	fSetupTPRCaching = true;
2798	}
2799
2800	Log2(("\nE"));
2801
2802	#ifdef VBOX_STRICT
2803	{
2804	RTCCUINTREG val2;
2805
2806	rc2 = VMXReadVMCS(VMX_VMCS_CTRL_PIN_EXEC_CONTROLS, &val2);
2807	AssertRC(rc2);
2808	Log2(("VMX_VMCS_CTRL_PIN_EXEC_CONTROLS = %08x\n", val2));
2809
2810	/* allowed zero */
2811	if ((val2 & pVM->hwaccm.s.vmx.msr.vmx_pin_ctls.n.disallowed0) != pVM->hwaccm.s.vmx.msr.vmx_pin_ctls.n.disallowed0)
2812	Log(("Invalid VMX_VMCS_CTRL_PIN_EXEC_CONTROLS: zero\n"));
2813
2814	/* allowed one */
2815	if ((val2 & ~pVM->hwaccm.s.vmx.msr.vmx_pin_ctls.n.allowed1) != 0)
2816	Log(("Invalid VMX_VMCS_CTRL_PIN_EXEC_CONTROLS: one\n"));
2817
2818	rc2 = VMXReadVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, &val2);
2819	AssertRC(rc2);
2820	Log2(("VMX_VMCS_CTRL_PROC_EXEC_CONTROLS = %08x\n", val2));
2821
2822	/*
2823	* Must be set according to the MSR, but can be cleared if nested paging is used.
2824	*/
2825	if (pVM->hwaccm.s.fNestedPaging)
2826	{
2827	val2 \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INVLPG_EXIT
2828	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
2829	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT;
2830	}
2831
2832	/* allowed zero */
2833	if ((val2 & pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.disallowed0) != pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.disallowed0)
2834	Log(("Invalid VMX_VMCS_CTRL_PROC_EXEC_CONTROLS: zero\n"));
2835
2836	/* allowed one */
2837	if ((val2 & ~pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1) != 0)
2838	Log(("Invalid VMX_VMCS_CTRL_PROC_EXEC_CONTROLS: one\n"));
2839
2840	rc2 = VMXReadVMCS(VMX_VMCS_CTRL_ENTRY_CONTROLS, &val2);
2841	AssertRC(rc2);
2842	Log2(("VMX_VMCS_CTRL_ENTRY_CONTROLS = %08x\n", val2));
2843
2844	/* allowed zero */
2845	if ((val2 & pVM->hwaccm.s.vmx.msr.vmx_entry.n.disallowed0) != pVM->hwaccm.s.vmx.msr.vmx_entry.n.disallowed0)
2846	Log(("Invalid VMX_VMCS_CTRL_ENTRY_CONTROLS: zero\n"));
2847
2848	/* allowed one */
2849	if ((val2 & ~pVM->hwaccm.s.vmx.msr.vmx_entry.n.allowed1) != 0)
2850	Log(("Invalid VMX_VMCS_CTRL_ENTRY_CONTROLS: one\n"));
2851
2852	rc2 = VMXReadVMCS(VMX_VMCS_CTRL_EXIT_CONTROLS, &val2);
2853	AssertRC(rc2);
2854	Log2(("VMX_VMCS_CTRL_EXIT_CONTROLS = %08x\n", val2));
2855
2856	/* allowed zero */
2857	if ((val2 & pVM->hwaccm.s.vmx.msr.vmx_exit.n.disallowed0) != pVM->hwaccm.s.vmx.msr.vmx_exit.n.disallowed0)
2858	Log(("Invalid VMX_VMCS_CTRL_EXIT_CONTROLS: zero\n"));
2859
2860	/* allowed one */
2861	if ((val2 & ~pVM->hwaccm.s.vmx.msr.vmx_exit.n.allowed1) != 0)
2862	Log(("Invalid VMX_VMCS_CTRL_EXIT_CONTROLS: one\n"));
2863	}
2864	fWasInLongMode = CPUMIsGuestInLongModeEx(pCtx);
2865	#endif /* VBOX_STRICT */
2866
2867	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
2868	pVCpu->hwaccm.s.vmx.VMCSCache.u64TimeEntry = RTTimeNanoTS();
2869	#endif
2870
2871	/*
2872	* We can jump to this point to resume execution after determining that a VM-exit is innocent.
2873	*/
2874	ResumeExecution:
2875	if (!STAM_REL_PROFILE_ADV_IS_RUNNING(&pVCpu->hwaccm.s.StatEntry))
2876	STAM_REL_PROFILE_ADV_STOP_START(&pVCpu->hwaccm.s.StatExit2, &pVCpu->hwaccm.s.StatEntry, x);
2877	AssertMsg(pVCpu->hwaccm.s.idEnteredCpu == RTMpCpuId(),
2878	("Expected %d, I'm %d; cResume=%d exitReason=%RGv exitQualification=%RGv\n",
2879	(int)pVCpu->hwaccm.s.idEnteredCpu, (int)RTMpCpuId(), cResume, exitReason, exitQualification));
2880	Assert(!HWACCMR0SuspendPending());
2881	/* Not allowed to switch modes without reloading the host state (32->64 switcher)!! */
2882	Assert(fWasInLongMode == CPUMIsGuestInLongModeEx(pCtx));
2883
2884	/*
2885	* Safety precaution; looping for too long here can have a very bad effect on the host.
2886	*/
2887	if (RT_UNLIKELY(++cResume > pVM->hwaccm.s.cMaxResumeLoops))
2888	{
2889	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitMaxResume);
2890	rc = VINF_EM_RAW_INTERRUPT;
2891	goto end;
2892	}
2893
2894	/*
2895	* Check for IRQ inhibition due to instruction fusing (sti, mov ss).
2896	*/
2897	if (VMCPU_FF_ISSET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
2898	{
2899	Log(("VM_FF_INHIBIT_INTERRUPTS at %RGv successor %RGv\n", (RTGCPTR)pCtx->rip, EMGetInhibitInterruptsPC(pVCpu)));
2900	if (pCtx->rip != EMGetInhibitInterruptsPC(pVCpu))
2901	{
2902	/*
2903	* Note: we intentionally don't clear VM_FF_INHIBIT_INTERRUPTS here.
2904	* Before we are able to execute this instruction in raw mode (iret to guest code) an external interrupt might
2905	* force a world switch again. Possibly allowing a guest interrupt to be dispatched in the process. This could
2906	* break the guest. Sounds very unlikely, but such timing sensitive problems are not as rare as you might think.
2907	*/
2908	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
2909	/* Irq inhibition is no longer active; clear the corresponding VMX state. */
2910	rc2 = VMXWriteVMCS(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, 0);
2911	AssertRC(rc2);
2912	}
2913	}
2914	else
2915	{
2916	/* Irq inhibition is no longer active; clear the corresponding VMX state. */
2917	rc2 = VMXWriteVMCS(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, 0);
2918	AssertRC(rc2);
2919	}
2920
2921	#ifdef VBOX_HIGH_RES_TIMERS_HACK_IN_RING0
2922	if (RT_UNLIKELY((cResume & 0xf) == 0))
2923	{
2924	uint64_t u64CurTime = RTTimeMilliTS();
2925
2926	if (RT_UNLIKELY(u64CurTime > u64LastTime))
2927	{
2928	u64LastTime = u64CurTime;
2929	TMTimerPollVoid(pVM, pVCpu);
2930	}
2931	}
2932	#endif
2933
2934	/*
2935	* Check for pending actions that force us to go back to ring-3.
2936	*/
2937	if ( VM_FF_ISPENDING(pVM, VM_FF_HWACCM_TO_R3_MASK \| VM_FF_REQUEST \| VM_FF_PGM_POOL_FLUSH_PENDING \| VM_FF_PDM_DMA)
2938	\|\| VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_HWACCM_TO_R3_MASK \| VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL \| VMCPU_FF_REQUEST))
2939	{
2940	/* Check if a sync operation is pending. */
2941	if (VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
2942	{
2943	rc = PGMSyncCR3(pVCpu, pCtx->cr0, pCtx->cr3, pCtx->cr4, VMCPU_FF_ISSET(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
2944	if (rc != VINF_SUCCESS)
2945	{
2946	AssertRC(VBOXSTRICTRC_VAL(rc));
2947	Log(("Pending pool sync is forcing us back to ring 3; rc=%d\n", VBOXSTRICTRC_VAL(rc)));
2948	goto end;
2949	}
2950	}
2951
2952	#ifdef DEBUG
2953	/* Intercept X86_XCPT_DB if stepping is enabled */
2954	if (!DBGFIsStepping(pVCpu))
2955	#endif
2956	{
2957	if ( VM_FF_ISPENDING(pVM, VM_FF_HWACCM_TO_R3_MASK)
2958	\|\| VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_HWACCM_TO_R3_MASK))
2959	{
2960	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatSwitchToR3);
2961	rc = RT_UNLIKELY(VM_FF_ISPENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
2962	goto end;
2963	}
2964	}
2965
2966	/* Pending request packets might contain actions that need immediate attention, such as pending hardware interrupts. */
2967	if ( VM_FF_ISPENDING(pVM, VM_FF_REQUEST)
2968	\|\| VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_REQUEST))
2969	{
2970	rc = VINF_EM_PENDING_REQUEST;
2971	goto end;
2972	}
2973
2974	/* Check if a pgm pool flush is in progress. */
2975	if (VM_FF_ISPENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
2976	{
2977	rc = VINF_PGM_POOL_FLUSH_PENDING;
2978	goto end;
2979	}
2980
2981	/* Check if DMA work is pending (2nd+ run). */
2982	if (VM_FF_ISPENDING(pVM, VM_FF_PDM_DMA) && cResume > 1)
2983	{
2984	rc = VINF_EM_RAW_TO_R3;
2985	goto end;
2986	}
2987	}
2988
2989	#ifdef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
2990	/*
2991	* Exit to ring-3 preemption/work is pending.
2992	*
2993	* Interrupts are disabled before the call to make sure we don't miss any interrupt
2994	* that would flag preemption (IPI, timer tick, ++). (Would've been nice to do this
2995	* further down, but hmR0VmxCheckPendingInterrupt makes that impossible.)
2996	*
2997	* Note! Interrupts must be disabled done before we check for TLB flushes; TLB
2998	* shootdowns rely on this.
2999	*/
3000	uOldEFlags = ASMIntDisableFlags();
3001	if (RTThreadPreemptIsPending(NIL_RTTHREAD))
3002	{
3003	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitPreemptPending);
3004	rc = VINF_EM_RAW_INTERRUPT;
3005	goto end;
3006	}
3007	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
3008	#endif
3009
3010	/*
3011	* When external interrupts are pending, we should exit the VM when IF is et.
3012	* Note: After VM_FF_INHIBIT_INTERRUPTS check!
3013	*/
3014	rc = hmR0VmxCheckPendingInterrupt(pVM, pVCpu, pCtx);
3015	if (RT_FAILURE(rc))
3016	goto end;
3017
3018	/** @todo check timers?? */
3019
3020	/*
3021	* TPR caching using CR8 is only available in 64-bit mode.
3022	* Note: The 32-bit exception for AMD (X86_CPUID_AMD_FEATURE_ECX_CR8L), but this appears missing in Intel CPUs.
3023	* Note: We can't do this in LoadGuestState() as PDMApicGetTPR can jump back to ring-3 (lock)!! (no longer true) .
3024	*/
3025	/** @todo query and update the TPR only when it could have been changed (mmio
3026	* access & wrsmr (x2apic) */
3027	if (fSetupTPRCaching)
3028	{
3029	/* TPR caching in CR8 */
3030	bool fPending;
3031
3032	rc2 = PDMApicGetTPR(pVCpu, &u8LastTPR, &fPending);
3033	AssertRC(rc2);
3034	/* The TPR can be found at offset 0x80 in the APIC mmio page. */
3035	pVCpu->hwaccm.s.vmx.pbVAPIC[0x80] = u8LastTPR;
3036
3037	/*
3038	* Two options here:
3039	* - external interrupt pending, but masked by the TPR value.
3040	* -> a CR8 update that lower the current TPR value should cause an exit
3041	* - no pending interrupts
3042	* -> We don't need to be explicitely notified. There are enough world switches for detecting pending interrupts.
3043	*/
3044
3045	/* cr8 bits 3-0 correspond to bits 7-4 of the task priority mmio register. */
3046	rc = VMXWriteVMCS(VMX_VMCS_CTRL_TPR_THRESHOLD, (fPending) ? (u8LastTPR >> 4) : 0);
3047	AssertRC(VBOXSTRICTRC_VAL(rc));
3048
3049	if (pVM->hwaccm.s.fTPRPatchingActive)
3050	{
3051	Assert(!CPUMIsGuestInLongModeEx(pCtx));
3052	/* Our patch code uses LSTAR for TPR caching. */
3053	pCtx->msrLSTAR = u8LastTPR;
3054
3055	/** @todo r=ramshankar: we should check for MSR-bitmap support here. */
3056	if (fPending)
3057	{
3058	/* A TPR change could activate a pending interrupt, so catch lstar writes. */
3059	hmR0VmxSetMSRPermission(pVCpu, MSR_K8_LSTAR, true, false);
3060	}
3061	else
3062	{
3063	/*
3064	* No interrupts are pending, so we don't need to be explicitely notified.
3065	* There are enough world switches for detecting pending interrupts.
3066	*/
3067	hmR0VmxSetMSRPermission(pVCpu, MSR_K8_LSTAR, true, true);
3068	}
3069	}
3070	}
3071
3072	#ifdef LOG_ENABLED
3073	if ( pVM->hwaccm.s.fNestedPaging
3074	\|\| pVM->hwaccm.s.vmx.fVPID)
3075	{
3076	PHMGLOBLCPUINFO pCpu = HWACCMR0GetCurrentCpu();
3077	if (pVCpu->hwaccm.s.idLastCpu != pCpu->idCpu)
3078	{
3079	LogFlow(("Force TLB flush due to rescheduling to a different cpu (%d vs %d)\n", pVCpu->hwaccm.s.idLastCpu,
3080	pCpu->idCpu));
3081	}
3082	else if (pVCpu->hwaccm.s.cTLBFlushes != pCpu->cTLBFlushes)
3083	{
3084	LogFlow(("Force TLB flush due to changed TLB flush count (%x vs %x)\n", pVCpu->hwaccm.s.cTLBFlushes,
3085	pCpu->cTLBFlushes));
3086	}
3087	else if (VMCPU_FF_ISSET(pVCpu, VMCPU_FF_TLB_FLUSH))
3088	LogFlow(("Manual TLB flush\n"));
3089	}
3090	#endif
3091	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
3092	PGMRZDynMapFlushAutoSet(pVCpu);
3093	#endif
3094
3095	/*
3096	* NOTE: DO NOT DO ANYTHING AFTER THIS POINT THAT MIGHT JUMP BACK TO RING-3!
3097	* (until the actual world switch)
3098	*/
3099	#ifdef VBOX_STRICT
3100	idCpuCheck = RTMpCpuId();
3101	#endif
3102	#ifdef LOG_ENABLED
3103	VMMR0LogFlushDisable(pVCpu);
3104	#endif
3105
3106	/*
3107	* Save the host state first.
3108	*/
3109	if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_HOST_CONTEXT)
3110	{
3111	rc = VMXR0SaveHostState(pVM, pVCpu);
3112	if (RT_UNLIKELY(rc != VINF_SUCCESS))
3113	{
3114	VMMR0LogFlushEnable(pVCpu);
3115	goto end;
3116	}
3117	}
3118
3119	/*
3120	* Load the guest state.
3121	*/
3122	if (!pVCpu->hwaccm.s.fContextUseFlags)
3123	{
3124	VMXR0LoadMinimalGuestState(pVM, pVCpu, pCtx);
3125	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatLoadMinimal);
3126	}
3127	else
3128	{
3129	rc = VMXR0LoadGuestState(pVM, pVCpu, pCtx);
3130	if (RT_UNLIKELY(rc != VINF_SUCCESS))
3131	{
3132	VMMR0LogFlushEnable(pVCpu);
3133	goto end;
3134	}
3135	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatLoadFull);
3136	}
3137
3138	#ifndef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
3139	/*
3140	* Disable interrupts to make sure a poke will interrupt execution.
3141	* This must be done before we check for TLB flushes; TLB shootdowns rely on this.
3142	*/
3143	uOldEFlags = ASMIntDisableFlags();
3144	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
3145	#endif
3146
3147	/* Non-register state Guest Context */
3148	/** @todo change me according to cpu state */
3149	rc2 = VMXWriteVMCS(VMX_VMCS32_GUEST_ACTIVITY_STATE, VMX_CMS_GUEST_ACTIVITY_ACTIVE);
3150	AssertRC(rc2);
3151
3152	/* Set TLB flush state as checked until we return from the world switch. */
3153	ASMAtomicWriteBool(&pVCpu->hwaccm.s.fCheckedTLBFlush, true);
3154	/* Deal with tagged TLB setup and invalidation. */
3155	pVM->hwaccm.s.vmx.pfnSetupTaggedTLB(pVM, pVCpu);
3156
3157	/*
3158	* Manual save and restore:
3159	* - General purpose registers except RIP, RSP
3160	*
3161	* Trashed:
3162	* - CR2 (we don't care)
3163	* - LDTR (reset to 0)
3164	* - DRx (presumably not changed at all)
3165	* - DR7 (reset to 0x400)
3166	* - EFLAGS (reset to RT_BIT(1); not relevant)
3167	*/
3168
3169	/* All done! Let's start VM execution. */
3170	STAM_PROFILE_ADV_STOP_START(&pVCpu->hwaccm.s.StatEntry, &pVCpu->hwaccm.s.StatInGC, x);
3171	Assert(idCpuCheck == RTMpCpuId());
3172
3173	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
3174	pVCpu->hwaccm.s.vmx.VMCSCache.cResume = cResume;
3175	pVCpu->hwaccm.s.vmx.VMCSCache.u64TimeSwitch = RTTimeNanoTS();
3176	#endif
3177
3178	/*
3179	* Save the current TPR value in the LSTAR MSR so our patches can access it.
3180	*/
3181	if (pVM->hwaccm.s.fTPRPatchingActive)
3182	{
3183	Assert(pVM->hwaccm.s.fTPRPatchingActive);
3184	u64OldLSTAR = ASMRdMsr(MSR_K8_LSTAR);
3185	ASMWrMsr(MSR_K8_LSTAR, u8LastTPR);
3186	}
3187
3188	TMNotifyStartOfExecution(pVCpu);
3189
3190	#ifndef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
3191	/*
3192	* Save the current Host TSC_AUX and write the guest TSC_AUX to the host, so that
3193	* RDTSCPs (that don't cause exits) reads the guest MSR. See @bugref{3324}.
3194	*/
3195	if ( (pVCpu->hwaccm.s.vmx.proc_ctls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
3196	&& !(pVCpu->hwaccm.s.vmx.proc_ctls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT))
3197	{
3198	pVCpu->hwaccm.s.u64HostTSCAux = ASMRdMsr(MSR_K8_TSC_AUX);
3199	uint64_t u64GuestTSCAux = 0;
3200	rc2 = CPUMQueryGuestMsr(pVCpu, MSR_K8_TSC_AUX, &u64GuestTSCAux);
3201	AssertRC(rc2);
3202	ASMWrMsr(MSR_K8_TSC_AUX, u64GuestTSCAux);
3203	}
3204	#endif
3205
3206	#ifdef VBOX_WITH_KERNEL_USING_XMM
3207	rc = hwaccmR0VMXStartVMWrapXMM(pVCpu->hwaccm.s.fResumeVM, pCtx, &pVCpu->hwaccm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hwaccm.s.vmx.pfnStartVM);
3208	#else
3209	rc = pVCpu->hwaccm.s.vmx.pfnStartVM(pVCpu->hwaccm.s.fResumeVM, pCtx, &pVCpu->hwaccm.s.vmx.VMCSCache, pVM, pVCpu);
3210	#endif
3211	ASMAtomicWriteBool(&pVCpu->hwaccm.s.fCheckedTLBFlush, false);
3212	ASMAtomicIncU32(&pVCpu->hwaccm.s.cWorldSwitchExits);
3213
3214	/* Possibly the last TSC value seen by the guest (too high) (only when we're in TSC offset mode). */
3215	if (!(pVCpu->hwaccm.s.vmx.proc_ctls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT))
3216	{
3217	#ifndef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
3218	/* Restore host's TSC_AUX. */
3219	if (pVCpu->hwaccm.s.vmx.proc_ctls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
3220	ASMWrMsr(MSR_K8_TSC_AUX, pVCpu->hwaccm.s.u64HostTSCAux);
3221	#endif
3222
3223	TMCpuTickSetLastSeen(pVCpu,
3224	ASMReadTSC() + pVCpu->hwaccm.s.vmx.u64TSCOffset - 0x400 /* guestimate of world switch overhead in clock ticks */);
3225	}
3226
3227	TMNotifyEndOfExecution(pVCpu);
3228	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED);
3229	Assert(!(ASMGetFlags() & X86_EFL_IF));
3230
3231	/*
3232	* Restore the host LSTAR MSR if the guest could have changed it.
3233	*/
3234	if (pVM->hwaccm.s.fTPRPatchingActive)
3235	{
3236	Assert(pVM->hwaccm.s.fTPRPatchingActive);
3237	pVCpu->hwaccm.s.vmx.pbVAPIC[0x80] = pCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR);
3238	ASMWrMsr(MSR_K8_LSTAR, u64OldLSTAR);
3239	}
3240
3241	STAM_PROFILE_ADV_STOP_START(&pVCpu->hwaccm.s.StatInGC, &pVCpu->hwaccm.s.StatExit1, x);
3242	ASMSetFlags(uOldEFlags);
3243	#ifdef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
3244	uOldEFlags = ~(RTCCUINTREG)0;
3245	#endif
3246
3247	AssertMsg(!pVCpu->hwaccm.s.vmx.VMCSCache.Write.cValidEntries, ("pVCpu->hwaccm.s.vmx.VMCSCache.Write.cValidEntries=%d\n",
3248	pVCpu->hwaccm.s.vmx.VMCSCache.Write.cValidEntries));
3249
3250	/* In case we execute a goto ResumeExecution later on. */
3251	pVCpu->hwaccm.s.fResumeVM = true;
3252	pVCpu->hwaccm.s.fForceTLBFlush = false;
3253
3254	/*
3255	* !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
3256	* IMPORTANT: WE CAN'T DO ANY LOGGING OR OPERATIONS THAT CAN DO A LONGJMP BACK TO RING 3 BEFORE WE'VE SYNCED BACK (MOST OF) THE GUEST STATE
3257	* !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
3258	*/
3259
3260	if (RT_UNLIKELY(rc != VINF_SUCCESS))
3261	{
3262	hmR0VmxReportWorldSwitchError(pVM, pVCpu, rc, pCtx);
3263	VMMR0LogFlushEnable(pVCpu);
3264	goto end;
3265	}
3266
3267	/* Success. Query the guest state and figure out what has happened. */
3268
3269	/* Investigate why there was a VM-exit. */
3270	rc2 = VMXReadCachedVMCS(VMX_VMCS32_RO_EXIT_REASON, &exitReason);
3271	STAM_COUNTER_INC(&pVCpu->hwaccm.s.paStatExitReasonR0[exitReason & MASK_EXITREASON_STAT]);
3272
3273	exitReason &= 0xffff; /* bit 0-15 contain the exit code. */
3274	rc2 \|= VMXReadCachedVMCS(VMX_VMCS32_RO_VM_INSTR_ERROR, &instrError);
3275	rc2 \|= VMXReadCachedVMCS(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &cbInstr);
3276	rc2 \|= VMXReadCachedVMCS(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &intInfo);
3277	/* might not be valid; depends on VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID. */
3278	rc2 \|= VMXReadCachedVMCS(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERRCODE, &errCode);
3279	rc2 \|= VMXReadCachedVMCS(VMX_VMCS32_RO_EXIT_INSTR_INFO, &instrInfo);
3280	rc2 \|= VMXReadCachedVMCS(VMX_VMCS_RO_EXIT_QUALIFICATION, &exitQualification);
3281	AssertRC(rc2);
3282
3283	/*
3284	* Sync back the guest state.
3285	*/
3286	rc2 = VMXR0SaveGuestState(pVM, pVCpu, pCtx);
3287	AssertRC(rc2);
3288
3289	/* Note! NOW IT'S SAFE FOR LOGGING! */
3290	VMMR0LogFlushEnable(pVCpu);
3291	Log2(("Raw exit reason %08x\n", exitReason));
3292	#if ARCH_BITS == 64 /* for the time being */
3293	VBOXVMM_R0_HMVMX_VMEXIT(pVCpu, pCtx, exitReason);
3294	#endif
3295
3296	/*
3297	* Check if an injected event was interrupted prematurely.
3298	*/
3299	rc2 = VMXReadCachedVMCS(VMX_VMCS32_RO_IDT_INFO, &val);
3300	AssertRC(rc2);
3301	pVCpu->hwaccm.s.Event.intInfo = VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(val);
3302	if ( VMX_EXIT_INTERRUPTION_INFO_VALID(pVCpu->hwaccm.s.Event.intInfo)
3303	/* Ignore 'int xx' as they'll be restarted anyway. */
3304	&& VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hwaccm.s.Event.intInfo) != VMX_EXIT_INTERRUPTION_INFO_TYPE_SW
3305	/* Ignore software exceptions (such as int3) as they'll reoccur when we restart the instruction anyway. */
3306	&& VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hwaccm.s.Event.intInfo) != VMX_EXIT_INTERRUPTION_INFO_TYPE_SWEXCPT)
3307	{
3308	Assert(!pVCpu->hwaccm.s.Event.fPending);
3309	pVCpu->hwaccm.s.Event.fPending = true;
3310	/* Error code present? */
3311	if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(pVCpu->hwaccm.s.Event.intInfo))
3312	{
3313	rc2 = VMXReadCachedVMCS(VMX_VMCS32_RO_IDT_ERRCODE, &val);
3314	AssertRC(rc2);
3315	pVCpu->hwaccm.s.Event.errCode = val;
3316	Log(("Pending inject %RX64 at %RGv exit=%08x intInfo=%08x exitQualification=%RGv pending error=%RX64\n",
3317	pVCpu->hwaccm.s.Event.intInfo, (RTGCPTR)pCtx->rip, exitReason, intInfo, exitQualification, val));
3318	}
3319	else
3320	{
3321	Log(("Pending inject %RX64 at %RGv exit=%08x intInfo=%08x exitQualification=%RGv\n", pVCpu->hwaccm.s.Event.intInfo,
3322	(RTGCPTR)pCtx->rip, exitReason, intInfo, exitQualification));
3323	pVCpu->hwaccm.s.Event.errCode = 0;
3324	}
3325	}
3326	#ifdef VBOX_STRICT
3327	else if ( VMX_EXIT_INTERRUPTION_INFO_VALID(pVCpu->hwaccm.s.Event.intInfo)
3328	/* Ignore software exceptions (such as int3) as they're reoccur when we restart the instruction anyway. */
3329	&& VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hwaccm.s.Event.intInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_SWEXCPT)
3330	{
3331	Log(("Ignore pending inject %RX64 at %RGv exit=%08x intInfo=%08x exitQualification=%RGv\n",
3332	pVCpu->hwaccm.s.Event.intInfo, (RTGCPTR)pCtx->rip, exitReason, intInfo, exitQualification));
3333	}
3334
3335	if (exitReason == VMX_EXIT_ERR_INVALID_GUEST_STATE)
3336	HWACCMDumpRegs(pVM, pVCpu, pCtx);
3337	#endif
3338
3339	Log2(("E%d: New EIP=%x:%RGv\n", (uint32_t)exitReason, pCtx->cs.Sel, (RTGCPTR)pCtx->rip));
3340	Log2(("Exit reason %d, exitQualification %RGv\n", (uint32_t)exitReason, exitQualification));
3341	Log2(("instrInfo=%d instrError=%d instr length=%d\n", (uint32_t)instrInfo, (uint32_t)instrError, (uint32_t)cbInstr));
3342	Log2(("Interruption error code %d\n", (uint32_t)errCode));
3343	Log2(("IntInfo = %08x\n", (uint32_t)intInfo));
3344
3345	/*
3346	* Sync back the TPR if it was changed.
3347	*/
3348	if ( fSetupTPRCaching
3349	&& u8LastTPR != pVCpu->hwaccm.s.vmx.pbVAPIC[0x80])
3350	{
3351	rc2 = PDMApicSetTPR(pVCpu, pVCpu->hwaccm.s.vmx.pbVAPIC[0x80]);
3352	AssertRC(rc2);
3353	}
3354
3355	#ifdef DBGFTRACE_ENABLED /** @todo DTrace later. */
3356	RTTraceBufAddMsgF(pVM->CTX_SUFF(hTraceBuf), "vmexit %08x %016RX64 at %04:%08RX64 %RX64",
3357	exitReason, (uint64_t)exitQualification, pCtx->cs.Sel, pCtx->rip, (uint64_t)intInfo);
3358	#endif
3359	STAM_PROFILE_ADV_STOP_START(&pVCpu->hwaccm.s.StatExit1, &pVCpu->hwaccm.s.StatExit2, x);
3360
3361	/* Some cases don't need a complete resync of the guest CPU state; handle them here. */
3362	Assert(rc == VINF_SUCCESS); /* might consider VERR_IPE_UNINITIALIZED_STATUS here later... */
3363	switch (exitReason)
3364	{
3365	case VMX_EXIT_EXCEPTION: /* 0 Exception or non-maskable interrupt (NMI). */
3366	case VMX_EXIT_EXTERNAL_IRQ: /* 1 External interrupt. */
3367	{
3368	uint32_t vector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(intInfo);
3369
3370	if (!VMX_EXIT_INTERRUPTION_INFO_VALID(intInfo))
3371	{
3372	Assert(exitReason == VMX_EXIT_EXTERNAL_IRQ);
3373	#if 0 //def VBOX_WITH_VMMR0_DISABLE_PREEMPTION
3374	if ( RTThreadPreemptIsPendingTrusty()
3375	&& !RTThreadPreemptIsPending(NIL_RTTHREAD))
3376	goto ResumeExecution;
3377	#endif
3378	/* External interrupt; leave to allow it to be dispatched again. */
3379	rc = VINF_EM_RAW_INTERRUPT;
3380	break;
3381	}
3382	STAM_PROFILE_ADV_START(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3383	switch (VMX_EXIT_INTERRUPTION_INFO_TYPE(intInfo))
3384	{
3385	case VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI: /* Non-maskable interrupt. */
3386	/* External interrupt; leave to allow it to be dispatched again. */
3387	rc = VINF_EM_RAW_INTERRUPT;
3388	break;
3389
3390	case VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT: /* External hardware interrupt. */
3391	AssertFailed(); /* can't come here; fails the first check. */
3392	break;
3393
3394	case VMX_EXIT_INTERRUPTION_INFO_TYPE_DBEXCPT: /* Unknown why we get this type for #DB */
3395	case VMX_EXIT_INTERRUPTION_INFO_TYPE_SWEXCPT: /* Software exception. (#BP or #OF) */
3396	Assert(vector == 1 \|\| vector == 3 \|\| vector == 4);
3397	/* no break */
3398	case VMX_EXIT_INTERRUPTION_INFO_TYPE_HWEXCPT: /* Hardware exception. */
3399	Log2(("Hardware/software interrupt %d\n", vector));
3400	switch (vector)
3401	{
3402	case X86_XCPT_NM:
3403	{
3404	Log(("#NM fault at %RGv error code %x\n", (RTGCPTR)pCtx->rip, errCode));
3405
3406	/** @todo don't intercept #NM exceptions anymore when we've activated the guest FPU state. */
3407	/* If we sync the FPU/XMM state on-demand, then we can continue execution as if nothing has happened. */
3408	rc = CPUMR0LoadGuestFPU(pVM, pVCpu, pCtx);
3409	if (rc == VINF_SUCCESS)
3410	{
3411	Assert(CPUMIsGuestFPUStateActive(pVCpu));
3412
3413	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitShadowNM);
3414
3415	/* Continue execution. */
3416	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_GUEST_CR0;
3417
3418	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3419	goto ResumeExecution;
3420	}
3421
3422	Log(("Forward #NM fault to the guest\n"));
3423	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestNM);
3424	rc2 = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo),
3425	cbInstr, 0);
3426	AssertRC(rc2);
3427	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3428	goto ResumeExecution;
3429	}
3430
3431	case X86_XCPT_PF: /* Page fault */
3432	{
3433	#ifdef VBOX_ALWAYS_TRAP_PF
3434	if (pVM->hwaccm.s.fNestedPaging)
3435	{
3436	/*
3437	* A genuine pagefault. Forward the trap to the guest by injecting the exception and resuming execution.
3438	*/
3439	Log(("Guest page fault at %RGv cr2=%RGv error code %RGv rsp=%RGv\n", (RTGCPTR)pCtx->rip, exitQualification,
3440	errCode, (RTGCPTR)pCtx->rsp));
3441
3442	Assert(CPUMIsGuestInPagedProtectedModeEx(pCtx));
3443
3444	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestPF);
3445
3446	/* Now we must update CR2. */
3447	pCtx->cr2 = exitQualification;
3448	rc2 = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo),
3449	cbInstr, errCode);
3450	AssertRC(rc2);
3451
3452	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3453	goto ResumeExecution;
3454	}
3455	#else
3456	Assert(!pVM->hwaccm.s.fNestedPaging);
3457	#endif
3458
3459	#ifdef VBOX_HWACCM_WITH_GUEST_PATCHING
3460	/* Shortcut for APIC TPR reads and writes; 32 bits guests only */
3461	if ( pVM->hwaccm.s.fTRPPatchingAllowed
3462	&& pVM->hwaccm.s.pGuestPatchMem
3463	&& (exitQualification & 0xfff) == 0x080
3464	&& !(errCode & X86_TRAP_PF_P) /* not present */
3465	&& CPUMGetGuestCPL(pVCpu) == 0
3466	&& !CPUMIsGuestInLongModeEx(pCtx)
3467	&& pVM->hwaccm.s.cPatches < RT_ELEMENTS(pVM->hwaccm.s.aPatches))
3468	{
3469	RTGCPHYS GCPhysApicBase, GCPhys;
3470	PDMApicGetBase(pVM, &GCPhysApicBase); /** @todo cache this */
3471	GCPhysApicBase &= PAGE_BASE_GC_MASK;
3472
3473	rc = PGMGstGetPage(pVCpu, (RTGCPTR)exitQualification, NULL, &GCPhys);
3474	if ( rc == VINF_SUCCESS
3475	&& GCPhys == GCPhysApicBase)
3476	{
3477	/* Only attempt to patch the instruction once. */
3478	PHWACCMTPRPATCH pPatch = (PHWACCMTPRPATCH)RTAvloU32Get(&pVM->hwaccm.s.PatchTree, (AVLOU32KEY)pCtx->eip);
3479	if (!pPatch)
3480	{
3481	rc = VINF_EM_HWACCM_PATCH_TPR_INSTR;
3482	break;
3483	}
3484	}
3485	}
3486	#endif
3487
3488	Log2(("Page fault at %RGv error code %x\n", exitQualification, errCode));
3489	/* Exit qualification contains the linear address of the page fault. */
3490	TRPMAssertTrap(pVCpu, X86_XCPT_PF, TRPM_TRAP);
3491	TRPMSetErrorCode(pVCpu, errCode);
3492	TRPMSetFaultAddress(pVCpu, exitQualification);
3493
3494	/* Shortcut for APIC TPR reads and writes. */
3495	if ( (exitQualification & 0xfff) == 0x080
3496	&& !(errCode & X86_TRAP_PF_P) /* not present */
3497	&& fSetupTPRCaching
3498	&& (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC))
3499	{
3500	RTGCPHYS GCPhysApicBase, GCPhys;
3501	PDMApicGetBase(pVM, &GCPhysApicBase); /* @todo cache this */
3502	GCPhysApicBase &= PAGE_BASE_GC_MASK;
3503
3504	rc = PGMGstGetPage(pVCpu, (RTGCPTR)exitQualification, NULL, &GCPhys);
3505	if ( rc == VINF_SUCCESS
3506	&& GCPhys == GCPhysApicBase)
3507	{
3508	Log(("Enable VT-x virtual APIC access filtering\n"));
3509	rc2 = IOMMMIOMapMMIOHCPage(pVM, GCPhysApicBase, pVM->hwaccm.s.vmx.pAPICPhys, X86_PTE_RW \| X86_PTE_P);
3510	AssertRC(rc2);
3511	}
3512	}
3513
3514	/* Forward it to our trap handler first, in case our shadow pages are out of sync. */
3515	rc = PGMTrap0eHandler(pVCpu, errCode, CPUMCTX2CORE(pCtx), (RTGCPTR)exitQualification);
3516	Log2(("PGMTrap0eHandler %RGv returned %Rrc\n", (RTGCPTR)pCtx->rip, VBOXSTRICTRC_VAL(rc)));
3517
3518	if (rc == VINF_SUCCESS)
3519	{ /* We've successfully synced our shadow pages, so let's just continue execution. */
3520	Log2(("Shadow page fault at %RGv cr2=%RGv error code %x\n", (RTGCPTR)pCtx->rip, exitQualification ,errCode));
3521	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitShadowPF);
3522
3523	TRPMResetTrap(pVCpu);
3524	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3525	goto ResumeExecution;
3526	}
3527	else if (rc == VINF_EM_RAW_GUEST_TRAP)
3528	{
3529	/*
3530	* A genuine pagefault. Forward the trap to the guest by injecting the exception and resuming execution.
3531	*/
3532	Log2(("Forward page fault to the guest\n"));
3533
3534	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestPF);
3535	/* The error code might have been changed. */
3536	errCode = TRPMGetErrorCode(pVCpu);
3537
3538	TRPMResetTrap(pVCpu);
3539
3540	/* Now we must update CR2. */
3541	pCtx->cr2 = exitQualification;
3542	rc2 = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo),
3543	cbInstr, errCode);
3544	AssertRC(rc2);
3545
3546	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3547	goto ResumeExecution;
3548	}
3549	#ifdef VBOX_STRICT
3550	if (rc != VINF_EM_RAW_EMULATE_INSTR && rc != VINF_EM_RAW_EMULATE_IO_BLOCK)
3551	Log2(("PGMTrap0eHandler failed with %d\n", VBOXSTRICTRC_VAL(rc)));
3552	#endif
3553	/* Need to go back to the recompiler to emulate the instruction. */
3554	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitShadowPFEM);
3555	TRPMResetTrap(pVCpu);
3556	break;
3557	}
3558
3559	case X86_XCPT_MF: /* Floating point exception. */
3560	{
3561	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestMF);
3562	if (!(pCtx->cr0 & X86_CR0_NE))
3563	{
3564	/* old style FPU error reporting needs some extra work. */
3565	/** @todo don't fall back to the recompiler, but do it manually. */
3566	rc = VINF_EM_RAW_EMULATE_INSTR;
3567	break;
3568	}
3569	Log(("Trap %x at %04X:%RGv\n", vector, pCtx->cs.Sel, (RTGCPTR)pCtx->rip));
3570	rc2 = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo),
3571	cbInstr, errCode);
3572	AssertRC(rc2);
3573
3574	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3575	goto ResumeExecution;
3576	}
3577
3578	case X86_XCPT_DB: /* Debug exception. */
3579	{
3580	uint64_t uDR6;
3581
3582	/*
3583	* DR6, DR7.GD and IA32_DEBUGCTL.LBR are not updated yet.
3584	*
3585	* Exit qualification bits:
3586	* 3:0 B0-B3 which breakpoint condition was met
3587	* 12:4 Reserved (0)
3588	* 13 BD - debug register access detected
3589	* 14 BS - single step execution or branch taken
3590	* 63:15 Reserved (0)
3591	*/
3592	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestDB);
3593
3594	/* Note that we don't support guest and host-initiated debugging at the same time. */
3595
3596	uDR6 = X86_DR6_INIT_VAL;
3597	uDR6 \|= (exitQualification & (X86_DR6_B0\|X86_DR6_B1\|X86_DR6_B2\|X86_DR6_B3\|X86_DR6_BD\|X86_DR6_BS));
3598	rc = DBGFRZTrap01Handler(pVM, pVCpu, CPUMCTX2CORE(pCtx), uDR6);
3599	if (rc == VINF_EM_RAW_GUEST_TRAP)
3600	{
3601	/* Update DR6 here. */
3602	pCtx->dr[6] = uDR6;
3603
3604	/* Resync DR6 if the debug state is active. */
3605	if (CPUMIsGuestDebugStateActive(pVCpu))
3606	ASMSetDR6(pCtx->dr[6]);
3607
3608	/* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
3609	pCtx->dr[7] &= ~X86_DR7_GD;
3610
3611	/* Paranoia. */
3612	pCtx->dr[7] &= 0xffffffff; /* upper 32 bits reserved */
3613	pCtx->dr[7] &= ~(RT_BIT(11) \| RT_BIT(12) \| RT_BIT(14) \| RT_BIT(15)); /* must be zero */
3614	pCtx->dr[7] \|= 0x400; /* must be one */
3615
3616	/* Resync DR7 */
3617	rc2 = VMXWriteVMCS64(VMX_VMCS64_GUEST_DR7, pCtx->dr[7]);
3618	AssertRC(rc2);
3619
3620	Log(("Trap %x (debug) at %RGv exit qualification %RX64 dr6=%x dr7=%x\n", vector, (RTGCPTR)pCtx->rip,
3621	exitQualification, (uint32_t)pCtx->dr[6], (uint32_t)pCtx->dr[7]));
3622	rc2 = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo),
3623	cbInstr, errCode);
3624	AssertRC(rc2);
3625
3626	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3627	goto ResumeExecution;
3628	}
3629	/* Return to ring 3 to deal with the debug exit code. */
3630	Log(("Debugger hardware BP at %04x:%RGv (rc=%Rrc)\n", pCtx->cs.Sel, pCtx->rip, VBOXSTRICTRC_VAL(rc)));
3631	break;
3632	}
3633
3634	case X86_XCPT_BP: /* Breakpoint. */
3635	{
3636	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestBP);
3637	rc = DBGFRZTrap03Handler(pVM, pVCpu, CPUMCTX2CORE(pCtx));
3638	if (rc == VINF_EM_RAW_GUEST_TRAP)
3639	{
3640	Log(("Guest #BP at %04x:%RGv\n", pCtx->cs.Sel, pCtx->rip));
3641	rc2 = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo),
3642	cbInstr, errCode);
3643	AssertRC(rc2);
3644	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3645	goto ResumeExecution;
3646	}
3647	if (rc == VINF_SUCCESS)
3648	{
3649	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3650	goto ResumeExecution;
3651	}
3652	Log(("Debugger BP at %04x:%RGv (rc=%Rrc)\n", pCtx->cs.Sel, pCtx->rip, VBOXSTRICTRC_VAL(rc)));
3653	break;
3654	}
3655
3656	case X86_XCPT_GP: /* General protection failure exception. */
3657	{
3658	uint32_t cbOp;
3659	PDISCPUSTATE pDis = &pVCpu->hwaccm.s.DisState;
3660
3661	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestGP);
3662	#ifdef VBOX_STRICT
3663	if ( !CPUMIsGuestInRealModeEx(pCtx)
3664	\|\| !pVM->hwaccm.s.vmx.pRealModeTSS)
3665	{
3666	Log(("Trap %x at %04X:%RGv errorCode=%RGv\n", vector, pCtx->cs.Sel, (RTGCPTR)pCtx->rip, errCode));
3667	rc2 = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo),
3668	cbInstr, errCode);
3669	AssertRC(rc2);
3670	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3671	goto ResumeExecution;
3672	}
3673	#endif
3674	Assert(CPUMIsGuestInRealModeEx(pCtx));
3675
3676	LogFlow(("Real mode X86_XCPT_GP instruction emulation at %x:%RGv\n", pCtx->cs.Sel, (RTGCPTR)pCtx->rip));
3677
3678	rc2 = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbOp);
3679	if (RT_SUCCESS(rc2))
3680	{
3681	bool fUpdateRIP = true;
3682
3683	rc = VINF_SUCCESS;
3684	Assert(cbOp == pDis->cbInstr);
3685	switch (pDis->pCurInstr->uOpcode)
3686	{
3687	case OP_CLI:
3688	pCtx->eflags.Bits.u1IF = 0;
3689	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitCli);
3690	break;
3691
3692	case OP_STI:
3693	pCtx->eflags.Bits.u1IF = 1;
3694	EMSetInhibitInterruptsPC(pVCpu, pCtx->rip + pDis->cbInstr);
3695	Assert(VMCPU_FF_ISSET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
3696	rc2 = VMXWriteVMCS(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE,
3697	VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
3698	AssertRC(rc2);
3699	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitSti);
3700	break;
3701
3702	case OP_HLT:
3703	fUpdateRIP = false;
3704	rc = VINF_EM_HALT;
3705	pCtx->rip += pDis->cbInstr;
3706	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitHlt);
3707	break;
3708
3709	case OP_POPF:
3710	{
3711	RTGCPTR GCPtrStack;
3712	uint32_t cbParm;
3713	uint32_t uMask;
3714	X86EFLAGS eflags;
3715
3716	if (pDis->fPrefix & DISPREFIX_OPSIZE)
3717	{
3718	cbParm = 4;
3719	uMask = 0xffffffff;
3720	}
3721	else
3722	{
3723	cbParm = 2;
3724	uMask = 0xffff;
3725	}
3726
3727	rc2 = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pCtx), pCtx->esp & uMask, 0, &GCPtrStack);
3728	if (RT_FAILURE(rc2))
3729	{
3730	rc = VERR_EM_INTERPRETER;
3731	break;
3732	}
3733	eflags.u = 0;
3734	rc2 = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &eflags.u, cbParm);
3735	if (RT_FAILURE(rc2))
3736	{
3737	rc = VERR_EM_INTERPRETER;
3738	break;
3739	}
3740	LogFlow(("POPF %x -> %RGv mask=%x\n", eflags.u, pCtx->rsp, uMask));
3741	pCtx->eflags.u = (pCtx->eflags.u & ~(X86_EFL_POPF_BITS & uMask))
3742	\| (eflags.u & X86_EFL_POPF_BITS & uMask);
3743	/* RF cleared when popped in real mode; see pushf description in AMD manual. */
3744	pCtx->eflags.Bits.u1RF = 0;
3745	pCtx->esp += cbParm;
3746	pCtx->esp &= uMask;
3747
3748	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitPopf);
3749	break;
3750	}
3751
3752	case OP_PUSHF:
3753	{
3754	RTGCPTR GCPtrStack;
3755	uint32_t cbParm;
3756	uint32_t uMask;
3757	X86EFLAGS eflags;
3758
3759	if (pDis->fPrefix & DISPREFIX_OPSIZE)
3760	{
3761	cbParm = 4;
3762	uMask = 0xffffffff;
3763	}
3764	else
3765	{
3766	cbParm = 2;
3767	uMask = 0xffff;
3768	}
3769
3770	rc2 = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pCtx), (pCtx->esp - cbParm) & uMask, 0,
3771	&GCPtrStack);
3772	if (RT_FAILURE(rc2))
3773	{
3774	rc = VERR_EM_INTERPRETER;
3775	break;
3776	}
3777	eflags = pCtx->eflags;
3778	/* RF & VM cleared when pushed in real mode; see pushf description in AMD manual. */
3779	eflags.Bits.u1RF = 0;
3780	eflags.Bits.u1VM = 0;
3781
3782	rc2 = PGMPhysWrite(pVM, (RTGCPHYS)GCPtrStack, &eflags.u, cbParm);
3783	if (RT_FAILURE(rc2))
3784	{
3785	rc = VERR_EM_INTERPRETER;
3786	break;
3787	}
3788	LogFlow(("PUSHF %x -> %RGv\n", eflags.u, GCPtrStack));
3789	pCtx->esp -= cbParm;
3790	pCtx->esp &= uMask;
3791	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitPushf);
3792	break;
3793	}
3794
3795	case OP_IRET:
3796	{
3797	RTGCPTR GCPtrStack;
3798	uint32_t uMask = 0xffff;
3799	uint16_t aIretFrame[3];
3800
3801	if (pDis->fPrefix & (DISPREFIX_OPSIZE \| DISPREFIX_ADDRSIZE))
3802	{
3803	rc = VERR_EM_INTERPRETER;
3804	break;
3805	}
3806
3807	rc2 = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pCtx), pCtx->esp & uMask, 0, &GCPtrStack);
3808	if (RT_FAILURE(rc2))
3809	{
3810	rc = VERR_EM_INTERPRETER;
3811	break;
3812	}
3813	rc2 = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &aIretFrame[0], sizeof(aIretFrame));
3814	if (RT_FAILURE(rc2))
3815	{
3816	rc = VERR_EM_INTERPRETER;
3817	break;
3818	}
3819	pCtx->ip = aIretFrame[0];
3820	pCtx->cs.Sel = aIretFrame[1];
3821	pCtx->cs.ValidSel = aIretFrame[1];
3822	pCtx->cs.u64Base = (uint32_t)pCtx->cs.Sel << 4;
3823	pCtx->eflags.u = (pCtx->eflags.u & ~(X86_EFL_POPF_BITS & uMask))
3824	\| (aIretFrame[2] & X86_EFL_POPF_BITS & uMask);
3825	pCtx->sp += sizeof(aIretFrame);
3826
3827	LogFlow(("iret to %04x:%x\n", pCtx->cs.Sel, pCtx->ip));
3828	fUpdateRIP = false;
3829	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitIret);
3830	break;
3831	}
3832
3833	case OP_INT:
3834	{
3835	uint32_t intInfo2;
3836
3837	LogFlow(("Realmode: INT %x\n", pDis->Param1.uValue & 0xff));
3838	intInfo2 = pDis->Param1.uValue & 0xff;
3839	intInfo2 \|= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
3840	intInfo2 \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
3841
3842	rc = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, intInfo2, cbOp, 0);
3843	AssertRC(VBOXSTRICTRC_VAL(rc));
3844	fUpdateRIP = false;
3845	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitInt);
3846	break;
3847	}
3848
3849	case OP_INTO:
3850	{
3851	if (pCtx->eflags.Bits.u1OF)
3852	{
3853	uint32_t intInfo2;
3854
3855	LogFlow(("Realmode: INTO\n"));
3856	intInfo2 = X86_XCPT_OF;
3857	intInfo2 \|= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
3858	intInfo2 \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
3859
3860	rc = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, intInfo2, cbOp, 0);
3861	AssertRC(VBOXSTRICTRC_VAL(rc));
3862	fUpdateRIP = false;
3863	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitInt);
3864	}
3865	break;
3866	}
3867
3868	case OP_INT3:
3869	{
3870	uint32_t intInfo2;
3871
3872	LogFlow(("Realmode: INT 3\n"));
3873	intInfo2 = 3;
3874	intInfo2 \|= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
3875	intInfo2 \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
3876
3877	rc = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, intInfo2, cbOp, 0);
3878	AssertRC(VBOXSTRICTRC_VAL(rc));
3879	fUpdateRIP = false;
3880	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitInt);
3881	break;
3882	}
3883
3884	default:
3885	rc = EMInterpretInstructionDisasState(pVCpu, pDis, CPUMCTX2CORE(pCtx), 0, EMCODETYPE_SUPERVISOR);
3886	fUpdateRIP = false;
3887	break;
3888	}
3889
3890	if (rc == VINF_SUCCESS)
3891	{
3892	if (fUpdateRIP)
3893	pCtx->rip += cbOp; /* Move on to the next instruction. */
3894
3895	/*
3896	* LIDT, LGDT can end up here. In the future CRx changes as well. Just reload the
3897	* whole context to be done with it.
3898	*/
3899	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_ALL;
3900
3901	/* Only resume if successful. */
3902	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3903	goto ResumeExecution;
3904	}
3905	}
3906	else
3907	rc = VERR_EM_INTERPRETER;
3908
3909	AssertMsg(rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_EM_HALT,
3910	("Unexpected rc=%Rrc\n", VBOXSTRICTRC_VAL(rc)));
3911	break;
3912	}
3913
3914	#ifdef VBOX_STRICT
3915	case X86_XCPT_XF: /* SIMD exception. */
3916	case X86_XCPT_DE: /* Divide error. */
3917	case X86_XCPT_UD: /* Unknown opcode exception. */
3918	case X86_XCPT_SS: /* Stack segment exception. */
3919	case X86_XCPT_NP: /* Segment not present exception. */
3920	{
3921	switch (vector)
3922	{
3923	case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestDE); break;
3924	case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestUD); break;
3925	case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestSS); break;
3926	case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestNP); break;
3927	case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestXF); break;
3928	}
3929
3930	Log(("Trap %x at %04X:%RGv\n", vector, pCtx->cs.Sel, (RTGCPTR)pCtx->rip));
3931	rc2 = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo),
3932	cbInstr, errCode);
3933	AssertRC(rc2);
3934
3935	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3936	goto ResumeExecution;
3937	}
3938	#endif
3939	default:
3940	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestXcpUnk);
3941	if ( CPUMIsGuestInRealModeEx(pCtx)
3942	&& pVM->hwaccm.s.vmx.pRealModeTSS)
3943	{
3944	Log(("Real Mode Trap %x at %04x:%04X error code %x\n", vector, pCtx->cs.Sel, pCtx->eip, errCode));
3945	rc = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo),
3946	cbInstr, errCode);
3947	AssertRC(VBOXSTRICTRC_VAL(rc)); /* Strict RC check below. */
3948
3949	/* Go back to ring-3 in case of a triple fault. */
3950	if ( vector == X86_XCPT_DF
3951	&& rc == VINF_EM_RESET)
3952	{
3953	break;
3954	}
3955
3956	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3957	goto ResumeExecution;
3958	}
3959	AssertMsgFailed(("Unexpected vm-exit caused by exception %x\n", vector));
3960	rc = VERR_VMX_UNEXPECTED_EXCEPTION;
3961	break;
3962	} /* switch (vector) */
3963
3964	break;
3965
3966	default:
3967	rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_CODE;
3968	AssertMsgFailed(("Unexpected interruption code %x\n", intInfo));
3969	break;
3970	}
3971
3972	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3973	break;
3974	}
3975
3976	/*
3977	* 48 EPT violation. An attempt to access memory with a guest-physical address was disallowed
3978	* by the configuration of the EPT paging structures.
3979	*/
3980	case VMX_EXIT_EPT_VIOLATION:
3981	{
3982	RTGCPHYS GCPhys;
3983
3984	Assert(pVM->hwaccm.s.fNestedPaging);
3985
3986	rc2 = VMXReadVMCS64(VMX_VMCS_EXIT_PHYS_ADDR_FULL, &GCPhys);
3987	AssertRC(rc2);
3988	Assert(((exitQualification >> 7) & 3) != 2);
3989
3990	/* Determine the kind of violation. */
3991	errCode = 0;
3992	if (exitQualification & VMX_EXIT_QUALIFICATION_EPT_INSTR_FETCH)
3993	errCode \|= X86_TRAP_PF_ID;
3994
3995	if (exitQualification & VMX_EXIT_QUALIFICATION_EPT_DATA_WRITE)
3996	errCode \|= X86_TRAP_PF_RW;
3997
3998	/* If the page is present, then it's a page level protection fault. */
3999	if (exitQualification & VMX_EXIT_QUALIFICATION_EPT_ENTRY_PRESENT)
4000	errCode \|= X86_TRAP_PF_P;
4001	else
4002	{
4003	/* Shortcut for APIC TPR reads and writes. */
4004	if ( (GCPhys & 0xfff) == 0x080
4005	&& GCPhys > 0x1000000 /* to skip VGA frame buffer accesses */
4006	&& fSetupTPRCaching
4007	&& (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC))
4008	{
4009	RTGCPHYS GCPhysApicBase;
4010	PDMApicGetBase(pVM, &GCPhysApicBase); /* @todo cache this */
4011	GCPhysApicBase &= PAGE_BASE_GC_MASK;
4012	if (GCPhys == GCPhysApicBase + 0x80)
4013	{
4014	Log(("Enable VT-x virtual APIC access filtering\n"));
4015	rc2 = IOMMMIOMapMMIOHCPage(pVM, GCPhysApicBase, pVM->hwaccm.s.vmx.pAPICPhys, X86_PTE_RW \| X86_PTE_P);
4016	AssertRC(rc2);
4017	}
4018	}
4019	}
4020	Log(("EPT Page fault %x at %RGp error code %x\n", (uint32_t)exitQualification, GCPhys, errCode));
4021
4022	/* GCPhys contains the guest physical address of the page fault. */
4023	TRPMAssertTrap(pVCpu, X86_XCPT_PF, TRPM_TRAP);
4024	TRPMSetErrorCode(pVCpu, errCode);
4025	TRPMSetFaultAddress(pVCpu, GCPhys);
4026
4027	/* Handle the pagefault trap for the nested shadow table. */
4028	rc = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, errCode, CPUMCTX2CORE(pCtx), GCPhys);
4029
4030	/*
4031	* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment below, @bugref{6043}.
4032	*/
4033	if ( rc == VINF_SUCCESS
4034	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
4035	\|\| rc == VERR_PAGE_NOT_PRESENT)
4036	{
4037	/* We've successfully synced our shadow pages, so let's just continue execution. */
4038	Log2(("Shadow page fault at %RGv cr2=%RGp error code %x\n", (RTGCPTR)pCtx->rip, exitQualification , errCode));
4039	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitReasonNPF);
4040
4041	TRPMResetTrap(pVCpu);
4042	goto ResumeExecution;
4043	}
4044
4045	#ifdef VBOX_STRICT
4046	if (rc != VINF_EM_RAW_EMULATE_INSTR)
4047	LogFlow(("PGMTrap0eHandlerNestedPaging at %RGv failed with %Rrc\n", (RTGCPTR)pCtx->rip, VBOXSTRICTRC_VAL(rc)));
4048	#endif
4049	/* Need to go back to the recompiler to emulate the instruction. */
4050	TRPMResetTrap(pVCpu);
4051	break;
4052	}
4053
4054	case VMX_EXIT_EPT_MISCONFIG:
4055	{
4056	RTGCPHYS GCPhys;
4057
4058	Assert(pVM->hwaccm.s.fNestedPaging);
4059
4060	rc2 = VMXReadVMCS64(VMX_VMCS_EXIT_PHYS_ADDR_FULL, &GCPhys);
4061	AssertRC(rc2);
4062	Log(("VMX_EXIT_EPT_MISCONFIG for %RGp\n", GCPhys));
4063
4064	/* Shortcut for APIC TPR reads and writes. */
4065	if ( (GCPhys & 0xfff) == 0x080
4066	&& GCPhys > 0x1000000 /* to skip VGA frame buffer accesses */
4067	&& fSetupTPRCaching
4068	&& (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC))
4069	{
4070	RTGCPHYS GCPhysApicBase;
4071	PDMApicGetBase(pVM, &GCPhysApicBase); /* @todo cache this */
4072	GCPhysApicBase &= PAGE_BASE_GC_MASK;
4073	if (GCPhys == GCPhysApicBase + 0x80)
4074	{
4075	Log(("Enable VT-x virtual APIC access filtering\n"));
4076	rc2 = IOMMMIOMapMMIOHCPage(pVM, GCPhysApicBase, pVM->hwaccm.s.vmx.pAPICPhys, X86_PTE_RW \| X86_PTE_P);
4077	AssertRC(rc2);
4078	}
4079	}
4080
4081	rc = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pCtx), GCPhys, UINT32_MAX);
4082
4083	/*
4084	* If we succeed, resume execution.
4085	* Or, if fail in interpreting the instruction because we couldn't get the guest physical address
4086	* of the page containing the instruction via the guest's page tables (we would invalidate the guest page
4087	* in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
4088	* weird case. See @bugref{6043}.
4089	*/
4090	if ( rc == VINF_SUCCESS
4091	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
4092	\|\| rc == VERR_PAGE_NOT_PRESENT)
4093	{
4094	Log2(("PGMR0Trap0eHandlerNPMisconfig(,,,%RGp) at %RGv -> resume\n", GCPhys, (RTGCPTR)pCtx->rip));
4095	goto ResumeExecution;
4096	}
4097
4098	Log2(("PGMR0Trap0eHandlerNPMisconfig(,,,%RGp) at %RGv -> %Rrc\n", GCPhys, (RTGCPTR)pCtx->rip, VBOXSTRICTRC_VAL(rc)));
4099	break;
4100	}
4101
4102	case VMX_EXIT_IRQ_WINDOW: /* 7 Interrupt window. */
4103	/* Clear VM-exit on IF=1 change. */
4104	LogFlow(("VMX_EXIT_IRQ_WINDOW %RGv pending=%d IF=%d\n", (RTGCPTR)pCtx->rip,
4105	VMCPU_FF_ISPENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC\|VMCPU_FF_INTERRUPT_PIC)), pCtx->eflags.Bits.u1IF));
4106	pVCpu->hwaccm.s.vmx.proc_ctls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_IRQ_WINDOW_EXIT;
4107	rc2 = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
4108	AssertRC(rc2);
4109	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitIrqWindow);
4110	goto ResumeExecution; /* we check for pending guest interrupts there */
4111
4112	case VMX_EXIT_WBINVD: /* 54 Guest software attempted to execute WBINVD. (conditional) */
4113	case VMX_EXIT_INVD: /* 13 Guest software attempted to execute INVD. (unconditional) */
4114	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitInvd);
4115	/* Skip instruction and continue directly. */
4116	pCtx->rip += cbInstr;
4117	/* Continue execution.*/
4118	goto ResumeExecution;
4119
4120	case VMX_EXIT_CPUID: /* 10 Guest software attempted to execute CPUID. */
4121	{
4122	Log2(("VMX: Cpuid %x\n", pCtx->eax));
4123	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitCpuid);
4124	rc = EMInterpretCpuId(pVM, pVCpu, CPUMCTX2CORE(pCtx));
4125	if (rc == VINF_SUCCESS)
4126	{
4127	/* Update EIP and continue execution. */
4128	Assert(cbInstr == 2);
4129	pCtx->rip += cbInstr;
4130	goto ResumeExecution;
4131	}
4132	AssertMsgFailed(("EMU: cpuid failed with %Rrc\n", VBOXSTRICTRC_VAL(rc)));
4133	rc = VINF_EM_RAW_EMULATE_INSTR;
4134	break;
4135	}
4136
4137	case VMX_EXIT_RDPMC: /* 15 Guest software attempted to execute RDPMC. */
4138	{
4139	Log2(("VMX: Rdpmc %x\n", pCtx->ecx));
4140	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitRdpmc);
4141	rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pCtx));
4142	if (rc == VINF_SUCCESS)
4143	{
4144	/* Update EIP and continue execution. */
4145	Assert(cbInstr == 2);
4146	pCtx->rip += cbInstr;
4147	goto ResumeExecution;
4148	}
4149	rc = VINF_EM_RAW_EMULATE_INSTR;
4150	break;
4151	}
4152
4153	case VMX_EXIT_RDTSC: /* 16 Guest software attempted to execute RDTSC. */
4154	{
4155	Log2(("VMX: Rdtsc\n"));
4156	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitRdtsc);
4157	rc = EMInterpretRdtsc(pVM, pVCpu, CPUMCTX2CORE(pCtx));
4158	if (rc == VINF_SUCCESS)
4159	{
4160	/* Update EIP and continue execution. */
4161	Assert(cbInstr == 2);
4162	pCtx->rip += cbInstr;
4163	goto ResumeExecution;
4164	}
4165	rc = VINF_EM_RAW_EMULATE_INSTR;
4166	break;
4167	}
4168
4169	case VMX_EXIT_RDTSCP: /* 51 Guest software attempted to execute RDTSCP. */
4170	{
4171	Log2(("VMX: Rdtscp\n"));
4172	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitRdtscp);
4173	rc = EMInterpretRdtscp(pVM, pVCpu, pCtx);
4174	if (rc == VINF_SUCCESS)
4175	{
4176	/* Update EIP and continue execution. */
4177	Assert(cbInstr == 3);
4178	pCtx->rip += cbInstr;
4179	goto ResumeExecution;
4180	}
4181	rc = VINF_EM_RAW_EMULATE_INSTR;
4182	break;
4183	}
4184
4185	case VMX_EXIT_INVLPG: /* 14 Guest software attempted to execute INVLPG. */
4186	{
4187	Log2(("VMX: invlpg\n"));
4188	Assert(!pVM->hwaccm.s.fNestedPaging);
4189
4190	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitInvlpg);
4191	rc = EMInterpretInvlpg(pVM, pVCpu, CPUMCTX2CORE(pCtx), exitQualification);
4192	if (rc == VINF_SUCCESS)
4193	{
4194	/* Update EIP and continue execution. */
4195	pCtx->rip += cbInstr;
4196	goto ResumeExecution;
4197	}
4198	AssertMsg(rc == VERR_EM_INTERPRETER, ("EMU: invlpg %RGv failed with %Rrc\n", exitQualification, VBOXSTRICTRC_VAL(rc)));
4199	break;
4200	}
4201
4202	case VMX_EXIT_MONITOR: /* 39 Guest software attempted to execute MONITOR. */
4203	{
4204	Log2(("VMX: monitor\n"));
4205
4206	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitMonitor);
4207	rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pCtx));
4208	if (rc == VINF_SUCCESS)
4209	{
4210	/* Update EIP and continue execution. */
4211	pCtx->rip += cbInstr;
4212	goto ResumeExecution;
4213	}
4214	AssertMsg(rc == VERR_EM_INTERPRETER, ("EMU: monitor failed with %Rrc\n", VBOXSTRICTRC_VAL(rc)));
4215	break;
4216	}
4217
4218	case VMX_EXIT_WRMSR: /* 32 WRMSR. Guest software attempted to execute WRMSR. */
4219	/* When an interrupt is pending, we'll let MSR_K8_LSTAR writes fault in our TPR patch code. */
4220	if ( pVM->hwaccm.s.fTPRPatchingActive
4221	&& pCtx->ecx == MSR_K8_LSTAR)
4222	{
4223	Assert(!CPUMIsGuestInLongModeEx(pCtx));
4224	if ((pCtx->eax & 0xff) != u8LastTPR)
4225	{
4226	Log(("VMX: Faulting MSR_K8_LSTAR write with new TPR value %x\n", pCtx->eax & 0xff));
4227
4228	/* Our patch code uses LSTAR for TPR caching. */
4229	rc2 = PDMApicSetTPR(pVCpu, pCtx->eax & 0xff);
4230	AssertRC(rc2);
4231	}
4232
4233	/* Skip the instruction and continue. */
4234	pCtx->rip += cbInstr; /* wrmsr = [0F 30] */
4235
4236	/* Only resume if successful. */
4237	goto ResumeExecution;
4238	}
4239	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_GUEST_MSR;
4240	/* no break */
4241	case VMX_EXIT_RDMSR: /* 31 RDMSR. Guest software attempted to execute RDMSR. */
4242	{
4243	STAM_COUNTER_INC((exitReason == VMX_EXIT_RDMSR) ? &pVCpu->hwaccm.s.StatExitRdmsr : &pVCpu->hwaccm.s.StatExitWrmsr);
4244
4245	/*
4246	* Note: The Intel spec. claims there's an REX version of RDMSR that's slightly different,
4247	* so we play safe by completely disassembling the instruction.
4248	*/
4249	Log2(("VMX: %s\n", (exitReason == VMX_EXIT_RDMSR) ? "rdmsr" : "wrmsr"));
4250	rc = EMInterpretInstruction(pVCpu, CPUMCTX2CORE(pCtx), 0);
4251	if (rc == VINF_SUCCESS)
4252	{
4253	/* EIP has been updated already. */
4254	/* Only resume if successful. */
4255	goto ResumeExecution;
4256	}
4257	AssertMsg(rc == VERR_EM_INTERPRETER, ("EMU: %s failed with %Rrc\n",
4258	(exitReason == VMX_EXIT_RDMSR) ? "rdmsr" : "wrmsr", VBOXSTRICTRC_VAL(rc)));
4259	break;
4260	}
4261
4262	case VMX_EXIT_CRX_MOVE: /* 28 Control-register accesses. */
4263	{
4264	STAM_PROFILE_ADV_START(&pVCpu->hwaccm.s.StatExit2Sub2, y2);
4265
4266	switch (VMX_EXIT_QUALIFICATION_CRX_ACCESS(exitQualification))
4267	{
4268	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_WRITE:
4269	{
4270	Log2(("VMX: %RGv mov cr%d, x\n", (RTGCPTR)pCtx->rip, VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification)));
4271	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitCRxWrite[VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification)]);
4272	rc = EMInterpretCRxWrite(pVM, pVCpu, CPUMCTX2CORE(pCtx),
4273	VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification),
4274	VMX_EXIT_QUALIFICATION_CRX_GENREG(exitQualification));
4275	switch (VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification))
4276	{
4277	case 0:
4278	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_GUEST_CR0 \| HWACCM_CHANGED_GUEST_CR3;
4279	break;
4280	case 2:
4281	break;
4282	case 3:
4283	Assert(!pVM->hwaccm.s.fNestedPaging \|\| !CPUMIsGuestInPagedProtectedModeEx(pCtx));
4284	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_GUEST_CR3;
4285	break;
4286	case 4:
4287	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_GUEST_CR4;
4288	break;
4289	case 8:
4290	/* CR8 contains the APIC TPR */
4291	Assert(!(pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1
4292	& VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW));
4293	break;
4294
4295	default:
4296	AssertFailed();
4297	break;
4298	}
4299	break;
4300	}
4301
4302	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ:
4303	{
4304	Log2(("VMX: mov x, crx\n"));
4305	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitCRxRead[VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification)]);
4306
4307	Assert( !pVM->hwaccm.s.fNestedPaging
4308	\|\| !CPUMIsGuestInPagedProtectedModeEx(pCtx)
4309	\|\| VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification) != DISCREG_CR3);
4310
4311	/* CR8 reads only cause an exit when the TPR shadow feature isn't present. */
4312	Assert( VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification) != 8
4313	\|\| !(pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW));
4314
4315	rc = EMInterpretCRxRead(pVM, pVCpu, CPUMCTX2CORE(pCtx),
4316	VMX_EXIT_QUALIFICATION_CRX_GENREG(exitQualification),
4317	VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification));
4318	break;
4319	}
4320
4321	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_CLTS:
4322	{
4323	Log2(("VMX: clts\n"));
4324	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitCLTS);
4325	rc = EMInterpretCLTS(pVM, pVCpu);
4326	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_GUEST_CR0;
4327	break;
4328	}
4329
4330	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_LMSW:
4331	{
4332	Log2(("VMX: lmsw %x\n", VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(exitQualification)));
4333	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitLMSW);
4334	rc = EMInterpretLMSW(pVM, pVCpu, CPUMCTX2CORE(pCtx), VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(exitQualification));
4335	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_GUEST_CR0;
4336	break;
4337	}
4338	}
4339
4340	/* Update EIP if no error occurred. */
4341	if (RT_SUCCESS(rc))
4342	pCtx->rip += cbInstr;
4343
4344	if (rc == VINF_SUCCESS)
4345	{
4346	/* Only resume if successful. */
4347	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub2, y2);
4348	goto ResumeExecution;
4349	}
4350	Assert(rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_PGM_SYNC_CR3);
4351	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub2, y2);
4352	break;
4353	}
4354
4355	case VMX_EXIT_DRX_MOVE: /* 29 Debug-register accesses. */
4356	{
4357	if ( !DBGFIsStepping(pVCpu)
4358	&& !CPUMIsHyperDebugStateActive(pVCpu))
4359	{
4360	/* Disable DRx move intercepts. */
4361	pVCpu->hwaccm.s.vmx.proc_ctls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT;
4362	rc2 = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
4363	AssertRC(rc2);
4364
4365	/* Save the host and load the guest debug state. */
4366	rc2 = CPUMR0LoadGuestDebugState(pVM, pVCpu, pCtx, true /* include DR6 */);
4367	AssertRC(rc2);
4368
4369	#ifdef LOG_ENABLED
4370	if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(exitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
4371	{
4372	Log(("VMX_EXIT_DRX_MOVE: write DR%d genreg %d\n", VMX_EXIT_QUALIFICATION_DRX_REGISTER(exitQualification),
4373	VMX_EXIT_QUALIFICATION_DRX_GENREG(exitQualification)));
4374	}
4375	else
4376	Log(("VMX_EXIT_DRX_MOVE: read DR%d\n", VMX_EXIT_QUALIFICATION_DRX_REGISTER(exitQualification)));
4377	#endif
4378
4379	#ifdef VBOX_WITH_STATISTICS
4380	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatDRxContextSwitch);
4381	if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(exitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
4382	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitDRxWrite);
4383	else
4384	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitDRxRead);
4385	#endif
4386
4387	goto ResumeExecution;
4388	}
4389
4390	/** @todo clear VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT after the first
4391	* time and restore DRx registers afterwards */
4392	if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(exitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
4393	{
4394	Log2(("VMX: mov DRx%d, genreg%d\n", VMX_EXIT_QUALIFICATION_DRX_REGISTER(exitQualification),
4395	VMX_EXIT_QUALIFICATION_DRX_GENREG(exitQualification)));
4396	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitDRxWrite);
4397	rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pCtx),
4398	VMX_EXIT_QUALIFICATION_DRX_REGISTER(exitQualification),
4399	VMX_EXIT_QUALIFICATION_DRX_GENREG(exitQualification));
4400	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_GUEST_DEBUG;
4401	Log2(("DR7=%08x\n", pCtx->dr[7]));
4402	}
4403	else
4404	{
4405	Log2(("VMX: mov x, DRx\n"));
4406	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitDRxRead);
4407	rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pCtx),
4408	VMX_EXIT_QUALIFICATION_DRX_GENREG(exitQualification),
4409	VMX_EXIT_QUALIFICATION_DRX_REGISTER(exitQualification));
4410	}
4411	/* Update EIP if no error occurred. */
4412	if (RT_SUCCESS(rc))
4413	pCtx->rip += cbInstr;
4414
4415	if (rc == VINF_SUCCESS)
4416	{
4417	/* Only resume if successful. */
4418	goto ResumeExecution;
4419	}
4420	Assert(rc == VERR_EM_INTERPRETER);
4421	break;
4422	}
4423
4424	/* Note: We'll get a #GP if the IO instruction isn't allowed (IOPL or TSS bitmap); no need to double check. */
4425	case VMX_EXIT_PORT_IO: /* 30 I/O instruction. */
4426	{
4427	STAM_PROFILE_ADV_START(&pVCpu->hwaccm.s.StatExit2Sub1, y1);
4428	uint32_t uPort;
4429	uint32_t uIOWidth = VMX_EXIT_QUALIFICATION_IO_WIDTH(exitQualification);
4430	bool fIOWrite = (VMX_EXIT_QUALIFICATION_IO_DIRECTION(exitQualification) == VMX_EXIT_QUALIFICATION_IO_DIRECTION_OUT);
4431
4432	/** @todo necessary to make the distinction? */
4433	if (VMX_EXIT_QUALIFICATION_IO_ENCODING(exitQualification) == VMX_EXIT_QUALIFICATION_IO_ENCODING_DX)
4434	uPort = pCtx->edx & 0xffff;
4435	else
4436	uPort = VMX_EXIT_QUALIFICATION_IO_PORT(exitQualification); /* Immediate encoding. */
4437
4438	if (RT_UNLIKELY(uIOWidth == 2 \|\| uIOWidth >= 4)) /* paranoia */
4439	{
4440	rc = fIOWrite ? VINF_IOM_R3_IOPORT_WRITE : VINF_IOM_R3_IOPORT_READ;
4441	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub1, y1);
4442	break;
4443	}
4444
4445	uint32_t cbSize = g_aIOSize[uIOWidth];
4446	if (VMX_EXIT_QUALIFICATION_IO_STRING(exitQualification))
4447	{
4448	/* ins/outs */
4449	PDISCPUSTATE pDis = &pVCpu->hwaccm.s.DisState;
4450
4451	/* Disassemble manually to deal with segment prefixes. */
4452	/** @todo VMX_VMCS_EXIT_GUEST_LINEAR_ADDR contains the flat pointer operand of the instruction. */
4453	/** @todo VMX_VMCS32_RO_EXIT_INSTR_INFO also contains segment prefix info. */
4454	rc2 = EMInterpretDisasCurrent(pVM, pVCpu, pDis, NULL);
4455	if (RT_SUCCESS(rc))
4456	{
4457	if (fIOWrite)
4458	{
4459	Log2(("IOMInterpretOUTSEx %RGv %x size=%d\n", (RTGCPTR)pCtx->rip, uPort, cbSize));
4460	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitIOStringWrite);
4461	rc = IOMInterpretOUTSEx(pVM, CPUMCTX2CORE(pCtx), uPort, pDis->fPrefix, (DISCPUMODE)pDis->uAddrMode, cbSize);
4462	}
4463	else
4464	{
4465	Log2(("IOMInterpretINSEx %RGv %x size=%d\n", (RTGCPTR)pCtx->rip, uPort, cbSize));
4466	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitIOStringRead);
4467	rc = IOMInterpretINSEx(pVM, CPUMCTX2CORE(pCtx), uPort, pDis->fPrefix, (DISCPUMODE)pDis->uAddrMode, cbSize);
4468	}
4469	}
4470	else
4471	rc = VINF_EM_RAW_EMULATE_INSTR;
4472	}
4473	else
4474	{
4475	/* Normal in/out */
4476	uint32_t uAndVal = g_aIOOpAnd[uIOWidth];
4477
4478	Assert(!VMX_EXIT_QUALIFICATION_IO_REP(exitQualification));
4479
4480	if (fIOWrite)
4481	{
4482	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitIOWrite);
4483	rc = IOMIOPortWrite(pVM, uPort, pCtx->eax & uAndVal, cbSize);
4484	if (rc == VINF_IOM_R3_IOPORT_WRITE)
4485	HWACCMR0SavePendingIOPortWrite(pVCpu, pCtx->rip, pCtx->rip + cbInstr, uPort, uAndVal, cbSize);
4486	}
4487	else
4488	{
4489	uint32_t u32Val = 0;
4490
4491	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitIORead);
4492	rc = IOMIOPortRead(pVM, uPort, &u32Val, cbSize);
4493	if (IOM_SUCCESS(rc))
4494	{
4495	/* Write back to the EAX register. */
4496	pCtx->eax = (pCtx->eax & ~uAndVal) \| (u32Val & uAndVal);
4497	}
4498	else
4499	if (rc == VINF_IOM_R3_IOPORT_READ)
4500	HWACCMR0SavePendingIOPortRead(pVCpu, pCtx->rip, pCtx->rip + cbInstr, uPort, uAndVal, cbSize);
4501	}
4502	}
4503
4504	/*
4505	* Handled the I/O return codes.
4506	* (The unhandled cases end up with rc == VINF_EM_RAW_EMULATE_INSTR.)
4507	*/
4508	if (IOM_SUCCESS(rc))
4509	{
4510	/* Update EIP and continue execution. */
4511	pCtx->rip += cbInstr;
4512	if (RT_LIKELY(rc == VINF_SUCCESS))
4513	{
4514	/* If any IO breakpoints are armed, then we should check if a debug trap needs to be generated. */
4515	if (pCtx->dr[7] & X86_DR7_ENABLED_MASK)
4516	{
4517	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatDRxIOCheck);
4518	for (unsigned i = 0; i < 4; i++)
4519	{
4520	unsigned uBPLen = g_aIOSize[X86_DR7_GET_LEN(pCtx->dr[7], i)];
4521
4522	if ( (uPort >= pCtx->dr[i] && uPort < pCtx->dr[i] + uBPLen)
4523	&& (pCtx->dr[7] & (X86_DR7_L(i) \| X86_DR7_G(i)))
4524	&& (pCtx->dr[7] & X86_DR7_RW(i, X86_DR7_RW_IO)) == X86_DR7_RW(i, X86_DR7_RW_IO))
4525	{
4526	uint64_t uDR6;
4527
4528	Assert(CPUMIsGuestDebugStateActive(pVCpu));
4529
4530	uDR6 = ASMGetDR6();
4531
4532	/* Clear all breakpoint status flags and set the one we just hit. */
4533	uDR6 &= ~(X86_DR6_B0\|X86_DR6_B1\|X86_DR6_B2\|X86_DR6_B3);
4534	uDR6 \|= (uint64_t)RT_BIT(i);
4535
4536	/*
4537	* Note: AMD64 Architecture Programmer's Manual 13.1:
4538	* Bits 15:13 of the DR6 register is never cleared by the processor and must
4539	* be cleared by software after the contents have been read.
4540	*/
4541	ASMSetDR6(uDR6);
4542
4543	/* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
4544	pCtx->dr[7] &= ~X86_DR7_GD;
4545
4546	/* Paranoia. */
4547	pCtx->dr[7] &= 0xffffffff; /* upper 32 bits reserved */
4548	pCtx->dr[7] &= ~(RT_BIT(11) \| RT_BIT(12) \| RT_BIT(14) \| RT_BIT(15)); /* must be zero */
4549	pCtx->dr[7] \|= 0x400; /* must be one */
4550
4551	/* Resync DR7 */
4552	rc2 = VMXWriteVMCS64(VMX_VMCS64_GUEST_DR7, pCtx->dr[7]);
4553	AssertRC(rc2);
4554
4555	/* Construct inject info. */
4556	intInfo = X86_XCPT_DB;
4557	intInfo \|= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
4558	intInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HWEXCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
4559
4560	Log(("Inject IO debug trap at %RGv\n", (RTGCPTR)pCtx->rip));
4561	rc2 = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo),
4562	0 /* cbInstr /, 0 / errCode */);
4563	AssertRC(rc2);
4564
4565	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub1, y1);
4566	goto ResumeExecution;
4567	}
4568	}
4569	}
4570	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub1, y1);
4571	goto ResumeExecution;
4572	}
4573	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub1, y1);
4574	break;
4575	}
4576
4577	#ifdef VBOX_STRICT
4578	if (rc == VINF_IOM_R3_IOPORT_READ)
4579	Assert(!fIOWrite);
4580	else if (rc == VINF_IOM_R3_IOPORT_WRITE)
4581	Assert(fIOWrite);
4582	else
4583	{
4584	AssertMsg( RT_FAILURE(rc)
4585	\|\| rc == VINF_EM_RAW_EMULATE_INSTR
4586	\|\| rc == VINF_EM_RAW_GUEST_TRAP
4587	\|\| rc == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", VBOXSTRICTRC_VAL(rc)));
4588	}
4589	#endif
4590	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub1, y1);
4591	break;
4592	}
4593
4594	case VMX_EXIT_TPR: /* 43 TPR below threshold. Guest software executed MOV to CR8. */
4595	LogFlow(("VMX_EXIT_TPR\n"));
4596	/* RIP is already set to the next instruction and the TPR has been synced back. Just resume. */
4597	goto ResumeExecution;
4598
4599	case VMX_EXIT_APIC_ACCESS: /* 44 APIC access. Guest software attempted to access memory at a physical address
4600	on the APIC-access page. */
4601	{
4602	LogFlow(("VMX_EXIT_APIC_ACCESS\n"));
4603	unsigned uAccessType = VMX_EXIT_QUALIFICATION_APIC_ACCESS_TYPE(exitQualification);
4604
4605	switch (uAccessType)
4606	{
4607	case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
4608	case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
4609	{
4610	RTGCPHYS GCPhys;
4611	PDMApicGetBase(pVM, &GCPhys);
4612	GCPhys &= PAGE_BASE_GC_MASK;
4613	GCPhys += VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(exitQualification);
4614
4615	LogFlow(("Apic access at %RGp\n", GCPhys));
4616	rc = IOMMMIOPhysHandler(pVM, (uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ) ? 0 : X86_TRAP_PF_RW,
4617	CPUMCTX2CORE(pCtx), GCPhys);
4618	if (rc == VINF_SUCCESS)
4619	goto ResumeExecution; /* rip already updated */
4620	break;
4621	}
4622
4623	default:
4624	rc = VINF_EM_RAW_EMULATE_INSTR;
4625	break;
4626	}
4627	break;
4628	}
4629
4630	case VMX_EXIT_PREEMPTION_TIMER: /* 52 VMX-preemption timer expired. The preemption timer counted down to zero. */
4631	if (!TMTimerPollBool(pVM, pVCpu))
4632	goto ResumeExecution;
4633	rc = VINF_EM_RAW_TIMER_PENDING;
4634	break;
4635
4636	default:
4637	/* The rest is handled after syncing the entire CPU state. */
4638	break;
4639	}
4640
4641
4642	/*
4643	* Note: The guest state is not entirely synced back at this stage!
4644	*/
4645
4646	/* Investigate why there was a VM-exit. (part 2) */
4647	switch (exitReason)
4648	{
4649	case VMX_EXIT_EXCEPTION: /* 0 Exception or non-maskable interrupt (NMI). */
4650	case VMX_EXIT_EXTERNAL_IRQ: /* 1 External interrupt. */
4651	case VMX_EXIT_EPT_VIOLATION:
4652	case VMX_EXIT_EPT_MISCONFIG: /* 49 EPT misconfig is used by the PGM/MMIO optimizations. */
4653	case VMX_EXIT_PREEMPTION_TIMER: /* 52 VMX-preemption timer expired. The preemption timer counted down to zero. */
4654	/* Already handled above. */
4655	break;
4656
4657	case VMX_EXIT_TRIPLE_FAULT: /* 2 Triple fault. */
4658	rc = VINF_EM_RESET; /* Triple fault equals a reset. */
4659	break;
4660
4661	case VMX_EXIT_INIT_SIGNAL: /* 3 INIT signal. */
4662	case VMX_EXIT_SIPI: /* 4 Start-up IPI (SIPI). */
4663	rc = VINF_EM_RAW_INTERRUPT;
4664	AssertFailed(); /* Can't happen. Yet. */
4665	break;
4666
4667	case VMX_EXIT_IO_SMI_IRQ: /* 5 I/O system-management interrupt (SMI). */
4668	case VMX_EXIT_SMI_IRQ: /* 6 Other SMI. */
4669	rc = VINF_EM_RAW_INTERRUPT;
4670	AssertFailed(); /* Can't happen afaik. */
4671	break;
4672
4673	case VMX_EXIT_TASK_SWITCH: /* 9 Task switch: too complicated to emulate, so fall back to the recompiler */
4674	Log(("VMX_EXIT_TASK_SWITCH: exit=%RX64\n", exitQualification));
4675	if ( (VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE(exitQualification) == VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE_IDT)
4676	&& pVCpu->hwaccm.s.Event.fPending)
4677	{
4678	/* Caused by an injected interrupt. */
4679	pVCpu->hwaccm.s.Event.fPending = false;
4680
4681	Log(("VMX_EXIT_TASK_SWITCH: reassert trap %d\n", VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVCpu->hwaccm.s.Event.intInfo)));
4682	Assert(!VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(pVCpu->hwaccm.s.Event.intInfo));
4683	rc2 = TRPMAssertTrap(pVCpu, VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVCpu->hwaccm.s.Event.intInfo), TRPM_HARDWARE_INT);
4684	AssertRC(rc2);
4685	}
4686	/* else Exceptions and software interrupts can just be restarted. */
4687	rc = VERR_EM_INTERPRETER;
4688	break;
4689
4690	case VMX_EXIT_HLT: /* 12 Guest software attempted to execute HLT. */
4691	/* Check if external interrupts are pending; if so, don't switch back. */
4692	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitHlt);
4693	pCtx->rip++; /* skip hlt */
4694	if (EMShouldContinueAfterHalt(pVCpu, pCtx))
4695	goto ResumeExecution;
4696
4697	rc = VINF_EM_HALT;
4698	break;
4699
4700	case VMX_EXIT_MWAIT: /* 36 Guest software executed MWAIT. */
4701	Log2(("VMX: mwait\n"));
4702	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitMwait);
4703	rc = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pCtx));
4704	if ( rc == VINF_EM_HALT
4705	\|\| rc == VINF_SUCCESS)
4706	{
4707	/* Update EIP and continue execution. */
4708	pCtx->rip += cbInstr;
4709
4710	/* Check if external interrupts are pending; if so, don't switch back. */
4711	if ( rc == VINF_SUCCESS
4712	\|\| ( rc == VINF_EM_HALT
4713	&& EMShouldContinueAfterHalt(pVCpu, pCtx))
4714	)
4715	goto ResumeExecution;
4716	}
4717	AssertMsg(rc == VERR_EM_INTERPRETER \|\| rc == VINF_EM_HALT, ("EMU: mwait failed with %Rrc\n", VBOXSTRICTRC_VAL(rc)));
4718	break;
4719
4720	case VMX_EXIT_RSM: /* 17 Guest software attempted to execute RSM in SMM. */
4721	AssertFailed(); /* can't happen. */
4722	rc = VERR_EM_INTERPRETER;
4723	break;
4724
4725	case VMX_EXIT_MTF: /* 37 Exit due to Monitor Trap Flag. */
4726	LogFlow(("VMX_EXIT_MTF at %RGv\n", (RTGCPTR)pCtx->rip));
4727	pVCpu->hwaccm.s.vmx.proc_ctls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_TRAP_FLAG;
4728	rc2 = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
4729	AssertRC(rc2);
4730	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitMTF);
4731	#if 0
4732	DBGFDoneStepping(pVCpu);
4733	#endif
4734	rc = VINF_EM_DBG_STOP;
4735	break;
4736
4737	case VMX_EXIT_VMCALL: /* 18 Guest software executed VMCALL. */
4738	case VMX_EXIT_VMCLEAR: /* 19 Guest software executed VMCLEAR. */
4739	case VMX_EXIT_VMLAUNCH: /* 20 Guest software executed VMLAUNCH. */
4740	case VMX_EXIT_VMPTRLD: /* 21 Guest software executed VMPTRLD. */
4741	case VMX_EXIT_VMPTRST: /* 22 Guest software executed VMPTRST. */
4742	case VMX_EXIT_VMREAD: /* 23 Guest software executed VMREAD. */
4743	case VMX_EXIT_VMRESUME: /* 24 Guest software executed VMRESUME. */
4744	case VMX_EXIT_VMWRITE: /* 25 Guest software executed VMWRITE. */
4745	case VMX_EXIT_VMXOFF: /* 26 Guest software executed VMXOFF. */
4746	case VMX_EXIT_VMXON: /* 27 Guest software executed VMXON. */
4747	/** @todo inject #UD immediately */
4748	rc = VERR_EM_INTERPRETER;
4749	break;
4750
4751	case VMX_EXIT_CPUID: /* 10 Guest software attempted to execute CPUID. */
4752	case VMX_EXIT_RDTSC: /* 16 Guest software attempted to execute RDTSC. */
4753	case VMX_EXIT_INVLPG: /* 14 Guest software attempted to execute INVLPG. */
4754	case VMX_EXIT_CRX_MOVE: /* 28 Control-register accesses. */
4755	case VMX_EXIT_DRX_MOVE: /* 29 Debug-register accesses. */
4756	case VMX_EXIT_PORT_IO: /* 30 I/O instruction. */
4757	case VMX_EXIT_RDPMC: /* 15 Guest software attempted to execute RDPMC. */
4758	case VMX_EXIT_RDTSCP: /* 51 Guest software attempted to execute RDTSCP. */
4759	/* already handled above */
4760	AssertMsg( rc == VINF_PGM_CHANGE_MODE
4761	\|\| rc == VINF_EM_RAW_INTERRUPT
4762	\|\| rc == VERR_EM_INTERPRETER
4763	\|\| rc == VINF_EM_RAW_EMULATE_INSTR
4764	\|\| rc == VINF_PGM_SYNC_CR3
4765	\|\| rc == VINF_IOM_R3_IOPORT_READ
4766	\|\| rc == VINF_IOM_R3_IOPORT_WRITE
4767	\|\| rc == VINF_EM_RAW_GUEST_TRAP
4768	\|\| rc == VINF_TRPM_XCPT_DISPATCHED
4769	\|\| rc == VINF_EM_RESCHEDULE_REM,
4770	("rc = %d\n", VBOXSTRICTRC_VAL(rc)));
4771	break;
4772
4773	case VMX_EXIT_TPR: /* 43 TPR below threshold. Guest software executed MOV to CR8. */
4774	case VMX_EXIT_RDMSR: /* 31 RDMSR. Guest software attempted to execute RDMSR. */
4775	case VMX_EXIT_WRMSR: /* 32 WRMSR. Guest software attempted to execute WRMSR. */
4776	case VMX_EXIT_PAUSE: /* 40 Guest software attempted to execute PAUSE. */
4777	case VMX_EXIT_MONITOR: /* 39 Guest software attempted to execute MONITOR. */
4778	case VMX_EXIT_APIC_ACCESS: /* 44 APIC access. Guest software attempted to access memory at a physical address
4779	on the APIC-access page. */
4780	{
4781	/*
4782	* If we decided to emulate them here, then we must sync the MSRs that could have been changed (sysenter, FS/GS base)
4783	*/
4784	rc = VERR_EM_INTERPRETER;
4785	break;
4786	}
4787
4788	case VMX_EXIT_IRQ_WINDOW: /* 7 Interrupt window. */
4789	Assert(rc == VINF_EM_RAW_INTERRUPT);
4790	break;
4791
4792	case VMX_EXIT_ERR_INVALID_GUEST_STATE: /* 33 VM-entry failure due to invalid guest state. */
4793	{
4794	#ifdef VBOX_STRICT
4795	RTCCUINTREG val2 = 0;
4796
4797	Log(("VMX_EXIT_ERR_INVALID_GUEST_STATE\n"));
4798
4799	VMXReadVMCS(VMX_VMCS64_GUEST_RIP, &val2);
4800	Log(("Old eip %RGv new %RGv\n", (RTGCPTR)pCtx->rip, (RTGCPTR)val2));
4801
4802	VMXReadVMCS(VMX_VMCS64_GUEST_CR0, &val2);
4803	Log(("VMX_VMCS_GUEST_CR0 %RX64\n", (uint64_t)val2));
4804
4805	VMXReadVMCS(VMX_VMCS64_GUEST_CR3, &val2);
4806	Log(("VMX_VMCS_GUEST_CR3 %RX64\n", (uint64_t)val2));
4807
4808	VMXReadVMCS(VMX_VMCS64_GUEST_CR4, &val2);
4809	Log(("VMX_VMCS_GUEST_CR4 %RX64\n", (uint64_t)val2));
4810
4811	VMXReadVMCS(VMX_VMCS_GUEST_RFLAGS, &val2);
4812	Log(("VMX_VMCS_GUEST_RFLAGS %08x\n", val2));
4813
4814	VMX_LOG_SELREG(CS, "CS", val2);
4815	VMX_LOG_SELREG(DS, "DS", val2);
4816	VMX_LOG_SELREG(ES, "ES", val2);
4817	VMX_LOG_SELREG(FS, "FS", val2);
4818	VMX_LOG_SELREG(GS, "GS", val2);
4819	VMX_LOG_SELREG(SS, "SS", val2);
4820	VMX_LOG_SELREG(TR, "TR", val2);
4821	VMX_LOG_SELREG(LDTR, "LDTR", val2);
4822
4823	VMXReadVMCS(VMX_VMCS64_GUEST_GDTR_BASE, &val2);
4824	Log(("VMX_VMCS_GUEST_GDTR_BASE %RX64\n", (uint64_t)val2));
4825	VMXReadVMCS(VMX_VMCS64_GUEST_IDTR_BASE, &val2);
4826	Log(("VMX_VMCS_GUEST_IDTR_BASE %RX64\n", (uint64_t)val2));
4827	#endif /* VBOX_STRICT */
4828	rc = VERR_VMX_INVALID_GUEST_STATE;
4829	break;
4830	}
4831
4832	case VMX_EXIT_ERR_MSR_LOAD: /* 34 VM-entry failure due to MSR loading. */
4833	case VMX_EXIT_ERR_MACHINE_CHECK: /* 41 VM-entry failure due to machine-check. */
4834	default:
4835	rc = VERR_VMX_UNEXPECTED_EXIT_CODE;
4836	AssertMsgFailed(("Unexpected exit code %d\n", exitReason)); /* Can't happen. */
4837	break;
4838
4839	}
4840
4841	end:
4842	/* We now going back to ring-3, so clear the action flag. */
4843	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
4844
4845	/*
4846	* Signal changes for the recompiler.
4847	*/
4848	CPUMSetChangedFlags(pVCpu,
4849	CPUM_CHANGED_SYSENTER_MSR
4850	\| CPUM_CHANGED_LDTR
4851	\| CPUM_CHANGED_GDTR
4852	\| CPUM_CHANGED_IDTR
4853	\| CPUM_CHANGED_TR
4854	\| CPUM_CHANGED_HIDDEN_SEL_REGS);
4855
4856	/*
4857	* If we executed vmlaunch/vmresume and an external IRQ was pending, then we don't have to do a full sync the next time.
4858	*/
4859	if ( exitReason == VMX_EXIT_EXTERNAL_IRQ
4860	&& !VMX_EXIT_INTERRUPTION_INFO_VALID(intInfo))
4861	{
4862	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatPendingHostIrq);
4863	/* On the next entry we'll only sync the host context. */
4864	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_HOST_CONTEXT;
4865	}
4866	else
4867	{
4868	/* On the next entry we'll sync everything. */
4869	/** @todo we can do better than this */
4870	/* Not in the VINF_PGM_CHANGE_MODE though! */
4871	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_ALL;
4872	}
4873
4874	/* Translate into a less severe return code */
4875	if (rc == VERR_EM_INTERPRETER)
4876	rc = VINF_EM_RAW_EMULATE_INSTR;
4877	else if (rc == VERR_VMX_INVALID_VMCS_PTR)
4878	{
4879	/* Try to extract more information about what might have gone wrong here. */
4880	VMXGetActivateVMCS(&pVCpu->hwaccm.s.vmx.lasterror.u64VMCSPhys);
4881	pVCpu->hwaccm.s.vmx.lasterror.ulVMCSRevision = (uint32_t )pVCpu->hwaccm.s.vmx.pvVMCS;
4882	pVCpu->hwaccm.s.vmx.lasterror.idEnteredCpu = pVCpu->hwaccm.s.idEnteredCpu;
4883	pVCpu->hwaccm.s.vmx.lasterror.idCurrentCpu = RTMpCpuId();
4884	}
4885
4886	/* Just set the correct state here instead of trying to catch every goto above. */
4887	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED, VMCPUSTATE_STARTED_EXEC);
4888
4889	#ifdef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
4890	/* Restore interrupts if we exited after disabling them. */
4891	if (uOldEFlags != ~(RTCCUINTREG)0)
4892	ASMSetFlags(uOldEFlags);
4893	#endif
4894
4895	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2, x);
4896	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit1, x);
4897	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatEntry, x);
4898	Log2(("X"));
4899	return VBOXSTRICTRC_TODO(rc);
4900	}
4901
4902
4903	/**
4904	* Enters the VT-x session.
4905	*
4906	* @returns VBox status code.
4907	* @param pVM Pointer to the VM.
4908	* @param pVCpu Pointer to the VMCPU.
4909	* @param pCpu Pointer to the CPU info struct.
4910	*/
4911	VMMR0DECL(int) VMXR0Enter(PVM pVM, PVMCPU pVCpu, PHMGLOBLCPUINFO pCpu)
4912	{
4913	Assert(pVM->hwaccm.s.vmx.fSupported);
4914	NOREF(pCpu);
4915
4916	unsigned cr4 = ASMGetCR4();
4917	if (!(cr4 & X86_CR4_VMXE))
4918	{
4919	AssertMsgFailed(("X86_CR4_VMXE should be set!\n"));
4920	return VERR_VMX_X86_CR4_VMXE_CLEARED;
4921	}
4922
4923	/* Activate the VMCS. */
4924	int rc = VMXActivateVMCS(pVCpu->hwaccm.s.vmx.HCPhysVMCS);
4925	if (RT_FAILURE(rc))
4926	return rc;
4927
4928	pVCpu->hwaccm.s.fResumeVM = false;
4929	return VINF_SUCCESS;
4930	}
4931
4932
4933	/**
4934	* Leaves the VT-x session.
4935	*
4936	* @returns VBox status code.
4937	* @param pVM Pointer to the VM.
4938	* @param pVCpu Pointer to the VMCPU.
4939	* @param pCtx Pointer to the guests CPU context.
4940	*/
4941	VMMR0DECL(int) VMXR0Leave(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
4942	{
4943	Assert(pVM->hwaccm.s.vmx.fSupported);
4944
4945	#ifdef DEBUG
4946	if (CPUMIsHyperDebugStateActive(pVCpu))
4947	{
4948	CPUMR0LoadHostDebugState(pVM, pVCpu);
4949	Assert(pVCpu->hwaccm.s.vmx.proc_ctls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT);
4950	}
4951	else
4952	#endif
4953
4954	/*
4955	* Save the guest debug state if necessary.
4956	*/
4957	if (CPUMIsGuestDebugStateActive(pVCpu))
4958	{
4959	CPUMR0SaveGuestDebugState(pVM, pVCpu, pCtx, true /* save DR6 */);
4960
4961	/* Enable DRx move intercepts again. */
4962	pVCpu->hwaccm.s.vmx.proc_ctls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT;
4963	int rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
4964	AssertRC(rc);
4965
4966	/* Resync the debug registers the next time. */
4967	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_GUEST_DEBUG;
4968	}
4969	else
4970	Assert(pVCpu->hwaccm.s.vmx.proc_ctls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT);
4971
4972	/*
4973	* Clear VMCS, marking it inactive, clearing implementation-specific data and writing
4974	* VMCS data back to memory.
4975	*/
4976	int rc = VMXClearVMCS(pVCpu->hwaccm.s.vmx.HCPhysVMCS);
4977	AssertRC(rc);
4978
4979	return VINF_SUCCESS;
4980	}
4981
4982
4983	/**
4984	* Flush the TLB using EPT.
4985	*
4986	* @returns VBox status code.
4987	* @param pVM Pointer to the VM.
4988	* @param pVCpu Pointer to the VMCPU.
4989	* @param enmFlush Type of flush.
4990	*/
4991	static void hmR0VmxFlushEPT(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_EPT enmFlush)
4992	{
4993	uint64_t descriptor[2];
4994
4995	LogFlow(("hmR0VmxFlushEPT %d\n", enmFlush));
4996	Assert(pVM->hwaccm.s.fNestedPaging);
4997	descriptor[0] = pVCpu->hwaccm.s.vmx.GCPhysEPTP;
4998	descriptor[1] = 0; /* MBZ. Intel spec. 33.3 VMX Instructions */
4999	int rc = VMXR0InvEPT(enmFlush, &descriptor[0]);
5000	AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %x %RGv failed with %d\n", enmFlush, pVCpu->hwaccm.s.vmx.GCPhysEPTP, rc));
5001	}
5002
5003
5004	/**
5005	* Flush the TLB using VPID.
5006	*
5007	* @returns VBox status code.
5008	* @param pVM Pointer to the VM.
5009	* @param pVCpu Pointer to the VMCPU (can be NULL depending on @a
5010	* enmFlush).
5011	* @param enmFlush Type of flush.
5012	* @param GCPtr Virtual address of the page to flush (can be 0 depending
5013	* on @a enmFlush).
5014	*/
5015	static void hmR0VmxFlushVPID(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_VPID enmFlush, RTGCPTR GCPtr)
5016	{
5017	uint64_t descriptor[2];
5018
5019	Assert(pVM->hwaccm.s.vmx.fVPID);
5020	if (enmFlush == VMX_FLUSH_VPID_ALL_CONTEXTS)
5021	{
5022	descriptor[0] = 0;
5023	descriptor[1] = 0;
5024	}
5025	else
5026	{
5027	AssertPtr(pVCpu);
5028	AssertMsg(pVCpu->hwaccm.s.uCurrentASID != 0, ("VMXR0InvVPID invalid ASID %lu\n", pVCpu->hwaccm.s.uCurrentASID));
5029	AssertMsg(pVCpu->hwaccm.s.uCurrentASID <= UINT16_MAX, ("VMXR0InvVPID invalid ASID %lu\n", pVCpu->hwaccm.s.uCurrentASID));
5030	descriptor[0] = pVCpu->hwaccm.s.uCurrentASID;
5031	descriptor[1] = GCPtr;
5032	}
5033	int rc = VMXR0InvVPID(enmFlush, &descriptor[0]); NOREF(rc);
5034	AssertMsg(rc == VINF_SUCCESS,
5035	("VMXR0InvVPID %x %x %RGv failed with %d\n", enmFlush, pVCpu ? pVCpu->hwaccm.s.uCurrentASID : 0, GCPtr, rc));
5036	}
5037
5038
5039	/**
5040	* Invalidates a guest page by guest virtual address. Only relevant for
5041	* EPT/VPID, otherwise there is nothing really to invalidate.
5042	*
5043	* @returns VBox status code.
5044	* @param pVM Pointer to the VM.
5045	* @param pVCpu Pointer to the VMCPU.
5046	* @param GCVirt Guest virtual address of the page to invalidate.
5047	*/
5048	VMMR0DECL(int) VMXR0InvalidatePage(PVM pVM, PVMCPU pVCpu, RTGCPTR GCVirt)
5049	{
5050	bool fFlushPending = VMCPU_FF_ISSET(pVCpu, VMCPU_FF_TLB_FLUSH);
5051
5052	Log2(("VMXR0InvalidatePage %RGv\n", GCVirt));
5053
5054	if (!fFlushPending)
5055	{
5056	/*
5057	* We must invalidate the guest TLB entry in either case, we cannot ignore it even for the EPT case
5058	* See @bugref{6043} and @bugref{6177}
5059	*
5060	* Set the VMCPU_FF_TLB_FLUSH force flag and flush before VMENTRY in hmR0VmxSetupTLB*() as this
5061	* function maybe called in a loop with individual addresses.
5062	*/
5063	if (pVM->hwaccm.s.vmx.fVPID)
5064	{
5065	/* If we can flush just this page do it, otherwise flush as little as possible. */
5066	if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVVPID_CAPS_INDIV_ADDR)
5067	hmR0VmxFlushVPID(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, GCVirt);
5068	else
5069	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
5070	}
5071	else if (pVM->hwaccm.s.fNestedPaging)
5072	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
5073	}
5074
5075	return VINF_SUCCESS;
5076	}
5077
5078
5079	/**
5080	* Invalidates a guest page by physical address. Only relevant for EPT/VPID,
5081	* otherwise there is nothing really to invalidate.
5082	*
5083	* NOTE: Assumes the current instruction references this physical page though a virtual address!!
5084	*
5085	* @returns VBox status code.
5086	* @param pVM Pointer to the VM.
5087	* @param pVCpu Pointer to the VMCPU.
5088	* @param GCPhys Guest physical address of the page to invalidate.
5089	*/
5090	VMMR0DECL(int) VMXR0InvalidatePhysPage(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys)
5091	{
5092	LogFlow(("VMXR0InvalidatePhysPage %RGp\n", GCPhys));
5093
5094	/*
5095	* We cannot flush a page by guest-physical address. invvpid takes only a linear address
5096	* while invept only flushes by EPT not individual addresses. We update the force flag here
5097	* and flush before VMENTRY in hmR0VmxSetupTLB*(). This function might be called in a loop.
5098	*/
5099	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
5100	return VINF_SUCCESS;
5101	}
5102
5103
5104	/**
5105	* Report world switch error and dump some useful debug info.
5106	*
5107	* @param pVM Pointer to the VM.
5108	* @param pVCpu Pointer to the VMCPU.
5109	* @param rc Return code.
5110	* @param pCtx Pointer to the current guest CPU context (not updated).
5111	*/
5112	static void hmR0VmxReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, VBOXSTRICTRC rc, PCPUMCTX pCtx)
5113	{
5114	NOREF(pVM);
5115
5116	switch (VBOXSTRICTRC_VAL(rc))
5117	{
5118	case VERR_VMX_INVALID_VMXON_PTR:
5119	AssertFailed();
5120	break;
5121
5122	case VERR_VMX_UNABLE_TO_START_VM:
5123	case VERR_VMX_UNABLE_TO_RESUME_VM:
5124	{
5125	int rc2;
5126	RTCCUINTREG exitReason, instrError;
5127
5128	rc2 = VMXReadVMCS(VMX_VMCS32_RO_EXIT_REASON, &exitReason);
5129	rc2 \|= VMXReadVMCS(VMX_VMCS32_RO_VM_INSTR_ERROR, &instrError);
5130	AssertRC(rc2);
5131	if (rc2 == VINF_SUCCESS)
5132	{
5133	Log(("Unable to start/resume VM for reason: %x. Instruction error %x\n", (uint32_t)exitReason,
5134	(uint32_t)instrError));
5135	Log(("Current stack %08x\n", &rc2));
5136
5137	pVCpu->hwaccm.s.vmx.lasterror.ulInstrError = instrError;
5138	pVCpu->hwaccm.s.vmx.lasterror.ulExitReason = exitReason;
5139
5140	#ifdef VBOX_STRICT
5141	RTGDTR gdtr;
5142	PCX86DESCHC pDesc;
5143	RTCCUINTREG val;
5144
5145	ASMGetGDTR(&gdtr);
5146
5147	VMXReadVMCS(VMX_VMCS64_GUEST_RIP, &val);
5148	Log(("Old eip %RGv new %RGv\n", (RTGCPTR)pCtx->rip, (RTGCPTR)val));
5149	VMXReadVMCS(VMX_VMCS_CTRL_PIN_EXEC_CONTROLS, &val);
5150	Log(("VMX_VMCS_CTRL_PIN_EXEC_CONTROLS %08x\n", val));
5151	VMXReadVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, &val);
5152	Log(("VMX_VMCS_CTRL_PROC_EXEC_CONTROLS %08x\n", val));
5153	VMXReadVMCS(VMX_VMCS_CTRL_ENTRY_CONTROLS, &val);
5154	Log(("VMX_VMCS_CTRL_ENTRY_CONTROLS %08x\n", val));
5155	VMXReadVMCS(VMX_VMCS_CTRL_EXIT_CONTROLS, &val);
5156	Log(("VMX_VMCS_CTRL_EXIT_CONTROLS %08x\n", val));
5157
5158	VMXReadVMCS(VMX_VMCS_HOST_CR0, &val);
5159	Log(("VMX_VMCS_HOST_CR0 %08x\n", val));
5160	VMXReadVMCS(VMX_VMCS_HOST_CR3, &val);
5161	Log(("VMX_VMCS_HOST_CR3 %08x\n", val));
5162	VMXReadVMCS(VMX_VMCS_HOST_CR4, &val);
5163	Log(("VMX_VMCS_HOST_CR4 %08x\n", val));
5164
5165	VMXReadVMCS(VMX_VMCS16_HOST_FIELD_CS, &val);
5166	Log(("VMX_VMCS_HOST_FIELD_CS %08x\n", val));
5167	VMXReadVMCS(VMX_VMCS_GUEST_RFLAGS, &val);
5168	Log(("VMX_VMCS_GUEST_RFLAGS %08x\n", val));
5169
5170	if (val < gdtr.cbGdt)
5171	{
5172	pDesc = (PCX86DESCHC)(gdtr.pGdt + (val & X86_SEL_MASK));
5173	HWACCMR0DumpDescriptor(pDesc, val, "CS: ");
5174	}
5175
5176	VMXReadVMCS(VMX_VMCS16_HOST_FIELD_DS, &val);
5177	Log(("VMX_VMCS_HOST_FIELD_DS %08x\n", val));
5178	if (val < gdtr.cbGdt)
5179	{
5180	pDesc = (PCX86DESCHC)(gdtr.pGdt + (val & X86_SEL_MASK));
5181	HWACCMR0DumpDescriptor(pDesc, val, "DS: ");
5182	}
5183
5184	VMXReadVMCS(VMX_VMCS16_HOST_FIELD_ES, &val);
5185	Log(("VMX_VMCS_HOST_FIELD_ES %08x\n", val));
5186	if (val < gdtr.cbGdt)
5187	{
5188	pDesc = (PCX86DESCHC)(gdtr.pGdt + (val & X86_SEL_MASK));
5189	HWACCMR0DumpDescriptor(pDesc, val, "ES: ");
5190	}
5191
5192	VMXReadVMCS(VMX_VMCS16_HOST_FIELD_FS, &val);
5193	Log(("VMX_VMCS16_HOST_FIELD_FS %08x\n", val));
5194	if (val < gdtr.cbGdt)
5195	{
5196	pDesc = (PCX86DESCHC)(gdtr.pGdt + (val & X86_SEL_MASK));
5197	HWACCMR0DumpDescriptor(pDesc, val, "FS: ");
5198	}
5199
5200	VMXReadVMCS(VMX_VMCS16_HOST_FIELD_GS, &val);
5201	Log(("VMX_VMCS16_HOST_FIELD_GS %08x\n", val));
5202	if (val < gdtr.cbGdt)
5203	{
5204	pDesc = (PCX86DESCHC)(gdtr.pGdt + (val & X86_SEL_MASK));
5205	HWACCMR0DumpDescriptor(pDesc, val, "GS: ");
5206	}
5207
5208	VMXReadVMCS(VMX_VMCS16_HOST_FIELD_SS, &val);
5209	Log(("VMX_VMCS16_HOST_FIELD_SS %08x\n", val));
5210	if (val < gdtr.cbGdt)
5211	{
5212	pDesc = (PCX86DESCHC)(gdtr.pGdt + (val & X86_SEL_MASK));
5213	HWACCMR0DumpDescriptor(pDesc, val, "SS: ");
5214	}
5215
5216	VMXReadVMCS(VMX_VMCS16_HOST_FIELD_TR, &val);
5217	Log(("VMX_VMCS16_HOST_FIELD_TR %08x\n", val));
5218	if (val < gdtr.cbGdt)
5219	{
5220	pDesc = (PCX86DESCHC)(gdtr.pGdt + (val & X86_SEL_MASK));
5221	HWACCMR0DumpDescriptor(pDesc, val, "TR: ");
5222	}
5223
5224	VMXReadVMCS(VMX_VMCS_HOST_TR_BASE, &val);
5225	Log(("VMX_VMCS_HOST_TR_BASE %RHv\n", val));
5226	VMXReadVMCS(VMX_VMCS_HOST_GDTR_BASE, &val);
5227	Log(("VMX_VMCS_HOST_GDTR_BASE %RHv\n", val));
5228	VMXReadVMCS(VMX_VMCS_HOST_IDTR_BASE, &val);
5229	Log(("VMX_VMCS_HOST_IDTR_BASE %RHv\n", val));
5230	VMXReadVMCS(VMX_VMCS32_HOST_SYSENTER_CS, &val);
5231	Log(("VMX_VMCS_HOST_SYSENTER_CS %08x\n", val));
5232	VMXReadVMCS(VMX_VMCS_HOST_SYSENTER_EIP, &val);
5233	Log(("VMX_VMCS_HOST_SYSENTER_EIP %RHv\n", val));
5234	VMXReadVMCS(VMX_VMCS_HOST_SYSENTER_ESP, &val);
5235	Log(("VMX_VMCS_HOST_SYSENTER_ESP %RHv\n", val));
5236	VMXReadVMCS(VMX_VMCS_HOST_RSP, &val);
5237	Log(("VMX_VMCS_HOST_RSP %RHv\n", val));
5238	VMXReadVMCS(VMX_VMCS_HOST_RIP, &val);
5239	Log(("VMX_VMCS_HOST_RIP %RHv\n", val));
5240	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
5241	if (VMX_IS_64BIT_HOST_MODE())
5242	{
5243	Log(("MSR_K6_EFER = %RX64\n", ASMRdMsr(MSR_K6_EFER)));
5244	Log(("MSR_K6_STAR = %RX64\n", ASMRdMsr(MSR_K6_STAR)));
5245	Log(("MSR_K8_LSTAR = %RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
5246	Log(("MSR_K8_CSTAR = %RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
5247	Log(("MSR_K8_SF_MASK = %RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
5248	Log(("MSR_K8_KERNEL_GS_BASE = %RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
5249	}
5250	# endif
5251	#endif /* VBOX_STRICT */
5252	}
5253	break;
5254	}
5255
5256	default:
5257	/* impossible */
5258	AssertMsgFailed(("%Rrc (%#x)\n", VBOXSTRICTRC_VAL(rc), VBOXSTRICTRC_VAL(rc)));
5259	break;
5260	}
5261	}
5262
5263
5264	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
5265	/**
5266	* Prepares for and executes VMLAUNCH (64 bits guest mode).
5267	*
5268	* @returns VBox status code.
5269	* @param fResume Whether to vmlauch/vmresume.
5270	* @param pCtx Pointer to the guest CPU context.
5271	* @param pCache Pointer to the VMCS cache.
5272	* @param pVM Pointer to the VM.
5273	* @param pVCpu Pointer to the VMCPU.
5274	*/
5275	DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
5276	{
5277	uint32_t aParam[6];
5278	PHMGLOBLCPUINFO pCpu;
5279	RTHCPHYS HCPhysCpuPage;
5280	int rc;
5281
5282	pCpu = HWACCMR0GetCurrentCpu();
5283	HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
5284
5285	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5286	pCache->uPos = 1;
5287	pCache->interPD = PGMGetInterPaeCR3(pVM);
5288	pCache->pSwitcher = (uint64_t)pVM->hwaccm.s.pfnHost32ToGuest64R0;
5289	#endif
5290
5291	#ifdef DEBUG
5292	pCache->TestIn.HCPhysCpuPage= 0;
5293	pCache->TestIn.HCPhysVMCS = 0;
5294	pCache->TestIn.pCache = 0;
5295	pCache->TestOut.HCPhysVMCS = 0;
5296	pCache->TestOut.pCache = 0;
5297	pCache->TestOut.pCtx = 0;
5298	pCache->TestOut.eflags = 0;
5299	#endif
5300
5301	aParam[0] = (uint32_t)(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
5302	aParam[1] = (uint32_t)(HCPhysCpuPage >> 32); /* Param 1: VMXON physical address - Hi. */
5303	aParam[2] = (uint32_t)(pVCpu->hwaccm.s.vmx.HCPhysVMCS); /* Param 2: VMCS physical address - Lo. */
5304	aParam[3] = (uint32_t)(pVCpu->hwaccm.s.vmx.HCPhysVMCS >> 32); /* Param 2: VMCS physical address - Hi. */
5305	aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hwaccm.s.vmx.VMCSCache);
5306	aParam[5] = 0;
5307
5308	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5309	pCtx->dr[4] = pVM->hwaccm.s.vmx.pScratchPhys + 16 + 8;
5310	(uint32_t )(pVM->hwaccm.s.vmx.pScratch + 16 + 8) = 1;
5311	#endif
5312	rc = VMXR0Execute64BitsHandler(pVM, pVCpu, pCtx, pVM->hwaccm.s.pfnVMXGCStartVM64, 6, &aParam[0]);
5313
5314	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5315	Assert((uint32_t )(pVM->hwaccm.s.vmx.pScratch + 16 + 8) == 5);
5316	Assert(pCtx->dr[4] == 10);
5317	(uint32_t )(pVM->hwaccm.s.vmx.pScratch + 16 + 8) = 0xff;
5318	#endif
5319
5320	#ifdef DEBUG
5321	AssertMsg(pCache->TestIn.HCPhysCpuPage== HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
5322	AssertMsg(pCache->TestIn.HCPhysVMCS == pVCpu->hwaccm.s.vmx.HCPhysVMCS, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVMCS,
5323	pVCpu->hwaccm.s.vmx.HCPhysVMCS));
5324	AssertMsg(pCache->TestIn.HCPhysVMCS == pCache->TestOut.HCPhysVMCS, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVMCS,
5325	pCache->TestOut.HCPhysVMCS));
5326	AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
5327	pCache->TestOut.pCache));
5328	AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hwaccm.s.vmx.VMCSCache),
5329	("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hwaccm.s.vmx.VMCSCache)));
5330	AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
5331	pCache->TestOut.pCtx));
5332	Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
5333	#endif
5334	return rc;
5335	}
5336
5337
5338	# ifdef VBOX_STRICT
5339	static bool hmR0VmxIsValidReadField(uint32_t idxField)
5340	{
5341	switch (idxField)
5342	{
5343	case VMX_VMCS64_GUEST_RIP:
5344	case VMX_VMCS64_GUEST_RSP:
5345	case VMX_VMCS_GUEST_RFLAGS:
5346	case VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE:
5347	case VMX_VMCS_CTRL_CR0_READ_SHADOW:
5348	case VMX_VMCS64_GUEST_CR0:
5349	case VMX_VMCS_CTRL_CR4_READ_SHADOW:
5350	case VMX_VMCS64_GUEST_CR4:
5351	case VMX_VMCS64_GUEST_DR7:
5352	case VMX_VMCS32_GUEST_SYSENTER_CS:
5353	case VMX_VMCS64_GUEST_SYSENTER_EIP:
5354	case VMX_VMCS64_GUEST_SYSENTER_ESP:
5355	case VMX_VMCS32_GUEST_GDTR_LIMIT:
5356	case VMX_VMCS64_GUEST_GDTR_BASE:
5357	case VMX_VMCS32_GUEST_IDTR_LIMIT:
5358	case VMX_VMCS64_GUEST_IDTR_BASE:
5359	case VMX_VMCS16_GUEST_FIELD_CS:
5360	case VMX_VMCS32_GUEST_CS_LIMIT:
5361	case VMX_VMCS64_GUEST_CS_BASE:
5362	case VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS:
5363	case VMX_VMCS16_GUEST_FIELD_DS:
5364	case VMX_VMCS32_GUEST_DS_LIMIT:
5365	case VMX_VMCS64_GUEST_DS_BASE:
5366	case VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS:
5367	case VMX_VMCS16_GUEST_FIELD_ES:
5368	case VMX_VMCS32_GUEST_ES_LIMIT:
5369	case VMX_VMCS64_GUEST_ES_BASE:
5370	case VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS:
5371	case VMX_VMCS16_GUEST_FIELD_FS:
5372	case VMX_VMCS32_GUEST_FS_LIMIT:
5373	case VMX_VMCS64_GUEST_FS_BASE:
5374	case VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS:
5375	case VMX_VMCS16_GUEST_FIELD_GS:
5376	case VMX_VMCS32_GUEST_GS_LIMIT:
5377	case VMX_VMCS64_GUEST_GS_BASE:
5378	case VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS:
5379	case VMX_VMCS16_GUEST_FIELD_SS:
5380	case VMX_VMCS32_GUEST_SS_LIMIT:
5381	case VMX_VMCS64_GUEST_SS_BASE:
5382	case VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS:
5383	case VMX_VMCS16_GUEST_FIELD_LDTR:
5384	case VMX_VMCS32_GUEST_LDTR_LIMIT:
5385	case VMX_VMCS64_GUEST_LDTR_BASE:
5386	case VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS:
5387	case VMX_VMCS16_GUEST_FIELD_TR:
5388	case VMX_VMCS32_GUEST_TR_LIMIT:
5389	case VMX_VMCS64_GUEST_TR_BASE:
5390	case VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS:
5391	case VMX_VMCS32_RO_EXIT_REASON:
5392	case VMX_VMCS32_RO_VM_INSTR_ERROR:
5393	case VMX_VMCS32_RO_EXIT_INSTR_LENGTH:
5394	case VMX_VMCS32_RO_EXIT_INTERRUPTION_ERRCODE:
5395	case VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO:
5396	case VMX_VMCS32_RO_EXIT_INSTR_INFO:
5397	case VMX_VMCS_RO_EXIT_QUALIFICATION:
5398	case VMX_VMCS32_RO_IDT_INFO:
5399	case VMX_VMCS32_RO_IDT_ERRCODE:
5400	case VMX_VMCS64_GUEST_CR3:
5401	case VMX_VMCS_EXIT_PHYS_ADDR_FULL:
5402	return true;
5403	}
5404	return false;
5405	}
5406
5407
5408	static bool hmR0VmxIsValidWriteField(uint32_t idxField)
5409	{
5410	switch (idxField)
5411	{
5412	case VMX_VMCS64_GUEST_LDTR_BASE:
5413	case VMX_VMCS64_GUEST_TR_BASE:
5414	case VMX_VMCS64_GUEST_GDTR_BASE:
5415	case VMX_VMCS64_GUEST_IDTR_BASE:
5416	case VMX_VMCS64_GUEST_SYSENTER_EIP:
5417	case VMX_VMCS64_GUEST_SYSENTER_ESP:
5418	case VMX_VMCS64_GUEST_CR0:
5419	case VMX_VMCS64_GUEST_CR4:
5420	case VMX_VMCS64_GUEST_CR3:
5421	case VMX_VMCS64_GUEST_DR7:
5422	case VMX_VMCS64_GUEST_RIP:
5423	case VMX_VMCS64_GUEST_RSP:
5424	case VMX_VMCS64_GUEST_CS_BASE:
5425	case VMX_VMCS64_GUEST_DS_BASE:
5426	case VMX_VMCS64_GUEST_ES_BASE:
5427	case VMX_VMCS64_GUEST_FS_BASE:
5428	case VMX_VMCS64_GUEST_GS_BASE:
5429	case VMX_VMCS64_GUEST_SS_BASE:
5430	return true;
5431	}
5432	return false;
5433	}
5434	# endif /* VBOX_STRICT */
5435
5436
5437	/**
5438	* Executes the specified handler in 64-bit mode.
5439	*
5440	* @returns VBox status code.
5441	* @param pVM Pointer to the VM.
5442	* @param pVCpu Pointer to the VMCPU.
5443	* @param pCtx Pointer to the guest CPU context.
5444	* @param pfnHandler Pointer to the RC handler function.
5445	* @param cbParam Number of parameters.
5446	* @param paParam Array of 32-bit parameters.
5447	*/
5448	VMMR0DECL(int) VMXR0Execute64BitsHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, RTRCPTR pfnHandler, uint32_t cbParam,
5449	uint32_t *paParam)
5450	{
5451	int rc, rc2;
5452	PHMGLOBLCPUINFO pCpu;
5453	RTHCPHYS HCPhysCpuPage;
5454	RTHCUINTREG uOldEFlags;
5455
5456	AssertReturn(pVM->hwaccm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
5457	Assert(pfnHandler);
5458	Assert(pVCpu->hwaccm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hwaccm.s.vmx.VMCSCache.Write.aField));
5459	Assert(pVCpu->hwaccm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hwaccm.s.vmx.VMCSCache.Read.aField));
5460
5461	#ifdef VBOX_STRICT
5462	for (unsigned i=0;i<pVCpu->hwaccm.s.vmx.VMCSCache.Write.cValidEntries;i++)
5463	Assert(hmR0VmxIsValidWriteField(pVCpu->hwaccm.s.vmx.VMCSCache.Write.aField[i]));
5464
5465	for (unsigned i=0;i<pVCpu->hwaccm.s.vmx.VMCSCache.Read.cValidEntries;i++)
5466	Assert(hmR0VmxIsValidReadField(pVCpu->hwaccm.s.vmx.VMCSCache.Read.aField[i]));
5467	#endif
5468
5469	/* Disable interrupts. */
5470	uOldEFlags = ASMIntDisableFlags();
5471
5472	#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
5473	RTCPUID idHostCpu = RTMpCpuId();
5474	CPUMR0SetLApic(pVM, idHostCpu);
5475	#endif
5476
5477	pCpu = HWACCMR0GetCurrentCpu();
5478	HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
5479
5480	/* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
5481	VMXClearVMCS(pVCpu->hwaccm.s.vmx.HCPhysVMCS);
5482
5483	/* Leave VMX Root Mode. */
5484	VMXDisable();
5485
5486	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
5487
5488	CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
5489	CPUMSetHyperEIP(pVCpu, pfnHandler);
5490	for (int i=(int)cbParam-1;i>=0;i--)
5491	CPUMPushHyper(pVCpu, paParam[i]);
5492
5493	STAM_PROFILE_ADV_START(&pVCpu->hwaccm.s.StatWorldSwitch3264, z);
5494
5495	/* Call switcher. */
5496	rc = pVM->hwaccm.s.pfnHost32ToGuest64R0(pVM, RT_OFFSETOF(VM, aCpus[pVCpu->idCpu].cpum) - RT_OFFSETOF(VM, cpum));
5497	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatWorldSwitch3264, z);
5498
5499	/* Make sure the VMX instructions don't cause #UD faults. */
5500	ASMSetCR4(ASMGetCR4() \| X86_CR4_VMXE);
5501
5502	/* Enter VMX Root Mode */
5503	rc2 = VMXEnable(HCPhysCpuPage);
5504	if (RT_FAILURE(rc2))
5505	{
5506	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
5507	ASMSetFlags(uOldEFlags);
5508	return VERR_VMX_VMXON_FAILED;
5509	}
5510
5511	rc2 = VMXActivateVMCS(pVCpu->hwaccm.s.vmx.HCPhysVMCS);
5512	AssertRC(rc2);
5513	Assert(!(ASMGetFlags() & X86_EFL_IF));
5514	ASMSetFlags(uOldEFlags);
5515	return rc;
5516	}
5517	#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL) */
5518
5519
5520	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
5521	/**
5522	* Executes VMWRITE.
5523	*
5524	* @returns VBox status code
5525	* @param pVCpu Pointer to the VMCPU.
5526	* @param idxField VMCS field index.
5527	* @param u64Val 16, 32 or 64 bits value.
5528	*/
5529	VMMR0DECL(int) VMXWriteVMCS64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5530	{
5531	int rc;
5532	switch (idxField)
5533	{
5534	case VMX_VMCS_CTRL_TSC_OFFSET_FULL:
5535	case VMX_VMCS_CTRL_IO_BITMAP_A_FULL:
5536	case VMX_VMCS_CTRL_IO_BITMAP_B_FULL:
5537	case VMX_VMCS_CTRL_MSR_BITMAP_FULL:
5538	case VMX_VMCS_CTRL_VMEXIT_MSR_STORE_FULL:
5539	case VMX_VMCS_CTRL_VMEXIT_MSR_LOAD_FULL:
5540	case VMX_VMCS_CTRL_VMENTRY_MSR_LOAD_FULL:
5541	case VMX_VMCS_CTRL_VAPIC_PAGEADDR_FULL:
5542	case VMX_VMCS_CTRL_APIC_ACCESSADDR_FULL:
5543	case VMX_VMCS_GUEST_LINK_PTR_FULL:
5544	case VMX_VMCS_GUEST_PDPTR0_FULL:
5545	case VMX_VMCS_GUEST_PDPTR1_FULL:
5546	case VMX_VMCS_GUEST_PDPTR2_FULL:
5547	case VMX_VMCS_GUEST_PDPTR3_FULL:
5548	case VMX_VMCS_GUEST_DEBUGCTL_FULL:
5549	case VMX_VMCS_GUEST_EFER_FULL:
5550	case VMX_VMCS_CTRL_EPTP_FULL:
5551	/* These fields consist of two parts, which are both writable in 32 bits mode. */
5552	rc = VMXWriteVMCS32(idxField, u64Val);
5553	rc \|= VMXWriteVMCS32(idxField + 1, (uint32_t)(u64Val >> 32ULL));
5554	AssertRC(rc);
5555	return rc;
5556
5557	case VMX_VMCS64_GUEST_LDTR_BASE:
5558	case VMX_VMCS64_GUEST_TR_BASE:
5559	case VMX_VMCS64_GUEST_GDTR_BASE:
5560	case VMX_VMCS64_GUEST_IDTR_BASE:
5561	case VMX_VMCS64_GUEST_SYSENTER_EIP:
5562	case VMX_VMCS64_GUEST_SYSENTER_ESP:
5563	case VMX_VMCS64_GUEST_CR0:
5564	case VMX_VMCS64_GUEST_CR4:
5565	case VMX_VMCS64_GUEST_CR3:
5566	case VMX_VMCS64_GUEST_DR7:
5567	case VMX_VMCS64_GUEST_RIP:
5568	case VMX_VMCS64_GUEST_RSP:
5569	case VMX_VMCS64_GUEST_CS_BASE:
5570	case VMX_VMCS64_GUEST_DS_BASE:
5571	case VMX_VMCS64_GUEST_ES_BASE:
5572	case VMX_VMCS64_GUEST_FS_BASE:
5573	case VMX_VMCS64_GUEST_GS_BASE:
5574	case VMX_VMCS64_GUEST_SS_BASE:
5575	/* Queue a 64 bits value as we can't set it in 32 bits host mode. */
5576	if (u64Val >> 32ULL)
5577	rc = VMXWriteCachedVMCSEx(pVCpu, idxField, u64Val);
5578	else
5579	rc = VMXWriteVMCS32(idxField, (uint32_t)u64Val);
5580
5581	return rc;
5582
5583	default:
5584	AssertMsgFailed(("Unexpected field %x\n", idxField));
5585	return VERR_INVALID_PARAMETER;
5586	}
5587	}
5588
5589
5590	/**
5591	* Cache VMCS writes for performance reasons (Darwin) and for running 64 bits guests on 32 bits hosts.
5592	*
5593	* @param pVCpu Pointer to the VMCPU.
5594	* @param idxField VMCS field index.
5595	* @param u64Val 16, 32 or 64 bits value.
5596	*/
5597	VMMR0DECL(int) VMXWriteCachedVMCSEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5598	{
5599	PVMCSCACHE pCache = &pVCpu->hwaccm.s.vmx.VMCSCache;
5600
5601	AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1,
5602	("entries=%x\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
5603
5604	/* Make sure there are no duplicates. */
5605	for (unsigned i = 0; i < pCache->Write.cValidEntries; i++)
5606	{
5607	if (pCache->Write.aField[i] == idxField)
5608	{
5609	pCache->Write.aFieldVal[i] = u64Val;
5610	return VINF_SUCCESS;
5611	}
5612	}
5613
5614	pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
5615	pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
5616	pCache->Write.cValidEntries++;
5617	return VINF_SUCCESS;
5618	}
5619
5620	#endif /* HC_ARCH_BITS == 32 && !VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0 */
5621

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMR0/HWVMXR0.cpp@ 43354

Download in other formats: