HWVMXR0.cpp@ 41834

Last change on this file since 41834 was 41834, checked in by vboxsync, 12 years ago
VMM/HWVMXR0: Fix using stale VPIDs on resume.
Property svn:eol-style set to `native` Property svn:keywords set to `Id`
File size: 215.9 KB

Line
1	/* $Id: HWVMXR0.cpp 41834 2012-06-19 15:27:49Z vboxsync $ */
2	/** @file
3	* HM VMX (VT-x) - Host Context Ring-0.
4	*/
5
6	/*
7	* Copyright (C) 2006-2012 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18
19	/*******************************************************************************
20	* Header Files *
21	*******************************************************************************/
22	#define LOG_GROUP LOG_GROUP_HWACCM
23	#include <iprt/asm-amd64-x86.h>
24	#include <VBox/vmm/hwaccm.h>
25	#include <VBox/vmm/pgm.h>
26	#include <VBox/vmm/dbgf.h>
27	#include <VBox/vmm/dbgftrace.h>
28	#include <VBox/vmm/selm.h>
29	#include <VBox/vmm/iom.h>
30	#ifdef VBOX_WITH_REM
31	# include <VBox/vmm/rem.h>
32	#endif
33	#include <VBox/vmm/tm.h>
34	#include "HWACCMInternal.h"
35	#include <VBox/vmm/vm.h>
36	#include <VBox/vmm/pdmapi.h>
37	#include <VBox/err.h>
38	#include <VBox/log.h>
39	#include <iprt/assert.h>
40	#include <iprt/param.h>
41	#include <iprt/string.h>
42	#include <iprt/time.h>
43	#ifdef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
44	# include <iprt/thread.h>
45	#endif
46	#include <iprt/x86.h>
47	#include "HWVMXR0.h"
48
49	#include "dtrace/VBoxVMM.h"
50
51
52	/*******************************************************************************
53	* Defined Constants And Macros *
54	*******************************************************************************/
55	#if defined(RT_ARCH_AMD64)
56	# define VMX_IS_64BIT_HOST_MODE() (true)
57	#elif defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
58	# define VMX_IS_64BIT_HOST_MODE() (g_fVMXIs64bitHost != 0)
59	#else
60	# define VMX_IS_64BIT_HOST_MODE() (false)
61	#endif
62
63
64	/*******************************************************************************
65	* Global Variables *
66	*******************************************************************************/
67	/* IO operation lookup arrays. */
68	static uint32_t const g_aIOSize[4] = {1, 2, 0, 4};
69	static uint32_t const g_aIOOpAnd[4] = {0xff, 0xffff, 0, 0xffffffff};
70
71	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
72	/** See HWACCMR0A.asm. */
73	extern "C" uint32_t g_fVMXIs64bitHost;
74	#endif
75
76
77	/*******************************************************************************
78	* Local Functions *
79	*******************************************************************************/
80	static DECLCALLBACK(void) hmR0VmxSetupTLBEPT(PVM pVM, PVMCPU pVCpu);
81	static DECLCALLBACK(void) hmR0VmxSetupTLBVPID(PVM pVM, PVMCPU pVCpu);
82	static DECLCALLBACK(void) hmR0VmxSetupTLBBoth(PVM pVM, PVMCPU pVCpu);
83	static DECLCALLBACK(void) hmR0VmxSetupTLBDummy(PVM pVM, PVMCPU pVCpu);
84	static void hmR0VmxFlushEPT(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_EPT enmFlush);
85	static void hmR0VmxFlushVPID(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_VPID enmFlush, RTGCPTR GCPtr);
86	static void hmR0VmxUpdateExceptionBitmap(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx);
87	static void hmR0VmxSetMSRPermission(PVMCPU pVCpu, unsigned ulMSR, bool fRead, bool fWrite);
88	static void hmR0VmxReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, VBOXSTRICTRC rc, PCPUMCTX pCtx);
89
90
91	/**
92	* Updates error from VMCS to HWACCMCPU's lasterror record.
93	*
94	* @param pVM Pointer to the VM.
95	* @param pVCpu Pointer to the VMCPU.
96	* @param rc The error code.
97	*/
98	static void hmR0VmxCheckError(PVM pVM, PVMCPU pVCpu, int rc)
99	{
100	if (rc == VERR_VMX_GENERIC)
101	{
102	RTCCUINTREG instrError;
103
104	VMXReadVMCS(VMX_VMCS32_RO_VM_INSTR_ERROR, &instrError);
105	pVCpu->hwaccm.s.vmx.lasterror.ulInstrError = instrError;
106	}
107	pVM->hwaccm.s.lLastError = rc;
108	}
109
110
111	/**
112	* Sets up and activates VT-x on the current CPU.
113	*
114	* @returns VBox status code.
115	* @param pCpu Pointer to the CPU info struct.
116	* @param pVM Pointer to the VM. (can be NULL after a resume!!)
117	* @param pvCpuPage Pointer to the global CPU page.
118	* @param HCPhysCpuPage Physical address of the global CPU page.
119	*/
120	VMMR0DECL(int) VMXR0EnableCpu(PHMGLOBLCPUINFO pCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
121	{
122	AssertReturn(HCPhysCpuPage != 0 && HCPhysCpuPage != NIL_RTHCPHYS, VERR_INVALID_PARAMETER);
123	AssertReturn(pvCpuPage, VERR_INVALID_PARAMETER);
124
125	if (pVM)
126	{
127	/* Set revision dword at the beginning of the VMXON structure. */
128	(uint32_t )pvCpuPage = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hwaccm.s.vmx.msr.vmx_basic_info);
129	}
130
131	/** @todo we should unmap the two pages from the virtual address space in order to prevent accidental corruption.
132	* (which can have very bad consequences!!!)
133	*/
134
135	if (ASMGetCR4() & X86_CR4_VMXE)
136	return VERR_VMX_IN_VMX_ROOT_MODE;
137
138	ASMSetCR4(ASMGetCR4() \| X86_CR4_VMXE); /* Make sure the VMX instructions don't cause #UD faults. */
139
140	/*
141	* Enter VM root mode.
142	*/
143	int rc = VMXEnable(HCPhysCpuPage);
144	if (RT_FAILURE(rc))
145	{
146	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
147	return VERR_VMX_VMXON_FAILED;
148	}
149
150	/*
151	* Flush all VPIDs (in case we or any other hypervisor have been using VPIDs) so that
152	* we can avoid an explicit flush while using new VPIDs. We would still need to flush
153	* each time while reusing a VPID after hitting the MaxASID limit once.
154	*/
155	if ( pVM
156	&& pVM->hwaccm.s.vmx.fVPID
157	&& (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVVPID_CAPS_ALL_CONTEXTS))
158	{
159	hmR0VmxFlushVPID(pVM, NULL /* pvCpu /, VMX_FLUSH_VPID_ALL_CONTEXTS, 0 / GCPtr */);
160	pCpu->fFlushASIDBeforeUse = false;
161	}
162	else
163	pCpu->fFlushASIDBeforeUse = true;
164
165	/*
166	* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}.
167	*/
168	pCpu->cTLBFlushes++;
169
170	return VINF_SUCCESS;
171	}
172
173
174	/**
175	* Deactivates VT-x on the current CPU.
176	*
177	* @returns VBox status code.
178	* @param pCpu Pointer to the CPU info struct.
179	* @param pvCpuPage Pointer to the global CPU page.
180	* @param HCPhysCpuPage Physical address of the global CPU page.
181	*/
182	VMMR0DECL(int) VMXR0DisableCpu(PHMGLOBLCPUINFO pCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
183	{
184	AssertReturn(HCPhysCpuPage != 0 && HCPhysCpuPage != NIL_RTHCPHYS, VERR_INVALID_PARAMETER);
185	AssertReturn(pvCpuPage, VERR_INVALID_PARAMETER);
186	NOREF(pCpu);
187
188	/* If we're somehow not in VMX root mode, then we shouldn't dare leaving it. */
189	if (!(ASMGetCR4() & X86_CR4_VMXE))
190	return VERR_VMX_NOT_IN_VMX_ROOT_MODE;
191
192	/* Leave VMX Root Mode. */
193	VMXDisable();
194
195	/* And clear the X86_CR4_VMXE bit. */
196	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
197	return VINF_SUCCESS;
198	}
199
200
201	/**
202	* Does Ring-0 per VM VT-x initialization.
203	*
204	* @returns VBox status code.
205	* @param pVM Pointer to the VM.
206	*/
207	VMMR0DECL(int) VMXR0InitVM(PVM pVM)
208	{
209	int rc;
210
211	#ifdef LOG_ENABLED
212	SUPR0Printf("VMXR0InitVM %x\n", pVM);
213	#endif
214
215	pVM->hwaccm.s.vmx.pMemObjAPIC = NIL_RTR0MEMOBJ;
216
217	if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
218	{
219	/* Allocate one page for the APIC physical page (serves for filtering accesses). */
220	rc = RTR0MemObjAllocCont(&pVM->hwaccm.s.vmx.pMemObjAPIC, PAGE_SIZE, true /* executable R0 mapping */);
221	AssertRC(rc);
222	if (RT_FAILURE(rc))
223	return rc;
224
225	pVM->hwaccm.s.vmx.pAPIC = (uint8_t *)RTR0MemObjAddress(pVM->hwaccm.s.vmx.pMemObjAPIC);
226	pVM->hwaccm.s.vmx.pAPICPhys = RTR0MemObjGetPagePhysAddr(pVM->hwaccm.s.vmx.pMemObjAPIC, 0);
227	ASMMemZero32(pVM->hwaccm.s.vmx.pAPIC, PAGE_SIZE);
228	}
229	else
230	{
231	pVM->hwaccm.s.vmx.pMemObjAPIC = 0;
232	pVM->hwaccm.s.vmx.pAPIC = 0;
233	pVM->hwaccm.s.vmx.pAPICPhys = 0;
234	}
235
236	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
237	{
238	rc = RTR0MemObjAllocCont(&pVM->hwaccm.s.vmx.pMemObjScratch, PAGE_SIZE, true /* executable R0 mapping */);
239	AssertRC(rc);
240	if (RT_FAILURE(rc))
241	return rc;
242
243	pVM->hwaccm.s.vmx.pScratch = (uint8_t *)RTR0MemObjAddress(pVM->hwaccm.s.vmx.pMemObjScratch);
244	pVM->hwaccm.s.vmx.pScratchPhys = RTR0MemObjGetPagePhysAddr(pVM->hwaccm.s.vmx.pMemObjScratch, 0);
245
246	ASMMemZero32(pVM->hwaccm.s.vmx.pScratch, PAGE_SIZE);
247	strcpy((char *)pVM->hwaccm.s.vmx.pScratch, "SCRATCH Magic");
248	(uint64_t )(pVM->hwaccm.s.vmx.pScratch + 16) = UINT64_C(0xDEADBEEFDEADBEEF);
249	}
250	#endif
251
252	/* Allocate VMCSs for all guest CPUs. */
253	for (VMCPUID i = 0; i < pVM->cCpus; i++)
254	{
255	PVMCPU pVCpu = &pVM->aCpus[i];
256
257	pVCpu->hwaccm.s.vmx.hMemObjVMCS = NIL_RTR0MEMOBJ;
258
259	/* Allocate one page for the VM control structure (VMCS). */
260	rc = RTR0MemObjAllocCont(&pVCpu->hwaccm.s.vmx.hMemObjVMCS, PAGE_SIZE, true /* executable R0 mapping */);
261	AssertRC(rc);
262	if (RT_FAILURE(rc))
263	return rc;
264
265	pVCpu->hwaccm.s.vmx.pvVMCS = RTR0MemObjAddress(pVCpu->hwaccm.s.vmx.hMemObjVMCS);
266	pVCpu->hwaccm.s.vmx.HCPhysVMCS = RTR0MemObjGetPagePhysAddr(pVCpu->hwaccm.s.vmx.hMemObjVMCS, 0);
267	ASMMemZeroPage(pVCpu->hwaccm.s.vmx.pvVMCS);
268
269	pVCpu->hwaccm.s.vmx.cr0_mask = 0;
270	pVCpu->hwaccm.s.vmx.cr4_mask = 0;
271
272	/* Allocate one page for the virtual APIC page for TPR caching. */
273	rc = RTR0MemObjAllocCont(&pVCpu->hwaccm.s.vmx.hMemObjVAPIC, PAGE_SIZE, true /* executable R0 mapping */);
274	AssertRC(rc);
275	if (RT_FAILURE(rc))
276	return rc;
277
278	pVCpu->hwaccm.s.vmx.pbVAPIC = (uint8_t *)RTR0MemObjAddress(pVCpu->hwaccm.s.vmx.hMemObjVAPIC);
279	pVCpu->hwaccm.s.vmx.HCPhysVAPIC = RTR0MemObjGetPagePhysAddr(pVCpu->hwaccm.s.vmx.hMemObjVAPIC, 0);
280	ASMMemZeroPage(pVCpu->hwaccm.s.vmx.pbVAPIC);
281
282	/* Allocate the MSR bitmap if this feature is supported. */
283	if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
284	{
285	rc = RTR0MemObjAllocCont(&pVCpu->hwaccm.s.vmx.pMemObjMSRBitmap, PAGE_SIZE, true /* executable R0 mapping */);
286	AssertRC(rc);
287	if (RT_FAILURE(rc))
288	return rc;
289
290	pVCpu->hwaccm.s.vmx.pMSRBitmap = (uint8_t *)RTR0MemObjAddress(pVCpu->hwaccm.s.vmx.pMemObjMSRBitmap);
291	pVCpu->hwaccm.s.vmx.pMSRBitmapPhys = RTR0MemObjGetPagePhysAddr(pVCpu->hwaccm.s.vmx.pMemObjMSRBitmap, 0);
292	memset(pVCpu->hwaccm.s.vmx.pMSRBitmap, 0xff, PAGE_SIZE);
293	}
294
295	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
296	/* Allocate one page for the guest MSR load area (for preloading guest MSRs during the world switch). */
297	rc = RTR0MemObjAllocCont(&pVCpu->hwaccm.s.vmx.pMemObjGuestMSR, PAGE_SIZE, true /* executable R0 mapping */);
298	AssertRC(rc);
299	if (RT_FAILURE(rc))
300	return rc;
301
302	pVCpu->hwaccm.s.vmx.pGuestMSR = (uint8_t *)RTR0MemObjAddress(pVCpu->hwaccm.s.vmx.pMemObjGuestMSR);
303	pVCpu->hwaccm.s.vmx.pGuestMSRPhys = RTR0MemObjGetPagePhysAddr(pVCpu->hwaccm.s.vmx.pMemObjGuestMSR, 0);
304	memset(pVCpu->hwaccm.s.vmx.pGuestMSR, 0, PAGE_SIZE);
305
306	/* Allocate one page for the host MSR load area (for restoring host MSRs after the world switch back). */
307	rc = RTR0MemObjAllocCont(&pVCpu->hwaccm.s.vmx.pMemObjHostMSR, PAGE_SIZE, true /* executable R0 mapping */);
308	AssertRC(rc);
309	if (RT_FAILURE(rc))
310	return rc;
311
312	pVCpu->hwaccm.s.vmx.pHostMSR = (uint8_t *)RTR0MemObjAddress(pVCpu->hwaccm.s.vmx.pMemObjHostMSR);
313	pVCpu->hwaccm.s.vmx.pHostMSRPhys = RTR0MemObjGetPagePhysAddr(pVCpu->hwaccm.s.vmx.pMemObjHostMSR, 0);
314	memset(pVCpu->hwaccm.s.vmx.pHostMSR, 0, PAGE_SIZE);
315	#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
316
317	/* Current guest paging mode. */
318	pVCpu->hwaccm.s.vmx.enmLastSeenGuestMode = PGMMODE_REAL;
319
320	#ifdef LOG_ENABLED
321	SUPR0Printf("VMXR0InitVM %x VMCS=%x (%x)\n", pVM, pVCpu->hwaccm.s.vmx.pvVMCS, (uint32_t)pVCpu->hwaccm.s.vmx.HCPhysVMCS);
322	#endif
323	}
324
325	return VINF_SUCCESS;
326	}
327
328
329	/**
330	* Does Ring-0 per VM VT-x termination.
331	*
332	* @returns VBox status code.
333	* @param pVM Pointer to the VM.
334	*/
335	VMMR0DECL(int) VMXR0TermVM(PVM pVM)
336	{
337	for (VMCPUID i = 0; i < pVM->cCpus; i++)
338	{
339	PVMCPU pVCpu = &pVM->aCpus[i];
340
341	if (pVCpu->hwaccm.s.vmx.hMemObjVMCS != NIL_RTR0MEMOBJ)
342	{
343	RTR0MemObjFree(pVCpu->hwaccm.s.vmx.hMemObjVMCS, false);
344	pVCpu->hwaccm.s.vmx.hMemObjVMCS = NIL_RTR0MEMOBJ;
345	pVCpu->hwaccm.s.vmx.pvVMCS = 0;
346	pVCpu->hwaccm.s.vmx.HCPhysVMCS = 0;
347	}
348	if (pVCpu->hwaccm.s.vmx.hMemObjVAPIC != NIL_RTR0MEMOBJ)
349	{
350	RTR0MemObjFree(pVCpu->hwaccm.s.vmx.hMemObjVAPIC, false);
351	pVCpu->hwaccm.s.vmx.hMemObjVAPIC = NIL_RTR0MEMOBJ;
352	pVCpu->hwaccm.s.vmx.pbVAPIC = 0;
353	pVCpu->hwaccm.s.vmx.HCPhysVAPIC = 0;
354	}
355	if (pVCpu->hwaccm.s.vmx.pMemObjMSRBitmap != NIL_RTR0MEMOBJ)
356	{
357	RTR0MemObjFree(pVCpu->hwaccm.s.vmx.pMemObjMSRBitmap, false);
358	pVCpu->hwaccm.s.vmx.pMemObjMSRBitmap = NIL_RTR0MEMOBJ;
359	pVCpu->hwaccm.s.vmx.pMSRBitmap = 0;
360	pVCpu->hwaccm.s.vmx.pMSRBitmapPhys = 0;
361	}
362	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
363	if (pVCpu->hwaccm.s.vmx.pMemObjHostMSR != NIL_RTR0MEMOBJ)
364	{
365	RTR0MemObjFree(pVCpu->hwaccm.s.vmx.pMemObjHostMSR, false);
366	pVCpu->hwaccm.s.vmx.pMemObjHostMSR = NIL_RTR0MEMOBJ;
367	pVCpu->hwaccm.s.vmx.pHostMSR = 0;
368	pVCpu->hwaccm.s.vmx.pHostMSRPhys = 0;
369	}
370	if (pVCpu->hwaccm.s.vmx.pMemObjGuestMSR != NIL_RTR0MEMOBJ)
371	{
372	RTR0MemObjFree(pVCpu->hwaccm.s.vmx.pMemObjGuestMSR, false);
373	pVCpu->hwaccm.s.vmx.pMemObjGuestMSR = NIL_RTR0MEMOBJ;
374	pVCpu->hwaccm.s.vmx.pGuestMSR = 0;
375	pVCpu->hwaccm.s.vmx.pGuestMSRPhys = 0;
376	}
377	#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
378	}
379	if (pVM->hwaccm.s.vmx.pMemObjAPIC != NIL_RTR0MEMOBJ)
380	{
381	RTR0MemObjFree(pVM->hwaccm.s.vmx.pMemObjAPIC, false);
382	pVM->hwaccm.s.vmx.pMemObjAPIC = NIL_RTR0MEMOBJ;
383	pVM->hwaccm.s.vmx.pAPIC = 0;
384	pVM->hwaccm.s.vmx.pAPICPhys = 0;
385	}
386	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
387	if (pVM->hwaccm.s.vmx.pMemObjScratch != NIL_RTR0MEMOBJ)
388	{
389	ASMMemZero32(pVM->hwaccm.s.vmx.pScratch, PAGE_SIZE);
390	RTR0MemObjFree(pVM->hwaccm.s.vmx.pMemObjScratch, false);
391	pVM->hwaccm.s.vmx.pMemObjScratch = NIL_RTR0MEMOBJ;
392	pVM->hwaccm.s.vmx.pScratch = 0;
393	pVM->hwaccm.s.vmx.pScratchPhys = 0;
394	}
395	#endif
396	return VINF_SUCCESS;
397	}
398
399
400	/**
401	* Sets up VT-x for the specified VM.
402	*
403	* @returns VBox status code.
404	* @param pVM Pointer to the VM.
405	*/
406	VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
407	{
408	int rc = VINF_SUCCESS;
409	uint32_t val;
410
411	AssertReturn(pVM, VERR_INVALID_PARAMETER);
412
413	/* Initialize these always, see hwaccmR3InitFinalizeR0().*/
414	pVM->hwaccm.s.vmx.enmFlushEPT = VMX_FLUSH_EPT_NONE;
415	pVM->hwaccm.s.vmx.enmFlushVPID = VMX_FLUSH_VPID_NONE;
416
417	/* Determine optimal flush type for EPT. */
418	if (pVM->hwaccm.s.fNestedPaging)
419	{
420	if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVEPT_CAPS_SINGLE_CONTEXT)
421	pVM->hwaccm.s.vmx.enmFlushEPT = VMX_FLUSH_EPT_SINGLE_CONTEXT;
422	else if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVEPT_CAPS_ALL_CONTEXTS)
423	pVM->hwaccm.s.vmx.enmFlushEPT = VMX_FLUSH_EPT_ALL_CONTEXTS;
424	else
425	{
426	/*
427	* Should never really happen. EPT is supported but no suitable flush types supported.
428	* We cannot ignore EPT at this point as we've already setup Unrestricted Guest execution.
429	*/
430	pVM->hwaccm.s.vmx.enmFlushEPT = VMX_FLUSH_EPT_NOT_SUPPORTED;
431	return VERR_VMX_GENERIC;
432	}
433	}
434
435	/* Determine optimal flush type for VPID. */
436	if (pVM->hwaccm.s.vmx.fVPID)
437	{
438	if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVVPID_CAPS_SINGLE_CONTEXT)
439	pVM->hwaccm.s.vmx.enmFlushVPID = VMX_FLUSH_VPID_SINGLE_CONTEXT;
440	else if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVVPID_CAPS_ALL_CONTEXTS)
441	pVM->hwaccm.s.vmx.enmFlushVPID = VMX_FLUSH_VPID_ALL_CONTEXTS;
442	else
443	{
444	/*
445	* Neither SINGLE nor ALL context flush types for VPID supported by the CPU.
446	* We do not handle other flush type combinations, ignore VPID capabilities.
447	*/
448	if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVVPID_CAPS_INDIV_ADDR)
449	Log(("VMXR0SetupVM: Only VMX_FLUSH_VPID_INDIV_ADDR supported. Ignoring VPID.\n"));
450	if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVVPID_CAPS_SINGLE_CONTEXT_RETAIN_GLOBALS)
451	Log(("VMXR0SetupVM: Only VMX_FLUSH_VPID_SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
452	pVM->hwaccm.s.vmx.enmFlushVPID = VMX_FLUSH_VPID_NOT_SUPPORTED;
453	pVM->hwaccm.s.vmx.fVPID = false;
454	}
455	}
456
457	for (VMCPUID i = 0; i < pVM->cCpus; i++)
458	{
459	PVMCPU pVCpu = &pVM->aCpus[i];
460
461	AssertPtr(pVCpu->hwaccm.s.vmx.pvVMCS);
462
463	/* Set revision dword at the beginning of the VMCS structure. */
464	(uint32_t )pVCpu->hwaccm.s.vmx.pvVMCS = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hwaccm.s.vmx.msr.vmx_basic_info);
465
466	/*
467	* Clear and activate the VMCS.
468	*/
469	Log(("HCPhysVMCS = %RHp\n", pVCpu->hwaccm.s.vmx.HCPhysVMCS));
470	rc = VMXClearVMCS(pVCpu->hwaccm.s.vmx.HCPhysVMCS);
471	if (RT_FAILURE(rc))
472	goto vmx_end;
473
474	rc = VMXActivateVMCS(pVCpu->hwaccm.s.vmx.HCPhysVMCS);
475	if (RT_FAILURE(rc))
476	goto vmx_end;
477
478	/*
479	* VMX_VMCS_CTRL_PIN_EXEC_CONTROLS
480	* Set required bits to one and zero according to the MSR capabilities.
481	*/
482	val = pVM->hwaccm.s.vmx.msr.vmx_pin_ctls.n.disallowed0;
483	val \|= VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_EXT_INT_EXIT /* External interrupts */
484	\| VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_NMI_EXIT; /* Non-maskable interrupts */
485
486	/*
487	* Enable the VMX preemption timer.
488	*/
489	if (pVM->hwaccm.s.vmx.fUsePreemptTimer)
490	val \|= VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_PREEMPT_TIMER;
491	val &= pVM->hwaccm.s.vmx.msr.vmx_pin_ctls.n.allowed1;
492
493	rc = VMXWriteVMCS(VMX_VMCS_CTRL_PIN_EXEC_CONTROLS, val);
494	AssertRC(rc);
495
496	/*
497	* VMX_VMCS_CTRL_PROC_EXEC_CONTROLS
498	* Set required bits to one and zero according to the MSR capabilities.
499	*/
500	val = pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.disallowed0;
501	/* Program which event cause VM-exits and which features we want to use. */
502	val \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_HLT_EXIT
503	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_TSC_OFFSET
504	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT
505	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_UNCOND_IO_EXIT
506	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDPMC_EXIT
507	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_EXIT
508	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MWAIT_EXIT; /* don't execute mwait or else we'll idle inside
509	the guest (host thinks the cpu load is high) */
510
511	/* Without nested paging we should intercept invlpg and cr3 mov instructions. */
512	if (!pVM->hwaccm.s.fNestedPaging)
513	{
514	val \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INVLPG_EXIT
515	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
516	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT;
517	}
518
519	/*
520	* VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MWAIT_EXIT might cause a vmlaunch
521	* failure with an invalid control fields error. (combined with some other exit reasons)
522	*/
523	if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
524	{
525	/* CR8 reads from the APIC shadow page; writes cause an exit is they lower the TPR below the threshold */
526	val \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW;
527	Assert(pVM->hwaccm.s.vmx.pAPIC);
528	}
529	else
530	/* Exit on CR8 reads & writes in case the TPR shadow feature isn't present. */
531	val \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR8_STORE_EXIT \| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR8_LOAD_EXIT;
532
533	if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
534	{
535	Assert(pVCpu->hwaccm.s.vmx.pMSRBitmapPhys);
536	val \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS;
537	}
538
539	/* We will use the secondary control if it's present. */
540	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
541
542	/* Mask away the bits that the CPU doesn't support */
543	/** @todo make sure they don't conflict with the above requirements. */
544	val &= pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1;
545	pVCpu->hwaccm.s.vmx.proc_ctls = val;
546
547	rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, val);
548	AssertRC(rc);
549
550	if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
551	{
552	/*
553	* VMX_VMCS_CTRL_PROC_EXEC_CONTROLS2
554	* Set required bits to one and zero according to the MSR capabilities.
555	*/
556	val = pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2.n.disallowed0;
557	val \|= VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT;
558
559	if (pVM->hwaccm.s.fNestedPaging)
560	val \|= VMX_VMCS_CTRL_PROC_EXEC2_EPT;
561
562	if (pVM->hwaccm.s.vmx.fVPID)
563	val \|= VMX_VMCS_CTRL_PROC_EXEC2_VPID;
564
565	if (pVM->hwaccm.s.fHasIoApic)
566	val \|= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC;
567
568	if (pVM->hwaccm.s.vmx.fUnrestrictedGuest)
569	val \|= VMX_VMCS_CTRL_PROC_EXEC2_REAL_MODE;
570
571	/* Mask away the bits that the CPU doesn't support */
572	/** @todo make sure they don't conflict with the above requirements. */
573	val &= pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2.n.allowed1;
574	pVCpu->hwaccm.s.vmx.proc_ctls2 = val;
575	rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS2, val);
576	AssertRC(rc);
577	}
578
579	/*
580	* VMX_VMCS_CTRL_CR3_TARGET_COUNT
581	* Set required bits to one and zero according to the MSR capabilities.
582	*/
583	rc = VMXWriteVMCS(VMX_VMCS_CTRL_CR3_TARGET_COUNT, 0);
584	AssertRC(rc);
585
586	/*
587	* Forward all exception except #NM & #PF to the guest.
588	* We always need to check pagefaults since our shadow page table can be out of sync.
589	* And we always lazily sync the FPU & XMM state. .
590	*/
591
592	/** @todo Possible optimization:
593	* Keep the FPU and XMM state current in the EM thread. That way there's no need to
594	* lazily sync anything, but the downside is that we can't use the FPU stack or XMM
595	* registers ourselves of course.
596	*
597	* Note: only possible if the current state is actually ours (X86_CR0_TS flag)
598	*/
599
600	/*
601	* Don't filter page faults, all of them should cause a world switch.
602	*/
603	rc = VMXWriteVMCS(VMX_VMCS_CTRL_PAGEFAULT_ERROR_MASK, 0);
604	AssertRC(rc);
605	rc = VMXWriteVMCS(VMX_VMCS_CTRL_PAGEFAULT_ERROR_MATCH, 0);
606	AssertRC(rc);
607
608	rc = VMXWriteVMCS64(VMX_VMCS_CTRL_TSC_OFFSET_FULL, 0);
609	AssertRC(rc);
610	rc = VMXWriteVMCS64(VMX_VMCS_CTRL_IO_BITMAP_A_FULL, 0);
611	AssertRC(rc);
612	rc = VMXWriteVMCS64(VMX_VMCS_CTRL_IO_BITMAP_B_FULL, 0);
613	AssertRC(rc);
614
615	/*
616	* Set the MSR bitmap address.
617	*/
618	if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
619	{
620	Assert(pVCpu->hwaccm.s.vmx.pMSRBitmapPhys);
621
622	rc = VMXWriteVMCS64(VMX_VMCS_CTRL_MSR_BITMAP_FULL, pVCpu->hwaccm.s.vmx.pMSRBitmapPhys);
623	AssertRC(rc);
624
625	/* Allow the guest to directly modify these MSRs; they are restored and saved automatically. */
626	hmR0VmxSetMSRPermission(pVCpu, MSR_IA32_SYSENTER_CS, true, true);
627	hmR0VmxSetMSRPermission(pVCpu, MSR_IA32_SYSENTER_ESP, true, true);
628	hmR0VmxSetMSRPermission(pVCpu, MSR_IA32_SYSENTER_EIP, true, true);
629	hmR0VmxSetMSRPermission(pVCpu, MSR_K8_LSTAR, true, true);
630	hmR0VmxSetMSRPermission(pVCpu, MSR_K6_STAR, true, true);
631	hmR0VmxSetMSRPermission(pVCpu, MSR_K8_SF_MASK, true, true);
632	hmR0VmxSetMSRPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, true, true);
633	hmR0VmxSetMSRPermission(pVCpu, MSR_K8_GS_BASE, true, true);
634	hmR0VmxSetMSRPermission(pVCpu, MSR_K8_FS_BASE, true, true);
635	}
636
637	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
638	/*
639	* Set the guest & host MSR load/store physical addresses.
640	*/
641	Assert(pVCpu->hwaccm.s.vmx.pGuestMSRPhys);
642	rc = VMXWriteVMCS64(VMX_VMCS_CTRL_VMENTRY_MSR_LOAD_FULL, pVCpu->hwaccm.s.vmx.pGuestMSRPhys);
643	AssertRC(rc);
644	rc = VMXWriteVMCS64(VMX_VMCS_CTRL_VMEXIT_MSR_STORE_FULL, pVCpu->hwaccm.s.vmx.pGuestMSRPhys);
645	AssertRC(rc);
646
647	Assert(pVCpu->hwaccm.s.vmx.pHostMSRPhys);
648	rc = VMXWriteVMCS64(VMX_VMCS_CTRL_VMEXIT_MSR_LOAD_FULL, pVCpu->hwaccm.s.vmx.pHostMSRPhys);
649	AssertRC(rc);
650	#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
651
652	rc = VMXWriteVMCS(VMX_VMCS_CTRL_ENTRY_MSR_LOAD_COUNT, 0);
653	AssertRC(rc);
654	rc = VMXWriteVMCS(VMX_VMCS_CTRL_EXIT_MSR_STORE_COUNT, 0);
655	AssertRC(rc);
656
657	if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
658	{
659	Assert(pVM->hwaccm.s.vmx.pMemObjAPIC);
660	/* Optional */
661	rc = VMXWriteVMCS(VMX_VMCS_CTRL_TPR_THRESHOLD, 0);
662	rc \|= VMXWriteVMCS64(VMX_VMCS_CTRL_VAPIC_PAGEADDR_FULL, pVCpu->hwaccm.s.vmx.HCPhysVAPIC);
663
664	if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
665	rc \|= VMXWriteVMCS64(VMX_VMCS_CTRL_APIC_ACCESSADDR_FULL, pVM->hwaccm.s.vmx.pAPICPhys);
666
667	AssertRC(rc);
668	}
669
670	/* Set link pointer to -1. Not currently used. */
671	rc = VMXWriteVMCS64(VMX_VMCS_GUEST_LINK_PTR_FULL, 0xFFFFFFFFFFFFFFFFULL);
672	AssertRC(rc);
673
674	/*
675	* Clear VMCS, marking it inactive. Clear implementation specific data and writing back
676	* VMCS data back to memory.
677	*/
678	rc = VMXClearVMCS(pVCpu->hwaccm.s.vmx.HCPhysVMCS);
679	AssertRC(rc);
680
681	/*
682	* Configure the VMCS read cache.
683	*/
684	PVMCSCACHE pCache = &pVCpu->hwaccm.s.vmx.VMCSCache;
685
686	VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_RIP);
687	VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_RSP);
688	VMXSetupCachedReadVMCS(pCache, VMX_VMCS_GUEST_RFLAGS);
689	VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE);
690	VMXSetupCachedReadVMCS(pCache, VMX_VMCS_CTRL_CR0_READ_SHADOW);
691	VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_CR0);
692	VMXSetupCachedReadVMCS(pCache, VMX_VMCS_CTRL_CR4_READ_SHADOW);
693	VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_CR4);
694	VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_DR7);
695	VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_GUEST_SYSENTER_CS);
696	VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_SYSENTER_EIP);
697	VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_SYSENTER_ESP);
698	VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_GUEST_GDTR_LIMIT);
699	VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_GDTR_BASE);
700	VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_GUEST_IDTR_LIMIT);
701	VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_IDTR_BASE);
702
703	VMX_SETUP_SELREG(ES, pCache);
704	VMX_SETUP_SELREG(SS, pCache);
705	VMX_SETUP_SELREG(CS, pCache);
706	VMX_SETUP_SELREG(DS, pCache);
707	VMX_SETUP_SELREG(FS, pCache);
708	VMX_SETUP_SELREG(GS, pCache);
709	VMX_SETUP_SELREG(LDTR, pCache);
710	VMX_SETUP_SELREG(TR, pCache);
711
712	/*
713	* Status code VMCS reads.
714	*/
715	VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_EXIT_REASON);
716	VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_VM_INSTR_ERROR);
717	VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_EXIT_INSTR_LENGTH);
718	VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_EXIT_INTERRUPTION_ERRCODE);
719	VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO);
720	VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_EXIT_INSTR_INFO);
721	VMXSetupCachedReadVMCS(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
722	VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_IDT_INFO);
723	VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_IDT_ERRCODE);
724
725	if (pVM->hwaccm.s.fNestedPaging)
726	{
727	VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_CR3);
728	VMXSetupCachedReadVMCS(pCache, VMX_VMCS_EXIT_PHYS_ADDR_FULL);
729	pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
730	}
731	else
732	pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
733	} /* for each VMCPU */
734
735	/*
736	* Setup the right TLB function based on CPU capabilities.
737	*/
738	if (pVM->hwaccm.s.fNestedPaging && pVM->hwaccm.s.vmx.fVPID)
739	pVM->hwaccm.s.vmx.pfnSetupTaggedTLB = hmR0VmxSetupTLBBoth;
740	else if (pVM->hwaccm.s.fNestedPaging)
741	pVM->hwaccm.s.vmx.pfnSetupTaggedTLB = hmR0VmxSetupTLBEPT;
742	else if (pVM->hwaccm.s.vmx.fVPID)
743	pVM->hwaccm.s.vmx.pfnSetupTaggedTLB = hmR0VmxSetupTLBVPID;
744	else
745	pVM->hwaccm.s.vmx.pfnSetupTaggedTLB = hmR0VmxSetupTLBDummy;
746
747	vmx_end:
748	hmR0VmxCheckError(pVM, &pVM->aCpus[0], rc);
749	return rc;
750	}
751
752
753	/**
754	* Sets the permission bits for the specified MSR.
755	*
756	* @param pVCpu Pointer to the VMCPU.
757	* @param ulMSR The MSR value.
758	* @param fRead Whether reading is allowed.
759	* @param fWrite Whether writing is allowed.
760	*/
761	static void hmR0VmxSetMSRPermission(PVMCPU pVCpu, unsigned ulMSR, bool fRead, bool fWrite)
762	{
763	unsigned ulBit;
764	uint8_t pMSRBitmap = (uint8_t )pVCpu->hwaccm.s.vmx.pMSRBitmap;
765
766	/*
767	* Layout:
768	* 0x000 - 0x3ff - Low MSR read bits
769	* 0x400 - 0x7ff - High MSR read bits
770	* 0x800 - 0xbff - Low MSR write bits
771	* 0xc00 - 0xfff - High MSR write bits
772	*/
773	if (ulMSR <= 0x00001FFF)
774	{
775	/* Pentium-compatible MSRs */
776	ulBit = ulMSR;
777	}
778	else if ( ulMSR >= 0xC0000000
779	&& ulMSR <= 0xC0001FFF)
780	{
781	/* AMD Sixth Generation x86 Processor MSRs */
782	ulBit = (ulMSR - 0xC0000000);
783	pMSRBitmap += 0x400;
784	}
785	else
786	{
787	AssertFailed();
788	return;
789	}
790
791	Assert(ulBit <= 0x1fff);
792	if (fRead)
793	ASMBitClear(pMSRBitmap, ulBit);
794	else
795	ASMBitSet(pMSRBitmap, ulBit);
796
797	if (fWrite)
798	ASMBitClear(pMSRBitmap + 0x800, ulBit);
799	else
800	ASMBitSet(pMSRBitmap + 0x800, ulBit);
801	}
802
803
804	/**
805	* Injects an event (trap or external interrupt).
806	*
807	* @returns VBox status code. Note that it may return VINF_EM_RESET to
808	* indicate a triple fault when injecting X86_XCPT_DF.
809	*
810	* @param pVM Pointer to the VM.
811	* @param pVCpu Pointer to the VMCPU.
812	* @param pCtx Pointer to the guest CPU Context.
813	* @param intInfo VMX interrupt info.
814	* @param cbInstr Opcode length of faulting instruction.
815	* @param errCode Error code (optional).
816	*/
817	static int hmR0VmxInjectEvent(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, uint32_t intInfo, uint32_t cbInstr, uint32_t errCode)
818	{
819	int rc;
820	uint32_t iGate = VMX_EXIT_INTERRUPTION_INFO_VECTOR(intInfo);
821
822	#ifdef VBOX_WITH_STATISTICS
823	STAM_COUNTER_INC(&pVCpu->hwaccm.s.paStatInjectedIrqsR0[iGate & MASK_INJECT_IRQ_STAT]);
824	#endif
825
826	#ifdef VBOX_STRICT
827	if (iGate == 0xE)
828	{
829	LogFlow(("hmR0VmxInjectEvent: Injecting interrupt %d at %RGv error code=%08x CR2=%RGv intInfo=%08x\n", iGate,
830	(RTGCPTR)pCtx->rip, errCode, pCtx->cr2, intInfo));
831	}
832	else if (iGate < 0x20)
833	{
834	LogFlow(("hmR0VmxInjectEvent: Injecting interrupt %d at %RGv error code=%08x\n", iGate, (RTGCPTR)pCtx->rip,
835	errCode));
836	}
837	else
838	{
839	LogFlow(("INJ-EI: %x at %RGv\n", iGate, (RTGCPTR)pCtx->rip));
840	Assert( VMX_EXIT_INTERRUPTION_INFO_TYPE(intInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW
841	\|\| !VMCPU_FF_ISSET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
842	Assert( VMX_EXIT_INTERRUPTION_INFO_TYPE(intInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW
843	\|\| pCtx->eflags.u32 & X86_EFL_IF);
844	}
845	#endif
846
847	if ( CPUMIsGuestInRealModeEx(pCtx)
848	&& pVM->hwaccm.s.vmx.pRealModeTSS)
849	{
850	RTGCPHYS GCPhysHandler;
851	uint16_t offset, ip;
852	RTSEL sel;
853
854	/*
855	* Injecting events doesn't work right with real mode emulation.
856	* (#GP if we try to inject external hardware interrupts)
857	* Inject the interrupt or trap directly instead.
858	*
859	* ASSUMES no access handlers for the bits we read or write below (should be safe).
860	*/
861	Log(("Manual interrupt/trap '%x' inject (real mode)\n", iGate));
862
863	/*
864	* Check if the interrupt handler is present.
865	*/
866	if (iGate * 4 + 3 > pCtx->idtr.cbIdt)
867	{
868	Log(("IDT cbIdt violation\n"));
869	if (iGate != X86_XCPT_DF)
870	{
871	uint32_t intInfo2;
872
873	intInfo2 = (iGate == X86_XCPT_GP) ? (uint32_t)X86_XCPT_DF : iGate;
874	intInfo2 \|= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
875	intInfo2 \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
876	intInfo2 \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HWEXCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
877
878	return hmR0VmxInjectEvent(pVM, pVCpu, pCtx, intInfo2, 0, 0 /* no error code according to the Intel docs */);
879	}
880	Log(("Triple fault -> reset the VM!\n"));
881	return VINF_EM_RESET;
882	}
883	if ( VMX_EXIT_INTERRUPTION_INFO_TYPE(intInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW
884	\|\| iGate == 3 /* Both #BP and #OF point to the instruction after. */
885	\|\| iGate == 4)
886	{
887	ip = pCtx->ip + cbInstr;
888	}
889	else
890	ip = pCtx->ip;
891
892	/*
893	* Read the selector:offset pair of the interrupt handler.
894	*/
895	GCPhysHandler = (RTGCPHYS)pCtx->idtr.pIdt + iGate * 4;
896	rc = PGMPhysSimpleReadGCPhys(pVM, &offset, GCPhysHandler, sizeof(offset)); AssertRC(rc);
897	rc = PGMPhysSimpleReadGCPhys(pVM, &sel, GCPhysHandler + 2, sizeof(sel)); AssertRC(rc);
898
899	LogFlow(("IDT handler %04X:%04X\n", sel, offset));
900
901	/*
902	* Construct the stack frame.
903	*/
904	/** @todo Check stack limit. */
905	pCtx->sp -= 2;
906	LogFlow(("ss:sp %04X:%04X eflags=%x\n", pCtx->ss, pCtx->sp, pCtx->eflags.u));
907	rc = PGMPhysSimpleWriteGCPhys(pVM, pCtx->ssHid.u64Base + pCtx->sp, &pCtx->eflags, sizeof(uint16_t)); AssertRC(rc);
908	pCtx->sp -= 2;
909	LogFlow(("ss:sp %04X:%04X cs=%x\n", pCtx->ss, pCtx->sp, pCtx->cs));
910	rc = PGMPhysSimpleWriteGCPhys(pVM, pCtx->ssHid.u64Base + pCtx->sp, &pCtx->cs, sizeof(uint16_t)); AssertRC(rc);
911	pCtx->sp -= 2;
912	LogFlow(("ss:sp %04X:%04X ip=%x\n", pCtx->ss, pCtx->sp, ip));
913	rc = PGMPhysSimpleWriteGCPhys(pVM, pCtx->ssHid.u64Base + pCtx->sp, &ip, sizeof(ip)); AssertRC(rc);
914
915	/*
916	* Update the CPU state for executing the handler.
917	*/
918	pCtx->rip = offset;
919	pCtx->cs = sel;
920	pCtx->csHid.u64Base = sel << 4;
921	pCtx->eflags.u &= ~(X86_EFL_IF \| X86_EFL_TF \| X86_EFL_RF \| X86_EFL_AC);
922
923	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_GUEST_SEGMENT_REGS;
924	return VINF_SUCCESS;
925	}
926
927	/*
928	* Set event injection state.
929	*/
930	rc = VMXWriteVMCS(VMX_VMCS_CTRL_ENTRY_IRQ_INFO, intInfo \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT));
931	rc \|= VMXWriteVMCS(VMX_VMCS_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
932	rc \|= VMXWriteVMCS(VMX_VMCS_CTRL_ENTRY_EXCEPTION_ERRCODE, errCode);
933
934	AssertRC(rc);
935	return rc;
936	}
937
938
939	/**
940	* Checks for pending guest interrupts and injects them.
941	*
942	* @returns VBox status code.
943	* @param pVM Pointer to the VM.
944	* @param pVCpu Pointer to the VMCPU.
945	* @param pCtx Pointer to the guest CPU context.
946	*/
947	static int hmR0VmxCheckPendingInterrupt(PVM pVM, PVMCPU pVCpu, CPUMCTX *pCtx)
948	{
949	int rc;
950
951	/*
952	* Dispatch any pending interrupts (injected before, but a VM exit occurred prematurely).
953	*/
954	if (pVCpu->hwaccm.s.Event.fPending)
955	{
956	Log(("CPU%d: Reinjecting event %RX64 %08x at %RGv cr2=%RX64\n", pVCpu->idCpu, pVCpu->hwaccm.s.Event.intInfo,
957	pVCpu->hwaccm.s.Event.errCode, (RTGCPTR)pCtx->rip, pCtx->cr2));
958	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatIntReinject);
959	rc = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, pVCpu->hwaccm.s.Event.intInfo, 0, pVCpu->hwaccm.s.Event.errCode);
960	AssertRC(rc);
961
962	pVCpu->hwaccm.s.Event.fPending = false;
963	return VINF_SUCCESS;
964	}
965
966	/*
967	* If an active trap is already pending, we must forward it first!
968	*/
969	if (!TRPMHasTrap(pVCpu))
970	{
971	if (VMCPU_FF_TESTANDCLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI))
972	{
973	RTGCUINTPTR intInfo;
974
975	Log(("CPU%d: injecting #NMI\n", pVCpu->idCpu));
976
977	intInfo = X86_XCPT_NMI;
978	intInfo \|= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
979	intInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
980
981	rc = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, intInfo, 0, 0);
982	AssertRC(rc);
983
984	return VINF_SUCCESS;
985	}
986
987	/** @todo SMI interrupts. */
988
989	/*
990	* When external interrupts are pending, we should exit the VM when IF is set.
991	*/
992	if (VMCPU_FF_ISPENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC\|VMCPU_FF_INTERRUPT_PIC)))
993	{
994	if (!(pCtx->eflags.u32 & X86_EFL_IF))
995	{
996	if (!(pVCpu->hwaccm.s.vmx.proc_ctls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_IRQ_WINDOW_EXIT))
997	{
998	LogFlow(("Enable irq window exit!\n"));
999	pVCpu->hwaccm.s.vmx.proc_ctls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_IRQ_WINDOW_EXIT;
1000	rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
1001	AssertRC(rc);
1002	}
1003	/* else nothing to do but wait */
1004	}
1005	else if (!VMCPU_FF_ISSET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
1006	{
1007	uint8_t u8Interrupt;
1008
1009	rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
1010	Log(("CPU%d: Dispatch interrupt: u8Interrupt=%x (%d) rc=%Rrc cs:rip=%04X:%RGv\n", pVCpu->idCpu,
1011	u8Interrupt, u8Interrupt, rc, pCtx->cs, (RTGCPTR)pCtx->rip));
1012	if (RT_SUCCESS(rc))
1013	{
1014	rc = TRPMAssertTrap(pVCpu, u8Interrupt, TRPM_HARDWARE_INT);
1015	AssertRC(rc);
1016	}
1017	else
1018	{
1019	/* Can only happen in rare cases where a pending interrupt is cleared behind our back */
1020	Assert(!VMCPU_FF_ISPENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC\|VMCPU_FF_INTERRUPT_PIC)));
1021	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatSwitchGuestIrq);
1022	/* Just continue */
1023	}
1024	}
1025	else
1026	Log(("Pending interrupt blocked at %RGv by VM_FF_INHIBIT_INTERRUPTS!!\n", (RTGCPTR)pCtx->rip));
1027	}
1028	}
1029
1030	#ifdef VBOX_STRICT
1031	if (TRPMHasTrap(pVCpu))
1032	{
1033	uint8_t u8Vector;
1034	rc = TRPMQueryTrapAll(pVCpu, &u8Vector, 0, 0, 0);
1035	AssertRC(rc);
1036	}
1037	#endif
1038
1039	if ( (pCtx->eflags.u32 & X86_EFL_IF)
1040	&& (!VMCPU_FF_ISSET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
1041	&& TRPMHasTrap(pVCpu)
1042	)
1043	{
1044	uint8_t u8Vector;
1045	TRPMEVENT enmType;
1046	RTGCUINTPTR intInfo;
1047	RTGCUINT errCode;
1048
1049	/*
1050	* If a new event is pending, dispatch it now.
1051	*/
1052	rc = TRPMQueryTrapAll(pVCpu, &u8Vector, &enmType, &errCode, 0);
1053	AssertRC(rc);
1054	Assert(pCtx->eflags.Bits.u1IF == 1 \|\| enmType == TRPM_TRAP);
1055	Assert(enmType != TRPM_SOFTWARE_INT);
1056
1057	/*
1058	* Clear the pending trap.
1059	*/
1060	rc = TRPMResetTrap(pVCpu);
1061	AssertRC(rc);
1062
1063	intInfo = u8Vector;
1064	intInfo \|= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
1065
1066	if (enmType == TRPM_TRAP)
1067	{
1068	switch (u8Vector)
1069	{
1070	case X86_XCPT_DF:
1071	case X86_XCPT_TS:
1072	case X86_XCPT_NP:
1073	case X86_XCPT_SS:
1074	case X86_XCPT_GP:
1075	case X86_XCPT_PF:
1076	case X86_XCPT_AC:
1077	{
1078	/* Valid error codes. */
1079	intInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
1080	break;
1081	}
1082
1083	default:
1084	break;
1085	}
1086
1087	if ( u8Vector == X86_XCPT_BP
1088	\|\| u8Vector == X86_XCPT_OF)
1089	{
1090	intInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SWEXCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
1091	}
1092	else
1093	intInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HWEXCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
1094	}
1095	else
1096	intInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
1097
1098	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatIntInject);
1099	rc = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, intInfo, 0, errCode);
1100	AssertRC(rc);
1101	} /* if (interrupts can be dispatched) */
1102
1103	return VINF_SUCCESS;
1104	}
1105
1106
1107	/**
1108	* Save the host state into the VMCS.
1109	*
1110	* @returns VBox status code.
1111	* @param pVM Pointer to the VM.
1112	* @param pVCpu Pointer to the VMCPU.
1113	*/
1114	VMMR0DECL(int) VMXR0SaveHostState(PVM pVM, PVMCPU pVCpu)
1115	{
1116	int rc = VINF_SUCCESS;
1117	NOREF(pVM);
1118
1119	/*
1120	* Host CPU Context.
1121	*/
1122	if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_HOST_CONTEXT)
1123	{
1124	RTIDTR idtr;
1125	RTGDTR gdtr;
1126	RTSEL SelTR;
1127	PCX86DESCHC pDesc;
1128	uintptr_t trBase;
1129	RTSEL cs;
1130	RTSEL ss;
1131	uint64_t cr3;
1132
1133	/*
1134	* Control registers.
1135	*/
1136	rc = VMXWriteVMCS(VMX_VMCS_HOST_CR0, ASMGetCR0());
1137	Log2(("VMX_VMCS_HOST_CR0 %08x\n", ASMGetCR0()));
1138	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
1139	if (VMX_IS_64BIT_HOST_MODE())
1140	{
1141	cr3 = hwaccmR0Get64bitCR3();
1142	rc \|= VMXWriteVMCS64(VMX_VMCS_HOST_CR3, cr3);
1143	}
1144	else
1145	#endif
1146	{
1147	cr3 = ASMGetCR3();
1148	rc \|= VMXWriteVMCS(VMX_VMCS_HOST_CR3, cr3);
1149	}
1150	Log2(("VMX_VMCS_HOST_CR3 %08RX64\n", cr3));
1151	rc \|= VMXWriteVMCS(VMX_VMCS_HOST_CR4, ASMGetCR4());
1152	Log2(("VMX_VMCS_HOST_CR4 %08x\n", ASMGetCR4()));
1153	AssertRC(rc);
1154
1155	/*
1156	* Selector registers.
1157	*/
1158	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
1159	if (VMX_IS_64BIT_HOST_MODE())
1160	{
1161	cs = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelCS;
1162	ss = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelSS;
1163	}
1164	else
1165	{
1166	/* sysenter loads LDT cs & ss, VMX doesn't like this. Load the GDT ones (safe). */
1167	cs = (RTSEL)(uintptr_t)&SUPR0AbsKernelCS;
1168	ss = (RTSEL)(uintptr_t)&SUPR0AbsKernelSS;
1169	}
1170	#else
1171	cs = ASMGetCS();
1172	ss = ASMGetSS();
1173	#endif
1174	Assert(!(cs & X86_SEL_LDT)); Assert((cs & X86_SEL_RPL) == 0);
1175	Assert(!(ss & X86_SEL_LDT)); Assert((ss & X86_SEL_RPL) == 0);
1176	rc = VMXWriteVMCS(VMX_VMCS16_HOST_FIELD_CS, cs);
1177	/* Note: VMX is (again) very picky about the RPL of the selectors here; we'll restore them manually. */
1178	rc \|= VMXWriteVMCS(VMX_VMCS16_HOST_FIELD_DS, 0);
1179	rc \|= VMXWriteVMCS(VMX_VMCS16_HOST_FIELD_ES, 0);
1180	#if HC_ARCH_BITS == 32
1181	if (!VMX_IS_64BIT_HOST_MODE())
1182	{
1183	rc \|= VMXWriteVMCS(VMX_VMCS16_HOST_FIELD_FS, 0);
1184	rc \|= VMXWriteVMCS(VMX_VMCS16_HOST_FIELD_GS, 0);
1185	}
1186	#endif
1187	rc \|= VMXWriteVMCS(VMX_VMCS16_HOST_FIELD_SS, ss);
1188	SelTR = ASMGetTR();
1189	rc \|= VMXWriteVMCS(VMX_VMCS16_HOST_FIELD_TR, SelTR);
1190	AssertRC(rc);
1191	Log2(("VMX_VMCS_HOST_FIELD_CS %08x (%08x)\n", cs, ASMGetSS()));
1192	Log2(("VMX_VMCS_HOST_FIELD_DS 00000000 (%08x)\n", ASMGetDS()));
1193	Log2(("VMX_VMCS_HOST_FIELD_ES 00000000 (%08x)\n", ASMGetES()));
1194	Log2(("VMX_VMCS_HOST_FIELD_FS 00000000 (%08x)\n", ASMGetFS()));
1195	Log2(("VMX_VMCS_HOST_FIELD_GS 00000000 (%08x)\n", ASMGetGS()));
1196	Log2(("VMX_VMCS_HOST_FIELD_SS %08x (%08x)\n", ss, ASMGetSS()));
1197	Log2(("VMX_VMCS_HOST_FIELD_TR %08x\n", ASMGetTR()));
1198
1199	/*
1200	* GDTR & IDTR.
1201	*/
1202	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
1203	if (VMX_IS_64BIT_HOST_MODE())
1204	{
1205	X86XDTR64 gdtr64, idtr64;
1206	hwaccmR0Get64bitGDTRandIDTR(&gdtr64, &idtr64);
1207	rc = VMXWriteVMCS64(VMX_VMCS_HOST_GDTR_BASE, gdtr64.uAddr);
1208	rc \|= VMXWriteVMCS64(VMX_VMCS_HOST_IDTR_BASE, gdtr64.uAddr);
1209	AssertRC(rc);
1210	Log2(("VMX_VMCS_HOST_GDTR_BASE %RX64\n", gdtr64.uAddr));
1211	Log2(("VMX_VMCS_HOST_IDTR_BASE %RX64\n", idtr64.uAddr));
1212	gdtr.cbGdt = gdtr64.cb;
1213	gdtr.pGdt = (uintptr_t)gdtr64.uAddr;
1214	}
1215	else
1216	#endif
1217	{
1218	ASMGetGDTR(&gdtr);
1219	rc = VMXWriteVMCS(VMX_VMCS_HOST_GDTR_BASE, gdtr.pGdt);
1220	ASMGetIDTR(&idtr);
1221	rc \|= VMXWriteVMCS(VMX_VMCS_HOST_IDTR_BASE, idtr.pIdt);
1222	AssertRC(rc);
1223	Log2(("VMX_VMCS_HOST_GDTR_BASE %RHv\n", gdtr.pGdt));
1224	Log2(("VMX_VMCS_HOST_IDTR_BASE %RHv\n", idtr.pIdt));
1225	}
1226
1227	/*
1228	* Save the base address of the TR selector.
1229	*/
1230	if (SelTR > gdtr.cbGdt)
1231	{
1232	AssertMsgFailed(("Invalid TR selector %x. GDTR.cbGdt=%x\n", SelTR, gdtr.cbGdt));
1233	return VERR_VMX_INVALID_HOST_STATE;
1234	}
1235
1236	pDesc = (PCX86DESCHC)(gdtr.pGdt + (SelTR & X86_SEL_MASK));
1237	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
1238	if (VMX_IS_64BIT_HOST_MODE())
1239	{
1240	uint64_t trBase64 = X86DESC64_BASE(*(PX86DESC64)pDesc);
1241	rc = VMXWriteVMCS64(VMX_VMCS_HOST_TR_BASE, trBase64);
1242	Log2(("VMX_VMCS_HOST_TR_BASE %RX64\n", trBase64));
1243	AssertRC(rc);
1244	}
1245	else
1246	#endif
1247	{
1248	#if HC_ARCH_BITS == 64
1249	trBase = X86DESC64_BASE(*pDesc);
1250	#else
1251	trBase = X86DESC_BASE(*pDesc);
1252	#endif
1253	rc = VMXWriteVMCS(VMX_VMCS_HOST_TR_BASE, trBase);
1254	AssertRC(rc);
1255	Log2(("VMX_VMCS_HOST_TR_BASE %RHv\n", trBase));
1256	}
1257
1258	/*
1259	* FS base and GS base.
1260	*/
1261	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
1262	if (VMX_IS_64BIT_HOST_MODE())
1263	{
1264	Log2(("MSR_K8_FS_BASE = %RX64\n", ASMRdMsr(MSR_K8_FS_BASE)));
1265	Log2(("MSR_K8_GS_BASE = %RX64\n", ASMRdMsr(MSR_K8_GS_BASE)));
1266	rc = VMXWriteVMCS64(VMX_VMCS_HOST_FS_BASE, ASMRdMsr(MSR_K8_FS_BASE));
1267	rc \|= VMXWriteVMCS64(VMX_VMCS_HOST_GS_BASE, ASMRdMsr(MSR_K8_GS_BASE));
1268	}
1269	#endif
1270	AssertRC(rc);
1271
1272	/*
1273	* Sysenter MSRs.
1274	*/
1275	/** @todo expensive!! */
1276	rc = VMXWriteVMCS(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
1277	Log2(("VMX_VMCS_HOST_SYSENTER_CS %08x\n", ASMRdMsr_Low(MSR_IA32_SYSENTER_CS)));
1278	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
1279	if (VMX_IS_64BIT_HOST_MODE())
1280	{
1281	Log2(("VMX_VMCS_HOST_SYSENTER_EIP %RX64\n", ASMRdMsr(MSR_IA32_SYSENTER_EIP)));
1282	Log2(("VMX_VMCS_HOST_SYSENTER_ESP %RX64\n", ASMRdMsr(MSR_IA32_SYSENTER_ESP)));
1283	rc \|= VMXWriteVMCS64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
1284	rc \|= VMXWriteVMCS64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
1285	}
1286	else
1287	{
1288	rc \|= VMXWriteVMCS(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
1289	rc \|= VMXWriteVMCS(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
1290	Log2(("VMX_VMCS_HOST_SYSENTER_EIP %RX32\n", ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP)));
1291	Log2(("VMX_VMCS_HOST_SYSENTER_ESP %RX32\n", ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP)));
1292	}
1293	#elif HC_ARCH_BITS == 32
1294	rc \|= VMXWriteVMCS(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
1295	rc \|= VMXWriteVMCS(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
1296	Log2(("VMX_VMCS_HOST_SYSENTER_EIP %RX32\n", ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP)));
1297	Log2(("VMX_VMCS_HOST_SYSENTER_ESP %RX32\n", ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP)));
1298	#else
1299	Log2(("VMX_VMCS_HOST_SYSENTER_EIP %RX64\n", ASMRdMsr(MSR_IA32_SYSENTER_EIP)));
1300	Log2(("VMX_VMCS_HOST_SYSENTER_ESP %RX64\n", ASMRdMsr(MSR_IA32_SYSENTER_ESP)));
1301	rc \|= VMXWriteVMCS64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
1302	rc \|= VMXWriteVMCS64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
1303	#endif
1304	AssertRC(rc);
1305
1306
1307	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
1308	/*
1309	* Store all host MSRs in the VM-Exit load area, so they will be reloaded after
1310	* the world switch back to the host.
1311	*/
1312	PVMXMSR pMsr = (PVMXMSR)pVCpu->hwaccm.s.vmx.pHostMSR;
1313	unsigned idxMsr = 0;
1314
1315	/*
1316	* Check if EFER MSR present.
1317	*/
1318	if (ASMCpuId_EDX(0x80000001) & (X86_CPUID_AMD_FEATURE_EDX_NX\|X86_CPUID_AMD_FEATURE_EDX_LONG_MODE))
1319	{
1320	if (ASMCpuId_EDX(0x80000001) & X86_CPUID_AMD_FEATURE_EDX_SEP)
1321	{
1322	pMsr->u32IndexMSR = MSR_K6_STAR;
1323	pMsr->u32Reserved = 0;
1324	pMsr->u64Value = ASMRdMsr(MSR_K6_STAR); /* legacy syscall eip, cs & ss */
1325	pMsr++; idxMsr++;
1326	}
1327
1328	pMsr->u32IndexMSR = MSR_K6_EFER;
1329	pMsr->u32Reserved = 0;
1330	# if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
1331	if (CPUMIsGuestInLongMode(pVCpu))
1332	{
1333	/* Must match the EFER value in our 64 bits switcher. */
1334	pMsr->u64Value = ASMRdMsr(MSR_K6_EFER) \| MSR_K6_EFER_LME \| MSR_K6_EFER_SCE \| MSR_K6_EFER_NXE;
1335	}
1336	else
1337	# endif
1338	pMsr->u64Value = ASMRdMsr(MSR_K6_EFER);
1339	pMsr++; idxMsr++;
1340	}
1341
1342	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
1343	if (VMX_IS_64BIT_HOST_MODE())
1344	{
1345	pMsr->u32IndexMSR = MSR_K8_LSTAR;
1346	pMsr->u32Reserved = 0;
1347	pMsr->u64Value = ASMRdMsr(MSR_K8_LSTAR); /* 64 bits mode syscall rip */
1348	pMsr++; idxMsr++;
1349	pMsr->u32IndexMSR = MSR_K8_SF_MASK;
1350	pMsr->u32Reserved = 0;
1351	pMsr->u64Value = ASMRdMsr(MSR_K8_SF_MASK); /* syscall flag mask */
1352	pMsr++; idxMsr++;
1353	pMsr->u32IndexMSR = MSR_K8_KERNEL_GS_BASE;
1354	pMsr->u32Reserved = 0;
1355	pMsr->u64Value = ASMRdMsr(MSR_K8_KERNEL_GS_BASE); /* swapgs exchange value */
1356	pMsr++; idxMsr++;
1357	}
1358	# endif
1359	rc = VMXWriteVMCS(VMX_VMCS_CTRL_EXIT_MSR_LOAD_COUNT, idxMsr);
1360	AssertRC(rc);
1361	#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
1362
1363	pVCpu->hwaccm.s.fContextUseFlags &= ~HWACCM_CHANGED_HOST_CONTEXT;
1364	}
1365	return rc;
1366	}
1367
1368
1369	/**
1370	* Loads the 4 PDPEs into the guest state when nested paging is used and the
1371	* guest operates in PAE mode.
1372	*
1373	* @returns VBox status code.
1374	* @param pVCpu Pointer to the VMCPU.
1375	* @param pCtx Pointer to the guest CPU context.
1376	*/
1377	static int hmR0VmxLoadPaePdpes(PVMCPU pVCpu, PCPUMCTX pCtx)
1378	{
1379	if (CPUMIsGuestInPAEModeEx(pCtx))
1380	{
1381	X86PDPE aPdpes[4];
1382	int rc = PGMGstGetPaePdpes(pVCpu, &aPdpes[0]);
1383	AssertRCReturn(rc, rc);
1384
1385	rc = VMXWriteVMCS64(VMX_VMCS_GUEST_PDPTR0_FULL, aPdpes[0].u); AssertRCReturn(rc, rc);
1386	rc = VMXWriteVMCS64(VMX_VMCS_GUEST_PDPTR1_FULL, aPdpes[1].u); AssertRCReturn(rc, rc);
1387	rc = VMXWriteVMCS64(VMX_VMCS_GUEST_PDPTR2_FULL, aPdpes[2].u); AssertRCReturn(rc, rc);
1388	rc = VMXWriteVMCS64(VMX_VMCS_GUEST_PDPTR3_FULL, aPdpes[3].u); AssertRCReturn(rc, rc);
1389	}
1390	return VINF_SUCCESS;
1391	}
1392
1393
1394	/**
1395	* Saves the 4 PDPEs into the guest state when nested paging is used and the
1396	* guest operates in PAE mode.
1397	*
1398	* @returns VBox status code.
1399	* @param pVCpu Pointer to the VM CPU.
1400	* @param pCtx Pointer to the guest CPU context.
1401	*
1402	* @remarks Tell PGM about CR3 changes before calling this helper.
1403	*/
1404	static int hmR0VmxSavePaePdpes(PVMCPU pVCpu, PCPUMCTX pCtx)
1405	{
1406	if (CPUMIsGuestInPAEModeEx(pCtx))
1407	{
1408	int rc;
1409	X86PDPE aPdpes[4];
1410	rc = VMXReadVMCS64(VMX_VMCS_GUEST_PDPTR0_FULL, &aPdpes[0].u); AssertRCReturn(rc, rc);
1411	rc = VMXReadVMCS64(VMX_VMCS_GUEST_PDPTR1_FULL, &aPdpes[1].u); AssertRCReturn(rc, rc);
1412	rc = VMXReadVMCS64(VMX_VMCS_GUEST_PDPTR2_FULL, &aPdpes[2].u); AssertRCReturn(rc, rc);
1413	rc = VMXReadVMCS64(VMX_VMCS_GUEST_PDPTR3_FULL, &aPdpes[3].u); AssertRCReturn(rc, rc);
1414
1415	rc = PGMGstUpdatePaePdpes(pVCpu, &aPdpes[0]);
1416	AssertRCReturn(rc, rc);
1417	}
1418	return VINF_SUCCESS;
1419	}
1420
1421
1422	/**
1423	* Update the exception bitmap according to the current CPU state.
1424	*
1425	* @param pVM Pointer to the VM.
1426	* @param pVCpu Pointer to the VMCPU.
1427	* @param pCtx Pointer to the guest CPU context.
1428	*/
1429	static void hmR0VmxUpdateExceptionBitmap(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
1430	{
1431	uint32_t u32TrapMask;
1432	Assert(pCtx);
1433
1434	/*
1435	* Set up a mask for intercepting traps.
1436	*/
1437	/** @todo Do we really need to always intercept #DB? */
1438	u32TrapMask = RT_BIT(X86_XCPT_DB)
1439	\| RT_BIT(X86_XCPT_NM)
1440	#ifdef VBOX_ALWAYS_TRAP_PF
1441	\| RT_BIT(X86_XCPT_PF)
1442	#endif
1443	#ifdef VBOX_STRICT
1444	\| RT_BIT(X86_XCPT_BP)
1445	\| RT_BIT(X86_XCPT_DB)
1446	\| RT_BIT(X86_XCPT_DE)
1447	\| RT_BIT(X86_XCPT_NM)
1448	\| RT_BIT(X86_XCPT_UD)
1449	\| RT_BIT(X86_XCPT_NP)
1450	\| RT_BIT(X86_XCPT_SS)
1451	\| RT_BIT(X86_XCPT_GP)
1452	\| RT_BIT(X86_XCPT_MF)
1453	#endif
1454	;
1455
1456	/*
1457	* Without nested paging, #PF must be intercepted to implement shadow paging.
1458	*/
1459	/** @todo NP state won't change so maybe we should build the initial trap mask up front? */
1460	if (!pVM->hwaccm.s.fNestedPaging)
1461	u32TrapMask \|= RT_BIT(X86_XCPT_PF);
1462
1463	/* Catch floating point exceptions if we need to report them to the guest in a different way. */
1464	if (!(pCtx->cr0 & X86_CR0_NE))
1465	u32TrapMask \|= RT_BIT(X86_XCPT_MF);
1466
1467	#ifdef VBOX_STRICT
1468	Assert(u32TrapMask & RT_BIT(X86_XCPT_GP));
1469	#endif
1470
1471	/*
1472	* Intercept all exceptions in real mode as none of them can be injected directly (#GP otherwise).
1473	*/
1474	/** @todo Despite the claim to intercept everything, with NP we do not intercept #PF. Should we? */
1475	if ( CPUMIsGuestInRealModeEx(pCtx)
1476	&& pVM->hwaccm.s.vmx.pRealModeTSS)
1477	{
1478	u32TrapMask \|= RT_BIT(X86_XCPT_DE)
1479	\| RT_BIT(X86_XCPT_DB)
1480	\| RT_BIT(X86_XCPT_NMI)
1481	\| RT_BIT(X86_XCPT_BP)
1482	\| RT_BIT(X86_XCPT_OF)
1483	\| RT_BIT(X86_XCPT_BR)
1484	\| RT_BIT(X86_XCPT_UD)
1485	\| RT_BIT(X86_XCPT_DF)
1486	\| RT_BIT(X86_XCPT_CO_SEG_OVERRUN)
1487	\| RT_BIT(X86_XCPT_TS)
1488	\| RT_BIT(X86_XCPT_NP)
1489	\| RT_BIT(X86_XCPT_SS)
1490	\| RT_BIT(X86_XCPT_GP)
1491	\| RT_BIT(X86_XCPT_MF)
1492	\| RT_BIT(X86_XCPT_AC)
1493	\| RT_BIT(X86_XCPT_MC)
1494	\| RT_BIT(X86_XCPT_XF)
1495	;
1496	}
1497
1498	int rc = VMXWriteVMCS(VMX_VMCS_CTRL_EXCEPTION_BITMAP, u32TrapMask);
1499	AssertRC(rc);
1500	}
1501
1502
1503	/**
1504	* Loads a minimal guest state.
1505	*
1506	* NOTE: Don't do anything here that can cause a jump back to ring 3!!!!!
1507	*
1508	* @param pVM Pointer to the VM.
1509	* @param pVCpu Pointer to the VMCPU.
1510	* @param pCtx Pointer to the guest CPU context.
1511	*/
1512	VMMR0DECL(void) VMXR0LoadMinimalGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
1513	{
1514	int rc;
1515	X86EFLAGS eflags;
1516
1517	Assert(!(pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_ALL_GUEST));
1518
1519	/*
1520	* Load EIP, ESP and EFLAGS.
1521	*/
1522	rc = VMXWriteVMCS64(VMX_VMCS64_GUEST_RIP, pCtx->rip);
1523	rc \|= VMXWriteVMCS64(VMX_VMCS64_GUEST_RSP, pCtx->rsp);
1524	AssertRC(rc);
1525
1526	/*
1527	* Bits 22-31, 15, 5 & 3 must be zero. Bit 1 must be 1.
1528	*/
1529	eflags = pCtx->eflags;
1530	eflags.u32 &= VMX_EFLAGS_RESERVED_0;
1531	eflags.u32 \|= VMX_EFLAGS_RESERVED_1;
1532
1533	/*
1534	* Check if real mode emulation using v86 mode.
1535	*/
1536	if ( CPUMIsGuestInRealModeEx(pCtx)
1537	&& pVM->hwaccm.s.vmx.pRealModeTSS)
1538	{
1539	pVCpu->hwaccm.s.vmx.RealMode.eflags = eflags;
1540
1541	eflags.Bits.u1VM = 1;
1542	eflags.Bits.u2IOPL = 0; /* must always be 0 or else certain instructions won't cause faults. */
1543	}
1544	rc = VMXWriteVMCS(VMX_VMCS_GUEST_RFLAGS, eflags.u32);
1545	AssertRC(rc);
1546	}
1547
1548
1549	/**
1550	* Loads the guest state.
1551	*
1552	* NOTE: Don't do anything here that can cause a jump back to ring 3!!!!!
1553	*
1554	* @returns VBox status code.
1555	* @param pVM Pointer to the VM.
1556	* @param pVCpu Pointer to the VMCPU.
1557	* @param pCtx Pointer to the guest CPU context.
1558	*/
1559	VMMR0DECL(int) VMXR0LoadGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
1560	{
1561	int rc = VINF_SUCCESS;
1562	RTGCUINTPTR val;
1563
1564	/*
1565	* VMX_VMCS_CTRL_ENTRY_CONTROLS
1566	* Set required bits to one and zero according to the MSR capabilities.
1567	*/
1568	val = pVM->hwaccm.s.vmx.msr.vmx_entry.n.disallowed0;
1569
1570	/*
1571	* Load guest debug controls (DR7 & IA32_DEBUGCTL_MSR).
1572	* Forced to 1 on the 'first' VT-x capable CPUs; this actually includes the newest Nehalem CPUs
1573	*/
1574	val \|= VMX_VMCS_CTRL_ENTRY_CONTROLS_LOAD_DEBUG;
1575
1576	if (CPUMIsGuestInLongModeEx(pCtx))
1577	val \|= VMX_VMCS_CTRL_ENTRY_CONTROLS_IA64_MODE;
1578	/* else Must be zero when AMD64 is not available. */
1579
1580	/*
1581	* Mask away the bits that the CPU doesn't support.
1582	*/
1583	val &= pVM->hwaccm.s.vmx.msr.vmx_entry.n.allowed1;
1584	rc = VMXWriteVMCS(VMX_VMCS_CTRL_ENTRY_CONTROLS, val);
1585	AssertRC(rc);
1586
1587	/*
1588	* VMX_VMCS_CTRL_EXIT_CONTROLS
1589	* Set required bits to one and zero according to the MSR capabilities.
1590	*/
1591	val = pVM->hwaccm.s.vmx.msr.vmx_exit.n.disallowed0;
1592
1593	/*
1594	* Save debug controls (DR7 & IA32_DEBUGCTL_MSR)
1595	* Forced to 1 on the 'first' VT-x capable CPUs; this actually includes the newest Nehalem CPUs
1596	*/
1597	val \|= VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_DEBUG;
1598
1599	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
1600	if (VMX_IS_64BIT_HOST_MODE())
1601	val \|= VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_AMD64;
1602	/* else Must be zero when AMD64 is not available. */
1603	#elif HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
1604	if (CPUMIsGuestInLongModeEx(pCtx))
1605	val \|= VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_AMD64; /* our switcher goes to long mode */
1606	else
1607	Assert(!(val & VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_AMD64));
1608	#endif
1609	val &= pVM->hwaccm.s.vmx.msr.vmx_exit.n.allowed1;
1610
1611	/*
1612	* Don't acknowledge external interrupts on VM-exit.
1613	*/
1614	rc = VMXWriteVMCS(VMX_VMCS_CTRL_EXIT_CONTROLS, val);
1615	AssertRC(rc);
1616
1617	/*
1618	* Guest CPU context: ES, CS, SS, DS, FS, GS.
1619	*/
1620	if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_SEGMENT_REGS)
1621	{
1622	if (pVM->hwaccm.s.vmx.pRealModeTSS)
1623	{
1624	PGMMODE enmGuestMode = PGMGetGuestMode(pVCpu);
1625	if (pVCpu->hwaccm.s.vmx.enmLastSeenGuestMode != enmGuestMode)
1626	{
1627	/*
1628	* Correct weird requirements for switching to protected mode.
1629	*/
1630	if ( pVCpu->hwaccm.s.vmx.enmLastSeenGuestMode == PGMMODE_REAL
1631	&& enmGuestMode >= PGMMODE_PROTECTED)
1632	{
1633	#ifdef VBOX_WITH_REM
1634	/*
1635	* Flush the recompiler code cache as it's not unlikely the guest will rewrite code
1636	* it will later execute in real mode (OpenBSD 4.0 is one such example)
1637	*/
1638	REMFlushTBs(pVM);
1639	#endif
1640
1641	/*
1642	* DPL of all hidden selector registers must match the current CPL (0).
1643	*/
1644	pCtx->csHid.Attr.n.u2Dpl = 0;
1645	pCtx->csHid.Attr.n.u4Type = X86_SEL_TYPE_CODE \| X86_SEL_TYPE_RW_ACC;
1646
1647	pCtx->dsHid.Attr.n.u2Dpl = 0;
1648	pCtx->esHid.Attr.n.u2Dpl = 0;
1649	pCtx->fsHid.Attr.n.u2Dpl = 0;
1650	pCtx->gsHid.Attr.n.u2Dpl = 0;
1651	pCtx->ssHid.Attr.n.u2Dpl = 0;
1652	}
1653	pVCpu->hwaccm.s.vmx.enmLastSeenGuestMode = enmGuestMode;
1654	}
1655	else if ( CPUMIsGuestInRealModeEx(pCtx)
1656	&& pCtx->csHid.u64Base == 0xffff0000)
1657	{
1658	/* VT-x will fail with a guest invalid state otherwise... (CPU state after a reset) */
1659	pCtx->csHid.u64Base = 0xf0000;
1660	pCtx->cs = 0xf000;
1661	}
1662	}
1663
1664	VMX_WRITE_SELREG(ES, es);
1665	AssertRC(rc);
1666
1667	VMX_WRITE_SELREG(CS, cs);
1668	AssertRC(rc);
1669
1670	VMX_WRITE_SELREG(SS, ss);
1671	AssertRC(rc);
1672
1673	VMX_WRITE_SELREG(DS, ds);
1674	AssertRC(rc);
1675
1676	VMX_WRITE_SELREG(FS, fs);
1677	AssertRC(rc);
1678
1679	VMX_WRITE_SELREG(GS, gs);
1680	AssertRC(rc);
1681	}
1682
1683	/*
1684	* Guest CPU context: LDTR.
1685	*/
1686	if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_LDTR)
1687	{
1688	if (pCtx->ldtr == 0)
1689	{
1690	rc = VMXWriteVMCS(VMX_VMCS16_GUEST_FIELD_LDTR, 0);
1691	rc \|= VMXWriteVMCS(VMX_VMCS32_GUEST_LDTR_LIMIT, 0);
1692	rc \|= VMXWriteVMCS64(VMX_VMCS64_GUEST_LDTR_BASE, 0);
1693	/* Note: vmlaunch will fail with 0 or just 0x02. No idea why. */
1694	rc \|= VMXWriteVMCS(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, 0x82 /* present, LDT */);
1695	}
1696	else
1697	{
1698	rc = VMXWriteVMCS(VMX_VMCS16_GUEST_FIELD_LDTR, pCtx->ldtr);
1699	rc \|= VMXWriteVMCS(VMX_VMCS32_GUEST_LDTR_LIMIT, pCtx->ldtrHid.u32Limit);
1700	rc \|= VMXWriteVMCS64(VMX_VMCS64_GUEST_LDTR_BASE, pCtx->ldtrHid.u64Base);
1701	rc \|= VMXWriteVMCS(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, pCtx->ldtrHid.Attr.u);
1702	}
1703	AssertRC(rc);
1704	}
1705
1706	/*
1707	* Guest CPU context: TR.
1708	*/
1709	if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_TR)
1710	{
1711	/*
1712	* Real mode emulation using v86 mode with CR4.VME (interrupt redirection
1713	* using the int bitmap in the TSS).
1714	*/
1715	if ( CPUMIsGuestInRealModeEx(pCtx)
1716	&& pVM->hwaccm.s.vmx.pRealModeTSS)
1717	{
1718	RTGCPHYS GCPhys;
1719
1720	/* We convert it here every time as PCI regions could be reconfigured. */
1721	rc = PDMVMMDevHeapR3ToGCPhys(pVM, pVM->hwaccm.s.vmx.pRealModeTSS, &GCPhys);
1722	AssertRC(rc);
1723
1724	rc = VMXWriteVMCS(VMX_VMCS16_GUEST_FIELD_TR, 0);
1725	rc \|= VMXWriteVMCS(VMX_VMCS32_GUEST_TR_LIMIT, HWACCM_VTX_TSS_SIZE);
1726	rc \|= VMXWriteVMCS64(VMX_VMCS64_GUEST_TR_BASE, GCPhys /* phys = virt in this mode */);
1727
1728	X86DESCATTR attr;
1729
1730	attr.u = 0;
1731	attr.n.u1Present = 1;
1732	attr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
1733	val = attr.u;
1734	}
1735	else
1736	{
1737	rc = VMXWriteVMCS(VMX_VMCS16_GUEST_FIELD_TR, pCtx->tr);
1738	rc \|= VMXWriteVMCS(VMX_VMCS32_GUEST_TR_LIMIT, pCtx->trHid.u32Limit);
1739	rc \|= VMXWriteVMCS64(VMX_VMCS64_GUEST_TR_BASE, pCtx->trHid.u64Base);
1740
1741	val = pCtx->trHid.Attr.u;
1742
1743	/* The TSS selector must be busy (REM bugs? see defect #XXXX). */
1744	if (!(val & X86_SEL_TYPE_SYS_TSS_BUSY_MASK))
1745	{
1746	if (val & 0xf)
1747	val \|= X86_SEL_TYPE_SYS_TSS_BUSY_MASK;
1748	else
1749	/* Default if no TR selector has been set (otherwise vmlaunch will fail!) */
1750	val = (val & ~0xF) \| X86_SEL_TYPE_SYS_386_TSS_BUSY;
1751	}
1752	AssertMsg((val & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY \|\| (val & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("%#x\n", val));
1753	}
1754	rc \|= VMXWriteVMCS(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, val);
1755	AssertRC(rc);
1756	}
1757
1758	/*
1759	* Guest CPU context: GDTR.
1760	*/
1761	if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_GDTR)
1762	{
1763	rc = VMXWriteVMCS(VMX_VMCS32_GUEST_GDTR_LIMIT, pCtx->gdtr.cbGdt);
1764	rc \|= VMXWriteVMCS64(VMX_VMCS64_GUEST_GDTR_BASE, pCtx->gdtr.pGdt);
1765	AssertRC(rc);
1766	}
1767
1768	/*
1769	* Guest CPU context: IDTR.
1770	*/
1771	if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_IDTR)
1772	{
1773	rc = VMXWriteVMCS(VMX_VMCS32_GUEST_IDTR_LIMIT, pCtx->idtr.cbIdt);
1774	rc \|= VMXWriteVMCS64(VMX_VMCS64_GUEST_IDTR_BASE, pCtx->idtr.pIdt);
1775	AssertRC(rc);
1776	}
1777
1778	/*
1779	* Sysenter MSRs.
1780	*/
1781	if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_MSR)
1782	{
1783	rc = VMXWriteVMCS(VMX_VMCS32_GUEST_SYSENTER_CS, pCtx->SysEnter.cs);
1784	rc \|= VMXWriteVMCS64(VMX_VMCS64_GUEST_SYSENTER_EIP, pCtx->SysEnter.eip);
1785	rc \|= VMXWriteVMCS64(VMX_VMCS64_GUEST_SYSENTER_ESP, pCtx->SysEnter.esp);
1786	AssertRC(rc);
1787	}
1788
1789	/*
1790	* Guest CPU context: Control registers.
1791	*/
1792	if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_CR0)
1793	{
1794	val = pCtx->cr0;
1795	rc = VMXWriteVMCS(VMX_VMCS_CTRL_CR0_READ_SHADOW, val);
1796	Log2(("Guest CR0-shadow %08x\n", val));
1797	if (CPUMIsGuestFPUStateActive(pVCpu) == false)
1798	{
1799	/* Always use #NM exceptions to load the FPU/XMM state on demand. */
1800	val \|= X86_CR0_TS \| X86_CR0_ET \| X86_CR0_NE \| X86_CR0_MP;
1801	}
1802	else
1803	{
1804	/** @todo check if we support the old style mess correctly. */
1805	if (!(val & X86_CR0_NE))
1806	Log(("Forcing X86_CR0_NE!!!\n"));
1807
1808	val \|= X86_CR0_NE; /* always turn on the native mechanism to report FPU errors (old style uses interrupts) */
1809	}
1810	/* Note: protected mode & paging are always enabled; we use them for emulating real and protected mode without paging too. */
1811	if (!pVM->hwaccm.s.vmx.fUnrestrictedGuest)
1812	val \|= X86_CR0_PE \| X86_CR0_PG;
1813
1814	if (pVM->hwaccm.s.fNestedPaging)
1815	{
1816	if (CPUMIsGuestInPagedProtectedModeEx(pCtx))
1817	{
1818	/* Disable CR3 read/write monitoring as we don't need it for EPT. */
1819	pVCpu->hwaccm.s.vmx.proc_ctls &= ~( VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
1820	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT);
1821	}
1822	else
1823	{
1824	/* Reenable CR3 read/write monitoring as our identity mapped page table is active. */
1825	pVCpu->hwaccm.s.vmx.proc_ctls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
1826	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT;
1827	}
1828	rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
1829	AssertRC(rc);
1830	}
1831	else
1832	{
1833	/* Note: We must also set this as we rely on protecting various pages for which supervisor writes must be caught. */
1834	val \|= X86_CR0_WP;
1835	}
1836
1837	/* Always enable caching. */
1838	val &= ~(X86_CR0_CD\|X86_CR0_NW);
1839
1840	rc \|= VMXWriteVMCS64(VMX_VMCS64_GUEST_CR0, val);
1841	Log2(("Guest CR0 %08x\n", val));
1842
1843	/*
1844	* CR0 flags owned by the host; if the guests attempts to change them, then the VM will exit.
1845	*/
1846	val = X86_CR0_PE /* Must monitor this bit (assumptions are made for real mode emulation) */
1847	\| X86_CR0_WP /* Must monitor this bit (it must always be enabled). */
1848	\| X86_CR0_PG /* Must monitor this bit (assumptions are made for real mode & protected mode without paging emulation) */
1849	\| X86_CR0_CD /* Bit not restored during VM-exit! */
1850	\| X86_CR0_NW /* Bit not restored during VM-exit! */
1851	\| X86_CR0_NE;
1852
1853	/*
1854	* When the guest's FPU state is active, then we no longer care about the FPU related bits.
1855	*/
1856	if (CPUMIsGuestFPUStateActive(pVCpu) == false)
1857	val \|= X86_CR0_TS \| X86_CR0_ET \| X86_CR0_MP;
1858
1859	pVCpu->hwaccm.s.vmx.cr0_mask = val;
1860
1861	rc \|= VMXWriteVMCS(VMX_VMCS_CTRL_CR0_MASK, val);
1862	Log2(("Guest CR0-mask %08x\n", val));
1863	AssertRC(rc);
1864	}
1865
1866	if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_CR4)
1867	{
1868	rc = VMXWriteVMCS(VMX_VMCS_CTRL_CR4_READ_SHADOW, pCtx->cr4);
1869	Log2(("Guest CR4-shadow %08x\n", pCtx->cr4));
1870	/* Set the required bits in cr4 too (currently X86_CR4_VMXE). */
1871	val = pCtx->cr4 \| (uint32_t)pVM->hwaccm.s.vmx.msr.vmx_cr4_fixed0;
1872
1873	if (!pVM->hwaccm.s.fNestedPaging)
1874	{
1875	switch(pVCpu->hwaccm.s.enmShadowMode)
1876	{
1877	case PGMMODE_REAL: /* Real mode -> emulated using v86 mode */
1878	case PGMMODE_PROTECTED: /* Protected mode, no paging -> emulated using identity mapping. */
1879	case PGMMODE_32_BIT: /* 32-bit paging. */
1880	val &= ~X86_CR4_PAE;
1881	break;
1882
1883	case PGMMODE_PAE: /* PAE paging. */
1884	case PGMMODE_PAE_NX: /* PAE paging with NX enabled. */
1885	/** Must use PAE paging as we could use physical memory > 4 GB */
1886	val \|= X86_CR4_PAE;
1887	break;
1888
1889	case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
1890	case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
1891	#ifdef VBOX_ENABLE_64_BITS_GUESTS
1892	break;
1893	#else
1894	AssertFailed();
1895	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
1896	#endif
1897	default: /* shut up gcc */
1898	AssertFailed();
1899	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
1900	}
1901	}
1902	else if ( !CPUMIsGuestInPagedProtectedModeEx(pCtx)
1903	&& !pVM->hwaccm.s.vmx.fUnrestrictedGuest)
1904	{
1905	/* We use 4 MB pages in our identity mapping page table for real and protected mode without paging. */
1906	val \|= X86_CR4_PSE;
1907	/* Our identity mapping is a 32 bits page directory. */
1908	val &= ~X86_CR4_PAE;
1909	}
1910
1911	/*
1912	* Turn off VME if we're in emulated real mode.
1913	*/
1914	if ( CPUMIsGuestInRealModeEx(pCtx)
1915	&& pVM->hwaccm.s.vmx.pRealModeTSS)
1916	{
1917	val &= ~X86_CR4_VME;
1918	}
1919
1920	rc \|= VMXWriteVMCS64(VMX_VMCS64_GUEST_CR4, val);
1921	Log2(("Guest CR4 %08x\n", val));
1922
1923	/*
1924	* CR4 flags owned by the host; if the guests attempts to change them, then the VM will exit.
1925	*/
1926	val = 0
1927	\| X86_CR4_VME
1928	\| X86_CR4_PAE
1929	\| X86_CR4_PGE
1930	\| X86_CR4_PSE
1931	\| X86_CR4_VMXE;
1932	pVCpu->hwaccm.s.vmx.cr4_mask = val;
1933
1934	rc \|= VMXWriteVMCS(VMX_VMCS_CTRL_CR4_MASK, val);
1935	Log2(("Guest CR4-mask %08x\n", val));
1936	AssertRC(rc);
1937	}
1938
1939	#if 0
1940	/* Enable single stepping if requested and CPU supports it. */
1941	if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_TRAP_FLAG)
1942	if (DBGFIsStepping(pVCpu))
1943	{
1944	pVCpu->hwaccm.s.vmx.proc_ctls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_TRAP_FLAG;
1945	rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
1946	AssertRC(rc);
1947	}
1948	#endif
1949
1950	if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_CR3)
1951	{
1952	if (pVM->hwaccm.s.fNestedPaging)
1953	{
1954	Assert(PGMGetHyperCR3(pVCpu));
1955	pVCpu->hwaccm.s.vmx.GCPhysEPTP = PGMGetHyperCR3(pVCpu);
1956
1957	Assert(!(pVCpu->hwaccm.s.vmx.GCPhysEPTP & 0xfff));
1958	/** @todo Check the IA32_VMX_EPT_VPID_CAP MSR for other supported memory types. */
1959	pVCpu->hwaccm.s.vmx.GCPhysEPTP \|= VMX_EPT_MEMTYPE_WB
1960	\| (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
1961
1962	rc = VMXWriteVMCS64(VMX_VMCS_CTRL_EPTP_FULL, pVCpu->hwaccm.s.vmx.GCPhysEPTP);
1963	AssertRC(rc);
1964
1965	if ( !CPUMIsGuestInPagedProtectedModeEx(pCtx)
1966	&& !pVM->hwaccm.s.vmx.fUnrestrictedGuest)
1967	{
1968	RTGCPHYS GCPhys;
1969
1970	/* We convert it here every time as PCI regions could be reconfigured. */
1971	rc = PDMVMMDevHeapR3ToGCPhys(pVM, pVM->hwaccm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
1972	AssertMsgRC(rc, ("pNonPagingModeEPTPageTable = %RGv\n", pVM->hwaccm.s.vmx.pNonPagingModeEPTPageTable));
1973
1974	/*
1975	* We use our identity mapping page table here as we need to map guest virtual to
1976	* guest physical addresses; EPT will take care of the translation to host physical addresses.
1977	*/
1978	val = GCPhys;
1979	}
1980	else
1981	{
1982	/* Save the real guest CR3 in VMX_VMCS_GUEST_CR3 */
1983	val = pCtx->cr3;
1984	rc = hmR0VmxLoadPaePdpes(pVCpu, pCtx);
1985	AssertRCReturn(rc, rc);
1986	}
1987	}
1988	else
1989	{
1990	val = PGMGetHyperCR3(pVCpu);
1991	Assert(val \|\| VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL));
1992	}
1993
1994	/* Save our shadow CR3 register. */
1995	rc = VMXWriteVMCS64(VMX_VMCS64_GUEST_CR3, val);
1996	AssertRC(rc);
1997	}
1998
1999	/*
2000	* Guest CPU context: Debug registers.
2001	*/
2002	if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_DEBUG)
2003	{
2004	pCtx->dr[6] \|= X86_DR6_INIT_VAL; /* set all reserved bits to 1. */
2005	pCtx->dr[6] &= ~RT_BIT(12); /* must be zero. */
2006
2007	pCtx->dr[7] &= 0xffffffff; /* upper 32 bits reserved */
2008	pCtx->dr[7] &= ~(RT_BIT(11) \| RT_BIT(12) \| RT_BIT(14) \| RT_BIT(15)); /* must be zero */
2009	pCtx->dr[7] \|= 0x400; /* must be one */
2010
2011	/* Resync DR7 */
2012	rc = VMXWriteVMCS64(VMX_VMCS64_GUEST_DR7, pCtx->dr[7]);
2013	AssertRC(rc);
2014
2015	#ifdef DEBUG
2016	/* Sync the hypervisor debug state now if any breakpoint is armed. */
2017	if ( CPUMGetHyperDR7(pVCpu) & (X86_DR7_ENABLED_MASK\|X86_DR7_GD)
2018	&& !CPUMIsHyperDebugStateActive(pVCpu)
2019	&& !DBGFIsStepping(pVCpu))
2020	{
2021	/* Save the host and load the hypervisor debug state. */
2022	rc = CPUMR0LoadHyperDebugState(pVM, pVCpu, pCtx, true /* include DR6 */);
2023	AssertRC(rc);
2024
2025	/* DRx intercepts remain enabled. */
2026
2027	/* Override dr7 with the hypervisor value. */
2028	rc = VMXWriteVMCS64(VMX_VMCS64_GUEST_DR7, CPUMGetHyperDR7(pVCpu));
2029	AssertRC(rc);
2030	}
2031	else
2032	#endif
2033	/* Sync the debug state now if any breakpoint is armed. */
2034	if ( (pCtx->dr[7] & (X86_DR7_ENABLED_MASK\|X86_DR7_GD))
2035	&& !CPUMIsGuestDebugStateActive(pVCpu)
2036	&& !DBGFIsStepping(pVCpu))
2037	{
2038	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatDRxArmed);
2039
2040	/* Disable DRx move intercepts. */
2041	pVCpu->hwaccm.s.vmx.proc_ctls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT;
2042	rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
2043	AssertRC(rc);
2044
2045	/* Save the host and load the guest debug state. */
2046	rc = CPUMR0LoadGuestDebugState(pVM, pVCpu, pCtx, true /* include DR6 */);
2047	AssertRC(rc);
2048	}
2049
2050	/* IA32_DEBUGCTL MSR. */
2051	rc = VMXWriteVMCS64(VMX_VMCS_GUEST_DEBUGCTL_FULL, 0);
2052	AssertRC(rc);
2053
2054	/** @todo do we really ever need this? */
2055	rc \|= VMXWriteVMCS(VMX_VMCS_GUEST_DEBUG_EXCEPTIONS, 0);
2056	AssertRC(rc);
2057	}
2058
2059	/*
2060	* 64-bit guest mode.
2061	*/
2062	if (CPUMIsGuestInLongModeEx(pCtx))
2063	{
2064	#if !defined(VBOX_ENABLE_64_BITS_GUESTS)
2065	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
2066	#elif HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2067	pVCpu->hwaccm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
2068	#else
2069	# ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2070	if (!pVM->hwaccm.s.fAllow64BitGuests)
2071	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
2072	# endif
2073	pVCpu->hwaccm.s.vmx.pfnStartVM = VMXR0StartVM64;
2074	#endif
2075	if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_MSR)
2076	{
2077	/* Update these as wrmsr might have changed them. */
2078	rc = VMXWriteVMCS64(VMX_VMCS64_GUEST_FS_BASE, pCtx->fsHid.u64Base);
2079	AssertRC(rc);
2080	rc = VMXWriteVMCS64(VMX_VMCS64_GUEST_GS_BASE, pCtx->gsHid.u64Base);
2081	AssertRC(rc);
2082	}
2083	}
2084	else
2085	{
2086	pVCpu->hwaccm.s.vmx.pfnStartVM = VMXR0StartVM32;
2087	}
2088
2089	hmR0VmxUpdateExceptionBitmap(pVM, pVCpu, pCtx);
2090
2091	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
2092	/* Store all guest MSRs in the VM-Entry load area, so they will be loaded during the world switch. */
2093	PVMXMSR pMsr = (PVMXMSR)pVCpu->hwaccm.s.vmx.pGuestMSR;
2094	unsigned idxMsr = 0;
2095
2096	uint32_t ulEdx;
2097	uint32_t ulTemp;
2098	CPUMGetGuestCpuId(pVCpu, 0x80000001, &ulTemp, &ulTemp, &ulTemp, &ulEdx);
2099	/* EFER MSR present? */
2100	if (ulEdx & (X86_CPUID_AMD_FEATURE_EDX_NX\|X86_CPUID_AMD_FEATURE_EDX_LONG_MODE))
2101	{
2102	pMsr->u32IndexMSR = MSR_K6_EFER;
2103	pMsr->u32Reserved = 0;
2104	pMsr->u64Value = pCtx->msrEFER;
2105	/* VT-x will complain if only MSR_K6_EFER_LME is set. */
2106	if (!CPUMIsGuestInLongModeEx(pCtx))
2107	pMsr->u64Value &= ~(MSR_K6_EFER_LMA\|MSR_K6_EFER_LME);
2108	pMsr++; idxMsr++;
2109
2110	if (ulEdx & X86_CPUID_AMD_FEATURE_EDX_LONG_MODE)
2111	{
2112	pMsr->u32IndexMSR = MSR_K8_LSTAR;
2113	pMsr->u32Reserved = 0;
2114	pMsr->u64Value = pCtx->msrLSTAR; /* 64 bits mode syscall rip */
2115	pMsr++; idxMsr++;
2116	pMsr->u32IndexMSR = MSR_K6_STAR;
2117	pMsr->u32Reserved = 0;
2118	pMsr->u64Value = pCtx->msrSTAR; /* legacy syscall eip, cs & ss */
2119	pMsr++; idxMsr++;
2120	pMsr->u32IndexMSR = MSR_K8_SF_MASK;
2121	pMsr->u32Reserved = 0;
2122	pMsr->u64Value = pCtx->msrSFMASK; /* syscall flag mask */
2123	pMsr++; idxMsr++;
2124	pMsr->u32IndexMSR = MSR_K8_KERNEL_GS_BASE;
2125	pMsr->u32Reserved = 0;
2126	pMsr->u64Value = pCtx->msrKERNELGSBASE; /* swapgs exchange value */
2127	pMsr++; idxMsr++;
2128	}
2129	}
2130	pVCpu->hwaccm.s.vmx.cCachedMSRs = idxMsr;
2131
2132	rc = VMXWriteVMCS(VMX_VMCS_CTRL_ENTRY_MSR_LOAD_COUNT, idxMsr);
2133	AssertRC(rc);
2134
2135	rc = VMXWriteVMCS(VMX_VMCS_CTRL_EXIT_MSR_STORE_COUNT, idxMsr);
2136	AssertRC(rc);
2137	#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
2138
2139	bool fOffsettedTsc;
2140	if (pVM->hwaccm.s.vmx.fUsePreemptTimer)
2141	{
2142	uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVCpu, &fOffsettedTsc, &pVCpu->hwaccm.s.vmx.u64TSCOffset);
2143
2144	/* Make sure the returned values have sane upper and lower boundaries. */
2145	uint64_t u64CpuHz = SUPGetCpuHzFromGIP(g_pSUPGlobalInfoPage);
2146
2147	cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64 of a second */
2148	cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
2149
2150	cTicksToDeadline >>= pVM->hwaccm.s.vmx.cPreemptTimerShift;
2151	uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
2152	rc = VMXWriteVMCS(VMX_VMCS32_GUEST_PREEMPTION_TIMER_VALUE, cPreemptionTickCount);
2153	AssertRC(rc);
2154	}
2155	else
2156	fOffsettedTsc = TMCpuTickCanUseRealTSC(pVCpu, &pVCpu->hwaccm.s.vmx.u64TSCOffset);
2157
2158	if (fOffsettedTsc)
2159	{
2160	uint64_t u64CurTSC = ASMReadTSC();
2161	if (u64CurTSC + pVCpu->hwaccm.s.vmx.u64TSCOffset >= TMCpuTickGetLastSeen(pVCpu))
2162	{
2163	/* Note: VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT takes precedence over TSC_OFFSET */
2164	rc = VMXWriteVMCS64(VMX_VMCS_CTRL_TSC_OFFSET_FULL, pVCpu->hwaccm.s.vmx.u64TSCOffset);
2165	AssertRC(rc);
2166
2167	pVCpu->hwaccm.s.vmx.proc_ctls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT;
2168	rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
2169	AssertRC(rc);
2170	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatTSCOffset);
2171	}
2172	else
2173	{
2174	/* Fall back to rdtsc emulation as we would otherwise pass decreasing tsc values to the guest. */
2175	LogFlow(("TSC %RX64 offset %RX64 time=%RX64 last=%RX64 (diff=%RX64, virt_tsc=%RX64)\n", u64CurTSC,
2176	pVCpu->hwaccm.s.vmx.u64TSCOffset, u64CurTSC + pVCpu->hwaccm.s.vmx.u64TSCOffset,
2177	TMCpuTickGetLastSeen(pVCpu), TMCpuTickGetLastSeen(pVCpu) - u64CurTSC - pVCpu->hwaccm.s.vmx.u64TSCOffset,
2178	TMCpuTickGet(pVCpu)));
2179	pVCpu->hwaccm.s.vmx.proc_ctls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT;
2180	rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
2181	AssertRC(rc);
2182	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatTSCInterceptOverFlow);
2183	}
2184	}
2185	else
2186	{
2187	pVCpu->hwaccm.s.vmx.proc_ctls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT;
2188	rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
2189	AssertRC(rc);
2190	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatTSCIntercept);
2191	}
2192
2193	/* Done with the major changes */
2194	pVCpu->hwaccm.s.fContextUseFlags &= ~HWACCM_CHANGED_ALL_GUEST;
2195
2196	/* Minimal guest state update (ESP, EIP, EFLAGS mostly) */
2197	VMXR0LoadMinimalGuestState(pVM, pVCpu, pCtx);
2198	return rc;
2199	}
2200
2201
2202	/**
2203	* Syncs back the guest state from VMCS.
2204	*
2205	* @returns VBox status code.
2206	* @param pVM Pointer to the VM.
2207	* @param pVCpu Pointer to the VMCPU.
2208	* @param pCtx Pointer to the guest CPU context.
2209	*/
2210	DECLINLINE(int) VMXR0SaveGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
2211	{
2212	RTGCUINTREG val, valShadow;
2213	RTGCUINTPTR uInterruptState;
2214	int rc;
2215
2216	/* First sync back EIP, ESP, and EFLAGS. */
2217	rc = VMXReadCachedVMCS(VMX_VMCS64_GUEST_RIP, &val);
2218	AssertRC(rc);
2219	pCtx->rip = val;
2220	rc = VMXReadCachedVMCS(VMX_VMCS64_GUEST_RSP, &val);
2221	AssertRC(rc);
2222	pCtx->rsp = val;
2223	rc = VMXReadCachedVMCS(VMX_VMCS_GUEST_RFLAGS, &val);
2224	AssertRC(rc);
2225	pCtx->eflags.u32 = val;
2226
2227	/* Take care of instruction fusing (sti, mov ss) */
2228	rc \|= VMXReadCachedVMCS(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &val);
2229	uInterruptState = val;
2230	if (uInterruptState != 0)
2231	{
2232	Assert(uInterruptState <= 2); /* only sti & mov ss */
2233	Log(("uInterruptState %x eip=%RGv\n", (uint32_t)uInterruptState, pCtx->rip));
2234	EMSetInhibitInterruptsPC(pVCpu, pCtx->rip);
2235	}
2236	else
2237	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
2238
2239	/* Control registers. */
2240	VMXReadCachedVMCS(VMX_VMCS_CTRL_CR0_READ_SHADOW, &valShadow);
2241	VMXReadCachedVMCS(VMX_VMCS64_GUEST_CR0, &val);
2242	val = (valShadow & pVCpu->hwaccm.s.vmx.cr0_mask) \| (val & ~pVCpu->hwaccm.s.vmx.cr0_mask);
2243	CPUMSetGuestCR0(pVCpu, val);
2244
2245	VMXReadCachedVMCS(VMX_VMCS_CTRL_CR4_READ_SHADOW, &valShadow);
2246	VMXReadCachedVMCS(VMX_VMCS64_GUEST_CR4, &val);
2247	val = (valShadow & pVCpu->hwaccm.s.vmx.cr4_mask) \| (val & ~pVCpu->hwaccm.s.vmx.cr4_mask);
2248	CPUMSetGuestCR4(pVCpu, val);
2249
2250	/*
2251	* No reason to sync back the CRx registers. They can't be changed by the guest unless in
2252	* the nested paging case where CR3 & CR4 can be changed by the guest.
2253	*/
2254	if ( pVM->hwaccm.s.fNestedPaging
2255	&& CPUMIsGuestInPagedProtectedModeEx(pCtx)) /** @todo check if we will always catch mode switches and such... */
2256	{
2257	PVMCSCACHE pCache = &pVCpu->hwaccm.s.vmx.VMCSCache;
2258
2259	/* Can be updated behind our back in the nested paging case. */
2260	CPUMSetGuestCR2(pVCpu, pCache->cr2);
2261
2262	VMXReadCachedVMCS(VMX_VMCS64_GUEST_CR3, &val);
2263
2264	if (val != pCtx->cr3)
2265	{
2266	CPUMSetGuestCR3(pVCpu, val);
2267	PGMUpdateCR3(pVCpu, val);
2268	}
2269	rc = hmR0VmxSavePaePdpes(pVCpu, pCtx);
2270	AssertRCReturn(rc, rc);
2271	}
2272
2273	/* Sync back DR7. */
2274	VMXReadCachedVMCS(VMX_VMCS64_GUEST_DR7, &val);
2275	pCtx->dr[7] = val;
2276
2277	/* Guest CPU context: ES, CS, SS, DS, FS, GS. */
2278	VMX_READ_SELREG(ES, es);
2279	VMX_READ_SELREG(SS, ss);
2280	VMX_READ_SELREG(CS, cs);
2281	VMX_READ_SELREG(DS, ds);
2282	VMX_READ_SELREG(FS, fs);
2283	VMX_READ_SELREG(GS, gs);
2284
2285	/* System MSRs */
2286	VMXReadCachedVMCS(VMX_VMCS32_GUEST_SYSENTER_CS, &val);
2287	pCtx->SysEnter.cs = val;
2288	VMXReadCachedVMCS(VMX_VMCS64_GUEST_SYSENTER_EIP, &val);
2289	pCtx->SysEnter.eip = val;
2290	VMXReadCachedVMCS(VMX_VMCS64_GUEST_SYSENTER_ESP, &val);
2291	pCtx->SysEnter.esp = val;
2292
2293	/* Misc. registers; must sync everything otherwise we can get out of sync when jumping to ring 3. */
2294	VMX_READ_SELREG(LDTR, ldtr);
2295
2296	VMXReadCachedVMCS(VMX_VMCS32_GUEST_GDTR_LIMIT, &val);
2297	pCtx->gdtr.cbGdt = val;
2298	VMXReadCachedVMCS(VMX_VMCS64_GUEST_GDTR_BASE, &val);
2299	pCtx->gdtr.pGdt = val;
2300
2301	VMXReadCachedVMCS(VMX_VMCS32_GUEST_IDTR_LIMIT, &val);
2302	pCtx->idtr.cbIdt = val;
2303	VMXReadCachedVMCS(VMX_VMCS64_GUEST_IDTR_BASE, &val);
2304	pCtx->idtr.pIdt = val;
2305
2306	/* Real mode emulation using v86 mode. */
2307	if ( CPUMIsGuestInRealModeEx(pCtx)
2308	&& pVM->hwaccm.s.vmx.pRealModeTSS)
2309	{
2310	/* Hide our emulation flags */
2311	pCtx->eflags.Bits.u1VM = 0;
2312
2313	/* Restore original IOPL setting as we always use 0. */
2314	pCtx->eflags.Bits.u2IOPL = pVCpu->hwaccm.s.vmx.RealMode.eflags.Bits.u2IOPL;
2315
2316	/* Force a TR resync every time in case we switch modes. */
2317	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_GUEST_TR;
2318	}
2319	else
2320	{
2321	/* In real mode we have a fake TSS, so only sync it back when it's supposed to be valid. */
2322	VMX_READ_SELREG(TR, tr);
2323	}
2324
2325	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
2326	/* Save the possibly changed MSRs that we automatically restore and save during a world switch. */
2327	for (unsigned i = 0; i < pVCpu->hwaccm.s.vmx.cCachedMSRs; i++)
2328	{
2329	PVMXMSR pMsr = (PVMXMSR)pVCpu->hwaccm.s.vmx.pGuestMSR;
2330	pMsr += i;
2331
2332	switch (pMsr->u32IndexMSR)
2333	{
2334	case MSR_K8_LSTAR:
2335	pCtx->msrLSTAR = pMsr->u64Value;
2336	break;
2337	case MSR_K6_STAR:
2338	pCtx->msrSTAR = pMsr->u64Value;
2339	break;
2340	case MSR_K8_SF_MASK:
2341	pCtx->msrSFMASK = pMsr->u64Value;
2342	break;
2343	case MSR_K8_KERNEL_GS_BASE:
2344	pCtx->msrKERNELGSBASE = pMsr->u64Value;
2345	break;
2346	case MSR_K6_EFER:
2347	/* EFER can't be changed without causing a VM-exit. */
2348	/* Assert(pCtx->msrEFER == pMsr->u64Value); */
2349	break;
2350	default:
2351	AssertFailed();
2352	return VERR_HM_UNEXPECTED_LD_ST_MSR;
2353	}
2354	}
2355	#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
2356	return VINF_SUCCESS;
2357	}
2358
2359
2360	/**
2361	* Dummy placeholder for TLB flush handling before VM-entry. Used in the case
2362	* where neither EPT nor VPID is supported by the CPU.
2363	*
2364	* @param pVM Pointer to the VM.
2365	* @param pVCpu Pointer to the VMCPU.
2366	*/
2367	static DECLCALLBACK(void) hmR0VmxSetupTLBDummy(PVM pVM, PVMCPU pVCpu)
2368	{
2369	NOREF(pVM);
2370	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
2371	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
2372	pVCpu->hwaccm.s.TlbShootdown.cPages = 0;
2373	return;
2374	}
2375
2376
2377	/**
2378	* Setup the tagged TLB for EPT+VPID.
2379	*
2380	* @param pVM Pointer to the VM.
2381	* @param pVCpu Pointer to the VMCPU.
2382	*/
2383	static DECLCALLBACK(void) hmR0VmxSetupTLBBoth(PVM pVM, PVMCPU pVCpu)
2384	{
2385	PHMGLOBLCPUINFO pCpu;
2386
2387	Assert(pVM->hwaccm.s.fNestedPaging && pVM->hwaccm.s.vmx.fVPID);
2388
2389	pCpu = HWACCMR0GetCurrentCpu();
2390
2391	/*
2392	* Force a TLB flush for the first world switch if the current CPU differs from the one we ran on last
2393	* This can happen both for start & resume due to long jumps back to ring-3.
2394	* If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB,
2395	* so we cannot reuse the current ASID anymore.
2396	*/
2397	bool fNewASID = false;
2398	if ( pVCpu->hwaccm.s.idLastCpu != pCpu->idCpu
2399	\|\| pVCpu->hwaccm.s.cTLBFlushes != pCpu->cTLBFlushes)
2400	{
2401	pVCpu->hwaccm.s.fForceTLBFlush = true;
2402	fNewASID = true;
2403	}
2404
2405	/*
2406	* Check for explicit TLB shootdowns.
2407	*/
2408	if (VMCPU_FF_TESTANDCLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2409	pVCpu->hwaccm.s.fForceTLBFlush = true;
2410
2411	pVCpu->hwaccm.s.idLastCpu = pCpu->idCpu;
2412
2413	if (pVCpu->hwaccm.s.fForceTLBFlush)
2414	{
2415	if (fNewASID)
2416	{
2417	++pCpu->uCurrentASID;
2418	if (pCpu->uCurrentASID >= pVM->hwaccm.s.uMaxASID)
2419	{
2420	pCpu->uCurrentASID = 1; /* start at 1; host uses 0 */
2421	pCpu->cTLBFlushes++;
2422	pCpu->fFlushASIDBeforeUse = true;
2423	}
2424
2425	pVCpu->hwaccm.s.uCurrentASID = pCpu->uCurrentASID;
2426	if (pCpu->fFlushASIDBeforeUse)
2427	{
2428	hmR0VmxFlushVPID(pVM, pVCpu, pVM->hwaccm.s.vmx.enmFlushVPID, 0 /* GCPtr */);
2429	#ifdef VBOX_WITH_STATISTICS
2430	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatFlushASID);
2431	#endif
2432	}
2433	}
2434	else
2435	{
2436	if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVVPID_CAPS_SINGLE_CONTEXT)
2437	hmR0VmxFlushVPID(pVM, pVCpu, VMX_FLUSH_VPID_SINGLE_CONTEXT, 0 /* GCPtr */);
2438	else
2439	hmR0VmxFlushEPT(pVM, pVCpu, pVM->hwaccm.s.vmx.enmFlushEPT);
2440
2441	#ifdef VBOX_WITH_STATISTICS
2442	/*
2443	* This is not terribly accurate (i.e. we don't have any StatFlushEPT counter). We currently count these
2444	* as ASID flushes too, better than including them under StatFlushTLBWorldSwitch.
2445	*/
2446	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatFlushASID);
2447	#endif
2448	}
2449
2450	pVCpu->hwaccm.s.cTLBFlushes = pCpu->cTLBFlushes;
2451	pVCpu->hwaccm.s.fForceTLBFlush = false;
2452	}
2453	else
2454	{
2455	AssertMsg(pVCpu->hwaccm.s.uCurrentASID && pCpu->uCurrentASID, ("hwaccm uCurrentASID=%lu cpu uCurrentASID=%lu\n",
2456	pVCpu->hwaccm.s.uCurrentASID, pCpu->uCurrentASID));
2457
2458	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
2459	* not be executed. See hwaccmQueueInvlPage() where it is commented
2460	* out. Support individual entry flushing someday. */
2461	if (VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
2462	{
2463	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatTlbShootdown);
2464
2465	/*
2466	* Flush individual guest entries using VPID from the TLB or as little as possible with EPT
2467	* as supported by the CPU.
2468	*/
2469	if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVVPID_CAPS_INDIV_ADDR)
2470	{
2471	for (unsigned i = 0; i < pVCpu->hwaccm.s.TlbShootdown.cPages; i++)
2472	hmR0VmxFlushVPID(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, pVCpu->hwaccm.s.TlbShootdown.aPages[i]);
2473	}
2474	else
2475	hmR0VmxFlushEPT(pVM, pVCpu, pVM->hwaccm.s.vmx.enmFlushEPT);
2476	}
2477	else
2478	{
2479	#ifdef VBOX_WITH_STATISTICS
2480	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatNoFlushTLBWorldSwitch);
2481	#endif
2482	}
2483	}
2484	pVCpu->hwaccm.s.TlbShootdown.cPages = 0;
2485	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
2486
2487	AssertMsg(pVCpu->hwaccm.s.cTLBFlushes == pCpu->cTLBFlushes,
2488	("Flush count mismatch for cpu %d (%x vs %x)\n", pCpu->idCpu, pVCpu->hwaccm.s.cTLBFlushes, pCpu->cTLBFlushes));
2489	AssertMsg(pCpu->uCurrentASID >= 1 && pCpu->uCurrentASID < pVM->hwaccm.s.uMaxASID,
2490	("cpu%d uCurrentASID = %x\n", pCpu->idCpu, pCpu->uCurrentASID));
2491	AssertMsg(pVCpu->hwaccm.s.uCurrentASID >= 1 && pVCpu->hwaccm.s.uCurrentASID < pVM->hwaccm.s.uMaxASID,
2492	("cpu%d VM uCurrentASID = %x\n", pCpu->idCpu, pVCpu->hwaccm.s.uCurrentASID));
2493
2494	/* Update VMCS with the VPID. */
2495	int rc = VMXWriteVMCS(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hwaccm.s.uCurrentASID);
2496	AssertRC(rc);
2497	}
2498
2499
2500	/**
2501	* Setup the tagged TLB for EPT only.
2502	*
2503	* @returns VBox status code.
2504	* @param pVM Pointer to the VM.
2505	* @param pVCpu Pointer to the VMCPU.
2506	*/
2507	static DECLCALLBACK(void) hmR0VmxSetupTLBEPT(PVM pVM, PVMCPU pVCpu)
2508	{
2509	PHMGLOBLCPUINFO pCpu;
2510
2511	Assert(pVM->hwaccm.s.fNestedPaging);
2512	Assert(!pVM->hwaccm.s.vmx.fVPID);
2513
2514	pCpu = HWACCMR0GetCurrentCpu();
2515
2516	/*
2517	* Force a TLB flush for the first world switch if the current CPU differs from the one we ran on last
2518	* This can happen both for start & resume due to long jumps back to ring-3.
2519	* If the TLB flush count shouldn't really change in this EPT-only case.
2520	*/
2521	if ( pVCpu->hwaccm.s.idLastCpu != pCpu->idCpu
2522	\|\| pVCpu->hwaccm.s.cTLBFlushes != pCpu->cTLBFlushes)
2523	{
2524	pVCpu->hwaccm.s.fForceTLBFlush = true;
2525	}
2526
2527	/*
2528	* Check for explicit TLB shootdown flushes.
2529	*/
2530	if (VMCPU_FF_TESTANDCLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2531	pVCpu->hwaccm.s.fForceTLBFlush = true;
2532
2533	pVCpu->hwaccm.s.idLastCpu = pCpu->idCpu;
2534	pVCpu->hwaccm.s.cTLBFlushes = pCpu->cTLBFlushes;
2535
2536	if (pVCpu->hwaccm.s.fForceTLBFlush)
2537	hmR0VmxFlushEPT(pVM, pVCpu, pVM->hwaccm.s.vmx.enmFlushEPT);
2538	else
2539	{
2540	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
2541	* not be executed. See hwaccmQueueInvlPage() where it is commented
2542	* out. Support individual entry flushing someday. */
2543	if (VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
2544	{
2545	/*
2546	* We cannot flush individual entries without VPID support. Flush using EPT.
2547	*/
2548	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatTlbShootdown);
2549	hmR0VmxFlushEPT(pVM, pVCpu, pVM->hwaccm.s.vmx.enmFlushEPT);
2550	}
2551	}
2552	pVCpu->hwaccm.s.TlbShootdown.cPages= 0;
2553	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
2554
2555	#ifdef VBOX_WITH_STATISTICS
2556	if (pVCpu->hwaccm.s.fForceTLBFlush)
2557	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatFlushTLBWorldSwitch);
2558	else
2559	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatNoFlushTLBWorldSwitch);
2560	#endif
2561	}
2562
2563
2564	/**
2565	* Setup the tagged TLB for VPID.
2566	*
2567	* @returns VBox status code.
2568	* @param pVM Pointer to the VM.
2569	* @param pVCpu Pointer to the VMCPU.
2570	*/
2571	static DECLCALLBACK(void) hmR0VmxSetupTLBVPID(PVM pVM, PVMCPU pVCpu)
2572	{
2573	PHMGLOBLCPUINFO pCpu;
2574
2575	Assert(pVM->hwaccm.s.vmx.fVPID);
2576	Assert(!pVM->hwaccm.s.fNestedPaging);
2577
2578	pCpu = HWACCMR0GetCurrentCpu();
2579
2580	/*
2581	* Force a TLB flush for the first world switch if the current CPU differs from the one we ran on last
2582	* This can happen both for start & resume due to long jumps back to ring-3.
2583	* If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB,
2584	* so we cannot reuse the current ASID anymore.
2585	*/
2586	if ( pVCpu->hwaccm.s.idLastCpu != pCpu->idCpu
2587	\|\| pVCpu->hwaccm.s.cTLBFlushes != pCpu->cTLBFlushes)
2588	{
2589	/* Force a TLB flush on VM entry. */
2590	pVCpu->hwaccm.s.fForceTLBFlush = true;
2591	}
2592
2593	/*
2594	* Check for explicit TLB shootdown flushes.
2595	*/
2596	if (VMCPU_FF_TESTANDCLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2597	pVCpu->hwaccm.s.fForceTLBFlush = true;
2598
2599	pVCpu->hwaccm.s.idLastCpu = pCpu->idCpu;
2600
2601	if (pVCpu->hwaccm.s.fForceTLBFlush)
2602	{
2603	++pCpu->uCurrentASID;
2604	if (pCpu->uCurrentASID >= pVM->hwaccm.s.uMaxASID)
2605	{
2606	pCpu->uCurrentASID = 1; /* start at 1; host uses 0 */
2607	pCpu->cTLBFlushes++;
2608	pCpu->fFlushASIDBeforeUse = true;
2609	}
2610	else
2611	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatFlushASID);
2612
2613	pVCpu->hwaccm.s.fForceTLBFlush = false;
2614	pVCpu->hwaccm.s.cTLBFlushes = pCpu->cTLBFlushes;
2615	pVCpu->hwaccm.s.uCurrentASID = pCpu->uCurrentASID;
2616	if (pCpu->fFlushASIDBeforeUse)
2617	hmR0VmxFlushVPID(pVM, pVCpu, pVM->hwaccm.s.vmx.enmFlushVPID, 0 /* GCPtr */);
2618	}
2619	else
2620	{
2621	Assert(pVCpu->hwaccm.s.uCurrentASID && pCpu->uCurrentASID);
2622
2623	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
2624	* not be executed. See hwaccmQueueInvlPage() where it is commented
2625	* out. Support individual entry flushing someday. */
2626	if (VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
2627	{
2628	/*
2629	* Flush individual guest entries using VPID from the TLB or as little as possible with EPT
2630	* as supported by the CPU.
2631	*/
2632	if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVVPID_CAPS_INDIV_ADDR)
2633	{
2634	for (unsigned i = 0; i < pVCpu->hwaccm.s.TlbShootdown.cPages; i++)
2635	hmR0VmxFlushVPID(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, pVCpu->hwaccm.s.TlbShootdown.aPages[i]);
2636	}
2637	else
2638	hmR0VmxFlushVPID(pVM, pVCpu, pVM->hwaccm.s.vmx.enmFlushVPID, 0 /* GCPtr */);
2639	}
2640	}
2641	pVCpu->hwaccm.s.TlbShootdown.cPages = 0;
2642	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
2643
2644	AssertMsg(pVCpu->hwaccm.s.cTLBFlushes == pCpu->cTLBFlushes, ("Flush count mismatch for cpu %d (%x vs %x)\n", pCpu->idCpu, pVCpu->hwaccm.s.cTLBFlushes, pCpu->cTLBFlushes));
2645	AssertMsg(pCpu->uCurrentASID >= 1 && pCpu->uCurrentASID < pVM->hwaccm.s.uMaxASID, ("cpu%d uCurrentASID = %x\n", pCpu->idCpu, pCpu->uCurrentASID));
2646	AssertMsg(pVCpu->hwaccm.s.uCurrentASID >= 1 && pVCpu->hwaccm.s.uCurrentASID < pVM->hwaccm.s.uMaxASID, ("cpu%d VM uCurrentASID = %x\n", pCpu->idCpu, pVCpu->hwaccm.s.uCurrentASID));
2647
2648	int rc = VMXWriteVMCS(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hwaccm.s.uCurrentASID);
2649	AssertRC(rc);
2650
2651	# ifdef VBOX_WITH_STATISTICS
2652	if (pVCpu->hwaccm.s.fForceTLBFlush)
2653	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatFlushTLBWorldSwitch);
2654	else
2655	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatNoFlushTLBWorldSwitch);
2656	# endif
2657	}
2658
2659
2660	/**
2661	* Runs guest code in a VT-x VM.
2662	*
2663	* @returns VBox status code.
2664	* @param pVM Pointer to the VM.
2665	* @param pVCpu Pointer to the VMCPU.
2666	* @param pCtx Pointer to the guest CPU context.
2667	*/
2668	VMMR0DECL(int) VMXR0RunGuestCode(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
2669	{
2670	STAM_PROFILE_ADV_START(&pVCpu->hwaccm.s.StatEntry, x);
2671	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hwaccm.s.StatExit1);
2672	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hwaccm.s.StatExit2);
2673
2674	VBOXSTRICTRC rc = VINF_SUCCESS;
2675	int rc2;
2676	RTGCUINTREG val;
2677	RTGCUINTREG exitReason = (RTGCUINTREG)VMX_EXIT_INVALID;
2678	RTGCUINTREG instrError, cbInstr;
2679	RTGCUINTPTR exitQualification = 0;
2680	RTGCUINTPTR intInfo = 0; /* shut up buggy gcc 4 */
2681	RTGCUINTPTR errCode, instrInfo;
2682	bool fSetupTPRCaching = false;
2683	uint64_t u64OldLSTAR = 0;
2684	uint8_t u8LastTPR = 0;
2685	RTCCUINTREG uOldEFlags = ~(RTCCUINTREG)0;
2686	unsigned cResume = 0;
2687	#ifdef VBOX_STRICT
2688	RTCPUID idCpuCheck;
2689	bool fWasInLongMode = false;
2690	#endif
2691	#ifdef VBOX_HIGH_RES_TIMERS_HACK_IN_RING0
2692	uint64_t u64LastTime = RTTimeMilliTS();
2693	#endif
2694
2695	Assert(!(pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC) \|\| (pVCpu->hwaccm.s.vmx.pbVAPIC && pVM->hwaccm.s.vmx.pAPIC));
2696
2697	/*
2698	* Check if we need to use TPR shadowing.
2699	*/
2700	if ( CPUMIsGuestInLongModeEx(pCtx)
2701	\|\| ( ((pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC) \|\| pVM->hwaccm.s.fTRPPatchingAllowed)
2702	&& pVM->hwaccm.s.fHasIoApic)
2703	)
2704	{
2705	fSetupTPRCaching = true;
2706	}
2707
2708	Log2(("\nE"));
2709
2710	#ifdef VBOX_STRICT
2711	{
2712	RTCCUINTREG val2;
2713
2714	rc2 = VMXReadVMCS(VMX_VMCS_CTRL_PIN_EXEC_CONTROLS, &val2);
2715	AssertRC(rc2);
2716	Log2(("VMX_VMCS_CTRL_PIN_EXEC_CONTROLS = %08x\n", val2));
2717
2718	/* allowed zero */
2719	if ((val2 & pVM->hwaccm.s.vmx.msr.vmx_pin_ctls.n.disallowed0) != pVM->hwaccm.s.vmx.msr.vmx_pin_ctls.n.disallowed0)
2720	Log(("Invalid VMX_VMCS_CTRL_PIN_EXEC_CONTROLS: zero\n"));
2721
2722	/* allowed one */
2723	if ((val2 & ~pVM->hwaccm.s.vmx.msr.vmx_pin_ctls.n.allowed1) != 0)
2724	Log(("Invalid VMX_VMCS_CTRL_PIN_EXEC_CONTROLS: one\n"));
2725
2726	rc2 = VMXReadVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, &val2);
2727	AssertRC(rc2);
2728	Log2(("VMX_VMCS_CTRL_PROC_EXEC_CONTROLS = %08x\n", val2));
2729
2730	/*
2731	* Must be set according to the MSR, but can be cleared if nested paging is used.
2732	*/
2733	if (pVM->hwaccm.s.fNestedPaging)
2734	{
2735	val2 \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INVLPG_EXIT
2736	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
2737	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT;
2738	}
2739
2740	/* allowed zero */
2741	if ((val2 & pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.disallowed0) != pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.disallowed0)
2742	Log(("Invalid VMX_VMCS_CTRL_PROC_EXEC_CONTROLS: zero\n"));
2743
2744	/* allowed one */
2745	if ((val2 & ~pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1) != 0)
2746	Log(("Invalid VMX_VMCS_CTRL_PROC_EXEC_CONTROLS: one\n"));
2747
2748	rc2 = VMXReadVMCS(VMX_VMCS_CTRL_ENTRY_CONTROLS, &val2);
2749	AssertRC(rc2);
2750	Log2(("VMX_VMCS_CTRL_ENTRY_CONTROLS = %08x\n", val2));
2751
2752	/* allowed zero */
2753	if ((val2 & pVM->hwaccm.s.vmx.msr.vmx_entry.n.disallowed0) != pVM->hwaccm.s.vmx.msr.vmx_entry.n.disallowed0)
2754	Log(("Invalid VMX_VMCS_CTRL_ENTRY_CONTROLS: zero\n"));
2755
2756	/* allowed one */
2757	if ((val2 & ~pVM->hwaccm.s.vmx.msr.vmx_entry.n.allowed1) != 0)
2758	Log(("Invalid VMX_VMCS_CTRL_ENTRY_CONTROLS: one\n"));
2759
2760	rc2 = VMXReadVMCS(VMX_VMCS_CTRL_EXIT_CONTROLS, &val2);
2761	AssertRC(rc2);
2762	Log2(("VMX_VMCS_CTRL_EXIT_CONTROLS = %08x\n", val2));
2763
2764	/* allowed zero */
2765	if ((val2 & pVM->hwaccm.s.vmx.msr.vmx_exit.n.disallowed0) != pVM->hwaccm.s.vmx.msr.vmx_exit.n.disallowed0)
2766	Log(("Invalid VMX_VMCS_CTRL_EXIT_CONTROLS: zero\n"));
2767
2768	/* allowed one */
2769	if ((val2 & ~pVM->hwaccm.s.vmx.msr.vmx_exit.n.allowed1) != 0)
2770	Log(("Invalid VMX_VMCS_CTRL_EXIT_CONTROLS: one\n"));
2771	}
2772	fWasInLongMode = CPUMIsGuestInLongModeEx(pCtx);
2773	#endif /* VBOX_STRICT */
2774
2775	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
2776	pVCpu->hwaccm.s.vmx.VMCSCache.u64TimeEntry = RTTimeNanoTS();
2777	#endif
2778
2779	/*
2780	* We can jump to this point to resume execution after determining that a VM-exit is innocent.
2781	*/
2782	ResumeExecution:
2783	if (!STAM_REL_PROFILE_ADV_IS_RUNNING(&pVCpu->hwaccm.s.StatEntry))
2784	STAM_REL_PROFILE_ADV_STOP_START(&pVCpu->hwaccm.s.StatExit2, &pVCpu->hwaccm.s.StatEntry, x);
2785	AssertMsg(pVCpu->hwaccm.s.idEnteredCpu == RTMpCpuId(),
2786	("Expected %d, I'm %d; cResume=%d exitReason=%RGv exitQualification=%RGv\n",
2787	(int)pVCpu->hwaccm.s.idEnteredCpu, (int)RTMpCpuId(), cResume, exitReason, exitQualification));
2788	Assert(!HWACCMR0SuspendPending());
2789	/* Not allowed to switch modes without reloading the host state (32->64 switcher)!! */
2790	Assert(fWasInLongMode == CPUMIsGuestInLongModeEx(pCtx));
2791
2792	/*
2793	* Safety precaution; looping for too long here can have a very bad effect on the host.
2794	*/
2795	if (RT_UNLIKELY(++cResume > pVM->hwaccm.s.cMaxResumeLoops))
2796	{
2797	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitMaxResume);
2798	rc = VINF_EM_RAW_INTERRUPT;
2799	goto end;
2800	}
2801
2802	/*
2803	* Check for IRQ inhibition due to instruction fusing (sti, mov ss).
2804	*/
2805	if (VMCPU_FF_ISSET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
2806	{
2807	Log(("VM_FF_INHIBIT_INTERRUPTS at %RGv successor %RGv\n", (RTGCPTR)pCtx->rip, EMGetInhibitInterruptsPC(pVCpu)));
2808	if (pCtx->rip != EMGetInhibitInterruptsPC(pVCpu))
2809	{
2810	/*
2811	* Note: we intentionally don't clear VM_FF_INHIBIT_INTERRUPTS here.
2812	* Before we are able to execute this instruction in raw mode (iret to guest code) an external interrupt might
2813	* force a world switch again. Possibly allowing a guest interrupt to be dispatched in the process. This could
2814	* break the guest. Sounds very unlikely, but such timing sensitive problems are not as rare as you might think.
2815	*/
2816	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
2817	/* Irq inhibition is no longer active; clear the corresponding VMX state. */
2818	rc2 = VMXWriteVMCS(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, 0);
2819	AssertRC(rc2);
2820	}
2821	}
2822	else
2823	{
2824	/* Irq inhibition is no longer active; clear the corresponding VMX state. */
2825	rc2 = VMXWriteVMCS(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, 0);
2826	AssertRC(rc2);
2827	}
2828
2829	#ifdef VBOX_HIGH_RES_TIMERS_HACK_IN_RING0
2830	if (RT_UNLIKELY((cResume & 0xf) == 0))
2831	{
2832	uint64_t u64CurTime = RTTimeMilliTS();
2833
2834	if (RT_UNLIKELY(u64CurTime > u64LastTime))
2835	{
2836	u64LastTime = u64CurTime;
2837	TMTimerPollVoid(pVM, pVCpu);
2838	}
2839	}
2840	#endif
2841
2842	/*
2843	* Check for pending actions that force us to go back to ring-3.
2844	*/
2845	if ( VM_FF_ISPENDING(pVM, VM_FF_HWACCM_TO_R3_MASK \| VM_FF_REQUEST \| VM_FF_PGM_POOL_FLUSH_PENDING \| VM_FF_PDM_DMA)
2846	\|\| VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_HWACCM_TO_R3_MASK \| VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL \| VMCPU_FF_REQUEST))
2847	{
2848	/* Check if a sync operation is pending. */
2849	if (VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
2850	{
2851	rc = PGMSyncCR3(pVCpu, pCtx->cr0, pCtx->cr3, pCtx->cr4, VMCPU_FF_ISSET(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
2852	if (rc != VINF_SUCCESS)
2853	{
2854	AssertRC(VBOXSTRICTRC_VAL(rc));
2855	Log(("Pending pool sync is forcing us back to ring 3; rc=%d\n", VBOXSTRICTRC_VAL(rc)));
2856	goto end;
2857	}
2858	}
2859
2860	#ifdef DEBUG
2861	/* Intercept X86_XCPT_DB if stepping is enabled */
2862	if (!DBGFIsStepping(pVCpu))
2863	#endif
2864	{
2865	if ( VM_FF_ISPENDING(pVM, VM_FF_HWACCM_TO_R3_MASK)
2866	\|\| VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_HWACCM_TO_R3_MASK))
2867	{
2868	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatSwitchToR3);
2869	rc = RT_UNLIKELY(VM_FF_ISPENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
2870	goto end;
2871	}
2872	}
2873
2874	/* Pending request packets might contain actions that need immediate attention, such as pending hardware interrupts. */
2875	if ( VM_FF_ISPENDING(pVM, VM_FF_REQUEST)
2876	\|\| VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_REQUEST))
2877	{
2878	rc = VINF_EM_PENDING_REQUEST;
2879	goto end;
2880	}
2881
2882	/* Check if a pgm pool flush is in progress. */
2883	if (VM_FF_ISPENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
2884	{
2885	rc = VINF_PGM_POOL_FLUSH_PENDING;
2886	goto end;
2887	}
2888
2889	/* Check if DMA work is pending (2nd+ run). */
2890	if (VM_FF_ISPENDING(pVM, VM_FF_PDM_DMA) && cResume > 1)
2891	{
2892	rc = VINF_EM_RAW_TO_R3;
2893	goto end;
2894	}
2895	}
2896
2897	#ifdef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
2898	/*
2899	* Exit to ring-3 preemption/work is pending.
2900	*
2901	* Interrupts are disabled before the call to make sure we don't miss any interrupt
2902	* that would flag preemption (IPI, timer tick, ++). (Would've been nice to do this
2903	* further down, but hmR0VmxCheckPendingInterrupt makes that impossible.)
2904	*
2905	* Note! Interrupts must be disabled done before we check for TLB flushes; TLB
2906	* shootdowns rely on this.
2907	*/
2908	uOldEFlags = ASMIntDisableFlags();
2909	if (RTThreadPreemptIsPending(NIL_RTTHREAD))
2910	{
2911	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitPreemptPending);
2912	rc = VINF_EM_RAW_INTERRUPT;
2913	goto end;
2914	}
2915	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
2916	#endif
2917
2918	/*
2919	* When external interrupts are pending, we should exit the VM when IF is et.
2920	* Note: After VM_FF_INHIBIT_INTERRUPTS check!
2921	*/
2922	rc = hmR0VmxCheckPendingInterrupt(pVM, pVCpu, pCtx);
2923	if (RT_FAILURE(rc))
2924	goto end;
2925
2926	/** @todo check timers?? */
2927
2928	/*
2929	* TPR caching using CR8 is only available in 64-bit mode.
2930	* Note: The 32-bit exception for AMD (X86_CPUID_AMD_FEATURE_ECX_CR8L), but this appears missing in Intel CPUs.
2931	* Note: We can't do this in LoadGuestState() as PDMApicGetTPR can jump back to ring-3 (lock)!! (no longer true) .
2932	*/
2933	/** @todo query and update the TPR only when it could have been changed (mmio
2934	* access & wrsmr (x2apic) */
2935	if (fSetupTPRCaching)
2936	{
2937	/* TPR caching in CR8 */
2938	bool fPending;
2939
2940	rc2 = PDMApicGetTPR(pVCpu, &u8LastTPR, &fPending);
2941	AssertRC(rc2);
2942	/* The TPR can be found at offset 0x80 in the APIC mmio page. */
2943	pVCpu->hwaccm.s.vmx.pbVAPIC[0x80] = u8LastTPR;
2944
2945	/*
2946	* Two options here:
2947	* - external interrupt pending, but masked by the TPR value.
2948	* -> a CR8 update that lower the current TPR value should cause an exit
2949	* - no pending interrupts
2950	* -> We don't need to be explicitely notified. There are enough world switches for detecting pending interrupts.
2951	*/
2952	rc = VMXWriteVMCS(VMX_VMCS_CTRL_TPR_THRESHOLD, (fPending) ? (u8LastTPR >> 4) : 0); /* cr8 bits 3-0 correspond to bits 7-4 of the task priority mmio register. */
2953	AssertRC(VBOXSTRICTRC_VAL(rc));
2954
2955	if (pVM->hwaccm.s.fTPRPatchingActive)
2956	{
2957	Assert(!CPUMIsGuestInLongModeEx(pCtx));
2958	/* Our patch code uses LSTAR for TPR caching. */
2959	pCtx->msrLSTAR = u8LastTPR;
2960
2961	if (fPending)
2962	{
2963	/* A TPR change could activate a pending interrupt, so catch lstar writes. */
2964	hmR0VmxSetMSRPermission(pVCpu, MSR_K8_LSTAR, true, false);
2965	}
2966	else
2967	{
2968	/*
2969	* No interrupts are pending, so we don't need to be explicitely notified.
2970	* There are enough world switches for detecting pending interrupts.
2971	*/
2972	hmR0VmxSetMSRPermission(pVCpu, MSR_K8_LSTAR, true, true);
2973	}
2974	}
2975	}
2976
2977	#ifdef LOG_ENABLED
2978	if ( pVM->hwaccm.s.fNestedPaging
2979	\|\| pVM->hwaccm.s.vmx.fVPID)
2980	{
2981	PHMGLOBLCPUINFO pCpu = HWACCMR0GetCurrentCpu();
2982	if ( pVCpu->hwaccm.s.idLastCpu != pCpu->idCpu
2983	\|\| pVCpu->hwaccm.s.cTLBFlushes != pCpu->cTLBFlushes)
2984	{
2985	if (pVCpu->hwaccm.s.idLastCpu != pCpu->idCpu)
2986	LogFlow(("Force TLB flush due to rescheduling to a different cpu (%d vs %d)\n", pVCpu->hwaccm.s.idLastCpu, pCpu->idCpu));
2987	else
2988	LogFlow(("Force TLB flush due to changed TLB flush count (%x vs %x)\n", pVCpu->hwaccm.s.cTLBFlushes, pCpu->cTLBFlushes));
2989	}
2990	else if (VMCPU_FF_ISSET(pVCpu, VMCPU_FF_TLB_FLUSH))
2991	LogFlow(("Manual TLB flush\n"));
2992	}
2993	#endif
2994	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
2995	PGMRZDynMapFlushAutoSet(pVCpu);
2996	#endif
2997
2998	/*
2999	* NOTE: DO NOT DO ANYTHING AFTER THIS POINT THAT MIGHT JUMP BACK TO RING-3!
3000	* (until the actual world switch)
3001	*/
3002	#ifdef VBOX_STRICT
3003	idCpuCheck = RTMpCpuId();
3004	#endif
3005	#ifdef LOG_ENABLED
3006	VMMR0LogFlushDisable(pVCpu);
3007	#endif
3008
3009	/*
3010	* Save the host state first.
3011	*/
3012	if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_HOST_CONTEXT)
3013	{
3014	rc = VMXR0SaveHostState(pVM, pVCpu);
3015	if (RT_UNLIKELY(rc != VINF_SUCCESS))
3016	{
3017	VMMR0LogFlushEnable(pVCpu);
3018	goto end;
3019	}
3020	}
3021
3022	/*
3023	* Load the guest state.
3024	*/
3025	if (!pVCpu->hwaccm.s.fContextUseFlags)
3026	{
3027	VMXR0LoadMinimalGuestState(pVM, pVCpu, pCtx);
3028	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatLoadMinimal);
3029	}
3030	else
3031	{
3032	rc = VMXR0LoadGuestState(pVM, pVCpu, pCtx);
3033	if (RT_UNLIKELY(rc != VINF_SUCCESS))
3034	{
3035	VMMR0LogFlushEnable(pVCpu);
3036	goto end;
3037	}
3038	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatLoadFull);
3039	}
3040
3041	#ifndef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
3042	/*
3043	* Disable interrupts to make sure a poke will interrupt execution.
3044	* This must be done before we check for TLB flushes; TLB shootdowns rely on this.
3045	*/
3046	uOldEFlags = ASMIntDisableFlags();
3047	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
3048	#endif
3049
3050	/* Non-register state Guest Context */
3051	/** @todo change me according to cpu state */
3052	rc2 = VMXWriteVMCS(VMX_VMCS32_GUEST_ACTIVITY_STATE, VMX_CMS_GUEST_ACTIVITY_ACTIVE);
3053	AssertRC(rc2);
3054
3055	/* Set TLB flush state as checked until we return from the world switch. */
3056	ASMAtomicWriteBool(&pVCpu->hwaccm.s.fCheckedTLBFlush, true);
3057	/* Deal with tagged TLB setup and invalidation. */
3058	pVM->hwaccm.s.vmx.pfnSetupTaggedTLB(pVM, pVCpu);
3059
3060	/*
3061	* Manual save and restore:
3062	* - General purpose registers except RIP, RSP
3063	*
3064	* Trashed:
3065	* - CR2 (we don't care)
3066	* - LDTR (reset to 0)
3067	* - DRx (presumably not changed at all)
3068	* - DR7 (reset to 0x400)
3069	* - EFLAGS (reset to RT_BIT(1); not relevant)
3070	*/
3071
3072	/* All done! Let's start VM execution. */
3073	STAM_PROFILE_ADV_STOP_START(&pVCpu->hwaccm.s.StatEntry, &pVCpu->hwaccm.s.StatInGC, x);
3074	Assert(idCpuCheck == RTMpCpuId());
3075
3076	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
3077	pVCpu->hwaccm.s.vmx.VMCSCache.cResume = cResume;
3078	pVCpu->hwaccm.s.vmx.VMCSCache.u64TimeSwitch = RTTimeNanoTS();
3079	#endif
3080
3081	/*
3082	* Save the current TPR value in the LSTAR MSR so our patches can access it.
3083	*/
3084	if (pVM->hwaccm.s.fTPRPatchingActive)
3085	{
3086	Assert(pVM->hwaccm.s.fTPRPatchingActive);
3087	u64OldLSTAR = ASMRdMsr(MSR_K8_LSTAR);
3088	ASMWrMsr(MSR_K8_LSTAR, u8LastTPR);
3089	}
3090
3091	TMNotifyStartOfExecution(pVCpu);
3092	#ifdef VBOX_WITH_KERNEL_USING_XMM
3093	rc = hwaccmR0VMXStartVMWrapXMM(pVCpu->hwaccm.s.fResumeVM, pCtx, &pVCpu->hwaccm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hwaccm.s.vmx.pfnStartVM);
3094	#else
3095	rc = pVCpu->hwaccm.s.vmx.pfnStartVM(pVCpu->hwaccm.s.fResumeVM, pCtx, &pVCpu->hwaccm.s.vmx.VMCSCache, pVM, pVCpu);
3096	#endif
3097	ASMAtomicWriteBool(&pVCpu->hwaccm.s.fCheckedTLBFlush, false);
3098	ASMAtomicIncU32(&pVCpu->hwaccm.s.cWorldSwitchExits);
3099	/* Possibly the last TSC value seen by the guest (too high) (only when we're in TSC offset mode). */
3100	if (!(pVCpu->hwaccm.s.vmx.proc_ctls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT))
3101	{
3102	TMCpuTickSetLastSeen(pVCpu,
3103	ASMReadTSC() + pVCpu->hwaccm.s.vmx.u64TSCOffset - 0x400 /* guestimate of world switch overhead in clock ticks */);
3104	}
3105
3106	TMNotifyEndOfExecution(pVCpu);
3107	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED);
3108	Assert(!(ASMGetFlags() & X86_EFL_IF));
3109
3110	/*
3111	* Restore the host LSTAR MSR if the guest could have changed it.
3112	*/
3113	if (pVM->hwaccm.s.fTPRPatchingActive)
3114	{
3115	Assert(pVM->hwaccm.s.fTPRPatchingActive);
3116	pVCpu->hwaccm.s.vmx.pbVAPIC[0x80] = pCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR);
3117	ASMWrMsr(MSR_K8_LSTAR, u64OldLSTAR);
3118	}
3119
3120	STAM_PROFILE_ADV_STOP_START(&pVCpu->hwaccm.s.StatInGC, &pVCpu->hwaccm.s.StatExit1, x);
3121	ASMSetFlags(uOldEFlags);
3122	#ifdef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
3123	uOldEFlags = ~(RTCCUINTREG)0;
3124	#endif
3125
3126	AssertMsg(!pVCpu->hwaccm.s.vmx.VMCSCache.Write.cValidEntries, ("pVCpu->hwaccm.s.vmx.VMCSCache.Write.cValidEntries=%d\n", pVCpu->hwaccm.s.vmx.VMCSCache.Write.cValidEntries));
3127
3128	/* In case we execute a goto ResumeExecution later on. */
3129	pVCpu->hwaccm.s.fResumeVM = true;
3130	pVCpu->hwaccm.s.fForceTLBFlush = false;
3131
3132	/*
3133	* !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
3134	* IMPORTANT: WE CAN'T DO ANY LOGGING OR OPERATIONS THAT CAN DO A LONGJMP BACK TO RING 3 BEFORE WE'VE SYNCED BACK (MOST OF) THE GUEST STATE
3135	* !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
3136	*/
3137
3138	if (RT_UNLIKELY(rc != VINF_SUCCESS))
3139	{
3140	hmR0VmxReportWorldSwitchError(pVM, pVCpu, rc, pCtx);
3141	VMMR0LogFlushEnable(pVCpu);
3142	goto end;
3143	}
3144
3145	/* Success. Query the guest state and figure out what has happened. */
3146
3147	/* Investigate why there was a VM-exit. */
3148	rc2 = VMXReadCachedVMCS(VMX_VMCS32_RO_EXIT_REASON, &exitReason);
3149	STAM_COUNTER_INC(&pVCpu->hwaccm.s.paStatExitReasonR0[exitReason & MASK_EXITREASON_STAT]);
3150
3151	exitReason &= 0xffff; /* bit 0-15 contain the exit code. */
3152	rc2 \|= VMXReadCachedVMCS(VMX_VMCS32_RO_VM_INSTR_ERROR, &instrError);
3153	rc2 \|= VMXReadCachedVMCS(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &cbInstr);
3154	rc2 \|= VMXReadCachedVMCS(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &intInfo);
3155	/* might not be valid; depends on VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID. */
3156	rc2 \|= VMXReadCachedVMCS(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERRCODE, &errCode);
3157	rc2 \|= VMXReadCachedVMCS(VMX_VMCS32_RO_EXIT_INSTR_INFO, &instrInfo);
3158	rc2 \|= VMXReadCachedVMCS(VMX_VMCS_RO_EXIT_QUALIFICATION, &exitQualification);
3159	AssertRC(rc2);
3160
3161	/*
3162	* Sync back the guest state.
3163	*/
3164	rc2 = VMXR0SaveGuestState(pVM, pVCpu, pCtx);
3165	AssertRC(rc2);
3166
3167	/* Note! NOW IT'S SAFE FOR LOGGING! */
3168	VMMR0LogFlushEnable(pVCpu);
3169	Log2(("Raw exit reason %08x\n", exitReason));
3170	#if ARCH_BITS == 64 /* for the time being */
3171	VBOXVMM_R0_HMVMX_VMEXIT(pVCpu, pCtx, exitReason);
3172	#endif
3173
3174	/*
3175	* Check if an injected event was interrupted prematurely.
3176	*/
3177	rc2 = VMXReadCachedVMCS(VMX_VMCS32_RO_IDT_INFO, &val);
3178	AssertRC(rc2);
3179	pVCpu->hwaccm.s.Event.intInfo = VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(val);
3180	if ( VMX_EXIT_INTERRUPTION_INFO_VALID(pVCpu->hwaccm.s.Event.intInfo)
3181	/* Ignore 'int xx' as they'll be restarted anyway. */
3182	&& VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hwaccm.s.Event.intInfo) != VMX_EXIT_INTERRUPTION_INFO_TYPE_SW
3183	/* Ignore software exceptions (such as int3) as they'll reoccur when we restart the instruction anyway. */
3184	&& VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hwaccm.s.Event.intInfo) != VMX_EXIT_INTERRUPTION_INFO_TYPE_SWEXCPT)
3185	{
3186	Assert(!pVCpu->hwaccm.s.Event.fPending);
3187	pVCpu->hwaccm.s.Event.fPending = true;
3188	/* Error code present? */
3189	if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(pVCpu->hwaccm.s.Event.intInfo))
3190	{
3191	rc2 = VMXReadCachedVMCS(VMX_VMCS32_RO_IDT_ERRCODE, &val);
3192	AssertRC(rc2);
3193	pVCpu->hwaccm.s.Event.errCode = val;
3194	Log(("Pending inject %RX64 at %RGv exit=%08x intInfo=%08x exitQualification=%RGv pending error=%RX64\n",
3195	pVCpu->hwaccm.s.Event.intInfo, (RTGCPTR)pCtx->rip, exitReason, intInfo, exitQualification, val));
3196	}
3197	else
3198	{
3199	Log(("Pending inject %RX64 at %RGv exit=%08x intInfo=%08x exitQualification=%RGv\n", pVCpu->hwaccm.s.Event.intInfo,
3200	(RTGCPTR)pCtx->rip, exitReason, intInfo, exitQualification));
3201	pVCpu->hwaccm.s.Event.errCode = 0;
3202	}
3203	}
3204	#ifdef VBOX_STRICT
3205	else if ( VMX_EXIT_INTERRUPTION_INFO_VALID(pVCpu->hwaccm.s.Event.intInfo)
3206	/* Ignore software exceptions (such as int3) as they're reoccur when we restart the instruction anyway. */
3207	&& VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hwaccm.s.Event.intInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_SWEXCPT)
3208	{
3209	Log(("Ignore pending inject %RX64 at %RGv exit=%08x intInfo=%08x exitQualification=%RGv\n",
3210	pVCpu->hwaccm.s.Event.intInfo, (RTGCPTR)pCtx->rip, exitReason, intInfo, exitQualification));
3211	}
3212
3213	if (exitReason == VMX_EXIT_ERR_INVALID_GUEST_STATE)
3214	HWACCMDumpRegs(pVM, pVCpu, pCtx);
3215	#endif
3216
3217	Log2(("E%d: New EIP=%x:%RGv\n", (uint32_t)exitReason, pCtx->cs, (RTGCPTR)pCtx->rip));
3218	Log2(("Exit reason %d, exitQualification %RGv\n", (uint32_t)exitReason, exitQualification));
3219	Log2(("instrInfo=%d instrError=%d instr length=%d\n", (uint32_t)instrInfo, (uint32_t)instrError, (uint32_t)cbInstr));
3220	Log2(("Interruption error code %d\n", (uint32_t)errCode));
3221	Log2(("IntInfo = %08x\n", (uint32_t)intInfo));
3222
3223	/*
3224	* Sync back the TPR if it was changed.
3225	*/
3226	if ( fSetupTPRCaching
3227	&& u8LastTPR != pVCpu->hwaccm.s.vmx.pbVAPIC[0x80])
3228	{
3229	rc2 = PDMApicSetTPR(pVCpu, pVCpu->hwaccm.s.vmx.pbVAPIC[0x80]);
3230	AssertRC(rc2);
3231	}
3232
3233	#ifdef DBGFTRACE_ENABLED /** @todo DTrace later. */
3234	RTTraceBufAddMsgF(pVM->CTX_SUFF(hTraceBuf), "vmexit %08x %016RX64 at %04:%08RX64 %RX64",
3235	exitReason, (uint64_t)exitQualification, pCtx->cs, pCtx->rip, (uint64_t)intInfo);
3236	#endif
3237	STAM_PROFILE_ADV_STOP_START(&pVCpu->hwaccm.s.StatExit1, &pVCpu->hwaccm.s.StatExit2, x);
3238
3239	/* Some cases don't need a complete resync of the guest CPU state; handle them here. */
3240	Assert(rc == VINF_SUCCESS); /* might consider VERR_IPE_UNINITIALIZED_STATUS here later... */
3241	switch (exitReason)
3242	{
3243	case VMX_EXIT_EXCEPTION: /* 0 Exception or non-maskable interrupt (NMI). */
3244	case VMX_EXIT_EXTERNAL_IRQ: /* 1 External interrupt. */
3245	{
3246	uint32_t vector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(intInfo);
3247
3248	if (!VMX_EXIT_INTERRUPTION_INFO_VALID(intInfo))
3249	{
3250	Assert(exitReason == VMX_EXIT_EXTERNAL_IRQ);
3251	#if 0 //def VBOX_WITH_VMMR0_DISABLE_PREEMPTION
3252	if ( RTThreadPreemptIsPendingTrusty()
3253	&& !RTThreadPreemptIsPending(NIL_RTTHREAD))
3254	goto ResumeExecution;
3255	#endif
3256	/* External interrupt; leave to allow it to be dispatched again. */
3257	rc = VINF_EM_RAW_INTERRUPT;
3258	break;
3259	}
3260	STAM_PROFILE_ADV_START(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3261	switch (VMX_EXIT_INTERRUPTION_INFO_TYPE(intInfo))
3262	{
3263	case VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI: /* Non-maskable interrupt. */
3264	/* External interrupt; leave to allow it to be dispatched again. */
3265	rc = VINF_EM_RAW_INTERRUPT;
3266	break;
3267
3268	case VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT: /* External hardware interrupt. */
3269	AssertFailed(); /* can't come here; fails the first check. */
3270	break;
3271
3272	case VMX_EXIT_INTERRUPTION_INFO_TYPE_DBEXCPT: /* Unknown why we get this type for #DB */
3273	case VMX_EXIT_INTERRUPTION_INFO_TYPE_SWEXCPT: /* Software exception. (#BP or #OF) */
3274	Assert(vector == 1 \|\| vector == 3 \|\| vector == 4);
3275	/* no break */
3276	case VMX_EXIT_INTERRUPTION_INFO_TYPE_HWEXCPT: /* Hardware exception. */
3277	Log2(("Hardware/software interrupt %d\n", vector));
3278	switch (vector)
3279	{
3280	case X86_XCPT_NM:
3281	{
3282	Log(("#NM fault at %RGv error code %x\n", (RTGCPTR)pCtx->rip, errCode));
3283
3284	/** @todo don't intercept #NM exceptions anymore when we've activated the guest FPU state. */
3285	/* If we sync the FPU/XMM state on-demand, then we can continue execution as if nothing has happened. */
3286	rc = CPUMR0LoadGuestFPU(pVM, pVCpu, pCtx);
3287	if (rc == VINF_SUCCESS)
3288	{
3289	Assert(CPUMIsGuestFPUStateActive(pVCpu));
3290
3291	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitShadowNM);
3292
3293	/* Continue execution. */
3294	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_GUEST_CR0;
3295
3296	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3297	goto ResumeExecution;
3298	}
3299
3300	Log(("Forward #NM fault to the guest\n"));
3301	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestNM);
3302	rc2 = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo),
3303	cbInstr, 0);
3304	AssertRC(rc2);
3305	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3306	goto ResumeExecution;
3307	}
3308
3309	case X86_XCPT_PF: /* Page fault */
3310	{
3311	#ifdef VBOX_ALWAYS_TRAP_PF
3312	if (pVM->hwaccm.s.fNestedPaging)
3313	{
3314	/*
3315	* A genuine pagefault. Forward the trap to the guest by injecting the exception and resuming execution.
3316	*/
3317	Log(("Guest page fault at %RGv cr2=%RGv error code %RGv rsp=%RGv\n", (RTGCPTR)pCtx->rip, exitQualification,
3318	errCode, (RTGCPTR)pCtx->rsp));
3319
3320	Assert(CPUMIsGuestInPagedProtectedModeEx(pCtx));
3321
3322	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestPF);
3323
3324	/* Now we must update CR2. */
3325	pCtx->cr2 = exitQualification;
3326	rc2 = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo),
3327	cbInstr, errCode);
3328	AssertRC(rc2);
3329
3330	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3331	goto ResumeExecution;
3332	}
3333	#else
3334	Assert(!pVM->hwaccm.s.fNestedPaging);
3335	#endif
3336
3337	#ifdef VBOX_HWACCM_WITH_GUEST_PATCHING
3338	/* Shortcut for APIC TPR reads and writes; 32 bits guests only */
3339	if ( pVM->hwaccm.s.fTRPPatchingAllowed
3340	&& pVM->hwaccm.s.pGuestPatchMem
3341	&& (exitQualification & 0xfff) == 0x080
3342	&& !(errCode & X86_TRAP_PF_P) /* not present */
3343	&& CPUMGetGuestCPL(pVCpu, CPUMCTX2CORE(pCtx)) == 0
3344	&& !CPUMIsGuestInLongModeEx(pCtx)
3345	&& pVM->hwaccm.s.cPatches < RT_ELEMENTS(pVM->hwaccm.s.aPatches))
3346	{
3347	RTGCPHYS GCPhysApicBase, GCPhys;
3348	PDMApicGetBase(pVM, &GCPhysApicBase); /** @todo cache this */
3349	GCPhysApicBase &= PAGE_BASE_GC_MASK;
3350
3351	rc = PGMGstGetPage(pVCpu, (RTGCPTR)exitQualification, NULL, &GCPhys);
3352	if ( rc == VINF_SUCCESS
3353	&& GCPhys == GCPhysApicBase)
3354	{
3355	/* Only attempt to patch the instruction once. */
3356	PHWACCMTPRPATCH pPatch = (PHWACCMTPRPATCH)RTAvloU32Get(&pVM->hwaccm.s.PatchTree, (AVLOU32KEY)pCtx->eip);
3357	if (!pPatch)
3358	{
3359	rc = VINF_EM_HWACCM_PATCH_TPR_INSTR;
3360	break;
3361	}
3362	}
3363	}
3364	#endif
3365
3366	Log2(("Page fault at %RGv error code %x\n", exitQualification, errCode));
3367	/* Exit qualification contains the linear address of the page fault. */
3368	TRPMAssertTrap(pVCpu, X86_XCPT_PF, TRPM_TRAP);
3369	TRPMSetErrorCode(pVCpu, errCode);
3370	TRPMSetFaultAddress(pVCpu, exitQualification);
3371
3372	/* Shortcut for APIC TPR reads and writes. */
3373	if ( (exitQualification & 0xfff) == 0x080
3374	&& !(errCode & X86_TRAP_PF_P) /* not present */
3375	&& fSetupTPRCaching
3376	&& (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC))
3377	{
3378	RTGCPHYS GCPhysApicBase, GCPhys;
3379	PDMApicGetBase(pVM, &GCPhysApicBase); /* @todo cache this */
3380	GCPhysApicBase &= PAGE_BASE_GC_MASK;
3381
3382	rc = PGMGstGetPage(pVCpu, (RTGCPTR)exitQualification, NULL, &GCPhys);
3383	if ( rc == VINF_SUCCESS
3384	&& GCPhys == GCPhysApicBase)
3385	{
3386	Log(("Enable VT-x virtual APIC access filtering\n"));
3387	rc2 = IOMMMIOMapMMIOHCPage(pVM, GCPhysApicBase, pVM->hwaccm.s.vmx.pAPICPhys, X86_PTE_RW \| X86_PTE_P);
3388	AssertRC(rc2);
3389	}
3390	}
3391
3392	/* Forward it to our trap handler first, in case our shadow pages are out of sync. */
3393	rc = PGMTrap0eHandler(pVCpu, errCode, CPUMCTX2CORE(pCtx), (RTGCPTR)exitQualification);
3394	Log2(("PGMTrap0eHandler %RGv returned %Rrc\n", (RTGCPTR)pCtx->rip, VBOXSTRICTRC_VAL(rc)));
3395
3396	if (rc == VINF_SUCCESS)
3397	{ /* We've successfully synced our shadow pages, so let's just continue execution. */
3398	Log2(("Shadow page fault at %RGv cr2=%RGv error code %x\n", (RTGCPTR)pCtx->rip, exitQualification ,errCode));
3399	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitShadowPF);
3400
3401	TRPMResetTrap(pVCpu);
3402	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3403	goto ResumeExecution;
3404	}
3405	else if (rc == VINF_EM_RAW_GUEST_TRAP)
3406	{
3407	/*
3408	* A genuine pagefault. Forward the trap to the guest by injecting the exception and resuming execution.
3409	*/
3410	Log2(("Forward page fault to the guest\n"));
3411
3412	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestPF);
3413	/* The error code might have been changed. */
3414	errCode = TRPMGetErrorCode(pVCpu);
3415
3416	TRPMResetTrap(pVCpu);
3417
3418	/* Now we must update CR2. */
3419	pCtx->cr2 = exitQualification;
3420	rc2 = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo),
3421	cbInstr, errCode);
3422	AssertRC(rc2);
3423
3424	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3425	goto ResumeExecution;
3426	}
3427	#ifdef VBOX_STRICT
3428	if (rc != VINF_EM_RAW_EMULATE_INSTR && rc != VINF_EM_RAW_EMULATE_IO_BLOCK)
3429	Log2(("PGMTrap0eHandler failed with %d\n", VBOXSTRICTRC_VAL(rc)));
3430	#endif
3431	/* Need to go back to the recompiler to emulate the instruction. */
3432	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitShadowPFEM);
3433	TRPMResetTrap(pVCpu);
3434	break;
3435	}
3436
3437	case X86_XCPT_MF: /* Floating point exception. */
3438	{
3439	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestMF);
3440	if (!(pCtx->cr0 & X86_CR0_NE))
3441	{
3442	/* old style FPU error reporting needs some extra work. */
3443	/** @todo don't fall back to the recompiler, but do it manually. */
3444	rc = VINF_EM_RAW_EMULATE_INSTR;
3445	break;
3446	}
3447	Log(("Trap %x at %04X:%RGv\n", vector, pCtx->cs, (RTGCPTR)pCtx->rip));
3448	rc2 = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo),
3449	cbInstr, errCode);
3450	AssertRC(rc2);
3451
3452	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3453	goto ResumeExecution;
3454	}
3455
3456	case X86_XCPT_DB: /* Debug exception. */
3457	{
3458	uint64_t uDR6;
3459
3460	/*
3461	* DR6, DR7.GD and IA32_DEBUGCTL.LBR are not updated yet.
3462	*
3463	* Exit qualification bits:
3464	* 3:0 B0-B3 which breakpoint condition was met
3465	* 12:4 Reserved (0)
3466	* 13 BD - debug register access detected
3467	* 14 BS - single step execution or branch taken
3468	* 63:15 Reserved (0)
3469	*/
3470	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestDB);
3471
3472	/* Note that we don't support guest and host-initiated debugging at the same time. */
3473
3474	uDR6 = X86_DR6_INIT_VAL;
3475	uDR6 \|= (exitQualification & (X86_DR6_B0\|X86_DR6_B1\|X86_DR6_B2\|X86_DR6_B3\|X86_DR6_BD\|X86_DR6_BS));
3476	rc = DBGFRZTrap01Handler(pVM, pVCpu, CPUMCTX2CORE(pCtx), uDR6);
3477	if (rc == VINF_EM_RAW_GUEST_TRAP)
3478	{
3479	/* Update DR6 here. */
3480	pCtx->dr[6] = uDR6;
3481
3482	/* Resync DR6 if the debug state is active. */
3483	if (CPUMIsGuestDebugStateActive(pVCpu))
3484	ASMSetDR6(pCtx->dr[6]);
3485
3486	/* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
3487	pCtx->dr[7] &= ~X86_DR7_GD;
3488
3489	/* Paranoia. */
3490	pCtx->dr[7] &= 0xffffffff; /* upper 32 bits reserved */
3491	pCtx->dr[7] &= ~(RT_BIT(11) \| RT_BIT(12) \| RT_BIT(14) \| RT_BIT(15)); /* must be zero */
3492	pCtx->dr[7] \|= 0x400; /* must be one */
3493
3494	/* Resync DR7 */
3495	rc2 = VMXWriteVMCS64(VMX_VMCS64_GUEST_DR7, pCtx->dr[7]);
3496	AssertRC(rc2);
3497
3498	Log(("Trap %x (debug) at %RGv exit qualification %RX64 dr6=%x dr7=%x\n", vector, (RTGCPTR)pCtx->rip,
3499	exitQualification, (uint32_t)pCtx->dr[6], (uint32_t)pCtx->dr[7]));
3500	rc2 = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo),
3501	cbInstr, errCode);
3502	AssertRC(rc2);
3503
3504	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3505	goto ResumeExecution;
3506	}
3507	/* Return to ring 3 to deal with the debug exit code. */
3508	Log(("Debugger hardware BP at %04x:%RGv (rc=%Rrc)\n", pCtx->cs, pCtx->rip, VBOXSTRICTRC_VAL(rc)));
3509	break;
3510	}
3511
3512	case X86_XCPT_BP: /* Breakpoint. */
3513	{
3514	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestBP);
3515	rc = DBGFRZTrap03Handler(pVM, pVCpu, CPUMCTX2CORE(pCtx));
3516	if (rc == VINF_EM_RAW_GUEST_TRAP)
3517	{
3518	Log(("Guest #BP at %04x:%RGv\n", pCtx->cs, pCtx->rip));
3519	rc2 = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo),
3520	cbInstr, errCode);
3521	AssertRC(rc2);
3522	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3523	goto ResumeExecution;
3524	}
3525	if (rc == VINF_SUCCESS)
3526	{
3527	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3528	goto ResumeExecution;
3529	}
3530	Log(("Debugger BP at %04x:%RGv (rc=%Rrc)\n", pCtx->cs, pCtx->rip, VBOXSTRICTRC_VAL(rc)));
3531	break;
3532	}
3533
3534	case X86_XCPT_GP: /* General protection failure exception. */
3535	{
3536	uint32_t cbOp;
3537	PDISCPUSTATE pDis = &pVCpu->hwaccm.s.DisState;
3538
3539	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestGP);
3540	#ifdef VBOX_STRICT
3541	if ( !CPUMIsGuestInRealModeEx(pCtx)
3542	\|\| !pVM->hwaccm.s.vmx.pRealModeTSS)
3543	{
3544	Log(("Trap %x at %04X:%RGv errorCode=%RGv\n", vector, pCtx->cs, (RTGCPTR)pCtx->rip, errCode));
3545	rc2 = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo),
3546	cbInstr, errCode);
3547	AssertRC(rc2);
3548	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3549	goto ResumeExecution;
3550	}
3551	#endif
3552	Assert(CPUMIsGuestInRealModeEx(pCtx));
3553
3554	LogFlow(("Real mode X86_XCPT_GP instruction emulation at %x:%RGv\n", pCtx->cs, (RTGCPTR)pCtx->rip));
3555
3556	rc2 = EMInterpretDisasOne(pVM, pVCpu, CPUMCTX2CORE(pCtx), pDis, &cbOp);
3557	if (RT_SUCCESS(rc2))
3558	{
3559	bool fUpdateRIP = true;
3560
3561	rc = VINF_SUCCESS;
3562	Assert(cbOp == pDis->cbInstr);
3563	switch (pDis->pCurInstr->uOpcode)
3564	{
3565	case OP_CLI:
3566	pCtx->eflags.Bits.u1IF = 0;
3567	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitCli);
3568	break;
3569
3570	case OP_STI:
3571	pCtx->eflags.Bits.u1IF = 1;
3572	EMSetInhibitInterruptsPC(pVCpu, pCtx->rip + pDis->cbInstr);
3573	Assert(VMCPU_FF_ISSET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
3574	rc2 = VMXWriteVMCS(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE,
3575	VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
3576	AssertRC(rc2);
3577	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitSti);
3578	break;
3579
3580	case OP_HLT:
3581	fUpdateRIP = false;
3582	rc = VINF_EM_HALT;
3583	pCtx->rip += pDis->cbInstr;
3584	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitHlt);
3585	break;
3586
3587	case OP_POPF:
3588	{
3589	RTGCPTR GCPtrStack;
3590	uint32_t cbParm;
3591	uint32_t uMask;
3592	X86EFLAGS eflags;
3593
3594	if (pDis->fPrefix & DISPREFIX_OPSIZE)
3595	{
3596	cbParm = 4;
3597	uMask = 0xffffffff;
3598	}
3599	else
3600	{
3601	cbParm = 2;
3602	uMask = 0xffff;
3603	}
3604
3605	rc2 = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pCtx), pCtx->esp & uMask, 0, &GCPtrStack);
3606	if (RT_FAILURE(rc2))
3607	{
3608	rc = VERR_EM_INTERPRETER;
3609	break;
3610	}
3611	eflags.u = 0;
3612	rc2 = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &eflags.u, cbParm);
3613	if (RT_FAILURE(rc2))
3614	{
3615	rc = VERR_EM_INTERPRETER;
3616	break;
3617	}
3618	LogFlow(("POPF %x -> %RGv mask=%x\n", eflags.u, pCtx->rsp, uMask));
3619	pCtx->eflags.u = (pCtx->eflags.u & ~(X86_EFL_POPF_BITS & uMask)) \| (eflags.u & X86_EFL_POPF_BITS & uMask);
3620	/* RF cleared when popped in real mode; see pushf description in AMD manual. */
3621	pCtx->eflags.Bits.u1RF = 0;
3622	pCtx->esp += cbParm;
3623	pCtx->esp &= uMask;
3624
3625	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitPopf);
3626	break;
3627	}
3628
3629	case OP_PUSHF:
3630	{
3631	RTGCPTR GCPtrStack;
3632	uint32_t cbParm;
3633	uint32_t uMask;
3634	X86EFLAGS eflags;
3635
3636	if (pDis->fPrefix & DISPREFIX_OPSIZE)
3637	{
3638	cbParm = 4;
3639	uMask = 0xffffffff;
3640	}
3641	else
3642	{
3643	cbParm = 2;
3644	uMask = 0xffff;
3645	}
3646
3647	rc2 = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pCtx), (pCtx->esp - cbParm) & uMask, 0,
3648	&GCPtrStack);
3649	if (RT_FAILURE(rc2))
3650	{
3651	rc = VERR_EM_INTERPRETER;
3652	break;
3653	}
3654	eflags = pCtx->eflags;
3655	/* RF & VM cleared when pushed in real mode; see pushf description in AMD manual. */
3656	eflags.Bits.u1RF = 0;
3657	eflags.Bits.u1VM = 0;
3658
3659	rc2 = PGMPhysWrite(pVM, (RTGCPHYS)GCPtrStack, &eflags.u, cbParm);
3660	if (RT_FAILURE(rc2))
3661	{
3662	rc = VERR_EM_INTERPRETER;
3663	break;
3664	}
3665	LogFlow(("PUSHF %x -> %RGv\n", eflags.u, GCPtrStack));
3666	pCtx->esp -= cbParm;
3667	pCtx->esp &= uMask;
3668	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitPushf);
3669	break;
3670	}
3671
3672	case OP_IRET:
3673	{
3674	RTGCPTR GCPtrStack;
3675	uint32_t uMask = 0xffff;
3676	uint16_t aIretFrame[3];
3677
3678	if (pDis->fPrefix & (DISPREFIX_OPSIZE \| DISPREFIX_ADDRSIZE))
3679	{
3680	rc = VERR_EM_INTERPRETER;
3681	break;
3682	}
3683
3684	rc2 = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pCtx), pCtx->esp & uMask, 0, &GCPtrStack);
3685	if (RT_FAILURE(rc2))
3686	{
3687	rc = VERR_EM_INTERPRETER;
3688	break;
3689	}
3690	rc2 = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &aIretFrame[0], sizeof(aIretFrame));
3691	if (RT_FAILURE(rc2))
3692	{
3693	rc = VERR_EM_INTERPRETER;
3694	break;
3695	}
3696	pCtx->ip = aIretFrame[0];
3697	pCtx->cs = aIretFrame[1];
3698	pCtx->csHid.u64Base = pCtx->cs << 4;
3699	pCtx->eflags.u = (pCtx->eflags.u & ~(X86_EFL_POPF_BITS & uMask))
3700	\| (aIretFrame[2] & X86_EFL_POPF_BITS & uMask);
3701	pCtx->sp += sizeof(aIretFrame);
3702
3703	LogFlow(("iret to %04x:%x\n", pCtx->cs, pCtx->ip));
3704	fUpdateRIP = false;
3705	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitIret);
3706	break;
3707	}
3708
3709	case OP_INT:
3710	{
3711	uint32_t intInfo2;
3712
3713	LogFlow(("Realmode: INT %x\n", pDis->Param1.uValue & 0xff));
3714	intInfo2 = pDis->Param1.uValue & 0xff;
3715	intInfo2 \|= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
3716	intInfo2 \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
3717
3718	rc = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, intInfo2, cbOp, 0);
3719	AssertRC(VBOXSTRICTRC_VAL(rc));
3720	fUpdateRIP = false;
3721	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitInt);
3722	break;
3723	}
3724
3725	case OP_INTO:
3726	{
3727	if (pCtx->eflags.Bits.u1OF)
3728	{
3729	uint32_t intInfo2;
3730
3731	LogFlow(("Realmode: INTO\n"));
3732	intInfo2 = X86_XCPT_OF;
3733	intInfo2 \|= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
3734	intInfo2 \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
3735
3736	rc = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, intInfo2, cbOp, 0);
3737	AssertRC(VBOXSTRICTRC_VAL(rc));
3738	fUpdateRIP = false;
3739	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitInt);
3740	}
3741	break;
3742	}
3743
3744	case OP_INT3:
3745	{
3746	uint32_t intInfo2;
3747
3748	LogFlow(("Realmode: INT 3\n"));
3749	intInfo2 = 3;
3750	intInfo2 \|= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
3751	intInfo2 \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
3752
3753	rc = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, intInfo2, cbOp, 0);
3754	AssertRC(VBOXSTRICTRC_VAL(rc));
3755	fUpdateRIP = false;
3756	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitInt);
3757	break;
3758	}
3759
3760	default:
3761	rc = EMInterpretInstructionDisasState(pVCpu, pDis, CPUMCTX2CORE(pCtx), 0, EMCODETYPE_SUPERVISOR);
3762	fUpdateRIP = false;
3763	break;
3764	}
3765
3766	if (rc == VINF_SUCCESS)
3767	{
3768	if (fUpdateRIP)
3769	pCtx->rip += cbOp; /* Move on to the next instruction. */
3770
3771	/*
3772	* LIDT, LGDT can end up here. In the future CRx changes as well. Just reload the
3773	* whole context to be done with it.
3774	*/
3775	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_ALL;
3776
3777	/* Only resume if successful. */
3778	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3779	goto ResumeExecution;
3780	}
3781	}
3782	else
3783	rc = VERR_EM_INTERPRETER;
3784
3785	AssertMsg(rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_EM_HALT, ("Unexpected rc=%Rrc\n", VBOXSTRICTRC_VAL(rc)));
3786	break;
3787	}
3788
3789	#ifdef VBOX_STRICT
3790	case X86_XCPT_XF: /* SIMD exception. */
3791	case X86_XCPT_DE: /* Divide error. */
3792	case X86_XCPT_UD: /* Unknown opcode exception. */
3793	case X86_XCPT_SS: /* Stack segment exception. */
3794	case X86_XCPT_NP: /* Segment not present exception. */
3795	{
3796	switch(vector)
3797	{
3798	case X86_XCPT_DE:
3799	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestDE);
3800	break;
3801	case X86_XCPT_UD:
3802	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestUD);
3803	break;
3804	case X86_XCPT_SS:
3805	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestSS);
3806	break;
3807	case X86_XCPT_NP:
3808	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestNP);
3809	break;
3810	case X86_XCPT_XF:
3811	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestXF);
3812	break;
3813	}
3814
3815	Log(("Trap %x at %04X:%RGv\n", vector, pCtx->cs, (RTGCPTR)pCtx->rip));
3816	rc2 = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo),
3817	cbInstr, errCode);
3818	AssertRC(rc2);
3819
3820	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3821	goto ResumeExecution;
3822	}
3823	#endif
3824	default:
3825	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestXcpUnk);
3826	if ( CPUMIsGuestInRealModeEx(pCtx)
3827	&& pVM->hwaccm.s.vmx.pRealModeTSS)
3828	{
3829	Log(("Real Mode Trap %x at %04x:%04X error code %x\n", vector, pCtx->cs, pCtx->eip, errCode));
3830	rc = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo),
3831	cbInstr, errCode);
3832	AssertRC(VBOXSTRICTRC_VAL(rc)); /* Strict RC check below. */
3833
3834	/* Go back to ring-3 in case of a triple fault. */
3835	if ( vector == X86_XCPT_DF
3836	&& rc == VINF_EM_RESET)
3837	{
3838	break;
3839	}
3840
3841	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3842	goto ResumeExecution;
3843	}
3844	AssertMsgFailed(("Unexpected vm-exit caused by exception %x\n", vector));
3845	rc = VERR_VMX_UNEXPECTED_EXCEPTION;
3846	break;
3847	} /* switch (vector) */
3848
3849	break;
3850
3851	default:
3852	rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_CODE;
3853	AssertMsgFailed(("Unexpected interruption code %x\n", intInfo));
3854	break;
3855	}
3856
3857	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3858	break;
3859	}
3860
3861	/*
3862	* 48 EPT violation. An attemp to access memory with a guest-physical address was disallowed
3863	* by the configuration of the EPT paging structures.
3864	*/
3865	case VMX_EXIT_EPT_VIOLATION:
3866	{
3867	RTGCPHYS GCPhys;
3868
3869	Assert(pVM->hwaccm.s.fNestedPaging);
3870
3871	rc2 = VMXReadVMCS64(VMX_VMCS_EXIT_PHYS_ADDR_FULL, &GCPhys);
3872	AssertRC(rc2);
3873	Assert(((exitQualification >> 7) & 3) != 2);
3874
3875	/* Determine the kind of violation. */
3876	errCode = 0;
3877	if (exitQualification & VMX_EXIT_QUALIFICATION_EPT_INSTR_FETCH)
3878	errCode \|= X86_TRAP_PF_ID;
3879
3880	if (exitQualification & VMX_EXIT_QUALIFICATION_EPT_DATA_WRITE)
3881	errCode \|= X86_TRAP_PF_RW;
3882
3883	/* If the page is present, then it's a page level protection fault. */
3884	if (exitQualification & VMX_EXIT_QUALIFICATION_EPT_ENTRY_PRESENT)
3885	errCode \|= X86_TRAP_PF_P;
3886	else
3887	{
3888	/* Shortcut for APIC TPR reads and writes. */
3889	if ( (GCPhys & 0xfff) == 0x080
3890	&& GCPhys > 0x1000000 /* to skip VGA frame buffer accesses */
3891	&& fSetupTPRCaching
3892	&& (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC))
3893	{
3894	RTGCPHYS GCPhysApicBase;
3895	PDMApicGetBase(pVM, &GCPhysApicBase); /* @todo cache this */
3896	GCPhysApicBase &= PAGE_BASE_GC_MASK;
3897	if (GCPhys == GCPhysApicBase + 0x80)
3898	{
3899	Log(("Enable VT-x virtual APIC access filtering\n"));
3900	rc2 = IOMMMIOMapMMIOHCPage(pVM, GCPhysApicBase, pVM->hwaccm.s.vmx.pAPICPhys, X86_PTE_RW \| X86_PTE_P);
3901	AssertRC(rc2);
3902	}
3903	}
3904	}
3905	Log(("EPT Page fault %x at %RGp error code %x\n", (uint32_t)exitQualification, GCPhys, errCode));
3906
3907	/* GCPhys contains the guest physical address of the page fault. */
3908	TRPMAssertTrap(pVCpu, X86_XCPT_PF, TRPM_TRAP);
3909	TRPMSetErrorCode(pVCpu, errCode);
3910	TRPMSetFaultAddress(pVCpu, GCPhys);
3911
3912	/* Handle the pagefault trap for the nested shadow table. */
3913	rc = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, errCode, CPUMCTX2CORE(pCtx), GCPhys);
3914
3915	/*
3916	* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment below, @bugref{6043}.
3917	*/
3918	if ( rc == VINF_SUCCESS
3919	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
3920	\|\| rc == VERR_PAGE_NOT_PRESENT)
3921	{
3922	/* We've successfully synced our shadow pages, so let's just continue execution. */
3923	Log2(("Shadow page fault at %RGv cr2=%RGp error code %x\n", (RTGCPTR)pCtx->rip, exitQualification , errCode));
3924	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitReasonNPF);
3925
3926	TRPMResetTrap(pVCpu);
3927	goto ResumeExecution;
3928	}
3929
3930	#ifdef VBOX_STRICT
3931	if (rc != VINF_EM_RAW_EMULATE_INSTR)
3932	LogFlow(("PGMTrap0eHandlerNestedPaging at %RGv failed with %Rrc\n", (RTGCPTR)pCtx->rip, VBOXSTRICTRC_VAL(rc)));
3933	#endif
3934	/* Need to go back to the recompiler to emulate the instruction. */
3935	TRPMResetTrap(pVCpu);
3936	break;
3937	}
3938
3939	case VMX_EXIT_EPT_MISCONFIG:
3940	{
3941	RTGCPHYS GCPhys;
3942
3943	Assert(pVM->hwaccm.s.fNestedPaging);
3944
3945	rc2 = VMXReadVMCS64(VMX_VMCS_EXIT_PHYS_ADDR_FULL, &GCPhys);
3946	AssertRC(rc2);
3947	Log(("VMX_EXIT_EPT_MISCONFIG for %RGp\n", GCPhys));
3948
3949	/* Shortcut for APIC TPR reads and writes. */
3950	if ( (GCPhys & 0xfff) == 0x080
3951	&& GCPhys > 0x1000000 /* to skip VGA frame buffer accesses */
3952	&& fSetupTPRCaching
3953	&& (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC))
3954	{
3955	RTGCPHYS GCPhysApicBase;
3956	PDMApicGetBase(pVM, &GCPhysApicBase); /* @todo cache this */
3957	GCPhysApicBase &= PAGE_BASE_GC_MASK;
3958	if (GCPhys == GCPhysApicBase + 0x80)
3959	{
3960	Log(("Enable VT-x virtual APIC access filtering\n"));
3961	rc2 = IOMMMIOMapMMIOHCPage(pVM, GCPhysApicBase, pVM->hwaccm.s.vmx.pAPICPhys, X86_PTE_RW \| X86_PTE_P);
3962	AssertRC(rc2);
3963	}
3964	}
3965
3966	rc = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pCtx), GCPhys, UINT32_MAX);
3967
3968	/*
3969	* If we succeed, resume execution.
3970	* Or, if fail in interpreting the instruction because we couldn't get the guest physical address
3971	* of the page containing the instruction via the guest's page tables (we would invalidate the guest page
3972	* in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
3973	* weird case. See @bugref{6043}.
3974	*/
3975	if ( rc == VINF_SUCCESS
3976	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
3977	\|\| rc == VERR_PAGE_NOT_PRESENT)
3978	{
3979	Log2(("PGMR0Trap0eHandlerNPMisconfig(,,,%RGp) at %RGv -> resume\n", GCPhys, (RTGCPTR)pCtx->rip));
3980	goto ResumeExecution;
3981	}
3982
3983	Log2(("PGMR0Trap0eHandlerNPMisconfig(,,,%RGp) at %RGv -> %Rrc\n", GCPhys, (RTGCPTR)pCtx->rip, VBOXSTRICTRC_VAL(rc)));
3984	break;
3985	}
3986
3987	case VMX_EXIT_IRQ_WINDOW: /* 7 Interrupt window. */
3988	/* Clear VM-exit on IF=1 change. */
3989	LogFlow(("VMX_EXIT_IRQ_WINDOW %RGv pending=%d IF=%d\n", (RTGCPTR)pCtx->rip,
3990	VMCPU_FF_ISPENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC\|VMCPU_FF_INTERRUPT_PIC)), pCtx->eflags.Bits.u1IF));
3991	pVCpu->hwaccm.s.vmx.proc_ctls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_IRQ_WINDOW_EXIT;
3992	rc2 = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
3993	AssertRC(rc2);
3994	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitIrqWindow);
3995	goto ResumeExecution; /* we check for pending guest interrupts there */
3996
3997	case VMX_EXIT_WBINVD: /* 54 Guest software attempted to execute WBINVD. (conditional) */
3998	case VMX_EXIT_INVD: /* 13 Guest software attempted to execute INVD. (unconditional) */
3999	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitInvd);
4000	/* Skip instruction and continue directly. */
4001	pCtx->rip += cbInstr;
4002	/* Continue execution.*/
4003	goto ResumeExecution;
4004
4005	case VMX_EXIT_CPUID: /* 10 Guest software attempted to execute CPUID. */
4006	{
4007	Log2(("VMX: Cpuid %x\n", pCtx->eax));
4008	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitCpuid);
4009	rc = EMInterpretCpuId(pVM, pVCpu, CPUMCTX2CORE(pCtx));
4010	if (rc == VINF_SUCCESS)
4011	{
4012	/* Update EIP and continue execution. */
4013	Assert(cbInstr == 2);
4014	pCtx->rip += cbInstr;
4015	goto ResumeExecution;
4016	}
4017	AssertMsgFailed(("EMU: cpuid failed with %Rrc\n", VBOXSTRICTRC_VAL(rc)));
4018	rc = VINF_EM_RAW_EMULATE_INSTR;
4019	break;
4020	}
4021
4022	case VMX_EXIT_RDPMC: /* 15 Guest software attempted to execute RDPMC. */
4023	{
4024	Log2(("VMX: Rdpmc %x\n", pCtx->ecx));
4025	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitRdpmc);
4026	rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pCtx));
4027	if (rc == VINF_SUCCESS)
4028	{
4029	/* Update EIP and continue execution. */
4030	Assert(cbInstr == 2);
4031	pCtx->rip += cbInstr;
4032	goto ResumeExecution;
4033	}
4034	rc = VINF_EM_RAW_EMULATE_INSTR;
4035	break;
4036	}
4037
4038	case VMX_EXIT_RDTSC: /* 16 Guest software attempted to execute RDTSC. */
4039	{
4040	Log2(("VMX: Rdtsc\n"));
4041	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitRdtsc);
4042	rc = EMInterpretRdtsc(pVM, pVCpu, CPUMCTX2CORE(pCtx));
4043	if (rc == VINF_SUCCESS)
4044	{
4045	/* Update EIP and continue execution. */
4046	Assert(cbInstr == 2);
4047	pCtx->rip += cbInstr;
4048	goto ResumeExecution;
4049	}
4050	rc = VINF_EM_RAW_EMULATE_INSTR;
4051	break;
4052	}
4053
4054	case VMX_EXIT_INVPG: /* 14 Guest software attempted to execute INVPG. */
4055	{
4056	Log2(("VMX: invlpg\n"));
4057	Assert(!pVM->hwaccm.s.fNestedPaging);
4058
4059	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitInvpg);
4060	rc = EMInterpretInvlpg(pVM, pVCpu, CPUMCTX2CORE(pCtx), exitQualification);
4061	if (rc == VINF_SUCCESS)
4062	{
4063	/* Update EIP and continue execution. */
4064	pCtx->rip += cbInstr;
4065	goto ResumeExecution;
4066	}
4067	AssertMsg(rc == VERR_EM_INTERPRETER, ("EMU: invlpg %RGv failed with %Rrc\n", exitQualification, VBOXSTRICTRC_VAL(rc)));
4068	break;
4069	}
4070
4071	case VMX_EXIT_MONITOR: /* 39 Guest software attempted to execute MONITOR. */
4072	{
4073	Log2(("VMX: monitor\n"));
4074
4075	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitMonitor);
4076	rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pCtx));
4077	if (rc == VINF_SUCCESS)
4078	{
4079	/* Update EIP and continue execution. */
4080	pCtx->rip += cbInstr;
4081	goto ResumeExecution;
4082	}
4083	AssertMsg(rc == VERR_EM_INTERPRETER, ("EMU: monitor failed with %Rrc\n", VBOXSTRICTRC_VAL(rc)));
4084	break;
4085	}
4086
4087	case VMX_EXIT_WRMSR: /* 32 WRMSR. Guest software attempted to execute WRMSR. */
4088	/* When an interrupt is pending, we'll let MSR_K8_LSTAR writes fault in our TPR patch code. */
4089	if ( pVM->hwaccm.s.fTPRPatchingActive
4090	&& pCtx->ecx == MSR_K8_LSTAR)
4091	{
4092	Assert(!CPUMIsGuestInLongModeEx(pCtx));
4093	if ((pCtx->eax & 0xff) != u8LastTPR)
4094	{
4095	Log(("VMX: Faulting MSR_K8_LSTAR write with new TPR value %x\n", pCtx->eax & 0xff));
4096
4097	/* Our patch code uses LSTAR for TPR caching. */
4098	rc2 = PDMApicSetTPR(pVCpu, pCtx->eax & 0xff);
4099	AssertRC(rc2);
4100	}
4101
4102	/* Skip the instruction and continue. */
4103	pCtx->rip += cbInstr; /* wrmsr = [0F 30] */
4104
4105	/* Only resume if successful. */
4106	goto ResumeExecution;
4107	}
4108	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_GUEST_MSR;
4109	/* no break */
4110	case VMX_EXIT_RDMSR: /* 31 RDMSR. Guest software attempted to execute RDMSR. */
4111	{
4112	STAM_COUNTER_INC((exitReason == VMX_EXIT_RDMSR) ? &pVCpu->hwaccm.s.StatExitRdmsr : &pVCpu->hwaccm.s.StatExitWrmsr);
4113
4114	/*
4115	* Note: The Intel spec. claims there's an REX version of RDMSR that's slightly different,
4116	* so we play safe by completely disassembling the instruction.
4117	*/
4118	Log2(("VMX: %s\n", (exitReason == VMX_EXIT_RDMSR) ? "rdmsr" : "wrmsr"));
4119	rc = EMInterpretInstruction(pVCpu, CPUMCTX2CORE(pCtx), 0);
4120	if (rc == VINF_SUCCESS)
4121	{
4122	/* EIP has been updated already. */
4123	/* Only resume if successful. */
4124	goto ResumeExecution;
4125	}
4126	AssertMsg(rc == VERR_EM_INTERPRETER, ("EMU: %s failed with %Rrc\n",
4127	(exitReason == VMX_EXIT_RDMSR) ? "rdmsr" : "wrmsr", VBOXSTRICTRC_VAL(rc)));
4128	break;
4129	}
4130
4131	case VMX_EXIT_CRX_MOVE: /* 28 Control-register accesses. */
4132	{
4133	STAM_PROFILE_ADV_START(&pVCpu->hwaccm.s.StatExit2Sub2, y2);
4134
4135	switch (VMX_EXIT_QUALIFICATION_CRX_ACCESS(exitQualification))
4136	{
4137	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_WRITE:
4138	{
4139	Log2(("VMX: %RGv mov cr%d, x\n", (RTGCPTR)pCtx->rip, VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification)));
4140	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitCRxWrite[VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification)]);
4141	rc = EMInterpretCRxWrite(pVM, pVCpu, CPUMCTX2CORE(pCtx),
4142	VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification),
4143	VMX_EXIT_QUALIFICATION_CRX_GENREG(exitQualification));
4144	switch (VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification))
4145	{
4146	case 0:
4147	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_GUEST_CR0 \| HWACCM_CHANGED_GUEST_CR3;
4148	break;
4149	case 2:
4150	break;
4151	case 3:
4152	Assert(!pVM->hwaccm.s.fNestedPaging \|\| !CPUMIsGuestInPagedProtectedModeEx(pCtx));
4153	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_GUEST_CR3;
4154	break;
4155	case 4:
4156	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_GUEST_CR4;
4157	break;
4158	case 8:
4159	/* CR8 contains the APIC TPR */
4160	Assert(!(pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1
4161	& VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW));
4162	break;
4163
4164	default:
4165	AssertFailed();
4166	break;
4167	}
4168	break;
4169	}
4170
4171	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ:
4172	{
4173	Log2(("VMX: mov x, crx\n"));
4174	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitCRxRead[VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification)]);
4175
4176	Assert( !pVM->hwaccm.s.fNestedPaging
4177	\|\| !CPUMIsGuestInPagedProtectedModeEx(pCtx)
4178	\|\| VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification) != DISCREG_CR3);
4179
4180	/* CR8 reads only cause an exit when the TPR shadow feature isn't present. */
4181	Assert( VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification) != 8
4182	\|\| !(pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW));
4183
4184	rc = EMInterpretCRxRead(pVM, pVCpu, CPUMCTX2CORE(pCtx),
4185	VMX_EXIT_QUALIFICATION_CRX_GENREG(exitQualification),
4186	VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification));
4187	break;
4188	}
4189
4190	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_CLTS:
4191	{
4192	Log2(("VMX: clts\n"));
4193	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitCLTS);
4194	rc = EMInterpretCLTS(pVM, pVCpu);
4195	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_GUEST_CR0;
4196	break;
4197	}
4198
4199	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_LMSW:
4200	{
4201	Log2(("VMX: lmsw %x\n", VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(exitQualification)));
4202	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitLMSW);
4203	rc = EMInterpretLMSW(pVM, pVCpu, CPUMCTX2CORE(pCtx), VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(exitQualification));
4204	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_GUEST_CR0;
4205	break;
4206	}
4207	}
4208
4209	/* Update EIP if no error occurred. */
4210	if (RT_SUCCESS(rc))
4211	pCtx->rip += cbInstr;
4212
4213	if (rc == VINF_SUCCESS)
4214	{
4215	/* Only resume if successful. */
4216	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub2, y2);
4217	goto ResumeExecution;
4218	}
4219	Assert(rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_PGM_SYNC_CR3);
4220	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub2, y2);
4221	break;
4222	}
4223
4224	case VMX_EXIT_DRX_MOVE: /* 29 Debug-register accesses. */
4225	{
4226	if ( !DBGFIsStepping(pVCpu)
4227	&& !CPUMIsHyperDebugStateActive(pVCpu))
4228	{
4229	/* Disable DRx move intercepts. */
4230	pVCpu->hwaccm.s.vmx.proc_ctls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT;
4231	rc2 = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
4232	AssertRC(rc2);
4233
4234	/* Save the host and load the guest debug state. */
4235	rc2 = CPUMR0LoadGuestDebugState(pVM, pVCpu, pCtx, true /* include DR6 */);
4236	AssertRC(rc2);
4237
4238	#ifdef LOG_ENABLED
4239	if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(exitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
4240	{
4241	Log(("VMX_EXIT_DRX_MOVE: write DR%d genreg %d\n", VMX_EXIT_QUALIFICATION_DRX_REGISTER(exitQualification),
4242	VMX_EXIT_QUALIFICATION_DRX_GENREG(exitQualification)));
4243	}
4244	else
4245	Log(("VMX_EXIT_DRX_MOVE: read DR%d\n", VMX_EXIT_QUALIFICATION_DRX_REGISTER(exitQualification)));
4246	#endif
4247
4248	#ifdef VBOX_WITH_STATISTICS
4249	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatDRxContextSwitch);
4250	if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(exitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
4251	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitDRxWrite);
4252	else
4253	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitDRxRead);
4254	#endif
4255
4256	goto ResumeExecution;
4257	}
4258
4259	/** @todo clear VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT after the first
4260	* time and restore DRx registers afterwards */
4261	if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(exitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
4262	{
4263	Log2(("VMX: mov DRx%d, genreg%d\n", VMX_EXIT_QUALIFICATION_DRX_REGISTER(exitQualification),
4264	VMX_EXIT_QUALIFICATION_DRX_GENREG(exitQualification)));
4265	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitDRxWrite);
4266	rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pCtx),
4267	VMX_EXIT_QUALIFICATION_DRX_REGISTER(exitQualification),
4268	VMX_EXIT_QUALIFICATION_DRX_GENREG(exitQualification));
4269	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_GUEST_DEBUG;
4270	Log2(("DR7=%08x\n", pCtx->dr[7]));
4271	}
4272	else
4273	{
4274	Log2(("VMX: mov x, DRx\n"));
4275	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitDRxRead);
4276	rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pCtx),
4277	VMX_EXIT_QUALIFICATION_DRX_GENREG(exitQualification),
4278	VMX_EXIT_QUALIFICATION_DRX_REGISTER(exitQualification));
4279	}
4280	/* Update EIP if no error occurred. */
4281	if (RT_SUCCESS(rc))
4282	pCtx->rip += cbInstr;
4283
4284	if (rc == VINF_SUCCESS)
4285	{
4286	/* Only resume if successful. */
4287	goto ResumeExecution;
4288	}
4289	Assert(rc == VERR_EM_INTERPRETER);
4290	break;
4291	}
4292
4293	/* Note: We'll get a #GP if the IO instruction isn't allowed (IOPL or TSS bitmap); no need to double check. */
4294	case VMX_EXIT_PORT_IO: /* 30 I/O instruction. */
4295	{
4296	STAM_PROFILE_ADV_START(&pVCpu->hwaccm.s.StatExit2Sub1, y1);
4297	uint32_t uPort;
4298	uint32_t uIOWidth = VMX_EXIT_QUALIFICATION_IO_WIDTH(exitQualification);
4299	bool fIOWrite = (VMX_EXIT_QUALIFICATION_IO_DIRECTION(exitQualification) == VMX_EXIT_QUALIFICATION_IO_DIRECTION_OUT);
4300
4301	/** @todo necessary to make the distinction? */
4302	if (VMX_EXIT_QUALIFICATION_IO_ENCODING(exitQualification) == VMX_EXIT_QUALIFICATION_IO_ENCODING_DX)
4303	uPort = pCtx->edx & 0xffff;
4304	else
4305	uPort = VMX_EXIT_QUALIFICATION_IO_PORT(exitQualification); /* Immediate encoding. */
4306
4307	if (RT_UNLIKELY(uIOWidth == 2 \|\| uIOWidth >= 4)) /* paranoia */
4308	{
4309	rc = fIOWrite ? VINF_IOM_R3_IOPORT_WRITE : VINF_IOM_R3_IOPORT_READ;
4310	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub1, y1);
4311	break;
4312	}
4313
4314	uint32_t cbSize = g_aIOSize[uIOWidth];
4315	if (VMX_EXIT_QUALIFICATION_IO_STRING(exitQualification))
4316	{
4317	/* ins/outs */
4318	PDISCPUSTATE pDis = &pVCpu->hwaccm.s.DisState;
4319
4320	/* Disassemble manually to deal with segment prefixes. */
4321	/** @todo VMX_VMCS_EXIT_GUEST_LINEAR_ADDR contains the flat pointer operand of the instruction. */
4322	/** @todo VMX_VMCS32_RO_EXIT_INSTR_INFO also contains segment prefix info. */
4323	rc2 = EMInterpretDisasOne(pVM, pVCpu, CPUMCTX2CORE(pCtx), pDis, NULL);
4324	if (RT_SUCCESS(rc))
4325	{
4326	if (fIOWrite)
4327	{
4328	Log2(("IOMInterpretOUTSEx %RGv %x size=%d\n", (RTGCPTR)pCtx->rip, uPort, cbSize));
4329	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitIOStringWrite);
4330	rc = IOMInterpretOUTSEx(pVM, CPUMCTX2CORE(pCtx), uPort, pDis->fPrefix, (DISCPUMODE)pDis->uAddrMode, cbSize);
4331	}
4332	else
4333	{
4334	Log2(("IOMInterpretINSEx %RGv %x size=%d\n", (RTGCPTR)pCtx->rip, uPort, cbSize));
4335	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitIOStringRead);
4336	rc = IOMInterpretINSEx(pVM, CPUMCTX2CORE(pCtx), uPort, pDis->fPrefix, (DISCPUMODE)pDis->uAddrMode, cbSize);
4337	}
4338	}
4339	else
4340	rc = VINF_EM_RAW_EMULATE_INSTR;
4341	}
4342	else
4343	{
4344	/* Normal in/out */
4345	uint32_t uAndVal = g_aIOOpAnd[uIOWidth];
4346
4347	Assert(!VMX_EXIT_QUALIFICATION_IO_REP(exitQualification));
4348
4349	if (fIOWrite)
4350	{
4351	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitIOWrite);
4352	rc = IOMIOPortWrite(pVM, uPort, pCtx->eax & uAndVal, cbSize);
4353	if (rc == VINF_IOM_R3_IOPORT_WRITE)
4354	HWACCMR0SavePendingIOPortWrite(pVCpu, pCtx->rip, pCtx->rip + cbInstr, uPort, uAndVal, cbSize);
4355	}
4356	else
4357	{
4358	uint32_t u32Val = 0;
4359
4360	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitIORead);
4361	rc = IOMIOPortRead(pVM, uPort, &u32Val, cbSize);
4362	if (IOM_SUCCESS(rc))
4363	{
4364	/* Write back to the EAX register. */
4365	pCtx->eax = (pCtx->eax & ~uAndVal) \| (u32Val & uAndVal);
4366	}
4367	else
4368	if (rc == VINF_IOM_R3_IOPORT_READ)
4369	HWACCMR0SavePendingIOPortRead(pVCpu, pCtx->rip, pCtx->rip + cbInstr, uPort, uAndVal, cbSize);
4370	}
4371	}
4372
4373	/*
4374	* Handled the I/O return codes.
4375	* (The unhandled cases end up with rc == VINF_EM_RAW_EMULATE_INSTR.)
4376	*/
4377	if (IOM_SUCCESS(rc))
4378	{
4379	/* Update EIP and continue execution. */
4380	pCtx->rip += cbInstr;
4381	if (RT_LIKELY(rc == VINF_SUCCESS))
4382	{
4383	/* If any IO breakpoints are armed, then we should check if a debug trap needs to be generated. */
4384	if (pCtx->dr[7] & X86_DR7_ENABLED_MASK)
4385	{
4386	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatDRxIOCheck);
4387	for (unsigned i = 0; i < 4; i++)
4388	{
4389	unsigned uBPLen = g_aIOSize[X86_DR7_GET_LEN(pCtx->dr[7], i)];
4390
4391	if ( (uPort >= pCtx->dr[i] && uPort < pCtx->dr[i] + uBPLen)
4392	&& (pCtx->dr[7] & (X86_DR7_L(i) \| X86_DR7_G(i)))
4393	&& (pCtx->dr[7] & X86_DR7_RW(i, X86_DR7_RW_IO)) == X86_DR7_RW(i, X86_DR7_RW_IO))
4394	{
4395	uint64_t uDR6;
4396
4397	Assert(CPUMIsGuestDebugStateActive(pVCpu));
4398
4399	uDR6 = ASMGetDR6();
4400
4401	/* Clear all breakpoint status flags and set the one we just hit. */
4402	uDR6 &= ~(X86_DR6_B0\|X86_DR6_B1\|X86_DR6_B2\|X86_DR6_B3);
4403	uDR6 \|= (uint64_t)RT_BIT(i);
4404
4405	/*
4406	* Note: AMD64 Architecture Programmer's Manual 13.1:
4407	* Bits 15:13 of the DR6 register is never cleared by the processor and must
4408	* be cleared by software after the contents have been read.
4409	*/
4410	ASMSetDR6(uDR6);
4411
4412	/* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
4413	pCtx->dr[7] &= ~X86_DR7_GD;
4414
4415	/* Paranoia. */
4416	pCtx->dr[7] &= 0xffffffff; /* upper 32 bits reserved */
4417	pCtx->dr[7] &= ~(RT_BIT(11) \| RT_BIT(12) \| RT_BIT(14) \| RT_BIT(15)); /* must be zero */
4418	pCtx->dr[7] \|= 0x400; /* must be one */
4419
4420	/* Resync DR7 */
4421	rc2 = VMXWriteVMCS64(VMX_VMCS64_GUEST_DR7, pCtx->dr[7]);
4422	AssertRC(rc2);
4423
4424	/* Construct inject info. */
4425	intInfo = X86_XCPT_DB;
4426	intInfo \|= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
4427	intInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HWEXCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
4428
4429	Log(("Inject IO debug trap at %RGv\n", (RTGCPTR)pCtx->rip));
4430	rc2 = hmR0VmxInjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo),
4431	0 /* cbInstr /, 0 / errCode */);
4432	AssertRC(rc2);
4433
4434	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub1, y1);
4435	goto ResumeExecution;
4436	}
4437	}
4438	}
4439	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub1, y1);
4440	goto ResumeExecution;
4441	}
4442	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub1, y1);
4443	break;
4444	}
4445
4446	#ifdef VBOX_STRICT
4447	if (rc == VINF_IOM_R3_IOPORT_READ)
4448	Assert(!fIOWrite);
4449	else if (rc == VINF_IOM_R3_IOPORT_WRITE)
4450	Assert(fIOWrite);
4451	else
4452	{
4453	AssertMsg( RT_FAILURE(rc)
4454	\|\| rc == VINF_EM_RAW_EMULATE_INSTR
4455	\|\| rc == VINF_EM_RAW_GUEST_TRAP
4456	\|\| rc == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", VBOXSTRICTRC_VAL(rc)));
4457	}
4458	#endif
4459	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub1, y1);
4460	break;
4461	}
4462
4463	case VMX_EXIT_TPR: /* 43 TPR below threshold. Guest software executed MOV to CR8. */
4464	LogFlow(("VMX_EXIT_TPR\n"));
4465	/* RIP is already set to the next instruction and the TPR has been synced back. Just resume. */
4466	goto ResumeExecution;
4467
4468	case VMX_EXIT_APIC_ACCESS: /* 44 APIC access. Guest software attempted to access memory at a physical address
4469	on the APIC-access page. */
4470	{
4471	LogFlow(("VMX_EXIT_APIC_ACCESS\n"));
4472	unsigned uAccessType = VMX_EXIT_QUALIFICATION_APIC_ACCESS_TYPE(exitQualification);
4473
4474	switch(uAccessType)
4475	{
4476	case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
4477	case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
4478	{
4479	RTGCPHYS GCPhys;
4480	PDMApicGetBase(pVM, &GCPhys);
4481	GCPhys &= PAGE_BASE_GC_MASK;
4482	GCPhys += VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(exitQualification);
4483
4484	LogFlow(("Apic access at %RGp\n", GCPhys));
4485	rc = IOMMMIOPhysHandler(pVM, (uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ) ? 0 : X86_TRAP_PF_RW,
4486	CPUMCTX2CORE(pCtx), GCPhys);
4487	if (rc == VINF_SUCCESS)
4488	goto ResumeExecution; /* rip already updated */
4489	break;
4490	}
4491
4492	default:
4493	rc = VINF_EM_RAW_EMULATE_INSTR;
4494	break;
4495	}
4496	break;
4497	}
4498
4499	case VMX_EXIT_PREEMPTION_TIMER: /* 52 VMX-preemption timer expired. The preemption timer counted down to zero. */
4500	if (!TMTimerPollBool(pVM, pVCpu))
4501	goto ResumeExecution;
4502	rc = VINF_EM_RAW_TIMER_PENDING;
4503	break;
4504
4505	default:
4506	/* The rest is handled after syncing the entire CPU state. */
4507	break;
4508	}
4509
4510
4511	/*
4512	* Note: The guest state is not entirely synced back at this stage!
4513	*/
4514
4515	/* Investigate why there was a VM-exit. (part 2) */
4516	switch (exitReason)
4517	{
4518	case VMX_EXIT_EXCEPTION: /* 0 Exception or non-maskable interrupt (NMI). */
4519	case VMX_EXIT_EXTERNAL_IRQ: /* 1 External interrupt. */
4520	case VMX_EXIT_EPT_VIOLATION:
4521	case VMX_EXIT_EPT_MISCONFIG: /* 49 EPT misconfig is used by the PGM/MMIO optimizations. */
4522	case VMX_EXIT_PREEMPTION_TIMER: /* 52 VMX-preemption timer expired. The preemption timer counted down to zero. */
4523	/* Already handled above. */
4524	break;
4525
4526	case VMX_EXIT_TRIPLE_FAULT: /* 2 Triple fault. */
4527	rc = VINF_EM_RESET; /* Triple fault equals a reset. */
4528	break;
4529
4530	case VMX_EXIT_INIT_SIGNAL: /* 3 INIT signal. */
4531	case VMX_EXIT_SIPI: /* 4 Start-up IPI (SIPI). */
4532	rc = VINF_EM_RAW_INTERRUPT;
4533	AssertFailed(); /* Can't happen. Yet. */
4534	break;
4535
4536	case VMX_EXIT_IO_SMI_IRQ: /* 5 I/O system-management interrupt (SMI). */
4537	case VMX_EXIT_SMI_IRQ: /* 6 Other SMI. */
4538	rc = VINF_EM_RAW_INTERRUPT;
4539	AssertFailed(); /* Can't happen afaik. */
4540	break;
4541
4542	case VMX_EXIT_TASK_SWITCH: /* 9 Task switch: too complicated to emulate, so fall back to the recompiler */
4543	Log(("VMX_EXIT_TASK_SWITCH: exit=%RX64\n", exitQualification));
4544	if ( (VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE(exitQualification) == VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE_IDT)
4545	&& pVCpu->hwaccm.s.Event.fPending)
4546	{
4547	/* Caused by an injected interrupt. */
4548	pVCpu->hwaccm.s.Event.fPending = false;
4549
4550	Log(("VMX_EXIT_TASK_SWITCH: reassert trap %d\n", VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVCpu->hwaccm.s.Event.intInfo)));
4551	Assert(!VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(pVCpu->hwaccm.s.Event.intInfo));
4552	rc2 = TRPMAssertTrap(pVCpu, VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVCpu->hwaccm.s.Event.intInfo), TRPM_HARDWARE_INT);
4553	AssertRC(rc2);
4554	}
4555	/* else Exceptions and software interrupts can just be restarted. */
4556	rc = VERR_EM_INTERPRETER;
4557	break;
4558
4559	case VMX_EXIT_HLT: /* 12 Guest software attempted to execute HLT. */
4560	/* Check if external interrupts are pending; if so, don't switch back. */
4561	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitHlt);
4562	pCtx->rip++; /* skip hlt */
4563	if (EMShouldContinueAfterHalt(pVCpu, pCtx))
4564	goto ResumeExecution;
4565
4566	rc = VINF_EM_HALT;
4567	break;
4568
4569	case VMX_EXIT_MWAIT: /* 36 Guest software executed MWAIT. */
4570	Log2(("VMX: mwait\n"));
4571	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitMwait);
4572	rc = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pCtx));
4573	if ( rc == VINF_EM_HALT
4574	\|\| rc == VINF_SUCCESS)
4575	{
4576	/* Update EIP and continue execution. */
4577	pCtx->rip += cbInstr;
4578
4579	/* Check if external interrupts are pending; if so, don't switch back. */
4580	if ( rc == VINF_SUCCESS
4581	\|\| ( rc == VINF_EM_HALT
4582	&& EMShouldContinueAfterHalt(pVCpu, pCtx))
4583	)
4584	goto ResumeExecution;
4585	}
4586	AssertMsg(rc == VERR_EM_INTERPRETER \|\| rc == VINF_EM_HALT, ("EMU: mwait failed with %Rrc\n", VBOXSTRICTRC_VAL(rc)));
4587	break;
4588
4589	case VMX_EXIT_RSM: /* 17 Guest software attempted to execute RSM in SMM. */
4590	AssertFailed(); /* can't happen. */
4591	rc = VERR_EM_INTERPRETER;
4592	break;
4593
4594	case VMX_EXIT_MTF: /* 37 Exit due to Monitor Trap Flag. */
4595	LogFlow(("VMX_EXIT_MTF at %RGv\n", (RTGCPTR)pCtx->rip));
4596	pVCpu->hwaccm.s.vmx.proc_ctls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_TRAP_FLAG;
4597	rc2 = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
4598	AssertRC(rc2);
4599	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitMTF);
4600	#if 0
4601	DBGFDoneStepping(pVCpu);
4602	#endif
4603	rc = VINF_EM_DBG_STOP;
4604	break;
4605
4606	case VMX_EXIT_VMCALL: /* 18 Guest software executed VMCALL. */
4607	case VMX_EXIT_VMCLEAR: /* 19 Guest software executed VMCLEAR. */
4608	case VMX_EXIT_VMLAUNCH: /* 20 Guest software executed VMLAUNCH. */
4609	case VMX_EXIT_VMPTRLD: /* 21 Guest software executed VMPTRLD. */
4610	case VMX_EXIT_VMPTRST: /* 22 Guest software executed VMPTRST. */
4611	case VMX_EXIT_VMREAD: /* 23 Guest software executed VMREAD. */
4612	case VMX_EXIT_VMRESUME: /* 24 Guest software executed VMRESUME. */
4613	case VMX_EXIT_VMWRITE: /* 25 Guest software executed VMWRITE. */
4614	case VMX_EXIT_VMXOFF: /* 26 Guest software executed VMXOFF. */
4615	case VMX_EXIT_VMXON: /* 27 Guest software executed VMXON. */
4616	/** @todo inject #UD immediately */
4617	rc = VERR_EM_INTERPRETER;
4618	break;
4619
4620	case VMX_EXIT_CPUID: /* 10 Guest software attempted to execute CPUID. */
4621	case VMX_EXIT_RDTSC: /* 16 Guest software attempted to execute RDTSC. */
4622	case VMX_EXIT_INVPG: /* 14 Guest software attempted to execute INVPG. */
4623	case VMX_EXIT_CRX_MOVE: /* 28 Control-register accesses. */
4624	case VMX_EXIT_DRX_MOVE: /* 29 Debug-register accesses. */
4625	case VMX_EXIT_PORT_IO: /* 30 I/O instruction. */
4626	case VMX_EXIT_RDPMC: /* 15 Guest software attempted to execute RDPMC. */
4627	/* already handled above */
4628	AssertMsg( rc == VINF_PGM_CHANGE_MODE
4629	\|\| rc == VINF_EM_RAW_INTERRUPT
4630	\|\| rc == VERR_EM_INTERPRETER
4631	\|\| rc == VINF_EM_RAW_EMULATE_INSTR
4632	\|\| rc == VINF_PGM_SYNC_CR3
4633	\|\| rc == VINF_IOM_R3_IOPORT_READ
4634	\|\| rc == VINF_IOM_R3_IOPORT_WRITE
4635	\|\| rc == VINF_EM_RAW_GUEST_TRAP
4636	\|\| rc == VINF_TRPM_XCPT_DISPATCHED
4637	\|\| rc == VINF_EM_RESCHEDULE_REM,
4638	("rc = %d\n", VBOXSTRICTRC_VAL(rc)));
4639	break;
4640
4641	case VMX_EXIT_TPR: /* 43 TPR below threshold. Guest software executed MOV to CR8. */
4642	case VMX_EXIT_RDMSR: /* 31 RDMSR. Guest software attempted to execute RDMSR. */
4643	case VMX_EXIT_WRMSR: /* 32 WRMSR. Guest software attempted to execute WRMSR. */
4644	case VMX_EXIT_PAUSE: /* 40 Guest software attempted to execute PAUSE. */
4645	case VMX_EXIT_MONITOR: /* 39 Guest software attempted to execute MONITOR. */
4646	case VMX_EXIT_APIC_ACCESS: /* 44 APIC access. Guest software attempted to access memory at a physical address
4647	on the APIC-access page. */
4648	{
4649	/*
4650	* If we decided to emulate them here, then we must sync the MSRs that could have been changed (sysenter, FS/GS base)
4651	*/
4652	rc = VERR_EM_INTERPRETER;
4653	break;
4654	}
4655
4656	case VMX_EXIT_IRQ_WINDOW: /* 7 Interrupt window. */
4657	Assert(rc == VINF_EM_RAW_INTERRUPT);
4658	break;
4659
4660	case VMX_EXIT_ERR_INVALID_GUEST_STATE: /* 33 VM-entry failure due to invalid guest state. */
4661	{
4662	#ifdef VBOX_STRICT
4663	RTCCUINTREG val2 = 0;
4664
4665	Log(("VMX_EXIT_ERR_INVALID_GUEST_STATE\n"));
4666
4667	VMXReadVMCS(VMX_VMCS64_GUEST_RIP, &val2);
4668	Log(("Old eip %RGv new %RGv\n", (RTGCPTR)pCtx->rip, (RTGCPTR)val2));
4669
4670	VMXReadVMCS(VMX_VMCS64_GUEST_CR0, &val2);
4671	Log(("VMX_VMCS_GUEST_CR0 %RX64\n", (uint64_t)val2));
4672
4673	VMXReadVMCS(VMX_VMCS64_GUEST_CR3, &val2);
4674	Log(("VMX_VMCS_GUEST_CR3 %RX64\n", (uint64_t)val2));
4675
4676	VMXReadVMCS(VMX_VMCS64_GUEST_CR4, &val2);
4677	Log(("VMX_VMCS_GUEST_CR4 %RX64\n", (uint64_t)val2));
4678
4679	VMXReadVMCS(VMX_VMCS_GUEST_RFLAGS, &val2);
4680	Log(("VMX_VMCS_GUEST_RFLAGS %08x\n", val2));
4681
4682	VMX_LOG_SELREG(CS, "CS", val2);
4683	VMX_LOG_SELREG(DS, "DS", val2);
4684	VMX_LOG_SELREG(ES, "ES", val2);
4685	VMX_LOG_SELREG(FS, "FS", val2);
4686	VMX_LOG_SELREG(GS, "GS", val2);
4687	VMX_LOG_SELREG(SS, "SS", val2);
4688	VMX_LOG_SELREG(TR, "TR", val2);
4689	VMX_LOG_SELREG(LDTR, "LDTR", val2);
4690
4691	VMXReadVMCS(VMX_VMCS64_GUEST_GDTR_BASE, &val2);
4692	Log(("VMX_VMCS_GUEST_GDTR_BASE %RX64\n", (uint64_t)val2));
4693	VMXReadVMCS(VMX_VMCS64_GUEST_IDTR_BASE, &val2);
4694	Log(("VMX_VMCS_GUEST_IDTR_BASE %RX64\n", (uint64_t)val2));
4695	#endif /* VBOX_STRICT */
4696	rc = VERR_VMX_INVALID_GUEST_STATE;
4697	break;
4698	}
4699
4700	case VMX_EXIT_ERR_MSR_LOAD: /* 34 VM-entry failure due to MSR loading. */
4701	case VMX_EXIT_ERR_MACHINE_CHECK: /* 41 VM-entry failure due to machine-check. */
4702	default:
4703	rc = VERR_VMX_UNEXPECTED_EXIT_CODE;
4704	AssertMsgFailed(("Unexpected exit code %d\n", exitReason)); /* Can't happen. */
4705	break;
4706
4707	}
4708
4709	end:
4710	/* We now going back to ring-3, so clear the action flag. */
4711	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
4712
4713	/*
4714	* Signal changes for the recompiler.
4715	*/
4716	CPUMSetChangedFlags(pVCpu,
4717	CPUM_CHANGED_SYSENTER_MSR
4718	\| CPUM_CHANGED_LDTR
4719	\| CPUM_CHANGED_GDTR
4720	\| CPUM_CHANGED_IDTR
4721	\| CPUM_CHANGED_TR
4722	\| CPUM_CHANGED_HIDDEN_SEL_REGS);
4723
4724	/*
4725	* If we executed vmlaunch/vmresume and an external IRQ was pending, then we don't have to do a full sync the next time.
4726	*/
4727	if ( exitReason == VMX_EXIT_EXTERNAL_IRQ
4728	&& !VMX_EXIT_INTERRUPTION_INFO_VALID(intInfo))
4729	{
4730	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatPendingHostIrq);
4731	/* On the next entry we'll only sync the host context. */
4732	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_HOST_CONTEXT;
4733	}
4734	else
4735	{
4736	/* On the next entry we'll sync everything. */
4737	/** @todo we can do better than this */
4738	/* Not in the VINF_PGM_CHANGE_MODE though! */
4739	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_ALL;
4740	}
4741
4742	/* Translate into a less severe return code */
4743	if (rc == VERR_EM_INTERPRETER)
4744	rc = VINF_EM_RAW_EMULATE_INSTR;
4745	else if (rc == VERR_VMX_INVALID_VMCS_PTR)
4746	{
4747	/* Try to extract more information about what might have gone wrong here. */
4748	VMXGetActivateVMCS(&pVCpu->hwaccm.s.vmx.lasterror.u64VMCSPhys);
4749	pVCpu->hwaccm.s.vmx.lasterror.ulVMCSRevision = (uint32_t )pVCpu->hwaccm.s.vmx.pvVMCS;
4750	pVCpu->hwaccm.s.vmx.lasterror.idEnteredCpu = pVCpu->hwaccm.s.idEnteredCpu;
4751	pVCpu->hwaccm.s.vmx.lasterror.idCurrentCpu = RTMpCpuId();
4752	}
4753
4754	/* Just set the correct state here instead of trying to catch every goto above. */
4755	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED, VMCPUSTATE_STARTED_EXEC);
4756
4757	#ifdef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
4758	/* Restore interrupts if we exited after disabling them. */
4759	if (uOldEFlags != ~(RTCCUINTREG)0)
4760	ASMSetFlags(uOldEFlags);
4761	#endif
4762
4763	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2, x);
4764	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit1, x);
4765	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatEntry, x);
4766	Log2(("X"));
4767	return VBOXSTRICTRC_TODO(rc);
4768	}
4769
4770
4771	/**
4772	* Enters the VT-x session.
4773	*
4774	* @returns VBox status code.
4775	* @param pVM Pointer to the VM.
4776	* @param pVCpu Pointer to the VMCPU.
4777	* @param pCpu Pointer to the CPU info struct.
4778	*/
4779	VMMR0DECL(int) VMXR0Enter(PVM pVM, PVMCPU pVCpu, PHMGLOBLCPUINFO pCpu)
4780	{
4781	Assert(pVM->hwaccm.s.vmx.fSupported);
4782	NOREF(pCpu);
4783
4784	unsigned cr4 = ASMGetCR4();
4785	if (!(cr4 & X86_CR4_VMXE))
4786	{
4787	AssertMsgFailed(("X86_CR4_VMXE should be set!\n"));
4788	return VERR_VMX_X86_CR4_VMXE_CLEARED;
4789	}
4790
4791	/* Activate the VMCS. */
4792	int rc = VMXActivateVMCS(pVCpu->hwaccm.s.vmx.HCPhysVMCS);
4793	if (RT_FAILURE(rc))
4794	return rc;
4795
4796	pVCpu->hwaccm.s.fResumeVM = false;
4797	return VINF_SUCCESS;
4798	}
4799
4800
4801	/**
4802	* Leaves the VT-x session.
4803	*
4804	* @returns VBox status code.
4805	* @param pVM Pointer to the VM.
4806	* @param pVCpu Pointer to the VMCPU.
4807	* @param pCtx Pointer to the guests CPU context.
4808	*/
4809	VMMR0DECL(int) VMXR0Leave(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
4810	{
4811	Assert(pVM->hwaccm.s.vmx.fSupported);
4812
4813	#ifdef DEBUG
4814	if (CPUMIsHyperDebugStateActive(pVCpu))
4815	{
4816	CPUMR0LoadHostDebugState(pVM, pVCpu);
4817	Assert(pVCpu->hwaccm.s.vmx.proc_ctls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT);
4818	}
4819	else
4820	#endif
4821
4822	/*
4823	* Save the guest debug state if necessary.
4824	*/
4825	if (CPUMIsGuestDebugStateActive(pVCpu))
4826	{
4827	CPUMR0SaveGuestDebugState(pVM, pVCpu, pCtx, true /* save DR6 */);
4828
4829	/* Enable DRx move intercepts again. */
4830	pVCpu->hwaccm.s.vmx.proc_ctls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT;
4831	int rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
4832	AssertRC(rc);
4833
4834	/* Resync the debug registers the next time. */
4835	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_GUEST_DEBUG;
4836	}
4837	else
4838	Assert(pVCpu->hwaccm.s.vmx.proc_ctls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT);
4839
4840	/*
4841	* Clear VMCS, marking it inactive, clearing implementation-specific data and writing
4842	* VMCS data back to memory.
4843	*/
4844	int rc = VMXClearVMCS(pVCpu->hwaccm.s.vmx.HCPhysVMCS);
4845	AssertRC(rc);
4846
4847	return VINF_SUCCESS;
4848	}
4849
4850
4851	/**
4852	* Flush the TLB using EPT.
4853	*
4854	* @returns VBox status code.
4855	* @param pVM Pointer to the VM.
4856	* @param pVCpu Pointer to the VMCPU.
4857	* @param enmFlush Type of flush.
4858	*/
4859	static void hmR0VmxFlushEPT(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_EPT enmFlush)
4860	{
4861	uint64_t descriptor[2];
4862
4863	LogFlow(("hmR0VmxFlushEPT %d\n", enmFlush));
4864	Assert(pVM->hwaccm.s.fNestedPaging);
4865	descriptor[0] = pVCpu->hwaccm.s.vmx.GCPhysEPTP;
4866	descriptor[1] = 0; /* MBZ. Intel spec. 33.3 VMX Instructions */
4867	int rc = VMXR0InvEPT(enmFlush, &descriptor[0]);
4868	AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %x %RGv failed with %d\n", enmFlush, pVCpu->hwaccm.s.vmx.GCPhysEPTP, rc));
4869	}
4870
4871
4872	/**
4873	* Flush the TLB using VPID.
4874	*
4875	* @returns VBox status code.
4876	* @param pVM Pointer to the VM.
4877	* @param pVCpu Pointer to the VMCPU (can be NULL depending on @a
4878	* enmFlush).
4879	* @param enmFlush Type of flush.
4880	* @param GCPtr Virtual address of the page to flush (can be 0 depending
4881	* on @a enmFlush).
4882	*/
4883	static void hmR0VmxFlushVPID(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_VPID enmFlush, RTGCPTR GCPtr)
4884	{
4885	#if HC_ARCH_BITS == 32
4886	/*
4887	* If we get a flush in 64-bit guest mode, then force a full TLB flush. invvpid probably takes only 32-bit addresses.
4888	*/
4889	if ( CPUMIsGuestInLongMode(pVCpu)
4890	&& !VMX_IS_64BIT_HOST_MODE())
4891	{
4892	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
4893	}
4894	else
4895	#endif
4896	{
4897	uint64_t descriptor[2];
4898
4899	Assert(pVM->hwaccm.s.vmx.fVPID);
4900	if (enmFlush == VMX_FLUSH_VPID_ALL_CONTEXTS)
4901	{
4902	descriptor[0] = 0;
4903	descriptor[1] = 0;
4904	}
4905	else
4906	{
4907	AssertPtr(pVCpu);
4908	Assert(pVCpu->hwaccm.s.uCurrentASID != 0);
4909	descriptor[0] = pVCpu->hwaccm.s.uCurrentASID;
4910	descriptor[1] = GCPtr;
4911	}
4912	int rc = VMXR0InvVPID(enmFlush, &descriptor[0]); NOREF(rc);
4913	AssertMsg(rc == VINF_SUCCESS,
4914	("VMXR0InvVPID %x %x %RGv failed with %d\n", enmFlush, pVCpu ? pVCpu->hwaccm.s.uCurrentASID : 0, GCPtr, rc));
4915	}
4916	}
4917
4918
4919	/**
4920	* Invalidates a guest page by guest virtual address. Only relevant for
4921	* EPT/VPID, otherwise there is nothing really to invalidate.
4922	*
4923	* @returns VBox status code.
4924	* @param pVM Pointer to the VM.
4925	* @param pVCpu Pointer to the VMCPU.
4926	* @param GCVirt Guest virtual address of the page to invalidate.
4927	*/
4928	VMMR0DECL(int) VMXR0InvalidatePage(PVM pVM, PVMCPU pVCpu, RTGCPTR GCVirt)
4929	{
4930	bool fFlushPending = VMCPU_FF_ISSET(pVCpu, VMCPU_FF_TLB_FLUSH);
4931
4932	Log2(("VMXR0InvalidatePage %RGv\n", GCVirt));
4933
4934	if (!fFlushPending)
4935	{
4936	/*
4937	* We must invalidate the guest TLB entry in either case, we cannot ignore it even for the EPT case
4938	* See @bugref{6043} and @bugref{6177}
4939	*
4940	* Set the VMCPU_FF_TLB_FLUSH force flag and flush before VMENTRY in hmR0VmxSetupTLB*() as this
4941	* function maybe called in a loop with individual addresses.
4942	*/
4943	if (pVM->hwaccm.s.vmx.fVPID)
4944	{
4945	/* If we can flush just this page do it, otherwise flush as little as possible. */
4946	if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVVPID_CAPS_INDIV_ADDR)
4947	hmR0VmxFlushVPID(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, GCVirt);
4948	else
4949	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
4950	}
4951	else if (pVM->hwaccm.s.fNestedPaging)
4952	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
4953	}
4954
4955	return VINF_SUCCESS;
4956	}
4957
4958
4959	/**
4960	* Invalidates a guest page by physical address. Only relevant for EPT/VPID,
4961	* otherwise there is nothing really to invalidate.
4962	*
4963	* NOTE: Assumes the current instruction references this physical page though a virtual address!!
4964	*
4965	* @returns VBox status code.
4966	* @param pVM Pointer to the VM.
4967	* @param pVCpu Pointer to the VMCPU.
4968	* @param GCPhys Guest physical address of the page to invalidate.
4969	*/
4970	VMMR0DECL(int) VMXR0InvalidatePhysPage(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys)
4971	{
4972	LogFlow(("VMXR0InvalidatePhysPage %RGp\n", GCPhys));
4973
4974	/*
4975	* We cannot flush a page by guest-physical address. invvpid takes only a linear address
4976	* while invept only flushes by EPT not individual addresses. We update the force flag here
4977	* and flush before VMENTRY in hmR0VmxSetupTLB*(). This function might be called in a loop.
4978	*/
4979	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
4980	return VINF_SUCCESS;
4981	}
4982
4983
4984	/**
4985	* Report world switch error and dump some useful debug info.
4986	*
4987	* @param pVM Pointer to the VM.
4988	* @param pVCpu Pointer to the VMCPU.
4989	* @param rc Return code.
4990	* @param pCtx Pointer to the current guest CPU context (not updated).
4991	*/
4992	static void hmR0VmxReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, VBOXSTRICTRC rc, PCPUMCTX pCtx)
4993	{
4994	NOREF(pVM);
4995
4996	switch (VBOXSTRICTRC_VAL(rc))
4997	{
4998	case VERR_VMX_INVALID_VMXON_PTR:
4999	AssertFailed();
5000	break;
5001
5002	case VERR_VMX_UNABLE_TO_START_VM:
5003	case VERR_VMX_UNABLE_TO_RESUME_VM:
5004	{
5005	int rc2;
5006	RTCCUINTREG exitReason, instrError;
5007
5008	rc2 = VMXReadVMCS(VMX_VMCS32_RO_EXIT_REASON, &exitReason);
5009	rc2 \|= VMXReadVMCS(VMX_VMCS32_RO_VM_INSTR_ERROR, &instrError);
5010	AssertRC(rc2);
5011	if (rc2 == VINF_SUCCESS)
5012	{
5013	Log(("Unable to start/resume VM for reason: %x. Instruction error %x\n", (uint32_t)exitReason,
5014	(uint32_t)instrError));
5015	Log(("Current stack %08x\n", &rc2));
5016
5017	pVCpu->hwaccm.s.vmx.lasterror.ulInstrError = instrError;
5018	pVCpu->hwaccm.s.vmx.lasterror.ulExitReason = exitReason;
5019
5020	#ifdef VBOX_STRICT
5021	RTGDTR gdtr;
5022	PCX86DESCHC pDesc;
5023	RTCCUINTREG val;
5024
5025	ASMGetGDTR(&gdtr);
5026
5027	VMXReadVMCS(VMX_VMCS64_GUEST_RIP, &val);
5028	Log(("Old eip %RGv new %RGv\n", (RTGCPTR)pCtx->rip, (RTGCPTR)val));
5029	VMXReadVMCS(VMX_VMCS_CTRL_PIN_EXEC_CONTROLS, &val);
5030	Log(("VMX_VMCS_CTRL_PIN_EXEC_CONTROLS %08x\n", val));
5031	VMXReadVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, &val);
5032	Log(("VMX_VMCS_CTRL_PROC_EXEC_CONTROLS %08x\n", val));
5033	VMXReadVMCS(VMX_VMCS_CTRL_ENTRY_CONTROLS, &val);
5034	Log(("VMX_VMCS_CTRL_ENTRY_CONTROLS %08x\n", val));
5035	VMXReadVMCS(VMX_VMCS_CTRL_EXIT_CONTROLS, &val);
5036	Log(("VMX_VMCS_CTRL_EXIT_CONTROLS %08x\n", val));
5037
5038	VMXReadVMCS(VMX_VMCS_HOST_CR0, &val);
5039	Log(("VMX_VMCS_HOST_CR0 %08x\n", val));
5040	VMXReadVMCS(VMX_VMCS_HOST_CR3, &val);
5041	Log(("VMX_VMCS_HOST_CR3 %08x\n", val));
5042	VMXReadVMCS(VMX_VMCS_HOST_CR4, &val);
5043	Log(("VMX_VMCS_HOST_CR4 %08x\n", val));
5044
5045	VMXReadVMCS(VMX_VMCS16_HOST_FIELD_CS, &val);
5046	Log(("VMX_VMCS_HOST_FIELD_CS %08x\n", val));
5047	VMXReadVMCS(VMX_VMCS_GUEST_RFLAGS, &val);
5048	Log(("VMX_VMCS_GUEST_RFLAGS %08x\n", val));
5049
5050	if (val < gdtr.cbGdt)
5051	{
5052	pDesc = (PCX86DESCHC)(gdtr.pGdt + (val & X86_SEL_MASK));
5053	HWACCMR0DumpDescriptor(pDesc, val, "CS: ");
5054	}
5055
5056	VMXReadVMCS(VMX_VMCS16_HOST_FIELD_DS, &val);
5057	Log(("VMX_VMCS_HOST_FIELD_DS %08x\n", val));
5058	if (val < gdtr.cbGdt)
5059	{
5060	pDesc = (PCX86DESCHC)(gdtr.pGdt + (val & X86_SEL_MASK));
5061	HWACCMR0DumpDescriptor(pDesc, val, "DS: ");
5062	}
5063
5064	VMXReadVMCS(VMX_VMCS16_HOST_FIELD_ES, &val);
5065	Log(("VMX_VMCS_HOST_FIELD_ES %08x\n", val));
5066	if (val < gdtr.cbGdt)
5067	{
5068	pDesc = (PCX86DESCHC)(gdtr.pGdt + (val & X86_SEL_MASK));
5069	HWACCMR0DumpDescriptor(pDesc, val, "ES: ");
5070	}
5071
5072	VMXReadVMCS(VMX_VMCS16_HOST_FIELD_FS, &val);
5073	Log(("VMX_VMCS16_HOST_FIELD_FS %08x\n", val));
5074	if (val < gdtr.cbGdt)
5075	{
5076	pDesc = (PCX86DESCHC)(gdtr.pGdt + (val & X86_SEL_MASK));
5077	HWACCMR0DumpDescriptor(pDesc, val, "FS: ");
5078	}
5079
5080	VMXReadVMCS(VMX_VMCS16_HOST_FIELD_GS, &val);
5081	Log(("VMX_VMCS16_HOST_FIELD_GS %08x\n", val));
5082	if (val < gdtr.cbGdt)
5083	{
5084	pDesc = (PCX86DESCHC)(gdtr.pGdt + (val & X86_SEL_MASK));
5085	HWACCMR0DumpDescriptor(pDesc, val, "GS: ");
5086	}
5087
5088	VMXReadVMCS(VMX_VMCS16_HOST_FIELD_SS, &val);
5089	Log(("VMX_VMCS16_HOST_FIELD_SS %08x\n", val));
5090	if (val < gdtr.cbGdt)
5091	{
5092	pDesc = (PCX86DESCHC)(gdtr.pGdt + (val & X86_SEL_MASK));
5093	HWACCMR0DumpDescriptor(pDesc, val, "SS: ");
5094	}
5095
5096	VMXReadVMCS(VMX_VMCS16_HOST_FIELD_TR, &val);
5097	Log(("VMX_VMCS16_HOST_FIELD_TR %08x\n", val));
5098	if (val < gdtr.cbGdt)
5099	{
5100	pDesc = (PCX86DESCHC)(gdtr.pGdt + (val & X86_SEL_MASK));
5101	HWACCMR0DumpDescriptor(pDesc, val, "TR: ");
5102	}
5103
5104	VMXReadVMCS(VMX_VMCS_HOST_TR_BASE, &val);
5105	Log(("VMX_VMCS_HOST_TR_BASE %RHv\n", val));
5106	VMXReadVMCS(VMX_VMCS_HOST_GDTR_BASE, &val);
5107	Log(("VMX_VMCS_HOST_GDTR_BASE %RHv\n", val));
5108	VMXReadVMCS(VMX_VMCS_HOST_IDTR_BASE, &val);
5109	Log(("VMX_VMCS_HOST_IDTR_BASE %RHv\n", val));
5110	VMXReadVMCS(VMX_VMCS32_HOST_SYSENTER_CS, &val);
5111	Log(("VMX_VMCS_HOST_SYSENTER_CS %08x\n", val));
5112	VMXReadVMCS(VMX_VMCS_HOST_SYSENTER_EIP, &val);
5113	Log(("VMX_VMCS_HOST_SYSENTER_EIP %RHv\n", val));
5114	VMXReadVMCS(VMX_VMCS_HOST_SYSENTER_ESP, &val);
5115	Log(("VMX_VMCS_HOST_SYSENTER_ESP %RHv\n", val));
5116	VMXReadVMCS(VMX_VMCS_HOST_RSP, &val);
5117	Log(("VMX_VMCS_HOST_RSP %RHv\n", val));
5118	VMXReadVMCS(VMX_VMCS_HOST_RIP, &val);
5119	Log(("VMX_VMCS_HOST_RIP %RHv\n", val));
5120	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
5121	if (VMX_IS_64BIT_HOST_MODE())
5122	{
5123	Log(("MSR_K6_EFER = %RX64\n", ASMRdMsr(MSR_K6_EFER)));
5124	Log(("MSR_K6_STAR = %RX64\n", ASMRdMsr(MSR_K6_STAR)));
5125	Log(("MSR_K8_LSTAR = %RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
5126	Log(("MSR_K8_CSTAR = %RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
5127	Log(("MSR_K8_SF_MASK = %RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
5128	}
5129	# endif
5130	#endif /* VBOX_STRICT */
5131	}
5132	break;
5133	}
5134
5135	default:
5136	/* impossible */
5137	AssertMsgFailed(("%Rrc (%#x)\n", VBOXSTRICTRC_VAL(rc), VBOXSTRICTRC_VAL(rc)));
5138	break;
5139	}
5140	}
5141
5142
5143	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
5144	/**
5145	* Prepares for and executes VMLAUNCH (64 bits guest mode).
5146	*
5147	* @returns VBox status code.
5148	* @param fResume Whether to vmlauch/vmresume.
5149	* @param pCtx Pointer to the guest CPU context.
5150	* @param pCache Pointer to the VMCS cache.
5151	* @param pVM Pointer to the VM.
5152	* @param pVCpu Pointer to the VMCPU.
5153	*/
5154	DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
5155	{
5156	uint32_t aParam[6];
5157	PHMGLOBLCPUINFO pCpu;
5158	RTHCPHYS HCPhysCpuPage;
5159	int rc;
5160
5161	pCpu = HWACCMR0GetCurrentCpu();
5162	HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
5163
5164	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5165	pCache->uPos = 1;
5166	pCache->interPD = PGMGetInterPaeCR3(pVM);
5167	pCache->pSwitcher = (uint64_t)pVM->hwaccm.s.pfnHost32ToGuest64R0;
5168	#endif
5169
5170	#ifdef DEBUG
5171	pCache->TestIn.HCPhysCpuPage= 0;
5172	pCache->TestIn.HCPhysVMCS = 0;
5173	pCache->TestIn.pCache = 0;
5174	pCache->TestOut.HCPhysVMCS = 0;
5175	pCache->TestOut.pCache = 0;
5176	pCache->TestOut.pCtx = 0;
5177	pCache->TestOut.eflags = 0;
5178	#endif
5179
5180	aParam[0] = (uint32_t)(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
5181	aParam[1] = (uint32_t)(HCPhysCpuPage >> 32); /* Param 1: VMXON physical address - Hi. */
5182	aParam[2] = (uint32_t)(pVCpu->hwaccm.s.vmx.HCPhysVMCS); /* Param 2: VMCS physical address - Lo. */
5183	aParam[3] = (uint32_t)(pVCpu->hwaccm.s.vmx.HCPhysVMCS >> 32); /* Param 2: VMCS physical address - Hi. */
5184	aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hwaccm.s.vmx.VMCSCache);
5185	aParam[5] = 0;
5186
5187	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5188	pCtx->dr[4] = pVM->hwaccm.s.vmx.pScratchPhys + 16 + 8;
5189	(uint32_t )(pVM->hwaccm.s.vmx.pScratch + 16 + 8) = 1;
5190	#endif
5191	rc = VMXR0Execute64BitsHandler(pVM, pVCpu, pCtx, pVM->hwaccm.s.pfnVMXGCStartVM64, 6, &aParam[0]);
5192
5193	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5194	Assert((uint32_t )(pVM->hwaccm.s.vmx.pScratch + 16 + 8) == 5);
5195	Assert(pCtx->dr[4] == 10);
5196	(uint32_t )(pVM->hwaccm.s.vmx.pScratch + 16 + 8) = 0xff;
5197	#endif
5198
5199	#ifdef DEBUG
5200	AssertMsg(pCache->TestIn.HCPhysCpuPage== HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
5201	AssertMsg(pCache->TestIn.HCPhysVMCS == pVCpu->hwaccm.s.vmx.HCPhysVMCS, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVMCS,
5202	pVCpu->hwaccm.s.vmx.HCPhysVMCS));
5203	AssertMsg(pCache->TestIn.HCPhysVMCS == pCache->TestOut.HCPhysVMCS, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVMCS,
5204	pCache->TestOut.HCPhysVMCS));
5205	AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
5206	pCache->TestOut.pCache));
5207	AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hwaccm.s.vmx.VMCSCache),
5208	("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hwaccm.s.vmx.VMCSCache)));
5209	AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
5210	pCache->TestOut.pCtx));
5211	Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
5212	#endif
5213	return rc;
5214	}
5215
5216
5217	# ifdef VBOX_STRICT
5218	static bool hmR0VmxIsValidReadField(uint32_t idxField)
5219	{
5220	switch (idxField)
5221	{
5222	case VMX_VMCS64_GUEST_RIP:
5223	case VMX_VMCS64_GUEST_RSP:
5224	case VMX_VMCS_GUEST_RFLAGS:
5225	case VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE:
5226	case VMX_VMCS_CTRL_CR0_READ_SHADOW:
5227	case VMX_VMCS64_GUEST_CR0:
5228	case VMX_VMCS_CTRL_CR4_READ_SHADOW:
5229	case VMX_VMCS64_GUEST_CR4:
5230	case VMX_VMCS64_GUEST_DR7:
5231	case VMX_VMCS32_GUEST_SYSENTER_CS:
5232	case VMX_VMCS64_GUEST_SYSENTER_EIP:
5233	case VMX_VMCS64_GUEST_SYSENTER_ESP:
5234	case VMX_VMCS32_GUEST_GDTR_LIMIT:
5235	case VMX_VMCS64_GUEST_GDTR_BASE:
5236	case VMX_VMCS32_GUEST_IDTR_LIMIT:
5237	case VMX_VMCS64_GUEST_IDTR_BASE:
5238	case VMX_VMCS16_GUEST_FIELD_CS:
5239	case VMX_VMCS32_GUEST_CS_LIMIT:
5240	case VMX_VMCS64_GUEST_CS_BASE:
5241	case VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS:
5242	case VMX_VMCS16_GUEST_FIELD_DS:
5243	case VMX_VMCS32_GUEST_DS_LIMIT:
5244	case VMX_VMCS64_GUEST_DS_BASE:
5245	case VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS:
5246	case VMX_VMCS16_GUEST_FIELD_ES:
5247	case VMX_VMCS32_GUEST_ES_LIMIT:
5248	case VMX_VMCS64_GUEST_ES_BASE:
5249	case VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS:
5250	case VMX_VMCS16_GUEST_FIELD_FS:
5251	case VMX_VMCS32_GUEST_FS_LIMIT:
5252	case VMX_VMCS64_GUEST_FS_BASE:
5253	case VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS:
5254	case VMX_VMCS16_GUEST_FIELD_GS:
5255	case VMX_VMCS32_GUEST_GS_LIMIT:
5256	case VMX_VMCS64_GUEST_GS_BASE:
5257	case VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS:
5258	case VMX_VMCS16_GUEST_FIELD_SS:
5259	case VMX_VMCS32_GUEST_SS_LIMIT:
5260	case VMX_VMCS64_GUEST_SS_BASE:
5261	case VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS:
5262	case VMX_VMCS16_GUEST_FIELD_LDTR:
5263	case VMX_VMCS32_GUEST_LDTR_LIMIT:
5264	case VMX_VMCS64_GUEST_LDTR_BASE:
5265	case VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS:
5266	case VMX_VMCS16_GUEST_FIELD_TR:
5267	case VMX_VMCS32_GUEST_TR_LIMIT:
5268	case VMX_VMCS64_GUEST_TR_BASE:
5269	case VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS:
5270	case VMX_VMCS32_RO_EXIT_REASON:
5271	case VMX_VMCS32_RO_VM_INSTR_ERROR:
5272	case VMX_VMCS32_RO_EXIT_INSTR_LENGTH:
5273	case VMX_VMCS32_RO_EXIT_INTERRUPTION_ERRCODE:
5274	case VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO:
5275	case VMX_VMCS32_RO_EXIT_INSTR_INFO:
5276	case VMX_VMCS_RO_EXIT_QUALIFICATION:
5277	case VMX_VMCS32_RO_IDT_INFO:
5278	case VMX_VMCS32_RO_IDT_ERRCODE:
5279	case VMX_VMCS64_GUEST_CR3:
5280	case VMX_VMCS_EXIT_PHYS_ADDR_FULL:
5281	return true;
5282	}
5283	return false;
5284	}
5285
5286
5287	static bool hmR0VmxIsValidWriteField(uint32_t idxField)
5288	{
5289	switch (idxField)
5290	{
5291	case VMX_VMCS64_GUEST_LDTR_BASE:
5292	case VMX_VMCS64_GUEST_TR_BASE:
5293	case VMX_VMCS64_GUEST_GDTR_BASE:
5294	case VMX_VMCS64_GUEST_IDTR_BASE:
5295	case VMX_VMCS64_GUEST_SYSENTER_EIP:
5296	case VMX_VMCS64_GUEST_SYSENTER_ESP:
5297	case VMX_VMCS64_GUEST_CR0:
5298	case VMX_VMCS64_GUEST_CR4:
5299	case VMX_VMCS64_GUEST_CR3:
5300	case VMX_VMCS64_GUEST_DR7:
5301	case VMX_VMCS64_GUEST_RIP:
5302	case VMX_VMCS64_GUEST_RSP:
5303	case VMX_VMCS64_GUEST_CS_BASE:
5304	case VMX_VMCS64_GUEST_DS_BASE:
5305	case VMX_VMCS64_GUEST_ES_BASE:
5306	case VMX_VMCS64_GUEST_FS_BASE:
5307	case VMX_VMCS64_GUEST_GS_BASE:
5308	case VMX_VMCS64_GUEST_SS_BASE:
5309	return true;
5310	}
5311	return false;
5312	}
5313	# endif /* VBOX_STRICT */
5314
5315
5316	/**
5317	* Executes the specified handler in 64-bit mode.
5318	*
5319	* @returns VBox status code.
5320	* @param pVM Pointer to the VM.
5321	* @param pVCpu Pointer to the VMCPU.
5322	* @param pCtx Pointer to the guest CPU context.
5323	* @param pfnHandler Pointer to the RC handler function.
5324	* @param cbParam Number of parameters.
5325	* @param paParam Array of 32-bit parameters.
5326	*/
5327	VMMR0DECL(int) VMXR0Execute64BitsHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, RTRCPTR pfnHandler, uint32_t cbParam,
5328	uint32_t *paParam)
5329	{
5330	int rc, rc2;
5331	PHMGLOBLCPUINFO pCpu;
5332	RTHCPHYS HCPhysCpuPage;
5333	RTHCUINTREG uOldEFlags;
5334
5335	AssertReturn(pVM->hwaccm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
5336	Assert(pfnHandler);
5337	Assert(pVCpu->hwaccm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hwaccm.s.vmx.VMCSCache.Write.aField));
5338	Assert(pVCpu->hwaccm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hwaccm.s.vmx.VMCSCache.Read.aField));
5339
5340	#ifdef VBOX_STRICT
5341	for (unsigned i=0;i<pVCpu->hwaccm.s.vmx.VMCSCache.Write.cValidEntries;i++)
5342	Assert(hmR0VmxIsValidWriteField(pVCpu->hwaccm.s.vmx.VMCSCache.Write.aField[i]));
5343
5344	for (unsigned i=0;i<pVCpu->hwaccm.s.vmx.VMCSCache.Read.cValidEntries;i++)
5345	Assert(hmR0VmxIsValidReadField(pVCpu->hwaccm.s.vmx.VMCSCache.Read.aField[i]));
5346	#endif
5347
5348	/* Disable interrupts. */
5349	uOldEFlags = ASMIntDisableFlags();
5350
5351	#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
5352	RTCPUID idHostCpu = RTMpCpuId();
5353	CPUMR0SetLApic(pVM, idHostCpu);
5354	#endif
5355
5356	pCpu = HWACCMR0GetCurrentCpu();
5357	HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
5358
5359	/* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
5360	VMXClearVMCS(pVCpu->hwaccm.s.vmx.HCPhysVMCS);
5361
5362	/* Leave VMX Root Mode. */
5363	VMXDisable();
5364
5365	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
5366
5367	CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
5368	CPUMSetHyperEIP(pVCpu, pfnHandler);
5369	for (int i=(int)cbParam-1;i>=0;i--)
5370	CPUMPushHyper(pVCpu, paParam[i]);
5371
5372	STAM_PROFILE_ADV_START(&pVCpu->hwaccm.s.StatWorldSwitch3264, z);
5373
5374	/* Call switcher. */
5375	rc = pVM->hwaccm.s.pfnHost32ToGuest64R0(pVM, RT_OFFSETOF(VM, aCpus[pVCpu->idCpu].cpum) - RT_OFFSETOF(VM, cpum));
5376	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatWorldSwitch3264, z);
5377
5378	/* Make sure the VMX instructions don't cause #UD faults. */
5379	ASMSetCR4(ASMGetCR4() \| X86_CR4_VMXE);
5380
5381	/* Enter VMX Root Mode */
5382	rc2 = VMXEnable(HCPhysCpuPage);
5383	if (RT_FAILURE(rc2))
5384	{
5385	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
5386	ASMSetFlags(uOldEFlags);
5387	return VERR_VMX_VMXON_FAILED;
5388	}
5389
5390	rc2 = VMXActivateVMCS(pVCpu->hwaccm.s.vmx.HCPhysVMCS);
5391	AssertRC(rc2);
5392	Assert(!(ASMGetFlags() & X86_EFL_IF));
5393	ASMSetFlags(uOldEFlags);
5394	return rc;
5395	}
5396	#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL) */
5397
5398
5399	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
5400	/**
5401	* Executes VMWRITE.
5402	*
5403	* @returns VBox status code
5404	* @param pVCpu Pointer to the VMCPU.
5405	* @param idxField VMCS field index.
5406	* @param u64Val 16, 32 or 64 bits value.
5407	*/
5408	VMMR0DECL(int) VMXWriteVMCS64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5409	{
5410	int rc;
5411	switch (idxField)
5412	{
5413	case VMX_VMCS_CTRL_TSC_OFFSET_FULL:
5414	case VMX_VMCS_CTRL_IO_BITMAP_A_FULL:
5415	case VMX_VMCS_CTRL_IO_BITMAP_B_FULL:
5416	case VMX_VMCS_CTRL_MSR_BITMAP_FULL:
5417	case VMX_VMCS_CTRL_VMEXIT_MSR_STORE_FULL:
5418	case VMX_VMCS_CTRL_VMEXIT_MSR_LOAD_FULL:
5419	case VMX_VMCS_CTRL_VMENTRY_MSR_LOAD_FULL:
5420	case VMX_VMCS_CTRL_VAPIC_PAGEADDR_FULL:
5421	case VMX_VMCS_CTRL_APIC_ACCESSADDR_FULL:
5422	case VMX_VMCS_GUEST_LINK_PTR_FULL:
5423	case VMX_VMCS_GUEST_PDPTR0_FULL:
5424	case VMX_VMCS_GUEST_PDPTR1_FULL:
5425	case VMX_VMCS_GUEST_PDPTR2_FULL:
5426	case VMX_VMCS_GUEST_PDPTR3_FULL:
5427	case VMX_VMCS_GUEST_DEBUGCTL_FULL:
5428	case VMX_VMCS_GUEST_EFER_FULL:
5429	case VMX_VMCS_CTRL_EPTP_FULL:
5430	/* These fields consist of two parts, which are both writable in 32 bits mode. */
5431	rc = VMXWriteVMCS32(idxField, u64Val);
5432	rc \|= VMXWriteVMCS32(idxField + 1, (uint32_t)(u64Val >> 32ULL));
5433	AssertRC(rc);
5434	return rc;
5435
5436	case VMX_VMCS64_GUEST_LDTR_BASE:
5437	case VMX_VMCS64_GUEST_TR_BASE:
5438	case VMX_VMCS64_GUEST_GDTR_BASE:
5439	case VMX_VMCS64_GUEST_IDTR_BASE:
5440	case VMX_VMCS64_GUEST_SYSENTER_EIP:
5441	case VMX_VMCS64_GUEST_SYSENTER_ESP:
5442	case VMX_VMCS64_GUEST_CR0:
5443	case VMX_VMCS64_GUEST_CR4:
5444	case VMX_VMCS64_GUEST_CR3:
5445	case VMX_VMCS64_GUEST_DR7:
5446	case VMX_VMCS64_GUEST_RIP:
5447	case VMX_VMCS64_GUEST_RSP:
5448	case VMX_VMCS64_GUEST_CS_BASE:
5449	case VMX_VMCS64_GUEST_DS_BASE:
5450	case VMX_VMCS64_GUEST_ES_BASE:
5451	case VMX_VMCS64_GUEST_FS_BASE:
5452	case VMX_VMCS64_GUEST_GS_BASE:
5453	case VMX_VMCS64_GUEST_SS_BASE:
5454	/* Queue a 64 bits value as we can't set it in 32 bits host mode. */
5455	if (u64Val >> 32ULL)
5456	rc = VMXWriteCachedVMCSEx(pVCpu, idxField, u64Val);
5457	else
5458	rc = VMXWriteVMCS32(idxField, (uint32_t)u64Val);
5459
5460	return rc;
5461
5462	default:
5463	AssertMsgFailed(("Unexpected field %x\n", idxField));
5464	return VERR_INVALID_PARAMETER;
5465	}
5466	}
5467
5468
5469	/**
5470	* Cache VMCS writes for performance reasons (Darwin) and for running 64 bits guests on 32 bits hosts.
5471	*
5472	* @param pVCpu Pointer to the VMCPU.
5473	* @param idxField VMCS field index.
5474	* @param u64Val 16, 32 or 64 bits value.
5475	*/
5476	VMMR0DECL(int) VMXWriteCachedVMCSEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5477	{
5478	PVMCSCACHE pCache = &pVCpu->hwaccm.s.vmx.VMCSCache;
5479
5480	AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1,
5481	("entries=%x\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
5482
5483	/* Make sure there are no duplicates. */
5484	for (unsigned i = 0; i < pCache->Write.cValidEntries; i++)
5485	{
5486	if (pCache->Write.aField[i] == idxField)
5487	{
5488	pCache->Write.aFieldVal[i] = u64Val;
5489	return VINF_SUCCESS;
5490	}
5491	}
5492
5493	pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
5494	pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
5495	pCache->Write.cValidEntries++;
5496	return VINF_SUCCESS;
5497	}
5498
5499	#endif /* HC_ARCH_BITS == 32 && !VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0 */
5500

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMR0/HWVMXR0.cpp@ 41834

Download in other formats: