HWVMXR0.cpp@ 36641

Last change on this file since 36641 was 36641, checked in by vboxsync, 14 years ago
VMX: Do not mangle 16-bit TR into 32-bit one.
Property svn:eol-style set to `native` Property svn:keywords set to `Id`
File size: 198.7 KB

Line
1	/* $Id: HWVMXR0.cpp 36641 2011-04-11 12:50:36Z vboxsync $ */
2	/** @file
3	* HWACCM VMX - Host Context Ring 0.
4	*/
5
6	/*
7	* Copyright (C) 2006-2007 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18
19	/*******************************************************************************
20	* Header Files *
21	*******************************************************************************/
22	#define LOG_GROUP LOG_GROUP_HWACCM
23	#include <iprt/asm-amd64-x86.h>
24	#include <VBox/vmm/hwaccm.h>
25	#include <VBox/vmm/pgm.h>
26	#include <VBox/vmm/dbgf.h>
27	#include <VBox/vmm/selm.h>
28	#include <VBox/vmm/iom.h>
29	#include <VBox/vmm/rem.h>
30	#include <VBox/vmm/tm.h>
31	#include "HWACCMInternal.h"
32	#include <VBox/vmm/vm.h>
33	#include <VBox/x86.h>
34	#include <VBox/vmm/pdmapi.h>
35	#include <VBox/err.h>
36	#include <VBox/log.h>
37	#include <iprt/assert.h>
38	#include <iprt/param.h>
39	#include <iprt/string.h>
40	#include <iprt/time.h>
41	#ifdef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
42	# include <iprt/thread.h>
43	#endif
44	#include "HWVMXR0.h"
45
46	/*******************************************************************************
47	* Defined Constants And Macros *
48	*******************************************************************************/
49	#if defined(RT_ARCH_AMD64)
50	# define VMX_IS_64BIT_HOST_MODE() (true)
51	#elif defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
52	# define VMX_IS_64BIT_HOST_MODE() (g_fVMXIs64bitHost != 0)
53	#else
54	# define VMX_IS_64BIT_HOST_MODE() (false)
55	#endif
56
57	/*******************************************************************************
58	* Global Variables *
59	*******************************************************************************/
60	/* IO operation lookup arrays. */
61	static uint32_t const g_aIOSize[4] = {1, 2, 0, 4};
62	static uint32_t const g_aIOOpAnd[4] = {0xff, 0xffff, 0, 0xffffffff};
63
64	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
65	/** See HWACCMR0A.asm. */
66	extern "C" uint32_t g_fVMXIs64bitHost;
67	#endif
68
69	/*******************************************************************************
70	* Local Functions *
71	*******************************************************************************/
72	static void VMXR0ReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, VBOXSTRICTRC rc, PCPUMCTX pCtx);
73	static void vmxR0SetupTLBEPT(PVM pVM, PVMCPU pVCpu);
74	static void vmxR0SetupTLBVPID(PVM pVM, PVMCPU pVCpu);
75	static void vmxR0SetupTLBDummy(PVM pVM, PVMCPU pVCpu);
76	static void vmxR0FlushEPT(PVM pVM, PVMCPU pVCpu, VMX_FLUSH enmFlush, RTGCPHYS GCPhys);
77	static void vmxR0FlushVPID(PVM pVM, PVMCPU pVCpu, VMX_FLUSH enmFlush, RTGCPTR GCPtr);
78	static void vmxR0UpdateExceptionBitmap(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx);
79	#ifdef VBOX_STRICT
80	static bool vmxR0IsValidReadField(uint32_t idxField);
81	static bool vmxR0IsValidWriteField(uint32_t idxField);
82	#endif
83	static void vmxR0SetMSRPermission(PVMCPU pVCpu, unsigned ulMSR, bool fRead, bool fWrite);
84
85	static void VMXR0CheckError(PVM pVM, PVMCPU pVCpu, int rc)
86	{
87	if (rc == VERR_VMX_GENERIC)
88	{
89	RTCCUINTREG instrError;
90
91	VMXReadVMCS(VMX_VMCS32_RO_VM_INSTR_ERROR, &instrError);
92	pVCpu->hwaccm.s.vmx.lasterror.ulInstrError = instrError;
93	}
94	pVM->hwaccm.s.lLastError = rc;
95	}
96
97	/**
98	* Sets up and activates VT-x on the current CPU
99	*
100	* @returns VBox status code.
101	* @param pCpu CPU info struct
102	* @param pVM The VM to operate on. (can be NULL after a resume!!)
103	* @param pvPageCpu Pointer to the global cpu page
104	* @param pPageCpuPhys Physical address of the global cpu page
105	*/
106	VMMR0DECL(int) VMXR0EnableCpu(PHWACCM_CPUINFO pCpu, PVM pVM, void *pvPageCpu, RTHCPHYS pPageCpuPhys)
107	{
108	AssertReturn(pPageCpuPhys, VERR_INVALID_PARAMETER);
109	AssertReturn(pvPageCpu, VERR_INVALID_PARAMETER);
110
111	if (pVM)
112	{
113	/* Set revision dword at the beginning of the VMXON structure. */
114	(uint32_t )pvPageCpu = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hwaccm.s.vmx.msr.vmx_basic_info);
115	}
116
117	/** @todo we should unmap the two pages from the virtual address space in order to prevent accidental corruption.
118	* (which can have very bad consequences!!!)
119	*/
120
121	if (ASMGetCR4() & X86_CR4_VMXE)
122	return VERR_VMX_IN_VMX_ROOT_MODE;
123
124	/* Make sure the VMX instructions don't cause #UD faults. */
125	ASMSetCR4(ASMGetCR4() \| X86_CR4_VMXE);
126
127	/* Enter VMX Root Mode */
128	int rc = VMXEnable(pPageCpuPhys);
129	if (RT_FAILURE(rc))
130	{
131	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
132	return VERR_VMX_VMXON_FAILED;
133	}
134	return VINF_SUCCESS;
135	}
136
137	/**
138	* Deactivates VT-x on the current CPU
139	*
140	* @returns VBox status code.
141	* @param pCpu CPU info struct
142	* @param pvPageCpu Pointer to the global cpu page
143	* @param pPageCpuPhys Physical address of the global cpu page
144	*/
145	VMMR0DECL(int) VMXR0DisableCpu(PHWACCM_CPUINFO pCpu, void *pvPageCpu, RTHCPHYS pPageCpuPhys)
146	{
147	AssertReturn(pPageCpuPhys, VERR_INVALID_PARAMETER);
148	AssertReturn(pvPageCpu, VERR_INVALID_PARAMETER);
149
150	/* If we're somehow not in VMX root mode, then we shouldn't dare leaving it. */
151	if (!(ASMGetCR4() & X86_CR4_VMXE))
152	return VERR_VMX_NOT_IN_VMX_ROOT_MODE;
153
154	/* Leave VMX Root Mode. */
155	VMXDisable();
156
157	/* And clear the X86_CR4_VMXE bit */
158	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
159	return VINF_SUCCESS;
160	}
161
162	/**
163	* Does Ring-0 per VM VT-x init.
164	*
165	* @returns VBox status code.
166	* @param pVM The VM to operate on.
167	*/
168	VMMR0DECL(int) VMXR0InitVM(PVM pVM)
169	{
170	int rc;
171
172	#ifdef LOG_ENABLED
173	SUPR0Printf("VMXR0InitVM %x\n", pVM);
174	#endif
175
176	pVM->hwaccm.s.vmx.pMemObjAPIC = NIL_RTR0MEMOBJ;
177
178	if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
179	{
180	/* Allocate one page for the APIC physical page (serves for filtering accesses). */
181	rc = RTR0MemObjAllocCont(&pVM->hwaccm.s.vmx.pMemObjAPIC, 1 << PAGE_SHIFT, true /* executable R0 mapping */);
182	AssertRC(rc);
183	if (RT_FAILURE(rc))
184	return rc;
185
186	pVM->hwaccm.s.vmx.pAPIC = (uint8_t *)RTR0MemObjAddress(pVM->hwaccm.s.vmx.pMemObjAPIC);
187	pVM->hwaccm.s.vmx.pAPICPhys = RTR0MemObjGetPagePhysAddr(pVM->hwaccm.s.vmx.pMemObjAPIC, 0);
188	ASMMemZero32(pVM->hwaccm.s.vmx.pAPIC, PAGE_SIZE);
189	}
190	else
191	{
192	pVM->hwaccm.s.vmx.pMemObjAPIC = 0;
193	pVM->hwaccm.s.vmx.pAPIC = 0;
194	pVM->hwaccm.s.vmx.pAPICPhys = 0;
195	}
196
197	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
198	{
199	rc = RTR0MemObjAllocCont(&pVM->hwaccm.s.vmx.pMemObjScratch, 1 << PAGE_SHIFT, true /* executable R0 mapping */);
200	AssertRC(rc);
201	if (RT_FAILURE(rc))
202	return rc;
203
204	pVM->hwaccm.s.vmx.pScratch = (uint8_t *)RTR0MemObjAddress(pVM->hwaccm.s.vmx.pMemObjScratch);
205	pVM->hwaccm.s.vmx.pScratchPhys = RTR0MemObjGetPagePhysAddr(pVM->hwaccm.s.vmx.pMemObjScratch, 0);
206
207	ASMMemZero32(pVM->hwaccm.s.vmx.pScratch, PAGE_SIZE);
208	strcpy((char *)pVM->hwaccm.s.vmx.pScratch, "SCRATCH Magic");
209	(uint64_t )(pVM->hwaccm.s.vmx.pScratch + 16) = UINT64_C(0xDEADBEEFDEADBEEF);
210	}
211	#endif
212
213	/* Allocate VMCBs for all guest CPUs. */
214	for (VMCPUID i = 0; i < pVM->cCpus; i++)
215	{
216	PVMCPU pVCpu = &pVM->aCpus[i];
217
218	pVCpu->hwaccm.s.vmx.pMemObjVMCS = NIL_RTR0MEMOBJ;
219
220	/* Allocate one page for the VM control structure (VMCS). */
221	rc = RTR0MemObjAllocCont(&pVCpu->hwaccm.s.vmx.pMemObjVMCS, 1 << PAGE_SHIFT, true /* executable R0 mapping */);
222	AssertRC(rc);
223	if (RT_FAILURE(rc))
224	return rc;
225
226	pVCpu->hwaccm.s.vmx.pVMCS = RTR0MemObjAddress(pVCpu->hwaccm.s.vmx.pMemObjVMCS);
227	pVCpu->hwaccm.s.vmx.pVMCSPhys = RTR0MemObjGetPagePhysAddr(pVCpu->hwaccm.s.vmx.pMemObjVMCS, 0);
228	ASMMemZero32(pVCpu->hwaccm.s.vmx.pVMCS, PAGE_SIZE);
229
230	pVCpu->hwaccm.s.vmx.cr0_mask = 0;
231	pVCpu->hwaccm.s.vmx.cr4_mask = 0;
232
233	/* Allocate one page for the virtual APIC page for TPR caching. */
234	rc = RTR0MemObjAllocCont(&pVCpu->hwaccm.s.vmx.pMemObjVAPIC, 1 << PAGE_SHIFT, true /* executable R0 mapping */);
235	AssertRC(rc);
236	if (RT_FAILURE(rc))
237	return rc;
238
239	pVCpu->hwaccm.s.vmx.pVAPIC = (uint8_t *)RTR0MemObjAddress(pVCpu->hwaccm.s.vmx.pMemObjVAPIC);
240	pVCpu->hwaccm.s.vmx.pVAPICPhys = RTR0MemObjGetPagePhysAddr(pVCpu->hwaccm.s.vmx.pMemObjVAPIC, 0);
241	ASMMemZero32(pVCpu->hwaccm.s.vmx.pVAPIC, PAGE_SIZE);
242
243	/* Allocate the MSR bitmap if this feature is supported. */
244	if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
245	{
246	rc = RTR0MemObjAllocCont(&pVCpu->hwaccm.s.vmx.pMemObjMSRBitmap, 1 << PAGE_SHIFT, true /* executable R0 mapping */);
247	AssertRC(rc);
248	if (RT_FAILURE(rc))
249	return rc;
250
251	pVCpu->hwaccm.s.vmx.pMSRBitmap = (uint8_t *)RTR0MemObjAddress(pVCpu->hwaccm.s.vmx.pMemObjMSRBitmap);
252	pVCpu->hwaccm.s.vmx.pMSRBitmapPhys = RTR0MemObjGetPagePhysAddr(pVCpu->hwaccm.s.vmx.pMemObjMSRBitmap, 0);
253	memset(pVCpu->hwaccm.s.vmx.pMSRBitmap, 0xff, PAGE_SIZE);
254	}
255
256	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
257	/* Allocate one page for the guest MSR load area (for preloading guest MSRs during the world switch). */
258	rc = RTR0MemObjAllocCont(&pVCpu->hwaccm.s.vmx.pMemObjGuestMSR, 1 << PAGE_SHIFT, true /* executable R0 mapping */);
259	AssertRC(rc);
260	if (RT_FAILURE(rc))
261	return rc;
262
263	pVCpu->hwaccm.s.vmx.pGuestMSR = (uint8_t *)RTR0MemObjAddress(pVCpu->hwaccm.s.vmx.pMemObjGuestMSR);
264	pVCpu->hwaccm.s.vmx.pGuestMSRPhys = RTR0MemObjGetPagePhysAddr(pVCpu->hwaccm.s.vmx.pMemObjGuestMSR, 0);
265	memset(pVCpu->hwaccm.s.vmx.pGuestMSR, 0, PAGE_SIZE);
266
267	/* Allocate one page for the host MSR load area (for restoring host MSRs after the world switch back). */
268	rc = RTR0MemObjAllocCont(&pVCpu->hwaccm.s.vmx.pMemObjHostMSR, 1 << PAGE_SHIFT, true /* executable R0 mapping */);
269	AssertRC(rc);
270	if (RT_FAILURE(rc))
271	return rc;
272
273	pVCpu->hwaccm.s.vmx.pHostMSR = (uint8_t *)RTR0MemObjAddress(pVCpu->hwaccm.s.vmx.pMemObjHostMSR);
274	pVCpu->hwaccm.s.vmx.pHostMSRPhys = RTR0MemObjGetPagePhysAddr(pVCpu->hwaccm.s.vmx.pMemObjHostMSR, 0);
275	memset(pVCpu->hwaccm.s.vmx.pHostMSR, 0, PAGE_SIZE);
276	#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
277
278	/* Current guest paging mode. */
279	pVCpu->hwaccm.s.vmx.enmLastSeenGuestMode = PGMMODE_REAL;
280
281	#ifdef LOG_ENABLED
282	SUPR0Printf("VMXR0InitVM %x VMCS=%x (%x)\n", pVM, pVCpu->hwaccm.s.vmx.pVMCS, (uint32_t)pVCpu->hwaccm.s.vmx.pVMCSPhys);
283	#endif
284	}
285
286	return VINF_SUCCESS;
287	}
288
289	/**
290	* Does Ring-0 per VM VT-x termination.
291	*
292	* @returns VBox status code.
293	* @param pVM The VM to operate on.
294	*/
295	VMMR0DECL(int) VMXR0TermVM(PVM pVM)
296	{
297	for (VMCPUID i = 0; i < pVM->cCpus; i++)
298	{
299	PVMCPU pVCpu = &pVM->aCpus[i];
300
301	if (pVCpu->hwaccm.s.vmx.pMemObjVMCS != NIL_RTR0MEMOBJ)
302	{
303	RTR0MemObjFree(pVCpu->hwaccm.s.vmx.pMemObjVMCS, false);
304	pVCpu->hwaccm.s.vmx.pMemObjVMCS = NIL_RTR0MEMOBJ;
305	pVCpu->hwaccm.s.vmx.pVMCS = 0;
306	pVCpu->hwaccm.s.vmx.pVMCSPhys = 0;
307	}
308	if (pVCpu->hwaccm.s.vmx.pMemObjVAPIC != NIL_RTR0MEMOBJ)
309	{
310	RTR0MemObjFree(pVCpu->hwaccm.s.vmx.pMemObjVAPIC, false);
311	pVCpu->hwaccm.s.vmx.pMemObjVAPIC = NIL_RTR0MEMOBJ;
312	pVCpu->hwaccm.s.vmx.pVAPIC = 0;
313	pVCpu->hwaccm.s.vmx.pVAPICPhys = 0;
314	}
315	if (pVCpu->hwaccm.s.vmx.pMemObjMSRBitmap != NIL_RTR0MEMOBJ)
316	{
317	RTR0MemObjFree(pVCpu->hwaccm.s.vmx.pMemObjMSRBitmap, false);
318	pVCpu->hwaccm.s.vmx.pMemObjMSRBitmap = NIL_RTR0MEMOBJ;
319	pVCpu->hwaccm.s.vmx.pMSRBitmap = 0;
320	pVCpu->hwaccm.s.vmx.pMSRBitmapPhys = 0;
321	}
322	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
323	if (pVCpu->hwaccm.s.vmx.pMemObjHostMSR != NIL_RTR0MEMOBJ)
324	{
325	RTR0MemObjFree(pVCpu->hwaccm.s.vmx.pMemObjHostMSR, false);
326	pVCpu->hwaccm.s.vmx.pMemObjHostMSR = NIL_RTR0MEMOBJ;
327	pVCpu->hwaccm.s.vmx.pHostMSR = 0;
328	pVCpu->hwaccm.s.vmx.pHostMSRPhys = 0;
329	}
330	if (pVCpu->hwaccm.s.vmx.pMemObjGuestMSR != NIL_RTR0MEMOBJ)
331	{
332	RTR0MemObjFree(pVCpu->hwaccm.s.vmx.pMemObjGuestMSR, false);
333	pVCpu->hwaccm.s.vmx.pMemObjGuestMSR = NIL_RTR0MEMOBJ;
334	pVCpu->hwaccm.s.vmx.pGuestMSR = 0;
335	pVCpu->hwaccm.s.vmx.pGuestMSRPhys = 0;
336	}
337	#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
338	}
339	if (pVM->hwaccm.s.vmx.pMemObjAPIC != NIL_RTR0MEMOBJ)
340	{
341	RTR0MemObjFree(pVM->hwaccm.s.vmx.pMemObjAPIC, false);
342	pVM->hwaccm.s.vmx.pMemObjAPIC = NIL_RTR0MEMOBJ;
343	pVM->hwaccm.s.vmx.pAPIC = 0;
344	pVM->hwaccm.s.vmx.pAPICPhys = 0;
345	}
346	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
347	if (pVM->hwaccm.s.vmx.pMemObjScratch != NIL_RTR0MEMOBJ)
348	{
349	ASMMemZero32(pVM->hwaccm.s.vmx.pScratch, PAGE_SIZE);
350	RTR0MemObjFree(pVM->hwaccm.s.vmx.pMemObjScratch, false);
351	pVM->hwaccm.s.vmx.pMemObjScratch = NIL_RTR0MEMOBJ;
352	pVM->hwaccm.s.vmx.pScratch = 0;
353	pVM->hwaccm.s.vmx.pScratchPhys = 0;
354	}
355	#endif
356	return VINF_SUCCESS;
357	}
358
359	/**
360	* Sets up VT-x for the specified VM
361	*
362	* @returns VBox status code.
363	* @param pVM The VM to operate on.
364	*/
365	VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
366	{
367	int rc = VINF_SUCCESS;
368	uint32_t val;
369
370	AssertReturn(pVM, VERR_INVALID_PARAMETER);
371
372	for (VMCPUID i = 0; i < pVM->cCpus; i++)
373	{
374	PVMCPU pVCpu = &pVM->aCpus[i];
375
376	Assert(pVCpu->hwaccm.s.vmx.pVMCS);
377
378	/* Set revision dword at the beginning of the VMCS structure. */
379	(uint32_t )pVCpu->hwaccm.s.vmx.pVMCS = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hwaccm.s.vmx.msr.vmx_basic_info);
380
381	/* Clear VM Control Structure. */
382	Log(("pVMCSPhys = %RHp\n", pVCpu->hwaccm.s.vmx.pVMCSPhys));
383	rc = VMXClearVMCS(pVCpu->hwaccm.s.vmx.pVMCSPhys);
384	if (RT_FAILURE(rc))
385	goto vmx_end;
386
387	/* Activate the VM Control Structure. */
388	rc = VMXActivateVMCS(pVCpu->hwaccm.s.vmx.pVMCSPhys);
389	if (RT_FAILURE(rc))
390	goto vmx_end;
391
392	/* VMX_VMCS_CTRL_PIN_EXEC_CONTROLS
393	* Set required bits to one and zero according to the MSR capabilities.
394	*/
395	val = pVM->hwaccm.s.vmx.msr.vmx_pin_ctls.n.disallowed0;
396	/* External and non-maskable interrupts cause VM-exits. */
397	val \|= VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_EXT_INT_EXIT \| VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_NMI_EXIT;
398	/* enable the preemption timer. */
399	if (pVM->hwaccm.s.vmx.fUsePreemptTimer)
400	val \|= VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_PREEMPT_TIMER;
401	val &= pVM->hwaccm.s.vmx.msr.vmx_pin_ctls.n.allowed1;
402
403	rc = VMXWriteVMCS(VMX_VMCS_CTRL_PIN_EXEC_CONTROLS, val);
404	AssertRC(rc);
405
406	/* VMX_VMCS_CTRL_PROC_EXEC_CONTROLS
407	* Set required bits to one and zero according to the MSR capabilities.
408	*/
409	val = pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.disallowed0;
410	/* Program which event cause VM-exits and which features we want to use. */
411	val = val \| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_HLT_EXIT
412	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_TSC_OFFSET
413	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT
414	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_UNCOND_IO_EXIT
415	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDPMC_EXIT
416	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_EXIT
417	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MWAIT_EXIT; /* don't execute mwait or else we'll idle inside the guest (host thinks the cpu load is high) */
418
419	/* Without nested paging we should intercept invlpg and cr3 mov instructions. */
420	if (!pVM->hwaccm.s.fNestedPaging)
421	val \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INVLPG_EXIT
422	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
423	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT;
424
425	/* Note: VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MWAIT_EXIT might cause a vmlaunch failure with an invalid control fields error. (combined with some other exit reasons) */
426	if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
427	{
428	/* CR8 reads from the APIC shadow page; writes cause an exit is they lower the TPR below the threshold */
429	val \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW;
430	Assert(pVM->hwaccm.s.vmx.pAPIC);
431	}
432	else
433	/* Exit on CR8 reads & writes in case the TPR shadow feature isn't present. */
434	val \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR8_STORE_EXIT \| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR8_LOAD_EXIT;
435
436	if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
437	{
438	Assert(pVCpu->hwaccm.s.vmx.pMSRBitmapPhys);
439	val \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS;
440	}
441
442	/* We will use the secondary control if it's present. */
443	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
444
445	/* Mask away the bits that the CPU doesn't support */
446	/** @todo make sure they don't conflict with the above requirements. */
447	val &= pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1;
448	pVCpu->hwaccm.s.vmx.proc_ctls = val;
449
450	rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, val);
451	AssertRC(rc);
452
453	if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
454	{
455	/* VMX_VMCS_CTRL_PROC_EXEC_CONTROLS2
456	* Set required bits to one and zero according to the MSR capabilities.
457	*/
458	val = pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2.n.disallowed0;
459	val \|= VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT;
460
461	#ifdef HWACCM_VTX_WITH_EPT
462	if (pVM->hwaccm.s.fNestedPaging)
463	val \|= VMX_VMCS_CTRL_PROC_EXEC2_EPT;
464	#endif /* HWACCM_VTX_WITH_EPT */
465	#ifdef HWACCM_VTX_WITH_VPID
466	else
467	if (pVM->hwaccm.s.vmx.fVPID)
468	val \|= VMX_VMCS_CTRL_PROC_EXEC2_VPID;
469	#endif /* HWACCM_VTX_WITH_VPID */
470
471	if (pVM->hwaccm.s.fHasIoApic)
472	val \|= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC;
473
474	if (pVM->hwaccm.s.vmx.fUnrestrictedGuest)
475	val \|= VMX_VMCS_CTRL_PROC_EXEC2_REAL_MODE;
476
477	/* Mask away the bits that the CPU doesn't support */
478	/** @todo make sure they don't conflict with the above requirements. */
479	val &= pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2.n.allowed1;
480	pVCpu->hwaccm.s.vmx.proc_ctls2 = val;
481	rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS2, val);
482	AssertRC(rc);
483	}
484
485	/* VMX_VMCS_CTRL_CR3_TARGET_COUNT
486	* Set required bits to one and zero according to the MSR capabilities.
487	*/
488	rc = VMXWriteVMCS(VMX_VMCS_CTRL_CR3_TARGET_COUNT, 0);
489	AssertRC(rc);
490
491	/* Forward all exception except #NM & #PF to the guest.
492	* We always need to check pagefaults since our shadow page table can be out of sync.
493	* And we always lazily sync the FPU & XMM state.
494	*/
495
496	/** @todo Possible optimization:
497	* Keep the FPU and XMM state current in the EM thread. That way there's no need to
498	* lazily sync anything, but the downside is that we can't use the FPU stack or XMM
499	* registers ourselves of course.
500	*
501	* Note: only possible if the current state is actually ours (X86_CR0_TS flag)
502	*/
503
504	/* Don't filter page faults; all of them should cause a switch. */
505	rc = VMXWriteVMCS(VMX_VMCS_CTRL_PAGEFAULT_ERROR_MASK, 0);
506	rc \|= VMXWriteVMCS(VMX_VMCS_CTRL_PAGEFAULT_ERROR_MATCH, 0);
507	AssertRC(rc);
508
509	/* Init TSC offset to zero. */
510	rc = VMXWriteVMCS64(VMX_VMCS_CTRL_TSC_OFFSET_FULL, 0);
511	AssertRC(rc);
512
513	rc = VMXWriteVMCS64(VMX_VMCS_CTRL_IO_BITMAP_A_FULL, 0);
514	AssertRC(rc);
515
516	rc = VMXWriteVMCS64(VMX_VMCS_CTRL_IO_BITMAP_B_FULL, 0);
517	AssertRC(rc);
518
519	/* Set the MSR bitmap address. */
520	if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
521	{
522	Assert(pVCpu->hwaccm.s.vmx.pMSRBitmapPhys);
523
524	rc = VMXWriteVMCS64(VMX_VMCS_CTRL_MSR_BITMAP_FULL, pVCpu->hwaccm.s.vmx.pMSRBitmapPhys);
525	AssertRC(rc);
526
527	/* Allow the guest to directly modify these MSRs; they are restored and saved automatically. */
528	vmxR0SetMSRPermission(pVCpu, MSR_IA32_SYSENTER_CS, true, true);
529	vmxR0SetMSRPermission(pVCpu, MSR_IA32_SYSENTER_ESP, true, true);
530	vmxR0SetMSRPermission(pVCpu, MSR_IA32_SYSENTER_EIP, true, true);
531	vmxR0SetMSRPermission(pVCpu, MSR_K8_LSTAR, true, true);
532	vmxR0SetMSRPermission(pVCpu, MSR_K6_STAR, true, true);
533	vmxR0SetMSRPermission(pVCpu, MSR_K8_SF_MASK, true, true);
534	vmxR0SetMSRPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, true, true);
535	vmxR0SetMSRPermission(pVCpu, MSR_K8_GS_BASE, true, true);
536	vmxR0SetMSRPermission(pVCpu, MSR_K8_FS_BASE, true, true);
537	}
538
539	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
540	/* Set the guest & host MSR load/store physical addresses. */
541	Assert(pVCpu->hwaccm.s.vmx.pGuestMSRPhys);
542	rc = VMXWriteVMCS64(VMX_VMCS_CTRL_VMENTRY_MSR_LOAD_FULL, pVCpu->hwaccm.s.vmx.pGuestMSRPhys);
543	AssertRC(rc);
544	rc = VMXWriteVMCS64(VMX_VMCS_CTRL_VMEXIT_MSR_STORE_FULL, pVCpu->hwaccm.s.vmx.pGuestMSRPhys);
545	AssertRC(rc);
546
547	Assert(pVCpu->hwaccm.s.vmx.pHostMSRPhys);
548	rc = VMXWriteVMCS64(VMX_VMCS_CTRL_VMEXIT_MSR_LOAD_FULL, pVCpu->hwaccm.s.vmx.pHostMSRPhys);
549	AssertRC(rc);
550	#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
551
552	rc = VMXWriteVMCS(VMX_VMCS_CTRL_ENTRY_MSR_LOAD_COUNT, 0);
553	AssertRC(rc);
554
555	rc = VMXWriteVMCS(VMX_VMCS_CTRL_EXIT_MSR_STORE_COUNT, 0);
556	AssertRC(rc);
557
558	if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
559	{
560	Assert(pVM->hwaccm.s.vmx.pMemObjAPIC);
561	/* Optional */
562	rc = VMXWriteVMCS(VMX_VMCS_CTRL_TPR_THRESHOLD, 0);
563	rc \|= VMXWriteVMCS64(VMX_VMCS_CTRL_VAPIC_PAGEADDR_FULL, pVCpu->hwaccm.s.vmx.pVAPICPhys);
564
565	if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
566	rc \|= VMXWriteVMCS64(VMX_VMCS_CTRL_APIC_ACCESSADDR_FULL, pVM->hwaccm.s.vmx.pAPICPhys);
567
568	AssertRC(rc);
569	}
570
571	/* Set link pointer to -1. Not currently used. */
572	rc = VMXWriteVMCS64(VMX_VMCS_GUEST_LINK_PTR_FULL, 0xFFFFFFFFFFFFFFFFULL);
573	AssertRC(rc);
574
575	/* Clear VM Control Structure. Marking it inactive, clearing implementation specific data and writing back VMCS data to memory. */
576	rc = VMXClearVMCS(pVCpu->hwaccm.s.vmx.pVMCSPhys);
577	AssertRC(rc);
578
579	/* Configure the VMCS read cache. */
580	PVMCSCACHE pCache = &pVCpu->hwaccm.s.vmx.VMCSCache;
581
582	VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_RIP);
583	VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_RSP);
584	VMXSetupCachedReadVMCS(pCache, VMX_VMCS_GUEST_RFLAGS);
585	VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE);
586	VMXSetupCachedReadVMCS(pCache, VMX_VMCS_CTRL_CR0_READ_SHADOW);
587	VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_CR0);
588	VMXSetupCachedReadVMCS(pCache, VMX_VMCS_CTRL_CR4_READ_SHADOW);
589	VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_CR4);
590	VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_DR7);
591	VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_GUEST_SYSENTER_CS);
592	VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_SYSENTER_EIP);
593	VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_SYSENTER_ESP);
594	VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_GUEST_GDTR_LIMIT);
595	VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_GDTR_BASE);
596	VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_GUEST_IDTR_LIMIT);
597	VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_IDTR_BASE);
598
599	VMX_SETUP_SELREG(ES, pCache);
600	VMX_SETUP_SELREG(SS, pCache);
601	VMX_SETUP_SELREG(CS, pCache);
602	VMX_SETUP_SELREG(DS, pCache);
603	VMX_SETUP_SELREG(FS, pCache);
604	VMX_SETUP_SELREG(GS, pCache);
605	VMX_SETUP_SELREG(LDTR, pCache);
606	VMX_SETUP_SELREG(TR, pCache);
607
608	/* Status code VMCS reads. */
609	VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_EXIT_REASON);
610	VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_VM_INSTR_ERROR);
611	VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_EXIT_INSTR_LENGTH);
612	VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_EXIT_INTERRUPTION_ERRCODE);
613	VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO);
614	VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_EXIT_INSTR_INFO);
615	VMXSetupCachedReadVMCS(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
616	VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_IDT_INFO);
617	VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_IDT_ERRCODE);
618
619	if (pVM->hwaccm.s.fNestedPaging)
620	{
621	VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_CR3);
622	VMXSetupCachedReadVMCS(pCache, VMX_VMCS_EXIT_PHYS_ADDR_FULL);
623	pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
624	}
625	else
626	pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
627	} /* for each VMCPU */
628
629	/* Choose the right TLB setup function. */
630	if (pVM->hwaccm.s.fNestedPaging)
631	{
632	pVM->hwaccm.s.vmx.pfnSetupTaggedTLB = vmxR0SetupTLBEPT;
633
634	/* Default values for flushing. */
635	pVM->hwaccm.s.vmx.enmFlushPage = VMX_FLUSH_ALL_CONTEXTS;
636	pVM->hwaccm.s.vmx.enmFlushContext = VMX_FLUSH_ALL_CONTEXTS;
637
638	/* If the capabilities specify we can do more, then make use of it. */
639	if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVEPT_CAPS_INDIV)
640	pVM->hwaccm.s.vmx.enmFlushPage = VMX_FLUSH_PAGE;
641	else
642	if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVEPT_CAPS_CONTEXT)
643	pVM->hwaccm.s.vmx.enmFlushPage = VMX_FLUSH_SINGLE_CONTEXT;
644
645	if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVEPT_CAPS_CONTEXT)
646	pVM->hwaccm.s.vmx.enmFlushContext = VMX_FLUSH_SINGLE_CONTEXT;
647	}
648	#ifdef HWACCM_VTX_WITH_VPID
649	else
650	if (pVM->hwaccm.s.vmx.fVPID)
651	{
652	pVM->hwaccm.s.vmx.pfnSetupTaggedTLB = vmxR0SetupTLBVPID;
653
654	/* Default values for flushing. */
655	pVM->hwaccm.s.vmx.enmFlushPage = VMX_FLUSH_ALL_CONTEXTS;
656	pVM->hwaccm.s.vmx.enmFlushContext = VMX_FLUSH_ALL_CONTEXTS;
657
658	/* If the capabilities specify we can do more, then make use of it. */
659	if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVVPID_CAPS_INDIV)
660	pVM->hwaccm.s.vmx.enmFlushPage = VMX_FLUSH_PAGE;
661	else
662	if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVVPID_CAPS_CONTEXT)
663	pVM->hwaccm.s.vmx.enmFlushPage = VMX_FLUSH_SINGLE_CONTEXT;
664
665	if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVVPID_CAPS_CONTEXT)
666	pVM->hwaccm.s.vmx.enmFlushContext = VMX_FLUSH_SINGLE_CONTEXT;
667	}
668	#endif /* HWACCM_VTX_WITH_VPID */
669	else
670	pVM->hwaccm.s.vmx.pfnSetupTaggedTLB = vmxR0SetupTLBDummy;
671
672	vmx_end:
673	VMXR0CheckError(pVM, &pVM->aCpus[0], rc);
674	return rc;
675	}
676
677	/**
678	* Sets the permission bits for the specified MSR
679	*
680	* @param pVCpu The VMCPU to operate on.
681	* @param ulMSR MSR value
682	* @param fRead Reading allowed/disallowed
683	* @param fWrite Writing allowed/disallowed
684	*/
685	static void vmxR0SetMSRPermission(PVMCPU pVCpu, unsigned ulMSR, bool fRead, bool fWrite)
686	{
687	unsigned ulBit;
688	uint8_t pMSRBitmap = (uint8_t )pVCpu->hwaccm.s.vmx.pMSRBitmap;
689
690	/* Layout:
691	* 0x000 - 0x3ff - Low MSR read bits
692	* 0x400 - 0x7ff - High MSR read bits
693	* 0x800 - 0xbff - Low MSR write bits
694	* 0xc00 - 0xfff - High MSR write bits
695	*/
696	if (ulMSR <= 0x00001FFF)
697	{
698	/* Pentium-compatible MSRs */
699	ulBit = ulMSR;
700	}
701	else
702	if ( ulMSR >= 0xC0000000
703	&& ulMSR <= 0xC0001FFF)
704	{
705	/* AMD Sixth Generation x86 Processor MSRs */
706	ulBit = (ulMSR - 0xC0000000);
707	pMSRBitmap += 0x400;
708	}
709	else
710	{
711	AssertFailed();
712	return;
713	}
714
715	Assert(ulBit <= 0x1fff);
716	if (fRead)
717	ASMBitClear(pMSRBitmap, ulBit);
718	else
719	ASMBitSet(pMSRBitmap, ulBit);
720
721	if (fWrite)
722	ASMBitClear(pMSRBitmap + 0x800, ulBit);
723	else
724	ASMBitSet(pMSRBitmap + 0x800, ulBit);
725	}
726
727
728	/**
729	* Injects an event (trap or external interrupt)
730	*
731	* @returns VBox status code. Note that it may return VINF_EM_RESET to
732	* indicate a triple fault when injecting X86_XCPT_DF.
733	*
734	* @param pVM The VM to operate on.
735	* @param pVCpu The VMCPU to operate on.
736	* @param pCtx CPU Context
737	* @param intInfo VMX interrupt info
738	* @param cbInstr Opcode length of faulting instruction
739	* @param errCode Error code (optional)
740	*/
741	static int VMXR0InjectEvent(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, uint32_t intInfo, uint32_t cbInstr, uint32_t errCode)
742	{
743	int rc;
744	uint32_t iGate = VMX_EXIT_INTERRUPTION_INFO_VECTOR(intInfo);
745
746	#ifdef VBOX_WITH_STATISTICS
747	STAM_COUNTER_INC(&pVCpu->hwaccm.s.paStatInjectedIrqsR0[iGate & MASK_INJECT_IRQ_STAT]);
748	#endif
749
750	#ifdef VBOX_STRICT
751	if (iGate == 0xE)
752	LogFlow(("VMXR0InjectEvent: Injecting interrupt %d at %RGv error code=%08x CR2=%RGv intInfo=%08x\n", iGate, (RTGCPTR)pCtx->rip, errCode, pCtx->cr2, intInfo));
753	else
754	if (iGate < 0x20)
755	LogFlow(("VMXR0InjectEvent: Injecting interrupt %d at %RGv error code=%08x\n", iGate, (RTGCPTR)pCtx->rip, errCode));
756	else
757	{
758	LogFlow(("INJ-EI: %x at %RGv\n", iGate, (RTGCPTR)pCtx->rip));
759	Assert(VMX_EXIT_INTERRUPTION_INFO_TYPE(intInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW \|\| !VMCPU_FF_ISSET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
760	Assert(VMX_EXIT_INTERRUPTION_INFO_TYPE(intInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW \|\| pCtx->eflags.u32 & X86_EFL_IF);
761	}
762	#endif
763
764	if ( CPUMIsGuestInRealModeEx(pCtx)
765	&& pVM->hwaccm.s.vmx.pRealModeTSS)
766	{
767	RTGCPHYS GCPhysHandler;
768	uint16_t offset, ip;
769	RTSEL sel;
770
771	/* Injecting events doesn't work right with real mode emulation.
772	* (#GP if we try to inject external hardware interrupts)
773	* Inject the interrupt or trap directly instead.
774	*
775	* ASSUMES no access handlers for the bits we read or write below (should be safe).
776	*/
777	Log(("Manual interrupt/trap '%x' inject (real mode)\n", iGate));
778
779	/* Check if the interrupt handler is present. */
780	if (iGate * 4 + 3 > pCtx->idtr.cbIdt)
781	{
782	Log(("IDT cbIdt violation\n"));
783	if (iGate != X86_XCPT_DF)
784	{
785	uint32_t intInfo2;
786
787	intInfo2 = (iGate == X86_XCPT_GP) ? (uint32_t)X86_XCPT_DF : iGate;
788	intInfo2 \|= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
789	intInfo2 \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
790	intInfo2 \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HWEXCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
791
792	return VMXR0InjectEvent(pVM, pVCpu, pCtx, intInfo2, 0, 0 /* no error code according to the Intel docs */);
793	}
794	Log(("Triple fault -> reset the VM!\n"));
795	return VINF_EM_RESET;
796	}
797	if ( VMX_EXIT_INTERRUPTION_INFO_TYPE(intInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW
798	\|\| iGate == 3 /* Both #BP and #OF point to the instruction after. */
799	\|\| iGate == 4)
800	{
801	ip = pCtx->ip + cbInstr;
802	}
803	else
804	ip = pCtx->ip;
805
806	/* Read the selector:offset pair of the interrupt handler. */
807	GCPhysHandler = (RTGCPHYS)pCtx->idtr.pIdt + iGate * 4;
808	rc = PGMPhysSimpleReadGCPhys(pVM, &offset, GCPhysHandler, sizeof(offset)); AssertRC(rc);
809	rc = PGMPhysSimpleReadGCPhys(pVM, &sel, GCPhysHandler + 2, sizeof(sel)); AssertRC(rc);
810
811	LogFlow(("IDT handler %04X:%04X\n", sel, offset));
812
813	/* Construct the stack frame. */
814	/** @todo should check stack limit. */
815	pCtx->sp -= 2;
816	LogFlow(("ss:sp %04X:%04X eflags=%x\n", pCtx->ss, pCtx->sp, pCtx->eflags.u));
817	rc = PGMPhysSimpleWriteGCPhys(pVM, pCtx->ssHid.u64Base + pCtx->sp, &pCtx->eflags, sizeof(uint16_t)); AssertRC(rc);
818	pCtx->sp -= 2;
819	LogFlow(("ss:sp %04X:%04X cs=%x\n", pCtx->ss, pCtx->sp, pCtx->cs));
820	rc = PGMPhysSimpleWriteGCPhys(pVM, pCtx->ssHid.u64Base + pCtx->sp, &pCtx->cs, sizeof(uint16_t)); AssertRC(rc);
821	pCtx->sp -= 2;
822	LogFlow(("ss:sp %04X:%04X ip=%x\n", pCtx->ss, pCtx->sp, ip));
823	rc = PGMPhysSimpleWriteGCPhys(pVM, pCtx->ssHid.u64Base + pCtx->sp, &ip, sizeof(ip)); AssertRC(rc);
824
825	/* Update the CPU state for executing the handler. */
826	pCtx->rip = offset;
827	pCtx->cs = sel;
828	pCtx->csHid.u64Base = sel << 4;
829	pCtx->eflags.u &= ~(X86_EFL_IF\|X86_EFL_TF\|X86_EFL_RF\|X86_EFL_AC);
830
831	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_GUEST_SEGMENT_REGS;
832	return VINF_SUCCESS;
833	}
834
835	/* Set event injection state. */
836	rc = VMXWriteVMCS(VMX_VMCS_CTRL_ENTRY_IRQ_INFO, intInfo \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT));
837
838	rc \|= VMXWriteVMCS(VMX_VMCS_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
839	rc \|= VMXWriteVMCS(VMX_VMCS_CTRL_ENTRY_EXCEPTION_ERRCODE, errCode);
840
841	AssertRC(rc);
842	return rc;
843	}
844
845
846	/**
847	* Checks for pending guest interrupts and injects them
848	*
849	* @returns VBox status code.
850	* @param pVM The VM to operate on.
851	* @param pVCpu The VMCPU to operate on.
852	* @param pCtx CPU Context
853	*/
854	static int VMXR0CheckPendingInterrupt(PVM pVM, PVMCPU pVCpu, CPUMCTX *pCtx)
855	{
856	int rc;
857
858	/* Dispatch any pending interrupts. (injected before, but a VM exit occurred prematurely) */
859	if (pVCpu->hwaccm.s.Event.fPending)
860	{
861	Log(("CPU%d: Reinjecting event %RX64 %08x at %RGv cr2=%RX64\n", pVCpu->idCpu, pVCpu->hwaccm.s.Event.intInfo, pVCpu->hwaccm.s.Event.errCode, (RTGCPTR)pCtx->rip, pCtx->cr2));
862	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatIntReinject);
863	rc = VMXR0InjectEvent(pVM, pVCpu, pCtx, pVCpu->hwaccm.s.Event.intInfo, 0, pVCpu->hwaccm.s.Event.errCode);
864	AssertRC(rc);
865
866	pVCpu->hwaccm.s.Event.fPending = false;
867	return VINF_SUCCESS;
868	}
869
870	/* If an active trap is already pending, then we must forward it first! */
871	if (!TRPMHasTrap(pVCpu))
872	{
873	if (VMCPU_FF_TESTANDCLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI))
874	{
875	RTGCUINTPTR intInfo;
876
877	Log(("CPU%d: injecting #NMI\n", pVCpu->idCpu));
878
879	intInfo = X86_XCPT_NMI;
880	intInfo \|= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
881	intInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
882
883	rc = VMXR0InjectEvent(pVM, pVCpu, pCtx, intInfo, 0, 0);
884	AssertRC(rc);
885
886	return VINF_SUCCESS;
887	}
888
889	/* @todo SMI interrupts. */
890
891	/* When external interrupts are pending, we should exit the VM when IF is set. */
892	if (VMCPU_FF_ISPENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC\|VMCPU_FF_INTERRUPT_PIC)))
893	{
894	if (!(pCtx->eflags.u32 & X86_EFL_IF))
895	{
896	if (!(pVCpu->hwaccm.s.vmx.proc_ctls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_IRQ_WINDOW_EXIT))
897	{
898	LogFlow(("Enable irq window exit!\n"));
899	pVCpu->hwaccm.s.vmx.proc_ctls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_IRQ_WINDOW_EXIT;
900	rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
901	AssertRC(rc);
902	}
903	/* else nothing to do but wait */
904	}
905	else
906	if (!VMCPU_FF_ISSET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
907	{
908	uint8_t u8Interrupt;
909
910	rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
911	Log(("CPU%d: Dispatch interrupt: u8Interrupt=%x (%d) rc=%Rrc cs:rip=%04X:%RGv\n", pVCpu->idCpu, u8Interrupt, u8Interrupt, rc, pCtx->cs, (RTGCPTR)pCtx->rip));
912	if (RT_SUCCESS(rc))
913	{
914	rc = TRPMAssertTrap(pVCpu, u8Interrupt, TRPM_HARDWARE_INT);
915	AssertRC(rc);
916	}
917	else
918	{
919	/* Can only happen in rare cases where a pending interrupt is cleared behind our back */
920	Assert(!VMCPU_FF_ISPENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC\|VMCPU_FF_INTERRUPT_PIC)));
921	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatSwitchGuestIrq);
922	/* Just continue */
923	}
924	}
925	else
926	Log(("Pending interrupt blocked at %RGv by VM_FF_INHIBIT_INTERRUPTS!!\n", (RTGCPTR)pCtx->rip));
927	}
928	}
929
930	#ifdef VBOX_STRICT
931	if (TRPMHasTrap(pVCpu))
932	{
933	uint8_t u8Vector;
934	rc = TRPMQueryTrapAll(pVCpu, &u8Vector, 0, 0, 0);
935	AssertRC(rc);
936	}
937	#endif
938
939	if ( (pCtx->eflags.u32 & X86_EFL_IF)
940	&& (!VMCPU_FF_ISSET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
941	&& TRPMHasTrap(pVCpu)
942	)
943	{
944	uint8_t u8Vector;
945	TRPMEVENT enmType;
946	RTGCUINTPTR intInfo;
947	RTGCUINT errCode;
948
949	/* If a new event is pending, then dispatch it now. */
950	rc = TRPMQueryTrapAll(pVCpu, &u8Vector, &enmType, &errCode, 0);
951	AssertRC(rc);
952	Assert(pCtx->eflags.Bits.u1IF == 1 \|\| enmType == TRPM_TRAP);
953	Assert(enmType != TRPM_SOFTWARE_INT);
954
955	/* Clear the pending trap. */
956	rc = TRPMResetTrap(pVCpu);
957	AssertRC(rc);
958
959	intInfo = u8Vector;
960	intInfo \|= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
961
962	if (enmType == TRPM_TRAP)
963	{
964	switch (u8Vector) {
965	case 8:
966	case 10:
967	case 11:
968	case 12:
969	case 13:
970	case 14:
971	case 17:
972	/* Valid error codes. */
973	intInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
974	break;
975	default:
976	break;
977	}
978	if (u8Vector == X86_XCPT_BP \|\| u8Vector == X86_XCPT_OF)
979	intInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SWEXCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
980	else
981	intInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HWEXCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
982	}
983	else
984	intInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
985
986	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatIntInject);
987	rc = VMXR0InjectEvent(pVM, pVCpu, pCtx, intInfo, 0, errCode);
988	AssertRC(rc);
989	} /* if (interrupts can be dispatched) */
990
991	return VINF_SUCCESS;
992	}
993
994	/**
995	* Save the host state
996	*
997	* @returns VBox status code.
998	* @param pVM The VM to operate on.
999	* @param pVCpu The VMCPU to operate on.
1000	*/
1001	VMMR0DECL(int) VMXR0SaveHostState(PVM pVM, PVMCPU pVCpu)
1002	{
1003	int rc = VINF_SUCCESS;
1004
1005	/*
1006	* Host CPU Context
1007	*/
1008	if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_HOST_CONTEXT)
1009	{
1010	RTIDTR idtr;
1011	RTGDTR gdtr;
1012	RTSEL SelTR;
1013	PCX86DESCHC pDesc;
1014	uintptr_t trBase;
1015	RTSEL cs;
1016	RTSEL ss;
1017	uint64_t cr3;
1018
1019	/* Control registers */
1020	rc = VMXWriteVMCS(VMX_VMCS_HOST_CR0, ASMGetCR0());
1021	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
1022	if (VMX_IS_64BIT_HOST_MODE())
1023	{
1024	cr3 = hwaccmR0Get64bitCR3();
1025	rc \|= VMXWriteVMCS64(VMX_VMCS_HOST_CR3, cr3);
1026	}
1027	else
1028	#endif
1029	{
1030	cr3 = ASMGetCR3();
1031	rc \|= VMXWriteVMCS(VMX_VMCS_HOST_CR3, cr3);
1032	}
1033	rc \|= VMXWriteVMCS(VMX_VMCS_HOST_CR4, ASMGetCR4());
1034	AssertRC(rc);
1035	Log2(("VMX_VMCS_HOST_CR0 %08x\n", ASMGetCR0()));
1036	Log2(("VMX_VMCS_HOST_CR3 %08RX64\n", cr3));
1037	Log2(("VMX_VMCS_HOST_CR4 %08x\n", ASMGetCR4()));
1038
1039	/* Selector registers. */
1040	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
1041	if (VMX_IS_64BIT_HOST_MODE())
1042	{
1043	cs = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelCS;
1044	ss = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelSS;
1045	}
1046	else
1047	{
1048	/* sysenter loads LDT cs & ss, VMX doesn't like this. Load the GDT ones (safe). */
1049	cs = (RTSEL)(uintptr_t)&SUPR0AbsKernelCS;
1050	ss = (RTSEL)(uintptr_t)&SUPR0AbsKernelSS;
1051	}
1052	#else
1053	cs = ASMGetCS();
1054	ss = ASMGetSS();
1055	#endif
1056	Assert(!(cs & X86_SEL_LDT)); Assert((cs & X86_SEL_RPL) == 0);
1057	Assert(!(ss & X86_SEL_LDT)); Assert((ss & X86_SEL_RPL) == 0);
1058	rc = VMXWriteVMCS(VMX_VMCS16_HOST_FIELD_CS, cs);
1059	/* Note: VMX is (again) very picky about the RPL of the selectors here; we'll restore them manually. */
1060	rc \|= VMXWriteVMCS(VMX_VMCS16_HOST_FIELD_DS, 0);
1061	rc \|= VMXWriteVMCS(VMX_VMCS16_HOST_FIELD_ES, 0);
1062	#if HC_ARCH_BITS == 32
1063	if (!VMX_IS_64BIT_HOST_MODE())
1064	{
1065	rc \|= VMXWriteVMCS(VMX_VMCS16_HOST_FIELD_FS, 0);
1066	rc \|= VMXWriteVMCS(VMX_VMCS16_HOST_FIELD_GS, 0);
1067	}
1068	#endif
1069	rc \|= VMXWriteVMCS(VMX_VMCS16_HOST_FIELD_SS, ss);
1070	SelTR = ASMGetTR();
1071	rc \|= VMXWriteVMCS(VMX_VMCS16_HOST_FIELD_TR, SelTR);
1072	AssertRC(rc);
1073	Log2(("VMX_VMCS_HOST_FIELD_CS %08x (%08x)\n", cs, ASMGetSS()));
1074	Log2(("VMX_VMCS_HOST_FIELD_DS 00000000 (%08x)\n", ASMGetDS()));
1075	Log2(("VMX_VMCS_HOST_FIELD_ES 00000000 (%08x)\n", ASMGetES()));
1076	Log2(("VMX_VMCS_HOST_FIELD_FS 00000000 (%08x)\n", ASMGetFS()));
1077	Log2(("VMX_VMCS_HOST_FIELD_GS 00000000 (%08x)\n", ASMGetGS()));
1078	Log2(("VMX_VMCS_HOST_FIELD_SS %08x (%08x)\n", ss, ASMGetSS()));
1079	Log2(("VMX_VMCS_HOST_FIELD_TR %08x\n", ASMGetTR()));
1080
1081	/* GDTR & IDTR */
1082	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
1083	if (VMX_IS_64BIT_HOST_MODE())
1084	{
1085	X86XDTR64 gdtr64, idtr64;
1086	hwaccmR0Get64bitGDTRandIDTR(&gdtr64, &idtr64);
1087	rc = VMXWriteVMCS64(VMX_VMCS_HOST_GDTR_BASE, gdtr64.uAddr);
1088	rc \|= VMXWriteVMCS64(VMX_VMCS_HOST_IDTR_BASE, gdtr64.uAddr);
1089	AssertRC(rc);
1090	Log2(("VMX_VMCS_HOST_GDTR_BASE %RX64\n", gdtr64.uAddr));
1091	Log2(("VMX_VMCS_HOST_IDTR_BASE %RX64\n", idtr64.uAddr));
1092	gdtr.cbGdt = gdtr64.cb;
1093	gdtr.pGdt = (uintptr_t)gdtr64.uAddr;
1094	}
1095	else
1096	#endif
1097	{
1098	ASMGetGDTR(&gdtr);
1099	rc = VMXWriteVMCS(VMX_VMCS_HOST_GDTR_BASE, gdtr.pGdt);
1100	ASMGetIDTR(&idtr);
1101	rc \|= VMXWriteVMCS(VMX_VMCS_HOST_IDTR_BASE, idtr.pIdt);
1102	AssertRC(rc);
1103	Log2(("VMX_VMCS_HOST_GDTR_BASE %RHv\n", gdtr.pGdt));
1104	Log2(("VMX_VMCS_HOST_IDTR_BASE %RHv\n", idtr.pIdt));
1105	}
1106
1107	/* Save the base address of the TR selector. */
1108	if (SelTR > gdtr.cbGdt)
1109	{
1110	AssertMsgFailed(("Invalid TR selector %x. GDTR.cbGdt=%x\n", SelTR, gdtr.cbGdt));
1111	return VERR_VMX_INVALID_HOST_STATE;
1112	}
1113
1114	pDesc = (PCX86DESCHC)(gdtr.pGdt + (SelTR & X86_SEL_MASK));
1115	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
1116	if (VMX_IS_64BIT_HOST_MODE())
1117	{
1118	uint64_t trBase64 = X86DESC64_BASE(*(PX86DESC64)pDesc);
1119	rc = VMXWriteVMCS64(VMX_VMCS_HOST_TR_BASE, trBase64);
1120	Log2(("VMX_VMCS_HOST_TR_BASE %RX64\n", trBase64));
1121	AssertRC(rc);
1122	}
1123	else
1124	#endif
1125	{
1126	#if HC_ARCH_BITS == 64
1127	trBase = X86DESC64_BASE(*pDesc);
1128	#else
1129	trBase = X86DESC_BASE(*pDesc);
1130	#endif
1131	rc = VMXWriteVMCS(VMX_VMCS_HOST_TR_BASE, trBase);
1132	AssertRC(rc);
1133	Log2(("VMX_VMCS_HOST_TR_BASE %RHv\n", trBase));
1134	}
1135
1136	/* FS and GS base. */
1137	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
1138	if (VMX_IS_64BIT_HOST_MODE())
1139	{
1140	Log2(("MSR_K8_FS_BASE = %RX64\n", ASMRdMsr(MSR_K8_FS_BASE)));
1141	Log2(("MSR_K8_GS_BASE = %RX64\n", ASMRdMsr(MSR_K8_GS_BASE)));
1142	rc = VMXWriteVMCS64(VMX_VMCS_HOST_FS_BASE, ASMRdMsr(MSR_K8_FS_BASE));
1143	rc \|= VMXWriteVMCS64(VMX_VMCS_HOST_GS_BASE, ASMRdMsr(MSR_K8_GS_BASE));
1144	}
1145	#endif
1146	AssertRC(rc);
1147
1148	/* Sysenter MSRs. */
1149	/** @todo expensive!! */
1150	rc = VMXWriteVMCS(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
1151	Log2(("VMX_VMCS_HOST_SYSENTER_CS %08x\n", ASMRdMsr_Low(MSR_IA32_SYSENTER_CS)));
1152	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
1153	if (VMX_IS_64BIT_HOST_MODE())
1154	{
1155	Log2(("VMX_VMCS_HOST_SYSENTER_EIP %RX64\n", ASMRdMsr(MSR_IA32_SYSENTER_EIP)));
1156	Log2(("VMX_VMCS_HOST_SYSENTER_ESP %RX64\n", ASMRdMsr(MSR_IA32_SYSENTER_ESP)));
1157	rc \|= VMXWriteVMCS64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
1158	rc \|= VMXWriteVMCS64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
1159	}
1160	else
1161	{
1162	rc \|= VMXWriteVMCS(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
1163	rc \|= VMXWriteVMCS(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
1164	Log2(("VMX_VMCS_HOST_SYSENTER_EIP %RX32\n", ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP)));
1165	Log2(("VMX_VMCS_HOST_SYSENTER_ESP %RX32\n", ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP)));
1166	}
1167	#elif HC_ARCH_BITS == 32
1168	rc \|= VMXWriteVMCS(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
1169	rc \|= VMXWriteVMCS(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
1170	Log2(("VMX_VMCS_HOST_SYSENTER_EIP %RX32\n", ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP)));
1171	Log2(("VMX_VMCS_HOST_SYSENTER_ESP %RX32\n", ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP)));
1172	#else
1173	Log2(("VMX_VMCS_HOST_SYSENTER_EIP %RX64\n", ASMRdMsr(MSR_IA32_SYSENTER_EIP)));
1174	Log2(("VMX_VMCS_HOST_SYSENTER_ESP %RX64\n", ASMRdMsr(MSR_IA32_SYSENTER_ESP)));
1175	rc \|= VMXWriteVMCS64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
1176	rc \|= VMXWriteVMCS64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
1177	#endif
1178	AssertRC(rc);
1179
1180	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
1181	/* Store all host MSRs in the VM-Exit load area, so they will be reloaded after the world switch back to the host. */
1182	PVMXMSR pMsr = (PVMXMSR)pVCpu->hwaccm.s.vmx.pHostMSR;
1183	unsigned idxMsr = 0;
1184
1185	/* EFER MSR present? */
1186	if (ASMCpuId_EDX(0x80000001) & (X86_CPUID_AMD_FEATURE_EDX_NX\|X86_CPUID_AMD_FEATURE_EDX_LONG_MODE))
1187	{
1188	if (ASMCpuId_EDX(0x80000001) & X86_CPUID_AMD_FEATURE_EDX_SEP)
1189	{
1190	pMsr->u32IndexMSR = MSR_K6_STAR;
1191	pMsr->u32Reserved = 0;
1192	pMsr->u64Value = ASMRdMsr(MSR_K6_STAR); /* legacy syscall eip, cs & ss */
1193	pMsr++; idxMsr++;
1194	}
1195
1196	pMsr->u32IndexMSR = MSR_K6_EFER;
1197	pMsr->u32Reserved = 0;
1198	# if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
1199	if (CPUMIsGuestInLongMode(pVCpu))
1200	{
1201	/* Must match the efer value in our 64 bits switcher. */
1202	pMsr->u64Value = ASMRdMsr(MSR_K6_EFER) \| MSR_K6_EFER_LME \| MSR_K6_EFER_SCE \| MSR_K6_EFER_NXE;
1203	}
1204	else
1205	# endif
1206	pMsr->u64Value = ASMRdMsr(MSR_K6_EFER);
1207	pMsr++; idxMsr++;
1208	}
1209
1210	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
1211	if (VMX_IS_64BIT_HOST_MODE())
1212	{
1213	pMsr->u32IndexMSR = MSR_K8_LSTAR;
1214	pMsr->u32Reserved = 0;
1215	pMsr->u64Value = ASMRdMsr(MSR_K8_LSTAR); /* 64 bits mode syscall rip */
1216	pMsr++; idxMsr++;
1217	pMsr->u32IndexMSR = MSR_K8_SF_MASK;
1218	pMsr->u32Reserved = 0;
1219	pMsr->u64Value = ASMRdMsr(MSR_K8_SF_MASK); /* syscall flag mask */
1220	pMsr++; idxMsr++;
1221	pMsr->u32IndexMSR = MSR_K8_KERNEL_GS_BASE;
1222	pMsr->u32Reserved = 0;
1223	pMsr->u64Value = ASMRdMsr(MSR_K8_KERNEL_GS_BASE); /* swapgs exchange value */
1224	pMsr++; idxMsr++;
1225	}
1226	# endif
1227	rc = VMXWriteVMCS(VMX_VMCS_CTRL_EXIT_MSR_LOAD_COUNT, idxMsr);
1228	AssertRC(rc);
1229	#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
1230
1231	pVCpu->hwaccm.s.fContextUseFlags &= ~HWACCM_CHANGED_HOST_CONTEXT;
1232	}
1233	return rc;
1234	}
1235
1236	/**
1237	* Prefetch the 4 PDPT pointers (PAE and nested paging only)
1238	*
1239	* @returns VINF_SUCCESS or fatal error.
1240	* @param pVM The VM to operate on.
1241	* @param pVCpu The VMCPU to operate on.
1242	* @param pCtx Guest context
1243	*/
1244	static int vmxR0PrefetchPAEPdptrs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
1245	{
1246	if (CPUMIsGuestInPAEModeEx(pCtx))
1247	{
1248	for (unsigned i=0;i<4;i++)
1249	{
1250	X86PDPE Pdpe;
1251	int rc = PGMGstQueryPaePDPtr(pVCpu, i, &Pdpe);
1252	AssertRCReturn(rc, rc);
1253
1254	rc = VMXWriteVMCS64(VMX_VMCS_GUEST_PDPTR0_FULL + i*2, Pdpe.u);
1255	AssertRC(rc);
1256	}
1257	}
1258	return VINF_SUCCESS;
1259	}
1260
1261	/**
1262	* Update the exception bitmap according to the current CPU state
1263	*
1264	* @param pVM The VM to operate on.
1265	* @param pVCpu The VMCPU to operate on.
1266	* @param pCtx Guest context
1267	*/
1268	static void vmxR0UpdateExceptionBitmap(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
1269	{
1270	uint32_t u32TrapMask;
1271	Assert(pCtx);
1272
1273	u32TrapMask = HWACCM_VMX_TRAP_MASK;
1274	#ifndef DEBUG
1275	if (pVM->hwaccm.s.fNestedPaging)
1276	u32TrapMask &= ~RT_BIT(X86_XCPT_PF); /* no longer need to intercept #PF. */
1277	#endif
1278
1279	/* Also catch floating point exceptions as we need to report them to the guest in a different way. */
1280	if ( CPUMIsGuestFPUStateActive(pVCpu) == true
1281	&& !(pCtx->cr0 & X86_CR0_NE)
1282	&& !pVCpu->hwaccm.s.fFPUOldStyleOverride)
1283	{
1284	u32TrapMask \|= RT_BIT(X86_XCPT_MF);
1285	pVCpu->hwaccm.s.fFPUOldStyleOverride = true;
1286	}
1287
1288	#ifdef VBOX_STRICT
1289	Assert(u32TrapMask & RT_BIT(X86_XCPT_GP));
1290	#endif
1291
1292	/* Intercept all exceptions in real mode as none of them can be injected directly (#GP otherwise). */
1293	if ( CPUMIsGuestInRealModeEx(pCtx)
1294	&& pVM->hwaccm.s.vmx.pRealModeTSS)
1295	u32TrapMask \|= HWACCM_VMX_TRAP_MASK_REALMODE;
1296
1297	int rc = VMXWriteVMCS(VMX_VMCS_CTRL_EXCEPTION_BITMAP, u32TrapMask);
1298	AssertRC(rc);
1299	}
1300
1301	/**
1302	* Loads a minimal guest state
1303	*
1304	* NOTE: Don't do anything here that can cause a jump back to ring 3!!!!!
1305	*
1306	* @param pVM The VM to operate on.
1307	* @param pVCpu The VMCPU to operate on.
1308	* @param pCtx Guest context
1309	*/
1310	VMMR0DECL(void) VMXR0LoadMinimalGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
1311	{
1312	int rc;
1313	X86EFLAGS eflags;
1314
1315	Assert(!(pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_ALL_GUEST));
1316
1317	/* EIP, ESP and EFLAGS */
1318	rc = VMXWriteVMCS64(VMX_VMCS64_GUEST_RIP, pCtx->rip);
1319	rc \|= VMXWriteVMCS64(VMX_VMCS64_GUEST_RSP, pCtx->rsp);
1320	AssertRC(rc);
1321
1322	/* Bits 22-31, 15, 5 & 3 must be zero. Bit 1 must be 1. */
1323	eflags = pCtx->eflags;
1324	eflags.u32 &= VMX_EFLAGS_RESERVED_0;
1325	eflags.u32 \|= VMX_EFLAGS_RESERVED_1;
1326
1327	/* Real mode emulation using v86 mode. */
1328	if ( CPUMIsGuestInRealModeEx(pCtx)
1329	&& pVM->hwaccm.s.vmx.pRealModeTSS)
1330	{
1331	pVCpu->hwaccm.s.vmx.RealMode.eflags = eflags;
1332
1333	eflags.Bits.u1VM = 1;
1334	eflags.Bits.u2IOPL = 0; /* must always be 0 or else certain instructions won't cause faults. */
1335	}
1336	rc = VMXWriteVMCS(VMX_VMCS_GUEST_RFLAGS, eflags.u32);
1337	AssertRC(rc);
1338	}
1339
1340	/**
1341	* Loads the guest state
1342	*
1343	* NOTE: Don't do anything here that can cause a jump back to ring 3!!!!!
1344	*
1345	* @returns VBox status code.
1346	* @param pVM The VM to operate on.
1347	* @param pVCpu The VMCPU to operate on.
1348	* @param pCtx Guest context
1349	*/
1350	VMMR0DECL(int) VMXR0LoadGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
1351	{
1352	int rc = VINF_SUCCESS;
1353	RTGCUINTPTR val;
1354
1355	/* VMX_VMCS_CTRL_ENTRY_CONTROLS
1356	* Set required bits to one and zero according to the MSR capabilities.
1357	*/
1358	val = pVM->hwaccm.s.vmx.msr.vmx_entry.n.disallowed0;
1359	/* Load guest debug controls (dr7 & IA32_DEBUGCTL_MSR) (forced to 1 on the 'first' VT-x capable CPUs; this actually includes the newest Nehalem CPUs) */
1360	val \|= VMX_VMCS_CTRL_ENTRY_CONTROLS_LOAD_DEBUG;
1361	/* 64 bits guest mode? */
1362	if (CPUMIsGuestInLongModeEx(pCtx))
1363	val \|= VMX_VMCS_CTRL_ENTRY_CONTROLS_IA64_MODE;
1364	/* else Must be zero when AMD64 is not available. */
1365
1366	/* Mask away the bits that the CPU doesn't support */
1367	val &= pVM->hwaccm.s.vmx.msr.vmx_entry.n.allowed1;
1368	rc = VMXWriteVMCS(VMX_VMCS_CTRL_ENTRY_CONTROLS, val);
1369	AssertRC(rc);
1370
1371	/* VMX_VMCS_CTRL_EXIT_CONTROLS
1372	* Set required bits to one and zero according to the MSR capabilities.
1373	*/
1374	val = pVM->hwaccm.s.vmx.msr.vmx_exit.n.disallowed0;
1375
1376	/* Save debug controls (dr7 & IA32_DEBUGCTL_MSR) (forced to 1 on the 'first' VT-x capable CPUs; this actually includes the newest Nehalem CPUs) */
1377	val \|= VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_DEBUG;
1378
1379	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
1380	if (VMX_IS_64BIT_HOST_MODE())
1381	val \|= VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_AMD64;
1382	/* else: Must be zero when AMD64 is not available. */
1383	#elif HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
1384	if (CPUMIsGuestInLongModeEx(pCtx))
1385	val \|= VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_AMD64; /* our switcher goes to long mode */
1386	else
1387	Assert(!(val & VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_AMD64));
1388	#endif
1389	val &= pVM->hwaccm.s.vmx.msr.vmx_exit.n.allowed1;
1390	/* Don't acknowledge external interrupts on VM-exit. */
1391	rc = VMXWriteVMCS(VMX_VMCS_CTRL_EXIT_CONTROLS, val);
1392	AssertRC(rc);
1393
1394	/* Guest CPU context: ES, CS, SS, DS, FS, GS. */
1395	if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_SEGMENT_REGS)
1396	{
1397	if (pVM->hwaccm.s.vmx.pRealModeTSS)
1398	{
1399	PGMMODE enmGuestMode = PGMGetGuestMode(pVCpu);
1400	if (pVCpu->hwaccm.s.vmx.enmLastSeenGuestMode != enmGuestMode)
1401	{
1402	/* Correct weird requirements for switching to protected mode. */
1403	if ( pVCpu->hwaccm.s.vmx.enmLastSeenGuestMode == PGMMODE_REAL
1404	&& enmGuestMode >= PGMMODE_PROTECTED)
1405	{
1406	/* Flush the recompiler code cache as it's not unlikely
1407	* the guest will rewrite code it will later execute in real
1408	* mode (OpenBSD 4.0 is one such example)
1409	*/
1410	REMFlushTBs(pVM);
1411
1412	/* DPL of all hidden selector registers must match the current CPL (0). */
1413	pCtx->csHid.Attr.n.u2Dpl = 0;
1414	pCtx->csHid.Attr.n.u4Type = X86_SEL_TYPE_CODE \| X86_SEL_TYPE_RW_ACC;
1415
1416	pCtx->dsHid.Attr.n.u2Dpl = 0;
1417	pCtx->esHid.Attr.n.u2Dpl = 0;
1418	pCtx->fsHid.Attr.n.u2Dpl = 0;
1419	pCtx->gsHid.Attr.n.u2Dpl = 0;
1420	pCtx->ssHid.Attr.n.u2Dpl = 0;
1421
1422	/* The limit must correspond to the 32 bits setting. */
1423	if (!pCtx->csHid.Attr.n.u1DefBig)
1424	pCtx->csHid.u32Limit &= 0xffff;
1425	if (!pCtx->dsHid.Attr.n.u1DefBig)
1426	pCtx->dsHid.u32Limit &= 0xffff;
1427	if (!pCtx->esHid.Attr.n.u1DefBig)
1428	pCtx->esHid.u32Limit &= 0xffff;
1429	if (!pCtx->fsHid.Attr.n.u1DefBig)
1430	pCtx->fsHid.u32Limit &= 0xffff;
1431	if (!pCtx->gsHid.Attr.n.u1DefBig)
1432	pCtx->gsHid.u32Limit &= 0xffff;
1433	if (!pCtx->ssHid.Attr.n.u1DefBig)
1434	pCtx->ssHid.u32Limit &= 0xffff;
1435	}
1436	else
1437	/* Switching from protected mode to real mode. */
1438	if ( pVCpu->hwaccm.s.vmx.enmLastSeenGuestMode >= PGMMODE_PROTECTED
1439	&& enmGuestMode == PGMMODE_REAL)
1440	{
1441	/* The limit must also be set to 0xffff. */
1442	pCtx->csHid.u32Limit = 0xffff;
1443	pCtx->dsHid.u32Limit = 0xffff;
1444	pCtx->esHid.u32Limit = 0xffff;
1445	pCtx->fsHid.u32Limit = 0xffff;
1446	pCtx->gsHid.u32Limit = 0xffff;
1447	pCtx->ssHid.u32Limit = 0xffff;
1448
1449	Assert(pCtx->csHid.u64Base <= 0xfffff);
1450	Assert(pCtx->dsHid.u64Base <= 0xfffff);
1451	Assert(pCtx->esHid.u64Base <= 0xfffff);
1452	Assert(pCtx->fsHid.u64Base <= 0xfffff);
1453	Assert(pCtx->gsHid.u64Base <= 0xfffff);
1454	}
1455	pVCpu->hwaccm.s.vmx.enmLastSeenGuestMode = enmGuestMode;
1456	}
1457	else
1458	/* VT-x will fail with a guest invalid state otherwise... (CPU state after a reset) */
1459	if ( CPUMIsGuestInRealModeEx(pCtx)
1460	&& pCtx->csHid.u64Base == 0xffff0000)
1461	{
1462	pCtx->csHid.u64Base = 0xf0000;
1463	pCtx->cs = 0xf000;
1464	}
1465	}
1466
1467	VMX_WRITE_SELREG(ES, es);
1468	AssertRC(rc);
1469
1470	VMX_WRITE_SELREG(CS, cs);
1471	AssertRC(rc);
1472
1473	VMX_WRITE_SELREG(SS, ss);
1474	AssertRC(rc);
1475
1476	VMX_WRITE_SELREG(DS, ds);
1477	AssertRC(rc);
1478
1479	VMX_WRITE_SELREG(FS, fs);
1480	AssertRC(rc);
1481
1482	VMX_WRITE_SELREG(GS, gs);
1483	AssertRC(rc);
1484	}
1485
1486	/* Guest CPU context: LDTR. */
1487	if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_LDTR)
1488	{
1489	if (pCtx->ldtr == 0)
1490	{
1491	rc = VMXWriteVMCS(VMX_VMCS16_GUEST_FIELD_LDTR, 0);
1492	rc \|= VMXWriteVMCS(VMX_VMCS32_GUEST_LDTR_LIMIT, 0);
1493	rc \|= VMXWriteVMCS64(VMX_VMCS64_GUEST_LDTR_BASE, 0);
1494	/* Note: vmlaunch will fail with 0 or just 0x02. No idea why. */
1495	rc \|= VMXWriteVMCS(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, 0x82 /* present, LDT */);
1496	}
1497	else
1498	{
1499	rc = VMXWriteVMCS(VMX_VMCS16_GUEST_FIELD_LDTR, pCtx->ldtr);
1500	rc \|= VMXWriteVMCS(VMX_VMCS32_GUEST_LDTR_LIMIT, pCtx->ldtrHid.u32Limit);
1501	rc \|= VMXWriteVMCS64(VMX_VMCS64_GUEST_LDTR_BASE, pCtx->ldtrHid.u64Base);
1502	rc \|= VMXWriteVMCS(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, pCtx->ldtrHid.Attr.u);
1503	}
1504	AssertRC(rc);
1505	}
1506	/* Guest CPU context: TR. */
1507	if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_TR)
1508	{
1509	/* Real mode emulation using v86 mode with CR4.VME (interrupt redirection using the int bitmap in the TSS) */
1510	if ( CPUMIsGuestInRealModeEx(pCtx)
1511	&& pVM->hwaccm.s.vmx.pRealModeTSS)
1512	{
1513	RTGCPHYS GCPhys;
1514
1515	/* We convert it here every time as pci regions could be reconfigured. */
1516	rc = PDMVMMDevHeapR3ToGCPhys(pVM, pVM->hwaccm.s.vmx.pRealModeTSS, &GCPhys);
1517	AssertRC(rc);
1518
1519	rc = VMXWriteVMCS(VMX_VMCS16_GUEST_FIELD_TR, 0);
1520	rc \|= VMXWriteVMCS(VMX_VMCS32_GUEST_TR_LIMIT, HWACCM_VTX_TSS_SIZE);
1521	rc \|= VMXWriteVMCS64(VMX_VMCS64_GUEST_TR_BASE, GCPhys /* phys = virt in this mode */);
1522
1523	X86DESCATTR attr;
1524
1525	attr.u = 0;
1526	attr.n.u1Present = 1;
1527	attr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
1528	val = attr.u;
1529	}
1530	else
1531	{
1532	rc = VMXWriteVMCS(VMX_VMCS16_GUEST_FIELD_TR, pCtx->tr);
1533	rc \|= VMXWriteVMCS(VMX_VMCS32_GUEST_TR_LIMIT, pCtx->trHid.u32Limit);
1534	rc \|= VMXWriteVMCS64(VMX_VMCS64_GUEST_TR_BASE, pCtx->trHid.u64Base);
1535
1536	val = pCtx->trHid.Attr.u;
1537
1538	/* The TSS selector must be busy. */
1539	if ((val & 0xD) == X86_SEL_TYPE_SYS_286_TSS_AVAIL)
1540	val = (val & ~0xF) \| X86_SEL_TYPE_SYS_286_TSS_BUSY;
1541	else
1542	/* Default even if no TR selector has been set (otherwise vmlaunch will fail!) */
1543	val = (val & ~0xF) \| X86_SEL_TYPE_SYS_386_TSS_BUSY;
1544
1545	}
1546	rc \|= VMXWriteVMCS(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, val);
1547	AssertRC(rc);
1548	}
1549	/* Guest CPU context: GDTR. */
1550	if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_GDTR)
1551	{
1552	rc = VMXWriteVMCS(VMX_VMCS32_GUEST_GDTR_LIMIT, pCtx->gdtr.cbGdt);
1553	rc \|= VMXWriteVMCS64(VMX_VMCS64_GUEST_GDTR_BASE, pCtx->gdtr.pGdt);
1554	AssertRC(rc);
1555	}
1556	/* Guest CPU context: IDTR. */
1557	if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_IDTR)
1558	{
1559	rc = VMXWriteVMCS(VMX_VMCS32_GUEST_IDTR_LIMIT, pCtx->idtr.cbIdt);
1560	rc \|= VMXWriteVMCS64(VMX_VMCS64_GUEST_IDTR_BASE, pCtx->idtr.pIdt);
1561	AssertRC(rc);
1562	}
1563
1564	/*
1565	* Sysenter MSRs
1566	*/
1567	if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_MSR)
1568	{
1569	rc = VMXWriteVMCS(VMX_VMCS32_GUEST_SYSENTER_CS, pCtx->SysEnter.cs);
1570	rc \|= VMXWriteVMCS64(VMX_VMCS64_GUEST_SYSENTER_EIP, pCtx->SysEnter.eip);
1571	rc \|= VMXWriteVMCS64(VMX_VMCS64_GUEST_SYSENTER_ESP, pCtx->SysEnter.esp);
1572	AssertRC(rc);
1573	}
1574
1575	/* Control registers */
1576	if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_CR0)
1577	{
1578	val = pCtx->cr0;
1579	rc = VMXWriteVMCS(VMX_VMCS_CTRL_CR0_READ_SHADOW, val);
1580	Log2(("Guest CR0-shadow %08x\n", val));
1581	if (CPUMIsGuestFPUStateActive(pVCpu) == false)
1582	{
1583	/* Always use #NM exceptions to load the FPU/XMM state on demand. */
1584	val \|= X86_CR0_TS \| X86_CR0_ET \| X86_CR0_NE \| X86_CR0_MP;
1585	}
1586	else
1587	{
1588	/** @todo check if we support the old style mess correctly. */
1589	if (!(val & X86_CR0_NE))
1590	Log(("Forcing X86_CR0_NE!!!\n"));
1591
1592	val \|= X86_CR0_NE; /* always turn on the native mechanism to report FPU errors (old style uses interrupts) */
1593	}
1594	/* Note: protected mode & paging are always enabled; we use them for emulating real and protected mode without paging too. */
1595	if (!pVM->hwaccm.s.vmx.fUnrestrictedGuest)
1596	val \|= X86_CR0_PE \| X86_CR0_PG;
1597
1598	if (pVM->hwaccm.s.fNestedPaging)
1599	{
1600	if (CPUMIsGuestInPagedProtectedModeEx(pCtx))
1601	{
1602	/* Disable cr3 read/write monitoring as we don't need it for EPT. */
1603	pVCpu->hwaccm.s.vmx.proc_ctls &= ~( VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
1604	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT);
1605	}
1606	else
1607	{
1608	/* Reenable cr3 read/write monitoring as our identity mapped page table is active. */
1609	pVCpu->hwaccm.s.vmx.proc_ctls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
1610	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT;
1611	}
1612	rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
1613	AssertRC(rc);
1614	}
1615	else
1616	{
1617	/* Note: We must also set this as we rely on protecting various pages for which supervisor writes must be caught. */
1618	val \|= X86_CR0_WP;
1619	}
1620
1621	/* Always enable caching. */
1622	val &= ~(X86_CR0_CD\|X86_CR0_NW);
1623
1624	rc \|= VMXWriteVMCS64(VMX_VMCS64_GUEST_CR0, val);
1625	Log2(("Guest CR0 %08x\n", val));
1626	/* CR0 flags owned by the host; if the guests attempts to change them, then
1627	* the VM will exit.
1628	*/
1629	val = X86_CR0_PE /* Must monitor this bit (assumptions are made for real mode emulation) */
1630	\| X86_CR0_WP /* Must monitor this bit (it must always be enabled). */
1631	\| X86_CR0_PG /* Must monitor this bit (assumptions are made for real mode & protected mode without paging emulation) */
1632	\| X86_CR0_CD /* Bit not restored during VM-exit! */
1633	\| X86_CR0_NW /* Bit not restored during VM-exit! */
1634	\| X86_CR0_NE;
1635
1636	/* When the guest's FPU state is active, then we no longer care about
1637	* the FPU related bits.
1638	*/
1639	if (CPUMIsGuestFPUStateActive(pVCpu) == false)
1640	val \|= X86_CR0_TS \| X86_CR0_ET \| X86_CR0_MP;
1641
1642	pVCpu->hwaccm.s.vmx.cr0_mask = val;
1643
1644	rc \|= VMXWriteVMCS(VMX_VMCS_CTRL_CR0_MASK, val);
1645	Log2(("Guest CR0-mask %08x\n", val));
1646	AssertRC(rc);
1647	}
1648	if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_CR4)
1649	{
1650	/* CR4 */
1651	rc = VMXWriteVMCS(VMX_VMCS_CTRL_CR4_READ_SHADOW, pCtx->cr4);
1652	Log2(("Guest CR4-shadow %08x\n", pCtx->cr4));
1653	/* Set the required bits in cr4 too (currently X86_CR4_VMXE). */
1654	val = pCtx->cr4 \| (uint32_t)pVM->hwaccm.s.vmx.msr.vmx_cr4_fixed0;
1655
1656	if (!pVM->hwaccm.s.fNestedPaging)
1657	{
1658	switch(pVCpu->hwaccm.s.enmShadowMode)
1659	{
1660	case PGMMODE_REAL: /* Real mode -> emulated using v86 mode */
1661	case PGMMODE_PROTECTED: /* Protected mode, no paging -> emulated using identity mapping. */
1662	case PGMMODE_32_BIT: /* 32-bit paging. */
1663	val &= ~X86_CR4_PAE;
1664	break;
1665
1666	case PGMMODE_PAE: /* PAE paging. */
1667	case PGMMODE_PAE_NX: /* PAE paging with NX enabled. */
1668	/** Must use PAE paging as we could use physical memory > 4 GB */
1669	val \|= X86_CR4_PAE;
1670	break;
1671
1672	case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
1673	case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
1674	#ifdef VBOX_ENABLE_64_BITS_GUESTS
1675	break;
1676	#else
1677	AssertFailed();
1678	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
1679	#endif
1680	default: /* shut up gcc */
1681	AssertFailed();
1682	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
1683	}
1684	}
1685	else
1686	if ( !CPUMIsGuestInPagedProtectedModeEx(pCtx)
1687	&& !pVM->hwaccm.s.vmx.fUnrestrictedGuest)
1688	{
1689	/* We use 4 MB pages in our identity mapping page table for real and protected mode without paging. */
1690	val \|= X86_CR4_PSE;
1691	/* Our identity mapping is a 32 bits page directory. */
1692	val &= ~X86_CR4_PAE;
1693	}
1694
1695	/* Turn off VME if we're in emulated real mode. */
1696	if ( CPUMIsGuestInRealModeEx(pCtx)
1697	&& pVM->hwaccm.s.vmx.pRealModeTSS)
1698	val &= ~X86_CR4_VME;
1699
1700	rc \|= VMXWriteVMCS64(VMX_VMCS64_GUEST_CR4, val);
1701	Log2(("Guest CR4 %08x\n", val));
1702	/* CR4 flags owned by the host; if the guests attempts to change them, then
1703	* the VM will exit.
1704	*/
1705	val = 0
1706	\| X86_CR4_VME
1707	\| X86_CR4_PAE
1708	\| X86_CR4_PGE
1709	\| X86_CR4_PSE
1710	\| X86_CR4_VMXE;
1711	pVCpu->hwaccm.s.vmx.cr4_mask = val;
1712
1713	rc \|= VMXWriteVMCS(VMX_VMCS_CTRL_CR4_MASK, val);
1714	Log2(("Guest CR4-mask %08x\n", val));
1715	AssertRC(rc);
1716	}
1717
1718	if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_CR3)
1719	{
1720	if (pVM->hwaccm.s.fNestedPaging)
1721	{
1722	Assert(PGMGetHyperCR3(pVCpu));
1723	pVCpu->hwaccm.s.vmx.GCPhysEPTP = PGMGetHyperCR3(pVCpu);
1724
1725	Assert(!(pVCpu->hwaccm.s.vmx.GCPhysEPTP & 0xfff));
1726	/** @todo Check the IA32_VMX_EPT_VPID_CAP MSR for other supported memory types. */
1727	pVCpu->hwaccm.s.vmx.GCPhysEPTP \|= VMX_EPT_MEMTYPE_WB
1728	\| (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
1729
1730	rc = VMXWriteVMCS64(VMX_VMCS_CTRL_EPTP_FULL, pVCpu->hwaccm.s.vmx.GCPhysEPTP);
1731	AssertRC(rc);
1732
1733	if ( !CPUMIsGuestInPagedProtectedModeEx(pCtx)
1734	&& !pVM->hwaccm.s.vmx.fUnrestrictedGuest)
1735	{
1736	RTGCPHYS GCPhys;
1737
1738	/* We convert it here every time as pci regions could be reconfigured. */
1739	rc = PDMVMMDevHeapR3ToGCPhys(pVM, pVM->hwaccm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
1740	AssertMsgRC(rc, ("pNonPagingModeEPTPageTable = %RGv\n", pVM->hwaccm.s.vmx.pNonPagingModeEPTPageTable));
1741
1742	/* We use our identity mapping page table here as we need to map guest virtual to guest physical addresses; EPT will
1743	* take care of the translation to host physical addresses.
1744	*/
1745	val = GCPhys;
1746	}
1747	else
1748	{
1749	/* Save the real guest CR3 in VMX_VMCS_GUEST_CR3 */
1750	val = pCtx->cr3;
1751	/* Prefetch the four PDPT entries in PAE mode. */
1752	rc = vmxR0PrefetchPAEPdptrs(pVM, pVCpu, pCtx);
1753	AssertRCReturn(rc, rc);
1754	}
1755	}
1756	else
1757	{
1758	val = PGMGetHyperCR3(pVCpu);
1759	Assert(val \|\| VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL));
1760	}
1761
1762	/* Save our shadow CR3 register. */
1763	rc = VMXWriteVMCS64(VMX_VMCS64_GUEST_CR3, val);
1764	AssertRC(rc);
1765	}
1766
1767	/* Debug registers. */
1768	if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_DEBUG)
1769	{
1770	pCtx->dr[6] \|= X86_DR6_INIT_VAL; /* set all reserved bits to 1. */
1771	pCtx->dr[6] &= ~RT_BIT(12); /* must be zero. */
1772
1773	pCtx->dr[7] &= 0xffffffff; /* upper 32 bits reserved */
1774	pCtx->dr[7] &= ~(RT_BIT(11) \| RT_BIT(12) \| RT_BIT(14) \| RT_BIT(15)); /* must be zero */
1775	pCtx->dr[7] \|= 0x400; /* must be one */
1776
1777	/* Resync DR7 */
1778	rc = VMXWriteVMCS64(VMX_VMCS64_GUEST_DR7, pCtx->dr[7]);
1779	AssertRC(rc);
1780
1781	#ifdef DEBUG
1782	/* Sync the hypervisor debug state now if any breakpoint is armed. */
1783	if ( CPUMGetHyperDR7(pVCpu) & (X86_DR7_ENABLED_MASK\|X86_DR7_GD)
1784	&& !CPUMIsHyperDebugStateActive(pVCpu)
1785	&& !DBGFIsStepping(pVCpu))
1786	{
1787	/* Save the host and load the hypervisor debug state. */
1788	rc = CPUMR0LoadHyperDebugState(pVM, pVCpu, pCtx, true /* include DR6 */);
1789	AssertRC(rc);
1790
1791	/* DRx intercepts remain enabled. */
1792
1793	/* Override dr7 with the hypervisor value. */
1794	rc = VMXWriteVMCS64(VMX_VMCS64_GUEST_DR7, CPUMGetHyperDR7(pVCpu));
1795	AssertRC(rc);
1796	}
1797	else
1798	#endif
1799	/* Sync the debug state now if any breakpoint is armed. */
1800	if ( (pCtx->dr[7] & (X86_DR7_ENABLED_MASK\|X86_DR7_GD))
1801	&& !CPUMIsGuestDebugStateActive(pVCpu)
1802	&& !DBGFIsStepping(pVCpu))
1803	{
1804	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatDRxArmed);
1805
1806	/* Disable drx move intercepts. */
1807	pVCpu->hwaccm.s.vmx.proc_ctls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT;
1808	rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
1809	AssertRC(rc);
1810
1811	/* Save the host and load the guest debug state. */
1812	rc = CPUMR0LoadGuestDebugState(pVM, pVCpu, pCtx, true /* include DR6 */);
1813	AssertRC(rc);
1814	}
1815
1816	/* IA32_DEBUGCTL MSR. */
1817	rc = VMXWriteVMCS64(VMX_VMCS_GUEST_DEBUGCTL_FULL, 0);
1818	AssertRC(rc);
1819
1820	/** @todo do we really ever need this? */
1821	rc \|= VMXWriteVMCS(VMX_VMCS_GUEST_DEBUG_EXCEPTIONS, 0);
1822	AssertRC(rc);
1823	}
1824
1825	/* 64 bits guest mode? */
1826	if (CPUMIsGuestInLongModeEx(pCtx))
1827	{
1828	#if !defined(VBOX_ENABLE_64_BITS_GUESTS)
1829	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
1830	#elif HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
1831	pVCpu->hwaccm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
1832	#else
1833	# ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
1834	if (!pVM->hwaccm.s.fAllow64BitGuests)
1835	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
1836	# endif
1837	pVCpu->hwaccm.s.vmx.pfnStartVM = VMXR0StartVM64;
1838	#endif
1839	if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_MSR)
1840	{
1841	/* Update these as wrmsr might have changed them. */
1842	rc = VMXWriteVMCS64(VMX_VMCS64_GUEST_FS_BASE, pCtx->fsHid.u64Base);
1843	AssertRC(rc);
1844	rc = VMXWriteVMCS64(VMX_VMCS64_GUEST_GS_BASE, pCtx->gsHid.u64Base);
1845	AssertRC(rc);
1846	}
1847	}
1848	else
1849	{
1850	pVCpu->hwaccm.s.vmx.pfnStartVM = VMXR0StartVM32;
1851	}
1852
1853	vmxR0UpdateExceptionBitmap(pVM, pVCpu, pCtx);
1854
1855	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
1856	/* Store all guest MSRs in the VM-Entry load area, so they will be loaded during the world switch. */
1857	PVMXMSR pMsr = (PVMXMSR)pVCpu->hwaccm.s.vmx.pGuestMSR;
1858	unsigned idxMsr = 0;
1859
1860	uint32_t ulEdx;
1861	uint32_t ulTemp;
1862	CPUMGetGuestCpuId(pVCpu, 0x80000001, &ulTemp, &ulTemp, &ulTemp, &ulEdx);
1863	/* EFER MSR present? */
1864	if (ulEdx & (X86_CPUID_AMD_FEATURE_EDX_NX\|X86_CPUID_AMD_FEATURE_EDX_LONG_MODE))
1865	{
1866	pMsr->u32IndexMSR = MSR_K6_EFER;
1867	pMsr->u32Reserved = 0;
1868	pMsr->u64Value = pCtx->msrEFER;
1869	/* VT-x will complain if only MSR_K6_EFER_LME is set. */
1870	if (!CPUMIsGuestInLongModeEx(pCtx))
1871	pMsr->u64Value &= ~(MSR_K6_EFER_LMA\|MSR_K6_EFER_LME);
1872	pMsr++; idxMsr++;
1873
1874	if (ulEdx & X86_CPUID_AMD_FEATURE_EDX_LONG_MODE)
1875	{
1876	pMsr->u32IndexMSR = MSR_K8_LSTAR;
1877	pMsr->u32Reserved = 0;
1878	pMsr->u64Value = pCtx->msrLSTAR; /* 64 bits mode syscall rip */
1879	pMsr++; idxMsr++;
1880	pMsr->u32IndexMSR = MSR_K6_STAR;
1881	pMsr->u32Reserved = 0;
1882	pMsr->u64Value = pCtx->msrSTAR; /* legacy syscall eip, cs & ss */
1883	pMsr++; idxMsr++;
1884	pMsr->u32IndexMSR = MSR_K8_SF_MASK;
1885	pMsr->u32Reserved = 0;
1886	pMsr->u64Value = pCtx->msrSFMASK; /* syscall flag mask */
1887	pMsr++; idxMsr++;
1888	pMsr->u32IndexMSR = MSR_K8_KERNEL_GS_BASE;
1889	pMsr->u32Reserved = 0;
1890	pMsr->u64Value = pCtx->msrKERNELGSBASE; /* swapgs exchange value */
1891	pMsr++; idxMsr++;
1892	}
1893	}
1894	pVCpu->hwaccm.s.vmx.cCachedMSRs = idxMsr;
1895
1896	rc = VMXWriteVMCS(VMX_VMCS_CTRL_ENTRY_MSR_LOAD_COUNT, idxMsr);
1897	AssertRC(rc);
1898
1899	rc = VMXWriteVMCS(VMX_VMCS_CTRL_EXIT_MSR_STORE_COUNT, idxMsr);
1900	AssertRC(rc);
1901	#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
1902
1903	bool fOffsettedTsc;
1904	if (pVM->hwaccm.s.vmx.fUsePreemptTimer)
1905	{
1906	uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVCpu, &fOffsettedTsc, &pVCpu->hwaccm.s.vmx.u64TSCOffset);
1907
1908	/* Make sure the returned values have sane upper and lower boundaries. */
1909	uint64_t u64CpuHz = SUPGetCpuHzFromGIP(g_pSUPGlobalInfoPage);
1910
1911	cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64 of a second */
1912	cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
1913
1914	cTicksToDeadline >>= pVM->hwaccm.s.vmx.cPreemptTimerShift;
1915	uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
1916	rc = VMXWriteVMCS(VMX_VMCS32_GUEST_PREEMPTION_TIMER_VALUE, cPreemptionTickCount);
1917	AssertRC(rc);
1918	}
1919	else
1920	fOffsettedTsc = TMCpuTickCanUseRealTSC(pVCpu, &pVCpu->hwaccm.s.vmx.u64TSCOffset);
1921	if (fOffsettedTsc)
1922	{
1923	uint64_t u64CurTSC = ASMReadTSC();
1924	if (u64CurTSC + pVCpu->hwaccm.s.vmx.u64TSCOffset >= TMCpuTickGetLastSeen(pVCpu))
1925	{
1926	/* Note: VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT takes precedence over TSC_OFFSET */
1927	rc = VMXWriteVMCS64(VMX_VMCS_CTRL_TSC_OFFSET_FULL, pVCpu->hwaccm.s.vmx.u64TSCOffset);
1928	AssertRC(rc);
1929
1930	pVCpu->hwaccm.s.vmx.proc_ctls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT;
1931	rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
1932	AssertRC(rc);
1933	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatTSCOffset);
1934	}
1935	else
1936	{
1937	/* Fall back to rdtsc emulation as we would otherwise pass decreasing tsc values to the guest. */
1938	LogFlow(("TSC %RX64 offset %RX64 time=%RX64 last=%RX64 (diff=%RX64, virt_tsc=%RX64)\n", u64CurTSC, pVCpu->hwaccm.s.vmx.u64TSCOffset, u64CurTSC + pVCpu->hwaccm.s.vmx.u64TSCOffset, TMCpuTickGetLastSeen(pVCpu), TMCpuTickGetLastSeen(pVCpu) - u64CurTSC - pVCpu->hwaccm.s.vmx.u64TSCOffset, TMCpuTickGet(pVCpu)));
1939	pVCpu->hwaccm.s.vmx.proc_ctls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT;
1940	rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
1941	AssertRC(rc);
1942	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatTSCInterceptOverFlow);
1943	}
1944	}
1945	else
1946	{
1947	pVCpu->hwaccm.s.vmx.proc_ctls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT;
1948	rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
1949	AssertRC(rc);
1950	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatTSCIntercept);
1951	}
1952
1953	/* Done with the major changes */
1954	pVCpu->hwaccm.s.fContextUseFlags &= ~HWACCM_CHANGED_ALL_GUEST;
1955
1956	/* Minimal guest state update (esp, eip, eflags mostly) */
1957	VMXR0LoadMinimalGuestState(pVM, pVCpu, pCtx);
1958	return rc;
1959	}
1960
1961	/**
1962	* Syncs back the guest state
1963	*
1964	* @returns VBox status code.
1965	* @param pVM The VM to operate on.
1966	* @param pVCpu The VMCPU to operate on.
1967	* @param pCtx Guest context
1968	*/
1969	DECLINLINE(int) VMXR0SaveGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
1970	{
1971	RTGCUINTREG val, valShadow;
1972	RTGCUINTPTR uInterruptState;
1973	int rc;
1974
1975	/* Let's first sync back eip, esp, and eflags. */
1976	rc = VMXReadCachedVMCS(VMX_VMCS64_GUEST_RIP, &val);
1977	AssertRC(rc);
1978	pCtx->rip = val;
1979	rc = VMXReadCachedVMCS(VMX_VMCS64_GUEST_RSP, &val);
1980	AssertRC(rc);
1981	pCtx->rsp = val;
1982	rc = VMXReadCachedVMCS(VMX_VMCS_GUEST_RFLAGS, &val);
1983	AssertRC(rc);
1984	pCtx->eflags.u32 = val;
1985
1986	/* Take care of instruction fusing (sti, mov ss) */
1987	rc \|= VMXReadCachedVMCS(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &val);
1988	uInterruptState = val;
1989	if (uInterruptState != 0)
1990	{
1991	Assert(uInterruptState <= 2); /* only sti & mov ss */
1992	Log(("uInterruptState %x eip=%RGv\n", (uint32_t)uInterruptState, pCtx->rip));
1993	EMSetInhibitInterruptsPC(pVCpu, pCtx->rip);
1994	}
1995	else
1996	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
1997
1998	/* Control registers. */
1999	VMXReadCachedVMCS(VMX_VMCS_CTRL_CR0_READ_SHADOW, &valShadow);
2000	VMXReadCachedVMCS(VMX_VMCS64_GUEST_CR0, &val);
2001	val = (valShadow & pVCpu->hwaccm.s.vmx.cr0_mask) \| (val & ~pVCpu->hwaccm.s.vmx.cr0_mask);
2002	CPUMSetGuestCR0(pVCpu, val);
2003
2004	VMXReadCachedVMCS(VMX_VMCS_CTRL_CR4_READ_SHADOW, &valShadow);
2005	VMXReadCachedVMCS(VMX_VMCS64_GUEST_CR4, &val);
2006	val = (valShadow & pVCpu->hwaccm.s.vmx.cr4_mask) \| (val & ~pVCpu->hwaccm.s.vmx.cr4_mask);
2007	CPUMSetGuestCR4(pVCpu, val);
2008
2009	/* Note: no reason to sync back the CRx registers. They can't be changed by the guest. */
2010	/* Note: only in the nested paging case can CR3 & CR4 be changed by the guest. */
2011	if ( pVM->hwaccm.s.fNestedPaging
2012	&& CPUMIsGuestInPagedProtectedModeEx(pCtx))
2013	{
2014	PVMCSCACHE pCache = &pVCpu->hwaccm.s.vmx.VMCSCache;
2015
2016	/* Can be updated behind our back in the nested paging case. */
2017	CPUMSetGuestCR2(pVCpu, pCache->cr2);
2018
2019	VMXReadCachedVMCS(VMX_VMCS64_GUEST_CR3, &val);
2020
2021	if (val != pCtx->cr3)
2022	{
2023	CPUMSetGuestCR3(pVCpu, val);
2024	PGMUpdateCR3(pVCpu, val);
2025	}
2026	/* Prefetch the four PDPT entries in PAE mode. */
2027	rc = vmxR0PrefetchPAEPdptrs(pVM, pVCpu, pCtx);
2028	AssertRCReturn(rc, rc);
2029	}
2030
2031	/* Sync back DR7 here. */
2032	VMXReadCachedVMCS(VMX_VMCS64_GUEST_DR7, &val);
2033	pCtx->dr[7] = val;
2034
2035	/* Guest CPU context: ES, CS, SS, DS, FS, GS. */
2036	VMX_READ_SELREG(ES, es);
2037	VMX_READ_SELREG(SS, ss);
2038	VMX_READ_SELREG(CS, cs);
2039	VMX_READ_SELREG(DS, ds);
2040	VMX_READ_SELREG(FS, fs);
2041	VMX_READ_SELREG(GS, gs);
2042
2043	/*
2044	* System MSRs
2045	*/
2046	VMXReadCachedVMCS(VMX_VMCS32_GUEST_SYSENTER_CS, &val);
2047	pCtx->SysEnter.cs = val;
2048	VMXReadCachedVMCS(VMX_VMCS64_GUEST_SYSENTER_EIP, &val);
2049	pCtx->SysEnter.eip = val;
2050	VMXReadCachedVMCS(VMX_VMCS64_GUEST_SYSENTER_ESP, &val);
2051	pCtx->SysEnter.esp = val;
2052
2053	/* Misc. registers; must sync everything otherwise we can get out of sync when jumping to ring 3. */
2054	VMX_READ_SELREG(LDTR, ldtr);
2055
2056	VMXReadCachedVMCS(VMX_VMCS32_GUEST_GDTR_LIMIT, &val);
2057	pCtx->gdtr.cbGdt = val;
2058	VMXReadCachedVMCS(VMX_VMCS64_GUEST_GDTR_BASE, &val);
2059	pCtx->gdtr.pGdt = val;
2060
2061	VMXReadCachedVMCS(VMX_VMCS32_GUEST_IDTR_LIMIT, &val);
2062	pCtx->idtr.cbIdt = val;
2063	VMXReadCachedVMCS(VMX_VMCS64_GUEST_IDTR_BASE, &val);
2064	pCtx->idtr.pIdt = val;
2065
2066	/* Real mode emulation using v86 mode. */
2067	if ( CPUMIsGuestInRealModeEx(pCtx)
2068	&& pVM->hwaccm.s.vmx.pRealModeTSS)
2069	{
2070	/* Hide our emulation flags */
2071	pCtx->eflags.Bits.u1VM = 0;
2072
2073	/* Restore original IOPL setting as we always use 0. */
2074	pCtx->eflags.Bits.u2IOPL = pVCpu->hwaccm.s.vmx.RealMode.eflags.Bits.u2IOPL;
2075
2076	/* Force a TR resync every time in case we switch modes. */
2077	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_GUEST_TR;
2078	}
2079	else
2080	{
2081	/* In real mode we have a fake TSS, so only sync it back when it's supposed to be valid. */
2082	VMX_READ_SELREG(TR, tr);
2083	}
2084
2085	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
2086	/* Save the possibly changed MSRs that we automatically restore and save during a world switch. */
2087	for (unsigned i = 0; i < pVCpu->hwaccm.s.vmx.cCachedMSRs; i++)
2088	{
2089	PVMXMSR pMsr = (PVMXMSR)pVCpu->hwaccm.s.vmx.pGuestMSR;
2090	pMsr += i;
2091
2092	switch (pMsr->u32IndexMSR)
2093	{
2094	case MSR_K8_LSTAR:
2095	pCtx->msrLSTAR = pMsr->u64Value;
2096	break;
2097	case MSR_K6_STAR:
2098	pCtx->msrSTAR = pMsr->u64Value;
2099	break;
2100	case MSR_K8_SF_MASK:
2101	pCtx->msrSFMASK = pMsr->u64Value;
2102	break;
2103	case MSR_K8_KERNEL_GS_BASE:
2104	pCtx->msrKERNELGSBASE = pMsr->u64Value;
2105	break;
2106	case MSR_K6_EFER:
2107	/* EFER can't be changed without causing a VM-exit. */
2108	// Assert(pCtx->msrEFER == pMsr->u64Value);
2109	break;
2110	default:
2111	AssertFailed();
2112	return VERR_INTERNAL_ERROR;
2113	}
2114	}
2115	#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
2116	return VINF_SUCCESS;
2117	}
2118
2119	/**
2120	* Dummy placeholder
2121	*
2122	* @param pVM The VM to operate on.
2123	* @param pVCpu The VMCPU to operate on.
2124	*/
2125	static void vmxR0SetupTLBDummy(PVM pVM, PVMCPU pVCpu)
2126	{
2127	NOREF(pVM);
2128	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
2129	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
2130	pVCpu->hwaccm.s.TlbShootdown.cPages = 0;
2131	return;
2132	}
2133
2134	/**
2135	* Setup the tagged TLB for EPT
2136	*
2137	* @returns VBox status code.
2138	* @param pVM The VM to operate on.
2139	* @param pVCpu The VMCPU to operate on.
2140	*/
2141	static void vmxR0SetupTLBEPT(PVM pVM, PVMCPU pVCpu)
2142	{
2143	PHWACCM_CPUINFO pCpu;
2144
2145	Assert(pVM->hwaccm.s.fNestedPaging);
2146	Assert(!pVM->hwaccm.s.vmx.fVPID);
2147
2148	/* Deal with tagged TLBs if VPID or EPT is supported. */
2149	pCpu = HWACCMR0GetCurrentCpu();
2150	/* Force a TLB flush for the first world switch if the current cpu differs from the one we ran on last. */
2151	/* Note that this can happen both for start and resume due to long jumps back to ring 3. */
2152	if ( pVCpu->hwaccm.s.idLastCpu != pCpu->idCpu
2153	/* if the tlb flush count has changed, another VM has flushed the TLB of this cpu, so we can't use our current ASID anymore. */
2154	\|\| pVCpu->hwaccm.s.cTLBFlushes != pCpu->cTLBFlushes)
2155	{
2156	/* Force a TLB flush on VM entry. */
2157	pVCpu->hwaccm.s.fForceTLBFlush = true;
2158	}
2159	else
2160	Assert(!pCpu->fFlushTLB);
2161
2162	/* Check for tlb shootdown flushes. */
2163	if (VMCPU_FF_TESTANDCLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2164	pVCpu->hwaccm.s.fForceTLBFlush = true;
2165
2166	pVCpu->hwaccm.s.idLastCpu = pCpu->idCpu;
2167	pCpu->fFlushTLB = false;
2168
2169	if (pVCpu->hwaccm.s.fForceTLBFlush)
2170	{
2171	vmxR0FlushEPT(pVM, pVCpu, pVM->hwaccm.s.vmx.enmFlushContext, 0);
2172	}
2173	else
2174	if (VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
2175	{
2176	/* Deal with pending TLB shootdown actions which were queued when we were not executing code. */
2177	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatTlbShootdown);
2178
2179	for (unsigned i=0;i<pVCpu->hwaccm.s.TlbShootdown.cPages;i++)
2180	{
2181	/* aTlbShootdownPages contains physical addresses in this case. */
2182	vmxR0FlushEPT(pVM, pVCpu, pVM->hwaccm.s.vmx.enmFlushPage, pVCpu->hwaccm.s.TlbShootdown.aPages[i]);
2183	}
2184	}
2185	pVCpu->hwaccm.s.TlbShootdown.cPages= 0;
2186	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
2187
2188	#ifdef VBOX_WITH_STATISTICS
2189	if (pVCpu->hwaccm.s.fForceTLBFlush)
2190	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatFlushTLBWorldSwitch);
2191	else
2192	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatNoFlushTLBWorldSwitch);
2193	#endif
2194	}
2195
2196	#ifdef HWACCM_VTX_WITH_VPID
2197	/**
2198	* Setup the tagged TLB for VPID
2199	*
2200	* @returns VBox status code.
2201	* @param pVM The VM to operate on.
2202	* @param pVCpu The VMCPU to operate on.
2203	*/
2204	static void vmxR0SetupTLBVPID(PVM pVM, PVMCPU pVCpu)
2205	{
2206	PHWACCM_CPUINFO pCpu;
2207
2208	Assert(pVM->hwaccm.s.vmx.fVPID);
2209	Assert(!pVM->hwaccm.s.fNestedPaging);
2210
2211	/* Deal with tagged TLBs if VPID or EPT is supported. */
2212	pCpu = HWACCMR0GetCurrentCpu();
2213	/* Force a TLB flush for the first world switch if the current cpu differs from the one we ran on last. */
2214	/* Note that this can happen both for start and resume due to long jumps back to ring 3. */
2215	if ( pVCpu->hwaccm.s.idLastCpu != pCpu->idCpu
2216	/* if the tlb flush count has changed, another VM has flushed the TLB of this cpu, so we can't use our current ASID anymore. */
2217	\|\| pVCpu->hwaccm.s.cTLBFlushes != pCpu->cTLBFlushes)
2218	{
2219	/* Force a TLB flush on VM entry. */
2220	pVCpu->hwaccm.s.fForceTLBFlush = true;
2221	}
2222	else
2223	Assert(!pCpu->fFlushTLB);
2224
2225	pVCpu->hwaccm.s.idLastCpu = pCpu->idCpu;
2226
2227	/* Check for tlb shootdown flushes. */
2228	if (VMCPU_FF_TESTANDCLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2229	pVCpu->hwaccm.s.fForceTLBFlush = true;
2230
2231	/* Make sure we flush the TLB when required. Switch ASID to achieve the same thing, but without actually flushing the whole TLB (which is expensive). */
2232	if (pVCpu->hwaccm.s.fForceTLBFlush)
2233	{
2234	if ( ++pCpu->uCurrentASID >= pVM->hwaccm.s.uMaxASID
2235	\|\| pCpu->fFlushTLB)
2236	{
2237	pCpu->fFlushTLB = false;
2238	pCpu->uCurrentASID = 1; /* start at 1; host uses 0 */
2239	pCpu->cTLBFlushes++;
2240	vmxR0FlushVPID(pVM, pVCpu, VMX_FLUSH_ALL_CONTEXTS, 0);
2241	}
2242	else
2243	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatFlushASID);
2244
2245	pVCpu->hwaccm.s.fForceTLBFlush = false;
2246	pVCpu->hwaccm.s.cTLBFlushes = pCpu->cTLBFlushes;
2247	pVCpu->hwaccm.s.uCurrentASID = pCpu->uCurrentASID;
2248	}
2249	else
2250	{
2251	Assert(!pCpu->fFlushTLB);
2252	Assert(pVCpu->hwaccm.s.uCurrentASID && pCpu->uCurrentASID);
2253
2254	if (VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
2255	{
2256	/* Deal with pending TLB shootdown actions which were queued when we were not executing code. */
2257	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatTlbShootdown);
2258	for (unsigned i=0;i<pVCpu->hwaccm.s.TlbShootdown.cPages;i++)
2259	vmxR0FlushVPID(pVM, pVCpu, pVM->hwaccm.s.vmx.enmFlushPage, pVCpu->hwaccm.s.TlbShootdown.aPages[i]);
2260	}
2261	}
2262	pVCpu->hwaccm.s.TlbShootdown.cPages = 0;
2263	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
2264
2265	AssertMsg(pVCpu->hwaccm.s.cTLBFlushes == pCpu->cTLBFlushes, ("Flush count mismatch for cpu %d (%x vs %x)\n", pCpu->idCpu, pVCpu->hwaccm.s.cTLBFlushes, pCpu->cTLBFlushes));
2266	AssertMsg(pCpu->uCurrentASID >= 1 && pCpu->uCurrentASID < pVM->hwaccm.s.uMaxASID, ("cpu%d uCurrentASID = %x\n", pCpu->idCpu, pCpu->uCurrentASID));
2267	AssertMsg(pVCpu->hwaccm.s.uCurrentASID >= 1 && pVCpu->hwaccm.s.uCurrentASID < pVM->hwaccm.s.uMaxASID, ("cpu%d VM uCurrentASID = %x\n", pCpu->idCpu, pVCpu->hwaccm.s.uCurrentASID));
2268
2269	int rc = VMXWriteVMCS(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hwaccm.s.uCurrentASID);
2270	AssertRC(rc);
2271
2272	if (pVCpu->hwaccm.s.fForceTLBFlush)
2273	vmxR0FlushVPID(pVM, pVCpu, pVM->hwaccm.s.vmx.enmFlushContext, 0);
2274
2275	#ifdef VBOX_WITH_STATISTICS
2276	if (pVCpu->hwaccm.s.fForceTLBFlush)
2277	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatFlushTLBWorldSwitch);
2278	else
2279	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatNoFlushTLBWorldSwitch);
2280	#endif
2281	}
2282	#endif /* HWACCM_VTX_WITH_VPID */
2283
2284	/**
2285	* Runs guest code in a VT-x VM.
2286	*
2287	* @returns VBox status code.
2288	* @param pVM The VM to operate on.
2289	* @param pVCpu The VMCPU to operate on.
2290	* @param pCtx Guest context
2291	*/
2292	VMMR0DECL(int) VMXR0RunGuestCode(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
2293	{
2294	STAM_PROFILE_ADV_START(&pVCpu->hwaccm.s.StatEntry, x);
2295	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hwaccm.s.StatExit1);
2296	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hwaccm.s.StatExit2);
2297
2298	VBOXSTRICTRC rc = VINF_SUCCESS;
2299	int rc2;
2300	RTGCUINTREG val;
2301	RTGCUINTREG exitReason = (RTGCUINTREG)VMX_EXIT_INVALID;
2302	RTGCUINTREG instrError, cbInstr;
2303	RTGCUINTPTR exitQualification = 0;
2304	RTGCUINTPTR intInfo = 0; /* shut up buggy gcc 4 */
2305	RTGCUINTPTR errCode, instrInfo;
2306	bool fSetupTPRCaching = false;
2307	uint64_t u64OldLSTAR = 0;
2308	uint8_t u8LastTPR = 0;
2309	RTCCUINTREG uOldEFlags = ~(RTCCUINTREG)0;
2310	unsigned cResume = 0;
2311	#ifdef VBOX_STRICT
2312	RTCPUID idCpuCheck;
2313	bool fWasInLongMode = false;
2314	#endif
2315	#ifdef VBOX_HIGH_RES_TIMERS_HACK_IN_RING0
2316	uint64_t u64LastTime = RTTimeMilliTS();
2317	#endif
2318
2319	Assert(!(pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC) \|\| (pVCpu->hwaccm.s.vmx.pVAPIC && pVM->hwaccm.s.vmx.pAPIC));
2320
2321	/* Check if we need to use TPR shadowing. */
2322	if ( CPUMIsGuestInLongModeEx(pCtx)
2323	\|\| ( ((pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC) \|\| pVM->hwaccm.s.fTRPPatchingAllowed)
2324	&& pVM->hwaccm.s.fHasIoApic)
2325	)
2326	{
2327	fSetupTPRCaching = true;
2328	}
2329
2330	Log2(("\nE"));
2331
2332	#ifdef VBOX_STRICT
2333	{
2334	RTCCUINTREG val2;
2335
2336	rc2 = VMXReadVMCS(VMX_VMCS_CTRL_PIN_EXEC_CONTROLS, &val2);
2337	AssertRC(rc2);
2338	Log2(("VMX_VMCS_CTRL_PIN_EXEC_CONTROLS = %08x\n", val2));
2339
2340	/* allowed zero */
2341	if ((val2 & pVM->hwaccm.s.vmx.msr.vmx_pin_ctls.n.disallowed0) != pVM->hwaccm.s.vmx.msr.vmx_pin_ctls.n.disallowed0)
2342	Log(("Invalid VMX_VMCS_CTRL_PIN_EXEC_CONTROLS: zero\n"));
2343
2344	/* allowed one */
2345	if ((val2 & ~pVM->hwaccm.s.vmx.msr.vmx_pin_ctls.n.allowed1) != 0)
2346	Log(("Invalid VMX_VMCS_CTRL_PIN_EXEC_CONTROLS: one\n"));
2347
2348	rc2 = VMXReadVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, &val2);
2349	AssertRC(rc2);
2350	Log2(("VMX_VMCS_CTRL_PROC_EXEC_CONTROLS = %08x\n", val2));
2351
2352	/* Must be set according to the MSR, but can be cleared in case of EPT. */
2353	if (pVM->hwaccm.s.fNestedPaging)
2354	val2 \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INVLPG_EXIT
2355	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
2356	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT;
2357
2358	/* allowed zero */
2359	if ((val2 & pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.disallowed0) != pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.disallowed0)
2360	Log(("Invalid VMX_VMCS_CTRL_PROC_EXEC_CONTROLS: zero\n"));
2361
2362	/* allowed one */
2363	if ((val2 & ~pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1) != 0)
2364	Log(("Invalid VMX_VMCS_CTRL_PROC_EXEC_CONTROLS: one\n"));
2365
2366	rc2 = VMXReadVMCS(VMX_VMCS_CTRL_ENTRY_CONTROLS, &val2);
2367	AssertRC(rc2);
2368	Log2(("VMX_VMCS_CTRL_ENTRY_CONTROLS = %08x\n", val2));
2369
2370	/* allowed zero */
2371	if ((val2 & pVM->hwaccm.s.vmx.msr.vmx_entry.n.disallowed0) != pVM->hwaccm.s.vmx.msr.vmx_entry.n.disallowed0)
2372	Log(("Invalid VMX_VMCS_CTRL_ENTRY_CONTROLS: zero\n"));
2373
2374	/* allowed one */
2375	if ((val2 & ~pVM->hwaccm.s.vmx.msr.vmx_entry.n.allowed1) != 0)
2376	Log(("Invalid VMX_VMCS_CTRL_ENTRY_CONTROLS: one\n"));
2377
2378	rc2 = VMXReadVMCS(VMX_VMCS_CTRL_EXIT_CONTROLS, &val2);
2379	AssertRC(rc2);
2380	Log2(("VMX_VMCS_CTRL_EXIT_CONTROLS = %08x\n", val2));
2381
2382	/* allowed zero */
2383	if ((val2 & pVM->hwaccm.s.vmx.msr.vmx_exit.n.disallowed0) != pVM->hwaccm.s.vmx.msr.vmx_exit.n.disallowed0)
2384	Log(("Invalid VMX_VMCS_CTRL_EXIT_CONTROLS: zero\n"));
2385
2386	/* allowed one */
2387	if ((val2 & ~pVM->hwaccm.s.vmx.msr.vmx_exit.n.allowed1) != 0)
2388	Log(("Invalid VMX_VMCS_CTRL_EXIT_CONTROLS: one\n"));
2389	}
2390	fWasInLongMode = CPUMIsGuestInLongModeEx(pCtx);
2391	#endif /* VBOX_STRICT */
2392
2393	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
2394	pVCpu->hwaccm.s.vmx.VMCSCache.u64TimeEntry = RTTimeNanoTS();
2395	#endif
2396
2397	/* We can jump to this point to resume execution after determining that a VM-exit is innocent.
2398	*/
2399	ResumeExecution:
2400	if (!STAM_REL_PROFILE_ADV_IS_RUNNING(&pVCpu->hwaccm.s.StatEntry))
2401	STAM_REL_PROFILE_ADV_STOP_START(&pVCpu->hwaccm.s.StatExit2, &pVCpu->hwaccm.s.StatEntry, x);
2402	AssertMsg(pVCpu->hwaccm.s.idEnteredCpu == RTMpCpuId(),
2403	("Expected %d, I'm %d; cResume=%d exitReason=%RGv exitQualification=%RGv\n",
2404	(int)pVCpu->hwaccm.s.idEnteredCpu, (int)RTMpCpuId(), cResume, exitReason, exitQualification));
2405	Assert(!HWACCMR0SuspendPending());
2406	/* Not allowed to switch modes without reloading the host state (32->64 switcher)!! */
2407	Assert(fWasInLongMode == CPUMIsGuestInLongModeEx(pCtx));
2408
2409	/* Safety precaution; looping for too long here can have a very bad effect on the host */
2410	if (RT_UNLIKELY(++cResume > pVM->hwaccm.s.cMaxResumeLoops))
2411	{
2412	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitMaxResume);
2413	rc = VINF_EM_RAW_INTERRUPT;
2414	goto end;
2415	}
2416
2417	/* Check for irq inhibition due to instruction fusing (sti, mov ss). */
2418	if (VMCPU_FF_ISSET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
2419	{
2420	Log(("VM_FF_INHIBIT_INTERRUPTS at %RGv successor %RGv\n", (RTGCPTR)pCtx->rip, EMGetInhibitInterruptsPC(pVCpu)));
2421	if (pCtx->rip != EMGetInhibitInterruptsPC(pVCpu))
2422	{
2423	/* Note: we intentionally don't clear VM_FF_INHIBIT_INTERRUPTS here.
2424	* Before we are able to execute this instruction in raw mode (iret to guest code) an external interrupt might
2425	* force a world switch again. Possibly allowing a guest interrupt to be dispatched in the process. This could
2426	* break the guest. Sounds very unlikely, but such timing sensitive problems are not as rare as you might think.
2427	*/
2428	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
2429	/* Irq inhibition is no longer active; clear the corresponding VMX state. */
2430	rc2 = VMXWriteVMCS(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, 0);
2431	AssertRC(rc2);
2432	}
2433	}
2434	else
2435	{
2436	/* Irq inhibition is no longer active; clear the corresponding VMX state. */
2437	rc2 = VMXWriteVMCS(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, 0);
2438	AssertRC(rc2);
2439	}
2440
2441	#ifdef VBOX_HIGH_RES_TIMERS_HACK_IN_RING0
2442	if (RT_UNLIKELY((cResume & 0xf) == 0))
2443	{
2444	uint64_t u64CurTime = RTTimeMilliTS();
2445
2446	if (RT_UNLIKELY(u64CurTime > u64LastTime))
2447	{
2448	u64LastTime = u64CurTime;
2449	TMTimerPollVoid(pVM, pVCpu);
2450	}
2451	}
2452	#endif
2453
2454	/* Check for pending actions that force us to go back to ring 3. */
2455	if ( VM_FF_ISPENDING(pVM, VM_FF_HWACCM_TO_R3_MASK \| VM_FF_REQUEST \| VM_FF_PGM_POOL_FLUSH_PENDING \| VM_FF_PDM_DMA)
2456	\|\| VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_HWACCM_TO_R3_MASK \| VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL \| VMCPU_FF_REQUEST))
2457	{
2458	/* Check if a sync operation is pending. */
2459	if (VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
2460	{
2461	rc = PGMSyncCR3(pVCpu, pCtx->cr0, pCtx->cr3, pCtx->cr4, VMCPU_FF_ISSET(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
2462	if (rc != VINF_SUCCESS)
2463	{
2464	AssertRC(VBOXSTRICTRC_VAL(rc));
2465	Log(("Pending pool sync is forcing us back to ring 3; rc=%d\n", VBOXSTRICTRC_VAL(rc)));
2466	goto end;
2467	}
2468	}
2469
2470	#ifdef DEBUG
2471	/* Intercept X86_XCPT_DB if stepping is enabled */
2472	if (!DBGFIsStepping(pVCpu))
2473	#endif
2474	{
2475	if ( VM_FF_ISPENDING(pVM, VM_FF_HWACCM_TO_R3_MASK)
2476	\|\| VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_HWACCM_TO_R3_MASK))
2477	{
2478	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatSwitchToR3);
2479	rc = RT_UNLIKELY(VM_FF_ISPENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
2480	goto end;
2481	}
2482	}
2483
2484	/* Pending request packets might contain actions that need immediate attention, such as pending hardware interrupts. */
2485	if ( VM_FF_ISPENDING(pVM, VM_FF_REQUEST)
2486	\|\| VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_REQUEST))
2487	{
2488	rc = VINF_EM_PENDING_REQUEST;
2489	goto end;
2490	}
2491
2492	/* Check if a pgm pool flush is in progress. */
2493	if (VM_FF_ISPENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
2494	{
2495	rc = VINF_PGM_POOL_FLUSH_PENDING;
2496	goto end;
2497	}
2498
2499	/* Check if DMA work is pending (2nd+ run). */
2500	if (VM_FF_ISPENDING(pVM, VM_FF_PDM_DMA) && cResume > 1)
2501	{
2502	rc = VINF_EM_RAW_TO_R3;
2503	goto end;
2504	}
2505	}
2506
2507	#ifdef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
2508	/*
2509	* Exit to ring-3 preemption/work is pending.
2510	*
2511	* Interrupts are disabled before the call to make sure we don't miss any interrupt
2512	* that would flag preemption (IPI, timer tick, ++). (Would've been nice to do this
2513	* further down, but VMXR0CheckPendingInterrupt makes that impossible.)
2514	*
2515	* Note! Interrupts must be disabled done before we check for TLB flushes; TLB
2516	* shootdowns rely on this.
2517	*/
2518	uOldEFlags = ASMIntDisableFlags();
2519	if (RTThreadPreemptIsPending(NIL_RTTHREAD))
2520	{
2521	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitPreemptPending);
2522	rc = VINF_EM_RAW_INTERRUPT;
2523	goto end;
2524	}
2525	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
2526	#endif
2527
2528	/* When external interrupts are pending, we should exit the VM when IF is set. */
2529	/* Note! After VM_FF_INHIBIT_INTERRUPTS check!!! */
2530	rc = VMXR0CheckPendingInterrupt(pVM, pVCpu, pCtx);
2531	if (RT_FAILURE(rc))
2532	goto end;
2533
2534	/** @todo check timers?? */
2535
2536	/* TPR caching using CR8 is only available in 64 bits mode */
2537	/* Note the 32 bits exception for AMD (X86_CPUID_AMD_FEATURE_ECX_CR8L), but that appears missing in Intel CPUs */
2538	/* Note: we can't do this in LoadGuestState as PDMApicGetTPR can jump back to ring 3 (lock)!!!!! (no longer true) */
2539	/**
2540	* @todo query and update the TPR only when it could have been changed (mmio access & wrmsr (x2apic))
2541	*/
2542	if (fSetupTPRCaching)
2543	{
2544	/* TPR caching in CR8 */
2545	bool fPending;
2546
2547	rc2 = PDMApicGetTPR(pVCpu, &u8LastTPR, &fPending);
2548	AssertRC(rc2);
2549	/* The TPR can be found at offset 0x80 in the APIC mmio page. */
2550	pVCpu->hwaccm.s.vmx.pVAPIC[0x80] = u8LastTPR;
2551
2552	/* Two options here:
2553	* - external interrupt pending, but masked by the TPR value.
2554	* -> a CR8 update that lower the current TPR value should cause an exit
2555	* - no pending interrupts
2556	* -> We don't need to be explicitely notified. There are enough world switches for detecting pending interrupts.
2557	*/
2558	rc = VMXWriteVMCS(VMX_VMCS_CTRL_TPR_THRESHOLD, (fPending) ? (u8LastTPR >> 4) : 0); /* cr8 bits 3-0 correspond to bits 7-4 of the task priority mmio register. */
2559	AssertRC(VBOXSTRICTRC_VAL(rc));
2560
2561	if (pVM->hwaccm.s.fTPRPatchingActive)
2562	{
2563	Assert(!CPUMIsGuestInLongModeEx(pCtx));
2564	/* Our patch code uses LSTAR for TPR caching. */
2565	pCtx->msrLSTAR = u8LastTPR;
2566
2567	if (fPending)
2568	{
2569	/* A TPR change could activate a pending interrupt, so catch lstar writes. */
2570	vmxR0SetMSRPermission(pVCpu, MSR_K8_LSTAR, true, false);
2571	}
2572	else
2573	{
2574	/* No interrupts are pending, so we don't need to be explicitely notified.
2575	* There are enough world switches for detecting pending interrupts.
2576	*/
2577	vmxR0SetMSRPermission(pVCpu, MSR_K8_LSTAR, true, true);
2578	}
2579	}
2580	}
2581
2582	#if defined(HWACCM_VTX_WITH_EPT) && defined(LOG_ENABLED)
2583	if ( pVM->hwaccm.s.fNestedPaging
2584	# ifdef HWACCM_VTX_WITH_VPID
2585	\|\| pVM->hwaccm.s.vmx.fVPID
2586	# endif /* HWACCM_VTX_WITH_VPID */
2587	)
2588	{
2589	PHWACCM_CPUINFO pCpu;
2590
2591	pCpu = HWACCMR0GetCurrentCpu();
2592	if ( pVCpu->hwaccm.s.idLastCpu != pCpu->idCpu
2593	\|\| pVCpu->hwaccm.s.cTLBFlushes != pCpu->cTLBFlushes)
2594	{
2595	if (pVCpu->hwaccm.s.idLastCpu != pCpu->idCpu)
2596	LogFlow(("Force TLB flush due to rescheduling to a different cpu (%d vs %d)\n", pVCpu->hwaccm.s.idLastCpu, pCpu->idCpu));
2597	else
2598	LogFlow(("Force TLB flush due to changed TLB flush count (%x vs %x)\n", pVCpu->hwaccm.s.cTLBFlushes, pCpu->cTLBFlushes));
2599	}
2600	if (pCpu->fFlushTLB)
2601	LogFlow(("Force TLB flush: first time cpu %d is used -> flush\n", pCpu->idCpu));
2602	else
2603	if (pVCpu->hwaccm.s.fForceTLBFlush)
2604	LogFlow(("Manual TLB flush\n"));
2605	}
2606	#endif
2607	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
2608	PGMRZDynMapFlushAutoSet(pVCpu);
2609	#endif
2610
2611	/*
2612	* NOTE: DO NOT DO ANYTHING AFTER THIS POINT THAT MIGHT JUMP BACK TO RING 3!
2613	* (until the actual world switch)
2614	*/
2615	#ifdef VBOX_STRICT
2616	idCpuCheck = RTMpCpuId();
2617	#endif
2618	#ifdef LOG_ENABLED
2619	VMMR0LogFlushDisable(pVCpu);
2620	#endif
2621	/* Save the host state first. */
2622	if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_HOST_CONTEXT)
2623	{
2624	rc = VMXR0SaveHostState(pVM, pVCpu);
2625	if (RT_UNLIKELY(rc != VINF_SUCCESS))
2626	{
2627	VMMR0LogFlushEnable(pVCpu);
2628	goto end;
2629	}
2630	}
2631
2632	/* Load the guest state */
2633	if (!pVCpu->hwaccm.s.fContextUseFlags)
2634	{
2635	VMXR0LoadMinimalGuestState(pVM, pVCpu, pCtx);
2636	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatLoadMinimal);
2637	}
2638	else
2639	{
2640	rc = VMXR0LoadGuestState(pVM, pVCpu, pCtx);
2641	if (RT_UNLIKELY(rc != VINF_SUCCESS))
2642	{
2643	VMMR0LogFlushEnable(pVCpu);
2644	goto end;
2645	}
2646	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatLoadFull);
2647	}
2648
2649	#ifndef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
2650	/* Disable interrupts to make sure a poke will interrupt execution.
2651	* This must be done before we check for TLB flushes; TLB shootdowns rely on this.
2652	*/
2653	uOldEFlags = ASMIntDisableFlags();
2654	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
2655	#endif
2656
2657	/* Non-register state Guest Context */
2658	/** @todo change me according to cpu state */
2659	rc2 = VMXWriteVMCS(VMX_VMCS32_GUEST_ACTIVITY_STATE, VMX_CMS_GUEST_ACTIVITY_ACTIVE);
2660	AssertRC(rc2);
2661
2662	/** Set TLB flush state as checked until we return from the world switch. */
2663	ASMAtomicWriteU8(&pVCpu->hwaccm.s.fCheckedTLBFlush, true);
2664	/* Deal with tagged TLB setup and invalidation. */
2665	pVM->hwaccm.s.vmx.pfnSetupTaggedTLB(pVM, pVCpu);
2666
2667	/* Manual save and restore:
2668	* - General purpose registers except RIP, RSP
2669	*
2670	* Trashed:
2671	* - CR2 (we don't care)
2672	* - LDTR (reset to 0)
2673	* - DRx (presumably not changed at all)
2674	* - DR7 (reset to 0x400)
2675	* - EFLAGS (reset to RT_BIT(1); not relevant)
2676	*
2677	*/
2678
2679	/* All done! Let's start VM execution. */
2680	STAM_PROFILE_ADV_STOP_START(&pVCpu->hwaccm.s.StatEntry, &pVCpu->hwaccm.s.StatInGC, x);
2681	Assert(idCpuCheck == RTMpCpuId());
2682
2683	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
2684	pVCpu->hwaccm.s.vmx.VMCSCache.cResume = cResume;
2685	pVCpu->hwaccm.s.vmx.VMCSCache.u64TimeSwitch = RTTimeNanoTS();
2686	#endif
2687
2688	/* Save the current TPR value in the LSTAR msr so our patches can access it. */
2689	if (pVM->hwaccm.s.fTPRPatchingActive)
2690	{
2691	Assert(pVM->hwaccm.s.fTPRPatchingActive);
2692	u64OldLSTAR = ASMRdMsr(MSR_K8_LSTAR);
2693	ASMWrMsr(MSR_K8_LSTAR, u8LastTPR);
2694	}
2695
2696	TMNotifyStartOfExecution(pVCpu);
2697	#ifdef VBOX_WITH_KERNEL_USING_XMM
2698	rc = hwaccmR0VMXStartVMWrapXMM(pVCpu->hwaccm.s.fResumeVM, pCtx, &pVCpu->hwaccm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hwaccm.s.vmx.pfnStartVM);
2699	#else
2700	rc = pVCpu->hwaccm.s.vmx.pfnStartVM(pVCpu->hwaccm.s.fResumeVM, pCtx, &pVCpu->hwaccm.s.vmx.VMCSCache, pVM, pVCpu);
2701	#endif
2702	ASMAtomicWriteU8(&pVCpu->hwaccm.s.fCheckedTLBFlush, false);
2703	ASMAtomicIncU32(&pVCpu->hwaccm.s.cWorldSwitchExit);
2704	/* Possibly the last TSC value seen by the guest (too high) (only when we're in tsc offset mode). */
2705	if (!(pVCpu->hwaccm.s.vmx.proc_ctls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT))
2706	TMCpuTickSetLastSeen(pVCpu, ASMReadTSC() + pVCpu->hwaccm.s.vmx.u64TSCOffset - 0x400 /* guestimate of world switch overhead in clock ticks */);
2707
2708	TMNotifyEndOfExecution(pVCpu);
2709	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED);
2710	Assert(!(ASMGetFlags() & X86_EFL_IF));
2711
2712	/* Restore the host LSTAR msr if the guest could have changed it. */
2713	if (pVM->hwaccm.s.fTPRPatchingActive)
2714	{
2715	Assert(pVM->hwaccm.s.fTPRPatchingActive);
2716	pVCpu->hwaccm.s.vmx.pVAPIC[0x80] = pCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR);
2717	ASMWrMsr(MSR_K8_LSTAR, u64OldLSTAR);
2718	}
2719
2720	STAM_PROFILE_ADV_STOP_START(&pVCpu->hwaccm.s.StatInGC, &pVCpu->hwaccm.s.StatExit1, x);
2721	ASMSetFlags(uOldEFlags);
2722	#ifdef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
2723	uOldEFlags = ~(RTCCUINTREG)0;
2724	#endif
2725
2726	AssertMsg(!pVCpu->hwaccm.s.vmx.VMCSCache.Write.cValidEntries, ("pVCpu->hwaccm.s.vmx.VMCSCache.Write.cValidEntries=%d\n", pVCpu->hwaccm.s.vmx.VMCSCache.Write.cValidEntries));
2727
2728	/* In case we execute a goto ResumeExecution later on. */
2729	pVCpu->hwaccm.s.fResumeVM = true;
2730	pVCpu->hwaccm.s.fForceTLBFlush = false;
2731
2732	/*
2733	* !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
2734	* IMPORTANT: WE CAN'T DO ANY LOGGING OR OPERATIONS THAT CAN DO A LONGJMP BACK TO RING 3 BEFORE WE'VE SYNCED BACK (MOST OF) THE GUEST STATE
2735	* !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
2736	*/
2737
2738	if (RT_UNLIKELY(rc != VINF_SUCCESS))
2739	{
2740	VMXR0ReportWorldSwitchError(pVM, pVCpu, rc, pCtx);
2741	VMMR0LogFlushEnable(pVCpu);
2742	goto end;
2743	}
2744
2745	/* Success. Query the guest state and figure out what has happened. */
2746
2747	/* Investigate why there was a VM-exit. */
2748	rc2 = VMXReadCachedVMCS(VMX_VMCS32_RO_EXIT_REASON, &exitReason);
2749	STAM_COUNTER_INC(&pVCpu->hwaccm.s.paStatExitReasonR0[exitReason & MASK_EXITREASON_STAT]);
2750
2751	exitReason &= 0xffff; /* bit 0-15 contain the exit code. */
2752	rc2 \|= VMXReadCachedVMCS(VMX_VMCS32_RO_VM_INSTR_ERROR, &instrError);
2753	rc2 \|= VMXReadCachedVMCS(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &cbInstr);
2754	rc2 \|= VMXReadCachedVMCS(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &intInfo);
2755	/* might not be valid; depends on VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID. */
2756	rc2 \|= VMXReadCachedVMCS(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERRCODE, &errCode);
2757	rc2 \|= VMXReadCachedVMCS(VMX_VMCS32_RO_EXIT_INSTR_INFO, &instrInfo);
2758	rc2 \|= VMXReadCachedVMCS(VMX_VMCS_RO_EXIT_QUALIFICATION, &exitQualification);
2759	AssertRC(rc2);
2760
2761	/* Sync back the guest state */
2762	rc2 = VMXR0SaveGuestState(pVM, pVCpu, pCtx);
2763	AssertRC(rc2);
2764
2765	/* Note! NOW IT'S SAFE FOR LOGGING! */
2766	VMMR0LogFlushEnable(pVCpu);
2767	Log2(("Raw exit reason %08x\n", exitReason));
2768
2769	/* Check if an injected event was interrupted prematurely. */
2770	rc2 = VMXReadCachedVMCS(VMX_VMCS32_RO_IDT_INFO, &val);
2771	AssertRC(rc2);
2772	pVCpu->hwaccm.s.Event.intInfo = VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(val);
2773	if ( VMX_EXIT_INTERRUPTION_INFO_VALID(pVCpu->hwaccm.s.Event.intInfo)
2774	/* Ignore 'int xx' as they'll be restarted anyway. */
2775	&& VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hwaccm.s.Event.intInfo) != VMX_EXIT_INTERRUPTION_INFO_TYPE_SW
2776	/* Ignore software exceptions (such as int3) as they'll reoccur when we restart the instruction anyway. */
2777	&& VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hwaccm.s.Event.intInfo) != VMX_EXIT_INTERRUPTION_INFO_TYPE_SWEXCPT)
2778	{
2779	Assert(!pVCpu->hwaccm.s.Event.fPending);
2780	pVCpu->hwaccm.s.Event.fPending = true;
2781	/* Error code present? */
2782	if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(pVCpu->hwaccm.s.Event.intInfo))
2783	{
2784	rc2 = VMXReadCachedVMCS(VMX_VMCS32_RO_IDT_ERRCODE, &val);
2785	AssertRC(rc2);
2786	pVCpu->hwaccm.s.Event.errCode = val;
2787	Log(("Pending inject %RX64 at %RGv exit=%08x intInfo=%08x exitQualification=%RGv pending error=%RX64\n", pVCpu->hwaccm.s.Event.intInfo, (RTGCPTR)pCtx->rip, exitReason, intInfo, exitQualification, val));
2788	}
2789	else
2790	{
2791	Log(("Pending inject %RX64 at %RGv exit=%08x intInfo=%08x exitQualification=%RGv\n", pVCpu->hwaccm.s.Event.intInfo, (RTGCPTR)pCtx->rip, exitReason, intInfo, exitQualification));
2792	pVCpu->hwaccm.s.Event.errCode = 0;
2793	}
2794	}
2795	#ifdef VBOX_STRICT
2796	else
2797	if ( VMX_EXIT_INTERRUPTION_INFO_VALID(pVCpu->hwaccm.s.Event.intInfo)
2798	/* Ignore software exceptions (such as int3) as they're reoccur when we restart the instruction anyway. */
2799	&& VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hwaccm.s.Event.intInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_SWEXCPT)
2800	{
2801	Log(("Ignore pending inject %RX64 at %RGv exit=%08x intInfo=%08x exitQualification=%RGv\n", pVCpu->hwaccm.s.Event.intInfo, (RTGCPTR)pCtx->rip, exitReason, intInfo, exitQualification));
2802	}
2803
2804	if (exitReason == VMX_EXIT_ERR_INVALID_GUEST_STATE)
2805	HWACCMDumpRegs(pVM, pVCpu, pCtx);
2806	#endif
2807
2808	Log2(("E%d: New EIP=%x:%RGv\n", (uint32_t)exitReason, pCtx->cs, (RTGCPTR)pCtx->rip));
2809	Log2(("Exit reason %d, exitQualification %RGv\n", (uint32_t)exitReason, exitQualification));
2810	Log2(("instrInfo=%d instrError=%d instr length=%d\n", (uint32_t)instrInfo, (uint32_t)instrError, (uint32_t)cbInstr));
2811	Log2(("Interruption error code %d\n", (uint32_t)errCode));
2812	Log2(("IntInfo = %08x\n", (uint32_t)intInfo));
2813
2814	/* Sync back the TPR if it was changed. */
2815	if ( fSetupTPRCaching
2816	&& u8LastTPR != pVCpu->hwaccm.s.vmx.pVAPIC[0x80])
2817	{
2818	rc2 = PDMApicSetTPR(pVCpu, pVCpu->hwaccm.s.vmx.pVAPIC[0x80]);
2819	AssertRC(rc2);
2820	}
2821
2822	STAM_PROFILE_ADV_STOP_START(&pVCpu->hwaccm.s.StatExit1, &pVCpu->hwaccm.s.StatExit2, x);
2823
2824	/* Some cases don't need a complete resync of the guest CPU state; handle them here. */
2825	Assert(rc == VINF_SUCCESS); /* might consider VERR_IPE_UNINITIALIZED_STATUS here later... */
2826	switch (exitReason)
2827	{
2828	case VMX_EXIT_EXCEPTION: /* 0 Exception or non-maskable interrupt (NMI). */
2829	case VMX_EXIT_EXTERNAL_IRQ: /* 1 External interrupt. */
2830	{
2831	uint32_t vector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(intInfo);
2832
2833	if (!VMX_EXIT_INTERRUPTION_INFO_VALID(intInfo))
2834	{
2835	Assert(exitReason == VMX_EXIT_EXTERNAL_IRQ);
2836	#if 0 //def VBOX_WITH_VMMR0_DISABLE_PREEMPTION
2837	if ( RTThreadPreemptIsPendingTrusty()
2838	&& !RTThreadPreemptIsPending(NIL_RTTHREAD))
2839	goto ResumeExecution;
2840	#endif
2841	/* External interrupt; leave to allow it to be dispatched again. */
2842	rc = VINF_EM_RAW_INTERRUPT;
2843	break;
2844	}
2845	STAM_PROFILE_ADV_START(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
2846	switch (VMX_EXIT_INTERRUPTION_INFO_TYPE(intInfo))
2847	{
2848	case VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI: /* Non-maskable interrupt. */
2849	/* External interrupt; leave to allow it to be dispatched again. */
2850	rc = VINF_EM_RAW_INTERRUPT;
2851	break;
2852
2853	case VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT: /* External hardware interrupt. */
2854	AssertFailed(); /* can't come here; fails the first check. */
2855	break;
2856
2857	case VMX_EXIT_INTERRUPTION_INFO_TYPE_DBEXCPT: /* Unknown why we get this type for #DB */
2858	case VMX_EXIT_INTERRUPTION_INFO_TYPE_SWEXCPT: /* Software exception. (#BP or #OF) */
2859	Assert(vector == 1 \|\| vector == 3 \|\| vector == 4);
2860	/* no break */
2861	case VMX_EXIT_INTERRUPTION_INFO_TYPE_HWEXCPT: /* Hardware exception. */
2862	Log2(("Hardware/software interrupt %d\n", vector));
2863	switch (vector)
2864	{
2865	case X86_XCPT_NM:
2866	{
2867	Log(("#NM fault at %RGv error code %x\n", (RTGCPTR)pCtx->rip, errCode));
2868
2869	/** @todo don't intercept #NM exceptions anymore when we've activated the guest FPU state. */
2870	/* If we sync the FPU/XMM state on-demand, then we can continue execution as if nothing has happened. */
2871	rc = CPUMR0LoadGuestFPU(pVM, pVCpu, pCtx);
2872	if (rc == VINF_SUCCESS)
2873	{
2874	Assert(CPUMIsGuestFPUStateActive(pVCpu));
2875
2876	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitShadowNM);
2877
2878	/* Continue execution. */
2879	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_GUEST_CR0;
2880
2881	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
2882	goto ResumeExecution;
2883	}
2884
2885	Log(("Forward #NM fault to the guest\n"));
2886	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestNM);
2887	rc2 = VMXR0InjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), cbInstr, 0);
2888	AssertRC(rc2);
2889	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
2890	goto ResumeExecution;
2891	}
2892
2893	case X86_XCPT_PF: /* Page fault */
2894	{
2895	#ifdef DEBUG
2896	if (pVM->hwaccm.s.fNestedPaging)
2897	{ /* A genuine pagefault.
2898	* Forward the trap to the guest by injecting the exception and resuming execution.
2899	*/
2900	Log(("Guest page fault at %RGv cr2=%RGv error code %RGv rsp=%RGv\n", (RTGCPTR)pCtx->rip, exitQualification, errCode, (RTGCPTR)pCtx->rsp));
2901
2902	Assert(CPUMIsGuestInPagedProtectedModeEx(pCtx));
2903
2904	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestPF);
2905
2906	/* Now we must update CR2. */
2907	pCtx->cr2 = exitQualification;
2908	rc2 = VMXR0InjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), cbInstr, errCode);
2909	AssertRC(rc2);
2910
2911	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
2912	goto ResumeExecution;
2913	}
2914	#endif
2915	Assert(!pVM->hwaccm.s.fNestedPaging);
2916
2917	#ifdef VBOX_HWACCM_WITH_GUEST_PATCHING
2918	/* Shortcut for APIC TPR reads and writes; 32 bits guests only */
2919	if ( pVM->hwaccm.s.fTRPPatchingAllowed
2920	&& pVM->hwaccm.s.pGuestPatchMem
2921	&& (exitQualification & 0xfff) == 0x080
2922	&& !(errCode & X86_TRAP_PF_P) /* not present */
2923	&& CPUMGetGuestCPL(pVCpu, CPUMCTX2CORE(pCtx)) == 0
2924	&& !CPUMIsGuestInLongModeEx(pCtx)
2925	&& pVM->hwaccm.s.cPatches < RT_ELEMENTS(pVM->hwaccm.s.aPatches))
2926	{
2927	RTGCPHYS GCPhysApicBase, GCPhys;
2928	PDMApicGetBase(pVM, &GCPhysApicBase); /* @todo cache this */
2929	GCPhysApicBase &= PAGE_BASE_GC_MASK;
2930
2931	rc = PGMGstGetPage(pVCpu, (RTGCPTR)exitQualification, NULL, &GCPhys);
2932	if ( rc == VINF_SUCCESS
2933	&& GCPhys == GCPhysApicBase)
2934	{
2935	/* Only attempt to patch the instruction once. */
2936	PHWACCMTPRPATCH pPatch = (PHWACCMTPRPATCH)RTAvloU32Get(&pVM->hwaccm.s.PatchTree, (AVLOU32KEY)pCtx->eip);
2937	if (!pPatch)
2938	{
2939	rc = VINF_EM_HWACCM_PATCH_TPR_INSTR;
2940	break;
2941	}
2942	}
2943	}
2944	#endif
2945
2946	Log2(("Page fault at %RGv error code %x\n", exitQualification, errCode));
2947	/* Exit qualification contains the linear address of the page fault. */
2948	TRPMAssertTrap(pVCpu, X86_XCPT_PF, TRPM_TRAP);
2949	TRPMSetErrorCode(pVCpu, errCode);
2950	TRPMSetFaultAddress(pVCpu, exitQualification);
2951
2952	/* Shortcut for APIC TPR reads and writes. */
2953	if ( (exitQualification & 0xfff) == 0x080
2954	&& !(errCode & X86_TRAP_PF_P) /* not present */
2955	&& fSetupTPRCaching
2956	&& (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC))
2957	{
2958	RTGCPHYS GCPhysApicBase, GCPhys;
2959	PDMApicGetBase(pVM, &GCPhysApicBase); /* @todo cache this */
2960	GCPhysApicBase &= PAGE_BASE_GC_MASK;
2961
2962	rc = PGMGstGetPage(pVCpu, (RTGCPTR)exitQualification, NULL, &GCPhys);
2963	if ( rc == VINF_SUCCESS
2964	&& GCPhys == GCPhysApicBase)
2965	{
2966	Log(("Enable VT-x virtual APIC access filtering\n"));
2967	rc2 = IOMMMIOMapMMIOHCPage(pVM, GCPhysApicBase, pVM->hwaccm.s.vmx.pAPICPhys, X86_PTE_RW \| X86_PTE_P);
2968	AssertRC(rc2);
2969	}
2970	}
2971
2972	/* Forward it to our trap handler first, in case our shadow pages are out of sync. */
2973	rc = PGMTrap0eHandler(pVCpu, errCode, CPUMCTX2CORE(pCtx), (RTGCPTR)exitQualification);
2974	Log2(("PGMTrap0eHandler %RGv returned %Rrc\n", (RTGCPTR)pCtx->rip, VBOXSTRICTRC_VAL(rc)));
2975
2976	if (rc == VINF_SUCCESS)
2977	{ /* We've successfully synced our shadow pages, so let's just continue execution. */
2978	Log2(("Shadow page fault at %RGv cr2=%RGv error code %x\n", (RTGCPTR)pCtx->rip, exitQualification ,errCode));
2979	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitShadowPF);
2980
2981	TRPMResetTrap(pVCpu);
2982	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
2983	goto ResumeExecution;
2984	}
2985	else
2986	if (rc == VINF_EM_RAW_GUEST_TRAP)
2987	{ /* A genuine pagefault.
2988	* Forward the trap to the guest by injecting the exception and resuming execution.
2989	*/
2990	Log2(("Forward page fault to the guest\n"));
2991
2992	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestPF);
2993	/* The error code might have been changed. */
2994	errCode = TRPMGetErrorCode(pVCpu);
2995
2996	TRPMResetTrap(pVCpu);
2997
2998	/* Now we must update CR2. */
2999	pCtx->cr2 = exitQualification;
3000	rc2 = VMXR0InjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), cbInstr, errCode);
3001	AssertRC(rc2);
3002
3003	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3004	goto ResumeExecution;
3005	}
3006	#ifdef VBOX_STRICT
3007	if (rc != VINF_EM_RAW_EMULATE_INSTR && rc != VINF_EM_RAW_EMULATE_IO_BLOCK)
3008	Log2(("PGMTrap0eHandler failed with %d\n", VBOXSTRICTRC_VAL(rc)));
3009	#endif
3010	/* Need to go back to the recompiler to emulate the instruction. */
3011	TRPMResetTrap(pVCpu);
3012	break;
3013	}
3014
3015	case X86_XCPT_MF: /* Floating point exception. */
3016	{
3017	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestMF);
3018	if (!(pCtx->cr0 & X86_CR0_NE))
3019	{
3020	/* old style FPU error reporting needs some extra work. */
3021	/** @todo don't fall back to the recompiler, but do it manually. */
3022	rc = VINF_EM_RAW_EMULATE_INSTR;
3023	break;
3024	}
3025	Log(("Trap %x at %04X:%RGv\n", vector, pCtx->cs, (RTGCPTR)pCtx->rip));
3026	rc2 = VMXR0InjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), cbInstr, errCode);
3027	AssertRC(rc2);
3028
3029	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3030	goto ResumeExecution;
3031	}
3032
3033	case X86_XCPT_DB: /* Debug exception. */
3034	{
3035	uint64_t uDR6;
3036
3037	/* DR6, DR7.GD and IA32_DEBUGCTL.LBR are not updated yet.
3038	*
3039	* Exit qualification bits:
3040	* 3:0 B0-B3 which breakpoint condition was met
3041	* 12:4 Reserved (0)
3042	* 13 BD - debug register access detected
3043	* 14 BS - single step execution or branch taken
3044	* 63:15 Reserved (0)
3045	*/
3046	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestDB);
3047
3048	/* Note that we don't support guest and host-initiated debugging at the same time. */
3049
3050	uDR6 = X86_DR6_INIT_VAL;
3051	uDR6 \|= (exitQualification & (X86_DR6_B0\|X86_DR6_B1\|X86_DR6_B2\|X86_DR6_B3\|X86_DR6_BD\|X86_DR6_BS));
3052	rc = DBGFRZTrap01Handler(pVM, pVCpu, CPUMCTX2CORE(pCtx), uDR6);
3053	if (rc == VINF_EM_RAW_GUEST_TRAP)
3054	{
3055	/* Update DR6 here. */
3056	pCtx->dr[6] = uDR6;
3057
3058	/* Resync DR6 if the debug state is active. */
3059	if (CPUMIsGuestDebugStateActive(pVCpu))
3060	ASMSetDR6(pCtx->dr[6]);
3061
3062	/* X86_DR7_GD will be cleared if drx accesses should be trapped inside the guest. */
3063	pCtx->dr[7] &= ~X86_DR7_GD;
3064
3065	/* Paranoia. */
3066	pCtx->dr[7] &= 0xffffffff; /* upper 32 bits reserved */
3067	pCtx->dr[7] &= ~(RT_BIT(11) \| RT_BIT(12) \| RT_BIT(14) \| RT_BIT(15)); /* must be zero */
3068	pCtx->dr[7] \|= 0x400; /* must be one */
3069
3070	/* Resync DR7 */
3071	rc2 = VMXWriteVMCS64(VMX_VMCS64_GUEST_DR7, pCtx->dr[7]);
3072	AssertRC(rc2);
3073
3074	Log(("Trap %x (debug) at %RGv exit qualification %RX64 dr6=%x dr7=%x\n", vector, (RTGCPTR)pCtx->rip, exitQualification, (uint32_t)pCtx->dr[6], (uint32_t)pCtx->dr[7]));
3075	rc2 = VMXR0InjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), cbInstr, errCode);
3076	AssertRC(rc2);
3077
3078	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3079	goto ResumeExecution;
3080	}
3081	/* Return to ring 3 to deal with the debug exit code. */
3082	Log(("Debugger hardware BP at %04x:%RGv (rc=%Rrc)\n", pCtx->cs, pCtx->rip, VBOXSTRICTRC_VAL(rc)));
3083	break;
3084	}
3085
3086	case X86_XCPT_BP: /* Breakpoint. */
3087	{
3088	rc = DBGFRZTrap03Handler(pVM, pVCpu, CPUMCTX2CORE(pCtx));
3089	if (rc == VINF_EM_RAW_GUEST_TRAP)
3090	{
3091	Log(("Guest #BP at %04x:%RGv\n", pCtx->cs, pCtx->rip));
3092	rc2 = VMXR0InjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), cbInstr, errCode);
3093	AssertRC(rc2);
3094	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3095	goto ResumeExecution;
3096	}
3097	if (rc == VINF_SUCCESS)
3098	{
3099	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3100	goto ResumeExecution;
3101	}
3102	Log(("Debugger BP at %04x:%RGv (rc=%Rrc)\n", pCtx->cs, pCtx->rip, VBOXSTRICTRC_VAL(rc)));
3103	break;
3104	}
3105
3106	case X86_XCPT_GP: /* General protection failure exception.*/
3107	{
3108	uint32_t cbOp;
3109	uint32_t cbSize;
3110	PDISCPUSTATE pDis = &pVCpu->hwaccm.s.DisState;
3111
3112	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestGP);
3113	#ifdef VBOX_STRICT
3114	if ( !CPUMIsGuestInRealModeEx(pCtx)
3115	\|\| !pVM->hwaccm.s.vmx.pRealModeTSS)
3116	{
3117	Log(("Trap %x at %04X:%RGv errorCode=%RGv\n", vector, pCtx->cs, (RTGCPTR)pCtx->rip, errCode));
3118	rc2 = VMXR0InjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), cbInstr, errCode);
3119	AssertRC(rc2);
3120	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3121	goto ResumeExecution;
3122	}
3123	#endif
3124	Assert(CPUMIsGuestInRealModeEx(pCtx));
3125
3126	LogFlow(("Real mode X86_XCPT_GP instruction emulation at %x:%RGv\n", pCtx->cs, (RTGCPTR)pCtx->rip));
3127
3128	rc2 = EMInterpretDisasOne(pVM, pVCpu, CPUMCTX2CORE(pCtx), pDis, &cbOp);
3129	if (RT_SUCCESS(rc2))
3130	{
3131	bool fUpdateRIP = true;
3132
3133	rc = VINF_SUCCESS;
3134	Assert(cbOp == pDis->opsize);
3135	switch (pDis->pCurInstr->opcode)
3136	{
3137	case OP_CLI:
3138	pCtx->eflags.Bits.u1IF = 0;
3139	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitCli);
3140	break;
3141
3142	case OP_STI:
3143	pCtx->eflags.Bits.u1IF = 1;
3144	EMSetInhibitInterruptsPC(pVCpu, pCtx->rip + pDis->opsize);
3145	Assert(VMCPU_FF_ISSET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
3146	rc2 = VMXWriteVMCS(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
3147	AssertRC(rc2);
3148	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitSti);
3149	break;
3150
3151	case OP_HLT:
3152	fUpdateRIP = false;
3153	rc = VINF_EM_HALT;
3154	pCtx->rip += pDis->opsize;
3155	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitHlt);
3156	break;
3157
3158	case OP_POPF:
3159	{
3160	RTGCPTR GCPtrStack;
3161	uint32_t cbParm;
3162	uint32_t uMask;
3163	X86EFLAGS eflags;
3164
3165	if (pDis->prefix & PREFIX_OPSIZE)
3166	{
3167	cbParm = 4;
3168	uMask = 0xffffffff;
3169	}
3170	else
3171	{
3172	cbParm = 2;
3173	uMask = 0xffff;
3174	}
3175
3176	rc2 = SELMToFlatEx(pVM, DIS_SELREG_SS, CPUMCTX2CORE(pCtx), pCtx->esp & uMask, 0, &GCPtrStack);
3177	if (RT_FAILURE(rc2))
3178	{
3179	rc = VERR_EM_INTERPRETER;
3180	break;
3181	}
3182	eflags.u = 0;
3183	rc2 = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &eflags.u, cbParm);
3184	if (RT_FAILURE(rc2))
3185	{
3186	rc = VERR_EM_INTERPRETER;
3187	break;
3188	}
3189	LogFlow(("POPF %x -> %RGv mask=%x\n", eflags.u, pCtx->rsp, uMask));
3190	pCtx->eflags.u = (pCtx->eflags.u & ~(X86_EFL_POPF_BITS & uMask)) \| (eflags.u & X86_EFL_POPF_BITS & uMask);
3191	/* RF cleared when popped in real mode; see pushf description in AMD manual. */
3192	pCtx->eflags.Bits.u1RF = 0;
3193	pCtx->esp += cbParm;
3194	pCtx->esp &= uMask;
3195
3196	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitPopf);
3197	break;
3198	}
3199
3200	case OP_PUSHF:
3201	{
3202	RTGCPTR GCPtrStack;
3203	uint32_t cbParm;
3204	uint32_t uMask;
3205	X86EFLAGS eflags;
3206
3207	if (pDis->prefix & PREFIX_OPSIZE)
3208	{
3209	cbParm = 4;
3210	uMask = 0xffffffff;
3211	}
3212	else
3213	{
3214	cbParm = 2;
3215	uMask = 0xffff;
3216	}
3217
3218	rc2 = SELMToFlatEx(pVM, DIS_SELREG_SS, CPUMCTX2CORE(pCtx), (pCtx->esp - cbParm) & uMask, 0, &GCPtrStack);
3219	if (RT_FAILURE(rc2))
3220	{
3221	rc = VERR_EM_INTERPRETER;
3222	break;
3223	}
3224	eflags = pCtx->eflags;
3225	/* RF & VM cleared when pushed in real mode; see pushf description in AMD manual. */
3226	eflags.Bits.u1RF = 0;
3227	eflags.Bits.u1VM = 0;
3228
3229	rc2 = PGMPhysWrite(pVM, (RTGCPHYS)GCPtrStack, &eflags.u, cbParm);
3230	if (RT_FAILURE(rc2))
3231	{
3232	rc = VERR_EM_INTERPRETER;
3233	break;
3234	}
3235	LogFlow(("PUSHF %x -> %RGv\n", eflags.u, GCPtrStack));
3236	pCtx->esp -= cbParm;
3237	pCtx->esp &= uMask;
3238	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitPushf);
3239	break;
3240	}
3241
3242	case OP_IRET:
3243	{
3244	RTGCPTR GCPtrStack;
3245	uint32_t uMask = 0xffff;
3246	uint16_t aIretFrame[3];
3247
3248	if (pDis->prefix & (PREFIX_OPSIZE \| PREFIX_ADDRSIZE))
3249	{
3250	rc = VERR_EM_INTERPRETER;
3251	break;
3252	}
3253
3254	rc2 = SELMToFlatEx(pVM, DIS_SELREG_SS, CPUMCTX2CORE(pCtx), pCtx->esp & uMask, 0, &GCPtrStack);
3255	if (RT_FAILURE(rc2))
3256	{
3257	rc = VERR_EM_INTERPRETER;
3258	break;
3259	}
3260	rc2 = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &aIretFrame[0], sizeof(aIretFrame));
3261	if (RT_FAILURE(rc2))
3262	{
3263	rc = VERR_EM_INTERPRETER;
3264	break;
3265	}
3266	pCtx->ip = aIretFrame[0];
3267	pCtx->cs = aIretFrame[1];
3268	pCtx->csHid.u64Base = pCtx->cs << 4;
3269	pCtx->eflags.u = (pCtx->eflags.u & ~(X86_EFL_POPF_BITS & uMask)) \| (aIretFrame[2] & X86_EFL_POPF_BITS & uMask);
3270	pCtx->sp += sizeof(aIretFrame);
3271
3272	LogFlow(("iret to %04x:%x\n", pCtx->cs, pCtx->ip));
3273	fUpdateRIP = false;
3274	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitIret);
3275	break;
3276	}
3277
3278	case OP_INT:
3279	{
3280	uint32_t intInfo2;
3281
3282	LogFlow(("Realmode: INT %x\n", pDis->param1.parval & 0xff));
3283	intInfo2 = pDis->param1.parval & 0xff;
3284	intInfo2 \|= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
3285	intInfo2 \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
3286
3287	rc = VMXR0InjectEvent(pVM, pVCpu, pCtx, intInfo2, cbOp, 0);
3288	AssertRC(VBOXSTRICTRC_VAL(rc));
3289	fUpdateRIP = false;
3290	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitInt);
3291	break;
3292	}
3293
3294	case OP_INTO:
3295	{
3296	if (pCtx->eflags.Bits.u1OF)
3297	{
3298	uint32_t intInfo2;
3299
3300	LogFlow(("Realmode: INTO\n"));
3301	intInfo2 = X86_XCPT_OF;
3302	intInfo2 \|= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
3303	intInfo2 \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
3304
3305	rc = VMXR0InjectEvent(pVM, pVCpu, pCtx, intInfo2, cbOp, 0);
3306	AssertRC(VBOXSTRICTRC_VAL(rc));
3307	fUpdateRIP = false;
3308	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitInt);
3309	}
3310	break;
3311	}
3312
3313	case OP_INT3:
3314	{
3315	uint32_t intInfo2;
3316
3317	LogFlow(("Realmode: INT 3\n"));
3318	intInfo2 = 3;
3319	intInfo2 \|= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
3320	intInfo2 \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
3321
3322	rc = VMXR0InjectEvent(pVM, pVCpu, pCtx, intInfo2, cbOp, 0);
3323	AssertRC(VBOXSTRICTRC_VAL(rc));
3324	fUpdateRIP = false;
3325	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitInt);
3326	break;
3327	}
3328
3329	default:
3330	rc = EMInterpretInstructionCPU(pVM, pVCpu, pDis, CPUMCTX2CORE(pCtx), 0, EMCODETYPE_SUPERVISOR, &cbSize);
3331	break;
3332	}
3333
3334	if (rc == VINF_SUCCESS)
3335	{
3336	if (fUpdateRIP)
3337	pCtx->rip += cbOp; /* Move on to the next instruction. */
3338
3339	/* lidt, lgdt can end up here. In the future crx changes as well. Just reload the whole context to be done with it. */
3340	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_ALL;
3341
3342	/* Only resume if successful. */
3343	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3344	goto ResumeExecution;
3345	}
3346	}
3347	else
3348	rc = VERR_EM_INTERPRETER;
3349
3350	AssertMsg(rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_EM_HALT, ("Unexpected rc=%Rrc\n", VBOXSTRICTRC_VAL(rc)));
3351	break;
3352	}
3353
3354	#ifdef VBOX_STRICT
3355	case X86_XCPT_XF: /* SIMD exception. */
3356	case X86_XCPT_DE: /* Divide error. */
3357	case X86_XCPT_UD: /* Unknown opcode exception. */
3358	case X86_XCPT_SS: /* Stack segment exception. */
3359	case X86_XCPT_NP: /* Segment not present exception. */
3360	{
3361	switch(vector)
3362	{
3363	case X86_XCPT_DE:
3364	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestDE);
3365	break;
3366	case X86_XCPT_UD:
3367	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestUD);
3368	break;
3369	case X86_XCPT_SS:
3370	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestSS);
3371	break;
3372	case X86_XCPT_NP:
3373	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestNP);
3374	break;
3375	}
3376
3377	Log(("Trap %x at %04X:%RGv\n", vector, pCtx->cs, (RTGCPTR)pCtx->rip));
3378	rc2 = VMXR0InjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), cbInstr, errCode);
3379	AssertRC(rc2);
3380
3381	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3382	goto ResumeExecution;
3383	}
3384	#endif
3385	default:
3386	if ( CPUMIsGuestInRealModeEx(pCtx)
3387	&& pVM->hwaccm.s.vmx.pRealModeTSS)
3388	{
3389	Log(("Real Mode Trap %x at %04x:%04X error code %x\n", vector, pCtx->cs, pCtx->eip, errCode));
3390	rc = VMXR0InjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), cbInstr, errCode);
3391	AssertRC(VBOXSTRICTRC_VAL(rc)); /* Strict RC check below. */
3392
3393	/* Go back to ring 3 in case of a triple fault. */
3394	if ( vector == X86_XCPT_DF
3395	&& rc == VINF_EM_RESET)
3396	break;
3397
3398	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3399	goto ResumeExecution;
3400	}
3401	AssertMsgFailed(("Unexpected vm-exit caused by exception %x\n", vector));
3402	rc = VERR_VMX_UNEXPECTED_EXCEPTION;
3403	break;
3404	} /* switch (vector) */
3405
3406	break;
3407
3408	default:
3409	rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_CODE;
3410	AssertMsgFailed(("Unexpected interruption code %x\n", intInfo));
3411	break;
3412	}
3413
3414	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3415	break;
3416	}
3417
3418	case VMX_EXIT_EPT_VIOLATION: /* 48 EPT violation. An attempt to access memory with a guest-physical address was disallowed by the configuration of the EPT paging structures. */
3419	{
3420	RTGCPHYS GCPhys;
3421
3422	Assert(pVM->hwaccm.s.fNestedPaging);
3423
3424	rc2 = VMXReadVMCS64(VMX_VMCS_EXIT_PHYS_ADDR_FULL, &GCPhys);
3425	AssertRC(rc2);
3426	Assert(((exitQualification >> 7) & 3) != 2);
3427
3428	/* Determine the kind of violation. */
3429	errCode = 0;
3430	if (exitQualification & VMX_EXIT_QUALIFICATION_EPT_INSTR_FETCH)
3431	errCode \|= X86_TRAP_PF_ID;
3432
3433	if (exitQualification & VMX_EXIT_QUALIFICATION_EPT_DATA_WRITE)
3434	errCode \|= X86_TRAP_PF_RW;
3435
3436	/* If the page is present, then it's a page level protection fault. */
3437	if (exitQualification & VMX_EXIT_QUALIFICATION_EPT_ENTRY_PRESENT)
3438	{
3439	errCode \|= X86_TRAP_PF_P;
3440	}
3441	else
3442	{
3443	/* Shortcut for APIC TPR reads and writes. */
3444	if ( (GCPhys & 0xfff) == 0x080
3445	&& GCPhys > 0x1000000 /* to skip VGA frame buffer accesses */
3446	&& fSetupTPRCaching
3447	&& (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC))
3448	{
3449	RTGCPHYS GCPhysApicBase;
3450	PDMApicGetBase(pVM, &GCPhysApicBase); /* @todo cache this */
3451	GCPhysApicBase &= PAGE_BASE_GC_MASK;
3452	if (GCPhys == GCPhysApicBase + 0x80)
3453	{
3454	Log(("Enable VT-x virtual APIC access filtering\n"));
3455	rc2 = IOMMMIOMapMMIOHCPage(pVM, GCPhysApicBase, pVM->hwaccm.s.vmx.pAPICPhys, X86_PTE_RW \| X86_PTE_P);
3456	AssertRC(rc2);
3457	}
3458	}
3459	}
3460	Log(("EPT Page fault %x at %RGp error code %x\n", (uint32_t)exitQualification, GCPhys, errCode));
3461
3462	/* GCPhys contains the guest physical address of the page fault. */
3463	TRPMAssertTrap(pVCpu, X86_XCPT_PF, TRPM_TRAP);
3464	TRPMSetErrorCode(pVCpu, errCode);
3465	TRPMSetFaultAddress(pVCpu, GCPhys);
3466
3467	/* Handle the pagefault trap for the nested shadow table. */
3468	rc = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, errCode, CPUMCTX2CORE(pCtx), GCPhys);
3469	Log2(("PGMR0Trap0eHandlerNestedPaging %RGv returned %Rrc\n", (RTGCPTR)pCtx->rip, VBOXSTRICTRC_VAL(rc)));
3470	if (rc == VINF_SUCCESS)
3471	{ /* We've successfully synced our shadow pages, so let's just continue execution. */
3472	Log2(("Shadow page fault at %RGv cr2=%RGp error code %x\n", (RTGCPTR)pCtx->rip, exitQualification , errCode));
3473	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitReasonNPF);
3474
3475	TRPMResetTrap(pVCpu);
3476	goto ResumeExecution;
3477	}
3478
3479	#ifdef VBOX_STRICT
3480	if (rc != VINF_EM_RAW_EMULATE_INSTR)
3481	LogFlow(("PGMTrap0eHandlerNestedPaging failed with %d\n", VBOXSTRICTRC_VAL(rc)));
3482	#endif
3483	/* Need to go back to the recompiler to emulate the instruction. */
3484	TRPMResetTrap(pVCpu);
3485	break;
3486	}
3487
3488	case VMX_EXIT_EPT_MISCONFIG:
3489	{
3490	RTGCPHYS GCPhys;
3491
3492	Assert(pVM->hwaccm.s.fNestedPaging);
3493
3494	rc2 = VMXReadVMCS64(VMX_VMCS_EXIT_PHYS_ADDR_FULL, &GCPhys);
3495	AssertRC(rc2);
3496	Log(("VMX_EXIT_EPT_MISCONFIG for %RGp\n", GCPhys));
3497
3498	/* Shortcut for APIC TPR reads and writes. */
3499	if ( (GCPhys & 0xfff) == 0x080
3500	&& GCPhys > 0x1000000 /* to skip VGA frame buffer accesses */
3501	&& fSetupTPRCaching
3502	&& (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC))
3503	{
3504	RTGCPHYS GCPhysApicBase;
3505	PDMApicGetBase(pVM, &GCPhysApicBase); /* @todo cache this */
3506	GCPhysApicBase &= PAGE_BASE_GC_MASK;
3507	if (GCPhys == GCPhysApicBase + 0x80)
3508	{
3509	Log(("Enable VT-x virtual APIC access filtering\n"));
3510	rc2 = IOMMMIOMapMMIOHCPage(pVM, GCPhysApicBase, pVM->hwaccm.s.vmx.pAPICPhys, X86_PTE_RW \| X86_PTE_P);
3511	AssertRC(rc2);
3512	}
3513	}
3514
3515	rc = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pCtx), GCPhys, UINT32_MAX);
3516	if (rc == VINF_SUCCESS)
3517	{
3518	Log2(("PGMR0Trap0eHandlerNPMisconfig(,,,%RGp) at %RGv -> resume\n", GCPhys, (RTGCPTR)pCtx->rip));
3519	goto ResumeExecution;
3520	}
3521
3522	Log2(("PGMR0Trap0eHandlerNPMisconfig(,,,%RGp) at %RGv -> %Rrc\n", GCPhys, (RTGCPTR)pCtx->rip, VBOXSTRICTRC_VAL(rc)));
3523	break;
3524	}
3525
3526	case VMX_EXIT_IRQ_WINDOW: /* 7 Interrupt window. */
3527	/* Clear VM-exit on IF=1 change. */
3528	LogFlow(("VMX_EXIT_IRQ_WINDOW %RGv pending=%d IF=%d\n", (RTGCPTR)pCtx->rip, VMCPU_FF_ISPENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC\|VMCPU_FF_INTERRUPT_PIC)), pCtx->eflags.Bits.u1IF));
3529	pVCpu->hwaccm.s.vmx.proc_ctls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_IRQ_WINDOW_EXIT;
3530	rc2 = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
3531	AssertRC(rc2);
3532	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitIrqWindow);
3533	goto ResumeExecution; /* we check for pending guest interrupts there */
3534
3535	case VMX_EXIT_WBINVD: /* 54 Guest software attempted to execute WBINVD. (conditional) */
3536	case VMX_EXIT_INVD: /* 13 Guest software attempted to execute INVD. (unconditional) */
3537	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitInvd);
3538	/* Skip instruction and continue directly. */
3539	pCtx->rip += cbInstr;
3540	/* Continue execution.*/
3541	goto ResumeExecution;
3542
3543	case VMX_EXIT_CPUID: /* 10 Guest software attempted to execute CPUID. */
3544	{
3545	Log2(("VMX: Cpuid %x\n", pCtx->eax));
3546	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitCpuid);
3547	rc = EMInterpretCpuId(pVM, pVCpu, CPUMCTX2CORE(pCtx));
3548	if (rc == VINF_SUCCESS)
3549	{
3550	/* Update EIP and continue execution. */
3551	Assert(cbInstr == 2);
3552	pCtx->rip += cbInstr;
3553	goto ResumeExecution;
3554	}
3555	AssertMsgFailed(("EMU: cpuid failed with %Rrc\n", VBOXSTRICTRC_VAL(rc)));
3556	rc = VINF_EM_RAW_EMULATE_INSTR;
3557	break;
3558	}
3559
3560	case VMX_EXIT_RDPMC: /* 15 Guest software attempted to execute RDPMC. */
3561	{
3562	Log2(("VMX: Rdpmc %x\n", pCtx->ecx));
3563	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitRdpmc);
3564	rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pCtx));
3565	if (rc == VINF_SUCCESS)
3566	{
3567	/* Update EIP and continue execution. */
3568	Assert(cbInstr == 2);
3569	pCtx->rip += cbInstr;
3570	goto ResumeExecution;
3571	}
3572	rc = VINF_EM_RAW_EMULATE_INSTR;
3573	break;
3574	}
3575
3576	case VMX_EXIT_RDTSC: /* 16 Guest software attempted to execute RDTSC. */
3577	{
3578	Log2(("VMX: Rdtsc\n"));
3579	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitRdtsc);
3580	rc = EMInterpretRdtsc(pVM, pVCpu, CPUMCTX2CORE(pCtx));
3581	if (rc == VINF_SUCCESS)
3582	{
3583	/* Update EIP and continue execution. */
3584	Assert(cbInstr == 2);
3585	pCtx->rip += cbInstr;
3586	goto ResumeExecution;
3587	}
3588	rc = VINF_EM_RAW_EMULATE_INSTR;
3589	break;
3590	}
3591
3592	case VMX_EXIT_INVPG: /* 14 Guest software attempted to execute INVPG. */
3593	{
3594	Log2(("VMX: invlpg\n"));
3595	Assert(!pVM->hwaccm.s.fNestedPaging);
3596
3597	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitInvpg);
3598	rc = EMInterpretInvlpg(pVM, pVCpu, CPUMCTX2CORE(pCtx), exitQualification);
3599	if (rc == VINF_SUCCESS)
3600	{
3601	/* Update EIP and continue execution. */
3602	pCtx->rip += cbInstr;
3603	goto ResumeExecution;
3604	}
3605	AssertMsg(rc == VERR_EM_INTERPRETER, ("EMU: invlpg %RGv failed with %Rrc\n", exitQualification, VBOXSTRICTRC_VAL(rc)));
3606	break;
3607	}
3608
3609	case VMX_EXIT_MONITOR: /* 39 Guest software attempted to execute MONITOR. */
3610	{
3611	Log2(("VMX: monitor\n"));
3612
3613	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitMonitor);
3614	rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pCtx));
3615	if (rc == VINF_SUCCESS)
3616	{
3617	/* Update EIP and continue execution. */
3618	pCtx->rip += cbInstr;
3619	goto ResumeExecution;
3620	}
3621	AssertMsg(rc == VERR_EM_INTERPRETER, ("EMU: monitor failed with %Rrc\n", VBOXSTRICTRC_VAL(rc)));
3622	break;
3623	}
3624
3625	case VMX_EXIT_WRMSR: /* 32 WRMSR. Guest software attempted to execute WRMSR. */
3626	/* When an interrupt is pending, we'll let MSR_K8_LSTAR writes fault in our TPR patch code. */
3627	if ( pVM->hwaccm.s.fTPRPatchingActive
3628	&& pCtx->ecx == MSR_K8_LSTAR)
3629	{
3630	Assert(!CPUMIsGuestInLongModeEx(pCtx));
3631	if ((pCtx->eax & 0xff) != u8LastTPR)
3632	{
3633	Log(("VMX: Faulting MSR_K8_LSTAR write with new TPR value %x\n", pCtx->eax & 0xff));
3634
3635	/* Our patch code uses LSTAR for TPR caching. */
3636	rc2 = PDMApicSetTPR(pVCpu, pCtx->eax & 0xff);
3637	AssertRC(rc2);
3638	}
3639
3640	/* Skip the instruction and continue. */
3641	pCtx->rip += cbInstr; /* wrmsr = [0F 30] */
3642
3643	/* Only resume if successful. */
3644	goto ResumeExecution;
3645	}
3646	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_GUEST_MSR;
3647	/* no break */
3648	case VMX_EXIT_RDMSR: /* 31 RDMSR. Guest software attempted to execute RDMSR. */
3649	{
3650	uint32_t cbSize;
3651
3652	STAM_COUNTER_INC((exitReason == VMX_EXIT_RDMSR) ? &pVCpu->hwaccm.s.StatExitRdmsr : &pVCpu->hwaccm.s.StatExitWrmsr);
3653
3654	/* Note: the intel manual claims there's a REX version of RDMSR that's slightly different, so we play safe by completely disassembling the instruction. */
3655	Log2(("VMX: %s\n", (exitReason == VMX_EXIT_RDMSR) ? "rdmsr" : "wrmsr"));
3656	rc = EMInterpretInstruction(pVM, pVCpu, CPUMCTX2CORE(pCtx), 0, &cbSize);
3657	if (rc == VINF_SUCCESS)
3658	{
3659	/* EIP has been updated already. */
3660
3661	/* Only resume if successful. */
3662	goto ResumeExecution;
3663	}
3664	AssertMsg(rc == VERR_EM_INTERPRETER, ("EMU: %s failed with %Rrc\n", (exitReason == VMX_EXIT_RDMSR) ? "rdmsr" : "wrmsr", VBOXSTRICTRC_VAL(rc)));
3665	break;
3666	}
3667
3668	case VMX_EXIT_CRX_MOVE: /* 28 Control-register accesses. */
3669	{
3670	STAM_PROFILE_ADV_START(&pVCpu->hwaccm.s.StatExit2Sub2, y2);
3671
3672	switch (VMX_EXIT_QUALIFICATION_CRX_ACCESS(exitQualification))
3673	{
3674	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_WRITE:
3675	Log2(("VMX: %RGv mov cr%d, x\n", (RTGCPTR)pCtx->rip, VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification)));
3676	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitCRxWrite[VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification)]);
3677	rc = EMInterpretCRxWrite(pVM, pVCpu, CPUMCTX2CORE(pCtx),
3678	VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification),
3679	VMX_EXIT_QUALIFICATION_CRX_GENREG(exitQualification));
3680
3681	switch (VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification))
3682	{
3683	case 0:
3684	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_GUEST_CR0 \| HWACCM_CHANGED_GUEST_CR3;
3685	break;
3686	case 2:
3687	break;
3688	case 3:
3689	Assert(!pVM->hwaccm.s.fNestedPaging \|\| !CPUMIsGuestInPagedProtectedModeEx(pCtx));
3690	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_GUEST_CR3;
3691	break;
3692	case 4:
3693	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_GUEST_CR4;
3694	break;
3695	case 8:
3696	/* CR8 contains the APIC TPR */
3697	Assert(!(pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW));
3698	break;
3699
3700	default:
3701	AssertFailed();
3702	break;
3703	}
3704	break;
3705
3706	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ:
3707	Log2(("VMX: mov x, crx\n"));
3708	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitCRxRead[VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification)]);
3709
3710	Assert(!pVM->hwaccm.s.fNestedPaging \|\| !CPUMIsGuestInPagedProtectedModeEx(pCtx) \|\| VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification) != USE_REG_CR3);
3711
3712	/* CR8 reads only cause an exit when the TPR shadow feature isn't present. */
3713	Assert(VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification) != 8 \|\| !(pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW));
3714
3715	rc = EMInterpretCRxRead(pVM, pVCpu, CPUMCTX2CORE(pCtx),
3716	VMX_EXIT_QUALIFICATION_CRX_GENREG(exitQualification),
3717	VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification));
3718	break;
3719
3720	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_CLTS:
3721	Log2(("VMX: clts\n"));
3722	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitCLTS);
3723	rc = EMInterpretCLTS(pVM, pVCpu);
3724	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_GUEST_CR0;
3725	break;
3726
3727	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_LMSW:
3728	Log2(("VMX: lmsw %x\n", VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(exitQualification)));
3729	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitLMSW);
3730	rc = EMInterpretLMSW(pVM, pVCpu, CPUMCTX2CORE(pCtx), VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(exitQualification));
3731	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_GUEST_CR0;
3732	break;
3733	}
3734
3735	/* Update EIP if no error occurred. */
3736	if (RT_SUCCESS(rc))
3737	pCtx->rip += cbInstr;
3738
3739	if (rc == VINF_SUCCESS)
3740	{
3741	/* Only resume if successful. */
3742	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub2, y2);
3743	goto ResumeExecution;
3744	}
3745	Assert(rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_PGM_SYNC_CR3);
3746	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub2, y2);
3747	break;
3748	}
3749
3750	case VMX_EXIT_DRX_MOVE: /* 29 Debug-register accesses. */
3751	{
3752	if ( !DBGFIsStepping(pVCpu)
3753	&& !CPUMIsHyperDebugStateActive(pVCpu))
3754	{
3755	/* Disable drx move intercepts. */
3756	pVCpu->hwaccm.s.vmx.proc_ctls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT;
3757	rc2 = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
3758	AssertRC(rc2);
3759
3760	/* Save the host and load the guest debug state. */
3761	rc2 = CPUMR0LoadGuestDebugState(pVM, pVCpu, pCtx, true /* include DR6 */);
3762	AssertRC(rc2);
3763
3764	#ifdef LOG_ENABLED
3765	if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(exitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
3766	Log(("VMX_EXIT_DRX_MOVE: write DR%d genreg %d\n", VMX_EXIT_QUALIFICATION_DRX_REGISTER(exitQualification), VMX_EXIT_QUALIFICATION_DRX_GENREG(exitQualification)));
3767	else
3768	Log(("VMX_EXIT_DRX_MOVE: read DR%d\n", VMX_EXIT_QUALIFICATION_DRX_REGISTER(exitQualification)));
3769	#endif
3770
3771	#ifdef VBOX_WITH_STATISTICS
3772	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatDRxContextSwitch);
3773	if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(exitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
3774	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitDRxWrite);
3775	else
3776	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitDRxRead);
3777	#endif
3778
3779	goto ResumeExecution;
3780	}
3781
3782	/** @todo clear VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT after the first time and restore drx registers afterwards */
3783	if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(exitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
3784	{
3785	Log2(("VMX: mov drx%d, genreg%d\n", VMX_EXIT_QUALIFICATION_DRX_REGISTER(exitQualification), VMX_EXIT_QUALIFICATION_DRX_GENREG(exitQualification)));
3786	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitDRxWrite);
3787	rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pCtx),
3788	VMX_EXIT_QUALIFICATION_DRX_REGISTER(exitQualification),
3789	VMX_EXIT_QUALIFICATION_DRX_GENREG(exitQualification));
3790	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_GUEST_DEBUG;
3791	Log2(("DR7=%08x\n", pCtx->dr[7]));
3792	}
3793	else
3794	{
3795	Log2(("VMX: mov x, drx\n"));
3796	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitDRxRead);
3797	rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pCtx),
3798	VMX_EXIT_QUALIFICATION_DRX_GENREG(exitQualification),
3799	VMX_EXIT_QUALIFICATION_DRX_REGISTER(exitQualification));
3800	}
3801	/* Update EIP if no error occurred. */
3802	if (RT_SUCCESS(rc))
3803	pCtx->rip += cbInstr;
3804
3805	if (rc == VINF_SUCCESS)
3806	{
3807	/* Only resume if successful. */
3808	goto ResumeExecution;
3809	}
3810	Assert(rc == VERR_EM_INTERPRETER);
3811	break;
3812	}
3813
3814	/* Note: We'll get a #GP if the IO instruction isn't allowed (IOPL or TSS bitmap); no need to double check. */
3815	case VMX_EXIT_PORT_IO: /* 30 I/O instruction. */
3816	{
3817	STAM_PROFILE_ADV_START(&pVCpu->hwaccm.s.StatExit2Sub1, y1);
3818	uint32_t uIOWidth = VMX_EXIT_QUALIFICATION_IO_WIDTH(exitQualification);
3819	uint32_t uPort;
3820	bool fIOWrite = (VMX_EXIT_QUALIFICATION_IO_DIRECTION(exitQualification) == VMX_EXIT_QUALIFICATION_IO_DIRECTION_OUT);
3821
3822	/** @todo necessary to make the distinction? */
3823	if (VMX_EXIT_QUALIFICATION_IO_ENCODING(exitQualification) == VMX_EXIT_QUALIFICATION_IO_ENCODING_DX)
3824	{
3825	uPort = pCtx->edx & 0xffff;
3826	}
3827	else
3828	uPort = VMX_EXIT_QUALIFICATION_IO_PORT(exitQualification); /* Immediate encoding. */
3829
3830	/* paranoia */
3831	if (RT_UNLIKELY(uIOWidth == 2 \|\| uIOWidth >= 4))
3832	{
3833	rc = fIOWrite ? VINF_IOM_HC_IOPORT_WRITE : VINF_IOM_HC_IOPORT_READ;
3834	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub1, y1);
3835	break;
3836	}
3837
3838	uint32_t cbSize = g_aIOSize[uIOWidth];
3839
3840	if (VMX_EXIT_QUALIFICATION_IO_STRING(exitQualification))
3841	{
3842	/* ins/outs */
3843	PDISCPUSTATE pDis = &pVCpu->hwaccm.s.DisState;
3844
3845	/* Disassemble manually to deal with segment prefixes. */
3846	/** @todo VMX_VMCS_EXIT_GUEST_LINEAR_ADDR contains the flat pointer operand of the instruction. */
3847	/** @todo VMX_VMCS32_RO_EXIT_INSTR_INFO also contains segment prefix info. */
3848	rc2 = EMInterpretDisasOne(pVM, pVCpu, CPUMCTX2CORE(pCtx), pDis, NULL);
3849	if (RT_SUCCESS(rc))
3850	{
3851	if (fIOWrite)
3852	{
3853	Log2(("IOMInterpretOUTSEx %RGv %x size=%d\n", (RTGCPTR)pCtx->rip, uPort, cbSize));
3854	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitIOStringWrite);
3855	rc = IOMInterpretOUTSEx(pVM, CPUMCTX2CORE(pCtx), uPort, pDis->prefix, cbSize);
3856	}
3857	else
3858	{
3859	Log2(("IOMInterpretINSEx %RGv %x size=%d\n", (RTGCPTR)pCtx->rip, uPort, cbSize));
3860	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitIOStringRead);
3861	rc = IOMInterpretINSEx(pVM, CPUMCTX2CORE(pCtx), uPort, pDis->prefix, cbSize);
3862	}
3863	}
3864	else
3865	rc = VINF_EM_RAW_EMULATE_INSTR;
3866	}
3867	else
3868	{
3869	/* normal in/out */
3870	uint32_t uAndVal = g_aIOOpAnd[uIOWidth];
3871
3872	Assert(!VMX_EXIT_QUALIFICATION_IO_REP(exitQualification));
3873
3874	if (fIOWrite)
3875	{
3876	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitIOWrite);
3877	rc = IOMIOPortWrite(pVM, uPort, pCtx->eax & uAndVal, cbSize);
3878	if (rc == VINF_IOM_HC_IOPORT_WRITE)
3879	HWACCMR0SavePendingIOPortWrite(pVCpu, pCtx->rip, pCtx->rip + cbInstr, uPort, uAndVal, cbSize);
3880	}
3881	else
3882	{
3883	uint32_t u32Val = 0;
3884
3885	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitIORead);
3886	rc = IOMIOPortRead(pVM, uPort, &u32Val, cbSize);
3887	if (IOM_SUCCESS(rc))
3888	{
3889	/* Write back to the EAX register. */
3890	pCtx->eax = (pCtx->eax & ~uAndVal) \| (u32Val & uAndVal);
3891	}
3892	else
3893	if (rc == VINF_IOM_HC_IOPORT_READ)
3894	HWACCMR0SavePendingIOPortRead(pVCpu, pCtx->rip, pCtx->rip + cbInstr, uPort, uAndVal, cbSize);
3895	}
3896	}
3897	/*
3898	* Handled the I/O return codes.
3899	* (The unhandled cases end up with rc == VINF_EM_RAW_EMULATE_INSTR.)
3900	*/
3901	if (IOM_SUCCESS(rc))
3902	{
3903	/* Update EIP and continue execution. */
3904	pCtx->rip += cbInstr;
3905	if (RT_LIKELY(rc == VINF_SUCCESS))
3906	{
3907	/* If any IO breakpoints are armed, then we should check if a debug trap needs to be generated. */
3908	if (pCtx->dr[7] & X86_DR7_ENABLED_MASK)
3909	{
3910	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatDRxIOCheck);
3911	for (unsigned i=0;i<4;i++)
3912	{
3913	unsigned uBPLen = g_aIOSize[X86_DR7_GET_LEN(pCtx->dr[7], i)];
3914
3915	if ( (uPort >= pCtx->dr[i] && uPort < pCtx->dr[i] + uBPLen)
3916	&& (pCtx->dr[7] & (X86_DR7_L(i) \| X86_DR7_G(i)))
3917	&& (pCtx->dr[7] & X86_DR7_RW(i, X86_DR7_RW_IO)) == X86_DR7_RW(i, X86_DR7_RW_IO))
3918	{
3919	uint64_t uDR6;
3920
3921	Assert(CPUMIsGuestDebugStateActive(pVCpu));
3922
3923	uDR6 = ASMGetDR6();
3924
3925	/* Clear all breakpoint status flags and set the one we just hit. */
3926	uDR6 &= ~(X86_DR6_B0\|X86_DR6_B1\|X86_DR6_B2\|X86_DR6_B3);
3927	uDR6 \|= (uint64_t)RT_BIT(i);
3928
3929	/* Note: AMD64 Architecture Programmer's Manual 13.1:
3930	* Bits 15:13 of the DR6 register is never cleared by the processor and must be cleared by software after
3931	* the contents have been read.
3932	*/
3933	ASMSetDR6(uDR6);
3934
3935	/* X86_DR7_GD will be cleared if drx accesses should be trapped inside the guest. */
3936	pCtx->dr[7] &= ~X86_DR7_GD;
3937
3938	/* Paranoia. */
3939	pCtx->dr[7] &= 0xffffffff; /* upper 32 bits reserved */
3940	pCtx->dr[7] &= ~(RT_BIT(11) \| RT_BIT(12) \| RT_BIT(14) \| RT_BIT(15)); /* must be zero */
3941	pCtx->dr[7] \|= 0x400; /* must be one */
3942
3943	/* Resync DR7 */
3944	rc2 = VMXWriteVMCS64(VMX_VMCS64_GUEST_DR7, pCtx->dr[7]);
3945	AssertRC(rc2);
3946
3947	/* Construct inject info. */
3948	intInfo = X86_XCPT_DB;
3949	intInfo \|= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
3950	intInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HWEXCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
3951
3952	Log(("Inject IO debug trap at %RGv\n", (RTGCPTR)pCtx->rip));
3953	rc2 = VMXR0InjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), 0, 0);
3954	AssertRC(rc2);
3955
3956	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub1, y1);
3957	goto ResumeExecution;
3958	}
3959	}
3960	}
3961	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub1, y1);
3962	goto ResumeExecution;
3963	}
3964	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub1, y1);
3965	break;
3966	}
3967
3968	#ifdef VBOX_STRICT
3969	if (rc == VINF_IOM_HC_IOPORT_READ)
3970	Assert(!fIOWrite);
3971	else if (rc == VINF_IOM_HC_IOPORT_WRITE)
3972	Assert(fIOWrite);
3973	else
3974	AssertMsg(RT_FAILURE(rc) \|\| rc == VINF_EM_RAW_EMULATE_INSTR \|\| rc == VINF_EM_RAW_GUEST_TRAP \|\| rc == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", VBOXSTRICTRC_VAL(rc)));
3975	#endif
3976	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub1, y1);
3977	break;
3978	}
3979
3980	case VMX_EXIT_TPR: /* 43 TPR below threshold. Guest software executed MOV to CR8. */
3981	LogFlow(("VMX_EXIT_TPR\n"));
3982	/* RIP is already set to the next instruction and the TPR has been synced back. Just resume. */
3983	goto ResumeExecution;
3984
3985	case VMX_EXIT_APIC_ACCESS: /* 44 APIC access. Guest software attempted to access memory at a physical address on the APIC-access page. */
3986	{
3987	LogFlow(("VMX_EXIT_APIC_ACCESS\n"));
3988	unsigned uAccessType = VMX_EXIT_QUALIFICATION_APIC_ACCESS_TYPE(exitQualification);
3989
3990	switch(uAccessType)
3991	{
3992	case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
3993	case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
3994	{
3995	RTGCPHYS GCPhys;
3996	PDMApicGetBase(pVM, &GCPhys);
3997	GCPhys &= PAGE_BASE_GC_MASK;
3998	GCPhys += VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(exitQualification);
3999
4000	LogFlow(("Apic access at %RGp\n", GCPhys));
4001	rc = IOMMMIOPhysHandler(pVM, (uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ) ? 0 : X86_TRAP_PF_RW, CPUMCTX2CORE(pCtx), GCPhys);
4002	if (rc == VINF_SUCCESS)
4003	goto ResumeExecution; /* rip already updated */
4004	break;
4005	}
4006
4007	default:
4008	rc = VINF_EM_RAW_EMULATE_INSTR;
4009	break;
4010	}
4011	break;
4012	}
4013
4014	case VMX_EXIT_PREEMPTION_TIMER: /* 52 VMX-preemption timer expired. The preemption timer counted down to zero. */
4015	if (!TMTimerPollBool(pVM, pVCpu))
4016	goto ResumeExecution;
4017	rc = VINF_EM_RAW_TIMER_PENDING;
4018	break;
4019
4020	default:
4021	/* The rest is handled after syncing the entire CPU state. */
4022	break;
4023	}
4024
4025	/* Note: the guest state isn't entirely synced back at this stage. */
4026
4027	/* Investigate why there was a VM-exit. (part 2) */
4028	switch (exitReason)
4029	{
4030	case VMX_EXIT_EXCEPTION: /* 0 Exception or non-maskable interrupt (NMI). */
4031	case VMX_EXIT_EXTERNAL_IRQ: /* 1 External interrupt. */
4032	case VMX_EXIT_EPT_VIOLATION:
4033	case VMX_EXIT_EPT_MISCONFIG: /* 49 EPT misconfig is used by the PGM/MMIO optimizations. */
4034	case VMX_EXIT_PREEMPTION_TIMER: /* 52 VMX-preemption timer expired. The preemption timer counted down to zero. */
4035	/* Already handled above. */
4036	break;
4037
4038	case VMX_EXIT_TRIPLE_FAULT: /* 2 Triple fault. */
4039	rc = VINF_EM_RESET; /* Triple fault equals a reset. */
4040	break;
4041
4042	case VMX_EXIT_INIT_SIGNAL: /* 3 INIT signal. */
4043	case VMX_EXIT_SIPI: /* 4 Start-up IPI (SIPI). */
4044	rc = VINF_EM_RAW_INTERRUPT;
4045	AssertFailed(); /* Can't happen. Yet. */
4046	break;
4047
4048	case VMX_EXIT_IO_SMI_IRQ: /* 5 I/O system-management interrupt (SMI). */
4049	case VMX_EXIT_SMI_IRQ: /* 6 Other SMI. */
4050	rc = VINF_EM_RAW_INTERRUPT;
4051	AssertFailed(); /* Can't happen afaik. */
4052	break;
4053
4054	case VMX_EXIT_TASK_SWITCH: /* 9 Task switch: too complicated to emulate, so fall back to the recompiler */
4055	Log(("VMX_EXIT_TASK_SWITCH: exit=%RX64\n", exitQualification));
4056	if ( (VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE(exitQualification) == VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE_IDT)
4057	&& pVCpu->hwaccm.s.Event.fPending)
4058	{
4059	/* Caused by an injected interrupt. */
4060	pVCpu->hwaccm.s.Event.fPending = false;
4061
4062	Log(("VMX_EXIT_TASK_SWITCH: reassert trap %d\n", VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVCpu->hwaccm.s.Event.intInfo)));
4063	Assert(!VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(pVCpu->hwaccm.s.Event.intInfo));
4064	rc2 = TRPMAssertTrap(pVCpu, VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVCpu->hwaccm.s.Event.intInfo), TRPM_HARDWARE_INT);
4065	AssertRC(rc2);
4066	}
4067	/* else Exceptions and software interrupts can just be restarted. */
4068	rc = VERR_EM_INTERPRETER;
4069	break;
4070
4071	case VMX_EXIT_HLT: /* 12 Guest software attempted to execute HLT. */
4072	/** Check if external interrupts are pending; if so, don't switch back. */
4073	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitHlt);
4074	pCtx->rip++; /* skip hlt */
4075	if (EMShouldContinueAfterHalt(pVCpu, pCtx))
4076	goto ResumeExecution;
4077
4078	rc = VINF_EM_HALT;
4079	break;
4080
4081	case VMX_EXIT_MWAIT: /* 36 Guest software executed MWAIT. */
4082	Log2(("VMX: mwait\n"));
4083	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitMwait);
4084	rc = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pCtx));
4085	if ( rc == VINF_EM_HALT
4086	\|\| rc == VINF_SUCCESS)
4087	{
4088	/* Update EIP and continue execution. */
4089	pCtx->rip += cbInstr;
4090
4091	/** Check if external interrupts are pending; if so, don't switch back. */
4092	if ( rc == VINF_SUCCESS
4093	\|\| ( rc == VINF_EM_HALT
4094	&& EMShouldContinueAfterHalt(pVCpu, pCtx))
4095	)
4096	goto ResumeExecution;
4097	}
4098	AssertMsg(rc == VERR_EM_INTERPRETER \|\| rc == VINF_EM_HALT, ("EMU: mwait failed with %Rrc\n", VBOXSTRICTRC_VAL(rc)));
4099	break;
4100
4101	case VMX_EXIT_RSM: /* 17 Guest software attempted to execute RSM in SMM. */
4102	AssertFailed(); /* can't happen. */
4103	rc = VERR_EM_INTERPRETER;
4104	break;
4105
4106	case VMX_EXIT_VMCALL: /* 18 Guest software executed VMCALL. */
4107	case VMX_EXIT_VMCLEAR: /* 19 Guest software executed VMCLEAR. */
4108	case VMX_EXIT_VMLAUNCH: /* 20 Guest software executed VMLAUNCH. */
4109	case VMX_EXIT_VMPTRLD: /* 21 Guest software executed VMPTRLD. */
4110	case VMX_EXIT_VMPTRST: /* 22 Guest software executed VMPTRST. */
4111	case VMX_EXIT_VMREAD: /* 23 Guest software executed VMREAD. */
4112	case VMX_EXIT_VMRESUME: /* 24 Guest software executed VMRESUME. */
4113	case VMX_EXIT_VMWRITE: /* 25 Guest software executed VMWRITE. */
4114	case VMX_EXIT_VMXOFF: /* 26 Guest software executed VMXOFF. */
4115	case VMX_EXIT_VMXON: /* 27 Guest software executed VMXON. */
4116	/** @todo inject #UD immediately */
4117	rc = VERR_EM_INTERPRETER;
4118	break;
4119
4120	case VMX_EXIT_CPUID: /* 10 Guest software attempted to execute CPUID. */
4121	case VMX_EXIT_RDTSC: /* 16 Guest software attempted to execute RDTSC. */
4122	case VMX_EXIT_INVPG: /* 14 Guest software attempted to execute INVPG. */
4123	case VMX_EXIT_CRX_MOVE: /* 28 Control-register accesses. */
4124	case VMX_EXIT_DRX_MOVE: /* 29 Debug-register accesses. */
4125	case VMX_EXIT_PORT_IO: /* 30 I/O instruction. */
4126	case VMX_EXIT_RDPMC: /* 15 Guest software attempted to execute RDPMC. */
4127	/* already handled above */
4128	AssertMsg( rc == VINF_PGM_CHANGE_MODE
4129	\|\| rc == VINF_EM_RAW_INTERRUPT
4130	\|\| rc == VERR_EM_INTERPRETER
4131	\|\| rc == VINF_EM_RAW_EMULATE_INSTR
4132	\|\| rc == VINF_PGM_SYNC_CR3
4133	\|\| rc == VINF_IOM_HC_IOPORT_READ
4134	\|\| rc == VINF_IOM_HC_IOPORT_WRITE
4135	\|\| rc == VINF_EM_RAW_GUEST_TRAP
4136	\|\| rc == VINF_TRPM_XCPT_DISPATCHED
4137	\|\| rc == VINF_EM_RESCHEDULE_REM,
4138	("rc = %d\n", VBOXSTRICTRC_VAL(rc)));
4139	break;
4140
4141	case VMX_EXIT_TPR: /* 43 TPR below threshold. Guest software executed MOV to CR8. */
4142	case VMX_EXIT_APIC_ACCESS: /* 44 APIC access. Guest software attempted to access memory at a physical address on the APIC-access page. */
4143	case VMX_EXIT_RDMSR: /* 31 RDMSR. Guest software attempted to execute RDMSR. */
4144	case VMX_EXIT_WRMSR: /* 32 WRMSR. Guest software attempted to execute WRMSR. */
4145	case VMX_EXIT_PAUSE: /* 40 Guest software attempted to execute PAUSE. */
4146	case VMX_EXIT_MONITOR: /* 39 Guest software attempted to execute MONITOR. */
4147	/* Note: If we decide to emulate them here, then we must sync the MSRs that could have been changed (sysenter, fs/gs base)!!! */
4148	rc = VERR_EM_INTERPRETER;
4149	break;
4150
4151	case VMX_EXIT_IRQ_WINDOW: /* 7 Interrupt window. */
4152	Assert(rc == VINF_EM_RAW_INTERRUPT);
4153	break;
4154
4155	case VMX_EXIT_ERR_INVALID_GUEST_STATE: /* 33 VM-entry failure due to invalid guest state. */
4156	{
4157	#ifdef VBOX_STRICT
4158	RTCCUINTREG val2 = 0;
4159
4160	Log(("VMX_EXIT_ERR_INVALID_GUEST_STATE\n"));
4161
4162	VMXReadVMCS(VMX_VMCS64_GUEST_RIP, &val2);
4163	Log(("Old eip %RGv new %RGv\n", (RTGCPTR)pCtx->rip, (RTGCPTR)val2));
4164
4165	VMXReadVMCS(VMX_VMCS64_GUEST_CR0, &val2);
4166	Log(("VMX_VMCS_GUEST_CR0 %RX64\n", (uint64_t)val2));
4167
4168	VMXReadVMCS(VMX_VMCS64_GUEST_CR3, &val2);
4169	Log(("VMX_VMCS_GUEST_CR3 %RX64\n", (uint64_t)val2));
4170
4171	VMXReadVMCS(VMX_VMCS64_GUEST_CR4, &val2);
4172	Log(("VMX_VMCS_GUEST_CR4 %RX64\n", (uint64_t)val2));
4173
4174	VMXReadVMCS(VMX_VMCS_GUEST_RFLAGS, &val2);
4175	Log(("VMX_VMCS_GUEST_RFLAGS %08x\n", val2));
4176
4177	VMX_LOG_SELREG(CS, "CS", val2);
4178	VMX_LOG_SELREG(DS, "DS", val2);
4179	VMX_LOG_SELREG(ES, "ES", val2);
4180	VMX_LOG_SELREG(FS, "FS", val2);
4181	VMX_LOG_SELREG(GS, "GS", val2);
4182	VMX_LOG_SELREG(SS, "SS", val2);
4183	VMX_LOG_SELREG(TR, "TR", val2);
4184	VMX_LOG_SELREG(LDTR, "LDTR", val2);
4185
4186	VMXReadVMCS(VMX_VMCS64_GUEST_GDTR_BASE, &val2);
4187	Log(("VMX_VMCS_GUEST_GDTR_BASE %RX64\n", (uint64_t)val2));
4188	VMXReadVMCS(VMX_VMCS64_GUEST_IDTR_BASE, &val2);
4189	Log(("VMX_VMCS_GUEST_IDTR_BASE %RX64\n", (uint64_t)val2));
4190	#endif /* VBOX_STRICT */
4191	rc = VERR_VMX_INVALID_GUEST_STATE;
4192	break;
4193	}
4194
4195	case VMX_EXIT_ERR_MSR_LOAD: /* 34 VM-entry failure due to MSR loading. */
4196	case VMX_EXIT_ERR_MACHINE_CHECK: /* 41 VM-entry failure due to machine-check. */
4197	default:
4198	rc = VERR_VMX_UNEXPECTED_EXIT_CODE;
4199	AssertMsgFailed(("Unexpected exit code %d\n", exitReason)); /* Can't happen. */
4200	break;
4201
4202	}
4203	end:
4204
4205	/* We now going back to ring-3, so clear the action flag. */
4206	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
4207
4208	/* Signal changes for the recompiler. */
4209	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR \| CPUM_CHANGED_LDTR \| CPUM_CHANGED_GDTR \| CPUM_CHANGED_IDTR \| CPUM_CHANGED_TR \| CPUM_CHANGED_HIDDEN_SEL_REGS);
4210
4211	/* If we executed vmlaunch/vmresume and an external irq was pending, then we don't have to do a full sync the next time. */
4212	if ( exitReason == VMX_EXIT_EXTERNAL_IRQ
4213	&& !VMX_EXIT_INTERRUPTION_INFO_VALID(intInfo))
4214	{
4215	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatPendingHostIrq);
4216	/* On the next entry we'll only sync the host context. */
4217	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_HOST_CONTEXT;
4218	}
4219	else
4220	{
4221	/* On the next entry we'll sync everything. */
4222	/** @todo we can do better than this */
4223	/* Not in the VINF_PGM_CHANGE_MODE though! */
4224	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_ALL;
4225	}
4226
4227	/* translate into a less severe return code */
4228	if (rc == VERR_EM_INTERPRETER)
4229	rc = VINF_EM_RAW_EMULATE_INSTR;
4230	else
4231	/* Try to extract more information about what might have gone wrong here. */
4232	if (rc == VERR_VMX_INVALID_VMCS_PTR)
4233	{
4234	VMXGetActivateVMCS(&pVCpu->hwaccm.s.vmx.lasterror.u64VMCSPhys);
4235	pVCpu->hwaccm.s.vmx.lasterror.ulVMCSRevision = (uint32_t )pVCpu->hwaccm.s.vmx.pVMCS;
4236	pVCpu->hwaccm.s.vmx.lasterror.idEnteredCpu = pVCpu->hwaccm.s.idEnteredCpu;
4237	pVCpu->hwaccm.s.vmx.lasterror.idCurrentCpu = RTMpCpuId();
4238	}
4239
4240	/* Just set the correct state here instead of trying to catch every goto above. */
4241	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED, VMCPUSTATE_STARTED_EXEC);
4242
4243	#ifdef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
4244	/* Restore interrupts if we exitted after disabling them. */
4245	if (uOldEFlags != ~(RTCCUINTREG)0)
4246	ASMSetFlags(uOldEFlags);
4247	#endif
4248
4249	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2, x);
4250	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit1, x);
4251	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatEntry, x);
4252	Log2(("X"));
4253	return VBOXSTRICTRC_TODO(rc);
4254	}
4255
4256
4257	/**
4258	* Enters the VT-x session
4259	*
4260	* @returns VBox status code.
4261	* @param pVM The VM to operate on.
4262	* @param pVCpu The VMCPU to operate on.
4263	* @param pCpu CPU info struct
4264	*/
4265	VMMR0DECL(int) VMXR0Enter(PVM pVM, PVMCPU pVCpu, PHWACCM_CPUINFO pCpu)
4266	{
4267	Assert(pVM->hwaccm.s.vmx.fSupported);
4268
4269	unsigned cr4 = ASMGetCR4();
4270	if (!(cr4 & X86_CR4_VMXE))
4271	{
4272	AssertMsgFailed(("X86_CR4_VMXE should be set!\n"));
4273	return VERR_VMX_X86_CR4_VMXE_CLEARED;
4274	}
4275
4276	/* Activate the VM Control Structure. */
4277	int rc = VMXActivateVMCS(pVCpu->hwaccm.s.vmx.pVMCSPhys);
4278	if (RT_FAILURE(rc))
4279	return rc;
4280
4281	pVCpu->hwaccm.s.fResumeVM = false;
4282	return VINF_SUCCESS;
4283	}
4284
4285
4286	/**
4287	* Leaves the VT-x session
4288	*
4289	* @returns VBox status code.
4290	* @param pVM The VM to operate on.
4291	* @param pVCpu The VMCPU to operate on.
4292	* @param pCtx CPU context
4293	*/
4294	VMMR0DECL(int) VMXR0Leave(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
4295	{
4296	Assert(pVM->hwaccm.s.vmx.fSupported);
4297
4298	#ifdef DEBUG
4299	if (CPUMIsHyperDebugStateActive(pVCpu))
4300	{
4301	CPUMR0LoadHostDebugState(pVM, pVCpu);
4302	Assert(pVCpu->hwaccm.s.vmx.proc_ctls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT);
4303	}
4304	else
4305	#endif
4306	/* Save the guest debug state if necessary. */
4307	if (CPUMIsGuestDebugStateActive(pVCpu))
4308	{
4309	CPUMR0SaveGuestDebugState(pVM, pVCpu, pCtx, true /* save DR6 */);
4310
4311	/* Enable drx move intercepts again. */
4312	pVCpu->hwaccm.s.vmx.proc_ctls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT;
4313	int rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
4314	AssertRC(rc);
4315
4316	/* Resync the debug registers the next time. */
4317	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_GUEST_DEBUG;
4318	}
4319	else
4320	Assert(pVCpu->hwaccm.s.vmx.proc_ctls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT);
4321
4322	/* Clear VM Control Structure. Marking it inactive, clearing implementation specific data and writing back VMCS data to memory. */
4323	int rc = VMXClearVMCS(pVCpu->hwaccm.s.vmx.pVMCSPhys);
4324	AssertRC(rc);
4325
4326	return VINF_SUCCESS;
4327	}
4328
4329	/**
4330	* Flush the TLB (EPT)
4331	*
4332	* @returns VBox status code.
4333	* @param pVM The VM to operate on.
4334	* @param pVCpu The VM CPU to operate on.
4335	* @param enmFlush Type of flush
4336	* @param GCPhys Physical address of the page to flush
4337	*/
4338	static void vmxR0FlushEPT(PVM pVM, PVMCPU pVCpu, VMX_FLUSH enmFlush, RTGCPHYS GCPhys)
4339	{
4340	uint64_t descriptor[2];
4341
4342	LogFlow(("vmxR0FlushEPT %d %RGv\n", enmFlush, GCPhys));
4343	Assert(pVM->hwaccm.s.fNestedPaging);
4344	descriptor[0] = pVCpu->hwaccm.s.vmx.GCPhysEPTP;
4345	descriptor[1] = GCPhys;
4346	int rc = VMXR0InvEPT(enmFlush, &descriptor[0]);
4347	AssertRC(rc);
4348	}
4349
4350	#ifdef HWACCM_VTX_WITH_VPID
4351	/**
4352	* Flush the TLB (EPT)
4353	*
4354	* @returns VBox status code.
4355	* @param pVM The VM to operate on.
4356	* @param pVCpu The VM CPU to operate on.
4357	* @param enmFlush Type of flush
4358	* @param GCPtr Virtual address of the page to flush
4359	*/
4360	static void vmxR0FlushVPID(PVM pVM, PVMCPU pVCpu, VMX_FLUSH enmFlush, RTGCPTR GCPtr)
4361	{
4362	#if HC_ARCH_BITS == 32
4363	/* If we get a flush in 64 bits guest mode, then force a full TLB flush. Invvpid probably takes only 32 bits addresses. (@todo) */
4364	if ( CPUMIsGuestInLongMode(pVCpu)
4365	&& !VMX_IS_64BIT_HOST_MODE())
4366	{
4367	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
4368	}
4369	else
4370	#endif
4371	{
4372	uint64_t descriptor[2];
4373
4374	Assert(pVM->hwaccm.s.vmx.fVPID);
4375	descriptor[0] = pVCpu->hwaccm.s.uCurrentASID;
4376	descriptor[1] = GCPtr;
4377	int rc = VMXR0InvVPID(enmFlush, &descriptor[0]);
4378	AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvVPID %x %x %RGv failed with %d\n", enmFlush, pVCpu->hwaccm.s.uCurrentASID, GCPtr, rc));
4379	}
4380	}
4381	#endif /* HWACCM_VTX_WITH_VPID */
4382
4383	/**
4384	* Invalidates a guest page
4385	*
4386	* @returns VBox status code.
4387	* @param pVM The VM to operate on.
4388	* @param pVCpu The VM CPU to operate on.
4389	* @param GCVirt Page to invalidate
4390	*/
4391	VMMR0DECL(int) VMXR0InvalidatePage(PVM pVM, PVMCPU pVCpu, RTGCPTR GCVirt)
4392	{
4393	bool fFlushPending = VMCPU_FF_ISSET(pVCpu, VMCPU_FF_TLB_FLUSH);
4394
4395	Log2(("VMXR0InvalidatePage %RGv\n", GCVirt));
4396
4397	/* Only relevant if we want to use VPID.
4398	* In the nested paging case we still see such calls, but
4399	* can safely ignore them. (e.g. after cr3 updates)
4400	*/
4401	#ifdef HWACCM_VTX_WITH_VPID
4402	/* Skip it if a TLB flush is already pending. */
4403	if ( !fFlushPending
4404	&& pVM->hwaccm.s.vmx.fVPID)
4405	vmxR0FlushVPID(pVM, pVCpu, pVM->hwaccm.s.vmx.enmFlushPage, GCVirt);
4406	#endif /* HWACCM_VTX_WITH_VPID */
4407
4408	return VINF_SUCCESS;
4409	}
4410
4411	/**
4412	* Invalidates a guest page by physical address
4413	*
4414	* NOTE: Assumes the current instruction references this physical page though a virtual address!!
4415	*
4416	* @returns VBox status code.
4417	* @param pVM The VM to operate on.
4418	* @param pVCpu The VM CPU to operate on.
4419	* @param GCPhys Page to invalidate
4420	*/
4421	VMMR0DECL(int) VMXR0InvalidatePhysPage(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys)
4422	{
4423	bool fFlushPending = VMCPU_FF_ISSET(pVCpu, VMCPU_FF_TLB_FLUSH);
4424
4425	Assert(pVM->hwaccm.s.fNestedPaging);
4426
4427	LogFlow(("VMXR0InvalidatePhysPage %RGp\n", GCPhys));
4428
4429	/* Skip it if a TLB flush is already pending. */
4430	if (!fFlushPending)
4431	vmxR0FlushEPT(pVM, pVCpu, pVM->hwaccm.s.vmx.enmFlushPage, GCPhys);
4432
4433	return VINF_SUCCESS;
4434	}
4435
4436	/**
4437	* Report world switch error and dump some useful debug info
4438	*
4439	* @param pVM The VM to operate on.
4440	* @param pVCpu The VMCPU to operate on.
4441	* @param rc Return code
4442	* @param pCtx Current CPU context (not updated)
4443	*/
4444	static void VMXR0ReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, VBOXSTRICTRC rc, PCPUMCTX pCtx)
4445	{
4446	switch (VBOXSTRICTRC_VAL(rc))
4447	{
4448	case VERR_VMX_INVALID_VMXON_PTR:
4449	AssertFailed();
4450	break;
4451
4452	case VERR_VMX_UNABLE_TO_START_VM:
4453	case VERR_VMX_UNABLE_TO_RESUME_VM:
4454	{
4455	int rc2;
4456	RTCCUINTREG exitReason, instrError;
4457
4458	rc2 = VMXReadVMCS(VMX_VMCS32_RO_EXIT_REASON, &exitReason);
4459	rc2 \|= VMXReadVMCS(VMX_VMCS32_RO_VM_INSTR_ERROR, &instrError);
4460	AssertRC(rc2);
4461	if (rc2 == VINF_SUCCESS)
4462	{
4463	Log(("Unable to start/resume VM for reason: %x. Instruction error %x\n", (uint32_t)exitReason, (uint32_t)instrError));
4464	Log(("Current stack %08x\n", &rc2));
4465
4466	pVCpu->hwaccm.s.vmx.lasterror.ulInstrError = instrError;
4467	pVCpu->hwaccm.s.vmx.lasterror.ulExitReason = exitReason;
4468
4469	#ifdef VBOX_STRICT
4470	RTGDTR gdtr;
4471	PCX86DESCHC pDesc;
4472	RTCCUINTREG val;
4473
4474	ASMGetGDTR(&gdtr);
4475
4476	VMXReadVMCS(VMX_VMCS64_GUEST_RIP, &val);
4477	Log(("Old eip %RGv new %RGv\n", (RTGCPTR)pCtx->rip, (RTGCPTR)val));
4478	VMXReadVMCS(VMX_VMCS_CTRL_PIN_EXEC_CONTROLS, &val);
4479	Log(("VMX_VMCS_CTRL_PIN_EXEC_CONTROLS %08x\n", val));
4480	VMXReadVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, &val);
4481	Log(("VMX_VMCS_CTRL_PROC_EXEC_CONTROLS %08x\n", val));
4482	VMXReadVMCS(VMX_VMCS_CTRL_ENTRY_CONTROLS, &val);
4483	Log(("VMX_VMCS_CTRL_ENTRY_CONTROLS %08x\n", val));
4484	VMXReadVMCS(VMX_VMCS_CTRL_EXIT_CONTROLS, &val);
4485	Log(("VMX_VMCS_CTRL_EXIT_CONTROLS %08x\n", val));
4486
4487	VMXReadVMCS(VMX_VMCS_HOST_CR0, &val);
4488	Log(("VMX_VMCS_HOST_CR0 %08x\n", val));
4489
4490	VMXReadVMCS(VMX_VMCS_HOST_CR3, &val);
4491	Log(("VMX_VMCS_HOST_CR3 %08x\n", val));
4492
4493	VMXReadVMCS(VMX_VMCS_HOST_CR4, &val);
4494	Log(("VMX_VMCS_HOST_CR4 %08x\n", val));
4495
4496	VMXReadVMCS(VMX_VMCS16_HOST_FIELD_CS, &val);
4497	Log(("VMX_VMCS_HOST_FIELD_CS %08x\n", val));
4498
4499	VMXReadVMCS(VMX_VMCS_GUEST_RFLAGS, &val);
4500	Log(("VMX_VMCS_GUEST_RFLAGS %08x\n", val));
4501
4502	if (val < gdtr.cbGdt)
4503	{
4504	pDesc = (PCX86DESCHC)(gdtr.pGdt + (val & X86_SEL_MASK));
4505	HWACCMR0DumpDescriptor(pDesc, val, "CS: ");
4506	}
4507
4508	VMXReadVMCS(VMX_VMCS16_HOST_FIELD_DS, &val);
4509	Log(("VMX_VMCS_HOST_FIELD_DS %08x\n", val));
4510	if (val < gdtr.cbGdt)
4511	{
4512	pDesc = (PCX86DESCHC)(gdtr.pGdt + (val & X86_SEL_MASK));
4513	HWACCMR0DumpDescriptor(pDesc, val, "DS: ");
4514	}
4515
4516	VMXReadVMCS(VMX_VMCS16_HOST_FIELD_ES, &val);
4517	Log(("VMX_VMCS_HOST_FIELD_ES %08x\n", val));
4518	if (val < gdtr.cbGdt)
4519	{
4520	pDesc = (PCX86DESCHC)(gdtr.pGdt + (val & X86_SEL_MASK));
4521	HWACCMR0DumpDescriptor(pDesc, val, "ES: ");
4522	}
4523
4524	VMXReadVMCS(VMX_VMCS16_HOST_FIELD_FS, &val);
4525	Log(("VMX_VMCS16_HOST_FIELD_FS %08x\n", val));
4526	if (val < gdtr.cbGdt)
4527	{
4528	pDesc = (PCX86DESCHC)(gdtr.pGdt + (val & X86_SEL_MASK));
4529	HWACCMR0DumpDescriptor(pDesc, val, "FS: ");
4530	}
4531
4532	VMXReadVMCS(VMX_VMCS16_HOST_FIELD_GS, &val);
4533	Log(("VMX_VMCS16_HOST_FIELD_GS %08x\n", val));
4534	if (val < gdtr.cbGdt)
4535	{
4536	pDesc = (PCX86DESCHC)(gdtr.pGdt + (val & X86_SEL_MASK));
4537	HWACCMR0DumpDescriptor(pDesc, val, "GS: ");
4538	}
4539
4540	VMXReadVMCS(VMX_VMCS16_HOST_FIELD_SS, &val);
4541	Log(("VMX_VMCS16_HOST_FIELD_SS %08x\n", val));
4542	if (val < gdtr.cbGdt)
4543	{
4544	pDesc = (PCX86DESCHC)(gdtr.pGdt + (val & X86_SEL_MASK));
4545	HWACCMR0DumpDescriptor(pDesc, val, "SS: ");
4546	}
4547
4548	VMXReadVMCS(VMX_VMCS16_HOST_FIELD_TR, &val);
4549	Log(("VMX_VMCS16_HOST_FIELD_TR %08x\n", val));
4550	if (val < gdtr.cbGdt)
4551	{
4552	pDesc = (PCX86DESCHC)(gdtr.pGdt + (val & X86_SEL_MASK));
4553	HWACCMR0DumpDescriptor(pDesc, val, "TR: ");
4554	}
4555
4556	VMXReadVMCS(VMX_VMCS_HOST_TR_BASE, &val);
4557	Log(("VMX_VMCS_HOST_TR_BASE %RHv\n", val));
4558
4559	VMXReadVMCS(VMX_VMCS_HOST_GDTR_BASE, &val);
4560	Log(("VMX_VMCS_HOST_GDTR_BASE %RHv\n", val));
4561	VMXReadVMCS(VMX_VMCS_HOST_IDTR_BASE, &val);
4562	Log(("VMX_VMCS_HOST_IDTR_BASE %RHv\n", val));
4563
4564	VMXReadVMCS(VMX_VMCS32_HOST_SYSENTER_CS, &val);
4565	Log(("VMX_VMCS_HOST_SYSENTER_CS %08x\n", val));
4566
4567	VMXReadVMCS(VMX_VMCS_HOST_SYSENTER_EIP, &val);
4568	Log(("VMX_VMCS_HOST_SYSENTER_EIP %RHv\n", val));
4569
4570	VMXReadVMCS(VMX_VMCS_HOST_SYSENTER_ESP, &val);
4571	Log(("VMX_VMCS_HOST_SYSENTER_ESP %RHv\n", val));
4572
4573	VMXReadVMCS(VMX_VMCS_HOST_RSP, &val);
4574	Log(("VMX_VMCS_HOST_RSP %RHv\n", val));
4575	VMXReadVMCS(VMX_VMCS_HOST_RIP, &val);
4576	Log(("VMX_VMCS_HOST_RIP %RHv\n", val));
4577
4578	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4579	if (VMX_IS_64BIT_HOST_MODE())
4580	{
4581	Log(("MSR_K6_EFER = %RX64\n", ASMRdMsr(MSR_K6_EFER)));
4582	Log(("MSR_K6_STAR = %RX64\n", ASMRdMsr(MSR_K6_STAR)));
4583	Log(("MSR_K8_LSTAR = %RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
4584	Log(("MSR_K8_CSTAR = %RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
4585	Log(("MSR_K8_SF_MASK = %RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
4586	}
4587	# endif
4588	#endif /* VBOX_STRICT */
4589	}
4590	break;
4591	}
4592
4593	default:
4594	/* impossible */
4595	AssertMsgFailed(("%Rrc (%#x)\n", VBOXSTRICTRC_VAL(rc), VBOXSTRICTRC_VAL(rc)));
4596	break;
4597	}
4598	}
4599
4600	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4601	/**
4602	* Prepares for and executes VMLAUNCH (64 bits guest mode)
4603	*
4604	* @returns VBox status code
4605	* @param fResume vmlauch/vmresume
4606	* @param pCtx Guest context
4607	* @param pCache VMCS cache
4608	* @param pVM The VM to operate on.
4609	* @param pVCpu The VMCPU to operate on.
4610	*/
4611	DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
4612	{
4613	uint32_t aParam[6];
4614	PHWACCM_CPUINFO pCpu;
4615	RTHCPHYS pPageCpuPhys;
4616	int rc;
4617
4618	pCpu = HWACCMR0GetCurrentCpu();
4619	pPageCpuPhys = RTR0MemObjGetPagePhysAddr(pCpu->pMemObj, 0);
4620
4621	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4622	pCache->uPos = 1;
4623	pCache->interPD = PGMGetInterPaeCR3(pVM);
4624	pCache->pSwitcher = (uint64_t)pVM->hwaccm.s.pfnHost32ToGuest64R0;
4625	#endif
4626
4627	#ifdef DEBUG
4628	pCache->TestIn.pPageCpuPhys = 0;
4629	pCache->TestIn.pVMCSPhys = 0;
4630	pCache->TestIn.pCache = 0;
4631	pCache->TestOut.pVMCSPhys = 0;
4632	pCache->TestOut.pCache = 0;
4633	pCache->TestOut.pCtx = 0;
4634	pCache->TestOut.eflags = 0;
4635	#endif
4636
4637	aParam[0] = (uint32_t)(pPageCpuPhys); /* Param 1: VMXON physical address - Lo. */
4638	aParam[1] = (uint32_t)(pPageCpuPhys >> 32); /* Param 1: VMXON physical address - Hi. */
4639	aParam[2] = (uint32_t)(pVCpu->hwaccm.s.vmx.pVMCSPhys); /* Param 2: VMCS physical address - Lo. */
4640	aParam[3] = (uint32_t)(pVCpu->hwaccm.s.vmx.pVMCSPhys >> 32); /* Param 2: VMCS physical address - Hi. */
4641	aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hwaccm.s.vmx.VMCSCache);
4642	aParam[5] = 0;
4643
4644	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4645	pCtx->dr[4] = pVM->hwaccm.s.vmx.pScratchPhys + 16 + 8;
4646	(uint32_t )(pVM->hwaccm.s.vmx.pScratch + 16 + 8) = 1;
4647	#endif
4648	rc = VMXR0Execute64BitsHandler(pVM, pVCpu, pCtx, pVM->hwaccm.s.pfnVMXGCStartVM64, 6, &aParam[0]);
4649
4650	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4651	Assert((uint32_t )(pVM->hwaccm.s.vmx.pScratch + 16 + 8) == 5);
4652	Assert(pCtx->dr[4] == 10);
4653	(uint32_t )(pVM->hwaccm.s.vmx.pScratch + 16 + 8) = 0xff;
4654	#endif
4655
4656	#ifdef DEBUG
4657	AssertMsg(pCache->TestIn.pPageCpuPhys == pPageCpuPhys, ("%RHp vs %RHp\n", pCache->TestIn.pPageCpuPhys, pPageCpuPhys));
4658	AssertMsg(pCache->TestIn.pVMCSPhys == pVCpu->hwaccm.s.vmx.pVMCSPhys, ("%RHp vs %RHp\n", pCache->TestIn.pVMCSPhys, pVCpu->hwaccm.s.vmx.pVMCSPhys));
4659	AssertMsg(pCache->TestIn.pVMCSPhys == pCache->TestOut.pVMCSPhys, ("%RHp vs %RHp\n", pCache->TestIn.pVMCSPhys, pCache->TestOut.pVMCSPhys));
4660	AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache, pCache->TestOut.pCache));
4661	AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hwaccm.s.vmx.VMCSCache), ("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hwaccm.s.vmx.VMCSCache)));
4662	AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx, pCache->TestOut.pCtx));
4663	Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
4664	#endif
4665	return rc;
4666	}
4667
4668	/**
4669	* Executes the specified handler in 64 mode
4670	*
4671	* @returns VBox status code.
4672	* @param pVM The VM to operate on.
4673	* @param pVCpu The VMCPU to operate on.
4674	* @param pCtx Guest context
4675	* @param pfnHandler RC handler
4676	* @param cbParam Number of parameters
4677	* @param paParam Array of 32 bits parameters
4678	*/
4679	VMMR0DECL(int) VMXR0Execute64BitsHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, RTRCPTR pfnHandler, uint32_t cbParam, uint32_t *paParam)
4680	{
4681	int rc, rc2;
4682	PHWACCM_CPUINFO pCpu;
4683	RTHCPHYS pPageCpuPhys;
4684	RTHCUINTREG uOldEFlags;
4685
4686	AssertReturn(pVM->hwaccm.s.pfnHost32ToGuest64R0, VERR_INTERNAL_ERROR);
4687	Assert(pfnHandler);
4688	Assert(pVCpu->hwaccm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hwaccm.s.vmx.VMCSCache.Write.aField));
4689	Assert(pVCpu->hwaccm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hwaccm.s.vmx.VMCSCache.Read.aField));
4690
4691	#ifdef VBOX_STRICT
4692	for (unsigned i=0;i<pVCpu->hwaccm.s.vmx.VMCSCache.Write.cValidEntries;i++)
4693	Assert(vmxR0IsValidWriteField(pVCpu->hwaccm.s.vmx.VMCSCache.Write.aField[i]));
4694
4695	for (unsigned i=0;i<pVCpu->hwaccm.s.vmx.VMCSCache.Read.cValidEntries;i++)
4696	Assert(vmxR0IsValidReadField(pVCpu->hwaccm.s.vmx.VMCSCache.Read.aField[i]));
4697	#endif
4698
4699	/* Disable interrupts. */
4700	uOldEFlags = ASMIntDisableFlags();
4701
4702	pCpu = HWACCMR0GetCurrentCpu();
4703	pPageCpuPhys = RTR0MemObjGetPagePhysAddr(pCpu->pMemObj, 0);
4704
4705	/* Clear VM Control Structure. Marking it inactive, clearing implementation specific data and writing back VMCS data to memory. */
4706	VMXClearVMCS(pVCpu->hwaccm.s.vmx.pVMCSPhys);
4707
4708	/* Leave VMX Root Mode. */
4709	VMXDisable();
4710
4711	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
4712
4713	CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
4714	CPUMSetHyperEIP(pVCpu, pfnHandler);
4715	for (int i=(int)cbParam-1;i>=0;i--)
4716	CPUMPushHyper(pVCpu, paParam[i]);
4717
4718	STAM_PROFILE_ADV_START(&pVCpu->hwaccm.s.StatWorldSwitch3264, z);
4719	/* Call switcher. */
4720	rc = pVM->hwaccm.s.pfnHost32ToGuest64R0(pVM, RT_OFFSETOF(VM, aCpus[pVCpu->idCpu].cpum) - RT_OFFSETOF(VM, cpum));
4721	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatWorldSwitch3264, z);
4722
4723	/* Make sure the VMX instructions don't cause #UD faults. */
4724	ASMSetCR4(ASMGetCR4() \| X86_CR4_VMXE);
4725
4726	/* Enter VMX Root Mode */
4727	rc2 = VMXEnable(pPageCpuPhys);
4728	if (RT_FAILURE(rc2))
4729	{
4730	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
4731	ASMSetFlags(uOldEFlags);
4732	return VERR_VMX_VMXON_FAILED;
4733	}
4734
4735	rc2 = VMXActivateVMCS(pVCpu->hwaccm.s.vmx.pVMCSPhys);
4736	AssertRC(rc2);
4737	Assert(!(ASMGetFlags() & X86_EFL_IF));
4738	ASMSetFlags(uOldEFlags);
4739	return rc;
4740	}
4741
4742	#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL) */
4743
4744
4745	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
4746	/**
4747	* Executes VMWRITE
4748	*
4749	* @returns VBox status code
4750	* @param pVCpu The VMCPU to operate on.
4751	* @param idxField VMCS index
4752	* @param u64Val 16, 32 or 64 bits value
4753	*/
4754	VMMR0DECL(int) VMXWriteVMCS64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
4755	{
4756	int rc;
4757
4758	switch (idxField)
4759	{
4760	case VMX_VMCS_CTRL_TSC_OFFSET_FULL:
4761	case VMX_VMCS_CTRL_IO_BITMAP_A_FULL:
4762	case VMX_VMCS_CTRL_IO_BITMAP_B_FULL:
4763	case VMX_VMCS_CTRL_MSR_BITMAP_FULL:
4764	case VMX_VMCS_CTRL_VMEXIT_MSR_STORE_FULL:
4765	case VMX_VMCS_CTRL_VMEXIT_MSR_LOAD_FULL:
4766	case VMX_VMCS_CTRL_VMENTRY_MSR_LOAD_FULL:
4767	case VMX_VMCS_CTRL_VAPIC_PAGEADDR_FULL:
4768	case VMX_VMCS_CTRL_APIC_ACCESSADDR_FULL:
4769	case VMX_VMCS_GUEST_LINK_PTR_FULL:
4770	case VMX_VMCS_GUEST_PDPTR0_FULL:
4771	case VMX_VMCS_GUEST_PDPTR1_FULL:
4772	case VMX_VMCS_GUEST_PDPTR2_FULL:
4773	case VMX_VMCS_GUEST_PDPTR3_FULL:
4774	case VMX_VMCS_GUEST_DEBUGCTL_FULL:
4775	case VMX_VMCS_GUEST_EFER_FULL:
4776	case VMX_VMCS_CTRL_EPTP_FULL:
4777	/* These fields consist of two parts, which are both writable in 32 bits mode. */
4778	rc = VMXWriteVMCS32(idxField, u64Val);
4779	rc \|= VMXWriteVMCS32(idxField + 1, (uint32_t)(u64Val >> 32ULL));
4780	AssertRC(rc);
4781	return rc;
4782
4783	case VMX_VMCS64_GUEST_LDTR_BASE:
4784	case VMX_VMCS64_GUEST_TR_BASE:
4785	case VMX_VMCS64_GUEST_GDTR_BASE:
4786	case VMX_VMCS64_GUEST_IDTR_BASE:
4787	case VMX_VMCS64_GUEST_SYSENTER_EIP:
4788	case VMX_VMCS64_GUEST_SYSENTER_ESP:
4789	case VMX_VMCS64_GUEST_CR0:
4790	case VMX_VMCS64_GUEST_CR4:
4791	case VMX_VMCS64_GUEST_CR3:
4792	case VMX_VMCS64_GUEST_DR7:
4793	case VMX_VMCS64_GUEST_RIP:
4794	case VMX_VMCS64_GUEST_RSP:
4795	case VMX_VMCS64_GUEST_CS_BASE:
4796	case VMX_VMCS64_GUEST_DS_BASE:
4797	case VMX_VMCS64_GUEST_ES_BASE:
4798	case VMX_VMCS64_GUEST_FS_BASE:
4799	case VMX_VMCS64_GUEST_GS_BASE:
4800	case VMX_VMCS64_GUEST_SS_BASE:
4801	/* Queue a 64 bits value as we can't set it in 32 bits host mode. */
4802	if (u64Val >> 32ULL)
4803	rc = VMXWriteCachedVMCSEx(pVCpu, idxField, u64Val);
4804	else
4805	rc = VMXWriteVMCS32(idxField, (uint32_t)u64Val);
4806
4807	return rc;
4808
4809	default:
4810	AssertMsgFailed(("Unexpected field %x\n", idxField));
4811	return VERR_INVALID_PARAMETER;
4812	}
4813	}
4814
4815	/**
4816	* Cache VMCS writes for performance reasons (Darwin) and for running 64 bits guests on 32 bits hosts.
4817	*
4818	* @param pVCpu The VMCPU to operate on.
4819	* @param idxField VMCS field
4820	* @param u64Val Value
4821	*/
4822	VMMR0DECL(int) VMXWriteCachedVMCSEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
4823	{
4824	PVMCSCACHE pCache = &pVCpu->hwaccm.s.vmx.VMCSCache;
4825
4826	AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1, ("entries=%x\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
4827
4828	/* Make sure there are no duplicates. */
4829	for (unsigned i=0;i<pCache->Write.cValidEntries;i++)
4830	{
4831	if (pCache->Write.aField[i] == idxField)
4832	{
4833	pCache->Write.aFieldVal[i] = u64Val;
4834	return VINF_SUCCESS;
4835	}
4836	}
4837
4838	pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
4839	pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
4840	pCache->Write.cValidEntries++;
4841	return VINF_SUCCESS;
4842	}
4843
4844	#endif /* HC_ARCH_BITS == 32 && !VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0 */
4845
4846	#ifdef VBOX_STRICT
4847	static bool vmxR0IsValidReadField(uint32_t idxField)
4848	{
4849	switch(idxField)
4850	{
4851	case VMX_VMCS64_GUEST_RIP:
4852	case VMX_VMCS64_GUEST_RSP:
4853	case VMX_VMCS_GUEST_RFLAGS:
4854	case VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE:
4855	case VMX_VMCS_CTRL_CR0_READ_SHADOW:
4856	case VMX_VMCS64_GUEST_CR0:
4857	case VMX_VMCS_CTRL_CR4_READ_SHADOW:
4858	case VMX_VMCS64_GUEST_CR4:
4859	case VMX_VMCS64_GUEST_DR7:
4860	case VMX_VMCS32_GUEST_SYSENTER_CS:
4861	case VMX_VMCS64_GUEST_SYSENTER_EIP:
4862	case VMX_VMCS64_GUEST_SYSENTER_ESP:
4863	case VMX_VMCS32_GUEST_GDTR_LIMIT:
4864	case VMX_VMCS64_GUEST_GDTR_BASE:
4865	case VMX_VMCS32_GUEST_IDTR_LIMIT:
4866	case VMX_VMCS64_GUEST_IDTR_BASE:
4867	case VMX_VMCS16_GUEST_FIELD_CS:
4868	case VMX_VMCS32_GUEST_CS_LIMIT:
4869	case VMX_VMCS64_GUEST_CS_BASE:
4870	case VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS:
4871	case VMX_VMCS16_GUEST_FIELD_DS:
4872	case VMX_VMCS32_GUEST_DS_LIMIT:
4873	case VMX_VMCS64_GUEST_DS_BASE:
4874	case VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS:
4875	case VMX_VMCS16_GUEST_FIELD_ES:
4876	case VMX_VMCS32_GUEST_ES_LIMIT:
4877	case VMX_VMCS64_GUEST_ES_BASE:
4878	case VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS:
4879	case VMX_VMCS16_GUEST_FIELD_FS:
4880	case VMX_VMCS32_GUEST_FS_LIMIT:
4881	case VMX_VMCS64_GUEST_FS_BASE:
4882	case VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS:
4883	case VMX_VMCS16_GUEST_FIELD_GS:
4884	case VMX_VMCS32_GUEST_GS_LIMIT:
4885	case VMX_VMCS64_GUEST_GS_BASE:
4886	case VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS:
4887	case VMX_VMCS16_GUEST_FIELD_SS:
4888	case VMX_VMCS32_GUEST_SS_LIMIT:
4889	case VMX_VMCS64_GUEST_SS_BASE:
4890	case VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS:
4891	case VMX_VMCS16_GUEST_FIELD_LDTR:
4892	case VMX_VMCS32_GUEST_LDTR_LIMIT:
4893	case VMX_VMCS64_GUEST_LDTR_BASE:
4894	case VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS:
4895	case VMX_VMCS16_GUEST_FIELD_TR:
4896	case VMX_VMCS32_GUEST_TR_LIMIT:
4897	case VMX_VMCS64_GUEST_TR_BASE:
4898	case VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS:
4899	case VMX_VMCS32_RO_EXIT_REASON:
4900	case VMX_VMCS32_RO_VM_INSTR_ERROR:
4901	case VMX_VMCS32_RO_EXIT_INSTR_LENGTH:
4902	case VMX_VMCS32_RO_EXIT_INTERRUPTION_ERRCODE:
4903	case VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO:
4904	case VMX_VMCS32_RO_EXIT_INSTR_INFO:
4905	case VMX_VMCS_RO_EXIT_QUALIFICATION:
4906	case VMX_VMCS32_RO_IDT_INFO:
4907	case VMX_VMCS32_RO_IDT_ERRCODE:
4908	case VMX_VMCS64_GUEST_CR3:
4909	case VMX_VMCS_EXIT_PHYS_ADDR_FULL:
4910	return true;
4911	}
4912	return false;
4913	}
4914
4915	static bool vmxR0IsValidWriteField(uint32_t idxField)
4916	{
4917	switch(idxField)
4918	{
4919	case VMX_VMCS64_GUEST_LDTR_BASE:
4920	case VMX_VMCS64_GUEST_TR_BASE:
4921	case VMX_VMCS64_GUEST_GDTR_BASE:
4922	case VMX_VMCS64_GUEST_IDTR_BASE:
4923	case VMX_VMCS64_GUEST_SYSENTER_EIP:
4924	case VMX_VMCS64_GUEST_SYSENTER_ESP:
4925	case VMX_VMCS64_GUEST_CR0:
4926	case VMX_VMCS64_GUEST_CR4:
4927	case VMX_VMCS64_GUEST_CR3:
4928	case VMX_VMCS64_GUEST_DR7:
4929	case VMX_VMCS64_GUEST_RIP:
4930	case VMX_VMCS64_GUEST_RSP:
4931	case VMX_VMCS64_GUEST_CS_BASE:
4932	case VMX_VMCS64_GUEST_DS_BASE:
4933	case VMX_VMCS64_GUEST_ES_BASE:
4934	case VMX_VMCS64_GUEST_FS_BASE:
4935	case VMX_VMCS64_GUEST_GS_BASE:
4936	case VMX_VMCS64_GUEST_SS_BASE:
4937	return true;
4938	}
4939	return false;
4940	}
4941
4942	#endif

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMR0/HWVMXR0.cpp@ 36641

Download in other formats: