HWVMXR0.cpp@ 35346

Last change on this file since 35346 was 35346, checked in by vboxsync, 14 years ago
VMM reorg: Moving the public include files from include/VBox to include/VBox/vmm.
Property svn:eol-style set to `native` Property svn:keywords set to `Id`
File size: 198.5 KB

Line
1	/* $Id: HWVMXR0.cpp 35346 2010-12-27 16:13:13Z vboxsync $ */
2	/** @file
3	* HWACCM VMX - Host Context Ring 0.
4	*/
5
6	/*
7	* Copyright (C) 2006-2007 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18
19	/*******************************************************************************
20	* Header Files *
21	*******************************************************************************/
22	#define LOG_GROUP LOG_GROUP_HWACCM
23	#include <iprt/asm-amd64-x86.h>
24	#include <VBox/vmm/hwaccm.h>
25	#include <VBox/vmm/pgm.h>
26	#include <VBox/vmm/dbgf.h>
27	#include <VBox/vmm/selm.h>
28	#include <VBox/vmm/iom.h>
29	#include <VBox/vmm/rem.h>
30	#include <VBox/vmm/tm.h>
31	#include "HWACCMInternal.h"
32	#include <VBox/vmm/vm.h>
33	#include <VBox/x86.h>
34	#include <VBox/vmm/pdmapi.h>
35	#include <VBox/err.h>
36	#include <VBox/log.h>
37	#include <iprt/assert.h>
38	#include <iprt/param.h>
39	#include <iprt/string.h>
40	#include <iprt/time.h>
41	#ifdef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
42	# include <iprt/thread.h>
43	#endif
44	#include "HWVMXR0.h"
45
46	/*******************************************************************************
47	* Defined Constants And Macros *
48	*******************************************************************************/
49	#if defined(RT_ARCH_AMD64)
50	# define VMX_IS_64BIT_HOST_MODE() (true)
51	#elif defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
52	# define VMX_IS_64BIT_HOST_MODE() (g_fVMXIs64bitHost != 0)
53	#else
54	# define VMX_IS_64BIT_HOST_MODE() (false)
55	#endif
56
57	/*******************************************************************************
58	* Global Variables *
59	*******************************************************************************/
60	/* IO operation lookup arrays. */
61	static uint32_t const g_aIOSize[4] = {1, 2, 0, 4};
62	static uint32_t const g_aIOOpAnd[4] = {0xff, 0xffff, 0, 0xffffffff};
63
64	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
65	/** See HWACCMR0A.asm. */
66	extern "C" uint32_t g_fVMXIs64bitHost;
67	#endif
68
69	/*******************************************************************************
70	* Local Functions *
71	*******************************************************************************/
72	static void VMXR0ReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, VBOXSTRICTRC rc, PCPUMCTX pCtx);
73	static void vmxR0SetupTLBEPT(PVM pVM, PVMCPU pVCpu);
74	static void vmxR0SetupTLBVPID(PVM pVM, PVMCPU pVCpu);
75	static void vmxR0SetupTLBDummy(PVM pVM, PVMCPU pVCpu);
76	static void vmxR0FlushEPT(PVM pVM, PVMCPU pVCpu, VMX_FLUSH enmFlush, RTGCPHYS GCPhys);
77	static void vmxR0FlushVPID(PVM pVM, PVMCPU pVCpu, VMX_FLUSH enmFlush, RTGCPTR GCPtr);
78	static void vmxR0UpdateExceptionBitmap(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx);
79	#ifdef VBOX_STRICT
80	static bool vmxR0IsValidReadField(uint32_t idxField);
81	static bool vmxR0IsValidWriteField(uint32_t idxField);
82	#endif
83	static void vmxR0SetMSRPermission(PVMCPU pVCpu, unsigned ulMSR, bool fRead, bool fWrite);
84
85	static void VMXR0CheckError(PVM pVM, PVMCPU pVCpu, int rc)
86	{
87	if (rc == VERR_VMX_GENERIC)
88	{
89	RTCCUINTREG instrError;
90
91	VMXReadVMCS(VMX_VMCS32_RO_VM_INSTR_ERROR, &instrError);
92	pVCpu->hwaccm.s.vmx.lasterror.ulInstrError = instrError;
93	}
94	pVM->hwaccm.s.lLastError = rc;
95	}
96
97	/**
98	* Sets up and activates VT-x on the current CPU
99	*
100	* @returns VBox status code.
101	* @param pCpu CPU info struct
102	* @param pVM The VM to operate on. (can be NULL after a resume!!)
103	* @param pvPageCpu Pointer to the global cpu page
104	* @param pPageCpuPhys Physical address of the global cpu page
105	*/
106	VMMR0DECL(int) VMXR0EnableCpu(PHWACCM_CPUINFO pCpu, PVM pVM, void *pvPageCpu, RTHCPHYS pPageCpuPhys)
107	{
108	AssertReturn(pPageCpuPhys, VERR_INVALID_PARAMETER);
109	AssertReturn(pvPageCpu, VERR_INVALID_PARAMETER);
110
111	if (pVM)
112	{
113	/* Set revision dword at the beginning of the VMXON structure. */
114	(uint32_t )pvPageCpu = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hwaccm.s.vmx.msr.vmx_basic_info);
115	}
116
117	/** @todo we should unmap the two pages from the virtual address space in order to prevent accidental corruption.
118	* (which can have very bad consequences!!!)
119	*/
120
121	if (ASMGetCR4() & X86_CR4_VMXE)
122	return VERR_VMX_IN_VMX_ROOT_MODE;
123
124	/* Make sure the VMX instructions don't cause #UD faults. */
125	ASMSetCR4(ASMGetCR4() \| X86_CR4_VMXE);
126
127	/* Enter VMX Root Mode */
128	int rc = VMXEnable(pPageCpuPhys);
129	if (RT_FAILURE(rc))
130	{
131	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
132	return VERR_VMX_VMXON_FAILED;
133	}
134	return VINF_SUCCESS;
135	}
136
137	/**
138	* Deactivates VT-x on the current CPU
139	*
140	* @returns VBox status code.
141	* @param pCpu CPU info struct
142	* @param pvPageCpu Pointer to the global cpu page
143	* @param pPageCpuPhys Physical address of the global cpu page
144	*/
145	VMMR0DECL(int) VMXR0DisableCpu(PHWACCM_CPUINFO pCpu, void *pvPageCpu, RTHCPHYS pPageCpuPhys)
146	{
147	AssertReturn(pPageCpuPhys, VERR_INVALID_PARAMETER);
148	AssertReturn(pvPageCpu, VERR_INVALID_PARAMETER);
149
150	/* If we're somehow not in VMX root mode, then we shouldn't dare leaving it. */
151	if (!(ASMGetCR4() & X86_CR4_VMXE))
152	return VERR_VMX_NOT_IN_VMX_ROOT_MODE;
153
154	/* Leave VMX Root Mode. */
155	VMXDisable();
156
157	/* And clear the X86_CR4_VMXE bit */
158	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
159	return VINF_SUCCESS;
160	}
161
162	/**
163	* Does Ring-0 per VM VT-x init.
164	*
165	* @returns VBox status code.
166	* @param pVM The VM to operate on.
167	*/
168	VMMR0DECL(int) VMXR0InitVM(PVM pVM)
169	{
170	int rc;
171
172	#ifdef LOG_ENABLED
173	SUPR0Printf("VMXR0InitVM %x\n", pVM);
174	#endif
175
176	pVM->hwaccm.s.vmx.pMemObjAPIC = NIL_RTR0MEMOBJ;
177
178	if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
179	{
180	/* Allocate one page for the APIC physical page (serves for filtering accesses). */
181	rc = RTR0MemObjAllocCont(&pVM->hwaccm.s.vmx.pMemObjAPIC, 1 << PAGE_SHIFT, true /* executable R0 mapping */);
182	AssertRC(rc);
183	if (RT_FAILURE(rc))
184	return rc;
185
186	pVM->hwaccm.s.vmx.pAPIC = (uint8_t *)RTR0MemObjAddress(pVM->hwaccm.s.vmx.pMemObjAPIC);
187	pVM->hwaccm.s.vmx.pAPICPhys = RTR0MemObjGetPagePhysAddr(pVM->hwaccm.s.vmx.pMemObjAPIC, 0);
188	ASMMemZero32(pVM->hwaccm.s.vmx.pAPIC, PAGE_SIZE);
189	}
190	else
191	{
192	pVM->hwaccm.s.vmx.pMemObjAPIC = 0;
193	pVM->hwaccm.s.vmx.pAPIC = 0;
194	pVM->hwaccm.s.vmx.pAPICPhys = 0;
195	}
196
197	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
198	{
199	rc = RTR0MemObjAllocCont(&pVM->hwaccm.s.vmx.pMemObjScratch, 1 << PAGE_SHIFT, true /* executable R0 mapping */);
200	AssertRC(rc);
201	if (RT_FAILURE(rc))
202	return rc;
203
204	pVM->hwaccm.s.vmx.pScratch = (uint8_t *)RTR0MemObjAddress(pVM->hwaccm.s.vmx.pMemObjScratch);
205	pVM->hwaccm.s.vmx.pScratchPhys = RTR0MemObjGetPagePhysAddr(pVM->hwaccm.s.vmx.pMemObjScratch, 0);
206
207	ASMMemZero32(pVM->hwaccm.s.vmx.pScratch, PAGE_SIZE);
208	strcpy((char *)pVM->hwaccm.s.vmx.pScratch, "SCRATCH Magic");
209	(uint64_t )(pVM->hwaccm.s.vmx.pScratch + 16) = UINT64_C(0xDEADBEEFDEADBEEF);
210	}
211	#endif
212
213	/* Allocate VMCBs for all guest CPUs. */
214	for (VMCPUID i = 0; i < pVM->cCpus; i++)
215	{
216	PVMCPU pVCpu = &pVM->aCpus[i];
217
218	pVCpu->hwaccm.s.vmx.pMemObjVMCS = NIL_RTR0MEMOBJ;
219
220	/* Allocate one page for the VM control structure (VMCS). */
221	rc = RTR0MemObjAllocCont(&pVCpu->hwaccm.s.vmx.pMemObjVMCS, 1 << PAGE_SHIFT, true /* executable R0 mapping */);
222	AssertRC(rc);
223	if (RT_FAILURE(rc))
224	return rc;
225
226	pVCpu->hwaccm.s.vmx.pVMCS = RTR0MemObjAddress(pVCpu->hwaccm.s.vmx.pMemObjVMCS);
227	pVCpu->hwaccm.s.vmx.pVMCSPhys = RTR0MemObjGetPagePhysAddr(pVCpu->hwaccm.s.vmx.pMemObjVMCS, 0);
228	ASMMemZero32(pVCpu->hwaccm.s.vmx.pVMCS, PAGE_SIZE);
229
230	pVCpu->hwaccm.s.vmx.cr0_mask = 0;
231	pVCpu->hwaccm.s.vmx.cr4_mask = 0;
232
233	/* Allocate one page for the virtual APIC page for TPR caching. */
234	rc = RTR0MemObjAllocCont(&pVCpu->hwaccm.s.vmx.pMemObjVAPIC, 1 << PAGE_SHIFT, true /* executable R0 mapping */);
235	AssertRC(rc);
236	if (RT_FAILURE(rc))
237	return rc;
238
239	pVCpu->hwaccm.s.vmx.pVAPIC = (uint8_t *)RTR0MemObjAddress(pVCpu->hwaccm.s.vmx.pMemObjVAPIC);
240	pVCpu->hwaccm.s.vmx.pVAPICPhys = RTR0MemObjGetPagePhysAddr(pVCpu->hwaccm.s.vmx.pMemObjVAPIC, 0);
241	ASMMemZero32(pVCpu->hwaccm.s.vmx.pVAPIC, PAGE_SIZE);
242
243	/* Allocate the MSR bitmap if this feature is supported. */
244	if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
245	{
246	rc = RTR0MemObjAllocCont(&pVCpu->hwaccm.s.vmx.pMemObjMSRBitmap, 1 << PAGE_SHIFT, true /* executable R0 mapping */);
247	AssertRC(rc);
248	if (RT_FAILURE(rc))
249	return rc;
250
251	pVCpu->hwaccm.s.vmx.pMSRBitmap = (uint8_t *)RTR0MemObjAddress(pVCpu->hwaccm.s.vmx.pMemObjMSRBitmap);
252	pVCpu->hwaccm.s.vmx.pMSRBitmapPhys = RTR0MemObjGetPagePhysAddr(pVCpu->hwaccm.s.vmx.pMemObjMSRBitmap, 0);
253	memset(pVCpu->hwaccm.s.vmx.pMSRBitmap, 0xff, PAGE_SIZE);
254	}
255
256	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
257	/* Allocate one page for the guest MSR load area (for preloading guest MSRs during the world switch). */
258	rc = RTR0MemObjAllocCont(&pVCpu->hwaccm.s.vmx.pMemObjGuestMSR, 1 << PAGE_SHIFT, true /* executable R0 mapping */);
259	AssertRC(rc);
260	if (RT_FAILURE(rc))
261	return rc;
262
263	pVCpu->hwaccm.s.vmx.pGuestMSR = (uint8_t *)RTR0MemObjAddress(pVCpu->hwaccm.s.vmx.pMemObjGuestMSR);
264	pVCpu->hwaccm.s.vmx.pGuestMSRPhys = RTR0MemObjGetPagePhysAddr(pVCpu->hwaccm.s.vmx.pMemObjGuestMSR, 0);
265	memset(pVCpu->hwaccm.s.vmx.pGuestMSR, 0, PAGE_SIZE);
266
267	/* Allocate one page for the host MSR load area (for restoring host MSRs after the world switch back). */
268	rc = RTR0MemObjAllocCont(&pVCpu->hwaccm.s.vmx.pMemObjHostMSR, 1 << PAGE_SHIFT, true /* executable R0 mapping */);
269	AssertRC(rc);
270	if (RT_FAILURE(rc))
271	return rc;
272
273	pVCpu->hwaccm.s.vmx.pHostMSR = (uint8_t *)RTR0MemObjAddress(pVCpu->hwaccm.s.vmx.pMemObjHostMSR);
274	pVCpu->hwaccm.s.vmx.pHostMSRPhys = RTR0MemObjGetPagePhysAddr(pVCpu->hwaccm.s.vmx.pMemObjHostMSR, 0);
275	memset(pVCpu->hwaccm.s.vmx.pHostMSR, 0, PAGE_SIZE);
276	#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
277
278	/* Current guest paging mode. */
279	pVCpu->hwaccm.s.vmx.enmLastSeenGuestMode = PGMMODE_REAL;
280
281	#ifdef LOG_ENABLED
282	SUPR0Printf("VMXR0InitVM %x VMCS=%x (%x)\n", pVM, pVCpu->hwaccm.s.vmx.pVMCS, (uint32_t)pVCpu->hwaccm.s.vmx.pVMCSPhys);
283	#endif
284	}
285
286	return VINF_SUCCESS;
287	}
288
289	/**
290	* Does Ring-0 per VM VT-x termination.
291	*
292	* @returns VBox status code.
293	* @param pVM The VM to operate on.
294	*/
295	VMMR0DECL(int) VMXR0TermVM(PVM pVM)
296	{
297	for (VMCPUID i = 0; i < pVM->cCpus; i++)
298	{
299	PVMCPU pVCpu = &pVM->aCpus[i];
300
301	if (pVCpu->hwaccm.s.vmx.pMemObjVMCS != NIL_RTR0MEMOBJ)
302	{
303	RTR0MemObjFree(pVCpu->hwaccm.s.vmx.pMemObjVMCS, false);
304	pVCpu->hwaccm.s.vmx.pMemObjVMCS = NIL_RTR0MEMOBJ;
305	pVCpu->hwaccm.s.vmx.pVMCS = 0;
306	pVCpu->hwaccm.s.vmx.pVMCSPhys = 0;
307	}
308	if (pVCpu->hwaccm.s.vmx.pMemObjVAPIC != NIL_RTR0MEMOBJ)
309	{
310	RTR0MemObjFree(pVCpu->hwaccm.s.vmx.pMemObjVAPIC, false);
311	pVCpu->hwaccm.s.vmx.pMemObjVAPIC = NIL_RTR0MEMOBJ;
312	pVCpu->hwaccm.s.vmx.pVAPIC = 0;
313	pVCpu->hwaccm.s.vmx.pVAPICPhys = 0;
314	}
315	if (pVCpu->hwaccm.s.vmx.pMemObjMSRBitmap != NIL_RTR0MEMOBJ)
316	{
317	RTR0MemObjFree(pVCpu->hwaccm.s.vmx.pMemObjMSRBitmap, false);
318	pVCpu->hwaccm.s.vmx.pMemObjMSRBitmap = NIL_RTR0MEMOBJ;
319	pVCpu->hwaccm.s.vmx.pMSRBitmap = 0;
320	pVCpu->hwaccm.s.vmx.pMSRBitmapPhys = 0;
321	}
322	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
323	if (pVCpu->hwaccm.s.vmx.pMemObjHostMSR != NIL_RTR0MEMOBJ)
324	{
325	RTR0MemObjFree(pVCpu->hwaccm.s.vmx.pMemObjHostMSR, false);
326	pVCpu->hwaccm.s.vmx.pMemObjHostMSR = NIL_RTR0MEMOBJ;
327	pVCpu->hwaccm.s.vmx.pHostMSR = 0;
328	pVCpu->hwaccm.s.vmx.pHostMSRPhys = 0;
329	}
330	if (pVCpu->hwaccm.s.vmx.pMemObjGuestMSR != NIL_RTR0MEMOBJ)
331	{
332	RTR0MemObjFree(pVCpu->hwaccm.s.vmx.pMemObjGuestMSR, false);
333	pVCpu->hwaccm.s.vmx.pMemObjGuestMSR = NIL_RTR0MEMOBJ;
334	pVCpu->hwaccm.s.vmx.pGuestMSR = 0;
335	pVCpu->hwaccm.s.vmx.pGuestMSRPhys = 0;
336	}
337	#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
338	}
339	if (pVM->hwaccm.s.vmx.pMemObjAPIC != NIL_RTR0MEMOBJ)
340	{
341	RTR0MemObjFree(pVM->hwaccm.s.vmx.pMemObjAPIC, false);
342	pVM->hwaccm.s.vmx.pMemObjAPIC = NIL_RTR0MEMOBJ;
343	pVM->hwaccm.s.vmx.pAPIC = 0;
344	pVM->hwaccm.s.vmx.pAPICPhys = 0;
345	}
346	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
347	if (pVM->hwaccm.s.vmx.pMemObjScratch != NIL_RTR0MEMOBJ)
348	{
349	ASMMemZero32(pVM->hwaccm.s.vmx.pScratch, PAGE_SIZE);
350	RTR0MemObjFree(pVM->hwaccm.s.vmx.pMemObjScratch, false);
351	pVM->hwaccm.s.vmx.pMemObjScratch = NIL_RTR0MEMOBJ;
352	pVM->hwaccm.s.vmx.pScratch = 0;
353	pVM->hwaccm.s.vmx.pScratchPhys = 0;
354	}
355	#endif
356	return VINF_SUCCESS;
357	}
358
359	/**
360	* Sets up VT-x for the specified VM
361	*
362	* @returns VBox status code.
363	* @param pVM The VM to operate on.
364	*/
365	VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
366	{
367	int rc = VINF_SUCCESS;
368	uint32_t val;
369
370	AssertReturn(pVM, VERR_INVALID_PARAMETER);
371
372	for (VMCPUID i = 0; i < pVM->cCpus; i++)
373	{
374	PVMCPU pVCpu = &pVM->aCpus[i];
375
376	Assert(pVCpu->hwaccm.s.vmx.pVMCS);
377
378	/* Set revision dword at the beginning of the VMCS structure. */
379	(uint32_t )pVCpu->hwaccm.s.vmx.pVMCS = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hwaccm.s.vmx.msr.vmx_basic_info);
380
381	/* Clear VM Control Structure. */
382	Log(("pVMCSPhys = %RHp\n", pVCpu->hwaccm.s.vmx.pVMCSPhys));
383	rc = VMXClearVMCS(pVCpu->hwaccm.s.vmx.pVMCSPhys);
384	if (RT_FAILURE(rc))
385	goto vmx_end;
386
387	/* Activate the VM Control Structure. */
388	rc = VMXActivateVMCS(pVCpu->hwaccm.s.vmx.pVMCSPhys);
389	if (RT_FAILURE(rc))
390	goto vmx_end;
391
392	/* VMX_VMCS_CTRL_PIN_EXEC_CONTROLS
393	* Set required bits to one and zero according to the MSR capabilities.
394	*/
395	val = pVM->hwaccm.s.vmx.msr.vmx_pin_ctls.n.disallowed0;
396	/* External and non-maskable interrupts cause VM-exits. */
397	val \|= VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_EXT_INT_EXIT \| VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_NMI_EXIT;
398	/* enable the preemption timer. */
399	if (pVM->hwaccm.s.vmx.fUsePreemptTimer)
400	val \|= VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_PREEMPT_TIMER;
401	val &= pVM->hwaccm.s.vmx.msr.vmx_pin_ctls.n.allowed1;
402
403	rc = VMXWriteVMCS(VMX_VMCS_CTRL_PIN_EXEC_CONTROLS, val);
404	AssertRC(rc);
405
406	/* VMX_VMCS_CTRL_PROC_EXEC_CONTROLS
407	* Set required bits to one and zero according to the MSR capabilities.
408	*/
409	val = pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.disallowed0;
410	/* Program which event cause VM-exits and which features we want to use. */
411	val = val \| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_HLT_EXIT
412	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_TSC_OFFSET
413	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT
414	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_UNCOND_IO_EXIT
415	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDPMC_EXIT
416	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_EXIT
417	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MWAIT_EXIT; /* don't execute mwait or else we'll idle inside the guest (host thinks the cpu load is high) */
418
419	/* Without nested paging we should intercept invlpg and cr3 mov instructions. */
420	if (!pVM->hwaccm.s.fNestedPaging)
421	val \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INVLPG_EXIT
422	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
423	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT;
424
425	/* Note: VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MWAIT_EXIT might cause a vmlaunch failure with an invalid control fields error. (combined with some other exit reasons) */
426	if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
427	{
428	/* CR8 reads from the APIC shadow page; writes cause an exit is they lower the TPR below the threshold */
429	val \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW;
430	Assert(pVM->hwaccm.s.vmx.pAPIC);
431	}
432	else
433	/* Exit on CR8 reads & writes in case the TPR shadow feature isn't present. */
434	val \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR8_STORE_EXIT \| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR8_LOAD_EXIT;
435
436	if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
437	{
438	Assert(pVCpu->hwaccm.s.vmx.pMSRBitmapPhys);
439	val \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS;
440	}
441
442	/* We will use the secondary control if it's present. */
443	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
444
445	/* Mask away the bits that the CPU doesn't support */
446	/** @todo make sure they don't conflict with the above requirements. */
447	val &= pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1;
448	pVCpu->hwaccm.s.vmx.proc_ctls = val;
449
450	rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, val);
451	AssertRC(rc);
452
453	if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
454	{
455	/* VMX_VMCS_CTRL_PROC_EXEC_CONTROLS2
456	* Set required bits to one and zero according to the MSR capabilities.
457	*/
458	val = pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2.n.disallowed0;
459	val \|= VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT;
460
461	#ifdef HWACCM_VTX_WITH_EPT
462	if (pVM->hwaccm.s.fNestedPaging)
463	val \|= VMX_VMCS_CTRL_PROC_EXEC2_EPT;
464	#endif /* HWACCM_VTX_WITH_EPT */
465	#ifdef HWACCM_VTX_WITH_VPID
466	else
467	if (pVM->hwaccm.s.vmx.fVPID)
468	val \|= VMX_VMCS_CTRL_PROC_EXEC2_VPID;
469	#endif /* HWACCM_VTX_WITH_VPID */
470
471	if (pVM->hwaccm.s.fHasIoApic)
472	val \|= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC;
473
474	if (pVM->hwaccm.s.vmx.fUnrestrictedGuest)
475	val \|= VMX_VMCS_CTRL_PROC_EXEC2_REAL_MODE;
476
477	/* Mask away the bits that the CPU doesn't support */
478	/** @todo make sure they don't conflict with the above requirements. */
479	val &= pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2.n.allowed1;
480	pVCpu->hwaccm.s.vmx.proc_ctls2 = val;
481	rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS2, val);
482	AssertRC(rc);
483	}
484
485	/* VMX_VMCS_CTRL_CR3_TARGET_COUNT
486	* Set required bits to one and zero according to the MSR capabilities.
487	*/
488	rc = VMXWriteVMCS(VMX_VMCS_CTRL_CR3_TARGET_COUNT, 0);
489	AssertRC(rc);
490
491	/* Forward all exception except #NM & #PF to the guest.
492	* We always need to check pagefaults since our shadow page table can be out of sync.
493	* And we always lazily sync the FPU & XMM state.
494	*/
495
496	/** @todo Possible optimization:
497	* Keep the FPU and XMM state current in the EM thread. That way there's no need to
498	* lazily sync anything, but the downside is that we can't use the FPU stack or XMM
499	* registers ourselves of course.
500	*
501	* Note: only possible if the current state is actually ours (X86_CR0_TS flag)
502	*/
503
504	/* Don't filter page faults; all of them should cause a switch. */
505	rc = VMXWriteVMCS(VMX_VMCS_CTRL_PAGEFAULT_ERROR_MASK, 0);
506	rc \|= VMXWriteVMCS(VMX_VMCS_CTRL_PAGEFAULT_ERROR_MATCH, 0);
507	AssertRC(rc);
508
509	/* Init TSC offset to zero. */
510	rc = VMXWriteVMCS64(VMX_VMCS_CTRL_TSC_OFFSET_FULL, 0);
511	AssertRC(rc);
512
513	rc = VMXWriteVMCS64(VMX_VMCS_CTRL_IO_BITMAP_A_FULL, 0);
514	AssertRC(rc);
515
516	rc = VMXWriteVMCS64(VMX_VMCS_CTRL_IO_BITMAP_B_FULL, 0);
517	AssertRC(rc);
518
519	/* Set the MSR bitmap address. */
520	if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
521	{
522	Assert(pVCpu->hwaccm.s.vmx.pMSRBitmapPhys);
523
524	rc = VMXWriteVMCS64(VMX_VMCS_CTRL_MSR_BITMAP_FULL, pVCpu->hwaccm.s.vmx.pMSRBitmapPhys);
525	AssertRC(rc);
526
527	/* Allow the guest to directly modify these MSRs; they are restored and saved automatically. */
528	vmxR0SetMSRPermission(pVCpu, MSR_IA32_SYSENTER_CS, true, true);
529	vmxR0SetMSRPermission(pVCpu, MSR_IA32_SYSENTER_ESP, true, true);
530	vmxR0SetMSRPermission(pVCpu, MSR_IA32_SYSENTER_EIP, true, true);
531	vmxR0SetMSRPermission(pVCpu, MSR_K8_LSTAR, true, true);
532	vmxR0SetMSRPermission(pVCpu, MSR_K6_STAR, true, true);
533	vmxR0SetMSRPermission(pVCpu, MSR_K8_SF_MASK, true, true);
534	vmxR0SetMSRPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, true, true);
535	vmxR0SetMSRPermission(pVCpu, MSR_K8_GS_BASE, true, true);
536	vmxR0SetMSRPermission(pVCpu, MSR_K8_FS_BASE, true, true);
537	}
538
539	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
540	/* Set the guest & host MSR load/store physical addresses. */
541	Assert(pVCpu->hwaccm.s.vmx.pGuestMSRPhys);
542	rc = VMXWriteVMCS64(VMX_VMCS_CTRL_VMENTRY_MSR_LOAD_FULL, pVCpu->hwaccm.s.vmx.pGuestMSRPhys);
543	AssertRC(rc);
544	rc = VMXWriteVMCS64(VMX_VMCS_CTRL_VMEXIT_MSR_STORE_FULL, pVCpu->hwaccm.s.vmx.pGuestMSRPhys);
545	AssertRC(rc);
546
547	Assert(pVCpu->hwaccm.s.vmx.pHostMSRPhys);
548	rc = VMXWriteVMCS64(VMX_VMCS_CTRL_VMEXIT_MSR_LOAD_FULL, pVCpu->hwaccm.s.vmx.pHostMSRPhys);
549	AssertRC(rc);
550	#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
551
552	rc = VMXWriteVMCS(VMX_VMCS_CTRL_ENTRY_MSR_LOAD_COUNT, 0);
553	AssertRC(rc);
554
555	rc = VMXWriteVMCS(VMX_VMCS_CTRL_EXIT_MSR_STORE_COUNT, 0);
556	AssertRC(rc);
557
558	if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
559	{
560	Assert(pVM->hwaccm.s.vmx.pMemObjAPIC);
561	/* Optional */
562	rc = VMXWriteVMCS(VMX_VMCS_CTRL_TPR_THRESHOLD, 0);
563	rc \|= VMXWriteVMCS64(VMX_VMCS_CTRL_VAPIC_PAGEADDR_FULL, pVCpu->hwaccm.s.vmx.pVAPICPhys);
564
565	if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
566	rc \|= VMXWriteVMCS64(VMX_VMCS_CTRL_APIC_ACCESSADDR_FULL, pVM->hwaccm.s.vmx.pAPICPhys);
567
568	AssertRC(rc);
569	}
570
571	/* Set link pointer to -1. Not currently used. */
572	rc = VMXWriteVMCS64(VMX_VMCS_GUEST_LINK_PTR_FULL, 0xFFFFFFFFFFFFFFFFULL);
573	AssertRC(rc);
574
575	/* Clear VM Control Structure. Marking it inactive, clearing implementation specific data and writing back VMCS data to memory. */
576	rc = VMXClearVMCS(pVCpu->hwaccm.s.vmx.pVMCSPhys);
577	AssertRC(rc);
578
579	/* Configure the VMCS read cache. */
580	PVMCSCACHE pCache = &pVCpu->hwaccm.s.vmx.VMCSCache;
581
582	VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_RIP);
583	VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_RSP);
584	VMXSetupCachedReadVMCS(pCache, VMX_VMCS_GUEST_RFLAGS);
585	VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE);
586	VMXSetupCachedReadVMCS(pCache, VMX_VMCS_CTRL_CR0_READ_SHADOW);
587	VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_CR0);
588	VMXSetupCachedReadVMCS(pCache, VMX_VMCS_CTRL_CR4_READ_SHADOW);
589	VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_CR4);
590	VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_DR7);
591	VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_GUEST_SYSENTER_CS);
592	VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_SYSENTER_EIP);
593	VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_SYSENTER_ESP);
594	VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_GUEST_GDTR_LIMIT);
595	VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_GDTR_BASE);
596	VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_GUEST_IDTR_LIMIT);
597	VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_IDTR_BASE);
598
599	VMX_SETUP_SELREG(ES, pCache);
600	VMX_SETUP_SELREG(SS, pCache);
601	VMX_SETUP_SELREG(CS, pCache);
602	VMX_SETUP_SELREG(DS, pCache);
603	VMX_SETUP_SELREG(FS, pCache);
604	VMX_SETUP_SELREG(GS, pCache);
605	VMX_SETUP_SELREG(LDTR, pCache);
606	VMX_SETUP_SELREG(TR, pCache);
607
608	/* Status code VMCS reads. */
609	VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_EXIT_REASON);
610	VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_VM_INSTR_ERROR);
611	VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_EXIT_INSTR_LENGTH);
612	VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_EXIT_INTERRUPTION_ERRCODE);
613	VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO);
614	VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_EXIT_INSTR_INFO);
615	VMXSetupCachedReadVMCS(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
616	VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_IDT_INFO);
617	VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_IDT_ERRCODE);
618
619	if (pVM->hwaccm.s.fNestedPaging)
620	{
621	VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_CR3);
622	VMXSetupCachedReadVMCS(pCache, VMX_VMCS_EXIT_PHYS_ADDR_FULL);
623	pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
624	}
625	else
626	pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
627	} /* for each VMCPU */
628
629	/* Choose the right TLB setup function. */
630	if (pVM->hwaccm.s.fNestedPaging)
631	{
632	pVM->hwaccm.s.vmx.pfnSetupTaggedTLB = vmxR0SetupTLBEPT;
633
634	/* Default values for flushing. */
635	pVM->hwaccm.s.vmx.enmFlushPage = VMX_FLUSH_ALL_CONTEXTS;
636	pVM->hwaccm.s.vmx.enmFlushContext = VMX_FLUSH_ALL_CONTEXTS;
637
638	/* If the capabilities specify we can do more, then make use of it. */
639	if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVEPT_CAPS_INDIV)
640	pVM->hwaccm.s.vmx.enmFlushPage = VMX_FLUSH_PAGE;
641	else
642	if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVEPT_CAPS_CONTEXT)
643	pVM->hwaccm.s.vmx.enmFlushPage = VMX_FLUSH_SINGLE_CONTEXT;
644
645	if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVEPT_CAPS_CONTEXT)
646	pVM->hwaccm.s.vmx.enmFlushContext = VMX_FLUSH_SINGLE_CONTEXT;
647	}
648	#ifdef HWACCM_VTX_WITH_VPID
649	else
650	if (pVM->hwaccm.s.vmx.fVPID)
651	{
652	pVM->hwaccm.s.vmx.pfnSetupTaggedTLB = vmxR0SetupTLBVPID;
653
654	/* Default values for flushing. */
655	pVM->hwaccm.s.vmx.enmFlushPage = VMX_FLUSH_ALL_CONTEXTS;
656	pVM->hwaccm.s.vmx.enmFlushContext = VMX_FLUSH_ALL_CONTEXTS;
657
658	/* If the capabilities specify we can do more, then make use of it. */
659	if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVVPID_CAPS_INDIV)
660	pVM->hwaccm.s.vmx.enmFlushPage = VMX_FLUSH_PAGE;
661	else
662	if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVVPID_CAPS_CONTEXT)
663	pVM->hwaccm.s.vmx.enmFlushPage = VMX_FLUSH_SINGLE_CONTEXT;
664
665	if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVVPID_CAPS_CONTEXT)
666	pVM->hwaccm.s.vmx.enmFlushContext = VMX_FLUSH_SINGLE_CONTEXT;
667	}
668	#endif /* HWACCM_VTX_WITH_VPID */
669	else
670	pVM->hwaccm.s.vmx.pfnSetupTaggedTLB = vmxR0SetupTLBDummy;
671
672	vmx_end:
673	VMXR0CheckError(pVM, &pVM->aCpus[0], rc);
674	return rc;
675	}
676
677	/**
678	* Sets the permission bits for the specified MSR
679	*
680	* @param pVCpu The VMCPU to operate on.
681	* @param ulMSR MSR value
682	* @param fRead Reading allowed/disallowed
683	* @param fWrite Writing allowed/disallowed
684	*/
685	static void vmxR0SetMSRPermission(PVMCPU pVCpu, unsigned ulMSR, bool fRead, bool fWrite)
686	{
687	unsigned ulBit;
688	uint8_t pMSRBitmap = (uint8_t )pVCpu->hwaccm.s.vmx.pMSRBitmap;
689
690	/* Layout:
691	* 0x000 - 0x3ff - Low MSR read bits
692	* 0x400 - 0x7ff - High MSR read bits
693	* 0x800 - 0xbff - Low MSR write bits
694	* 0xc00 - 0xfff - High MSR write bits
695	*/
696	if (ulMSR <= 0x00001FFF)
697	{
698	/* Pentium-compatible MSRs */
699	ulBit = ulMSR;
700	}
701	else
702	if ( ulMSR >= 0xC0000000
703	&& ulMSR <= 0xC0001FFF)
704	{
705	/* AMD Sixth Generation x86 Processor MSRs */
706	ulBit = (ulMSR - 0xC0000000);
707	pMSRBitmap += 0x400;
708	}
709	else
710	{
711	AssertFailed();
712	return;
713	}
714
715	Assert(ulBit <= 0x1fff);
716	if (fRead)
717	ASMBitClear(pMSRBitmap, ulBit);
718	else
719	ASMBitSet(pMSRBitmap, ulBit);
720
721	if (fWrite)
722	ASMBitClear(pMSRBitmap + 0x800, ulBit);
723	else
724	ASMBitSet(pMSRBitmap + 0x800, ulBit);
725	}
726
727
728	/**
729	* Injects an event (trap or external interrupt)
730	*
731	* @returns VBox status code. Note that it may return VINF_EM_RESET to
732	* indicate a triple fault when injecting X86_XCPT_DF.
733	*
734	* @param pVM The VM to operate on.
735	* @param pVCpu The VMCPU to operate on.
736	* @param pCtx CPU Context
737	* @param intInfo VMX interrupt info
738	* @param cbInstr Opcode length of faulting instruction
739	* @param errCode Error code (optional)
740	*/
741	static int VMXR0InjectEvent(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, uint32_t intInfo, uint32_t cbInstr, uint32_t errCode)
742	{
743	int rc;
744	uint32_t iGate = VMX_EXIT_INTERRUPTION_INFO_VECTOR(intInfo);
745
746	#ifdef VBOX_WITH_STATISTICS
747	STAM_COUNTER_INC(&pVCpu->hwaccm.s.paStatInjectedIrqsR0[iGate & MASK_INJECT_IRQ_STAT]);
748	#endif
749
750	#ifdef VBOX_STRICT
751	if (iGate == 0xE)
752	LogFlow(("VMXR0InjectEvent: Injecting interrupt %d at %RGv error code=%08x CR2=%RGv intInfo=%08x\n", iGate, (RTGCPTR)pCtx->rip, errCode, pCtx->cr2, intInfo));
753	else
754	if (iGate < 0x20)
755	LogFlow(("VMXR0InjectEvent: Injecting interrupt %d at %RGv error code=%08x\n", iGate, (RTGCPTR)pCtx->rip, errCode));
756	else
757	{
758	LogFlow(("INJ-EI: %x at %RGv\n", iGate, (RTGCPTR)pCtx->rip));
759	Assert(VMX_EXIT_INTERRUPTION_INFO_TYPE(intInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW \|\| !VMCPU_FF_ISSET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
760	Assert(VMX_EXIT_INTERRUPTION_INFO_TYPE(intInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW \|\| pCtx->eflags.u32 & X86_EFL_IF);
761	}
762	#endif
763
764	if ( CPUMIsGuestInRealModeEx(pCtx)
765	&& pVM->hwaccm.s.vmx.pRealModeTSS)
766	{
767	RTGCPHYS GCPhysHandler;
768	uint16_t offset, ip;
769	RTSEL sel;
770
771	/* Injecting events doesn't work right with real mode emulation.
772	* (#GP if we try to inject external hardware interrupts)
773	* Inject the interrupt or trap directly instead.
774	*
775	* ASSUMES no access handlers for the bits we read or write below (should be safe).
776	*/
777	Log(("Manual interrupt/trap '%x' inject (real mode)\n", iGate));
778
779	/* Check if the interrupt handler is present. */
780	if (iGate * 4 + 3 > pCtx->idtr.cbIdt)
781	{
782	Log(("IDT cbIdt violation\n"));
783	if (iGate != X86_XCPT_DF)
784	{
785	uint32_t intInfo2;
786
787	intInfo2 = (iGate == X86_XCPT_GP) ? (uint32_t)X86_XCPT_DF : iGate;
788	intInfo2 \|= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
789	intInfo2 \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
790	intInfo2 \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HWEXCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
791
792	return VMXR0InjectEvent(pVM, pVCpu, pCtx, intInfo2, 0, 0 /* no error code according to the Intel docs */);
793	}
794	Log(("Triple fault -> reset the VM!\n"));
795	return VINF_EM_RESET;
796	}
797	if ( VMX_EXIT_INTERRUPTION_INFO_TYPE(intInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW
798	\|\| iGate == 3 /* Both #BP and #OF point to the instruction after. */
799	\|\| iGate == 4)
800	{
801	ip = pCtx->ip + cbInstr;
802	}
803	else
804	ip = pCtx->ip;
805
806	/* Read the selector:offset pair of the interrupt handler. */
807	GCPhysHandler = (RTGCPHYS)pCtx->idtr.pIdt + iGate * 4;
808	rc = PGMPhysSimpleReadGCPhys(pVM, &offset, GCPhysHandler, sizeof(offset)); AssertRC(rc);
809	rc = PGMPhysSimpleReadGCPhys(pVM, &sel, GCPhysHandler + 2, sizeof(sel)); AssertRC(rc);
810
811	LogFlow(("IDT handler %04X:%04X\n", sel, offset));
812
813	/* Construct the stack frame. */
814	/** @todo should check stack limit. */
815	pCtx->sp -= 2;
816	LogFlow(("ss:sp %04X:%04X eflags=%x\n", pCtx->ss, pCtx->sp, pCtx->eflags.u));
817	rc = PGMPhysSimpleWriteGCPhys(pVM, pCtx->ssHid.u64Base + pCtx->sp, &pCtx->eflags, sizeof(uint16_t)); AssertRC(rc);
818	pCtx->sp -= 2;
819	LogFlow(("ss:sp %04X:%04X cs=%x\n", pCtx->ss, pCtx->sp, pCtx->cs));
820	rc = PGMPhysSimpleWriteGCPhys(pVM, pCtx->ssHid.u64Base + pCtx->sp, &pCtx->cs, sizeof(uint16_t)); AssertRC(rc);
821	pCtx->sp -= 2;
822	LogFlow(("ss:sp %04X:%04X ip=%x\n", pCtx->ss, pCtx->sp, ip));
823	rc = PGMPhysSimpleWriteGCPhys(pVM, pCtx->ssHid.u64Base + pCtx->sp, &ip, sizeof(ip)); AssertRC(rc);
824
825	/* Update the CPU state for executing the handler. */
826	pCtx->rip = offset;
827	pCtx->cs = sel;
828	pCtx->csHid.u64Base = sel << 4;
829	pCtx->eflags.u &= ~(X86_EFL_IF\|X86_EFL_TF\|X86_EFL_RF\|X86_EFL_AC);
830
831	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_GUEST_SEGMENT_REGS;
832	return VINF_SUCCESS;
833	}
834
835	/* Set event injection state. */
836	rc = VMXWriteVMCS(VMX_VMCS_CTRL_ENTRY_IRQ_INFO, intInfo \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT));
837
838	rc \|= VMXWriteVMCS(VMX_VMCS_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
839	rc \|= VMXWriteVMCS(VMX_VMCS_CTRL_ENTRY_EXCEPTION_ERRCODE, errCode);
840
841	AssertRC(rc);
842	return rc;
843	}
844
845
846	/**
847	* Checks for pending guest interrupts and injects them
848	*
849	* @returns VBox status code.
850	* @param pVM The VM to operate on.
851	* @param pVCpu The VMCPU to operate on.
852	* @param pCtx CPU Context
853	*/
854	static int VMXR0CheckPendingInterrupt(PVM pVM, PVMCPU pVCpu, CPUMCTX *pCtx)
855	{
856	int rc;
857
858	/* Dispatch any pending interrupts. (injected before, but a VM exit occurred prematurely) */
859	if (pVCpu->hwaccm.s.Event.fPending)
860	{
861	Log(("CPU%d: Reinjecting event %RX64 %08x at %RGv cr2=%RX64\n", pVCpu->idCpu, pVCpu->hwaccm.s.Event.intInfo, pVCpu->hwaccm.s.Event.errCode, (RTGCPTR)pCtx->rip, pCtx->cr2));
862	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatIntReinject);
863	rc = VMXR0InjectEvent(pVM, pVCpu, pCtx, pVCpu->hwaccm.s.Event.intInfo, 0, pVCpu->hwaccm.s.Event.errCode);
864	AssertRC(rc);
865
866	pVCpu->hwaccm.s.Event.fPending = false;
867	return VINF_SUCCESS;
868	}
869
870	/* If an active trap is already pending, then we must forward it first! */
871	if (!TRPMHasTrap(pVCpu))
872	{
873	if (VMCPU_FF_TESTANDCLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI))
874	{
875	RTGCUINTPTR intInfo;
876
877	Log(("CPU%d: injecting #NMI\n", pVCpu->idCpu));
878
879	intInfo = X86_XCPT_NMI;
880	intInfo \|= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
881	intInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
882
883	rc = VMXR0InjectEvent(pVM, pVCpu, pCtx, intInfo, 0, 0);
884	AssertRC(rc);
885
886	return VINF_SUCCESS;
887	}
888
889	/* @todo SMI interrupts. */
890
891	/* When external interrupts are pending, we should exit the VM when IF is set. */
892	if (VMCPU_FF_ISPENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC\|VMCPU_FF_INTERRUPT_PIC)))
893	{
894	if (!(pCtx->eflags.u32 & X86_EFL_IF))
895	{
896	if (!(pVCpu->hwaccm.s.vmx.proc_ctls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_IRQ_WINDOW_EXIT))
897	{
898	LogFlow(("Enable irq window exit!\n"));
899	pVCpu->hwaccm.s.vmx.proc_ctls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_IRQ_WINDOW_EXIT;
900	rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
901	AssertRC(rc);
902	}
903	/* else nothing to do but wait */
904	}
905	else
906	if (!VMCPU_FF_ISSET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
907	{
908	uint8_t u8Interrupt;
909
910	rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
911	Log(("CPU%d: Dispatch interrupt: u8Interrupt=%x (%d) rc=%Rrc cs:rip=%04X:%RGv\n", pVCpu->idCpu, u8Interrupt, u8Interrupt, rc, pCtx->cs, (RTGCPTR)pCtx->rip));
912	if (RT_SUCCESS(rc))
913	{
914	rc = TRPMAssertTrap(pVCpu, u8Interrupt, TRPM_HARDWARE_INT);
915	AssertRC(rc);
916	}
917	else
918	{
919	/* Can only happen in rare cases where a pending interrupt is cleared behind our back */
920	Assert(!VMCPU_FF_ISPENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC\|VMCPU_FF_INTERRUPT_PIC)));
921	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatSwitchGuestIrq);
922	/* Just continue */
923	}
924	}
925	else
926	Log(("Pending interrupt blocked at %RGv by VM_FF_INHIBIT_INTERRUPTS!!\n", (RTGCPTR)pCtx->rip));
927	}
928	}
929
930	#ifdef VBOX_STRICT
931	if (TRPMHasTrap(pVCpu))
932	{
933	uint8_t u8Vector;
934	rc = TRPMQueryTrapAll(pVCpu, &u8Vector, 0, 0, 0);
935	AssertRC(rc);
936	}
937	#endif
938
939	if ( (pCtx->eflags.u32 & X86_EFL_IF)
940	&& (!VMCPU_FF_ISSET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
941	&& TRPMHasTrap(pVCpu)
942	)
943	{
944	uint8_t u8Vector;
945	TRPMEVENT enmType;
946	RTGCUINTPTR intInfo;
947	RTGCUINT errCode;
948
949	/* If a new event is pending, then dispatch it now. */
950	rc = TRPMQueryTrapAll(pVCpu, &u8Vector, &enmType, &errCode, 0);
951	AssertRC(rc);
952	Assert(pCtx->eflags.Bits.u1IF == 1 \|\| enmType == TRPM_TRAP);
953	Assert(enmType != TRPM_SOFTWARE_INT);
954
955	/* Clear the pending trap. */
956	rc = TRPMResetTrap(pVCpu);
957	AssertRC(rc);
958
959	intInfo = u8Vector;
960	intInfo \|= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
961
962	if (enmType == TRPM_TRAP)
963	{
964	switch (u8Vector) {
965	case 8:
966	case 10:
967	case 11:
968	case 12:
969	case 13:
970	case 14:
971	case 17:
972	/* Valid error codes. */
973	intInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
974	break;
975	default:
976	break;
977	}
978	if (u8Vector == X86_XCPT_BP \|\| u8Vector == X86_XCPT_OF)
979	intInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SWEXCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
980	else
981	intInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HWEXCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
982	}
983	else
984	intInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
985
986	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatIntInject);
987	rc = VMXR0InjectEvent(pVM, pVCpu, pCtx, intInfo, 0, errCode);
988	AssertRC(rc);
989	} /* if (interrupts can be dispatched) */
990
991	return VINF_SUCCESS;
992	}
993
994	/**
995	* Save the host state
996	*
997	* @returns VBox status code.
998	* @param pVM The VM to operate on.
999	* @param pVCpu The VMCPU to operate on.
1000	*/
1001	VMMR0DECL(int) VMXR0SaveHostState(PVM pVM, PVMCPU pVCpu)
1002	{
1003	int rc = VINF_SUCCESS;
1004
1005	/*
1006	* Host CPU Context
1007	*/
1008	if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_HOST_CONTEXT)
1009	{
1010	RTIDTR idtr;
1011	RTGDTR gdtr;
1012	RTSEL SelTR;
1013	PCX86DESCHC pDesc;
1014	uintptr_t trBase;
1015	RTSEL cs;
1016	RTSEL ss;
1017	uint64_t cr3;
1018
1019	/* Control registers */
1020	rc = VMXWriteVMCS(VMX_VMCS_HOST_CR0, ASMGetCR0());
1021	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
1022	if (VMX_IS_64BIT_HOST_MODE())
1023	{
1024	cr3 = hwaccmR0Get64bitCR3();
1025	rc \|= VMXWriteVMCS64(VMX_VMCS_HOST_CR3, cr3);
1026	}
1027	else
1028	#endif
1029	{
1030	cr3 = ASMGetCR3();
1031	rc \|= VMXWriteVMCS(VMX_VMCS_HOST_CR3, cr3);
1032	}
1033	rc \|= VMXWriteVMCS(VMX_VMCS_HOST_CR4, ASMGetCR4());
1034	AssertRC(rc);
1035	Log2(("VMX_VMCS_HOST_CR0 %08x\n", ASMGetCR0()));
1036	Log2(("VMX_VMCS_HOST_CR3 %08RX64\n", cr3));
1037	Log2(("VMX_VMCS_HOST_CR4 %08x\n", ASMGetCR4()));
1038
1039	/* Selector registers. */
1040	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
1041	if (VMX_IS_64BIT_HOST_MODE())
1042	{
1043	cs = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelCS;
1044	ss = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelSS;
1045	}
1046	else
1047	{
1048	/* sysenter loads LDT cs & ss, VMX doesn't like this. Load the GDT ones (safe). */
1049	cs = (RTSEL)(uintptr_t)&SUPR0AbsKernelCS;
1050	ss = (RTSEL)(uintptr_t)&SUPR0AbsKernelSS;
1051	}
1052	#else
1053	cs = ASMGetCS();
1054	ss = ASMGetSS();
1055	#endif
1056	Assert(!(cs & X86_SEL_LDT)); Assert((cs & X86_SEL_RPL) == 0);
1057	Assert(!(ss & X86_SEL_LDT)); Assert((ss & X86_SEL_RPL) == 0);
1058	rc = VMXWriteVMCS(VMX_VMCS16_HOST_FIELD_CS, cs);
1059	/* Note: VMX is (again) very picky about the RPL of the selectors here; we'll restore them manually. */
1060	rc \|= VMXWriteVMCS(VMX_VMCS16_HOST_FIELD_DS, 0);
1061	rc \|= VMXWriteVMCS(VMX_VMCS16_HOST_FIELD_ES, 0);
1062	#if HC_ARCH_BITS == 32
1063	if (!VMX_IS_64BIT_HOST_MODE())
1064	{
1065	rc \|= VMXWriteVMCS(VMX_VMCS16_HOST_FIELD_FS, 0);
1066	rc \|= VMXWriteVMCS(VMX_VMCS16_HOST_FIELD_GS, 0);
1067	}
1068	#endif
1069	rc \|= VMXWriteVMCS(VMX_VMCS16_HOST_FIELD_SS, ss);
1070	SelTR = ASMGetTR();
1071	rc \|= VMXWriteVMCS(VMX_VMCS16_HOST_FIELD_TR, SelTR);
1072	AssertRC(rc);
1073	Log2(("VMX_VMCS_HOST_FIELD_CS %08x (%08x)\n", cs, ASMGetSS()));
1074	Log2(("VMX_VMCS_HOST_FIELD_DS 00000000 (%08x)\n", ASMGetDS()));
1075	Log2(("VMX_VMCS_HOST_FIELD_ES 00000000 (%08x)\n", ASMGetES()));
1076	Log2(("VMX_VMCS_HOST_FIELD_FS 00000000 (%08x)\n", ASMGetFS()));
1077	Log2(("VMX_VMCS_HOST_FIELD_GS 00000000 (%08x)\n", ASMGetGS()));
1078	Log2(("VMX_VMCS_HOST_FIELD_SS %08x (%08x)\n", ss, ASMGetSS()));
1079	Log2(("VMX_VMCS_HOST_FIELD_TR %08x\n", ASMGetTR()));
1080
1081	/* GDTR & IDTR */
1082	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
1083	if (VMX_IS_64BIT_HOST_MODE())
1084	{
1085	X86XDTR64 gdtr64, idtr64;
1086	hwaccmR0Get64bitGDTRandIDTR(&gdtr64, &idtr64);
1087	rc = VMXWriteVMCS64(VMX_VMCS_HOST_GDTR_BASE, gdtr64.uAddr);
1088	rc \|= VMXWriteVMCS64(VMX_VMCS_HOST_IDTR_BASE, gdtr64.uAddr);
1089	AssertRC(rc);
1090	Log2(("VMX_VMCS_HOST_GDTR_BASE %RX64\n", gdtr64.uAddr));
1091	Log2(("VMX_VMCS_HOST_IDTR_BASE %RX64\n", idtr64.uAddr));
1092	gdtr.cbGdt = gdtr64.cb;
1093	gdtr.pGdt = (uintptr_t)gdtr64.uAddr;
1094	}
1095	else
1096	#endif
1097	{
1098	ASMGetGDTR(&gdtr);
1099	rc = VMXWriteVMCS(VMX_VMCS_HOST_GDTR_BASE, gdtr.pGdt);
1100	ASMGetIDTR(&idtr);
1101	rc \|= VMXWriteVMCS(VMX_VMCS_HOST_IDTR_BASE, idtr.pIdt);
1102	AssertRC(rc);
1103	Log2(("VMX_VMCS_HOST_GDTR_BASE %RHv\n", gdtr.pGdt));
1104	Log2(("VMX_VMCS_HOST_IDTR_BASE %RHv\n", idtr.pIdt));
1105	}
1106
1107	/* Save the base address of the TR selector. */
1108	if (SelTR > gdtr.cbGdt)
1109	{
1110	AssertMsgFailed(("Invalid TR selector %x. GDTR.cbGdt=%x\n", SelTR, gdtr.cbGdt));
1111	return VERR_VMX_INVALID_HOST_STATE;
1112	}
1113
1114	pDesc = (PCX86DESCHC)(gdtr.pGdt + (SelTR & X86_SEL_MASK));
1115	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
1116	if (VMX_IS_64BIT_HOST_MODE())
1117	{
1118	uint64_t trBase64 = X86DESC64_BASE(*(PX86DESC64)pDesc);
1119	rc = VMXWriteVMCS64(VMX_VMCS_HOST_TR_BASE, trBase64);
1120	Log2(("VMX_VMCS_HOST_TR_BASE %RX64\n", trBase64));
1121	AssertRC(rc);
1122	}
1123	else
1124	#endif
1125	{
1126	#if HC_ARCH_BITS == 64
1127	trBase = X86DESC64_BASE(*pDesc);
1128	#else
1129	trBase = X86DESC_BASE(*pDesc);
1130	#endif
1131	rc = VMXWriteVMCS(VMX_VMCS_HOST_TR_BASE, trBase);
1132	AssertRC(rc);
1133	Log2(("VMX_VMCS_HOST_TR_BASE %RHv\n", trBase));
1134	}
1135
1136	/* FS and GS base. */
1137	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
1138	if (VMX_IS_64BIT_HOST_MODE())
1139	{
1140	Log2(("MSR_K8_FS_BASE = %RX64\n", ASMRdMsr(MSR_K8_FS_BASE)));
1141	Log2(("MSR_K8_GS_BASE = %RX64\n", ASMRdMsr(MSR_K8_GS_BASE)));
1142	rc = VMXWriteVMCS64(VMX_VMCS_HOST_FS_BASE, ASMRdMsr(MSR_K8_FS_BASE));
1143	rc \|= VMXWriteVMCS64(VMX_VMCS_HOST_GS_BASE, ASMRdMsr(MSR_K8_GS_BASE));
1144	}
1145	#endif
1146	AssertRC(rc);
1147
1148	/* Sysenter MSRs. */
1149	/** @todo expensive!! */
1150	rc = VMXWriteVMCS(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
1151	Log2(("VMX_VMCS_HOST_SYSENTER_CS %08x\n", ASMRdMsr_Low(MSR_IA32_SYSENTER_CS)));
1152	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
1153	if (VMX_IS_64BIT_HOST_MODE())
1154	{
1155	Log2(("VMX_VMCS_HOST_SYSENTER_EIP %RX64\n", ASMRdMsr(MSR_IA32_SYSENTER_EIP)));
1156	Log2(("VMX_VMCS_HOST_SYSENTER_ESP %RX64\n", ASMRdMsr(MSR_IA32_SYSENTER_ESP)));
1157	rc \|= VMXWriteVMCS64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
1158	rc \|= VMXWriteVMCS64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
1159	}
1160	else
1161	{
1162	rc \|= VMXWriteVMCS(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
1163	rc \|= VMXWriteVMCS(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
1164	Log2(("VMX_VMCS_HOST_SYSENTER_EIP %RX32\n", ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP)));
1165	Log2(("VMX_VMCS_HOST_SYSENTER_ESP %RX32\n", ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP)));
1166	}
1167	#elif HC_ARCH_BITS == 32
1168	rc \|= VMXWriteVMCS(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
1169	rc \|= VMXWriteVMCS(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
1170	Log2(("VMX_VMCS_HOST_SYSENTER_EIP %RX32\n", ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP)));
1171	Log2(("VMX_VMCS_HOST_SYSENTER_ESP %RX32\n", ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP)));
1172	#else
1173	Log2(("VMX_VMCS_HOST_SYSENTER_EIP %RX64\n", ASMRdMsr(MSR_IA32_SYSENTER_EIP)));
1174	Log2(("VMX_VMCS_HOST_SYSENTER_ESP %RX64\n", ASMRdMsr(MSR_IA32_SYSENTER_ESP)));
1175	rc \|= VMXWriteVMCS64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
1176	rc \|= VMXWriteVMCS64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
1177	#endif
1178	AssertRC(rc);
1179
1180	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
1181	/* Store all host MSRs in the VM-Exit load area, so they will be reloaded after the world switch back to the host. */
1182	PVMXMSR pMsr = (PVMXMSR)pVCpu->hwaccm.s.vmx.pHostMSR;
1183	unsigned idxMsr = 0;
1184
1185	/* EFER MSR present? */
1186	if (ASMCpuId_EDX(0x80000001) & (X86_CPUID_AMD_FEATURE_EDX_NX\|X86_CPUID_AMD_FEATURE_EDX_LONG_MODE))
1187	{
1188	if (ASMCpuId_EDX(0x80000001) & X86_CPUID_AMD_FEATURE_EDX_SEP)
1189	{
1190	pMsr->u32IndexMSR = MSR_K6_STAR;
1191	pMsr->u32Reserved = 0;
1192	pMsr->u64Value = ASMRdMsr(MSR_K6_STAR); /* legacy syscall eip, cs & ss */
1193	pMsr++; idxMsr++;
1194	}
1195
1196	pMsr->u32IndexMSR = MSR_K6_EFER;
1197	pMsr->u32Reserved = 0;
1198	# if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
1199	if (CPUMIsGuestInLongMode(pVCpu))
1200	{
1201	/* Must match the efer value in our 64 bits switcher. */
1202	pMsr->u64Value = ASMRdMsr(MSR_K6_EFER) \| MSR_K6_EFER_LME \| MSR_K6_EFER_SCE \| MSR_K6_EFER_NXE;
1203	}
1204	else
1205	# endif
1206	pMsr->u64Value = ASMRdMsr(MSR_K6_EFER);
1207	pMsr++; idxMsr++;
1208	}
1209
1210	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
1211	if (VMX_IS_64BIT_HOST_MODE())
1212	{
1213	pMsr->u32IndexMSR = MSR_K8_LSTAR;
1214	pMsr->u32Reserved = 0;
1215	pMsr->u64Value = ASMRdMsr(MSR_K8_LSTAR); /* 64 bits mode syscall rip */
1216	pMsr++; idxMsr++;
1217	pMsr->u32IndexMSR = MSR_K8_SF_MASK;
1218	pMsr->u32Reserved = 0;
1219	pMsr->u64Value = ASMRdMsr(MSR_K8_SF_MASK); /* syscall flag mask */
1220	pMsr++; idxMsr++;
1221	pMsr->u32IndexMSR = MSR_K8_KERNEL_GS_BASE;
1222	pMsr->u32Reserved = 0;
1223	pMsr->u64Value = ASMRdMsr(MSR_K8_KERNEL_GS_BASE); /* swapgs exchange value */
1224	pMsr++; idxMsr++;
1225	}
1226	# endif
1227	rc = VMXWriteVMCS(VMX_VMCS_CTRL_EXIT_MSR_LOAD_COUNT, idxMsr);
1228	AssertRC(rc);
1229	#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
1230
1231	pVCpu->hwaccm.s.fContextUseFlags &= ~HWACCM_CHANGED_HOST_CONTEXT;
1232	}
1233	return rc;
1234	}
1235
1236	/**
1237	* Prefetch the 4 PDPT pointers (PAE and nested paging only)
1238	*
1239	* @param pVM The VM to operate on.
1240	* @param pVCpu The VMCPU to operate on.
1241	* @param pCtx Guest context
1242	*/
1243	static void vmxR0PrefetchPAEPdptrs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
1244	{
1245	if (CPUMIsGuestInPAEModeEx(pCtx))
1246	{
1247	X86PDPE Pdpe;
1248
1249	for (unsigned i=0;i<4;i++)
1250	{
1251	Pdpe = PGMGstGetPaePDPtr(pVCpu, i);
1252	int rc = VMXWriteVMCS64(VMX_VMCS_GUEST_PDPTR0_FULL + i*2, Pdpe.u);
1253	AssertRC(rc);
1254	}
1255	}
1256	}
1257
1258	/**
1259	* Update the exception bitmap according to the current CPU state
1260	*
1261	* @param pVM The VM to operate on.
1262	* @param pVCpu The VMCPU to operate on.
1263	* @param pCtx Guest context
1264	*/
1265	static void vmxR0UpdateExceptionBitmap(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
1266	{
1267	uint32_t u32TrapMask;
1268	Assert(pCtx);
1269
1270	u32TrapMask = HWACCM_VMX_TRAP_MASK;
1271	#ifndef DEBUG
1272	if (pVM->hwaccm.s.fNestedPaging)
1273	u32TrapMask &= ~RT_BIT(X86_XCPT_PF); /* no longer need to intercept #PF. */
1274	#endif
1275
1276	/* Also catch floating point exceptions as we need to report them to the guest in a different way. */
1277	if ( CPUMIsGuestFPUStateActive(pVCpu) == true
1278	&& !(pCtx->cr0 & X86_CR0_NE)
1279	&& !pVCpu->hwaccm.s.fFPUOldStyleOverride)
1280	{
1281	u32TrapMask \|= RT_BIT(X86_XCPT_MF);
1282	pVCpu->hwaccm.s.fFPUOldStyleOverride = true;
1283	}
1284
1285	#ifdef VBOX_STRICT
1286	Assert(u32TrapMask & RT_BIT(X86_XCPT_GP));
1287	#endif
1288
1289	/* Intercept all exceptions in real mode as none of them can be injected directly (#GP otherwise). */
1290	if ( CPUMIsGuestInRealModeEx(pCtx)
1291	&& pVM->hwaccm.s.vmx.pRealModeTSS)
1292	u32TrapMask \|= HWACCM_VMX_TRAP_MASK_REALMODE;
1293
1294	int rc = VMXWriteVMCS(VMX_VMCS_CTRL_EXCEPTION_BITMAP, u32TrapMask);
1295	AssertRC(rc);
1296	}
1297
1298	/**
1299	* Loads a minimal guest state
1300	*
1301	* NOTE: Don't do anything here that can cause a jump back to ring 3!!!!!
1302	*
1303	* @param pVM The VM to operate on.
1304	* @param pVCpu The VMCPU to operate on.
1305	* @param pCtx Guest context
1306	*/
1307	VMMR0DECL(void) VMXR0LoadMinimalGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
1308	{
1309	int rc;
1310	X86EFLAGS eflags;
1311
1312	Assert(!(pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_ALL_GUEST));
1313
1314	/* EIP, ESP and EFLAGS */
1315	rc = VMXWriteVMCS64(VMX_VMCS64_GUEST_RIP, pCtx->rip);
1316	rc \|= VMXWriteVMCS64(VMX_VMCS64_GUEST_RSP, pCtx->rsp);
1317	AssertRC(rc);
1318
1319	/* Bits 22-31, 15, 5 & 3 must be zero. Bit 1 must be 1. */
1320	eflags = pCtx->eflags;
1321	eflags.u32 &= VMX_EFLAGS_RESERVED_0;
1322	eflags.u32 \|= VMX_EFLAGS_RESERVED_1;
1323
1324	/* Real mode emulation using v86 mode. */
1325	if ( CPUMIsGuestInRealModeEx(pCtx)
1326	&& pVM->hwaccm.s.vmx.pRealModeTSS)
1327	{
1328	pVCpu->hwaccm.s.vmx.RealMode.eflags = eflags;
1329
1330	eflags.Bits.u1VM = 1;
1331	eflags.Bits.u2IOPL = 0; /* must always be 0 or else certain instructions won't cause faults. */
1332	}
1333	rc = VMXWriteVMCS(VMX_VMCS_GUEST_RFLAGS, eflags.u32);
1334	AssertRC(rc);
1335	}
1336
1337	/**
1338	* Loads the guest state
1339	*
1340	* NOTE: Don't do anything here that can cause a jump back to ring 3!!!!!
1341	*
1342	* @returns VBox status code.
1343	* @param pVM The VM to operate on.
1344	* @param pVCpu The VMCPU to operate on.
1345	* @param pCtx Guest context
1346	*/
1347	VMMR0DECL(int) VMXR0LoadGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
1348	{
1349	int rc = VINF_SUCCESS;
1350	RTGCUINTPTR val;
1351
1352	/* VMX_VMCS_CTRL_ENTRY_CONTROLS
1353	* Set required bits to one and zero according to the MSR capabilities.
1354	*/
1355	val = pVM->hwaccm.s.vmx.msr.vmx_entry.n.disallowed0;
1356	/* Load guest debug controls (dr7 & IA32_DEBUGCTL_MSR) (forced to 1 on the 'first' VT-x capable CPUs; this actually includes the newest Nehalem CPUs) */
1357	val \|= VMX_VMCS_CTRL_ENTRY_CONTROLS_LOAD_DEBUG;
1358	/* 64 bits guest mode? */
1359	if (CPUMIsGuestInLongModeEx(pCtx))
1360	val \|= VMX_VMCS_CTRL_ENTRY_CONTROLS_IA64_MODE;
1361	/* else Must be zero when AMD64 is not available. */
1362
1363	/* Mask away the bits that the CPU doesn't support */
1364	val &= pVM->hwaccm.s.vmx.msr.vmx_entry.n.allowed1;
1365	rc = VMXWriteVMCS(VMX_VMCS_CTRL_ENTRY_CONTROLS, val);
1366	AssertRC(rc);
1367
1368	/* VMX_VMCS_CTRL_EXIT_CONTROLS
1369	* Set required bits to one and zero according to the MSR capabilities.
1370	*/
1371	val = pVM->hwaccm.s.vmx.msr.vmx_exit.n.disallowed0;
1372
1373	/* Save debug controls (dr7 & IA32_DEBUGCTL_MSR) (forced to 1 on the 'first' VT-x capable CPUs; this actually includes the newest Nehalem CPUs) */
1374	val \|= VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_DEBUG;
1375
1376	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
1377	if (VMX_IS_64BIT_HOST_MODE())
1378	val \|= VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_AMD64;
1379	/* else: Must be zero when AMD64 is not available. */
1380	#elif HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
1381	if (CPUMIsGuestInLongModeEx(pCtx))
1382	val \|= VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_AMD64; /* our switcher goes to long mode */
1383	else
1384	Assert(!(val & VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_AMD64));
1385	#endif
1386	val &= pVM->hwaccm.s.vmx.msr.vmx_exit.n.allowed1;
1387	/* Don't acknowledge external interrupts on VM-exit. */
1388	rc = VMXWriteVMCS(VMX_VMCS_CTRL_EXIT_CONTROLS, val);
1389	AssertRC(rc);
1390
1391	/* Guest CPU context: ES, CS, SS, DS, FS, GS. */
1392	if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_SEGMENT_REGS)
1393	{
1394	if (pVM->hwaccm.s.vmx.pRealModeTSS)
1395	{
1396	PGMMODE enmGuestMode = PGMGetGuestMode(pVCpu);
1397	if (pVCpu->hwaccm.s.vmx.enmLastSeenGuestMode != enmGuestMode)
1398	{
1399	/* Correct weird requirements for switching to protected mode. */
1400	if ( pVCpu->hwaccm.s.vmx.enmLastSeenGuestMode == PGMMODE_REAL
1401	&& enmGuestMode >= PGMMODE_PROTECTED)
1402	{
1403	/* Flush the recompiler code cache as it's not unlikely
1404	* the guest will rewrite code it will later execute in real
1405	* mode (OpenBSD 4.0 is one such example)
1406	*/
1407	REMFlushTBs(pVM);
1408
1409	/* DPL of all hidden selector registers must match the current CPL (0). */
1410	pCtx->csHid.Attr.n.u2Dpl = 0;
1411	pCtx->csHid.Attr.n.u4Type = X86_SEL_TYPE_CODE \| X86_SEL_TYPE_RW_ACC;
1412
1413	pCtx->dsHid.Attr.n.u2Dpl = 0;
1414	pCtx->esHid.Attr.n.u2Dpl = 0;
1415	pCtx->fsHid.Attr.n.u2Dpl = 0;
1416	pCtx->gsHid.Attr.n.u2Dpl = 0;
1417	pCtx->ssHid.Attr.n.u2Dpl = 0;
1418
1419	/* The limit must correspond to the 32 bits setting. */
1420	if (!pCtx->csHid.Attr.n.u1DefBig)
1421	pCtx->csHid.u32Limit &= 0xffff;
1422	if (!pCtx->dsHid.Attr.n.u1DefBig)
1423	pCtx->dsHid.u32Limit &= 0xffff;
1424	if (!pCtx->esHid.Attr.n.u1DefBig)
1425	pCtx->esHid.u32Limit &= 0xffff;
1426	if (!pCtx->fsHid.Attr.n.u1DefBig)
1427	pCtx->fsHid.u32Limit &= 0xffff;
1428	if (!pCtx->gsHid.Attr.n.u1DefBig)
1429	pCtx->gsHid.u32Limit &= 0xffff;
1430	if (!pCtx->ssHid.Attr.n.u1DefBig)
1431	pCtx->ssHid.u32Limit &= 0xffff;
1432	}
1433	else
1434	/* Switching from protected mode to real mode. */
1435	if ( pVCpu->hwaccm.s.vmx.enmLastSeenGuestMode >= PGMMODE_PROTECTED
1436	&& enmGuestMode == PGMMODE_REAL)
1437	{
1438	/* The limit must also be set to 0xffff. */
1439	pCtx->csHid.u32Limit = 0xffff;
1440	pCtx->dsHid.u32Limit = 0xffff;
1441	pCtx->esHid.u32Limit = 0xffff;
1442	pCtx->fsHid.u32Limit = 0xffff;
1443	pCtx->gsHid.u32Limit = 0xffff;
1444	pCtx->ssHid.u32Limit = 0xffff;
1445
1446	Assert(pCtx->csHid.u64Base <= 0xfffff);
1447	Assert(pCtx->dsHid.u64Base <= 0xfffff);
1448	Assert(pCtx->esHid.u64Base <= 0xfffff);
1449	Assert(pCtx->fsHid.u64Base <= 0xfffff);
1450	Assert(pCtx->gsHid.u64Base <= 0xfffff);
1451	}
1452	pVCpu->hwaccm.s.vmx.enmLastSeenGuestMode = enmGuestMode;
1453	}
1454	else
1455	/* VT-x will fail with a guest invalid state otherwise... (CPU state after a reset) */
1456	if ( CPUMIsGuestInRealModeEx(pCtx)
1457	&& pCtx->csHid.u64Base == 0xffff0000)
1458	{
1459	pCtx->csHid.u64Base = 0xf0000;
1460	pCtx->cs = 0xf000;
1461	}
1462	}
1463
1464	VMX_WRITE_SELREG(ES, es);
1465	AssertRC(rc);
1466
1467	VMX_WRITE_SELREG(CS, cs);
1468	AssertRC(rc);
1469
1470	VMX_WRITE_SELREG(SS, ss);
1471	AssertRC(rc);
1472
1473	VMX_WRITE_SELREG(DS, ds);
1474	AssertRC(rc);
1475
1476	VMX_WRITE_SELREG(FS, fs);
1477	AssertRC(rc);
1478
1479	VMX_WRITE_SELREG(GS, gs);
1480	AssertRC(rc);
1481	}
1482
1483	/* Guest CPU context: LDTR. */
1484	if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_LDTR)
1485	{
1486	if (pCtx->ldtr == 0)
1487	{
1488	rc = VMXWriteVMCS(VMX_VMCS16_GUEST_FIELD_LDTR, 0);
1489	rc \|= VMXWriteVMCS(VMX_VMCS32_GUEST_LDTR_LIMIT, 0);
1490	rc \|= VMXWriteVMCS64(VMX_VMCS64_GUEST_LDTR_BASE, 0);
1491	/* Note: vmlaunch will fail with 0 or just 0x02. No idea why. */
1492	rc \|= VMXWriteVMCS(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, 0x82 /* present, LDT */);
1493	}
1494	else
1495	{
1496	rc = VMXWriteVMCS(VMX_VMCS16_GUEST_FIELD_LDTR, pCtx->ldtr);
1497	rc \|= VMXWriteVMCS(VMX_VMCS32_GUEST_LDTR_LIMIT, pCtx->ldtrHid.u32Limit);
1498	rc \|= VMXWriteVMCS64(VMX_VMCS64_GUEST_LDTR_BASE, pCtx->ldtrHid.u64Base);
1499	rc \|= VMXWriteVMCS(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, pCtx->ldtrHid.Attr.u);
1500	}
1501	AssertRC(rc);
1502	}
1503	/* Guest CPU context: TR. */
1504	if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_TR)
1505	{
1506	/* Real mode emulation using v86 mode with CR4.VME (interrupt redirection using the int bitmap in the TSS) */
1507	if ( CPUMIsGuestInRealModeEx(pCtx)
1508	&& pVM->hwaccm.s.vmx.pRealModeTSS)
1509	{
1510	RTGCPHYS GCPhys;
1511
1512	/* We convert it here every time as pci regions could be reconfigured. */
1513	rc = PDMVMMDevHeapR3ToGCPhys(pVM, pVM->hwaccm.s.vmx.pRealModeTSS, &GCPhys);
1514	AssertRC(rc);
1515
1516	rc = VMXWriteVMCS(VMX_VMCS16_GUEST_FIELD_TR, 0);
1517	rc \|= VMXWriteVMCS(VMX_VMCS32_GUEST_TR_LIMIT, HWACCM_VTX_TSS_SIZE);
1518	rc \|= VMXWriteVMCS64(VMX_VMCS64_GUEST_TR_BASE, GCPhys /* phys = virt in this mode */);
1519
1520	X86DESCATTR attr;
1521
1522	attr.u = 0;
1523	attr.n.u1Present = 1;
1524	attr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
1525	val = attr.u;
1526	}
1527	else
1528	{
1529	rc = VMXWriteVMCS(VMX_VMCS16_GUEST_FIELD_TR, pCtx->tr);
1530	rc \|= VMXWriteVMCS(VMX_VMCS32_GUEST_TR_LIMIT, pCtx->trHid.u32Limit);
1531	rc \|= VMXWriteVMCS64(VMX_VMCS64_GUEST_TR_BASE, pCtx->trHid.u64Base);
1532
1533	val = pCtx->trHid.Attr.u;
1534
1535	/* The TSS selector must be busy. */
1536	if ((val & 0xF) == X86_SEL_TYPE_SYS_286_TSS_AVAIL)
1537	val = (val & ~0xF) \| X86_SEL_TYPE_SYS_286_TSS_BUSY;
1538	else
1539	/* Default even if no TR selector has been set (otherwise vmlaunch will fail!) */
1540	val = (val & ~0xF) \| X86_SEL_TYPE_SYS_386_TSS_BUSY;
1541
1542	}
1543	rc \|= VMXWriteVMCS(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, val);
1544	AssertRC(rc);
1545	}
1546	/* Guest CPU context: GDTR. */
1547	if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_GDTR)
1548	{
1549	rc = VMXWriteVMCS(VMX_VMCS32_GUEST_GDTR_LIMIT, pCtx->gdtr.cbGdt);
1550	rc \|= VMXWriteVMCS64(VMX_VMCS64_GUEST_GDTR_BASE, pCtx->gdtr.pGdt);
1551	AssertRC(rc);
1552	}
1553	/* Guest CPU context: IDTR. */
1554	if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_IDTR)
1555	{
1556	rc = VMXWriteVMCS(VMX_VMCS32_GUEST_IDTR_LIMIT, pCtx->idtr.cbIdt);
1557	rc \|= VMXWriteVMCS64(VMX_VMCS64_GUEST_IDTR_BASE, pCtx->idtr.pIdt);
1558	AssertRC(rc);
1559	}
1560
1561	/*
1562	* Sysenter MSRs
1563	*/
1564	if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_MSR)
1565	{
1566	rc = VMXWriteVMCS(VMX_VMCS32_GUEST_SYSENTER_CS, pCtx->SysEnter.cs);
1567	rc \|= VMXWriteVMCS64(VMX_VMCS64_GUEST_SYSENTER_EIP, pCtx->SysEnter.eip);
1568	rc \|= VMXWriteVMCS64(VMX_VMCS64_GUEST_SYSENTER_ESP, pCtx->SysEnter.esp);
1569	AssertRC(rc);
1570	}
1571
1572	/* Control registers */
1573	if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_CR0)
1574	{
1575	val = pCtx->cr0;
1576	rc = VMXWriteVMCS(VMX_VMCS_CTRL_CR0_READ_SHADOW, val);
1577	Log2(("Guest CR0-shadow %08x\n", val));
1578	if (CPUMIsGuestFPUStateActive(pVCpu) == false)
1579	{
1580	/* Always use #NM exceptions to load the FPU/XMM state on demand. */
1581	val \|= X86_CR0_TS \| X86_CR0_ET \| X86_CR0_NE \| X86_CR0_MP;
1582	}
1583	else
1584	{
1585	/** @todo check if we support the old style mess correctly. */
1586	if (!(val & X86_CR0_NE))
1587	Log(("Forcing X86_CR0_NE!!!\n"));
1588
1589	val \|= X86_CR0_NE; /* always turn on the native mechanism to report FPU errors (old style uses interrupts) */
1590	}
1591	/* Note: protected mode & paging are always enabled; we use them for emulating real and protected mode without paging too. */
1592	if (!pVM->hwaccm.s.vmx.fUnrestrictedGuest)
1593	val \|= X86_CR0_PE \| X86_CR0_PG;
1594
1595	if (pVM->hwaccm.s.fNestedPaging)
1596	{
1597	if (CPUMIsGuestInPagedProtectedModeEx(pCtx))
1598	{
1599	/* Disable cr3 read/write monitoring as we don't need it for EPT. */
1600	pVCpu->hwaccm.s.vmx.proc_ctls &= ~( VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
1601	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT);
1602	}
1603	else
1604	{
1605	/* Reenable cr3 read/write monitoring as our identity mapped page table is active. */
1606	pVCpu->hwaccm.s.vmx.proc_ctls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
1607	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT;
1608	}
1609	rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
1610	AssertRC(rc);
1611	}
1612	else
1613	{
1614	/* Note: We must also set this as we rely on protecting various pages for which supervisor writes must be caught. */
1615	val \|= X86_CR0_WP;
1616	}
1617
1618	/* Always enable caching. */
1619	val &= ~(X86_CR0_CD\|X86_CR0_NW);
1620
1621	rc \|= VMXWriteVMCS64(VMX_VMCS64_GUEST_CR0, val);
1622	Log2(("Guest CR0 %08x\n", val));
1623	/* CR0 flags owned by the host; if the guests attempts to change them, then
1624	* the VM will exit.
1625	*/
1626	val = X86_CR0_PE /* Must monitor this bit (assumptions are made for real mode emulation) */
1627	\| X86_CR0_WP /* Must monitor this bit (it must always be enabled). */
1628	\| X86_CR0_PG /* Must monitor this bit (assumptions are made for real mode & protected mode without paging emulation) */
1629	\| X86_CR0_CD /* Bit not restored during VM-exit! */
1630	\| X86_CR0_NW /* Bit not restored during VM-exit! */
1631	\| X86_CR0_NE;
1632
1633	/* When the guest's FPU state is active, then we no longer care about
1634	* the FPU related bits.
1635	*/
1636	if (CPUMIsGuestFPUStateActive(pVCpu) == false)
1637	val \|= X86_CR0_TS \| X86_CR0_ET \| X86_CR0_MP;
1638
1639	pVCpu->hwaccm.s.vmx.cr0_mask = val;
1640
1641	rc \|= VMXWriteVMCS(VMX_VMCS_CTRL_CR0_MASK, val);
1642	Log2(("Guest CR0-mask %08x\n", val));
1643	AssertRC(rc);
1644	}
1645	if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_CR4)
1646	{
1647	/* CR4 */
1648	rc = VMXWriteVMCS(VMX_VMCS_CTRL_CR4_READ_SHADOW, pCtx->cr4);
1649	Log2(("Guest CR4-shadow %08x\n", pCtx->cr4));
1650	/* Set the required bits in cr4 too (currently X86_CR4_VMXE). */
1651	val = pCtx->cr4 \| (uint32_t)pVM->hwaccm.s.vmx.msr.vmx_cr4_fixed0;
1652
1653	if (!pVM->hwaccm.s.fNestedPaging)
1654	{
1655	switch(pVCpu->hwaccm.s.enmShadowMode)
1656	{
1657	case PGMMODE_REAL: /* Real mode -> emulated using v86 mode */
1658	case PGMMODE_PROTECTED: /* Protected mode, no paging -> emulated using identity mapping. */
1659	case PGMMODE_32_BIT: /* 32-bit paging. */
1660	val &= ~X86_CR4_PAE;
1661	break;
1662
1663	case PGMMODE_PAE: /* PAE paging. */
1664	case PGMMODE_PAE_NX: /* PAE paging with NX enabled. */
1665	/** Must use PAE paging as we could use physical memory > 4 GB */
1666	val \|= X86_CR4_PAE;
1667	break;
1668
1669	case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
1670	case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
1671	#ifdef VBOX_ENABLE_64_BITS_GUESTS
1672	break;
1673	#else
1674	AssertFailed();
1675	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
1676	#endif
1677	default: /* shut up gcc */
1678	AssertFailed();
1679	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
1680	}
1681	}
1682	else
1683	if ( !CPUMIsGuestInPagedProtectedModeEx(pCtx)
1684	&& !pVM->hwaccm.s.vmx.fUnrestrictedGuest)
1685	{
1686	/* We use 4 MB pages in our identity mapping page table for real and protected mode without paging. */
1687	val \|= X86_CR4_PSE;
1688	/* Our identity mapping is a 32 bits page directory. */
1689	val &= ~X86_CR4_PAE;
1690	}
1691
1692	/* Turn off VME if we're in emulated real mode. */
1693	if ( CPUMIsGuestInRealModeEx(pCtx)
1694	&& pVM->hwaccm.s.vmx.pRealModeTSS)
1695	val &= ~X86_CR4_VME;
1696
1697	rc \|= VMXWriteVMCS64(VMX_VMCS64_GUEST_CR4, val);
1698	Log2(("Guest CR4 %08x\n", val));
1699	/* CR4 flags owned by the host; if the guests attempts to change them, then
1700	* the VM will exit.
1701	*/
1702	val = 0
1703	\| X86_CR4_VME
1704	\| X86_CR4_PAE
1705	\| X86_CR4_PGE
1706	\| X86_CR4_PSE
1707	\| X86_CR4_VMXE;
1708	pVCpu->hwaccm.s.vmx.cr4_mask = val;
1709
1710	rc \|= VMXWriteVMCS(VMX_VMCS_CTRL_CR4_MASK, val);
1711	Log2(("Guest CR4-mask %08x\n", val));
1712	AssertRC(rc);
1713	}
1714
1715	if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_CR3)
1716	{
1717	if (pVM->hwaccm.s.fNestedPaging)
1718	{
1719	Assert(PGMGetHyperCR3(pVCpu));
1720	pVCpu->hwaccm.s.vmx.GCPhysEPTP = PGMGetHyperCR3(pVCpu);
1721
1722	Assert(!(pVCpu->hwaccm.s.vmx.GCPhysEPTP & 0xfff));
1723	/** @todo Check the IA32_VMX_EPT_VPID_CAP MSR for other supported memory types. */
1724	pVCpu->hwaccm.s.vmx.GCPhysEPTP \|= VMX_EPT_MEMTYPE_WB
1725	\| (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
1726
1727	rc = VMXWriteVMCS64(VMX_VMCS_CTRL_EPTP_FULL, pVCpu->hwaccm.s.vmx.GCPhysEPTP);
1728	AssertRC(rc);
1729
1730	if ( !CPUMIsGuestInPagedProtectedModeEx(pCtx)
1731	&& !pVM->hwaccm.s.vmx.fUnrestrictedGuest)
1732	{
1733	RTGCPHYS GCPhys;
1734
1735	/* We convert it here every time as pci regions could be reconfigured. */
1736	rc = PDMVMMDevHeapR3ToGCPhys(pVM, pVM->hwaccm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
1737	AssertMsgRC(rc, ("pNonPagingModeEPTPageTable = %RGv\n", pVM->hwaccm.s.vmx.pNonPagingModeEPTPageTable));
1738
1739	/* We use our identity mapping page table here as we need to map guest virtual to guest physical addresses; EPT will
1740	* take care of the translation to host physical addresses.
1741	*/
1742	val = GCPhys;
1743	}
1744	else
1745	{
1746	/* Save the real guest CR3 in VMX_VMCS_GUEST_CR3 */
1747	val = pCtx->cr3;
1748	/* Prefetch the four PDPT entries in PAE mode. */
1749	vmxR0PrefetchPAEPdptrs(pVM, pVCpu, pCtx);
1750	}
1751	}
1752	else
1753	{
1754	val = PGMGetHyperCR3(pVCpu);
1755	Assert(val \|\| VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL));
1756	}
1757
1758	/* Save our shadow CR3 register. */
1759	rc = VMXWriteVMCS64(VMX_VMCS64_GUEST_CR3, val);
1760	AssertRC(rc);
1761	}
1762
1763	/* Debug registers. */
1764	if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_DEBUG)
1765	{
1766	pCtx->dr[6] \|= X86_DR6_INIT_VAL; /* set all reserved bits to 1. */
1767	pCtx->dr[6] &= ~RT_BIT(12); /* must be zero. */
1768
1769	pCtx->dr[7] &= 0xffffffff; /* upper 32 bits reserved */
1770	pCtx->dr[7] &= ~(RT_BIT(11) \| RT_BIT(12) \| RT_BIT(14) \| RT_BIT(15)); /* must be zero */
1771	pCtx->dr[7] \|= 0x400; /* must be one */
1772
1773	/* Resync DR7 */
1774	rc = VMXWriteVMCS64(VMX_VMCS64_GUEST_DR7, pCtx->dr[7]);
1775	AssertRC(rc);
1776
1777	#ifdef DEBUG
1778	/* Sync the hypervisor debug state now if any breakpoint is armed. */
1779	if ( CPUMGetHyperDR7(pVCpu) & (X86_DR7_ENABLED_MASK\|X86_DR7_GD)
1780	&& !CPUMIsHyperDebugStateActive(pVCpu)
1781	&& !DBGFIsStepping(pVCpu))
1782	{
1783	/* Save the host and load the hypervisor debug state. */
1784	rc = CPUMR0LoadHyperDebugState(pVM, pVCpu, pCtx, true /* include DR6 */);
1785	AssertRC(rc);
1786
1787	/* DRx intercepts remain enabled. */
1788
1789	/* Override dr7 with the hypervisor value. */
1790	rc = VMXWriteVMCS64(VMX_VMCS64_GUEST_DR7, CPUMGetHyperDR7(pVCpu));
1791	AssertRC(rc);
1792	}
1793	else
1794	#endif
1795	/* Sync the debug state now if any breakpoint is armed. */
1796	if ( (pCtx->dr[7] & (X86_DR7_ENABLED_MASK\|X86_DR7_GD))
1797	&& !CPUMIsGuestDebugStateActive(pVCpu)
1798	&& !DBGFIsStepping(pVCpu))
1799	{
1800	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatDRxArmed);
1801
1802	/* Disable drx move intercepts. */
1803	pVCpu->hwaccm.s.vmx.proc_ctls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT;
1804	rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
1805	AssertRC(rc);
1806
1807	/* Save the host and load the guest debug state. */
1808	rc = CPUMR0LoadGuestDebugState(pVM, pVCpu, pCtx, true /* include DR6 */);
1809	AssertRC(rc);
1810	}
1811
1812	/* IA32_DEBUGCTL MSR. */
1813	rc = VMXWriteVMCS64(VMX_VMCS_GUEST_DEBUGCTL_FULL, 0);
1814	AssertRC(rc);
1815
1816	/** @todo do we really ever need this? */
1817	rc \|= VMXWriteVMCS(VMX_VMCS_GUEST_DEBUG_EXCEPTIONS, 0);
1818	AssertRC(rc);
1819	}
1820
1821	/* 64 bits guest mode? */
1822	if (CPUMIsGuestInLongModeEx(pCtx))
1823	{
1824	#if !defined(VBOX_ENABLE_64_BITS_GUESTS)
1825	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
1826	#elif HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
1827	pVCpu->hwaccm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
1828	#else
1829	# ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
1830	if (!pVM->hwaccm.s.fAllow64BitGuests)
1831	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
1832	# endif
1833	pVCpu->hwaccm.s.vmx.pfnStartVM = VMXR0StartVM64;
1834	#endif
1835	if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_MSR)
1836	{
1837	/* Update these as wrmsr might have changed them. */
1838	rc = VMXWriteVMCS64(VMX_VMCS64_GUEST_FS_BASE, pCtx->fsHid.u64Base);
1839	AssertRC(rc);
1840	rc = VMXWriteVMCS64(VMX_VMCS64_GUEST_GS_BASE, pCtx->gsHid.u64Base);
1841	AssertRC(rc);
1842	}
1843	}
1844	else
1845	{
1846	pVCpu->hwaccm.s.vmx.pfnStartVM = VMXR0StartVM32;
1847	}
1848
1849	vmxR0UpdateExceptionBitmap(pVM, pVCpu, pCtx);
1850
1851	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
1852	/* Store all guest MSRs in the VM-Entry load area, so they will be loaded during the world switch. */
1853	PVMXMSR pMsr = (PVMXMSR)pVCpu->hwaccm.s.vmx.pGuestMSR;
1854	unsigned idxMsr = 0;
1855
1856	uint32_t ulEdx;
1857	uint32_t ulTemp;
1858	CPUMGetGuestCpuId(pVCpu, 0x80000001, &ulTemp, &ulTemp, &ulTemp, &ulEdx);
1859	/* EFER MSR present? */
1860	if (ulEdx & (X86_CPUID_AMD_FEATURE_EDX_NX\|X86_CPUID_AMD_FEATURE_EDX_LONG_MODE))
1861	{
1862	pMsr->u32IndexMSR = MSR_K6_EFER;
1863	pMsr->u32Reserved = 0;
1864	pMsr->u64Value = pCtx->msrEFER;
1865	/* VT-x will complain if only MSR_K6_EFER_LME is set. */
1866	if (!CPUMIsGuestInLongModeEx(pCtx))
1867	pMsr->u64Value &= ~(MSR_K6_EFER_LMA\|MSR_K6_EFER_LME);
1868	pMsr++; idxMsr++;
1869
1870	if (ulEdx & X86_CPUID_AMD_FEATURE_EDX_LONG_MODE)
1871	{
1872	pMsr->u32IndexMSR = MSR_K8_LSTAR;
1873	pMsr->u32Reserved = 0;
1874	pMsr->u64Value = pCtx->msrLSTAR; /* 64 bits mode syscall rip */
1875	pMsr++; idxMsr++;
1876	pMsr->u32IndexMSR = MSR_K6_STAR;
1877	pMsr->u32Reserved = 0;
1878	pMsr->u64Value = pCtx->msrSTAR; /* legacy syscall eip, cs & ss */
1879	pMsr++; idxMsr++;
1880	pMsr->u32IndexMSR = MSR_K8_SF_MASK;
1881	pMsr->u32Reserved = 0;
1882	pMsr->u64Value = pCtx->msrSFMASK; /* syscall flag mask */
1883	pMsr++; idxMsr++;
1884	pMsr->u32IndexMSR = MSR_K8_KERNEL_GS_BASE;
1885	pMsr->u32Reserved = 0;
1886	pMsr->u64Value = pCtx->msrKERNELGSBASE; /* swapgs exchange value */
1887	pMsr++; idxMsr++;
1888	}
1889	}
1890	pVCpu->hwaccm.s.vmx.cCachedMSRs = idxMsr;
1891
1892	rc = VMXWriteVMCS(VMX_VMCS_CTRL_ENTRY_MSR_LOAD_COUNT, idxMsr);
1893	AssertRC(rc);
1894
1895	rc = VMXWriteVMCS(VMX_VMCS_CTRL_EXIT_MSR_STORE_COUNT, idxMsr);
1896	AssertRC(rc);
1897	#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
1898
1899	bool fOffsettedTsc;
1900	if (pVM->hwaccm.s.vmx.fUsePreemptTimer)
1901	{
1902	uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVCpu, &fOffsettedTsc, &pVCpu->hwaccm.s.vmx.u64TSCOffset);
1903
1904	/* Make sure the returned values have sane upper and lower boundaries. */
1905	uint64_t u64CpuHz = SUPGetCpuHzFromGIP(g_pSUPGlobalInfoPage);
1906
1907	cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64 of a second */
1908	cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
1909
1910	cTicksToDeadline >>= pVM->hwaccm.s.vmx.cPreemptTimerShift;
1911	uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
1912	rc = VMXWriteVMCS(VMX_VMCS32_GUEST_PREEMPTION_TIMER_VALUE, cPreemptionTickCount);
1913	AssertRC(rc);
1914	}
1915	else
1916	fOffsettedTsc = TMCpuTickCanUseRealTSC(pVCpu, &pVCpu->hwaccm.s.vmx.u64TSCOffset);
1917	if (fOffsettedTsc)
1918	{
1919	uint64_t u64CurTSC = ASMReadTSC();
1920	if (u64CurTSC + pVCpu->hwaccm.s.vmx.u64TSCOffset >= TMCpuTickGetLastSeen(pVCpu))
1921	{
1922	/* Note: VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT takes precedence over TSC_OFFSET */
1923	rc = VMXWriteVMCS64(VMX_VMCS_CTRL_TSC_OFFSET_FULL, pVCpu->hwaccm.s.vmx.u64TSCOffset);
1924	AssertRC(rc);
1925
1926	pVCpu->hwaccm.s.vmx.proc_ctls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT;
1927	rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
1928	AssertRC(rc);
1929	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatTSCOffset);
1930	}
1931	else
1932	{
1933	/* Fall back to rdtsc emulation as we would otherwise pass decreasing tsc values to the guest. */
1934	LogFlow(("TSC %RX64 offset %RX64 time=%RX64 last=%RX64 (diff=%RX64, virt_tsc=%RX64)\n", u64CurTSC, pVCpu->hwaccm.s.vmx.u64TSCOffset, u64CurTSC + pVCpu->hwaccm.s.vmx.u64TSCOffset, TMCpuTickGetLastSeen(pVCpu), TMCpuTickGetLastSeen(pVCpu) - u64CurTSC - pVCpu->hwaccm.s.vmx.u64TSCOffset, TMCpuTickGet(pVCpu)));
1935	pVCpu->hwaccm.s.vmx.proc_ctls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT;
1936	rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
1937	AssertRC(rc);
1938	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatTSCInterceptOverFlow);
1939	}
1940	}
1941	else
1942	{
1943	pVCpu->hwaccm.s.vmx.proc_ctls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT;
1944	rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
1945	AssertRC(rc);
1946	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatTSCIntercept);
1947	}
1948
1949	/* Done with the major changes */
1950	pVCpu->hwaccm.s.fContextUseFlags &= ~HWACCM_CHANGED_ALL_GUEST;
1951
1952	/* Minimal guest state update (esp, eip, eflags mostly) */
1953	VMXR0LoadMinimalGuestState(pVM, pVCpu, pCtx);
1954	return rc;
1955	}
1956
1957	/**
1958	* Syncs back the guest state
1959	*
1960	* @returns VBox status code.
1961	* @param pVM The VM to operate on.
1962	* @param pVCpu The VMCPU to operate on.
1963	* @param pCtx Guest context
1964	*/
1965	DECLINLINE(int) VMXR0SaveGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
1966	{
1967	RTGCUINTREG val, valShadow;
1968	RTGCUINTPTR uInterruptState;
1969	int rc;
1970
1971	/* Let's first sync back eip, esp, and eflags. */
1972	rc = VMXReadCachedVMCS(VMX_VMCS64_GUEST_RIP, &val);
1973	AssertRC(rc);
1974	pCtx->rip = val;
1975	rc = VMXReadCachedVMCS(VMX_VMCS64_GUEST_RSP, &val);
1976	AssertRC(rc);
1977	pCtx->rsp = val;
1978	rc = VMXReadCachedVMCS(VMX_VMCS_GUEST_RFLAGS, &val);
1979	AssertRC(rc);
1980	pCtx->eflags.u32 = val;
1981
1982	/* Take care of instruction fusing (sti, mov ss) */
1983	rc \|= VMXReadCachedVMCS(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &val);
1984	uInterruptState = val;
1985	if (uInterruptState != 0)
1986	{
1987	Assert(uInterruptState <= 2); /* only sti & mov ss */
1988	Log(("uInterruptState %x eip=%RGv\n", (uint32_t)uInterruptState, pCtx->rip));
1989	EMSetInhibitInterruptsPC(pVCpu, pCtx->rip);
1990	}
1991	else
1992	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
1993
1994	/* Control registers. */
1995	VMXReadCachedVMCS(VMX_VMCS_CTRL_CR0_READ_SHADOW, &valShadow);
1996	VMXReadCachedVMCS(VMX_VMCS64_GUEST_CR0, &val);
1997	val = (valShadow & pVCpu->hwaccm.s.vmx.cr0_mask) \| (val & ~pVCpu->hwaccm.s.vmx.cr0_mask);
1998	CPUMSetGuestCR0(pVCpu, val);
1999
2000	VMXReadCachedVMCS(VMX_VMCS_CTRL_CR4_READ_SHADOW, &valShadow);
2001	VMXReadCachedVMCS(VMX_VMCS64_GUEST_CR4, &val);
2002	val = (valShadow & pVCpu->hwaccm.s.vmx.cr4_mask) \| (val & ~pVCpu->hwaccm.s.vmx.cr4_mask);
2003	CPUMSetGuestCR4(pVCpu, val);
2004
2005	/* Note: no reason to sync back the CRx registers. They can't be changed by the guest. */
2006	/* Note: only in the nested paging case can CR3 & CR4 be changed by the guest. */
2007	if ( pVM->hwaccm.s.fNestedPaging
2008	&& CPUMIsGuestInPagedProtectedModeEx(pCtx))
2009	{
2010	PVMCSCACHE pCache = &pVCpu->hwaccm.s.vmx.VMCSCache;
2011
2012	/* Can be updated behind our back in the nested paging case. */
2013	CPUMSetGuestCR2(pVCpu, pCache->cr2);
2014
2015	VMXReadCachedVMCS(VMX_VMCS64_GUEST_CR3, &val);
2016
2017	if (val != pCtx->cr3)
2018	{
2019	CPUMSetGuestCR3(pVCpu, val);
2020	PGMUpdateCR3(pVCpu, val);
2021	}
2022	/* Prefetch the four PDPT entries in PAE mode. */
2023	vmxR0PrefetchPAEPdptrs(pVM, pVCpu, pCtx);
2024	}
2025
2026	/* Sync back DR7 here. */
2027	VMXReadCachedVMCS(VMX_VMCS64_GUEST_DR7, &val);
2028	pCtx->dr[7] = val;
2029
2030	/* Guest CPU context: ES, CS, SS, DS, FS, GS. */
2031	VMX_READ_SELREG(ES, es);
2032	VMX_READ_SELREG(SS, ss);
2033	VMX_READ_SELREG(CS, cs);
2034	VMX_READ_SELREG(DS, ds);
2035	VMX_READ_SELREG(FS, fs);
2036	VMX_READ_SELREG(GS, gs);
2037
2038	/*
2039	* System MSRs
2040	*/
2041	VMXReadCachedVMCS(VMX_VMCS32_GUEST_SYSENTER_CS, &val);
2042	pCtx->SysEnter.cs = val;
2043	VMXReadCachedVMCS(VMX_VMCS64_GUEST_SYSENTER_EIP, &val);
2044	pCtx->SysEnter.eip = val;
2045	VMXReadCachedVMCS(VMX_VMCS64_GUEST_SYSENTER_ESP, &val);
2046	pCtx->SysEnter.esp = val;
2047
2048	/* Misc. registers; must sync everything otherwise we can get out of sync when jumping to ring 3. */
2049	VMX_READ_SELREG(LDTR, ldtr);
2050
2051	VMXReadCachedVMCS(VMX_VMCS32_GUEST_GDTR_LIMIT, &val);
2052	pCtx->gdtr.cbGdt = val;
2053	VMXReadCachedVMCS(VMX_VMCS64_GUEST_GDTR_BASE, &val);
2054	pCtx->gdtr.pGdt = val;
2055
2056	VMXReadCachedVMCS(VMX_VMCS32_GUEST_IDTR_LIMIT, &val);
2057	pCtx->idtr.cbIdt = val;
2058	VMXReadCachedVMCS(VMX_VMCS64_GUEST_IDTR_BASE, &val);
2059	pCtx->idtr.pIdt = val;
2060
2061	/* Real mode emulation using v86 mode. */
2062	if ( CPUMIsGuestInRealModeEx(pCtx)
2063	&& pVM->hwaccm.s.vmx.pRealModeTSS)
2064	{
2065	/* Hide our emulation flags */
2066	pCtx->eflags.Bits.u1VM = 0;
2067
2068	/* Restore original IOPL setting as we always use 0. */
2069	pCtx->eflags.Bits.u2IOPL = pVCpu->hwaccm.s.vmx.RealMode.eflags.Bits.u2IOPL;
2070
2071	/* Force a TR resync every time in case we switch modes. */
2072	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_GUEST_TR;
2073	}
2074	else
2075	{
2076	/* In real mode we have a fake TSS, so only sync it back when it's supposed to be valid. */
2077	VMX_READ_SELREG(TR, tr);
2078	}
2079
2080	#ifdef VBOX_WITH_AUTO_MSR_LOAD_RESTORE
2081	/* Save the possibly changed MSRs that we automatically restore and save during a world switch. */
2082	for (unsigned i = 0; i < pVCpu->hwaccm.s.vmx.cCachedMSRs; i++)
2083	{
2084	PVMXMSR pMsr = (PVMXMSR)pVCpu->hwaccm.s.vmx.pGuestMSR;
2085	pMsr += i;
2086
2087	switch (pMsr->u32IndexMSR)
2088	{
2089	case MSR_K8_LSTAR:
2090	pCtx->msrLSTAR = pMsr->u64Value;
2091	break;
2092	case MSR_K6_STAR:
2093	pCtx->msrSTAR = pMsr->u64Value;
2094	break;
2095	case MSR_K8_SF_MASK:
2096	pCtx->msrSFMASK = pMsr->u64Value;
2097	break;
2098	case MSR_K8_KERNEL_GS_BASE:
2099	pCtx->msrKERNELGSBASE = pMsr->u64Value;
2100	break;
2101	case MSR_K6_EFER:
2102	/* EFER can't be changed without causing a VM-exit. */
2103	// Assert(pCtx->msrEFER == pMsr->u64Value);
2104	break;
2105	default:
2106	AssertFailed();
2107	return VERR_INTERNAL_ERROR;
2108	}
2109	}
2110	#endif /* VBOX_WITH_AUTO_MSR_LOAD_RESTORE */
2111	return VINF_SUCCESS;
2112	}
2113
2114	/**
2115	* Dummy placeholder
2116	*
2117	* @param pVM The VM to operate on.
2118	* @param pVCpu The VMCPU to operate on.
2119	*/
2120	static void vmxR0SetupTLBDummy(PVM pVM, PVMCPU pVCpu)
2121	{
2122	NOREF(pVM);
2123	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
2124	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
2125	pVCpu->hwaccm.s.TlbShootdown.cPages = 0;
2126	return;
2127	}
2128
2129	/**
2130	* Setup the tagged TLB for EPT
2131	*
2132	* @returns VBox status code.
2133	* @param pVM The VM to operate on.
2134	* @param pVCpu The VMCPU to operate on.
2135	*/
2136	static void vmxR0SetupTLBEPT(PVM pVM, PVMCPU pVCpu)
2137	{
2138	PHWACCM_CPUINFO pCpu;
2139
2140	Assert(pVM->hwaccm.s.fNestedPaging);
2141	Assert(!pVM->hwaccm.s.vmx.fVPID);
2142
2143	/* Deal with tagged TLBs if VPID or EPT is supported. */
2144	pCpu = HWACCMR0GetCurrentCpu();
2145	/* Force a TLB flush for the first world switch if the current cpu differs from the one we ran on last. */
2146	/* Note that this can happen both for start and resume due to long jumps back to ring 3. */
2147	if ( pVCpu->hwaccm.s.idLastCpu != pCpu->idCpu
2148	/* if the tlb flush count has changed, another VM has flushed the TLB of this cpu, so we can't use our current ASID anymore. */
2149	\|\| pVCpu->hwaccm.s.cTLBFlushes != pCpu->cTLBFlushes)
2150	{
2151	/* Force a TLB flush on VM entry. */
2152	pVCpu->hwaccm.s.fForceTLBFlush = true;
2153	}
2154	else
2155	Assert(!pCpu->fFlushTLB);
2156
2157	/* Check for tlb shootdown flushes. */
2158	if (VMCPU_FF_TESTANDCLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2159	pVCpu->hwaccm.s.fForceTLBFlush = true;
2160
2161	pVCpu->hwaccm.s.idLastCpu = pCpu->idCpu;
2162	pCpu->fFlushTLB = false;
2163
2164	if (pVCpu->hwaccm.s.fForceTLBFlush)
2165	{
2166	vmxR0FlushEPT(pVM, pVCpu, pVM->hwaccm.s.vmx.enmFlushContext, 0);
2167	}
2168	else
2169	if (VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
2170	{
2171	/* Deal with pending TLB shootdown actions which were queued when we were not executing code. */
2172	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatTlbShootdown);
2173
2174	for (unsigned i=0;i<pVCpu->hwaccm.s.TlbShootdown.cPages;i++)
2175	{
2176	/* aTlbShootdownPages contains physical addresses in this case. */
2177	vmxR0FlushEPT(pVM, pVCpu, pVM->hwaccm.s.vmx.enmFlushPage, pVCpu->hwaccm.s.TlbShootdown.aPages[i]);
2178	}
2179	}
2180	pVCpu->hwaccm.s.TlbShootdown.cPages= 0;
2181	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
2182
2183	#ifdef VBOX_WITH_STATISTICS
2184	if (pVCpu->hwaccm.s.fForceTLBFlush)
2185	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatFlushTLBWorldSwitch);
2186	else
2187	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatNoFlushTLBWorldSwitch);
2188	#endif
2189	}
2190
2191	#ifdef HWACCM_VTX_WITH_VPID
2192	/**
2193	* Setup the tagged TLB for VPID
2194	*
2195	* @returns VBox status code.
2196	* @param pVM The VM to operate on.
2197	* @param pVCpu The VMCPU to operate on.
2198	*/
2199	static void vmxR0SetupTLBVPID(PVM pVM, PVMCPU pVCpu)
2200	{
2201	PHWACCM_CPUINFO pCpu;
2202
2203	Assert(pVM->hwaccm.s.vmx.fVPID);
2204	Assert(!pVM->hwaccm.s.fNestedPaging);
2205
2206	/* Deal with tagged TLBs if VPID or EPT is supported. */
2207	pCpu = HWACCMR0GetCurrentCpu();
2208	/* Force a TLB flush for the first world switch if the current cpu differs from the one we ran on last. */
2209	/* Note that this can happen both for start and resume due to long jumps back to ring 3. */
2210	if ( pVCpu->hwaccm.s.idLastCpu != pCpu->idCpu
2211	/* if the tlb flush count has changed, another VM has flushed the TLB of this cpu, so we can't use our current ASID anymore. */
2212	\|\| pVCpu->hwaccm.s.cTLBFlushes != pCpu->cTLBFlushes)
2213	{
2214	/* Force a TLB flush on VM entry. */
2215	pVCpu->hwaccm.s.fForceTLBFlush = true;
2216	}
2217	else
2218	Assert(!pCpu->fFlushTLB);
2219
2220	pVCpu->hwaccm.s.idLastCpu = pCpu->idCpu;
2221
2222	/* Check for tlb shootdown flushes. */
2223	if (VMCPU_FF_TESTANDCLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2224	pVCpu->hwaccm.s.fForceTLBFlush = true;
2225
2226	/* Make sure we flush the TLB when required. Switch ASID to achieve the same thing, but without actually flushing the whole TLB (which is expensive). */
2227	if (pVCpu->hwaccm.s.fForceTLBFlush)
2228	{
2229	if ( ++pCpu->uCurrentASID >= pVM->hwaccm.s.uMaxASID
2230	\|\| pCpu->fFlushTLB)
2231	{
2232	pCpu->fFlushTLB = false;
2233	pCpu->uCurrentASID = 1; /* start at 1; host uses 0 */
2234	pCpu->cTLBFlushes++;
2235	vmxR0FlushVPID(pVM, pVCpu, VMX_FLUSH_ALL_CONTEXTS, 0);
2236	}
2237	else
2238	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatFlushASID);
2239
2240	pVCpu->hwaccm.s.fForceTLBFlush = false;
2241	pVCpu->hwaccm.s.cTLBFlushes = pCpu->cTLBFlushes;
2242	pVCpu->hwaccm.s.uCurrentASID = pCpu->uCurrentASID;
2243	}
2244	else
2245	{
2246	Assert(!pCpu->fFlushTLB);
2247	Assert(pVCpu->hwaccm.s.uCurrentASID && pCpu->uCurrentASID);
2248
2249	if (VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
2250	{
2251	/* Deal with pending TLB shootdown actions which were queued when we were not executing code. */
2252	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatTlbShootdown);
2253	for (unsigned i=0;i<pVCpu->hwaccm.s.TlbShootdown.cPages;i++)
2254	vmxR0FlushVPID(pVM, pVCpu, pVM->hwaccm.s.vmx.enmFlushPage, pVCpu->hwaccm.s.TlbShootdown.aPages[i]);
2255	}
2256	}
2257	pVCpu->hwaccm.s.TlbShootdown.cPages = 0;
2258	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
2259
2260	AssertMsg(pVCpu->hwaccm.s.cTLBFlushes == pCpu->cTLBFlushes, ("Flush count mismatch for cpu %d (%x vs %x)\n", pCpu->idCpu, pVCpu->hwaccm.s.cTLBFlushes, pCpu->cTLBFlushes));
2261	AssertMsg(pCpu->uCurrentASID >= 1 && pCpu->uCurrentASID < pVM->hwaccm.s.uMaxASID, ("cpu%d uCurrentASID = %x\n", pCpu->idCpu, pCpu->uCurrentASID));
2262	AssertMsg(pVCpu->hwaccm.s.uCurrentASID >= 1 && pVCpu->hwaccm.s.uCurrentASID < pVM->hwaccm.s.uMaxASID, ("cpu%d VM uCurrentASID = %x\n", pCpu->idCpu, pVCpu->hwaccm.s.uCurrentASID));
2263
2264	int rc = VMXWriteVMCS(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hwaccm.s.uCurrentASID);
2265	AssertRC(rc);
2266
2267	if (pVCpu->hwaccm.s.fForceTLBFlush)
2268	vmxR0FlushVPID(pVM, pVCpu, pVM->hwaccm.s.vmx.enmFlushContext, 0);
2269
2270	#ifdef VBOX_WITH_STATISTICS
2271	if (pVCpu->hwaccm.s.fForceTLBFlush)
2272	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatFlushTLBWorldSwitch);
2273	else
2274	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatNoFlushTLBWorldSwitch);
2275	#endif
2276	}
2277	#endif /* HWACCM_VTX_WITH_VPID */
2278
2279	/**
2280	* Runs guest code in a VT-x VM.
2281	*
2282	* @returns VBox status code.
2283	* @param pVM The VM to operate on.
2284	* @param pVCpu The VMCPU to operate on.
2285	* @param pCtx Guest context
2286	*/
2287	VMMR0DECL(int) VMXR0RunGuestCode(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
2288	{
2289	STAM_PROFILE_ADV_START(&pVCpu->hwaccm.s.StatEntry, x);
2290	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hwaccm.s.StatExit1);
2291	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hwaccm.s.StatExit2);
2292
2293	VBOXSTRICTRC rc = VINF_SUCCESS;
2294	int rc2;
2295	RTGCUINTREG val;
2296	RTGCUINTREG exitReason = (RTGCUINTREG)VMX_EXIT_INVALID;
2297	RTGCUINTREG instrError, cbInstr;
2298	RTGCUINTPTR exitQualification = 0;
2299	RTGCUINTPTR intInfo = 0; /* shut up buggy gcc 4 */
2300	RTGCUINTPTR errCode, instrInfo;
2301	bool fSetupTPRCaching = false;
2302	uint64_t u64OldLSTAR = 0;
2303	uint8_t u8LastTPR = 0;
2304	RTCCUINTREG uOldEFlags = ~(RTCCUINTREG)0;
2305	unsigned cResume = 0;
2306	#ifdef VBOX_STRICT
2307	RTCPUID idCpuCheck;
2308	bool fWasInLongMode = false;
2309	#endif
2310	#ifdef VBOX_HIGH_RES_TIMERS_HACK_IN_RING0
2311	uint64_t u64LastTime = RTTimeMilliTS();
2312	#endif
2313
2314	Assert(!(pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC) \|\| (pVCpu->hwaccm.s.vmx.pVAPIC && pVM->hwaccm.s.vmx.pAPIC));
2315
2316	/* Check if we need to use TPR shadowing. */
2317	if ( CPUMIsGuestInLongModeEx(pCtx)
2318	\|\| ( ((pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC) \|\| pVM->hwaccm.s.fTRPPatchingAllowed)
2319	&& pVM->hwaccm.s.fHasIoApic)
2320	)
2321	{
2322	fSetupTPRCaching = true;
2323	}
2324
2325	Log2(("\nE"));
2326
2327	#ifdef VBOX_STRICT
2328	{
2329	RTCCUINTREG val2;
2330
2331	rc2 = VMXReadVMCS(VMX_VMCS_CTRL_PIN_EXEC_CONTROLS, &val2);
2332	AssertRC(rc2);
2333	Log2(("VMX_VMCS_CTRL_PIN_EXEC_CONTROLS = %08x\n", val2));
2334
2335	/* allowed zero */
2336	if ((val2 & pVM->hwaccm.s.vmx.msr.vmx_pin_ctls.n.disallowed0) != pVM->hwaccm.s.vmx.msr.vmx_pin_ctls.n.disallowed0)
2337	Log(("Invalid VMX_VMCS_CTRL_PIN_EXEC_CONTROLS: zero\n"));
2338
2339	/* allowed one */
2340	if ((val2 & ~pVM->hwaccm.s.vmx.msr.vmx_pin_ctls.n.allowed1) != 0)
2341	Log(("Invalid VMX_VMCS_CTRL_PIN_EXEC_CONTROLS: one\n"));
2342
2343	rc2 = VMXReadVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, &val2);
2344	AssertRC(rc2);
2345	Log2(("VMX_VMCS_CTRL_PROC_EXEC_CONTROLS = %08x\n", val2));
2346
2347	/* Must be set according to the MSR, but can be cleared in case of EPT. */
2348	if (pVM->hwaccm.s.fNestedPaging)
2349	val2 \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INVLPG_EXIT
2350	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
2351	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT;
2352
2353	/* allowed zero */
2354	if ((val2 & pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.disallowed0) != pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.disallowed0)
2355	Log(("Invalid VMX_VMCS_CTRL_PROC_EXEC_CONTROLS: zero\n"));
2356
2357	/* allowed one */
2358	if ((val2 & ~pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1) != 0)
2359	Log(("Invalid VMX_VMCS_CTRL_PROC_EXEC_CONTROLS: one\n"));
2360
2361	rc2 = VMXReadVMCS(VMX_VMCS_CTRL_ENTRY_CONTROLS, &val2);
2362	AssertRC(rc2);
2363	Log2(("VMX_VMCS_CTRL_ENTRY_CONTROLS = %08x\n", val2));
2364
2365	/* allowed zero */
2366	if ((val2 & pVM->hwaccm.s.vmx.msr.vmx_entry.n.disallowed0) != pVM->hwaccm.s.vmx.msr.vmx_entry.n.disallowed0)
2367	Log(("Invalid VMX_VMCS_CTRL_ENTRY_CONTROLS: zero\n"));
2368
2369	/* allowed one */
2370	if ((val2 & ~pVM->hwaccm.s.vmx.msr.vmx_entry.n.allowed1) != 0)
2371	Log(("Invalid VMX_VMCS_CTRL_ENTRY_CONTROLS: one\n"));
2372
2373	rc2 = VMXReadVMCS(VMX_VMCS_CTRL_EXIT_CONTROLS, &val2);
2374	AssertRC(rc2);
2375	Log2(("VMX_VMCS_CTRL_EXIT_CONTROLS = %08x\n", val2));
2376
2377	/* allowed zero */
2378	if ((val2 & pVM->hwaccm.s.vmx.msr.vmx_exit.n.disallowed0) != pVM->hwaccm.s.vmx.msr.vmx_exit.n.disallowed0)
2379	Log(("Invalid VMX_VMCS_CTRL_EXIT_CONTROLS: zero\n"));
2380
2381	/* allowed one */
2382	if ((val2 & ~pVM->hwaccm.s.vmx.msr.vmx_exit.n.allowed1) != 0)
2383	Log(("Invalid VMX_VMCS_CTRL_EXIT_CONTROLS: one\n"));
2384	}
2385	fWasInLongMode = CPUMIsGuestInLongModeEx(pCtx);
2386	#endif /* VBOX_STRICT */
2387
2388	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
2389	pVCpu->hwaccm.s.vmx.VMCSCache.u64TimeEntry = RTTimeNanoTS();
2390	#endif
2391
2392	/* We can jump to this point to resume execution after determining that a VM-exit is innocent.
2393	*/
2394	ResumeExecution:
2395	if (!STAM_REL_PROFILE_ADV_IS_RUNNING(&pVCpu->hwaccm.s.StatEntry))
2396	STAM_REL_PROFILE_ADV_STOP_START(&pVCpu->hwaccm.s.StatExit2, &pVCpu->hwaccm.s.StatEntry, x);
2397	AssertMsg(pVCpu->hwaccm.s.idEnteredCpu == RTMpCpuId(),
2398	("Expected %d, I'm %d; cResume=%d exitReason=%RGv exitQualification=%RGv\n",
2399	(int)pVCpu->hwaccm.s.idEnteredCpu, (int)RTMpCpuId(), cResume, exitReason, exitQualification));
2400	Assert(!HWACCMR0SuspendPending());
2401	/* Not allowed to switch modes without reloading the host state (32->64 switcher)!! */
2402	Assert(fWasInLongMode == CPUMIsGuestInLongModeEx(pCtx));
2403
2404	/* Safety precaution; looping for too long here can have a very bad effect on the host */
2405	if (RT_UNLIKELY(++cResume > pVM->hwaccm.s.cMaxResumeLoops))
2406	{
2407	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitMaxResume);
2408	rc = VINF_EM_RAW_INTERRUPT;
2409	goto end;
2410	}
2411
2412	/* Check for irq inhibition due to instruction fusing (sti, mov ss). */
2413	if (VMCPU_FF_ISSET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
2414	{
2415	Log(("VM_FF_INHIBIT_INTERRUPTS at %RGv successor %RGv\n", (RTGCPTR)pCtx->rip, EMGetInhibitInterruptsPC(pVCpu)));
2416	if (pCtx->rip != EMGetInhibitInterruptsPC(pVCpu))
2417	{
2418	/* Note: we intentionally don't clear VM_FF_INHIBIT_INTERRUPTS here.
2419	* Before we are able to execute this instruction in raw mode (iret to guest code) an external interrupt might
2420	* force a world switch again. Possibly allowing a guest interrupt to be dispatched in the process. This could
2421	* break the guest. Sounds very unlikely, but such timing sensitive problems are not as rare as you might think.
2422	*/
2423	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
2424	/* Irq inhibition is no longer active; clear the corresponding VMX state. */
2425	rc2 = VMXWriteVMCS(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, 0);
2426	AssertRC(rc2);
2427	}
2428	}
2429	else
2430	{
2431	/* Irq inhibition is no longer active; clear the corresponding VMX state. */
2432	rc2 = VMXWriteVMCS(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, 0);
2433	AssertRC(rc2);
2434	}
2435
2436	#ifdef VBOX_HIGH_RES_TIMERS_HACK_IN_RING0
2437	if (RT_UNLIKELY((cResume & 0xf) == 0))
2438	{
2439	uint64_t u64CurTime = RTTimeMilliTS();
2440
2441	if (RT_UNLIKELY(u64CurTime > u64LastTime))
2442	{
2443	u64LastTime = u64CurTime;
2444	TMTimerPollVoid(pVM, pVCpu);
2445	}
2446	}
2447	#endif
2448
2449	/* Check for pending actions that force us to go back to ring 3. */
2450	if ( VM_FF_ISPENDING(pVM, VM_FF_HWACCM_TO_R3_MASK \| VM_FF_REQUEST \| VM_FF_PGM_POOL_FLUSH_PENDING \| VM_FF_PDM_DMA)
2451	\|\| VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_HWACCM_TO_R3_MASK \| VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL \| VMCPU_FF_REQUEST))
2452	{
2453	/* Check if a sync operation is pending. */
2454	if (VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
2455	{
2456	rc = PGMSyncCR3(pVCpu, pCtx->cr0, pCtx->cr3, pCtx->cr4, VMCPU_FF_ISSET(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
2457	if (rc != VINF_SUCCESS)
2458	{
2459	AssertRC(VBOXSTRICTRC_VAL(rc));
2460	Log(("Pending pool sync is forcing us back to ring 3; rc=%d\n", VBOXSTRICTRC_VAL(rc)));
2461	goto end;
2462	}
2463	}
2464
2465	#ifdef DEBUG
2466	/* Intercept X86_XCPT_DB if stepping is enabled */
2467	if (!DBGFIsStepping(pVCpu))
2468	#endif
2469	{
2470	if ( VM_FF_ISPENDING(pVM, VM_FF_HWACCM_TO_R3_MASK)
2471	\|\| VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_HWACCM_TO_R3_MASK))
2472	{
2473	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatSwitchToR3);
2474	rc = RT_UNLIKELY(VM_FF_ISPENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
2475	goto end;
2476	}
2477	}
2478
2479	/* Pending request packets might contain actions that need immediate attention, such as pending hardware interrupts. */
2480	if ( VM_FF_ISPENDING(pVM, VM_FF_REQUEST)
2481	\|\| VMCPU_FF_ISPENDING(pVCpu, VMCPU_FF_REQUEST))
2482	{
2483	rc = VINF_EM_PENDING_REQUEST;
2484	goto end;
2485	}
2486
2487	/* Check if a pgm pool flush is in progress. */
2488	if (VM_FF_ISPENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
2489	{
2490	rc = VINF_PGM_POOL_FLUSH_PENDING;
2491	goto end;
2492	}
2493
2494	/* Check if DMA work is pending (2nd+ run). */
2495	if (VM_FF_ISPENDING(pVM, VM_FF_PDM_DMA) && cResume > 1)
2496	{
2497	rc = VINF_EM_RAW_TO_R3;
2498	goto end;
2499	}
2500	}
2501
2502	#ifdef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
2503	/*
2504	* Exit to ring-3 preemption/work is pending.
2505	*
2506	* Interrupts are disabled before the call to make sure we don't miss any interrupt
2507	* that would flag preemption (IPI, timer tick, ++). (Would've been nice to do this
2508	* further down, but VMXR0CheckPendingInterrupt makes that impossible.)
2509	*
2510	* Note! Interrupts must be disabled done before we check for TLB flushes; TLB
2511	* shootdowns rely on this.
2512	*/
2513	uOldEFlags = ASMIntDisableFlags();
2514	if (RTThreadPreemptIsPending(NIL_RTTHREAD))
2515	{
2516	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitPreemptPending);
2517	rc = VINF_EM_RAW_INTERRUPT;
2518	goto end;
2519	}
2520	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
2521	#endif
2522
2523	/* When external interrupts are pending, we should exit the VM when IF is set. */
2524	/* Note! After VM_FF_INHIBIT_INTERRUPTS check!!! */
2525	rc = VMXR0CheckPendingInterrupt(pVM, pVCpu, pCtx);
2526	if (RT_FAILURE(rc))
2527	goto end;
2528
2529	/** @todo check timers?? */
2530
2531	/* TPR caching using CR8 is only available in 64 bits mode */
2532	/* Note the 32 bits exception for AMD (X86_CPUID_AMD_FEATURE_ECX_CR8L), but that appears missing in Intel CPUs */
2533	/* Note: we can't do this in LoadGuestState as PDMApicGetTPR can jump back to ring 3 (lock)!!!!! (no longer true) */
2534	/**
2535	* @todo query and update the TPR only when it could have been changed (mmio access & wrmsr (x2apic))
2536	*/
2537	if (fSetupTPRCaching)
2538	{
2539	/* TPR caching in CR8 */
2540	bool fPending;
2541
2542	rc2 = PDMApicGetTPR(pVCpu, &u8LastTPR, &fPending);
2543	AssertRC(rc2);
2544	/* The TPR can be found at offset 0x80 in the APIC mmio page. */
2545	pVCpu->hwaccm.s.vmx.pVAPIC[0x80] = u8LastTPR;
2546
2547	/* Two options here:
2548	* - external interrupt pending, but masked by the TPR value.
2549	* -> a CR8 update that lower the current TPR value should cause an exit
2550	* - no pending interrupts
2551	* -> We don't need to be explicitely notified. There are enough world switches for detecting pending interrupts.
2552	*/
2553	rc = VMXWriteVMCS(VMX_VMCS_CTRL_TPR_THRESHOLD, (fPending) ? (u8LastTPR >> 4) : 0); /* cr8 bits 3-0 correspond to bits 7-4 of the task priority mmio register. */
2554	AssertRC(VBOXSTRICTRC_VAL(rc));
2555
2556	if (pVM->hwaccm.s.fTPRPatchingActive)
2557	{
2558	Assert(!CPUMIsGuestInLongModeEx(pCtx));
2559	/* Our patch code uses LSTAR for TPR caching. */
2560	pCtx->msrLSTAR = u8LastTPR;
2561
2562	if (fPending)
2563	{
2564	/* A TPR change could activate a pending interrupt, so catch lstar writes. */
2565	vmxR0SetMSRPermission(pVCpu, MSR_K8_LSTAR, true, false);
2566	}
2567	else
2568	{
2569	/* No interrupts are pending, so we don't need to be explicitely notified.
2570	* There are enough world switches for detecting pending interrupts.
2571	*/
2572	vmxR0SetMSRPermission(pVCpu, MSR_K8_LSTAR, true, true);
2573	}
2574	}
2575	}
2576
2577	#if defined(HWACCM_VTX_WITH_EPT) && defined(LOG_ENABLED)
2578	if ( pVM->hwaccm.s.fNestedPaging
2579	# ifdef HWACCM_VTX_WITH_VPID
2580	\|\| pVM->hwaccm.s.vmx.fVPID
2581	# endif /* HWACCM_VTX_WITH_VPID */
2582	)
2583	{
2584	PHWACCM_CPUINFO pCpu;
2585
2586	pCpu = HWACCMR0GetCurrentCpu();
2587	if ( pVCpu->hwaccm.s.idLastCpu != pCpu->idCpu
2588	\|\| pVCpu->hwaccm.s.cTLBFlushes != pCpu->cTLBFlushes)
2589	{
2590	if (pVCpu->hwaccm.s.idLastCpu != pCpu->idCpu)
2591	LogFlow(("Force TLB flush due to rescheduling to a different cpu (%d vs %d)\n", pVCpu->hwaccm.s.idLastCpu, pCpu->idCpu));
2592	else
2593	LogFlow(("Force TLB flush due to changed TLB flush count (%x vs %x)\n", pVCpu->hwaccm.s.cTLBFlushes, pCpu->cTLBFlushes));
2594	}
2595	if (pCpu->fFlushTLB)
2596	LogFlow(("Force TLB flush: first time cpu %d is used -> flush\n", pCpu->idCpu));
2597	else
2598	if (pVCpu->hwaccm.s.fForceTLBFlush)
2599	LogFlow(("Manual TLB flush\n"));
2600	}
2601	#endif
2602	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
2603	PGMRZDynMapFlushAutoSet(pVCpu);
2604	#endif
2605
2606	/*
2607	* NOTE: DO NOT DO ANYTHING AFTER THIS POINT THAT MIGHT JUMP BACK TO RING 3!
2608	* (until the actual world switch)
2609	*/
2610	#ifdef VBOX_STRICT
2611	idCpuCheck = RTMpCpuId();
2612	#endif
2613	#ifdef LOG_ENABLED
2614	VMMR0LogFlushDisable(pVCpu);
2615	#endif
2616	/* Save the host state first. */
2617	if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_HOST_CONTEXT)
2618	{
2619	rc = VMXR0SaveHostState(pVM, pVCpu);
2620	if (RT_UNLIKELY(rc != VINF_SUCCESS))
2621	{
2622	VMMR0LogFlushEnable(pVCpu);
2623	goto end;
2624	}
2625	}
2626
2627	/* Load the guest state */
2628	if (!pVCpu->hwaccm.s.fContextUseFlags)
2629	{
2630	VMXR0LoadMinimalGuestState(pVM, pVCpu, pCtx);
2631	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatLoadMinimal);
2632	}
2633	else
2634	{
2635	rc = VMXR0LoadGuestState(pVM, pVCpu, pCtx);
2636	if (RT_UNLIKELY(rc != VINF_SUCCESS))
2637	{
2638	VMMR0LogFlushEnable(pVCpu);
2639	goto end;
2640	}
2641	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatLoadFull);
2642	}
2643
2644	#ifndef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
2645	/* Disable interrupts to make sure a poke will interrupt execution.
2646	* This must be done before we check for TLB flushes; TLB shootdowns rely on this.
2647	*/
2648	uOldEFlags = ASMIntDisableFlags();
2649	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
2650	#endif
2651
2652	/* Non-register state Guest Context */
2653	/** @todo change me according to cpu state */
2654	rc2 = VMXWriteVMCS(VMX_VMCS32_GUEST_ACTIVITY_STATE, VMX_CMS_GUEST_ACTIVITY_ACTIVE);
2655	AssertRC(rc2);
2656
2657	/** Set TLB flush state as checked until we return from the world switch. */
2658	ASMAtomicWriteU8(&pVCpu->hwaccm.s.fCheckedTLBFlush, true);
2659	/* Deal with tagged TLB setup and invalidation. */
2660	pVM->hwaccm.s.vmx.pfnSetupTaggedTLB(pVM, pVCpu);
2661
2662	/* Manual save and restore:
2663	* - General purpose registers except RIP, RSP
2664	*
2665	* Trashed:
2666	* - CR2 (we don't care)
2667	* - LDTR (reset to 0)
2668	* - DRx (presumably not changed at all)
2669	* - DR7 (reset to 0x400)
2670	* - EFLAGS (reset to RT_BIT(1); not relevant)
2671	*
2672	*/
2673
2674	/* All done! Let's start VM execution. */
2675	STAM_PROFILE_ADV_STOP_START(&pVCpu->hwaccm.s.StatEntry, &pVCpu->hwaccm.s.StatInGC, x);
2676	Assert(idCpuCheck == RTMpCpuId());
2677
2678	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
2679	pVCpu->hwaccm.s.vmx.VMCSCache.cResume = cResume;
2680	pVCpu->hwaccm.s.vmx.VMCSCache.u64TimeSwitch = RTTimeNanoTS();
2681	#endif
2682
2683	/* Save the current TPR value in the LSTAR msr so our patches can access it. */
2684	if (pVM->hwaccm.s.fTPRPatchingActive)
2685	{
2686	Assert(pVM->hwaccm.s.fTPRPatchingActive);
2687	u64OldLSTAR = ASMRdMsr(MSR_K8_LSTAR);
2688	ASMWrMsr(MSR_K8_LSTAR, u8LastTPR);
2689	}
2690
2691	TMNotifyStartOfExecution(pVCpu);
2692	#ifdef VBOX_WITH_KERNEL_USING_XMM
2693	rc = hwaccmR0VMXStartVMWrapXMM(pVCpu->hwaccm.s.fResumeVM, pCtx, &pVCpu->hwaccm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hwaccm.s.vmx.pfnStartVM);
2694	#else
2695	rc = pVCpu->hwaccm.s.vmx.pfnStartVM(pVCpu->hwaccm.s.fResumeVM, pCtx, &pVCpu->hwaccm.s.vmx.VMCSCache, pVM, pVCpu);
2696	#endif
2697	ASMAtomicWriteU8(&pVCpu->hwaccm.s.fCheckedTLBFlush, false);
2698	ASMAtomicIncU32(&pVCpu->hwaccm.s.cWorldSwitchExit);
2699	/* Possibly the last TSC value seen by the guest (too high) (only when we're in tsc offset mode). */
2700	if (!(pVCpu->hwaccm.s.vmx.proc_ctls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT))
2701	TMCpuTickSetLastSeen(pVCpu, ASMReadTSC() + pVCpu->hwaccm.s.vmx.u64TSCOffset - 0x400 /* guestimate of world switch overhead in clock ticks */);
2702
2703	TMNotifyEndOfExecution(pVCpu);
2704	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED);
2705	Assert(!(ASMGetFlags() & X86_EFL_IF));
2706
2707	/* Restore the host LSTAR msr if the guest could have changed it. */
2708	if (pVM->hwaccm.s.fTPRPatchingActive)
2709	{
2710	Assert(pVM->hwaccm.s.fTPRPatchingActive);
2711	pVCpu->hwaccm.s.vmx.pVAPIC[0x80] = pCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR);
2712	ASMWrMsr(MSR_K8_LSTAR, u64OldLSTAR);
2713	}
2714
2715	STAM_PROFILE_ADV_STOP_START(&pVCpu->hwaccm.s.StatInGC, &pVCpu->hwaccm.s.StatExit1, x);
2716	ASMSetFlags(uOldEFlags);
2717	#ifdef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
2718	uOldEFlags = ~(RTCCUINTREG)0;
2719	#endif
2720
2721	AssertMsg(!pVCpu->hwaccm.s.vmx.VMCSCache.Write.cValidEntries, ("pVCpu->hwaccm.s.vmx.VMCSCache.Write.cValidEntries=%d\n", pVCpu->hwaccm.s.vmx.VMCSCache.Write.cValidEntries));
2722
2723	/* In case we execute a goto ResumeExecution later on. */
2724	pVCpu->hwaccm.s.fResumeVM = true;
2725	pVCpu->hwaccm.s.fForceTLBFlush = false;
2726
2727	/*
2728	* !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
2729	* IMPORTANT: WE CAN'T DO ANY LOGGING OR OPERATIONS THAT CAN DO A LONGJMP BACK TO RING 3 BEFORE WE'VE SYNCED BACK (MOST OF) THE GUEST STATE
2730	* !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
2731	*/
2732
2733	if (RT_UNLIKELY(rc != VINF_SUCCESS))
2734	{
2735	VMXR0ReportWorldSwitchError(pVM, pVCpu, rc, pCtx);
2736	VMMR0LogFlushEnable(pVCpu);
2737	goto end;
2738	}
2739
2740	/* Success. Query the guest state and figure out what has happened. */
2741
2742	/* Investigate why there was a VM-exit. */
2743	rc2 = VMXReadCachedVMCS(VMX_VMCS32_RO_EXIT_REASON, &exitReason);
2744	STAM_COUNTER_INC(&pVCpu->hwaccm.s.paStatExitReasonR0[exitReason & MASK_EXITREASON_STAT]);
2745
2746	exitReason &= 0xffff; /* bit 0-15 contain the exit code. */
2747	rc2 \|= VMXReadCachedVMCS(VMX_VMCS32_RO_VM_INSTR_ERROR, &instrError);
2748	rc2 \|= VMXReadCachedVMCS(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &cbInstr);
2749	rc2 \|= VMXReadCachedVMCS(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &intInfo);
2750	/* might not be valid; depends on VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID. */
2751	rc2 \|= VMXReadCachedVMCS(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERRCODE, &errCode);
2752	rc2 \|= VMXReadCachedVMCS(VMX_VMCS32_RO_EXIT_INSTR_INFO, &instrInfo);
2753	rc2 \|= VMXReadCachedVMCS(VMX_VMCS_RO_EXIT_QUALIFICATION, &exitQualification);
2754	AssertRC(rc2);
2755
2756	/* Sync back the guest state */
2757	rc2 = VMXR0SaveGuestState(pVM, pVCpu, pCtx);
2758	AssertRC(rc2);
2759
2760	/* Note! NOW IT'S SAFE FOR LOGGING! */
2761	VMMR0LogFlushEnable(pVCpu);
2762	Log2(("Raw exit reason %08x\n", exitReason));
2763
2764	/* Check if an injected event was interrupted prematurely. */
2765	rc2 = VMXReadCachedVMCS(VMX_VMCS32_RO_IDT_INFO, &val);
2766	AssertRC(rc2);
2767	pVCpu->hwaccm.s.Event.intInfo = VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(val);
2768	if ( VMX_EXIT_INTERRUPTION_INFO_VALID(pVCpu->hwaccm.s.Event.intInfo)
2769	/* Ignore 'int xx' as they'll be restarted anyway. */
2770	&& VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hwaccm.s.Event.intInfo) != VMX_EXIT_INTERRUPTION_INFO_TYPE_SW
2771	/* Ignore software exceptions (such as int3) as they'll reoccur when we restart the instruction anyway. */
2772	&& VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hwaccm.s.Event.intInfo) != VMX_EXIT_INTERRUPTION_INFO_TYPE_SWEXCPT)
2773	{
2774	Assert(!pVCpu->hwaccm.s.Event.fPending);
2775	pVCpu->hwaccm.s.Event.fPending = true;
2776	/* Error code present? */
2777	if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(pVCpu->hwaccm.s.Event.intInfo))
2778	{
2779	rc2 = VMXReadCachedVMCS(VMX_VMCS32_RO_IDT_ERRCODE, &val);
2780	AssertRC(rc2);
2781	pVCpu->hwaccm.s.Event.errCode = val;
2782	Log(("Pending inject %RX64 at %RGv exit=%08x intInfo=%08x exitQualification=%RGv pending error=%RX64\n", pVCpu->hwaccm.s.Event.intInfo, (RTGCPTR)pCtx->rip, exitReason, intInfo, exitQualification, val));
2783	}
2784	else
2785	{
2786	Log(("Pending inject %RX64 at %RGv exit=%08x intInfo=%08x exitQualification=%RGv\n", pVCpu->hwaccm.s.Event.intInfo, (RTGCPTR)pCtx->rip, exitReason, intInfo, exitQualification));
2787	pVCpu->hwaccm.s.Event.errCode = 0;
2788	}
2789	}
2790	#ifdef VBOX_STRICT
2791	else
2792	if ( VMX_EXIT_INTERRUPTION_INFO_VALID(pVCpu->hwaccm.s.Event.intInfo)
2793	/* Ignore software exceptions (such as int3) as they're reoccur when we restart the instruction anyway. */
2794	&& VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hwaccm.s.Event.intInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_SWEXCPT)
2795	{
2796	Log(("Ignore pending inject %RX64 at %RGv exit=%08x intInfo=%08x exitQualification=%RGv\n", pVCpu->hwaccm.s.Event.intInfo, (RTGCPTR)pCtx->rip, exitReason, intInfo, exitQualification));
2797	}
2798
2799	if (exitReason == VMX_EXIT_ERR_INVALID_GUEST_STATE)
2800	HWACCMDumpRegs(pVM, pVCpu, pCtx);
2801	#endif
2802
2803	Log2(("E%d: New EIP=%x:%RGv\n", (uint32_t)exitReason, pCtx->cs, (RTGCPTR)pCtx->rip));
2804	Log2(("Exit reason %d, exitQualification %RGv\n", (uint32_t)exitReason, exitQualification));
2805	Log2(("instrInfo=%d instrError=%d instr length=%d\n", (uint32_t)instrInfo, (uint32_t)instrError, (uint32_t)cbInstr));
2806	Log2(("Interruption error code %d\n", (uint32_t)errCode));
2807	Log2(("IntInfo = %08x\n", (uint32_t)intInfo));
2808
2809	/* Sync back the TPR if it was changed. */
2810	if ( fSetupTPRCaching
2811	&& u8LastTPR != pVCpu->hwaccm.s.vmx.pVAPIC[0x80])
2812	{
2813	rc2 = PDMApicSetTPR(pVCpu, pVCpu->hwaccm.s.vmx.pVAPIC[0x80]);
2814	AssertRC(rc2);
2815	}
2816
2817	STAM_PROFILE_ADV_STOP_START(&pVCpu->hwaccm.s.StatExit1, &pVCpu->hwaccm.s.StatExit2, x);
2818
2819	/* Some cases don't need a complete resync of the guest CPU state; handle them here. */
2820	Assert(rc == VINF_SUCCESS); /* might consider VERR_IPE_UNINITIALIZED_STATUS here later... */
2821	switch (exitReason)
2822	{
2823	case VMX_EXIT_EXCEPTION: /* 0 Exception or non-maskable interrupt (NMI). */
2824	case VMX_EXIT_EXTERNAL_IRQ: /* 1 External interrupt. */
2825	{
2826	uint32_t vector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(intInfo);
2827
2828	if (!VMX_EXIT_INTERRUPTION_INFO_VALID(intInfo))
2829	{
2830	Assert(exitReason == VMX_EXIT_EXTERNAL_IRQ);
2831	#if 0 //def VBOX_WITH_VMMR0_DISABLE_PREEMPTION
2832	if ( RTThreadPreemptIsPendingTrusty()
2833	&& !RTThreadPreemptIsPending(NIL_RTTHREAD))
2834	goto ResumeExecution;
2835	#endif
2836	/* External interrupt; leave to allow it to be dispatched again. */
2837	rc = VINF_EM_RAW_INTERRUPT;
2838	break;
2839	}
2840	STAM_PROFILE_ADV_START(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
2841	switch (VMX_EXIT_INTERRUPTION_INFO_TYPE(intInfo))
2842	{
2843	case VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI: /* Non-maskable interrupt. */
2844	/* External interrupt; leave to allow it to be dispatched again. */
2845	rc = VINF_EM_RAW_INTERRUPT;
2846	break;
2847
2848	case VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT: /* External hardware interrupt. */
2849	AssertFailed(); /* can't come here; fails the first check. */
2850	break;
2851
2852	case VMX_EXIT_INTERRUPTION_INFO_TYPE_DBEXCPT: /* Unknown why we get this type for #DB */
2853	case VMX_EXIT_INTERRUPTION_INFO_TYPE_SWEXCPT: /* Software exception. (#BP or #OF) */
2854	Assert(vector == 1 \|\| vector == 3 \|\| vector == 4);
2855	/* no break */
2856	case VMX_EXIT_INTERRUPTION_INFO_TYPE_HWEXCPT: /* Hardware exception. */
2857	Log2(("Hardware/software interrupt %d\n", vector));
2858	switch (vector)
2859	{
2860	case X86_XCPT_NM:
2861	{
2862	Log(("#NM fault at %RGv error code %x\n", (RTGCPTR)pCtx->rip, errCode));
2863
2864	/** @todo don't intercept #NM exceptions anymore when we've activated the guest FPU state. */
2865	/* If we sync the FPU/XMM state on-demand, then we can continue execution as if nothing has happened. */
2866	rc = CPUMR0LoadGuestFPU(pVM, pVCpu, pCtx);
2867	if (rc == VINF_SUCCESS)
2868	{
2869	Assert(CPUMIsGuestFPUStateActive(pVCpu));
2870
2871	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitShadowNM);
2872
2873	/* Continue execution. */
2874	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_GUEST_CR0;
2875
2876	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
2877	goto ResumeExecution;
2878	}
2879
2880	Log(("Forward #NM fault to the guest\n"));
2881	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestNM);
2882	rc2 = VMXR0InjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), cbInstr, 0);
2883	AssertRC(rc2);
2884	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
2885	goto ResumeExecution;
2886	}
2887
2888	case X86_XCPT_PF: /* Page fault */
2889	{
2890	#ifdef DEBUG
2891	if (pVM->hwaccm.s.fNestedPaging)
2892	{ /* A genuine pagefault.
2893	* Forward the trap to the guest by injecting the exception and resuming execution.
2894	*/
2895	Log(("Guest page fault at %RGv cr2=%RGv error code %RGv rsp=%RGv\n", (RTGCPTR)pCtx->rip, exitQualification, errCode, (RTGCPTR)pCtx->rsp));
2896
2897	Assert(CPUMIsGuestInPagedProtectedModeEx(pCtx));
2898
2899	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestPF);
2900
2901	/* Now we must update CR2. */
2902	pCtx->cr2 = exitQualification;
2903	rc2 = VMXR0InjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), cbInstr, errCode);
2904	AssertRC(rc2);
2905
2906	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
2907	goto ResumeExecution;
2908	}
2909	#endif
2910	Assert(!pVM->hwaccm.s.fNestedPaging);
2911
2912	#ifdef VBOX_HWACCM_WITH_GUEST_PATCHING
2913	/* Shortcut for APIC TPR reads and writes; 32 bits guests only */
2914	if ( pVM->hwaccm.s.fTRPPatchingAllowed
2915	&& pVM->hwaccm.s.pGuestPatchMem
2916	&& (exitQualification & 0xfff) == 0x080
2917	&& !(errCode & X86_TRAP_PF_P) /* not present */
2918	&& CPUMGetGuestCPL(pVCpu, CPUMCTX2CORE(pCtx)) == 0
2919	&& !CPUMIsGuestInLongModeEx(pCtx)
2920	&& pVM->hwaccm.s.cPatches < RT_ELEMENTS(pVM->hwaccm.s.aPatches))
2921	{
2922	RTGCPHYS GCPhysApicBase, GCPhys;
2923	PDMApicGetBase(pVM, &GCPhysApicBase); /* @todo cache this */
2924	GCPhysApicBase &= PAGE_BASE_GC_MASK;
2925
2926	rc = PGMGstGetPage(pVCpu, (RTGCPTR)exitQualification, NULL, &GCPhys);
2927	if ( rc == VINF_SUCCESS
2928	&& GCPhys == GCPhysApicBase)
2929	{
2930	/* Only attempt to patch the instruction once. */
2931	PHWACCMTPRPATCH pPatch = (PHWACCMTPRPATCH)RTAvloU32Get(&pVM->hwaccm.s.PatchTree, (AVLOU32KEY)pCtx->eip);
2932	if (!pPatch)
2933	{
2934	rc = VINF_EM_HWACCM_PATCH_TPR_INSTR;
2935	break;
2936	}
2937	}
2938	}
2939	#endif
2940
2941	Log2(("Page fault at %RGv error code %x\n", exitQualification, errCode));
2942	/* Exit qualification contains the linear address of the page fault. */
2943	TRPMAssertTrap(pVCpu, X86_XCPT_PF, TRPM_TRAP);
2944	TRPMSetErrorCode(pVCpu, errCode);
2945	TRPMSetFaultAddress(pVCpu, exitQualification);
2946
2947	/* Shortcut for APIC TPR reads and writes. */
2948	if ( (exitQualification & 0xfff) == 0x080
2949	&& !(errCode & X86_TRAP_PF_P) /* not present */
2950	&& fSetupTPRCaching
2951	&& (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC))
2952	{
2953	RTGCPHYS GCPhysApicBase, GCPhys;
2954	PDMApicGetBase(pVM, &GCPhysApicBase); /* @todo cache this */
2955	GCPhysApicBase &= PAGE_BASE_GC_MASK;
2956
2957	rc = PGMGstGetPage(pVCpu, (RTGCPTR)exitQualification, NULL, &GCPhys);
2958	if ( rc == VINF_SUCCESS
2959	&& GCPhys == GCPhysApicBase)
2960	{
2961	Log(("Enable VT-x virtual APIC access filtering\n"));
2962	rc2 = IOMMMIOMapMMIOHCPage(pVM, GCPhysApicBase, pVM->hwaccm.s.vmx.pAPICPhys, X86_PTE_RW \| X86_PTE_P);
2963	AssertRC(rc2);
2964	}
2965	}
2966
2967	/* Forward it to our trap handler first, in case our shadow pages are out of sync. */
2968	rc = PGMTrap0eHandler(pVCpu, errCode, CPUMCTX2CORE(pCtx), (RTGCPTR)exitQualification);
2969	Log2(("PGMTrap0eHandler %RGv returned %Rrc\n", (RTGCPTR)pCtx->rip, VBOXSTRICTRC_VAL(rc)));
2970
2971	if (rc == VINF_SUCCESS)
2972	{ /* We've successfully synced our shadow pages, so let's just continue execution. */
2973	Log2(("Shadow page fault at %RGv cr2=%RGv error code %x\n", (RTGCPTR)pCtx->rip, exitQualification ,errCode));
2974	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitShadowPF);
2975
2976	TRPMResetTrap(pVCpu);
2977	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
2978	goto ResumeExecution;
2979	}
2980	else
2981	if (rc == VINF_EM_RAW_GUEST_TRAP)
2982	{ /* A genuine pagefault.
2983	* Forward the trap to the guest by injecting the exception and resuming execution.
2984	*/
2985	Log2(("Forward page fault to the guest\n"));
2986
2987	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestPF);
2988	/* The error code might have been changed. */
2989	errCode = TRPMGetErrorCode(pVCpu);
2990
2991	TRPMResetTrap(pVCpu);
2992
2993	/* Now we must update CR2. */
2994	pCtx->cr2 = exitQualification;
2995	rc2 = VMXR0InjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), cbInstr, errCode);
2996	AssertRC(rc2);
2997
2998	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
2999	goto ResumeExecution;
3000	}
3001	#ifdef VBOX_STRICT
3002	if (rc != VINF_EM_RAW_EMULATE_INSTR && rc != VINF_EM_RAW_EMULATE_IO_BLOCK)
3003	Log2(("PGMTrap0eHandler failed with %d\n", VBOXSTRICTRC_VAL(rc)));
3004	#endif
3005	/* Need to go back to the recompiler to emulate the instruction. */
3006	TRPMResetTrap(pVCpu);
3007	break;
3008	}
3009
3010	case X86_XCPT_MF: /* Floating point exception. */
3011	{
3012	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestMF);
3013	if (!(pCtx->cr0 & X86_CR0_NE))
3014	{
3015	/* old style FPU error reporting needs some extra work. */
3016	/** @todo don't fall back to the recompiler, but do it manually. */
3017	rc = VINF_EM_RAW_EMULATE_INSTR;
3018	break;
3019	}
3020	Log(("Trap %x at %04X:%RGv\n", vector, pCtx->cs, (RTGCPTR)pCtx->rip));
3021	rc2 = VMXR0InjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), cbInstr, errCode);
3022	AssertRC(rc2);
3023
3024	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3025	goto ResumeExecution;
3026	}
3027
3028	case X86_XCPT_DB: /* Debug exception. */
3029	{
3030	uint64_t uDR6;
3031
3032	/* DR6, DR7.GD and IA32_DEBUGCTL.LBR are not updated yet.
3033	*
3034	* Exit qualification bits:
3035	* 3:0 B0-B3 which breakpoint condition was met
3036	* 12:4 Reserved (0)
3037	* 13 BD - debug register access detected
3038	* 14 BS - single step execution or branch taken
3039	* 63:15 Reserved (0)
3040	*/
3041	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestDB);
3042
3043	/* Note that we don't support guest and host-initiated debugging at the same time. */
3044
3045	uDR6 = X86_DR6_INIT_VAL;
3046	uDR6 \|= (exitQualification & (X86_DR6_B0\|X86_DR6_B1\|X86_DR6_B2\|X86_DR6_B3\|X86_DR6_BD\|X86_DR6_BS));
3047	rc = DBGFRZTrap01Handler(pVM, pVCpu, CPUMCTX2CORE(pCtx), uDR6);
3048	if (rc == VINF_EM_RAW_GUEST_TRAP)
3049	{
3050	/* Update DR6 here. */
3051	pCtx->dr[6] = uDR6;
3052
3053	/* Resync DR6 if the debug state is active. */
3054	if (CPUMIsGuestDebugStateActive(pVCpu))
3055	ASMSetDR6(pCtx->dr[6]);
3056
3057	/* X86_DR7_GD will be cleared if drx accesses should be trapped inside the guest. */
3058	pCtx->dr[7] &= ~X86_DR7_GD;
3059
3060	/* Paranoia. */
3061	pCtx->dr[7] &= 0xffffffff; /* upper 32 bits reserved */
3062	pCtx->dr[7] &= ~(RT_BIT(11) \| RT_BIT(12) \| RT_BIT(14) \| RT_BIT(15)); /* must be zero */
3063	pCtx->dr[7] \|= 0x400; /* must be one */
3064
3065	/* Resync DR7 */
3066	rc2 = VMXWriteVMCS64(VMX_VMCS64_GUEST_DR7, pCtx->dr[7]);
3067	AssertRC(rc2);
3068
3069	Log(("Trap %x (debug) at %RGv exit qualification %RX64 dr6=%x dr7=%x\n", vector, (RTGCPTR)pCtx->rip, exitQualification, (uint32_t)pCtx->dr[6], (uint32_t)pCtx->dr[7]));
3070	rc2 = VMXR0InjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), cbInstr, errCode);
3071	AssertRC(rc2);
3072
3073	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3074	goto ResumeExecution;
3075	}
3076	/* Return to ring 3 to deal with the debug exit code. */
3077	Log(("Debugger hardware BP at %04x:%RGv (rc=%Rrc)\n", pCtx->cs, pCtx->rip, VBOXSTRICTRC_VAL(rc)));
3078	break;
3079	}
3080
3081	case X86_XCPT_BP: /* Breakpoint. */
3082	{
3083	rc = DBGFRZTrap03Handler(pVM, pVCpu, CPUMCTX2CORE(pCtx));
3084	if (rc == VINF_EM_RAW_GUEST_TRAP)
3085	{
3086	Log(("Guest #BP at %04x:%RGv\n", pCtx->cs, pCtx->rip));
3087	rc2 = VMXR0InjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), cbInstr, errCode);
3088	AssertRC(rc2);
3089	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3090	goto ResumeExecution;
3091	}
3092	if (rc == VINF_SUCCESS)
3093	{
3094	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3095	goto ResumeExecution;
3096	}
3097	Log(("Debugger BP at %04x:%RGv (rc=%Rrc)\n", pCtx->cs, pCtx->rip, VBOXSTRICTRC_VAL(rc)));
3098	break;
3099	}
3100
3101	case X86_XCPT_GP: /* General protection failure exception.*/
3102	{
3103	uint32_t cbOp;
3104	uint32_t cbSize;
3105	PDISCPUSTATE pDis = &pVCpu->hwaccm.s.DisState;
3106
3107	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestGP);
3108	#ifdef VBOX_STRICT
3109	if ( !CPUMIsGuestInRealModeEx(pCtx)
3110	\|\| !pVM->hwaccm.s.vmx.pRealModeTSS)
3111	{
3112	Log(("Trap %x at %04X:%RGv errorCode=%RGv\n", vector, pCtx->cs, (RTGCPTR)pCtx->rip, errCode));
3113	rc2 = VMXR0InjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), cbInstr, errCode);
3114	AssertRC(rc2);
3115	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3116	goto ResumeExecution;
3117	}
3118	#endif
3119	Assert(CPUMIsGuestInRealModeEx(pCtx));
3120
3121	LogFlow(("Real mode X86_XCPT_GP instruction emulation at %x:%RGv\n", pCtx->cs, (RTGCPTR)pCtx->rip));
3122
3123	rc2 = EMInterpretDisasOne(pVM, pVCpu, CPUMCTX2CORE(pCtx), pDis, &cbOp);
3124	if (RT_SUCCESS(rc2))
3125	{
3126	bool fUpdateRIP = true;
3127
3128	rc = VINF_SUCCESS;
3129	Assert(cbOp == pDis->opsize);
3130	switch (pDis->pCurInstr->opcode)
3131	{
3132	case OP_CLI:
3133	pCtx->eflags.Bits.u1IF = 0;
3134	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitCli);
3135	break;
3136
3137	case OP_STI:
3138	pCtx->eflags.Bits.u1IF = 1;
3139	EMSetInhibitInterruptsPC(pVCpu, pCtx->rip + pDis->opsize);
3140	Assert(VMCPU_FF_ISSET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
3141	rc2 = VMXWriteVMCS(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
3142	AssertRC(rc2);
3143	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitSti);
3144	break;
3145
3146	case OP_HLT:
3147	fUpdateRIP = false;
3148	rc = VINF_EM_HALT;
3149	pCtx->rip += pDis->opsize;
3150	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitHlt);
3151	break;
3152
3153	case OP_POPF:
3154	{
3155	RTGCPTR GCPtrStack;
3156	uint32_t cbParm;
3157	uint32_t uMask;
3158	X86EFLAGS eflags;
3159
3160	if (pDis->prefix & PREFIX_OPSIZE)
3161	{
3162	cbParm = 4;
3163	uMask = 0xffffffff;
3164	}
3165	else
3166	{
3167	cbParm = 2;
3168	uMask = 0xffff;
3169	}
3170
3171	rc2 = SELMToFlatEx(pVM, DIS_SELREG_SS, CPUMCTX2CORE(pCtx), pCtx->esp & uMask, 0, &GCPtrStack);
3172	if (RT_FAILURE(rc2))
3173	{
3174	rc = VERR_EM_INTERPRETER;
3175	break;
3176	}
3177	eflags.u = 0;
3178	rc2 = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &eflags.u, cbParm);
3179	if (RT_FAILURE(rc2))
3180	{
3181	rc = VERR_EM_INTERPRETER;
3182	break;
3183	}
3184	LogFlow(("POPF %x -> %RGv mask=%x\n", eflags.u, pCtx->rsp, uMask));
3185	pCtx->eflags.u = (pCtx->eflags.u & ~(X86_EFL_POPF_BITS & uMask)) \| (eflags.u & X86_EFL_POPF_BITS & uMask);
3186	/* RF cleared when popped in real mode; see pushf description in AMD manual. */
3187	pCtx->eflags.Bits.u1RF = 0;
3188	pCtx->esp += cbParm;
3189	pCtx->esp &= uMask;
3190
3191	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitPopf);
3192	break;
3193	}
3194
3195	case OP_PUSHF:
3196	{
3197	RTGCPTR GCPtrStack;
3198	uint32_t cbParm;
3199	uint32_t uMask;
3200	X86EFLAGS eflags;
3201
3202	if (pDis->prefix & PREFIX_OPSIZE)
3203	{
3204	cbParm = 4;
3205	uMask = 0xffffffff;
3206	}
3207	else
3208	{
3209	cbParm = 2;
3210	uMask = 0xffff;
3211	}
3212
3213	rc2 = SELMToFlatEx(pVM, DIS_SELREG_SS, CPUMCTX2CORE(pCtx), (pCtx->esp - cbParm) & uMask, 0, &GCPtrStack);
3214	if (RT_FAILURE(rc2))
3215	{
3216	rc = VERR_EM_INTERPRETER;
3217	break;
3218	}
3219	eflags = pCtx->eflags;
3220	/* RF & VM cleared when pushed in real mode; see pushf description in AMD manual. */
3221	eflags.Bits.u1RF = 0;
3222	eflags.Bits.u1VM = 0;
3223
3224	rc2 = PGMPhysWrite(pVM, (RTGCPHYS)GCPtrStack, &eflags.u, cbParm);
3225	if (RT_FAILURE(rc2))
3226	{
3227	rc = VERR_EM_INTERPRETER;
3228	break;
3229	}
3230	LogFlow(("PUSHF %x -> %RGv\n", eflags.u, GCPtrStack));
3231	pCtx->esp -= cbParm;
3232	pCtx->esp &= uMask;
3233	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitPushf);
3234	break;
3235	}
3236
3237	case OP_IRET:
3238	{
3239	RTGCPTR GCPtrStack;
3240	uint32_t uMask = 0xffff;
3241	uint16_t aIretFrame[3];
3242
3243	if (pDis->prefix & (PREFIX_OPSIZE \| PREFIX_ADDRSIZE))
3244	{
3245	rc = VERR_EM_INTERPRETER;
3246	break;
3247	}
3248
3249	rc2 = SELMToFlatEx(pVM, DIS_SELREG_SS, CPUMCTX2CORE(pCtx), pCtx->esp & uMask, 0, &GCPtrStack);
3250	if (RT_FAILURE(rc2))
3251	{
3252	rc = VERR_EM_INTERPRETER;
3253	break;
3254	}
3255	rc2 = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &aIretFrame[0], sizeof(aIretFrame));
3256	if (RT_FAILURE(rc2))
3257	{
3258	rc = VERR_EM_INTERPRETER;
3259	break;
3260	}
3261	pCtx->ip = aIretFrame[0];
3262	pCtx->cs = aIretFrame[1];
3263	pCtx->csHid.u64Base = pCtx->cs << 4;
3264	pCtx->eflags.u = (pCtx->eflags.u & ~(X86_EFL_POPF_BITS & uMask)) \| (aIretFrame[2] & X86_EFL_POPF_BITS & uMask);
3265	pCtx->sp += sizeof(aIretFrame);
3266
3267	LogFlow(("iret to %04x:%x\n", pCtx->cs, pCtx->ip));
3268	fUpdateRIP = false;
3269	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitIret);
3270	break;
3271	}
3272
3273	case OP_INT:
3274	{
3275	uint32_t intInfo2;
3276
3277	LogFlow(("Realmode: INT %x\n", pDis->param1.parval & 0xff));
3278	intInfo2 = pDis->param1.parval & 0xff;
3279	intInfo2 \|= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
3280	intInfo2 \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
3281
3282	rc = VMXR0InjectEvent(pVM, pVCpu, pCtx, intInfo2, cbOp, 0);
3283	AssertRC(VBOXSTRICTRC_VAL(rc));
3284	fUpdateRIP = false;
3285	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitInt);
3286	break;
3287	}
3288
3289	case OP_INTO:
3290	{
3291	if (pCtx->eflags.Bits.u1OF)
3292	{
3293	uint32_t intInfo2;
3294
3295	LogFlow(("Realmode: INTO\n"));
3296	intInfo2 = X86_XCPT_OF;
3297	intInfo2 \|= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
3298	intInfo2 \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
3299
3300	rc = VMXR0InjectEvent(pVM, pVCpu, pCtx, intInfo2, cbOp, 0);
3301	AssertRC(VBOXSTRICTRC_VAL(rc));
3302	fUpdateRIP = false;
3303	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitInt);
3304	}
3305	break;
3306	}
3307
3308	case OP_INT3:
3309	{
3310	uint32_t intInfo2;
3311
3312	LogFlow(("Realmode: INT 3\n"));
3313	intInfo2 = 3;
3314	intInfo2 \|= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
3315	intInfo2 \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
3316
3317	rc = VMXR0InjectEvent(pVM, pVCpu, pCtx, intInfo2, cbOp, 0);
3318	AssertRC(VBOXSTRICTRC_VAL(rc));
3319	fUpdateRIP = false;
3320	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitInt);
3321	break;
3322	}
3323
3324	default:
3325	rc = EMInterpretInstructionCPU(pVM, pVCpu, pDis, CPUMCTX2CORE(pCtx), 0, EMCODETYPE_SUPERVISOR, &cbSize);
3326	break;
3327	}
3328
3329	if (rc == VINF_SUCCESS)
3330	{
3331	if (fUpdateRIP)
3332	pCtx->rip += cbOp; /* Move on to the next instruction. */
3333
3334	/* lidt, lgdt can end up here. In the future crx changes as well. Just reload the whole context to be done with it. */
3335	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_ALL;
3336
3337	/* Only resume if successful. */
3338	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3339	goto ResumeExecution;
3340	}
3341	}
3342	else
3343	rc = VERR_EM_INTERPRETER;
3344
3345	AssertMsg(rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_EM_HALT, ("Unexpected rc=%Rrc\n", VBOXSTRICTRC_VAL(rc)));
3346	break;
3347	}
3348
3349	#ifdef VBOX_STRICT
3350	case X86_XCPT_XF: /* SIMD exception. */
3351	case X86_XCPT_DE: /* Divide error. */
3352	case X86_XCPT_UD: /* Unknown opcode exception. */
3353	case X86_XCPT_SS: /* Stack segment exception. */
3354	case X86_XCPT_NP: /* Segment not present exception. */
3355	{
3356	switch(vector)
3357	{
3358	case X86_XCPT_DE:
3359	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestDE);
3360	break;
3361	case X86_XCPT_UD:
3362	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestUD);
3363	break;
3364	case X86_XCPT_SS:
3365	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestSS);
3366	break;
3367	case X86_XCPT_NP:
3368	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestNP);
3369	break;
3370	}
3371
3372	Log(("Trap %x at %04X:%RGv\n", vector, pCtx->cs, (RTGCPTR)pCtx->rip));
3373	rc2 = VMXR0InjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), cbInstr, errCode);
3374	AssertRC(rc2);
3375
3376	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3377	goto ResumeExecution;
3378	}
3379	#endif
3380	default:
3381	if ( CPUMIsGuestInRealModeEx(pCtx)
3382	&& pVM->hwaccm.s.vmx.pRealModeTSS)
3383	{
3384	Log(("Real Mode Trap %x at %04x:%04X error code %x\n", vector, pCtx->cs, pCtx->eip, errCode));
3385	rc = VMXR0InjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), cbInstr, errCode);
3386	AssertRC(VBOXSTRICTRC_VAL(rc)); /* Strict RC check below. */
3387
3388	/* Go back to ring 3 in case of a triple fault. */
3389	if ( vector == X86_XCPT_DF
3390	&& rc == VINF_EM_RESET)
3391	break;
3392
3393	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3394	goto ResumeExecution;
3395	}
3396	AssertMsgFailed(("Unexpected vm-exit caused by exception %x\n", vector));
3397	rc = VERR_VMX_UNEXPECTED_EXCEPTION;
3398	break;
3399	} /* switch (vector) */
3400
3401	break;
3402
3403	default:
3404	rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_CODE;
3405	AssertMsgFailed(("Unexpected interruption code %x\n", intInfo));
3406	break;
3407	}
3408
3409	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
3410	break;
3411	}
3412
3413	case VMX_EXIT_EPT_VIOLATION: /* 48 EPT violation. An attempt to access memory with a guest-physical address was disallowed by the configuration of the EPT paging structures. */
3414	{
3415	RTGCPHYS GCPhys;
3416
3417	Assert(pVM->hwaccm.s.fNestedPaging);
3418
3419	rc2 = VMXReadVMCS64(VMX_VMCS_EXIT_PHYS_ADDR_FULL, &GCPhys);
3420	AssertRC(rc2);
3421	Assert(((exitQualification >> 7) & 3) != 2);
3422
3423	/* Determine the kind of violation. */
3424	errCode = 0;
3425	if (exitQualification & VMX_EXIT_QUALIFICATION_EPT_INSTR_FETCH)
3426	errCode \|= X86_TRAP_PF_ID;
3427
3428	if (exitQualification & VMX_EXIT_QUALIFICATION_EPT_DATA_WRITE)
3429	errCode \|= X86_TRAP_PF_RW;
3430
3431	/* If the page is present, then it's a page level protection fault. */
3432	if (exitQualification & VMX_EXIT_QUALIFICATION_EPT_ENTRY_PRESENT)
3433	{
3434	errCode \|= X86_TRAP_PF_P;
3435	}
3436	else
3437	{
3438	/* Shortcut for APIC TPR reads and writes. */
3439	if ( (GCPhys & 0xfff) == 0x080
3440	&& GCPhys > 0x1000000 /* to skip VGA frame buffer accesses */
3441	&& fSetupTPRCaching
3442	&& (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC))
3443	{
3444	RTGCPHYS GCPhysApicBase;
3445	PDMApicGetBase(pVM, &GCPhysApicBase); /* @todo cache this */
3446	GCPhysApicBase &= PAGE_BASE_GC_MASK;
3447	if (GCPhys == GCPhysApicBase + 0x80)
3448	{
3449	Log(("Enable VT-x virtual APIC access filtering\n"));
3450	rc2 = IOMMMIOMapMMIOHCPage(pVM, GCPhysApicBase, pVM->hwaccm.s.vmx.pAPICPhys, X86_PTE_RW \| X86_PTE_P);
3451	AssertRC(rc2);
3452	}
3453	}
3454	}
3455	Log(("EPT Page fault %x at %RGp error code %x\n", (uint32_t)exitQualification, GCPhys, errCode));
3456
3457	/* GCPhys contains the guest physical address of the page fault. */
3458	TRPMAssertTrap(pVCpu, X86_XCPT_PF, TRPM_TRAP);
3459	TRPMSetErrorCode(pVCpu, errCode);
3460	TRPMSetFaultAddress(pVCpu, GCPhys);
3461
3462	/* Handle the pagefault trap for the nested shadow table. */
3463	rc = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, errCode, CPUMCTX2CORE(pCtx), GCPhys);
3464	Log2(("PGMR0Trap0eHandlerNestedPaging %RGv returned %Rrc\n", (RTGCPTR)pCtx->rip, VBOXSTRICTRC_VAL(rc)));
3465	if (rc == VINF_SUCCESS)
3466	{ /* We've successfully synced our shadow pages, so let's just continue execution. */
3467	Log2(("Shadow page fault at %RGv cr2=%RGp error code %x\n", (RTGCPTR)pCtx->rip, exitQualification , errCode));
3468	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitReasonNPF);
3469
3470	TRPMResetTrap(pVCpu);
3471	goto ResumeExecution;
3472	}
3473
3474	#ifdef VBOX_STRICT
3475	if (rc != VINF_EM_RAW_EMULATE_INSTR)
3476	LogFlow(("PGMTrap0eHandlerNestedPaging failed with %d\n", VBOXSTRICTRC_VAL(rc)));
3477	#endif
3478	/* Need to go back to the recompiler to emulate the instruction. */
3479	TRPMResetTrap(pVCpu);
3480	break;
3481	}
3482
3483	case VMX_EXIT_EPT_MISCONFIG:
3484	{
3485	RTGCPHYS GCPhys;
3486
3487	Assert(pVM->hwaccm.s.fNestedPaging);
3488
3489	rc2 = VMXReadVMCS64(VMX_VMCS_EXIT_PHYS_ADDR_FULL, &GCPhys);
3490	AssertRC(rc2);
3491	Log(("VMX_EXIT_EPT_MISCONFIG for %RGp\n", GCPhys));
3492
3493	/* Shortcut for APIC TPR reads and writes. */
3494	if ( (GCPhys & 0xfff) == 0x080
3495	&& GCPhys > 0x1000000 /* to skip VGA frame buffer accesses */
3496	&& fSetupTPRCaching
3497	&& (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC))
3498	{
3499	RTGCPHYS GCPhysApicBase;
3500	PDMApicGetBase(pVM, &GCPhysApicBase); /* @todo cache this */
3501	GCPhysApicBase &= PAGE_BASE_GC_MASK;
3502	if (GCPhys == GCPhysApicBase + 0x80)
3503	{
3504	Log(("Enable VT-x virtual APIC access filtering\n"));
3505	rc2 = IOMMMIOMapMMIOHCPage(pVM, GCPhysApicBase, pVM->hwaccm.s.vmx.pAPICPhys, X86_PTE_RW \| X86_PTE_P);
3506	AssertRC(rc2);
3507	}
3508	}
3509
3510	rc = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pCtx), GCPhys, UINT32_MAX);
3511	if (rc == VINF_SUCCESS)
3512	{
3513	Log2(("PGMR0Trap0eHandlerNPMisconfig(,,,%RGp) at %RGv -> resume\n", GCPhys, (RTGCPTR)pCtx->rip));
3514	goto ResumeExecution;
3515	}
3516
3517	Log2(("PGMR0Trap0eHandlerNPMisconfig(,,,%RGp) at %RGv -> %Rrc\n", GCPhys, (RTGCPTR)pCtx->rip, VBOXSTRICTRC_VAL(rc)));
3518	break;
3519	}
3520
3521	case VMX_EXIT_IRQ_WINDOW: /* 7 Interrupt window. */
3522	/* Clear VM-exit on IF=1 change. */
3523	LogFlow(("VMX_EXIT_IRQ_WINDOW %RGv pending=%d IF=%d\n", (RTGCPTR)pCtx->rip, VMCPU_FF_ISPENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC\|VMCPU_FF_INTERRUPT_PIC)), pCtx->eflags.Bits.u1IF));
3524	pVCpu->hwaccm.s.vmx.proc_ctls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_IRQ_WINDOW_EXIT;
3525	rc2 = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
3526	AssertRC(rc2);
3527	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitIrqWindow);
3528	goto ResumeExecution; /* we check for pending guest interrupts there */
3529
3530	case VMX_EXIT_WBINVD: /* 54 Guest software attempted to execute WBINVD. (conditional) */
3531	case VMX_EXIT_INVD: /* 13 Guest software attempted to execute INVD. (unconditional) */
3532	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitInvd);
3533	/* Skip instruction and continue directly. */
3534	pCtx->rip += cbInstr;
3535	/* Continue execution.*/
3536	goto ResumeExecution;
3537
3538	case VMX_EXIT_CPUID: /* 10 Guest software attempted to execute CPUID. */
3539	{
3540	Log2(("VMX: Cpuid %x\n", pCtx->eax));
3541	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitCpuid);
3542	rc = EMInterpretCpuId(pVM, pVCpu, CPUMCTX2CORE(pCtx));
3543	if (rc == VINF_SUCCESS)
3544	{
3545	/* Update EIP and continue execution. */
3546	Assert(cbInstr == 2);
3547	pCtx->rip += cbInstr;
3548	goto ResumeExecution;
3549	}
3550	AssertMsgFailed(("EMU: cpuid failed with %Rrc\n", VBOXSTRICTRC_VAL(rc)));
3551	rc = VINF_EM_RAW_EMULATE_INSTR;
3552	break;
3553	}
3554
3555	case VMX_EXIT_RDPMC: /* 15 Guest software attempted to execute RDPMC. */
3556	{
3557	Log2(("VMX: Rdpmc %x\n", pCtx->ecx));
3558	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitRdpmc);
3559	rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pCtx));
3560	if (rc == VINF_SUCCESS)
3561	{
3562	/* Update EIP and continue execution. */
3563	Assert(cbInstr == 2);
3564	pCtx->rip += cbInstr;
3565	goto ResumeExecution;
3566	}
3567	rc = VINF_EM_RAW_EMULATE_INSTR;
3568	break;
3569	}
3570
3571	case VMX_EXIT_RDTSC: /* 16 Guest software attempted to execute RDTSC. */
3572	{
3573	Log2(("VMX: Rdtsc\n"));
3574	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitRdtsc);
3575	rc = EMInterpretRdtsc(pVM, pVCpu, CPUMCTX2CORE(pCtx));
3576	if (rc == VINF_SUCCESS)
3577	{
3578	/* Update EIP and continue execution. */
3579	Assert(cbInstr == 2);
3580	pCtx->rip += cbInstr;
3581	goto ResumeExecution;
3582	}
3583	rc = VINF_EM_RAW_EMULATE_INSTR;
3584	break;
3585	}
3586
3587	case VMX_EXIT_INVPG: /* 14 Guest software attempted to execute INVPG. */
3588	{
3589	Log2(("VMX: invlpg\n"));
3590	Assert(!pVM->hwaccm.s.fNestedPaging);
3591
3592	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitInvpg);
3593	rc = EMInterpretInvlpg(pVM, pVCpu, CPUMCTX2CORE(pCtx), exitQualification);
3594	if (rc == VINF_SUCCESS)
3595	{
3596	/* Update EIP and continue execution. */
3597	pCtx->rip += cbInstr;
3598	goto ResumeExecution;
3599	}
3600	AssertMsg(rc == VERR_EM_INTERPRETER, ("EMU: invlpg %RGv failed with %Rrc\n", exitQualification, VBOXSTRICTRC_VAL(rc)));
3601	break;
3602	}
3603
3604	case VMX_EXIT_MONITOR: /* 39 Guest software attempted to execute MONITOR. */
3605	{
3606	Log2(("VMX: monitor\n"));
3607
3608	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitMonitor);
3609	rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pCtx));
3610	if (rc == VINF_SUCCESS)
3611	{
3612	/* Update EIP and continue execution. */
3613	pCtx->rip += cbInstr;
3614	goto ResumeExecution;
3615	}
3616	AssertMsg(rc == VERR_EM_INTERPRETER, ("EMU: monitor failed with %Rrc\n", VBOXSTRICTRC_VAL(rc)));
3617	break;
3618	}
3619
3620	case VMX_EXIT_WRMSR: /* 32 WRMSR. Guest software attempted to execute WRMSR. */
3621	/* When an interrupt is pending, we'll let MSR_K8_LSTAR writes fault in our TPR patch code. */
3622	if ( pVM->hwaccm.s.fTPRPatchingActive
3623	&& pCtx->ecx == MSR_K8_LSTAR)
3624	{
3625	Assert(!CPUMIsGuestInLongModeEx(pCtx));
3626	if ((pCtx->eax & 0xff) != u8LastTPR)
3627	{
3628	Log(("VMX: Faulting MSR_K8_LSTAR write with new TPR value %x\n", pCtx->eax & 0xff));
3629
3630	/* Our patch code uses LSTAR for TPR caching. */
3631	rc2 = PDMApicSetTPR(pVCpu, pCtx->eax & 0xff);
3632	AssertRC(rc2);
3633	}
3634
3635	/* Skip the instruction and continue. */
3636	pCtx->rip += cbInstr; /* wrmsr = [0F 30] */
3637
3638	/* Only resume if successful. */
3639	goto ResumeExecution;
3640	}
3641	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_GUEST_MSR;
3642	/* no break */
3643	case VMX_EXIT_RDMSR: /* 31 RDMSR. Guest software attempted to execute RDMSR. */
3644	{
3645	uint32_t cbSize;
3646
3647	STAM_COUNTER_INC((exitReason == VMX_EXIT_RDMSR) ? &pVCpu->hwaccm.s.StatExitRdmsr : &pVCpu->hwaccm.s.StatExitWrmsr);
3648
3649	/* Note: the intel manual claims there's a REX version of RDMSR that's slightly different, so we play safe by completely disassembling the instruction. */
3650	Log2(("VMX: %s\n", (exitReason == VMX_EXIT_RDMSR) ? "rdmsr" : "wrmsr"));
3651	rc = EMInterpretInstruction(pVM, pVCpu, CPUMCTX2CORE(pCtx), 0, &cbSize);
3652	if (rc == VINF_SUCCESS)
3653	{
3654	/* EIP has been updated already. */
3655
3656	/* Only resume if successful. */
3657	goto ResumeExecution;
3658	}
3659	AssertMsg(rc == VERR_EM_INTERPRETER, ("EMU: %s failed with %Rrc\n", (exitReason == VMX_EXIT_RDMSR) ? "rdmsr" : "wrmsr", VBOXSTRICTRC_VAL(rc)));
3660	break;
3661	}
3662
3663	case VMX_EXIT_CRX_MOVE: /* 28 Control-register accesses. */
3664	{
3665	STAM_PROFILE_ADV_START(&pVCpu->hwaccm.s.StatExit2Sub2, y2);
3666
3667	switch (VMX_EXIT_QUALIFICATION_CRX_ACCESS(exitQualification))
3668	{
3669	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_WRITE:
3670	Log2(("VMX: %RGv mov cr%d, x\n", (RTGCPTR)pCtx->rip, VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification)));
3671	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitCRxWrite[VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification)]);
3672	rc = EMInterpretCRxWrite(pVM, pVCpu, CPUMCTX2CORE(pCtx),
3673	VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification),
3674	VMX_EXIT_QUALIFICATION_CRX_GENREG(exitQualification));
3675
3676	switch (VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification))
3677	{
3678	case 0:
3679	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_GUEST_CR0 \| HWACCM_CHANGED_GUEST_CR3;
3680	break;
3681	case 2:
3682	break;
3683	case 3:
3684	Assert(!pVM->hwaccm.s.fNestedPaging \|\| !CPUMIsGuestInPagedProtectedModeEx(pCtx));
3685	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_GUEST_CR3;
3686	break;
3687	case 4:
3688	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_GUEST_CR4;
3689	break;
3690	case 8:
3691	/* CR8 contains the APIC TPR */
3692	Assert(!(pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW));
3693	break;
3694
3695	default:
3696	AssertFailed();
3697	break;
3698	}
3699	break;
3700
3701	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ:
3702	Log2(("VMX: mov x, crx\n"));
3703	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitCRxRead[VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification)]);
3704
3705	Assert(!pVM->hwaccm.s.fNestedPaging \|\| !CPUMIsGuestInPagedProtectedModeEx(pCtx) \|\| VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification) != USE_REG_CR3);
3706
3707	/* CR8 reads only cause an exit when the TPR shadow feature isn't present. */
3708	Assert(VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification) != 8 \|\| !(pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW));
3709
3710	rc = EMInterpretCRxRead(pVM, pVCpu, CPUMCTX2CORE(pCtx),
3711	VMX_EXIT_QUALIFICATION_CRX_GENREG(exitQualification),
3712	VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification));
3713	break;
3714
3715	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_CLTS:
3716	Log2(("VMX: clts\n"));
3717	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitCLTS);
3718	rc = EMInterpretCLTS(pVM, pVCpu);
3719	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_GUEST_CR0;
3720	break;
3721
3722	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_LMSW:
3723	Log2(("VMX: lmsw %x\n", VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(exitQualification)));
3724	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitLMSW);
3725	rc = EMInterpretLMSW(pVM, pVCpu, CPUMCTX2CORE(pCtx), VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(exitQualification));
3726	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_GUEST_CR0;
3727	break;
3728	}
3729
3730	/* Update EIP if no error occurred. */
3731	if (RT_SUCCESS(rc))
3732	pCtx->rip += cbInstr;
3733
3734	if (rc == VINF_SUCCESS)
3735	{
3736	/* Only resume if successful. */
3737	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub2, y2);
3738	goto ResumeExecution;
3739	}
3740	Assert(rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_PGM_SYNC_CR3);
3741	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub2, y2);
3742	break;
3743	}
3744
3745	case VMX_EXIT_DRX_MOVE: /* 29 Debug-register accesses. */
3746	{
3747	if ( !DBGFIsStepping(pVCpu)
3748	&& !CPUMIsHyperDebugStateActive(pVCpu))
3749	{
3750	/* Disable drx move intercepts. */
3751	pVCpu->hwaccm.s.vmx.proc_ctls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT;
3752	rc2 = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
3753	AssertRC(rc2);
3754
3755	/* Save the host and load the guest debug state. */
3756	rc2 = CPUMR0LoadGuestDebugState(pVM, pVCpu, pCtx, true /* include DR6 */);
3757	AssertRC(rc2);
3758
3759	#ifdef LOG_ENABLED
3760	if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(exitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
3761	Log(("VMX_EXIT_DRX_MOVE: write DR%d genreg %d\n", VMX_EXIT_QUALIFICATION_DRX_REGISTER(exitQualification), VMX_EXIT_QUALIFICATION_DRX_GENREG(exitQualification)));
3762	else
3763	Log(("VMX_EXIT_DRX_MOVE: read DR%d\n", VMX_EXIT_QUALIFICATION_DRX_REGISTER(exitQualification)));
3764	#endif
3765
3766	#ifdef VBOX_WITH_STATISTICS
3767	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatDRxContextSwitch);
3768	if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(exitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
3769	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitDRxWrite);
3770	else
3771	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitDRxRead);
3772	#endif
3773
3774	goto ResumeExecution;
3775	}
3776
3777	/** @todo clear VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT after the first time and restore drx registers afterwards */
3778	if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(exitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
3779	{
3780	Log2(("VMX: mov drx%d, genreg%d\n", VMX_EXIT_QUALIFICATION_DRX_REGISTER(exitQualification), VMX_EXIT_QUALIFICATION_DRX_GENREG(exitQualification)));
3781	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitDRxWrite);
3782	rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pCtx),
3783	VMX_EXIT_QUALIFICATION_DRX_REGISTER(exitQualification),
3784	VMX_EXIT_QUALIFICATION_DRX_GENREG(exitQualification));
3785	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_GUEST_DEBUG;
3786	Log2(("DR7=%08x\n", pCtx->dr[7]));
3787	}
3788	else
3789	{
3790	Log2(("VMX: mov x, drx\n"));
3791	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitDRxRead);
3792	rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pCtx),
3793	VMX_EXIT_QUALIFICATION_DRX_GENREG(exitQualification),
3794	VMX_EXIT_QUALIFICATION_DRX_REGISTER(exitQualification));
3795	}
3796	/* Update EIP if no error occurred. */
3797	if (RT_SUCCESS(rc))
3798	pCtx->rip += cbInstr;
3799
3800	if (rc == VINF_SUCCESS)
3801	{
3802	/* Only resume if successful. */
3803	goto ResumeExecution;
3804	}
3805	Assert(rc == VERR_EM_INTERPRETER);
3806	break;
3807	}
3808
3809	/* Note: We'll get a #GP if the IO instruction isn't allowed (IOPL or TSS bitmap); no need to double check. */
3810	case VMX_EXIT_PORT_IO: /* 30 I/O instruction. */
3811	{
3812	STAM_PROFILE_ADV_START(&pVCpu->hwaccm.s.StatExit2Sub1, y1);
3813	uint32_t uIOWidth = VMX_EXIT_QUALIFICATION_IO_WIDTH(exitQualification);
3814	uint32_t uPort;
3815	bool fIOWrite = (VMX_EXIT_QUALIFICATION_IO_DIRECTION(exitQualification) == VMX_EXIT_QUALIFICATION_IO_DIRECTION_OUT);
3816
3817	/** @todo necessary to make the distinction? */
3818	if (VMX_EXIT_QUALIFICATION_IO_ENCODING(exitQualification) == VMX_EXIT_QUALIFICATION_IO_ENCODING_DX)
3819	{
3820	uPort = pCtx->edx & 0xffff;
3821	}
3822	else
3823	uPort = VMX_EXIT_QUALIFICATION_IO_PORT(exitQualification); /* Immediate encoding. */
3824
3825	/* paranoia */
3826	if (RT_UNLIKELY(uIOWidth == 2 \|\| uIOWidth >= 4))
3827	{
3828	rc = fIOWrite ? VINF_IOM_HC_IOPORT_WRITE : VINF_IOM_HC_IOPORT_READ;
3829	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub1, y1);
3830	break;
3831	}
3832
3833	uint32_t cbSize = g_aIOSize[uIOWidth];
3834
3835	if (VMX_EXIT_QUALIFICATION_IO_STRING(exitQualification))
3836	{
3837	/* ins/outs */
3838	PDISCPUSTATE pDis = &pVCpu->hwaccm.s.DisState;
3839
3840	/* Disassemble manually to deal with segment prefixes. */
3841	/** @todo VMX_VMCS_EXIT_GUEST_LINEAR_ADDR contains the flat pointer operand of the instruction. */
3842	/** @todo VMX_VMCS32_RO_EXIT_INSTR_INFO also contains segment prefix info. */
3843	rc2 = EMInterpretDisasOne(pVM, pVCpu, CPUMCTX2CORE(pCtx), pDis, NULL);
3844	if (RT_SUCCESS(rc))
3845	{
3846	if (fIOWrite)
3847	{
3848	Log2(("IOMInterpretOUTSEx %RGv %x size=%d\n", (RTGCPTR)pCtx->rip, uPort, cbSize));
3849	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitIOStringWrite);
3850	rc = IOMInterpretOUTSEx(pVM, CPUMCTX2CORE(pCtx), uPort, pDis->prefix, cbSize);
3851	}
3852	else
3853	{
3854	Log2(("IOMInterpretINSEx %RGv %x size=%d\n", (RTGCPTR)pCtx->rip, uPort, cbSize));
3855	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitIOStringRead);
3856	rc = IOMInterpretINSEx(pVM, CPUMCTX2CORE(pCtx), uPort, pDis->prefix, cbSize);
3857	}
3858	}
3859	else
3860	rc = VINF_EM_RAW_EMULATE_INSTR;
3861	}
3862	else
3863	{
3864	/* normal in/out */
3865	uint32_t uAndVal = g_aIOOpAnd[uIOWidth];
3866
3867	Assert(!VMX_EXIT_QUALIFICATION_IO_REP(exitQualification));
3868
3869	if (fIOWrite)
3870	{
3871	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitIOWrite);
3872	rc = IOMIOPortWrite(pVM, uPort, pCtx->eax & uAndVal, cbSize);
3873	if (rc == VINF_IOM_HC_IOPORT_WRITE)
3874	HWACCMR0SavePendingIOPortWrite(pVCpu, pCtx->rip, pCtx->rip + cbInstr, uPort, uAndVal, cbSize);
3875	}
3876	else
3877	{
3878	uint32_t u32Val = 0;
3879
3880	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitIORead);
3881	rc = IOMIOPortRead(pVM, uPort, &u32Val, cbSize);
3882	if (IOM_SUCCESS(rc))
3883	{
3884	/* Write back to the EAX register. */
3885	pCtx->eax = (pCtx->eax & ~uAndVal) \| (u32Val & uAndVal);
3886	}
3887	else
3888	if (rc == VINF_IOM_HC_IOPORT_READ)
3889	HWACCMR0SavePendingIOPortRead(pVCpu, pCtx->rip, pCtx->rip + cbInstr, uPort, uAndVal, cbSize);
3890	}
3891	}
3892	/*
3893	* Handled the I/O return codes.
3894	* (The unhandled cases end up with rc == VINF_EM_RAW_EMULATE_INSTR.)
3895	*/
3896	if (IOM_SUCCESS(rc))
3897	{
3898	/* Update EIP and continue execution. */
3899	pCtx->rip += cbInstr;
3900	if (RT_LIKELY(rc == VINF_SUCCESS))
3901	{
3902	/* If any IO breakpoints are armed, then we should check if a debug trap needs to be generated. */
3903	if (pCtx->dr[7] & X86_DR7_ENABLED_MASK)
3904	{
3905	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatDRxIOCheck);
3906	for (unsigned i=0;i<4;i++)
3907	{
3908	unsigned uBPLen = g_aIOSize[X86_DR7_GET_LEN(pCtx->dr[7], i)];
3909
3910	if ( (uPort >= pCtx->dr[i] && uPort < pCtx->dr[i] + uBPLen)
3911	&& (pCtx->dr[7] & (X86_DR7_L(i) \| X86_DR7_G(i)))
3912	&& (pCtx->dr[7] & X86_DR7_RW(i, X86_DR7_RW_IO)) == X86_DR7_RW(i, X86_DR7_RW_IO))
3913	{
3914	uint64_t uDR6;
3915
3916	Assert(CPUMIsGuestDebugStateActive(pVCpu));
3917
3918	uDR6 = ASMGetDR6();
3919
3920	/* Clear all breakpoint status flags and set the one we just hit. */
3921	uDR6 &= ~(X86_DR6_B0\|X86_DR6_B1\|X86_DR6_B2\|X86_DR6_B3);
3922	uDR6 \|= (uint64_t)RT_BIT(i);
3923
3924	/* Note: AMD64 Architecture Programmer's Manual 13.1:
3925	* Bits 15:13 of the DR6 register is never cleared by the processor and must be cleared by software after
3926	* the contents have been read.
3927	*/
3928	ASMSetDR6(uDR6);
3929
3930	/* X86_DR7_GD will be cleared if drx accesses should be trapped inside the guest. */
3931	pCtx->dr[7] &= ~X86_DR7_GD;
3932
3933	/* Paranoia. */
3934	pCtx->dr[7] &= 0xffffffff; /* upper 32 bits reserved */
3935	pCtx->dr[7] &= ~(RT_BIT(11) \| RT_BIT(12) \| RT_BIT(14) \| RT_BIT(15)); /* must be zero */
3936	pCtx->dr[7] \|= 0x400; /* must be one */
3937
3938	/* Resync DR7 */
3939	rc2 = VMXWriteVMCS64(VMX_VMCS64_GUEST_DR7, pCtx->dr[7]);
3940	AssertRC(rc2);
3941
3942	/* Construct inject info. */
3943	intInfo = X86_XCPT_DB;
3944	intInfo \|= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
3945	intInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HWEXCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
3946
3947	Log(("Inject IO debug trap at %RGv\n", (RTGCPTR)pCtx->rip));
3948	rc2 = VMXR0InjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), 0, 0);
3949	AssertRC(rc2);
3950
3951	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub1, y1);
3952	goto ResumeExecution;
3953	}
3954	}
3955	}
3956	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub1, y1);
3957	goto ResumeExecution;
3958	}
3959	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub1, y1);
3960	break;
3961	}
3962
3963	#ifdef VBOX_STRICT
3964	if (rc == VINF_IOM_HC_IOPORT_READ)
3965	Assert(!fIOWrite);
3966	else if (rc == VINF_IOM_HC_IOPORT_WRITE)
3967	Assert(fIOWrite);
3968	else
3969	AssertMsg(RT_FAILURE(rc) \|\| rc == VINF_EM_RAW_EMULATE_INSTR \|\| rc == VINF_EM_RAW_GUEST_TRAP \|\| rc == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", VBOXSTRICTRC_VAL(rc)));
3970	#endif
3971	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub1, y1);
3972	break;
3973	}
3974
3975	case VMX_EXIT_TPR: /* 43 TPR below threshold. Guest software executed MOV to CR8. */
3976	LogFlow(("VMX_EXIT_TPR\n"));
3977	/* RIP is already set to the next instruction and the TPR has been synced back. Just resume. */
3978	goto ResumeExecution;
3979
3980	case VMX_EXIT_APIC_ACCESS: /* 44 APIC access. Guest software attempted to access memory at a physical address on the APIC-access page. */
3981	{
3982	LogFlow(("VMX_EXIT_APIC_ACCESS\n"));
3983	unsigned uAccessType = VMX_EXIT_QUALIFICATION_APIC_ACCESS_TYPE(exitQualification);
3984
3985	switch(uAccessType)
3986	{
3987	case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
3988	case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
3989	{
3990	RTGCPHYS GCPhys;
3991	PDMApicGetBase(pVM, &GCPhys);
3992	GCPhys &= PAGE_BASE_GC_MASK;
3993	GCPhys += VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(exitQualification);
3994
3995	LogFlow(("Apic access at %RGp\n", GCPhys));
3996	rc = IOMMMIOPhysHandler(pVM, (uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ) ? 0 : X86_TRAP_PF_RW, CPUMCTX2CORE(pCtx), GCPhys);
3997	if (rc == VINF_SUCCESS)
3998	goto ResumeExecution; /* rip already updated */
3999	break;
4000	}
4001
4002	default:
4003	rc = VINF_EM_RAW_EMULATE_INSTR;
4004	break;
4005	}
4006	break;
4007	}
4008
4009	case VMX_EXIT_PREEMPTION_TIMER: /* 52 VMX-preemption timer expired. The preemption timer counted down to zero. */
4010	if (!TMTimerPollBool(pVM, pVCpu))
4011	goto ResumeExecution;
4012	rc = VINF_EM_RAW_TIMER_PENDING;
4013	break;
4014
4015	default:
4016	/* The rest is handled after syncing the entire CPU state. */
4017	break;
4018	}
4019
4020	/* Note: the guest state isn't entirely synced back at this stage. */
4021
4022	/* Investigate why there was a VM-exit. (part 2) */
4023	switch (exitReason)
4024	{
4025	case VMX_EXIT_EXCEPTION: /* 0 Exception or non-maskable interrupt (NMI). */
4026	case VMX_EXIT_EXTERNAL_IRQ: /* 1 External interrupt. */
4027	case VMX_EXIT_EPT_VIOLATION:
4028	case VMX_EXIT_EPT_MISCONFIG: /* 49 EPT misconfig is used by the PGM/MMIO optimizations. */
4029	case VMX_EXIT_PREEMPTION_TIMER: /* 52 VMX-preemption timer expired. The preemption timer counted down to zero. */
4030	/* Already handled above. */
4031	break;
4032
4033	case VMX_EXIT_TRIPLE_FAULT: /* 2 Triple fault. */
4034	rc = VINF_EM_RESET; /* Triple fault equals a reset. */
4035	break;
4036
4037	case VMX_EXIT_INIT_SIGNAL: /* 3 INIT signal. */
4038	case VMX_EXIT_SIPI: /* 4 Start-up IPI (SIPI). */
4039	rc = VINF_EM_RAW_INTERRUPT;
4040	AssertFailed(); /* Can't happen. Yet. */
4041	break;
4042
4043	case VMX_EXIT_IO_SMI_IRQ: /* 5 I/O system-management interrupt (SMI). */
4044	case VMX_EXIT_SMI_IRQ: /* 6 Other SMI. */
4045	rc = VINF_EM_RAW_INTERRUPT;
4046	AssertFailed(); /* Can't happen afaik. */
4047	break;
4048
4049	case VMX_EXIT_TASK_SWITCH: /* 9 Task switch: too complicated to emulate, so fall back to the recompiler */
4050	Log(("VMX_EXIT_TASK_SWITCH: exit=%RX64\n", exitQualification));
4051	if ( (VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE(exitQualification) == VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE_IDT)
4052	&& pVCpu->hwaccm.s.Event.fPending)
4053	{
4054	/* Caused by an injected interrupt. */
4055	pVCpu->hwaccm.s.Event.fPending = false;
4056
4057	Log(("VMX_EXIT_TASK_SWITCH: reassert trap %d\n", VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVCpu->hwaccm.s.Event.intInfo)));
4058	Assert(!VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(pVCpu->hwaccm.s.Event.intInfo));
4059	rc2 = TRPMAssertTrap(pVCpu, VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVCpu->hwaccm.s.Event.intInfo), TRPM_HARDWARE_INT);
4060	AssertRC(rc2);
4061	}
4062	/* else Exceptions and software interrupts can just be restarted. */
4063	rc = VERR_EM_INTERPRETER;
4064	break;
4065
4066	case VMX_EXIT_HLT: /* 12 Guest software attempted to execute HLT. */
4067	/** Check if external interrupts are pending; if so, don't switch back. */
4068	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitHlt);
4069	pCtx->rip++; /* skip hlt */
4070	if (EMShouldContinueAfterHalt(pVCpu, pCtx))
4071	goto ResumeExecution;
4072
4073	rc = VINF_EM_HALT;
4074	break;
4075
4076	case VMX_EXIT_MWAIT: /* 36 Guest software executed MWAIT. */
4077	Log2(("VMX: mwait\n"));
4078	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitMwait);
4079	rc = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pCtx));
4080	if ( rc == VINF_EM_HALT
4081	\|\| rc == VINF_SUCCESS)
4082	{
4083	/* Update EIP and continue execution. */
4084	pCtx->rip += cbInstr;
4085
4086	/** Check if external interrupts are pending; if so, don't switch back. */
4087	if ( rc == VINF_SUCCESS
4088	\|\| ( rc == VINF_EM_HALT
4089	&& EMShouldContinueAfterHalt(pVCpu, pCtx))
4090	)
4091	goto ResumeExecution;
4092	}
4093	AssertMsg(rc == VERR_EM_INTERPRETER \|\| rc == VINF_EM_HALT, ("EMU: mwait failed with %Rrc\n", VBOXSTRICTRC_VAL(rc)));
4094	break;
4095
4096	case VMX_EXIT_RSM: /* 17 Guest software attempted to execute RSM in SMM. */
4097	AssertFailed(); /* can't happen. */
4098	rc = VERR_EM_INTERPRETER;
4099	break;
4100
4101	case VMX_EXIT_VMCALL: /* 18 Guest software executed VMCALL. */
4102	case VMX_EXIT_VMCLEAR: /* 19 Guest software executed VMCLEAR. */
4103	case VMX_EXIT_VMLAUNCH: /* 20 Guest software executed VMLAUNCH. */
4104	case VMX_EXIT_VMPTRLD: /* 21 Guest software executed VMPTRLD. */
4105	case VMX_EXIT_VMPTRST: /* 22 Guest software executed VMPTRST. */
4106	case VMX_EXIT_VMREAD: /* 23 Guest software executed VMREAD. */
4107	case VMX_EXIT_VMRESUME: /* 24 Guest software executed VMRESUME. */
4108	case VMX_EXIT_VMWRITE: /* 25 Guest software executed VMWRITE. */
4109	case VMX_EXIT_VMXOFF: /* 26 Guest software executed VMXOFF. */
4110	case VMX_EXIT_VMXON: /* 27 Guest software executed VMXON. */
4111	/** @todo inject #UD immediately */
4112	rc = VERR_EM_INTERPRETER;
4113	break;
4114
4115	case VMX_EXIT_CPUID: /* 10 Guest software attempted to execute CPUID. */
4116	case VMX_EXIT_RDTSC: /* 16 Guest software attempted to execute RDTSC. */
4117	case VMX_EXIT_INVPG: /* 14 Guest software attempted to execute INVPG. */
4118	case VMX_EXIT_CRX_MOVE: /* 28 Control-register accesses. */
4119	case VMX_EXIT_DRX_MOVE: /* 29 Debug-register accesses. */
4120	case VMX_EXIT_PORT_IO: /* 30 I/O instruction. */
4121	case VMX_EXIT_RDPMC: /* 15 Guest software attempted to execute RDPMC. */
4122	/* already handled above */
4123	AssertMsg( rc == VINF_PGM_CHANGE_MODE
4124	\|\| rc == VINF_EM_RAW_INTERRUPT
4125	\|\| rc == VERR_EM_INTERPRETER
4126	\|\| rc == VINF_EM_RAW_EMULATE_INSTR
4127	\|\| rc == VINF_PGM_SYNC_CR3
4128	\|\| rc == VINF_IOM_HC_IOPORT_READ
4129	\|\| rc == VINF_IOM_HC_IOPORT_WRITE
4130	\|\| rc == VINF_EM_RAW_GUEST_TRAP
4131	\|\| rc == VINF_TRPM_XCPT_DISPATCHED
4132	\|\| rc == VINF_EM_RESCHEDULE_REM,
4133	("rc = %d\n", VBOXSTRICTRC_VAL(rc)));
4134	break;
4135
4136	case VMX_EXIT_TPR: /* 43 TPR below threshold. Guest software executed MOV to CR8. */
4137	case VMX_EXIT_APIC_ACCESS: /* 44 APIC access. Guest software attempted to access memory at a physical address on the APIC-access page. */
4138	case VMX_EXIT_RDMSR: /* 31 RDMSR. Guest software attempted to execute RDMSR. */
4139	case VMX_EXIT_WRMSR: /* 32 WRMSR. Guest software attempted to execute WRMSR. */
4140	case VMX_EXIT_PAUSE: /* 40 Guest software attempted to execute PAUSE. */
4141	case VMX_EXIT_MONITOR: /* 39 Guest software attempted to execute MONITOR. */
4142	/* Note: If we decide to emulate them here, then we must sync the MSRs that could have been changed (sysenter, fs/gs base)!!! */
4143	rc = VERR_EM_INTERPRETER;
4144	break;
4145
4146	case VMX_EXIT_IRQ_WINDOW: /* 7 Interrupt window. */
4147	Assert(rc == VINF_EM_RAW_INTERRUPT);
4148	break;
4149
4150	case VMX_EXIT_ERR_INVALID_GUEST_STATE: /* 33 VM-entry failure due to invalid guest state. */
4151	{
4152	#ifdef VBOX_STRICT
4153	RTCCUINTREG val2 = 0;
4154
4155	Log(("VMX_EXIT_ERR_INVALID_GUEST_STATE\n"));
4156
4157	VMXReadVMCS(VMX_VMCS64_GUEST_RIP, &val2);
4158	Log(("Old eip %RGv new %RGv\n", (RTGCPTR)pCtx->rip, (RTGCPTR)val2));
4159
4160	VMXReadVMCS(VMX_VMCS64_GUEST_CR0, &val2);
4161	Log(("VMX_VMCS_GUEST_CR0 %RX64\n", (uint64_t)val2));
4162
4163	VMXReadVMCS(VMX_VMCS64_GUEST_CR3, &val2);
4164	Log(("VMX_VMCS_GUEST_CR3 %RX64\n", (uint64_t)val2));
4165
4166	VMXReadVMCS(VMX_VMCS64_GUEST_CR4, &val2);
4167	Log(("VMX_VMCS_GUEST_CR4 %RX64\n", (uint64_t)val2));
4168
4169	VMXReadVMCS(VMX_VMCS_GUEST_RFLAGS, &val2);
4170	Log(("VMX_VMCS_GUEST_RFLAGS %08x\n", val2));
4171
4172	VMX_LOG_SELREG(CS, "CS", val2);
4173	VMX_LOG_SELREG(DS, "DS", val2);
4174	VMX_LOG_SELREG(ES, "ES", val2);
4175	VMX_LOG_SELREG(FS, "FS", val2);
4176	VMX_LOG_SELREG(GS, "GS", val2);
4177	VMX_LOG_SELREG(SS, "SS", val2);
4178	VMX_LOG_SELREG(TR, "TR", val2);
4179	VMX_LOG_SELREG(LDTR, "LDTR", val2);
4180
4181	VMXReadVMCS(VMX_VMCS64_GUEST_GDTR_BASE, &val2);
4182	Log(("VMX_VMCS_GUEST_GDTR_BASE %RX64\n", (uint64_t)val2));
4183	VMXReadVMCS(VMX_VMCS64_GUEST_IDTR_BASE, &val2);
4184	Log(("VMX_VMCS_GUEST_IDTR_BASE %RX64\n", (uint64_t)val2));
4185	#endif /* VBOX_STRICT */
4186	rc = VERR_VMX_INVALID_GUEST_STATE;
4187	break;
4188	}
4189
4190	case VMX_EXIT_ERR_MSR_LOAD: /* 34 VM-entry failure due to MSR loading. */
4191	case VMX_EXIT_ERR_MACHINE_CHECK: /* 41 VM-entry failure due to machine-check. */
4192	default:
4193	rc = VERR_VMX_UNEXPECTED_EXIT_CODE;
4194	AssertMsgFailed(("Unexpected exit code %d\n", exitReason)); /* Can't happen. */
4195	break;
4196
4197	}
4198	end:
4199
4200	/* We now going back to ring-3, so clear the action flag. */
4201	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
4202
4203	/* Signal changes for the recompiler. */
4204	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR \| CPUM_CHANGED_LDTR \| CPUM_CHANGED_GDTR \| CPUM_CHANGED_IDTR \| CPUM_CHANGED_TR \| CPUM_CHANGED_HIDDEN_SEL_REGS);
4205
4206	/* If we executed vmlaunch/vmresume and an external irq was pending, then we don't have to do a full sync the next time. */
4207	if ( exitReason == VMX_EXIT_EXTERNAL_IRQ
4208	&& !VMX_EXIT_INTERRUPTION_INFO_VALID(intInfo))
4209	{
4210	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatPendingHostIrq);
4211	/* On the next entry we'll only sync the host context. */
4212	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_HOST_CONTEXT;
4213	}
4214	else
4215	{
4216	/* On the next entry we'll sync everything. */
4217	/** @todo we can do better than this */
4218	/* Not in the VINF_PGM_CHANGE_MODE though! */
4219	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_ALL;
4220	}
4221
4222	/* translate into a less severe return code */
4223	if (rc == VERR_EM_INTERPRETER)
4224	rc = VINF_EM_RAW_EMULATE_INSTR;
4225	else
4226	/* Try to extract more information about what might have gone wrong here. */
4227	if (rc == VERR_VMX_INVALID_VMCS_PTR)
4228	{
4229	VMXGetActivateVMCS(&pVCpu->hwaccm.s.vmx.lasterror.u64VMCSPhys);
4230	pVCpu->hwaccm.s.vmx.lasterror.ulVMCSRevision = (uint32_t )pVCpu->hwaccm.s.vmx.pVMCS;
4231	pVCpu->hwaccm.s.vmx.lasterror.idEnteredCpu = pVCpu->hwaccm.s.idEnteredCpu;
4232	pVCpu->hwaccm.s.vmx.lasterror.idCurrentCpu = RTMpCpuId();
4233	}
4234
4235	/* Just set the correct state here instead of trying to catch every goto above. */
4236	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED, VMCPUSTATE_STARTED_EXEC);
4237
4238	#ifdef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
4239	/* Restore interrupts if we exitted after disabling them. */
4240	if (uOldEFlags != ~(RTCCUINTREG)0)
4241	ASMSetFlags(uOldEFlags);
4242	#endif
4243
4244	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2, x);
4245	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit1, x);
4246	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatEntry, x);
4247	Log2(("X"));
4248	return VBOXSTRICTRC_TODO(rc);
4249	}
4250
4251
4252	/**
4253	* Enters the VT-x session
4254	*
4255	* @returns VBox status code.
4256	* @param pVM The VM to operate on.
4257	* @param pVCpu The VMCPU to operate on.
4258	* @param pCpu CPU info struct
4259	*/
4260	VMMR0DECL(int) VMXR0Enter(PVM pVM, PVMCPU pVCpu, PHWACCM_CPUINFO pCpu)
4261	{
4262	Assert(pVM->hwaccm.s.vmx.fSupported);
4263
4264	unsigned cr4 = ASMGetCR4();
4265	if (!(cr4 & X86_CR4_VMXE))
4266	{
4267	AssertMsgFailed(("X86_CR4_VMXE should be set!\n"));
4268	return VERR_VMX_X86_CR4_VMXE_CLEARED;
4269	}
4270
4271	/* Activate the VM Control Structure. */
4272	int rc = VMXActivateVMCS(pVCpu->hwaccm.s.vmx.pVMCSPhys);
4273	if (RT_FAILURE(rc))
4274	return rc;
4275
4276	pVCpu->hwaccm.s.fResumeVM = false;
4277	return VINF_SUCCESS;
4278	}
4279
4280
4281	/**
4282	* Leaves the VT-x session
4283	*
4284	* @returns VBox status code.
4285	* @param pVM The VM to operate on.
4286	* @param pVCpu The VMCPU to operate on.
4287	* @param pCtx CPU context
4288	*/
4289	VMMR0DECL(int) VMXR0Leave(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
4290	{
4291	Assert(pVM->hwaccm.s.vmx.fSupported);
4292
4293	#ifdef DEBUG
4294	if (CPUMIsHyperDebugStateActive(pVCpu))
4295	{
4296	CPUMR0LoadHostDebugState(pVM, pVCpu);
4297	Assert(pVCpu->hwaccm.s.vmx.proc_ctls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT);
4298	}
4299	else
4300	#endif
4301	/* Save the guest debug state if necessary. */
4302	if (CPUMIsGuestDebugStateActive(pVCpu))
4303	{
4304	CPUMR0SaveGuestDebugState(pVM, pVCpu, pCtx, true /* save DR6 */);
4305
4306	/* Enable drx move intercepts again. */
4307	pVCpu->hwaccm.s.vmx.proc_ctls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT;
4308	int rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
4309	AssertRC(rc);
4310
4311	/* Resync the debug registers the next time. */
4312	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_GUEST_DEBUG;
4313	}
4314	else
4315	Assert(pVCpu->hwaccm.s.vmx.proc_ctls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT);
4316
4317	/* Clear VM Control Structure. Marking it inactive, clearing implementation specific data and writing back VMCS data to memory. */
4318	int rc = VMXClearVMCS(pVCpu->hwaccm.s.vmx.pVMCSPhys);
4319	AssertRC(rc);
4320
4321	return VINF_SUCCESS;
4322	}
4323
4324	/**
4325	* Flush the TLB (EPT)
4326	*
4327	* @returns VBox status code.
4328	* @param pVM The VM to operate on.
4329	* @param pVCpu The VM CPU to operate on.
4330	* @param enmFlush Type of flush
4331	* @param GCPhys Physical address of the page to flush
4332	*/
4333	static void vmxR0FlushEPT(PVM pVM, PVMCPU pVCpu, VMX_FLUSH enmFlush, RTGCPHYS GCPhys)
4334	{
4335	uint64_t descriptor[2];
4336
4337	LogFlow(("vmxR0FlushEPT %d %RGv\n", enmFlush, GCPhys));
4338	Assert(pVM->hwaccm.s.fNestedPaging);
4339	descriptor[0] = pVCpu->hwaccm.s.vmx.GCPhysEPTP;
4340	descriptor[1] = GCPhys;
4341	int rc = VMXR0InvEPT(enmFlush, &descriptor[0]);
4342	AssertRC(rc);
4343	}
4344
4345	#ifdef HWACCM_VTX_WITH_VPID
4346	/**
4347	* Flush the TLB (EPT)
4348	*
4349	* @returns VBox status code.
4350	* @param pVM The VM to operate on.
4351	* @param pVCpu The VM CPU to operate on.
4352	* @param enmFlush Type of flush
4353	* @param GCPtr Virtual address of the page to flush
4354	*/
4355	static void vmxR0FlushVPID(PVM pVM, PVMCPU pVCpu, VMX_FLUSH enmFlush, RTGCPTR GCPtr)
4356	{
4357	#if HC_ARCH_BITS == 32
4358	/* If we get a flush in 64 bits guest mode, then force a full TLB flush. Invvpid probably takes only 32 bits addresses. (@todo) */
4359	if ( CPUMIsGuestInLongMode(pVCpu)
4360	&& !VMX_IS_64BIT_HOST_MODE())
4361	{
4362	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
4363	}
4364	else
4365	#endif
4366	{
4367	uint64_t descriptor[2];
4368
4369	Assert(pVM->hwaccm.s.vmx.fVPID);
4370	descriptor[0] = pVCpu->hwaccm.s.uCurrentASID;
4371	descriptor[1] = GCPtr;
4372	int rc = VMXR0InvVPID(enmFlush, &descriptor[0]);
4373	AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvVPID %x %x %RGv failed with %d\n", enmFlush, pVCpu->hwaccm.s.uCurrentASID, GCPtr, rc));
4374	}
4375	}
4376	#endif /* HWACCM_VTX_WITH_VPID */
4377
4378	/**
4379	* Invalidates a guest page
4380	*
4381	* @returns VBox status code.
4382	* @param pVM The VM to operate on.
4383	* @param pVCpu The VM CPU to operate on.
4384	* @param GCVirt Page to invalidate
4385	*/
4386	VMMR0DECL(int) VMXR0InvalidatePage(PVM pVM, PVMCPU pVCpu, RTGCPTR GCVirt)
4387	{
4388	bool fFlushPending = VMCPU_FF_ISSET(pVCpu, VMCPU_FF_TLB_FLUSH);
4389
4390	Log2(("VMXR0InvalidatePage %RGv\n", GCVirt));
4391
4392	/* Only relevant if we want to use VPID.
4393	* In the nested paging case we still see such calls, but
4394	* can safely ignore them. (e.g. after cr3 updates)
4395	*/
4396	#ifdef HWACCM_VTX_WITH_VPID
4397	/* Skip it if a TLB flush is already pending. */
4398	if ( !fFlushPending
4399	&& pVM->hwaccm.s.vmx.fVPID)
4400	vmxR0FlushVPID(pVM, pVCpu, pVM->hwaccm.s.vmx.enmFlushPage, GCVirt);
4401	#endif /* HWACCM_VTX_WITH_VPID */
4402
4403	return VINF_SUCCESS;
4404	}
4405
4406	/**
4407	* Invalidates a guest page by physical address
4408	*
4409	* NOTE: Assumes the current instruction references this physical page though a virtual address!!
4410	*
4411	* @returns VBox status code.
4412	* @param pVM The VM to operate on.
4413	* @param pVCpu The VM CPU to operate on.
4414	* @param GCPhys Page to invalidate
4415	*/
4416	VMMR0DECL(int) VMXR0InvalidatePhysPage(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys)
4417	{
4418	bool fFlushPending = VMCPU_FF_ISSET(pVCpu, VMCPU_FF_TLB_FLUSH);
4419
4420	Assert(pVM->hwaccm.s.fNestedPaging);
4421
4422	LogFlow(("VMXR0InvalidatePhysPage %RGp\n", GCPhys));
4423
4424	/* Skip it if a TLB flush is already pending. */
4425	if (!fFlushPending)
4426	vmxR0FlushEPT(pVM, pVCpu, pVM->hwaccm.s.vmx.enmFlushPage, GCPhys);
4427
4428	return VINF_SUCCESS;
4429	}
4430
4431	/**
4432	* Report world switch error and dump some useful debug info
4433	*
4434	* @param pVM The VM to operate on.
4435	* @param pVCpu The VMCPU to operate on.
4436	* @param rc Return code
4437	* @param pCtx Current CPU context (not updated)
4438	*/
4439	static void VMXR0ReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, VBOXSTRICTRC rc, PCPUMCTX pCtx)
4440	{
4441	switch (VBOXSTRICTRC_VAL(rc))
4442	{
4443	case VERR_VMX_INVALID_VMXON_PTR:
4444	AssertFailed();
4445	break;
4446
4447	case VERR_VMX_UNABLE_TO_START_VM:
4448	case VERR_VMX_UNABLE_TO_RESUME_VM:
4449	{
4450	int rc2;
4451	RTCCUINTREG exitReason, instrError;
4452
4453	rc2 = VMXReadVMCS(VMX_VMCS32_RO_EXIT_REASON, &exitReason);
4454	rc2 \|= VMXReadVMCS(VMX_VMCS32_RO_VM_INSTR_ERROR, &instrError);
4455	AssertRC(rc2);
4456	if (rc2 == VINF_SUCCESS)
4457	{
4458	Log(("Unable to start/resume VM for reason: %x. Instruction error %x\n", (uint32_t)exitReason, (uint32_t)instrError));
4459	Log(("Current stack %08x\n", &rc2));
4460
4461	pVCpu->hwaccm.s.vmx.lasterror.ulInstrError = instrError;
4462	pVCpu->hwaccm.s.vmx.lasterror.ulExitReason = exitReason;
4463
4464	#ifdef VBOX_STRICT
4465	RTGDTR gdtr;
4466	PCX86DESCHC pDesc;
4467	RTCCUINTREG val;
4468
4469	ASMGetGDTR(&gdtr);
4470
4471	VMXReadVMCS(VMX_VMCS64_GUEST_RIP, &val);
4472	Log(("Old eip %RGv new %RGv\n", (RTGCPTR)pCtx->rip, (RTGCPTR)val));
4473	VMXReadVMCS(VMX_VMCS_CTRL_PIN_EXEC_CONTROLS, &val);
4474	Log(("VMX_VMCS_CTRL_PIN_EXEC_CONTROLS %08x\n", val));
4475	VMXReadVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, &val);
4476	Log(("VMX_VMCS_CTRL_PROC_EXEC_CONTROLS %08x\n", val));
4477	VMXReadVMCS(VMX_VMCS_CTRL_ENTRY_CONTROLS, &val);
4478	Log(("VMX_VMCS_CTRL_ENTRY_CONTROLS %08x\n", val));
4479	VMXReadVMCS(VMX_VMCS_CTRL_EXIT_CONTROLS, &val);
4480	Log(("VMX_VMCS_CTRL_EXIT_CONTROLS %08x\n", val));
4481
4482	VMXReadVMCS(VMX_VMCS_HOST_CR0, &val);
4483	Log(("VMX_VMCS_HOST_CR0 %08x\n", val));
4484
4485	VMXReadVMCS(VMX_VMCS_HOST_CR3, &val);
4486	Log(("VMX_VMCS_HOST_CR3 %08x\n", val));
4487
4488	VMXReadVMCS(VMX_VMCS_HOST_CR4, &val);
4489	Log(("VMX_VMCS_HOST_CR4 %08x\n", val));
4490
4491	VMXReadVMCS(VMX_VMCS16_HOST_FIELD_CS, &val);
4492	Log(("VMX_VMCS_HOST_FIELD_CS %08x\n", val));
4493
4494	VMXReadVMCS(VMX_VMCS_GUEST_RFLAGS, &val);
4495	Log(("VMX_VMCS_GUEST_RFLAGS %08x\n", val));
4496
4497	if (val < gdtr.cbGdt)
4498	{
4499	pDesc = (PCX86DESCHC)(gdtr.pGdt + (val & X86_SEL_MASK));
4500	HWACCMR0DumpDescriptor(pDesc, val, "CS: ");
4501	}
4502
4503	VMXReadVMCS(VMX_VMCS16_HOST_FIELD_DS, &val);
4504	Log(("VMX_VMCS_HOST_FIELD_DS %08x\n", val));
4505	if (val < gdtr.cbGdt)
4506	{
4507	pDesc = (PCX86DESCHC)(gdtr.pGdt + (val & X86_SEL_MASK));
4508	HWACCMR0DumpDescriptor(pDesc, val, "DS: ");
4509	}
4510
4511	VMXReadVMCS(VMX_VMCS16_HOST_FIELD_ES, &val);
4512	Log(("VMX_VMCS_HOST_FIELD_ES %08x\n", val));
4513	if (val < gdtr.cbGdt)
4514	{
4515	pDesc = (PCX86DESCHC)(gdtr.pGdt + (val & X86_SEL_MASK));
4516	HWACCMR0DumpDescriptor(pDesc, val, "ES: ");
4517	}
4518
4519	VMXReadVMCS(VMX_VMCS16_HOST_FIELD_FS, &val);
4520	Log(("VMX_VMCS16_HOST_FIELD_FS %08x\n", val));
4521	if (val < gdtr.cbGdt)
4522	{
4523	pDesc = (PCX86DESCHC)(gdtr.pGdt + (val & X86_SEL_MASK));
4524	HWACCMR0DumpDescriptor(pDesc, val, "FS: ");
4525	}
4526
4527	VMXReadVMCS(VMX_VMCS16_HOST_FIELD_GS, &val);
4528	Log(("VMX_VMCS16_HOST_FIELD_GS %08x\n", val));
4529	if (val < gdtr.cbGdt)
4530	{
4531	pDesc = (PCX86DESCHC)(gdtr.pGdt + (val & X86_SEL_MASK));
4532	HWACCMR0DumpDescriptor(pDesc, val, "GS: ");
4533	}
4534
4535	VMXReadVMCS(VMX_VMCS16_HOST_FIELD_SS, &val);
4536	Log(("VMX_VMCS16_HOST_FIELD_SS %08x\n", val));
4537	if (val < gdtr.cbGdt)
4538	{
4539	pDesc = (PCX86DESCHC)(gdtr.pGdt + (val & X86_SEL_MASK));
4540	HWACCMR0DumpDescriptor(pDesc, val, "SS: ");
4541	}
4542
4543	VMXReadVMCS(VMX_VMCS16_HOST_FIELD_TR, &val);
4544	Log(("VMX_VMCS16_HOST_FIELD_TR %08x\n", val));
4545	if (val < gdtr.cbGdt)
4546	{
4547	pDesc = (PCX86DESCHC)(gdtr.pGdt + (val & X86_SEL_MASK));
4548	HWACCMR0DumpDescriptor(pDesc, val, "TR: ");
4549	}
4550
4551	VMXReadVMCS(VMX_VMCS_HOST_TR_BASE, &val);
4552	Log(("VMX_VMCS_HOST_TR_BASE %RHv\n", val));
4553
4554	VMXReadVMCS(VMX_VMCS_HOST_GDTR_BASE, &val);
4555	Log(("VMX_VMCS_HOST_GDTR_BASE %RHv\n", val));
4556	VMXReadVMCS(VMX_VMCS_HOST_IDTR_BASE, &val);
4557	Log(("VMX_VMCS_HOST_IDTR_BASE %RHv\n", val));
4558
4559	VMXReadVMCS(VMX_VMCS32_HOST_SYSENTER_CS, &val);
4560	Log(("VMX_VMCS_HOST_SYSENTER_CS %08x\n", val));
4561
4562	VMXReadVMCS(VMX_VMCS_HOST_SYSENTER_EIP, &val);
4563	Log(("VMX_VMCS_HOST_SYSENTER_EIP %RHv\n", val));
4564
4565	VMXReadVMCS(VMX_VMCS_HOST_SYSENTER_ESP, &val);
4566	Log(("VMX_VMCS_HOST_SYSENTER_ESP %RHv\n", val));
4567
4568	VMXReadVMCS(VMX_VMCS_HOST_RSP, &val);
4569	Log(("VMX_VMCS_HOST_RSP %RHv\n", val));
4570	VMXReadVMCS(VMX_VMCS_HOST_RIP, &val);
4571	Log(("VMX_VMCS_HOST_RIP %RHv\n", val));
4572
4573	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4574	if (VMX_IS_64BIT_HOST_MODE())
4575	{
4576	Log(("MSR_K6_EFER = %RX64\n", ASMRdMsr(MSR_K6_EFER)));
4577	Log(("MSR_K6_STAR = %RX64\n", ASMRdMsr(MSR_K6_STAR)));
4578	Log(("MSR_K8_LSTAR = %RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
4579	Log(("MSR_K8_CSTAR = %RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
4580	Log(("MSR_K8_SF_MASK = %RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
4581	}
4582	# endif
4583	#endif /* VBOX_STRICT */
4584	}
4585	break;
4586	}
4587
4588	default:
4589	/* impossible */
4590	AssertMsgFailed(("%Rrc (%#x)\n", VBOXSTRICTRC_VAL(rc), VBOXSTRICTRC_VAL(rc)));
4591	break;
4592	}
4593	}
4594
4595	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4596	/**
4597	* Prepares for and executes VMLAUNCH (64 bits guest mode)
4598	*
4599	* @returns VBox status code
4600	* @param fResume vmlauch/vmresume
4601	* @param pCtx Guest context
4602	* @param pCache VMCS cache
4603	* @param pVM The VM to operate on.
4604	* @param pVCpu The VMCPU to operate on.
4605	*/
4606	DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
4607	{
4608	uint32_t aParam[6];
4609	PHWACCM_CPUINFO pCpu;
4610	RTHCPHYS pPageCpuPhys;
4611	int rc;
4612
4613	pCpu = HWACCMR0GetCurrentCpu();
4614	pPageCpuPhys = RTR0MemObjGetPagePhysAddr(pCpu->pMemObj, 0);
4615
4616	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4617	pCache->uPos = 1;
4618	pCache->interPD = PGMGetInterPaeCR3(pVM);
4619	pCache->pSwitcher = (uint64_t)pVM->hwaccm.s.pfnHost32ToGuest64R0;
4620	#endif
4621
4622	#ifdef DEBUG
4623	pCache->TestIn.pPageCpuPhys = 0;
4624	pCache->TestIn.pVMCSPhys = 0;
4625	pCache->TestIn.pCache = 0;
4626	pCache->TestOut.pVMCSPhys = 0;
4627	pCache->TestOut.pCache = 0;
4628	pCache->TestOut.pCtx = 0;
4629	pCache->TestOut.eflags = 0;
4630	#endif
4631
4632	aParam[0] = (uint32_t)(pPageCpuPhys); /* Param 1: VMXON physical address - Lo. */
4633	aParam[1] = (uint32_t)(pPageCpuPhys >> 32); /* Param 1: VMXON physical address - Hi. */
4634	aParam[2] = (uint32_t)(pVCpu->hwaccm.s.vmx.pVMCSPhys); /* Param 2: VMCS physical address - Lo. */
4635	aParam[3] = (uint32_t)(pVCpu->hwaccm.s.vmx.pVMCSPhys >> 32); /* Param 2: VMCS physical address - Hi. */
4636	aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hwaccm.s.vmx.VMCSCache);
4637	aParam[5] = 0;
4638
4639	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4640	pCtx->dr[4] = pVM->hwaccm.s.vmx.pScratchPhys + 16 + 8;
4641	(uint32_t )(pVM->hwaccm.s.vmx.pScratch + 16 + 8) = 1;
4642	#endif
4643	rc = VMXR0Execute64BitsHandler(pVM, pVCpu, pCtx, pVM->hwaccm.s.pfnVMXGCStartVM64, 6, &aParam[0]);
4644
4645	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4646	Assert((uint32_t )(pVM->hwaccm.s.vmx.pScratch + 16 + 8) == 5);
4647	Assert(pCtx->dr[4] == 10);
4648	(uint32_t )(pVM->hwaccm.s.vmx.pScratch + 16 + 8) = 0xff;
4649	#endif
4650
4651	#ifdef DEBUG
4652	AssertMsg(pCache->TestIn.pPageCpuPhys == pPageCpuPhys, ("%RHp vs %RHp\n", pCache->TestIn.pPageCpuPhys, pPageCpuPhys));
4653	AssertMsg(pCache->TestIn.pVMCSPhys == pVCpu->hwaccm.s.vmx.pVMCSPhys, ("%RHp vs %RHp\n", pCache->TestIn.pVMCSPhys, pVCpu->hwaccm.s.vmx.pVMCSPhys));
4654	AssertMsg(pCache->TestIn.pVMCSPhys == pCache->TestOut.pVMCSPhys, ("%RHp vs %RHp\n", pCache->TestIn.pVMCSPhys, pCache->TestOut.pVMCSPhys));
4655	AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache, pCache->TestOut.pCache));
4656	AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hwaccm.s.vmx.VMCSCache), ("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hwaccm.s.vmx.VMCSCache)));
4657	AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx, pCache->TestOut.pCtx));
4658	Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
4659	#endif
4660	return rc;
4661	}
4662
4663	/**
4664	* Executes the specified handler in 64 mode
4665	*
4666	* @returns VBox status code.
4667	* @param pVM The VM to operate on.
4668	* @param pVCpu The VMCPU to operate on.
4669	* @param pCtx Guest context
4670	* @param pfnHandler RC handler
4671	* @param cbParam Number of parameters
4672	* @param paParam Array of 32 bits parameters
4673	*/
4674	VMMR0DECL(int) VMXR0Execute64BitsHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, RTRCPTR pfnHandler, uint32_t cbParam, uint32_t *paParam)
4675	{
4676	int rc, rc2;
4677	PHWACCM_CPUINFO pCpu;
4678	RTHCPHYS pPageCpuPhys;
4679	RTHCUINTREG uOldEFlags;
4680
4681	AssertReturn(pVM->hwaccm.s.pfnHost32ToGuest64R0, VERR_INTERNAL_ERROR);
4682	Assert(pfnHandler);
4683	Assert(pVCpu->hwaccm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hwaccm.s.vmx.VMCSCache.Write.aField));
4684	Assert(pVCpu->hwaccm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hwaccm.s.vmx.VMCSCache.Read.aField));
4685
4686	#ifdef VBOX_STRICT
4687	for (unsigned i=0;i<pVCpu->hwaccm.s.vmx.VMCSCache.Write.cValidEntries;i++)
4688	Assert(vmxR0IsValidWriteField(pVCpu->hwaccm.s.vmx.VMCSCache.Write.aField[i]));
4689
4690	for (unsigned i=0;i<pVCpu->hwaccm.s.vmx.VMCSCache.Read.cValidEntries;i++)
4691	Assert(vmxR0IsValidReadField(pVCpu->hwaccm.s.vmx.VMCSCache.Read.aField[i]));
4692	#endif
4693
4694	/* Disable interrupts. */
4695	uOldEFlags = ASMIntDisableFlags();
4696
4697	pCpu = HWACCMR0GetCurrentCpu();
4698	pPageCpuPhys = RTR0MemObjGetPagePhysAddr(pCpu->pMemObj, 0);
4699
4700	/* Clear VM Control Structure. Marking it inactive, clearing implementation specific data and writing back VMCS data to memory. */
4701	VMXClearVMCS(pVCpu->hwaccm.s.vmx.pVMCSPhys);
4702
4703	/* Leave VMX Root Mode. */
4704	VMXDisable();
4705
4706	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
4707
4708	CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
4709	CPUMSetHyperEIP(pVCpu, pfnHandler);
4710	for (int i=(int)cbParam-1;i>=0;i--)
4711	CPUMPushHyper(pVCpu, paParam[i]);
4712
4713	STAM_PROFILE_ADV_START(&pVCpu->hwaccm.s.StatWorldSwitch3264, z);
4714	/* Call switcher. */
4715	rc = pVM->hwaccm.s.pfnHost32ToGuest64R0(pVM, RT_OFFSETOF(VM, aCpus[pVCpu->idCpu].cpum) - RT_OFFSETOF(VM, cpum));
4716	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatWorldSwitch3264, z);
4717
4718	/* Make sure the VMX instructions don't cause #UD faults. */
4719	ASMSetCR4(ASMGetCR4() \| X86_CR4_VMXE);
4720
4721	/* Enter VMX Root Mode */
4722	rc2 = VMXEnable(pPageCpuPhys);
4723	if (RT_FAILURE(rc2))
4724	{
4725	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
4726	ASMSetFlags(uOldEFlags);
4727	return VERR_VMX_VMXON_FAILED;
4728	}
4729
4730	rc2 = VMXActivateVMCS(pVCpu->hwaccm.s.vmx.pVMCSPhys);
4731	AssertRC(rc2);
4732	Assert(!(ASMGetFlags() & X86_EFL_IF));
4733	ASMSetFlags(uOldEFlags);
4734	return rc;
4735	}
4736
4737	#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL) */
4738
4739
4740	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
4741	/**
4742	* Executes VMWRITE
4743	*
4744	* @returns VBox status code
4745	* @param pVCpu The VMCPU to operate on.
4746	* @param idxField VMCS index
4747	* @param u64Val 16, 32 or 64 bits value
4748	*/
4749	VMMR0DECL(int) VMXWriteVMCS64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
4750	{
4751	int rc;
4752
4753	switch (idxField)
4754	{
4755	case VMX_VMCS_CTRL_TSC_OFFSET_FULL:
4756	case VMX_VMCS_CTRL_IO_BITMAP_A_FULL:
4757	case VMX_VMCS_CTRL_IO_BITMAP_B_FULL:
4758	case VMX_VMCS_CTRL_MSR_BITMAP_FULL:
4759	case VMX_VMCS_CTRL_VMEXIT_MSR_STORE_FULL:
4760	case VMX_VMCS_CTRL_VMEXIT_MSR_LOAD_FULL:
4761	case VMX_VMCS_CTRL_VMENTRY_MSR_LOAD_FULL:
4762	case VMX_VMCS_CTRL_VAPIC_PAGEADDR_FULL:
4763	case VMX_VMCS_CTRL_APIC_ACCESSADDR_FULL:
4764	case VMX_VMCS_GUEST_LINK_PTR_FULL:
4765	case VMX_VMCS_GUEST_PDPTR0_FULL:
4766	case VMX_VMCS_GUEST_PDPTR1_FULL:
4767	case VMX_VMCS_GUEST_PDPTR2_FULL:
4768	case VMX_VMCS_GUEST_PDPTR3_FULL:
4769	case VMX_VMCS_GUEST_DEBUGCTL_FULL:
4770	case VMX_VMCS_GUEST_EFER_FULL:
4771	case VMX_VMCS_CTRL_EPTP_FULL:
4772	/* These fields consist of two parts, which are both writable in 32 bits mode. */
4773	rc = VMXWriteVMCS32(idxField, u64Val);
4774	rc \|= VMXWriteVMCS32(idxField + 1, (uint32_t)(u64Val >> 32ULL));
4775	AssertRC(rc);
4776	return rc;
4777
4778	case VMX_VMCS64_GUEST_LDTR_BASE:
4779	case VMX_VMCS64_GUEST_TR_BASE:
4780	case VMX_VMCS64_GUEST_GDTR_BASE:
4781	case VMX_VMCS64_GUEST_IDTR_BASE:
4782	case VMX_VMCS64_GUEST_SYSENTER_EIP:
4783	case VMX_VMCS64_GUEST_SYSENTER_ESP:
4784	case VMX_VMCS64_GUEST_CR0:
4785	case VMX_VMCS64_GUEST_CR4:
4786	case VMX_VMCS64_GUEST_CR3:
4787	case VMX_VMCS64_GUEST_DR7:
4788	case VMX_VMCS64_GUEST_RIP:
4789	case VMX_VMCS64_GUEST_RSP:
4790	case VMX_VMCS64_GUEST_CS_BASE:
4791	case VMX_VMCS64_GUEST_DS_BASE:
4792	case VMX_VMCS64_GUEST_ES_BASE:
4793	case VMX_VMCS64_GUEST_FS_BASE:
4794	case VMX_VMCS64_GUEST_GS_BASE:
4795	case VMX_VMCS64_GUEST_SS_BASE:
4796	/* Queue a 64 bits value as we can't set it in 32 bits host mode. */
4797	if (u64Val >> 32ULL)
4798	rc = VMXWriteCachedVMCSEx(pVCpu, idxField, u64Val);
4799	else
4800	rc = VMXWriteVMCS32(idxField, (uint32_t)u64Val);
4801
4802	return rc;
4803
4804	default:
4805	AssertMsgFailed(("Unexpected field %x\n", idxField));
4806	return VERR_INVALID_PARAMETER;
4807	}
4808	}
4809
4810	/**
4811	* Cache VMCS writes for performance reasons (Darwin) and for running 64 bits guests on 32 bits hosts.
4812	*
4813	* @param pVCpu The VMCPU to operate on.
4814	* @param idxField VMCS field
4815	* @param u64Val Value
4816	*/
4817	VMMR0DECL(int) VMXWriteCachedVMCSEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
4818	{
4819	PVMCSCACHE pCache = &pVCpu->hwaccm.s.vmx.VMCSCache;
4820
4821	AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1, ("entries=%x\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
4822
4823	/* Make sure there are no duplicates. */
4824	for (unsigned i=0;i<pCache->Write.cValidEntries;i++)
4825	{
4826	if (pCache->Write.aField[i] == idxField)
4827	{
4828	pCache->Write.aFieldVal[i] = u64Val;
4829	return VINF_SUCCESS;
4830	}
4831	}
4832
4833	pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
4834	pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
4835	pCache->Write.cValidEntries++;
4836	return VINF_SUCCESS;
4837	}
4838
4839	#endif /* HC_ARCH_BITS == 32 && !VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0 */
4840
4841	#ifdef VBOX_STRICT
4842	static bool vmxR0IsValidReadField(uint32_t idxField)
4843	{
4844	switch(idxField)
4845	{
4846	case VMX_VMCS64_GUEST_RIP:
4847	case VMX_VMCS64_GUEST_RSP:
4848	case VMX_VMCS_GUEST_RFLAGS:
4849	case VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE:
4850	case VMX_VMCS_CTRL_CR0_READ_SHADOW:
4851	case VMX_VMCS64_GUEST_CR0:
4852	case VMX_VMCS_CTRL_CR4_READ_SHADOW:
4853	case VMX_VMCS64_GUEST_CR4:
4854	case VMX_VMCS64_GUEST_DR7:
4855	case VMX_VMCS32_GUEST_SYSENTER_CS:
4856	case VMX_VMCS64_GUEST_SYSENTER_EIP:
4857	case VMX_VMCS64_GUEST_SYSENTER_ESP:
4858	case VMX_VMCS32_GUEST_GDTR_LIMIT:
4859	case VMX_VMCS64_GUEST_GDTR_BASE:
4860	case VMX_VMCS32_GUEST_IDTR_LIMIT:
4861	case VMX_VMCS64_GUEST_IDTR_BASE:
4862	case VMX_VMCS16_GUEST_FIELD_CS:
4863	case VMX_VMCS32_GUEST_CS_LIMIT:
4864	case VMX_VMCS64_GUEST_CS_BASE:
4865	case VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS:
4866	case VMX_VMCS16_GUEST_FIELD_DS:
4867	case VMX_VMCS32_GUEST_DS_LIMIT:
4868	case VMX_VMCS64_GUEST_DS_BASE:
4869	case VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS:
4870	case VMX_VMCS16_GUEST_FIELD_ES:
4871	case VMX_VMCS32_GUEST_ES_LIMIT:
4872	case VMX_VMCS64_GUEST_ES_BASE:
4873	case VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS:
4874	case VMX_VMCS16_GUEST_FIELD_FS:
4875	case VMX_VMCS32_GUEST_FS_LIMIT:
4876	case VMX_VMCS64_GUEST_FS_BASE:
4877	case VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS:
4878	case VMX_VMCS16_GUEST_FIELD_GS:
4879	case VMX_VMCS32_GUEST_GS_LIMIT:
4880	case VMX_VMCS64_GUEST_GS_BASE:
4881	case VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS:
4882	case VMX_VMCS16_GUEST_FIELD_SS:
4883	case VMX_VMCS32_GUEST_SS_LIMIT:
4884	case VMX_VMCS64_GUEST_SS_BASE:
4885	case VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS:
4886	case VMX_VMCS16_GUEST_FIELD_LDTR:
4887	case VMX_VMCS32_GUEST_LDTR_LIMIT:
4888	case VMX_VMCS64_GUEST_LDTR_BASE:
4889	case VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS:
4890	case VMX_VMCS16_GUEST_FIELD_TR:
4891	case VMX_VMCS32_GUEST_TR_LIMIT:
4892	case VMX_VMCS64_GUEST_TR_BASE:
4893	case VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS:
4894	case VMX_VMCS32_RO_EXIT_REASON:
4895	case VMX_VMCS32_RO_VM_INSTR_ERROR:
4896	case VMX_VMCS32_RO_EXIT_INSTR_LENGTH:
4897	case VMX_VMCS32_RO_EXIT_INTERRUPTION_ERRCODE:
4898	case VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO:
4899	case VMX_VMCS32_RO_EXIT_INSTR_INFO:
4900	case VMX_VMCS_RO_EXIT_QUALIFICATION:
4901	case VMX_VMCS32_RO_IDT_INFO:
4902	case VMX_VMCS32_RO_IDT_ERRCODE:
4903	case VMX_VMCS64_GUEST_CR3:
4904	case VMX_VMCS_EXIT_PHYS_ADDR_FULL:
4905	return true;
4906	}
4907	return false;
4908	}
4909
4910	static bool vmxR0IsValidWriteField(uint32_t idxField)
4911	{
4912	switch(idxField)
4913	{
4914	case VMX_VMCS64_GUEST_LDTR_BASE:
4915	case VMX_VMCS64_GUEST_TR_BASE:
4916	case VMX_VMCS64_GUEST_GDTR_BASE:
4917	case VMX_VMCS64_GUEST_IDTR_BASE:
4918	case VMX_VMCS64_GUEST_SYSENTER_EIP:
4919	case VMX_VMCS64_GUEST_SYSENTER_ESP:
4920	case VMX_VMCS64_GUEST_CR0:
4921	case VMX_VMCS64_GUEST_CR4:
4922	case VMX_VMCS64_GUEST_CR3:
4923	case VMX_VMCS64_GUEST_DR7:
4924	case VMX_VMCS64_GUEST_RIP:
4925	case VMX_VMCS64_GUEST_RSP:
4926	case VMX_VMCS64_GUEST_CS_BASE:
4927	case VMX_VMCS64_GUEST_DS_BASE:
4928	case VMX_VMCS64_GUEST_ES_BASE:
4929	case VMX_VMCS64_GUEST_FS_BASE:
4930	case VMX_VMCS64_GUEST_GS_BASE:
4931	case VMX_VMCS64_GUEST_SS_BASE:
4932	return true;
4933	}
4934	return false;
4935	}
4936
4937	#endif

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMR0/HWVMXR0.cpp@ 35346

Download in other formats: