HWVMXR0.cpp@ 15440

Last change on this file since 15440 was 15440, checked in by vboxsync, 16 years ago
Sync CR2 properly for VT-x EPT (unable to access the 64 bits cr2 from 32 bits mode).
Property svn:eol-style set to `native` Property svn:keywords set to `Id`
File size: 146.6 KB

Line
1	/* $Id: HWVMXR0.cpp 15440 2008-12-13 13:09:30Z vboxsync $ */
2	/** @file
3	* HWACCM VMX - Host Context Ring 0.
4	*/
5
6	/*
7	* Copyright (C) 2006-2007 Sun Microsystems, Inc.
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*
17	* Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa
18	* Clara, CA 95054 USA or visit http://www.sun.com if you need
19	* additional information or have any questions.
20	*/
21
22
23	/*******************************************************************************
24	* Header Files *
25	*******************************************************************************/
26	#define LOG_GROUP LOG_GROUP_HWACCM
27	#include <VBox/hwaccm.h>
28	#include "HWACCMInternal.h"
29	#include <VBox/vm.h>
30	#include <VBox/x86.h>
31	#include <VBox/pgm.h>
32	#include <VBox/pdm.h>
33	#include <VBox/err.h>
34	#include <VBox/log.h>
35	#include <VBox/selm.h>
36	#include <VBox/iom.h>
37	#include <iprt/param.h>
38	#include <iprt/assert.h>
39	#include <iprt/asm.h>
40	#include <iprt/string.h>
41	#include "HWVMXR0.h"
42
43	/*******************************************************************************
44	* Defined Constants And Macros *
45	*******************************************************************************/
46	#if defined(RT_ARCH_AMD64)
47	# define VMX_IS_64BIT_HOST_MODE() (true)
48	#elif defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
49	# define VMX_IS_64BIT_HOST_MODE() (g_fVMXIs64bitHost != 0)
50	#else
51	# define VMX_IS_64BIT_HOST_MODE() (false)
52	#endif
53
54	/*******************************************************************************
55	* Global Variables *
56	*******************************************************************************/
57	/* IO operation lookup arrays. */
58	static uint32_t const g_aIOSize[4] = {1, 2, 0, 4};
59	static uint32_t const g_aIOOpAnd[4] = {0xff, 0xffff, 0, 0xffffffff};
60
61	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
62	/** See HWACCMR0A.asm. */
63	extern "C" uint32_t g_fVMXIs64bitHost;
64	#endif
65
66	/*******************************************************************************
67	* Local Functions *
68	*******************************************************************************/
69	static void VMXR0ReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, int rc, PCPUMCTX pCtx);
70	static void vmxR0SetupTLBEPT(PVM pVM, PVMCPU pVCpu);
71	static void vmxR0SetupTLBVPID(PVM pVM, PVMCPU pVCpu);
72	static void vmxR0SetupTLBDummy(PVM pVM, PVMCPU pVCpu);
73	static void vmxR0FlushEPT(PVM pVM, PVMCPU pVCpu, VMX_FLUSH enmFlush, RTGCPHYS GCPhys);
74	static void vmxR0FlushVPID(PVM pVM, PVMCPU pVCpu, VMX_FLUSH enmFlush, RTGCPTR GCPtr);
75	static void vmxR0UpdateExceptionBitmap(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx);
76
77
78	static void VMXR0CheckError(PVM pVM, PVMCPU pVCpu, int rc)
79	{
80	if (rc == VERR_VMX_GENERIC)
81	{
82	RTCCUINTREG instrError;
83
84	VMXReadVMCS(VMX_VMCS32_RO_VM_INSTR_ERROR, &instrError);
85	pVCpu->hwaccm.s.vmx.lasterror.ulInstrError = instrError;
86	}
87	pVM->hwaccm.s.lLastError = rc;
88	}
89
90	/**
91	* Sets up and activates VT-x on the current CPU
92	*
93	* @returns VBox status code.
94	* @param pCpu CPU info struct
95	* @param pVM The VM to operate on. (can be NULL after a resume!!)
96	* @param pvPageCpu Pointer to the global cpu page
97	* @param pPageCpuPhys Physical address of the global cpu page
98	*/
99	VMMR0DECL(int) VMXR0EnableCpu(PHWACCM_CPUINFO pCpu, PVM pVM, void *pvPageCpu, RTHCPHYS pPageCpuPhys)
100	{
101	AssertReturn(pPageCpuPhys, VERR_INVALID_PARAMETER);
102	AssertReturn(pvPageCpu, VERR_INVALID_PARAMETER);
103
104	#ifdef LOG_ENABLED
105	SUPR0Printf("VMXR0EnableCpu cpu %d page (%x) %x\n", pCpu->idCpu, pvPageCpu, (uint32_t)pPageCpuPhys);
106	#endif
107	if (pVM)
108	{
109	/* Set revision dword at the beginning of the VMXON structure. */
110	(uint32_t )pvPageCpu = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hwaccm.s.vmx.msr.vmx_basic_info);
111	}
112
113	/** @todo we should unmap the two pages from the virtual address space in order to prevent accidental corruption.
114	* (which can have very bad consequences!!!)
115	*/
116
117	/* Make sure the VMX instructions don't cause #UD faults. */
118	ASMSetCR4(ASMGetCR4() \| X86_CR4_VMXE);
119
120	/* Enter VMX Root Mode */
121	int rc = VMXEnable(pPageCpuPhys);
122	if (RT_FAILURE(rc))
123	{
124	if (pVM)
125	VMXR0CheckError(pVM, &pVM->aCpus[0], rc);
126	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
127	return VERR_VMX_VMXON_FAILED;
128	}
129	return VINF_SUCCESS;
130	}
131
132	/**
133	* Deactivates VT-x on the current CPU
134	*
135	* @returns VBox status code.
136	* @param pCpu CPU info struct
137	* @param pvPageCpu Pointer to the global cpu page
138	* @param pPageCpuPhys Physical address of the global cpu page
139	*/
140	VMMR0DECL(int) VMXR0DisableCpu(PHWACCM_CPUINFO pCpu, void *pvPageCpu, RTHCPHYS pPageCpuPhys)
141	{
142	AssertReturn(pPageCpuPhys, VERR_INVALID_PARAMETER);
143	AssertReturn(pvPageCpu, VERR_INVALID_PARAMETER);
144
145	/* Leave VMX Root Mode. */
146	VMXDisable();
147
148	/* And clear the X86_CR4_VMXE bit */
149	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
150
151	#ifdef LOG_ENABLED
152	SUPR0Printf("VMXR0DisableCpu cpu %d\n", pCpu->idCpu);
153	#endif
154	return VINF_SUCCESS;
155	}
156
157	/**
158	* Does Ring-0 per VM VT-x init.
159	*
160	* @returns VBox status code.
161	* @param pVM The VM to operate on.
162	*/
163	VMMR0DECL(int) VMXR0InitVM(PVM pVM)
164	{
165	int rc;
166
167	#ifdef LOG_ENABLED
168	SUPR0Printf("VMXR0InitVM %x\n", pVM);
169	#endif
170
171	pVM->hwaccm.s.vmx.pMemObjAPIC = NIL_RTR0MEMOBJ;
172
173	if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
174	{
175	/* Allocate one page for the virtual APIC mmio cache. */
176	rc = RTR0MemObjAllocCont(&pVM->hwaccm.s.vmx.pMemObjAPIC, 1 << PAGE_SHIFT, true /* executable R0 mapping */);
177	AssertRC(rc);
178	if (RT_FAILURE(rc))
179	return rc;
180
181	pVM->hwaccm.s.vmx.pAPIC = (uint8_t *)RTR0MemObjAddress(pVM->hwaccm.s.vmx.pMemObjAPIC);
182	pVM->hwaccm.s.vmx.pAPICPhys = RTR0MemObjGetPagePhysAddr(pVM->hwaccm.s.vmx.pMemObjAPIC, 0);
183	ASMMemZero32(pVM->hwaccm.s.vmx.pAPIC, PAGE_SIZE);
184	}
185	else
186	{
187	pVM->hwaccm.s.vmx.pMemObjAPIC = 0;
188	pVM->hwaccm.s.vmx.pAPIC = 0;
189	pVM->hwaccm.s.vmx.pAPICPhys = 0;
190	}
191
192	/* Allocate the MSR bitmap if this feature is supported. */
193	if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
194	{
195	rc = RTR0MemObjAllocCont(&pVM->hwaccm.s.vmx.pMemObjMSRBitmap, 1 << PAGE_SHIFT, true /* executable R0 mapping */);
196	AssertRC(rc);
197	if (RT_FAILURE(rc))
198	return rc;
199
200	pVM->hwaccm.s.vmx.pMSRBitmap = (uint8_t *)RTR0MemObjAddress(pVM->hwaccm.s.vmx.pMemObjMSRBitmap);
201	pVM->hwaccm.s.vmx.pMSRBitmapPhys = RTR0MemObjGetPagePhysAddr(pVM->hwaccm.s.vmx.pMemObjMSRBitmap, 0);
202	memset(pVM->hwaccm.s.vmx.pMSRBitmap, 0xff, PAGE_SIZE);
203	}
204
205	/* Allocate VMCBs for all guest CPUs. */
206	for (unsigned i=0;i<pVM->cCPUs;i++)
207	{
208	PVMCPU pVCpu = &pVM->aCpus[i];
209
210	pVCpu->hwaccm.s.vmx.pMemObjVMCS = NIL_RTR0MEMOBJ;
211
212	/* Allocate one page for the VM control structure (VMCS). */
213	rc = RTR0MemObjAllocCont(&pVCpu->hwaccm.s.vmx.pMemObjVMCS, 1 << PAGE_SHIFT, true /* executable R0 mapping */);
214	AssertRC(rc);
215	if (RT_FAILURE(rc))
216	return rc;
217
218	pVCpu->hwaccm.s.vmx.pVMCS = RTR0MemObjAddress(pVCpu->hwaccm.s.vmx.pMemObjVMCS);
219	pVCpu->hwaccm.s.vmx.pVMCSPhys = RTR0MemObjGetPagePhysAddr(pVCpu->hwaccm.s.vmx.pMemObjVMCS, 0);
220	ASMMemZero32(pVCpu->hwaccm.s.vmx.pVMCS, PAGE_SIZE);
221
222	pVCpu->hwaccm.s.vmx.cr0_mask = 0;
223	pVCpu->hwaccm.s.vmx.cr4_mask = 0;
224
225	/* Current guest paging mode. */
226	pVCpu->hwaccm.s.vmx.enmCurrGuestMode = PGMMODE_REAL;
227
228	#ifdef LOG_ENABLED
229	SUPR0Printf("VMXR0InitVM %x VMCS=%x (%x)\n", pVM, pVCpu->hwaccm.s.vmx.pVMCS, (uint32_t)pVCpu->hwaccm.s.vmx.pVMCSPhys);
230	#endif
231	}
232
233	return VINF_SUCCESS;
234	}
235
236	/**
237	* Does Ring-0 per VM VT-x termination.
238	*
239	* @returns VBox status code.
240	* @param pVM The VM to operate on.
241	*/
242	VMMR0DECL(int) VMXR0TermVM(PVM pVM)
243	{
244	for (unsigned i=0;i<pVM->cCPUs;i++)
245	{
246	if (pVM->aCpus[i].hwaccm.s.vmx.pMemObjVMCS != NIL_RTR0MEMOBJ)
247	{
248	RTR0MemObjFree(pVM->aCpus[i].hwaccm.s.vmx.pMemObjVMCS, false);
249	pVM->aCpus[i].hwaccm.s.vmx.pMemObjVMCS = NIL_RTR0MEMOBJ;
250	pVM->aCpus[i].hwaccm.s.vmx.pVMCS = 0;
251	pVM->aCpus[i].hwaccm.s.vmx.pVMCSPhys = 0;
252	}
253	}
254	if (pVM->hwaccm.s.vmx.pMemObjAPIC != NIL_RTR0MEMOBJ)
255	{
256	RTR0MemObjFree(pVM->hwaccm.s.vmx.pMemObjAPIC, false);
257	pVM->hwaccm.s.vmx.pMemObjAPIC = NIL_RTR0MEMOBJ;
258	pVM->hwaccm.s.vmx.pAPIC = 0;
259	pVM->hwaccm.s.vmx.pAPICPhys = 0;
260	}
261	if (pVM->hwaccm.s.vmx.pMemObjMSRBitmap != NIL_RTR0MEMOBJ)
262	{
263	RTR0MemObjFree(pVM->hwaccm.s.vmx.pMemObjMSRBitmap, false);
264	pVM->hwaccm.s.vmx.pMemObjMSRBitmap = NIL_RTR0MEMOBJ;
265	pVM->hwaccm.s.vmx.pMSRBitmap = 0;
266	pVM->hwaccm.s.vmx.pMSRBitmapPhys = 0;
267	}
268	return VINF_SUCCESS;
269	}
270
271	/**
272	* Sets up VT-x for the specified VM
273	*
274	* @returns VBox status code.
275	* @param pVM The VM to operate on.
276	*/
277	VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
278	{
279	int rc = VINF_SUCCESS;
280	uint32_t val;
281
282	AssertReturn(pVM, VERR_INVALID_PARAMETER);
283
284	for (unsigned i=0;i<pVM->cCPUs;i++)
285	{
286	PVMCPU pVCpu = &pVM->aCpus[i];
287
288	Assert(pVCpu->hwaccm.s.vmx.pVMCS);
289
290	/* Set revision dword at the beginning of the VMCS structure. */
291	(uint32_t )pVCpu->hwaccm.s.vmx.pVMCS = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hwaccm.s.vmx.msr.vmx_basic_info);
292
293	/* Clear VM Control Structure. */
294	Log(("pVMCSPhys = %RHp\n", pVCpu->hwaccm.s.vmx.pVMCSPhys));
295	rc = VMXClearVMCS(pVCpu->hwaccm.s.vmx.pVMCSPhys);
296	if (RT_FAILURE(rc))
297	goto vmx_end;
298
299	/* Activate the VM Control Structure. */
300	rc = VMXActivateVMCS(pVCpu->hwaccm.s.vmx.pVMCSPhys);
301	if (RT_FAILURE(rc))
302	goto vmx_end;
303
304	/* VMX_VMCS_CTRL_PIN_EXEC_CONTROLS
305	* Set required bits to one and zero according to the MSR capabilities.
306	*/
307	val = pVM->hwaccm.s.vmx.msr.vmx_pin_ctls.n.disallowed0;
308	/* External and non-maskable interrupts cause VM-exits. */
309	val = val \| VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_EXT_INT_EXIT \| VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_NMI_EXIT;
310	val &= pVM->hwaccm.s.vmx.msr.vmx_pin_ctls.n.allowed1;
311
312	rc = VMXWriteVMCS(VMX_VMCS_CTRL_PIN_EXEC_CONTROLS, val);
313	AssertRC(rc);
314
315	/* VMX_VMCS_CTRL_PROC_EXEC_CONTROLS
316	* Set required bits to one and zero according to the MSR capabilities.
317	*/
318	val = pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.disallowed0;
319	/* Program which event cause VM-exits and which features we want to use. */
320	val = val \| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_HLT_EXIT
321	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_TSC_OFFSET
322	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT
323	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_UNCOND_IO_EXIT
324	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MWAIT_EXIT; /* don't execute mwait or else we'll idle inside the guest (host thinks the cpu load is high) */
325
326	/* Without nested paging we should intercept invlpg and cr3 mov instructions. */
327	if (!pVM->hwaccm.s.fNestedPaging)
328	val \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INVLPG_EXIT
329	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
330	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT;
331
332	/* Note: VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MWAIT_EXIT might cause a vmlaunch failure with an invalid control fields error. (combined with some other exit reasons) */
333	if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
334	{
335	/* CR8 reads from the APIC shadow page; writes cause an exit is they lower the TPR below the threshold */
336	val \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW;
337	Assert(pVM->hwaccm.s.vmx.pAPIC);
338	}
339	else
340	/* Exit on CR8 reads & writes in case the TPR shadow feature isn't present. */
341	val \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR8_STORE_EXIT \| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR8_LOAD_EXIT;
342
343	#ifdef VBOX_WITH_VTX_MSR_BITMAPS
344	if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
345	{
346	Assert(pVM->hwaccm.s.vmx.pMSRBitmapPhys);
347	val \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS;
348	}
349	#endif
350
351	/* We will use the secondary control if it's present. */
352	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
353
354	/* Mask away the bits that the CPU doesn't support */
355	/** @todo make sure they don't conflict with the above requirements. */
356	val &= pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1;
357	pVCpu->hwaccm.s.vmx.proc_ctls = val;
358
359	rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, val);
360	AssertRC(rc);
361
362	if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
363	{
364	/* VMX_VMCS_CTRL_PROC_EXEC_CONTROLS2
365	* Set required bits to one and zero according to the MSR capabilities.
366	*/
367	val = pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2.n.disallowed0;
368	val \|= VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT;
369
370	#ifdef HWACCM_VTX_WITH_EPT
371	if (pVM->hwaccm.s.fNestedPaging)
372	val \|= VMX_VMCS_CTRL_PROC_EXEC2_EPT;
373	#endif /* HWACCM_VTX_WITH_EPT */
374	#ifdef HWACCM_VTX_WITH_VPID
375	else
376	if (pVM->hwaccm.s.vmx.fVPID)
377	val \|= VMX_VMCS_CTRL_PROC_EXEC2_VPID;
378	#endif /* HWACCM_VTX_WITH_VPID */
379
380	/* Mask away the bits that the CPU doesn't support */
381	/** @todo make sure they don't conflict with the above requirements. */
382	val &= pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2.n.allowed1;
383
384	rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS2, val);
385	AssertRC(rc);
386	}
387
388	/* VMX_VMCS_CTRL_CR3_TARGET_COUNT
389	* Set required bits to one and zero according to the MSR capabilities.
390	*/
391	rc = VMXWriteVMCS(VMX_VMCS_CTRL_CR3_TARGET_COUNT, 0);
392	AssertRC(rc);
393
394	/* Forward all exception except #NM & #PF to the guest.
395	* We always need to check pagefaults since our shadow page table can be out of sync.
396	* And we always lazily sync the FPU & XMM state.
397	*/
398
399	/** @todo Possible optimization:
400	* Keep the FPU and XMM state current in the EM thread. That way there's no need to
401	* lazily sync anything, but the downside is that we can't use the FPU stack or XMM
402	* registers ourselves of course.
403	*
404	* Note: only possible if the current state is actually ours (X86_CR0_TS flag)
405	*/
406
407	/* Don't filter page faults; all of them should cause a switch. */
408	rc = VMXWriteVMCS(VMX_VMCS_CTRL_PAGEFAULT_ERROR_MASK, 0);
409	rc \|= VMXWriteVMCS(VMX_VMCS_CTRL_PAGEFAULT_ERROR_MATCH, 0);
410	AssertRC(rc);
411
412	/* Init TSC offset to zero. */
413	rc = VMXWriteVMCS64(VMX_VMCS_CTRL_TSC_OFFSET_FULL, 0);
414	AssertRC(rc);
415
416	rc = VMXWriteVMCS64(VMX_VMCS_CTRL_IO_BITMAP_A_FULL, 0);
417	AssertRC(rc);
418
419	rc = VMXWriteVMCS64(VMX_VMCS_CTRL_IO_BITMAP_B_FULL, 0);
420	AssertRC(rc);
421
422	/* Set the MSR bitmap address. */
423	if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
424	{
425	/* Optional */
426	rc = VMXWriteVMCS64(VMX_VMCS_CTRL_MSR_BITMAP_FULL, pVM->hwaccm.s.vmx.pMSRBitmapPhys);
427	AssertRC(rc);
428	}
429
430	/* Clear MSR controls. */
431	rc = VMXWriteVMCS64(VMX_VMCS_CTRL_VMEXIT_MSR_STORE_FULL, 0);
432	rc \|= VMXWriteVMCS64(VMX_VMCS_CTRL_VMEXIT_MSR_LOAD_FULL, 0);
433	rc \|= VMXWriteVMCS64(VMX_VMCS_CTRL_VMENTRY_MSR_LOAD_FULL, 0);
434	rc \|= VMXWriteVMCS(VMX_VMCS_CTRL_EXIT_MSR_STORE_COUNT, 0);
435	rc \|= VMXWriteVMCS(VMX_VMCS_CTRL_EXIT_MSR_LOAD_COUNT, 0);
436	AssertRC(rc);
437
438	if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
439	{
440	Assert(pVM->hwaccm.s.vmx.pMemObjAPIC);
441	/* Optional */
442	rc = VMXWriteVMCS(VMX_VMCS_CTRL_TPR_THRESHOLD, 0);
443	rc \|= VMXWriteVMCS64(VMX_VMCS_CTRL_VAPIC_PAGEADDR_FULL, pVM->hwaccm.s.vmx.pAPICPhys);
444	AssertRC(rc);
445	}
446
447	/* Set link pointer to -1. Not currently used. */
448	rc = VMXWriteVMCS64(VMX_VMCS_GUEST_LINK_PTR_FULL, 0xFFFFFFFFFFFFFFFFULL);
449	AssertRC(rc);
450
451	/* Clear VM Control Structure. Marking it inactive, clearing implementation specific data and writing back VMCS data to memory. */
452	rc = VMXClearVMCS(pVCpu->hwaccm.s.vmx.pVMCSPhys);
453	AssertRC(rc);
454
455	/* Configure the VMCS read cache. */
456	PVMCSCACHE pCache = &pVCpu->hwaccm.s.vmx.VMCSCache;
457
458	VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_RIP);
459	VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_RSP);
460	VMXSetupCachedReadVMCS(pCache, VMX_VMCS_GUEST_RFLAGS);
461	VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE);
462	VMXSetupCachedReadVMCS(pCache, VMX_VMCS_CTRL_CR0_READ_SHADOW);
463	VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_CR0);
464	VMXSetupCachedReadVMCS(pCache, VMX_VMCS_CTRL_CR4_READ_SHADOW);
465	VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_CR4);
466	VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_DR7);
467	VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_GUEST_SYSENTER_CS);
468	VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_SYSENTER_EIP);
469	VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_SYSENTER_ESP);
470	VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_GUEST_GDTR_LIMIT);
471	VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_GDTR_BASE);
472	VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_GUEST_IDTR_LIMIT);
473	VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_IDTR_BASE);
474
475	VMX_SETUP_SELREG(ES, pCache);
476	VMX_SETUP_SELREG(SS, pCache);
477	VMX_SETUP_SELREG(CS, pCache);
478	VMX_SETUP_SELREG(DS, pCache);
479	VMX_SETUP_SELREG(FS, pCache);
480	VMX_SETUP_SELREG(GS, pCache);
481	VMX_SETUP_SELREG(LDTR, pCache);
482	VMX_SETUP_SELREG(TR, pCache);
483
484	/* Status code VMCS reads. */
485	VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_EXIT_REASON);
486	VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_VM_INSTR_ERROR);
487	VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_EXIT_INSTR_LENGTH);
488	VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_EXIT_INTERRUPTION_ERRCODE);
489	VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO);
490	VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_EXIT_INSTR_INFO);
491	VMXSetupCachedReadVMCS(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
492	VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_IDT_INFO);
493	VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_IDT_ERRCODE);
494
495	if (pVM->hwaccm.s.fNestedPaging)
496	{
497	VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_CR3);
498	VMXSetupCachedReadVMCS(pCache, VMX_VMCS_EXIT_PHYS_ADDR_FULL);
499	pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
500	}
501	else
502	pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
503	} /* for each VMCPU */
504
505	/* Choose the right TLB setup function. */
506	if (pVM->hwaccm.s.fNestedPaging)
507	{
508	pVM->hwaccm.s.vmx.pfnSetupTaggedTLB = vmxR0SetupTLBEPT;
509
510	/* Default values for flushing. */
511	pVM->hwaccm.s.vmx.enmFlushPage = VMX_FLUSH_ALL_CONTEXTS;
512	pVM->hwaccm.s.vmx.enmFlushContext = VMX_FLUSH_ALL_CONTEXTS;
513
514	/* If the capabilities specify we can do more, then make use of it. */
515	if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVEPT_CAPS_INDIV)
516	pVM->hwaccm.s.vmx.enmFlushPage = VMX_FLUSH_PAGE;
517	else
518	if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVEPT_CAPS_CONTEXT)
519	pVM->hwaccm.s.vmx.enmFlushPage = VMX_FLUSH_SINGLE_CONTEXT;
520
521	if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVEPT_CAPS_CONTEXT)
522	pVM->hwaccm.s.vmx.enmFlushContext = VMX_FLUSH_SINGLE_CONTEXT;
523	}
524	#ifdef HWACCM_VTX_WITH_VPID
525	else
526	if (pVM->hwaccm.s.vmx.fVPID)
527	{
528	pVM->hwaccm.s.vmx.pfnSetupTaggedTLB = vmxR0SetupTLBVPID;
529
530	/* Default values for flushing. */
531	pVM->hwaccm.s.vmx.enmFlushPage = VMX_FLUSH_ALL_CONTEXTS;
532	pVM->hwaccm.s.vmx.enmFlushContext = VMX_FLUSH_ALL_CONTEXTS;
533
534	/* If the capabilities specify we can do more, then make use of it. */
535	if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVVPID_CAPS_INDIV)
536	pVM->hwaccm.s.vmx.enmFlushPage = VMX_FLUSH_PAGE;
537	else
538	if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVVPID_CAPS_CONTEXT)
539	pVM->hwaccm.s.vmx.enmFlushPage = VMX_FLUSH_SINGLE_CONTEXT;
540
541	if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVVPID_CAPS_CONTEXT)
542	pVM->hwaccm.s.vmx.enmFlushContext = VMX_FLUSH_SINGLE_CONTEXT;
543	}
544	#endif /* HWACCM_VTX_WITH_VPID */
545	else
546	pVM->hwaccm.s.vmx.pfnSetupTaggedTLB = vmxR0SetupTLBDummy;
547
548	vmx_end:
549	VMXR0CheckError(pVM, &pVM->aCpus[0], rc);
550	return rc;
551	}
552
553
554	/**
555	* Injects an event (trap or external interrupt)
556	*
557	* @returns VBox status code.
558	* @param pVM The VM to operate on.
559	* @param pVCpu The VMCPU to operate on.
560	* @param pCtx CPU Context
561	* @param intInfo VMX interrupt info
562	* @param cbInstr Opcode length of faulting instruction
563	* @param errCode Error code (optional)
564	*/
565	static int VMXR0InjectEvent(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, uint32_t intInfo, uint32_t cbInstr, uint32_t errCode)
566	{
567	int rc;
568	uint32_t iGate = VMX_EXIT_INTERRUPTION_INFO_VECTOR(intInfo);
569
570	#ifdef VBOX_STRICT
571	if (iGate == 0xE)
572	LogFlow(("VMXR0InjectEvent: Injecting interrupt %d at %RGv error code=%08x CR2=%08x intInfo=%08x\n", iGate, (RTGCPTR)pCtx->rip, errCode, pCtx->cr2, intInfo));
573	else
574	if (iGate < 0x20)
575	LogFlow(("VMXR0InjectEvent: Injecting interrupt %d at %RGv error code=%08x\n", iGate, (RTGCPTR)pCtx->rip, errCode));
576	else
577	{
578	LogFlow(("INJ-EI: %x at %RGv\n", iGate, (RTGCPTR)pCtx->rip));
579	Assert(!VM_FF_ISSET(pVM, VM_FF_INHIBIT_INTERRUPTS));
580	Assert(pCtx->eflags.u32 & X86_EFL_IF);
581	}
582	#endif
583
584	#ifdef HWACCM_VMX_EMULATE_REALMODE
585	if (CPUMIsGuestInRealModeEx(pCtx))
586	{
587	RTGCPHYS GCPhysHandler;
588	uint16_t offset, ip;
589	RTSEL sel;
590
591	/* Injecting events doesn't work right with real mode emulation.
592	* (#GP if we try to inject external hardware interrupts)
593	* Inject the interrupt or trap directly instead.
594	*/
595	Log(("Manual interrupt/trap '%x' inject (real mode)\n", iGate));
596
597	/* Check if the interrupt handler is present. */
598	if (iGate * 4 + 3 > pCtx->idtr.cbIdt)
599	{
600	Log(("IDT cbIdt violation\n"));
601	if (iGate != X86_XCPT_DF)
602	{
603	RTGCUINTPTR intInfo;
604
605	intInfo = (iGate == X86_XCPT_GP) ? (uint32_t)X86_XCPT_DF : iGate;
606	intInfo \|= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
607	intInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
608	intInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HWEXCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
609
610	return VMXR0InjectEvent(pVM, pVCpu, pCtx, intInfo, 0, 0 /* no error code according to the Intel docs */);
611	}
612	Log(("Triple fault -> reset the VM!\n"));
613	return VINF_EM_RESET;
614	}
615	if ( VMX_EXIT_INTERRUPTION_INFO_TYPE(intInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW
616	\|\| iGate == 3 /* Both #BP and #OF point to the instruction after. */
617	\|\| iGate == 4)
618	{
619	ip = pCtx->ip + cbInstr;
620	}
621	else
622	ip = pCtx->ip;
623
624	/* Read the selector:offset pair of the interrupt handler. */
625	GCPhysHandler = (RTGCPHYS)pCtx->idtr.pIdt + iGate * 4;
626	PGMPhysRead(pVM, GCPhysHandler, &offset, sizeof(offset));
627	PGMPhysRead(pVM, GCPhysHandler + 2, &sel, sizeof(sel));
628
629	LogFlow(("IDT handler %04X:%04X\n", sel, offset));
630
631	/* Construct the stack frame. */
632	/** @todo should check stack limit. */
633	pCtx->sp -= 2;
634	LogFlow(("ss:sp %04X:%04X eflags=%x\n", pCtx->ss, pCtx->sp, pCtx->eflags.u));
635	PGMPhysWrite(pVM, pCtx->ssHid.u64Base + pCtx->sp, &pCtx->eflags, sizeof(uint16_t));
636	pCtx->sp -= 2;
637	LogFlow(("ss:sp %04X:%04X cs=%x\n", pCtx->ss, pCtx->sp, pCtx->cs));
638	PGMPhysWrite(pVM, pCtx->ssHid.u64Base + pCtx->sp, &pCtx->cs, sizeof(uint16_t));
639	pCtx->sp -= 2;
640	LogFlow(("ss:sp %04X:%04X ip=%x\n", pCtx->ss, pCtx->sp, ip));
641	PGMPhysWrite(pVM, pCtx->ssHid.u64Base + pCtx->sp, &ip, sizeof(ip));
642
643	/* Update the CPU state for executing the handler. */
644	pCtx->rip = offset;
645	pCtx->cs = sel;
646	pCtx->csHid.u64Base = sel << 4;
647	pCtx->eflags.u &= ~(X86_EFL_IF\|X86_EFL_TF\|X86_EFL_RF\|X86_EFL_AC);
648
649	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_GUEST_SEGMENT_REGS;
650	return VINF_SUCCESS;
651	}
652	#endif /* HWACCM_VMX_EMULATE_REALMODE */
653
654	/* Set event injection state. */
655	rc = VMXWriteVMCS(VMX_VMCS_CTRL_ENTRY_IRQ_INFO, intInfo \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT));
656
657	rc \|= VMXWriteVMCS(VMX_VMCS_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
658	rc \|= VMXWriteVMCS(VMX_VMCS_CTRL_ENTRY_EXCEPTION_ERRCODE, errCode);
659
660	AssertRC(rc);
661	return rc;
662	}
663
664
665	/**
666	* Checks for pending guest interrupts and injects them
667	*
668	* @returns VBox status code.
669	* @param pVM The VM to operate on.
670	* @param pVCpu The VMCPU to operate on.
671	* @param pCtx CPU Context
672	*/
673	static int VMXR0CheckPendingInterrupt(PVM pVM, PVMCPU pVCpu, CPUMCTX *pCtx)
674	{
675	int rc;
676
677	/* Dispatch any pending interrupts. (injected before, but a VM exit occurred prematurely) */
678	if (pVCpu->hwaccm.s.Event.fPending)
679	{
680	Log(("Reinjecting event %RX64 %08x at %RGv cr2=%RX64\n", pVCpu->hwaccm.s.Event.intInfo, pVCpu->hwaccm.s.Event.errCode, (RTGCPTR)pCtx->rip, pCtx->cr2));
681	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatIntReinject);
682	rc = VMXR0InjectEvent(pVM, pVCpu, pCtx, pVCpu->hwaccm.s.Event.intInfo, 0, pVCpu->hwaccm.s.Event.errCode);
683	AssertRC(rc);
684
685	pVCpu->hwaccm.s.Event.fPending = false;
686	return VINF_SUCCESS;
687	}
688
689	if (pVM->hwaccm.s.fInjectNMI)
690	{
691	RTGCUINTPTR intInfo;
692
693	intInfo = X86_XCPT_NMI;
694	intInfo \|= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
695	intInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
696
697	rc = VMXR0InjectEvent(pVM, pVCpu, pCtx, intInfo, 0, 0);
698	AssertRC(rc);
699
700	pVM->hwaccm.s.fInjectNMI = false;
701	return VINF_SUCCESS;
702	}
703
704	/* When external interrupts are pending, we should exit the VM when IF is set. */
705	if ( !TRPMHasTrap(pVM)
706	&& VM_FF_ISPENDING(pVM, (VM_FF_INTERRUPT_APIC\|VM_FF_INTERRUPT_PIC)))
707	{
708	if (!(pCtx->eflags.u32 & X86_EFL_IF))
709	{
710	if (!(pVCpu->hwaccm.s.vmx.proc_ctls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_IRQ_WINDOW_EXIT))
711	{
712	LogFlow(("Enable irq window exit!\n"));
713	pVCpu->hwaccm.s.vmx.proc_ctls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_IRQ_WINDOW_EXIT;
714	rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
715	AssertRC(rc);
716	}
717	/* else nothing to do but wait */
718	}
719	else
720	if (!VM_FF_ISSET(pVM, VM_FF_INHIBIT_INTERRUPTS))
721	{
722	uint8_t u8Interrupt;
723
724	rc = PDMGetInterrupt(pVM, &u8Interrupt);
725	Log(("Dispatch interrupt: u8Interrupt=%x (%d) rc=%Rrc cs:rip=%04X:%RGv\n", u8Interrupt, u8Interrupt, rc, pCtx->cs, (RTGCPTR)pCtx->rip));
726	if (RT_SUCCESS(rc))
727	{
728	rc = TRPMAssertTrap(pVM, u8Interrupt, TRPM_HARDWARE_INT);
729	AssertRC(rc);
730	}
731	else
732	{
733	/* Can only happen in rare cases where a pending interrupt is cleared behind our back */
734	Assert(!VM_FF_ISPENDING(pVM, (VM_FF_INTERRUPT_APIC\|VM_FF_INTERRUPT_PIC)));
735	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatSwitchGuestIrq);
736	/* Just continue */
737	}
738	}
739	else
740	Log(("Pending interrupt blocked at %RGv by VM_FF_INHIBIT_INTERRUPTS!!\n", (RTGCPTR)pCtx->rip));
741	}
742
743	#ifdef VBOX_STRICT
744	if (TRPMHasTrap(pVM))
745	{
746	uint8_t u8Vector;
747	rc = TRPMQueryTrapAll(pVM, &u8Vector, 0, 0, 0);
748	AssertRC(rc);
749	}
750	#endif
751
752	if ( pCtx->eflags.u32 & X86_EFL_IF
753	&& (!VM_FF_ISSET(pVM, VM_FF_INHIBIT_INTERRUPTS))
754	&& TRPMHasTrap(pVM)
755	)
756	{
757	uint8_t u8Vector;
758	int rc;
759	TRPMEVENT enmType;
760	RTGCUINTPTR intInfo;
761	RTGCUINT errCode;
762
763	/* If a new event is pending, then dispatch it now. */
764	rc = TRPMQueryTrapAll(pVM, &u8Vector, &enmType, &errCode, 0);
765	AssertRC(rc);
766	Assert(pCtx->eflags.Bits.u1IF == 1 \|\| enmType == TRPM_TRAP);
767	Assert(enmType != TRPM_SOFTWARE_INT);
768
769	/* Clear the pending trap. */
770	rc = TRPMResetTrap(pVM);
771	AssertRC(rc);
772
773	intInfo = u8Vector;
774	intInfo \|= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
775
776	if (enmType == TRPM_TRAP)
777	{
778	switch (u8Vector) {
779	case 8:
780	case 10:
781	case 11:
782	case 12:
783	case 13:
784	case 14:
785	case 17:
786	/* Valid error codes. */
787	intInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
788	break;
789	default:
790	break;
791	}
792	if (u8Vector == X86_XCPT_BP \|\| u8Vector == X86_XCPT_OF)
793	intInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SWEXCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
794	else
795	intInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HWEXCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
796	}
797	else
798	intInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
799
800	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatIntInject);
801	rc = VMXR0InjectEvent(pVM, pVCpu, pCtx, intInfo, 0, errCode);
802	AssertRC(rc);
803	} /* if (interrupts can be dispatched) */
804
805	return VINF_SUCCESS;
806	}
807
808	/**
809	* Save the host state
810	*
811	* @returns VBox status code.
812	* @param pVM The VM to operate on.
813	* @param pVCpu The VMCPU to operate on.
814	*/
815	VMMR0DECL(int) VMXR0SaveHostState(PVM pVM, PVMCPU pVCpu)
816	{
817	int rc = VINF_SUCCESS;
818
819	/*
820	* Host CPU Context
821	*/
822	if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_HOST_CONTEXT)
823	{
824	RTIDTR idtr;
825	RTGDTR gdtr;
826	RTSEL SelTR;
827	PX86DESCHC pDesc;
828	uintptr_t trBase;
829	RTSEL cs;
830	RTSEL ss;
831	uint64_t cr3;
832
833	/* Control registers */
834	rc = VMXWriteVMCS(VMX_VMCS_HOST_CR0, ASMGetCR0());
835	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
836	if (VMX_IS_64BIT_HOST_MODE())
837	{
838	cr3 = hwaccmR0Get64bitCR3();
839	rc \|= VMXWriteVMCS64(VMX_VMCS_HOST_CR3, cr3);
840	}
841	else
842	#endif
843	{
844	cr3 = ASMGetCR3();
845	rc \|= VMXWriteVMCS(VMX_VMCS_HOST_CR3, cr3);
846	}
847	rc \|= VMXWriteVMCS(VMX_VMCS_HOST_CR4, ASMGetCR4());
848	AssertRC(rc);
849	Log2(("VMX_VMCS_HOST_CR0 %08x\n", ASMGetCR0()));
850	Log2(("VMX_VMCS_HOST_CR3 %08RX64\n", cr3));
851	Log2(("VMX_VMCS_HOST_CR4 %08x\n", ASMGetCR4()));
852
853	/* Selector registers. */
854	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
855	if (VMX_IS_64BIT_HOST_MODE())
856	{
857	cs = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelCS;
858	ss = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelSS;
859	}
860	else
861	{
862	/* sysenter loads LDT cs & ss, VMX doesn't like this. Load the GDT ones (safe). */
863	cs = (RTSEL)(uintptr_t)&SUPR0AbsKernelCS;
864	ss = (RTSEL)(uintptr_t)&SUPR0AbsKernelSS;
865	}
866	#else
867	cs = ASMGetCS();
868	ss = ASMGetSS();
869	#endif
870	Assert(!(cs & X86_SEL_LDT)); Assert((cs & X86_SEL_RPL) == 0);
871	Assert(!(ss & X86_SEL_LDT)); Assert((ss & X86_SEL_RPL) == 0);
872	rc = VMXWriteVMCS(VMX_VMCS16_HOST_FIELD_CS, cs);
873	/* Note: VMX is (again) very picky about the RPL of the selectors here; we'll restore them manually. */
874	rc \|= VMXWriteVMCS(VMX_VMCS16_HOST_FIELD_DS, 0);
875	rc \|= VMXWriteVMCS(VMX_VMCS16_HOST_FIELD_ES, 0);
876	#if HC_ARCH_BITS == 32
877	if (!VMX_IS_64BIT_HOST_MODE())
878	{
879	rc \|= VMXWriteVMCS(VMX_VMCS16_HOST_FIELD_FS, 0);
880	rc \|= VMXWriteVMCS(VMX_VMCS16_HOST_FIELD_GS, 0);
881	}
882	#endif
883	rc \|= VMXWriteVMCS(VMX_VMCS16_HOST_FIELD_SS, ss);
884	SelTR = ASMGetTR();
885	rc \|= VMXWriteVMCS(VMX_VMCS16_HOST_FIELD_TR, SelTR);
886	AssertRC(rc);
887	Log2(("VMX_VMCS_HOST_FIELD_CS %08x (%08x)\n", cs, ASMGetSS()));
888	Log2(("VMX_VMCS_HOST_FIELD_DS 00000000 (%08x)\n", ASMGetDS()));
889	Log2(("VMX_VMCS_HOST_FIELD_ES 00000000 (%08x)\n", ASMGetES()));
890	Log2(("VMX_VMCS_HOST_FIELD_FS 00000000 (%08x)\n", ASMGetFS()));
891	Log2(("VMX_VMCS_HOST_FIELD_GS 00000000 (%08x)\n", ASMGetGS()));
892	Log2(("VMX_VMCS_HOST_FIELD_SS %08x (%08x)\n", ss, ASMGetSS()));
893	Log2(("VMX_VMCS_HOST_FIELD_TR %08x\n", ASMGetTR()));
894
895	/* GDTR & IDTR */
896	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
897	if (VMX_IS_64BIT_HOST_MODE())
898	{
899	X86XDTR64 gdtr64, idtr64;
900	hwaccmR0Get64bitGDTRandIDTR(&gdtr64, &idtr64);
901	rc = VMXWriteVMCS64(VMX_VMCS_HOST_GDTR_BASE, gdtr64.uAddr);
902	rc \|= VMXWriteVMCS64(VMX_VMCS_HOST_IDTR_BASE, gdtr64.uAddr);
903	AssertRC(rc);
904	Log2(("VMX_VMCS_HOST_GDTR_BASE %RX64\n", gdtr64.uAddr));
905	Log2(("VMX_VMCS_HOST_IDTR_BASE %RX64\n", idtr64.uAddr));
906	gdtr.cbGdt = gdtr64.cb;
907	gdtr.pGdt = (uintptr_t)gdtr64.uAddr;
908	}
909	else
910	#endif
911	{
912	ASMGetGDTR(&gdtr);
913	rc = VMXWriteVMCS(VMX_VMCS_HOST_GDTR_BASE, gdtr.pGdt);
914	ASMGetIDTR(&idtr);
915	rc \|= VMXWriteVMCS(VMX_VMCS_HOST_IDTR_BASE, idtr.pIdt);
916	AssertRC(rc);
917	Log2(("VMX_VMCS_HOST_GDTR_BASE %RHv\n", gdtr.pGdt));
918	Log2(("VMX_VMCS_HOST_IDTR_BASE %RHv\n", idtr.pIdt));
919	}
920
921
922	/* Save the base address of the TR selector. */
923	if (SelTR > gdtr.cbGdt)
924	{
925	AssertMsgFailed(("Invalid TR selector %x. GDTR.cbGdt=%x\n", SelTR, gdtr.cbGdt));
926	return VERR_VMX_INVALID_HOST_STATE;
927	}
928
929	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
930	if (VMX_IS_64BIT_HOST_MODE())
931	{
932	pDesc = &((PX86DESCHC)gdtr.pGdt)[SelTR >> X86_SEL_SHIFT_HC]; /// ????
933	uint64_t trBase64 = X86DESC64_BASE(*(PX86DESC64)pDesc);
934	rc = VMXWriteVMCS64(VMX_VMCS_HOST_TR_BASE, trBase64);
935	Log2(("VMX_VMCS_HOST_TR_BASE %RX64\n", trBase64));
936	AssertRC(rc);
937	}
938	else
939	#endif
940	{
941	pDesc = &((PX86DESCHC)gdtr.pGdt)[SelTR >> X86_SEL_SHIFT_HC];
942	#if HC_ARCH_BITS == 64
943	trBase = X86DESC64_BASE(*pDesc);
944	#else
945	trBase = X86DESC_BASE(*pDesc);
946	#endif
947	rc = VMXWriteVMCS(VMX_VMCS_HOST_TR_BASE, trBase);
948	AssertRC(rc);
949	Log2(("VMX_VMCS_HOST_TR_BASE %RHv\n", trBase));
950	}
951
952	/* FS and GS base. */
953	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
954	if (VMX_IS_64BIT_HOST_MODE())
955	{
956	Log2(("MSR_K8_FS_BASE = %RX64\n", ASMRdMsr(MSR_K8_FS_BASE)));
957	Log2(("MSR_K8_GS_BASE = %RX64\n", ASMRdMsr(MSR_K8_GS_BASE)));
958	rc = VMXWriteVMCS64(VMX_VMCS_HOST_FS_BASE, ASMRdMsr(MSR_K8_FS_BASE));
959	rc \|= VMXWriteVMCS64(VMX_VMCS_HOST_GS_BASE, ASMRdMsr(MSR_K8_GS_BASE));
960	}
961	#endif
962	AssertRC(rc);
963
964	/* Sysenter MSRs. */
965	/** @todo expensive!! */
966	rc = VMXWriteVMCS(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
967	Log2(("VMX_VMCS_HOST_SYSENTER_CS %08x\n", ASMRdMsr_Low(MSR_IA32_SYSENTER_CS)));
968	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
969	if (VMX_IS_64BIT_HOST_MODE())
970	{
971	Log2(("VMX_VMCS_HOST_SYSENTER_EIP %RX64\n", ASMRdMsr(MSR_IA32_SYSENTER_EIP)));
972	Log2(("VMX_VMCS_HOST_SYSENTER_ESP %RX64\n", ASMRdMsr(MSR_IA32_SYSENTER_ESP)));
973	rc \|= VMXWriteVMCS64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
974	rc \|= VMXWriteVMCS64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
975	}
976	else
977	{
978	rc \|= VMXWriteVMCS(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
979	rc \|= VMXWriteVMCS(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
980	Log2(("VMX_VMCS_HOST_SYSENTER_EIP %RX32\n", ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP)));
981	Log2(("VMX_VMCS_HOST_SYSENTER_ESP %RX32\n", ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP)));
982	}
983	#elif HC_ARCH_BITS == 32
984	rc \|= VMXWriteVMCS(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
985	rc \|= VMXWriteVMCS(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
986	Log2(("VMX_VMCS_HOST_SYSENTER_EIP %RX32\n", ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP)));
987	Log2(("VMX_VMCS_HOST_SYSENTER_ESP %RX32\n", ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP)));
988	#else
989	Log2(("VMX_VMCS_HOST_SYSENTER_EIP %RX64\n", ASMRdMsr(MSR_IA32_SYSENTER_EIP)));
990	Log2(("VMX_VMCS_HOST_SYSENTER_ESP %RX64\n", ASMRdMsr(MSR_IA32_SYSENTER_ESP)));
991	rc \|= VMXWriteVMCS64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
992	rc \|= VMXWriteVMCS64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
993	#endif
994	AssertRC(rc);
995
996	pVCpu->hwaccm.s.fContextUseFlags &= ~HWACCM_CHANGED_HOST_CONTEXT;
997	}
998	return rc;
999	}
1000
1001	/**
1002	* Prefetch the 4 PDPT pointers (PAE and nested paging only)
1003	*
1004	* @param pVM The VM to operate on.
1005	* @param pVCpu The VMCPU to operate on.
1006	* @param pCtx Guest context
1007	*/
1008	static void vmxR0PrefetchPAEPdptrs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
1009	{
1010	if (CPUMIsGuestInPAEModeEx(pCtx))
1011	{
1012	X86PDPE Pdpe;
1013
1014	for (unsigned i=0;i<4;i++)
1015	{
1016	Pdpe = PGMGstGetPaePDPtr(pVM, i);
1017	int rc = VMXWriteVMCS64(VMX_VMCS_GUEST_PDPTR0_FULL + i*2, Pdpe.u);
1018	AssertRC(rc);
1019	}
1020	}
1021	}
1022
1023	/**
1024	* Update the exception bitmap according to the current CPU state
1025	*
1026	* @param pVM The VM to operate on.
1027	* @param pVCpu The VMCPU to operate on.
1028	* @param pCtx Guest context
1029	*/
1030	static void vmxR0UpdateExceptionBitmap(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
1031	{
1032	uint32_t u32TrapMask;
1033	Assert(pCtx);
1034
1035	u32TrapMask = HWACCM_VMX_TRAP_MASK;
1036	#ifndef DEBUG
1037	if (pVM->hwaccm.s.fNestedPaging)
1038	u32TrapMask &= ~RT_BIT(X86_XCPT_PF); /* no longer need to intercept #PF. */
1039	#endif
1040
1041	/* Also catch floating point exceptions as we need to report them to the guest in a different way. */
1042	if ( CPUMIsGuestFPUStateActive(pVCpu) == true
1043	&& !(pCtx->cr0 & X86_CR0_NE)
1044	&& !pVCpu->hwaccm.s.fFPUOldStyleOverride)
1045	{
1046	u32TrapMask \|= RT_BIT(X86_XCPT_MF);
1047	pVCpu->hwaccm.s.fFPUOldStyleOverride = true;
1048	}
1049
1050	#ifdef DEBUG
1051	/* Intercept X86_XCPT_DB if stepping is enabled */
1052	if (DBGFIsStepping(pVM))
1053	u32TrapMask \|= RT_BIT(X86_XCPT_DB);
1054	#endif
1055
1056	#ifdef VBOX_STRICT
1057	Assert(u32TrapMask & RT_BIT(X86_XCPT_GP));
1058	#endif
1059
1060	# ifdef HWACCM_VMX_EMULATE_REALMODE
1061	/* Intercept all exceptions in real mode as none of them can be injected directly (#GP otherwise). */
1062	if (CPUMIsGuestInRealModeEx(pCtx))
1063	u32TrapMask \|= HWACCM_VMX_TRAP_MASK_REALMODE;
1064	# endif /* HWACCM_VMX_EMULATE_REALMODE */
1065
1066	int rc = VMXWriteVMCS(VMX_VMCS_CTRL_EXCEPTION_BITMAP, u32TrapMask);
1067	AssertRC(rc);
1068	}
1069
1070	/**
1071	* Loads the guest state
1072	*
1073	* NOTE: Don't do anything here that can cause a jump back to ring 3!!!!!
1074	*
1075	* @returns VBox status code.
1076	* @param pVM The VM to operate on.
1077	* @param pVCpu The VMCPU to operate on.
1078	* @param pCtx Guest context
1079	*/
1080	VMMR0DECL(int) VMXR0LoadGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
1081	{
1082	int rc = VINF_SUCCESS;
1083	RTGCUINTPTR val;
1084	X86EFLAGS eflags;
1085
1086	/* VMX_VMCS_CTRL_ENTRY_CONTROLS
1087	* Set required bits to one and zero according to the MSR capabilities.
1088	*/
1089	val = pVM->hwaccm.s.vmx.msr.vmx_entry.n.disallowed0;
1090	/* Load guest debug controls (dr7 & IA32_DEBUGCTL_MSR) (forced to 1 on the 'first' VT-x capable CPUs; this actually includes the newest Nehalem CPUs) */
1091	val \|= VMX_VMCS_CTRL_ENTRY_CONTROLS_LOAD_DEBUG;
1092	/* Required for the EFER write below, not supported on all CPUs. / /* @todo Sander, check this. */
1093	val \|= VMX_VMCS_CTRL_ENTRY_CONTROLS_LOAD_GUEST_EFER_MSR;
1094
1095	/* 64 bits guest mode? */
1096	if (pCtx->msrEFER & MSR_K6_EFER_LMA)
1097	val \|= VMX_VMCS_CTRL_ENTRY_CONTROLS_IA64_MODE;
1098	/* else Must be zero when AMD64 is not available. */
1099
1100	/* Mask away the bits that the CPU doesn't support */
1101	val &= pVM->hwaccm.s.vmx.msr.vmx_entry.n.allowed1;
1102	rc = VMXWriteVMCS(VMX_VMCS_CTRL_ENTRY_CONTROLS, val);
1103	AssertRC(rc);
1104
1105	/* VMX_VMCS_CTRL_EXIT_CONTROLS
1106	* Set required bits to one and zero according to the MSR capabilities.
1107	*/
1108	val = pVM->hwaccm.s.vmx.msr.vmx_exit.n.disallowed0;
1109
1110	/* Save debug controls (dr7 & IA32_DEBUGCTL_MSR) (forced to 1 on the 'first' VT-x capable CPUs; this actually includes the newest Nehalem CPUs) */
1111	val \|= VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_DEBUG;
1112	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
1113	if (VMX_IS_64BIT_HOST_MODE())
1114	val \|= VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_AMD64;
1115	/* else: Must be zero when AMD64 is not available. */
1116	#elif HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
1117	if (pCtx->msrEFER & MSR_K6_EFER_LMA)
1118	val \|= VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_AMD64; /* our switcher goes to long mode */
1119	else
1120	val &= ~VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_AMD64;
1121	#endif
1122	val &= pVM->hwaccm.s.vmx.msr.vmx_exit.n.allowed1;
1123	/* Don't acknowledge external interrupts on VM-exit. */
1124	rc = VMXWriteVMCS(VMX_VMCS_CTRL_EXIT_CONTROLS, val);
1125	AssertRC(rc);
1126
1127	/* Guest CPU context: ES, CS, SS, DS, FS, GS. */
1128	if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_SEGMENT_REGS)
1129	{
1130	#ifdef HWACCM_VMX_EMULATE_REALMODE
1131	PGMMODE enmGuestMode = PGMGetGuestMode(pVM);
1132	if (pVCpu->hwaccm.s.vmx.enmCurrGuestMode != enmGuestMode)
1133	{
1134	/* Correct weird requirements for switching to protected mode. */
1135	if ( pVCpu->hwaccm.s.vmx.enmCurrGuestMode == PGMMODE_REAL
1136	&& enmGuestMode >= PGMMODE_PROTECTED)
1137	{
1138	/* DPL of all hidden selector registers must match the current CPL (0). */
1139	pCtx->csHid.Attr.n.u2Dpl = 0;
1140	pCtx->csHid.Attr.n.u4Type = X86_SEL_TYPE_CODE \| X86_SEL_TYPE_RW_ACC;
1141
1142	pCtx->dsHid.Attr.n.u2Dpl = 0;
1143	pCtx->esHid.Attr.n.u2Dpl = 0;
1144	pCtx->fsHid.Attr.n.u2Dpl = 0;
1145	pCtx->gsHid.Attr.n.u2Dpl = 0;
1146	pCtx->ssHid.Attr.n.u2Dpl = 0;
1147	}
1148	else
1149	/* Switching from protected mode to real mode. */
1150	if ( pVCpu->hwaccm.s.vmx.enmCurrGuestMode >= PGMMODE_PROTECTED
1151	&& enmGuestMode == PGMMODE_REAL)
1152	{
1153	/* The limit must also be adjusted. */
1154	pCtx->csHid.u32Limit &= 0xffff;
1155	pCtx->dsHid.u32Limit &= 0xffff;
1156	pCtx->esHid.u32Limit &= 0xffff;
1157	pCtx->fsHid.u32Limit &= 0xffff;
1158	pCtx->gsHid.u32Limit &= 0xffff;
1159	pCtx->ssHid.u32Limit &= 0xffff;
1160
1161	Assert(pCtx->csHid.u64Base <= 0xfffff);
1162	Assert(pCtx->dsHid.u64Base <= 0xfffff);
1163	Assert(pCtx->esHid.u64Base <= 0xfffff);
1164	Assert(pCtx->fsHid.u64Base <= 0xfffff);
1165	Assert(pCtx->gsHid.u64Base <= 0xfffff);
1166	}
1167	pVCpu->hwaccm.s.vmx.enmCurrGuestMode = enmGuestMode;
1168	}
1169	else
1170	/* VT-x will fail with a guest invalid state otherwise... (CPU state after a reset) */
1171	if ( CPUMIsGuestInRealModeEx(pCtx)
1172	&& pCtx->csHid.u64Base == 0xffff0000)
1173	{
1174	pCtx->csHid.u64Base = 0xf0000;
1175	pCtx->cs = 0xf000;
1176	}
1177	#endif /* HWACCM_VMX_EMULATE_REALMODE */
1178
1179	VMX_WRITE_SELREG(ES, es);
1180	AssertRC(rc);
1181
1182	VMX_WRITE_SELREG(CS, cs);
1183	AssertRC(rc);
1184
1185	VMX_WRITE_SELREG(SS, ss);
1186	AssertRC(rc);
1187
1188	VMX_WRITE_SELREG(DS, ds);
1189	AssertRC(rc);
1190
1191	/* The base values in the hidden fs & gs registers are not in sync with the msrs; they are cut to 32 bits. */
1192	VMX_WRITE_SELREG(FS, fs);
1193	AssertRC(rc);
1194
1195	VMX_WRITE_SELREG(GS, gs);
1196	AssertRC(rc);
1197	}
1198
1199	/* Guest CPU context: LDTR. */
1200	if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_LDTR)
1201	{
1202	if (pCtx->ldtr == 0)
1203	{
1204	rc = VMXWriteVMCS(VMX_VMCS16_GUEST_FIELD_LDTR, 0);
1205	rc \|= VMXWriteVMCS(VMX_VMCS32_GUEST_LDTR_LIMIT, 0);
1206	rc \|= VMXWriteVMCS64(VMX_VMCS64_GUEST_LDTR_BASE, 0);
1207	/* Note: vmlaunch will fail with 0 or just 0x02. No idea why. */
1208	rc \|= VMXWriteVMCS(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, 0x82 /* present, LDT */);
1209	}
1210	else
1211	{
1212	rc = VMXWriteVMCS(VMX_VMCS16_GUEST_FIELD_LDTR, pCtx->ldtr);
1213	rc \|= VMXWriteVMCS(VMX_VMCS32_GUEST_LDTR_LIMIT, pCtx->ldtrHid.u32Limit);
1214	rc \|= VMXWriteVMCS64(VMX_VMCS64_GUEST_LDTR_BASE, pCtx->ldtrHid.u64Base);
1215	rc \|= VMXWriteVMCS(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, pCtx->ldtrHid.Attr.u);
1216	}
1217	AssertRC(rc);
1218	}
1219	/* Guest CPU context: TR. */
1220	if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_TR)
1221	{
1222	#ifdef HWACCM_VMX_EMULATE_REALMODE
1223	/* Real mode emulation using v86 mode with CR4.VME (interrupt redirection using the int bitmap in the TSS) */
1224	if (CPUMIsGuestInRealModeEx(pCtx))
1225	{
1226	RTGCPHYS GCPhys;
1227
1228	/* We convert it here every time as pci regions could be reconfigured. */
1229	rc = PDMVMMDevHeapR3ToGCPhys(pVM, pVM->hwaccm.s.vmx.pRealModeTSS, &GCPhys);
1230	AssertRC(rc);
1231
1232	rc = VMXWriteVMCS(VMX_VMCS16_GUEST_FIELD_TR, 0);
1233	rc \|= VMXWriteVMCS(VMX_VMCS32_GUEST_TR_LIMIT, HWACCM_VTX_TSS_SIZE);
1234	rc \|= VMXWriteVMCS64(VMX_VMCS64_GUEST_TR_BASE, GCPhys /* phys = virt in this mode */);
1235
1236	X86DESCATTR attr;
1237
1238	attr.u = 0;
1239	attr.n.u1Present = 1;
1240	attr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
1241	val = attr.u;
1242	}
1243	else
1244	#endif /* HWACCM_VMX_EMULATE_REALMODE */
1245	{
1246	rc = VMXWriteVMCS(VMX_VMCS16_GUEST_FIELD_TR, pCtx->tr);
1247	rc \|= VMXWriteVMCS(VMX_VMCS32_GUEST_TR_LIMIT, pCtx->trHid.u32Limit);
1248	rc \|= VMXWriteVMCS64(VMX_VMCS64_GUEST_TR_BASE, pCtx->trHid.u64Base);
1249
1250	val = pCtx->trHid.Attr.u;
1251
1252	/* The TSS selector must be busy. */
1253	if ((val & 0xF) == X86_SEL_TYPE_SYS_286_TSS_AVAIL)
1254	val = (val & ~0xF) \| X86_SEL_TYPE_SYS_286_TSS_BUSY;
1255	else
1256	/* Default even if no TR selector has been set (otherwise vmlaunch will fail!) */
1257	val = (val & ~0xF) \| X86_SEL_TYPE_SYS_386_TSS_BUSY;
1258
1259	}
1260	rc \|= VMXWriteVMCS(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, val);
1261	AssertRC(rc);
1262	}
1263	/* Guest CPU context: GDTR. */
1264	if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_GDTR)
1265	{
1266	rc = VMXWriteVMCS(VMX_VMCS32_GUEST_GDTR_LIMIT, pCtx->gdtr.cbGdt);
1267	rc \|= VMXWriteVMCS64(VMX_VMCS64_GUEST_GDTR_BASE, pCtx->gdtr.pGdt);
1268	AssertRC(rc);
1269	}
1270	/* Guest CPU context: IDTR. */
1271	if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_IDTR)
1272	{
1273	rc = VMXWriteVMCS(VMX_VMCS32_GUEST_IDTR_LIMIT, pCtx->idtr.cbIdt);
1274	rc \|= VMXWriteVMCS64(VMX_VMCS64_GUEST_IDTR_BASE, pCtx->idtr.pIdt);
1275	AssertRC(rc);
1276	}
1277
1278	/*
1279	* Sysenter MSRs (unconditional)
1280	*/
1281	rc = VMXWriteVMCS(VMX_VMCS32_GUEST_SYSENTER_CS, pCtx->SysEnter.cs);
1282	rc \|= VMXWriteVMCS64(VMX_VMCS64_GUEST_SYSENTER_EIP, pCtx->SysEnter.eip);
1283	rc \|= VMXWriteVMCS64(VMX_VMCS64_GUEST_SYSENTER_ESP, pCtx->SysEnter.esp);
1284	AssertRC(rc);
1285
1286	/* Control registers */
1287	if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_CR0)
1288	{
1289	val = pCtx->cr0;
1290	rc = VMXWriteVMCS(VMX_VMCS_CTRL_CR0_READ_SHADOW, val);
1291	Log2(("Guest CR0-shadow %08x\n", val));
1292	if (CPUMIsGuestFPUStateActive(pVCpu) == false)
1293	{
1294	/* Always use #NM exceptions to load the FPU/XMM state on demand. */
1295	val \|= X86_CR0_TS \| X86_CR0_ET \| X86_CR0_NE \| X86_CR0_MP;
1296	}
1297	else
1298	{
1299	/** @todo check if we support the old style mess correctly. */
1300	if (!(val & X86_CR0_NE))
1301	Log(("Forcing X86_CR0_NE!!!\n"));
1302
1303	val \|= X86_CR0_NE; /* always turn on the native mechanism to report FPU errors (old style uses interrupts) */
1304	}
1305	/* Note: protected mode & paging are always enabled; we use them for emulating real and protected mode without paging too. */
1306	val \|= X86_CR0_PE \| X86_CR0_PG;
1307	if (pVM->hwaccm.s.fNestedPaging)
1308	{
1309	if (CPUMIsGuestInPagedProtectedModeEx(pCtx))
1310	{
1311	/* Disable cr3 read/write monitoring as we don't need it for EPT. */
1312	pVCpu->hwaccm.s.vmx.proc_ctls &= ~( VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
1313	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT);
1314	}
1315	else
1316	{
1317	/* Reenable cr3 read/write monitoring as our identity mapped page table is active. */
1318	pVCpu->hwaccm.s.vmx.proc_ctls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
1319	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT;
1320	}
1321	rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
1322	AssertRC(rc);
1323	}
1324	else
1325	{
1326	/* Note: We must also set this as we rely on protecting various pages for which supervisor writes must be caught. */
1327	val \|= X86_CR0_WP;
1328	}
1329
1330	/* Always enable caching. */
1331	val &= ~(X86_CR0_CD\|X86_CR0_NW);
1332
1333	rc \|= VMXWriteVMCS64(VMX_VMCS64_GUEST_CR0, val);
1334	Log2(("Guest CR0 %08x\n", val));
1335	/* CR0 flags owned by the host; if the guests attempts to change them, then
1336	* the VM will exit.
1337	*/
1338	val = X86_CR0_PE /* Must monitor this bit (assumptions are made for real mode emulation) */
1339	\| X86_CR0_WP /* Must monitor this bit (it must always be enabled). */
1340	\| X86_CR0_PG /* Must monitor this bit (assumptions are made for real mode & protected mode without paging emulation) */
1341	\| X86_CR0_TS
1342	\| X86_CR0_ET /* Bit not restored during VM-exit! */
1343	\| X86_CR0_CD /* Bit not restored during VM-exit! */
1344	\| X86_CR0_NW /* Bit not restored during VM-exit! */
1345	\| X86_CR0_NE
1346	\| X86_CR0_MP;
1347	pVCpu->hwaccm.s.vmx.cr0_mask = val;
1348
1349	rc \|= VMXWriteVMCS(VMX_VMCS_CTRL_CR0_MASK, val);
1350	Log2(("Guest CR0-mask %08x\n", val));
1351	AssertRC(rc);
1352	}
1353	if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_CR4)
1354	{
1355	/* CR4 */
1356	rc = VMXWriteVMCS(VMX_VMCS_CTRL_CR4_READ_SHADOW, pCtx->cr4);
1357	Log2(("Guest CR4-shadow %08x\n", pCtx->cr4));
1358	/* Set the required bits in cr4 too (currently X86_CR4_VMXE). */
1359	val = pCtx->cr4 \| (uint32_t)pVM->hwaccm.s.vmx.msr.vmx_cr4_fixed0;
1360
1361	if (!pVM->hwaccm.s.fNestedPaging)
1362	{
1363	switch(pVCpu->hwaccm.s.enmShadowMode)
1364	{
1365	case PGMMODE_REAL: /* Real mode -> emulated using v86 mode */
1366	case PGMMODE_PROTECTED: /* Protected mode, no paging -> emulated using identity mapping. */
1367	case PGMMODE_32_BIT: /* 32-bit paging. */
1368	break;
1369
1370	case PGMMODE_PAE: /* PAE paging. */
1371	case PGMMODE_PAE_NX: /* PAE paging with NX enabled. */
1372	/** @todo use normal 32 bits paging */
1373	val \|= X86_CR4_PAE;
1374	break;
1375
1376	case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
1377	case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
1378	#ifdef VBOX_ENABLE_64_BITS_GUESTS
1379	break;
1380	#else
1381	AssertFailed();
1382	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
1383	#endif
1384	default: /* shut up gcc */
1385	AssertFailed();
1386	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
1387	}
1388	}
1389	else
1390	if (!CPUMIsGuestInPagedProtectedModeEx(pCtx))
1391	{
1392	/* We use 4 MB pages in our identity mapping page table for real and protected mode without paging. */
1393	val \|= X86_CR4_PSE;
1394	/* Our identity mapping is a 32 bits page directory. */
1395	val &= ~X86_CR4_PAE;
1396	}
1397
1398	#ifdef HWACCM_VMX_EMULATE_REALMODE
1399	/* Real mode emulation using v86 mode with CR4.VME (interrupt redirection using the int bitmap in the TSS) */
1400	if (CPUMIsGuestInRealModeEx(pCtx))
1401	val \|= X86_CR4_VME;
1402	#endif /* HWACCM_VMX_EMULATE_REALMODE */
1403
1404	rc \|= VMXWriteVMCS64(VMX_VMCS64_GUEST_CR4, val);
1405	Log2(("Guest CR4 %08x\n", val));
1406	/* CR4 flags owned by the host; if the guests attempts to change them, then
1407	* the VM will exit.
1408	*/
1409	val = 0
1410	#ifdef HWACCM_VMX_EMULATE_REALMODE
1411	\| X86_CR4_VME
1412	#endif
1413	\| X86_CR4_PAE
1414	\| X86_CR4_PGE
1415	\| X86_CR4_PSE
1416	\| X86_CR4_VMXE;
1417	pVCpu->hwaccm.s.vmx.cr4_mask = val;
1418
1419	rc \|= VMXWriteVMCS(VMX_VMCS_CTRL_CR4_MASK, val);
1420	Log2(("Guest CR4-mask %08x\n", val));
1421	AssertRC(rc);
1422	}
1423
1424	if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_CR3)
1425	{
1426	if (pVM->hwaccm.s.fNestedPaging)
1427	{
1428	AssertMsg( PGMGetEPTCR3(pVM) == PGMGetHyperCR3(pVM)
1429	\|\| VM_FF_ISPENDING(pVM, VM_FF_PGM_SYNC_CR3 \| VM_FF_PGM_SYNC_CR3_NON_GLOBAL),
1430	("%RHp vs %RHp\n", PGMGetEPTCR3(pVM), PGMGetHyperCR3(pVM)));
1431	pVCpu->hwaccm.s.vmx.GCPhysEPTP = PGMGetEPTCR3(pVM);
1432
1433	Assert(!(pVCpu->hwaccm.s.vmx.GCPhysEPTP & 0xfff));
1434	/** @todo Check the IA32_VMX_EPT_VPID_CAP MSR for other supported memory types. */
1435	pVCpu->hwaccm.s.vmx.GCPhysEPTP \|= VMX_EPT_MEMTYPE_WB
1436	\| (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
1437
1438	rc = VMXWriteVMCS64(VMX_VMCS_CTRL_EPTP_FULL, pVCpu->hwaccm.s.vmx.GCPhysEPTP);
1439	AssertRC(rc);
1440
1441	if (!CPUMIsGuestInPagedProtectedModeEx(pCtx))
1442	{
1443	RTGCPHYS GCPhys;
1444
1445	/* We convert it here every time as pci regions could be reconfigured. */
1446	rc = PDMVMMDevHeapR3ToGCPhys(pVM, pVM->hwaccm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
1447	AssertRC(rc);
1448
1449	/* We use our identity mapping page table here as we need to map guest virtual to guest physical addresses; EPT will
1450	* take care of the translation to host physical addresses.
1451	*/
1452	val = GCPhys;
1453	}
1454	else
1455	{
1456	/* Save the real guest CR3 in VMX_VMCS_GUEST_CR3 */
1457	val = pCtx->cr3;
1458	/* Prefetch the four PDPT entries in PAE mode. */
1459	vmxR0PrefetchPAEPdptrs(pVM, pVCpu, pCtx);
1460	}
1461	}
1462	else
1463	{
1464	val = PGMGetHyperCR3(pVM);
1465	Assert(val \|\| VM_FF_ISPENDING(pVM, VM_FF_PGM_SYNC_CR3 \| VM_FF_PGM_SYNC_CR3_NON_GLOBAL));
1466	}
1467
1468	/* Save our shadow CR3 register. */
1469	rc = VMXWriteVMCS64(VMX_VMCS64_GUEST_CR3, val);
1470	AssertRC(rc);
1471	}
1472
1473	/* Debug registers. */
1474	if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_DEBUG)
1475	{
1476	pCtx->dr[6] \|= X86_DR6_INIT_VAL; /* set all reserved bits to 1. */
1477	pCtx->dr[6] &= ~RT_BIT(12); /* must be zero. */
1478
1479	pCtx->dr[7] &= 0xffffffff; /* upper 32 bits reserved */
1480	pCtx->dr[7] &= ~(RT_BIT(11) \| RT_BIT(12) \| RT_BIT(14) \| RT_BIT(15)); /* must be zero */
1481	pCtx->dr[7] \|= 0x400; /* must be one */
1482
1483	/* Resync DR7 */
1484	rc = VMXWriteVMCS64(VMX_VMCS64_GUEST_DR7, pCtx->dr[7]);
1485	AssertRC(rc);
1486
1487	/* Sync the debug state now if any breakpoint is armed. */
1488	if ( (pCtx->dr[7] & (X86_DR7_ENABLED_MASK\|X86_DR7_GD))
1489	&& !CPUMIsGuestDebugStateActive(pVM)
1490	&& !DBGFIsStepping(pVM))
1491	{
1492	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatDRxArmed);
1493
1494	/* Disable drx move intercepts. */
1495	pVCpu->hwaccm.s.vmx.proc_ctls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT;
1496	rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
1497	AssertRC(rc);
1498
1499	/* Save the host and load the guest debug state. */
1500	rc = CPUMR0LoadGuestDebugState(pVM, pVCpu, pCtx, true /* include DR6 */);
1501	AssertRC(rc);
1502	}
1503
1504	/* IA32_DEBUGCTL MSR. */
1505	rc = VMXWriteVMCS64(VMX_VMCS_GUEST_DEBUGCTL_FULL, 0);
1506	AssertRC(rc);
1507
1508	/** @todo do we really ever need this? */
1509	rc \|= VMXWriteVMCS(VMX_VMCS_GUEST_DEBUG_EXCEPTIONS, 0);
1510	AssertRC(rc);
1511	}
1512
1513	/* EIP, ESP and EFLAGS */
1514	rc = VMXWriteVMCS64(VMX_VMCS64_GUEST_RIP, pCtx->rip);
1515	rc \|= VMXWriteVMCS64(VMX_VMCS64_GUEST_RSP, pCtx->rsp);
1516	AssertRC(rc);
1517
1518	/* Bits 22-31, 15, 5 & 3 must be zero. Bit 1 must be 1. */
1519	eflags = pCtx->eflags;
1520	eflags.u32 &= VMX_EFLAGS_RESERVED_0;
1521	eflags.u32 \|= VMX_EFLAGS_RESERVED_1;
1522
1523	#ifdef HWACCM_VMX_EMULATE_REALMODE
1524	/* Real mode emulation using v86 mode with CR4.VME (interrupt redirection using the int bitmap in the TSS) */
1525	if (CPUMIsGuestInRealModeEx(pCtx))
1526	{
1527	pVCpu->hwaccm.s.vmx.RealMode.eflags = eflags;
1528
1529	eflags.Bits.u1VM = 1;
1530	eflags.Bits.u2IOPL = 3;
1531	}
1532	#endif /* HWACCM_VMX_EMULATE_REALMODE */
1533	rc = VMXWriteVMCS(VMX_VMCS_GUEST_RFLAGS, eflags.u32);
1534	AssertRC(rc);
1535
1536	/* TSC offset. */
1537	uint64_t u64TSCOffset;
1538
1539	if (TMCpuTickCanUseRealTSC(pVM, &u64TSCOffset))
1540	{
1541	/* Note: VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT takes precedence over TSC_OFFSET */
1542	rc = VMXWriteVMCS64(VMX_VMCS_CTRL_TSC_OFFSET_FULL, u64TSCOffset);
1543	AssertRC(rc);
1544
1545	pVCpu->hwaccm.s.vmx.proc_ctls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT;
1546	rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
1547	AssertRC(rc);
1548	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatTSCOffset);
1549	}
1550	else
1551	{
1552	pVCpu->hwaccm.s.vmx.proc_ctls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT;
1553	rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
1554	AssertRC(rc);
1555	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatTSCIntercept);
1556	}
1557
1558	/* 64 bits guest mode? */
1559	if (pCtx->msrEFER & MSR_K6_EFER_LMA)
1560	{
1561	#if !defined(VBOX_ENABLE_64_BITS_GUESTS)
1562	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
1563	#elif HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
1564	pVCpu->hwaccm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
1565	#else
1566	# ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
1567	if (!pVM->hwaccm.s.fAllow64BitGuests)
1568	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
1569	# endif
1570	pVCpu->hwaccm.s.vmx.pfnStartVM = VMXR0StartVM64;
1571	#endif
1572	/* Unconditionally update these as wrmsr might have changed them. */
1573	rc = VMXWriteVMCS64(VMX_VMCS64_GUEST_FS_BASE, pCtx->fsHid.u64Base);
1574	AssertRC(rc);
1575	rc = VMXWriteVMCS64(VMX_VMCS64_GUEST_GS_BASE, pCtx->gsHid.u64Base);
1576	AssertRC(rc);
1577	}
1578	else
1579	{
1580	pVCpu->hwaccm.s.vmx.pfnStartVM = VMXR0StartVM32;
1581	}
1582
1583	/* Unconditionally update the guest EFER - on CPUs that supports it. */
1584	if (pVM->hwaccm.s.vmx.msr.vmx_entry.n.allowed1 & VMX_VMCS_CTRL_ENTRY_CONTROLS_LOAD_GUEST_EFER_MSR)
1585	{
1586	rc = VMXWriteVMCS64(VMX_VMCS_GUEST_EFER_FULL, pCtx->msrEFER);
1587	AssertRC(rc);
1588	}
1589
1590	vmxR0UpdateExceptionBitmap(pVM, pVCpu, pCtx);
1591
1592	/* Done. */
1593	pVCpu->hwaccm.s.fContextUseFlags &= ~HWACCM_CHANGED_ALL_GUEST;
1594
1595	return rc;
1596	}
1597
1598	/**
1599	* Syncs back the guest state
1600	*
1601	* @returns VBox status code.
1602	* @param pVM The VM to operate on.
1603	* @param pVCpu The VMCPU to operate on.
1604	* @param pCtx Guest context
1605	*/
1606	DECLINLINE(int) VMXR0SaveGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
1607	{
1608	RTGCUINTREG val, valShadow;
1609	RTGCUINTPTR uInterruptState;
1610	int rc;
1611
1612	/* Let's first sync back eip, esp, and eflags. */
1613	rc = VMXReadCachedVMCS(VMX_VMCS64_GUEST_RIP, &val);
1614	AssertRC(rc);
1615	pCtx->rip = val;
1616	rc = VMXReadCachedVMCS(VMX_VMCS64_GUEST_RSP, &val);
1617	AssertRC(rc);
1618	pCtx->rsp = val;
1619	rc = VMXReadCachedVMCS(VMX_VMCS_GUEST_RFLAGS, &val);
1620	AssertRC(rc);
1621	pCtx->eflags.u32 = val;
1622
1623	/* Take care of instruction fusing (sti, mov ss) */
1624	rc \|= VMXReadCachedVMCS(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &val);
1625	uInterruptState = val;
1626	if (uInterruptState != 0)
1627	{
1628	Assert(uInterruptState <= 2); /* only sti & mov ss */
1629	Log(("uInterruptState %x eip=%RGv\n", uInterruptState, pCtx->rip));
1630	EMSetInhibitInterruptsPC(pVM, pCtx->rip);
1631	}
1632	else
1633	VM_FF_CLEAR(pVM, VM_FF_INHIBIT_INTERRUPTS);
1634
1635	/* Control registers. */
1636	VMXReadCachedVMCS(VMX_VMCS_CTRL_CR0_READ_SHADOW, &valShadow);
1637	VMXReadCachedVMCS(VMX_VMCS64_GUEST_CR0, &val);
1638	val = (valShadow & pVCpu->hwaccm.s.vmx.cr0_mask) \| (val & ~pVCpu->hwaccm.s.vmx.cr0_mask);
1639	CPUMSetGuestCR0(pVM, val);
1640
1641	VMXReadCachedVMCS(VMX_VMCS_CTRL_CR4_READ_SHADOW, &valShadow);
1642	VMXReadCachedVMCS(VMX_VMCS64_GUEST_CR4, &val);
1643	val = (valShadow & pVCpu->hwaccm.s.vmx.cr4_mask) \| (val & ~pVCpu->hwaccm.s.vmx.cr4_mask);
1644	CPUMSetGuestCR4(pVM, val);
1645
1646	/* Note: no reason to sync back the CRx registers. They can't be changed by the guest. */
1647	/* Note: only in the nested paging case can CR3 & CR4 be changed by the guest. */
1648	if ( pVM->hwaccm.s.fNestedPaging
1649	&& CPUMIsGuestInPagedProtectedModeEx(pCtx))
1650	{
1651	PVMCSCACHE pCache = &pVCpu->hwaccm.s.vmx.VMCSCache;
1652
1653	/* Can be updated behind our back in the nested paging case. */
1654	CPUMSetGuestCR2(pVM, pCache->cr2);
1655
1656	VMXReadCachedVMCS(VMX_VMCS64_GUEST_CR3, &val);
1657
1658	if (val != pCtx->cr3)
1659	{
1660	CPUMSetGuestCR3(pVM, val);
1661	PGMUpdateCR3(pVM, val);
1662	}
1663	/* Prefetch the four PDPT entries in PAE mode. */
1664	vmxR0PrefetchPAEPdptrs(pVM, pVCpu, pCtx);
1665	}
1666
1667	/* Sync back DR7 here. */
1668	VMXReadCachedVMCS(VMX_VMCS64_GUEST_DR7, &val);
1669	pCtx->dr[7] = val;
1670
1671	/* Guest CPU context: ES, CS, SS, DS, FS, GS. */
1672	VMX_READ_SELREG(ES, es);
1673	VMX_READ_SELREG(SS, ss);
1674	VMX_READ_SELREG(CS, cs);
1675	VMX_READ_SELREG(DS, ds);
1676	VMX_READ_SELREG(FS, fs);
1677	VMX_READ_SELREG(GS, gs);
1678
1679	/*
1680	* System MSRs
1681	*/
1682	VMXReadCachedVMCS(VMX_VMCS32_GUEST_SYSENTER_CS, &val);
1683	pCtx->SysEnter.cs = val;
1684	VMXReadCachedVMCS(VMX_VMCS64_GUEST_SYSENTER_EIP, &val);
1685	pCtx->SysEnter.eip = val;
1686	VMXReadCachedVMCS(VMX_VMCS64_GUEST_SYSENTER_ESP, &val);
1687	pCtx->SysEnter.esp = val;
1688
1689	/* Misc. registers; must sync everything otherwise we can get out of sync when jumping to ring 3. */
1690	VMX_READ_SELREG(LDTR, ldtr);
1691
1692	VMXReadCachedVMCS(VMX_VMCS32_GUEST_GDTR_LIMIT, &val);
1693	pCtx->gdtr.cbGdt = val;
1694	VMXReadCachedVMCS(VMX_VMCS64_GUEST_GDTR_BASE, &val);
1695	pCtx->gdtr.pGdt = val;
1696
1697	VMXReadCachedVMCS(VMX_VMCS32_GUEST_IDTR_LIMIT, &val);
1698	pCtx->idtr.cbIdt = val;
1699	VMXReadCachedVMCS(VMX_VMCS64_GUEST_IDTR_BASE, &val);
1700	pCtx->idtr.pIdt = val;
1701
1702	#ifdef HWACCM_VMX_EMULATE_REALMODE
1703	/* Real mode emulation using v86 mode with CR4.VME (interrupt redirection using the int bitmap in the TSS) */
1704	if (CPUMIsGuestInRealModeEx(pCtx))
1705	{
1706	/* Hide our emulation flags */
1707	pCtx->eflags.Bits.u1VM = 0;
1708	pCtx->eflags.Bits.u2IOPL = pVCpu->hwaccm.s.vmx.RealMode.eflags.Bits.u2IOPL;
1709
1710	/* Force a TR resync every time in case we switch modes. */
1711	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_GUEST_TR;
1712	}
1713	else
1714	#endif /* HWACCM_VMX_EMULATE_REALMODE */
1715	{
1716	/* In real mode we have a fake TSS, so only sync it back when it's supposed to be valid. */
1717	VMX_READ_SELREG(TR, tr);
1718	}
1719	return VINF_SUCCESS;
1720	}
1721
1722	/**
1723	* Dummy placeholder
1724	*
1725	* @param pVM The VM to operate on.
1726	* @param pVCpu The VMCPU to operate on.
1727	*/
1728	static void vmxR0SetupTLBDummy(PVM pVM, PVMCPU pVCpu)
1729	{
1730	NOREF(pVM);
1731	NOREF(pVCpu);
1732	return;
1733	}
1734
1735	/**
1736	* Setup the tagged TLB for EPT
1737	*
1738	* @returns VBox status code.
1739	* @param pVM The VM to operate on.
1740	* @param pVCpu The VMCPU to operate on.
1741	*/
1742	static void vmxR0SetupTLBEPT(PVM pVM, PVMCPU pVCpu)
1743	{
1744	PHWACCM_CPUINFO pCpu;
1745
1746	Assert(pVM->hwaccm.s.fNestedPaging);
1747	Assert(!pVM->hwaccm.s.vmx.fVPID);
1748
1749	/* Deal with tagged TLBs if VPID or EPT is supported. */
1750	pCpu = HWACCMR0GetCurrentCpu();
1751	/* Force a TLB flush for the first world switch if the current cpu differs from the one we ran on last. */
1752	/* Note that this can happen both for start and resume due to long jumps back to ring 3. */
1753	if ( pVCpu->hwaccm.s.idLastCpu != pCpu->idCpu
1754	/* if the tlb flush count has changed, another VM has flushed the TLB of this cpu, so we can't use our current ASID anymore. */
1755	\|\| pVCpu->hwaccm.s.cTLBFlushes != pCpu->cTLBFlushes)
1756	{
1757	/* Force a TLB flush on VM entry. */
1758	pVCpu->hwaccm.s.fForceTLBFlush = true;
1759	}
1760	else
1761	Assert(!pCpu->fFlushTLB);
1762
1763	pVCpu->hwaccm.s.idLastCpu = pCpu->idCpu;
1764	pCpu->fFlushTLB = false;
1765
1766	if (pVCpu->hwaccm.s.fForceTLBFlush)
1767	vmxR0FlushEPT(pVM, pVCpu, pVM->hwaccm.s.vmx.enmFlushContext, 0);
1768
1769	#ifdef VBOX_WITH_STATISTICS
1770	if (pVCpu->hwaccm.s.fForceTLBFlush)
1771	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatFlushTLBWorldSwitch);
1772	else
1773	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatNoFlushTLBWorldSwitch);
1774	#endif
1775	}
1776
1777	#ifdef HWACCM_VTX_WITH_VPID
1778	/**
1779	* Setup the tagged TLB for VPID
1780	*
1781	* @returns VBox status code.
1782	* @param pVM The VM to operate on.
1783	* @param pVCpu The VMCPU to operate on.
1784	*/
1785	static void vmxR0SetupTLBVPID(PVM pVM, PVMCPU pVCpu)
1786	{
1787	PHWACCM_CPUINFO pCpu;
1788
1789	Assert(pVM->hwaccm.s.vmx.fVPID);
1790	Assert(!pVM->hwaccm.s.fNestedPaging);
1791
1792	/* Deal with tagged TLBs if VPID or EPT is supported. */
1793	pCpu = HWACCMR0GetCurrentCpu();
1794	/* Force a TLB flush for the first world switch if the current cpu differs from the one we ran on last. */
1795	/* Note that this can happen both for start and resume due to long jumps back to ring 3. */
1796	if ( pVCpu->hwaccm.s.idLastCpu != pCpu->idCpu
1797	/* if the tlb flush count has changed, another VM has flushed the TLB of this cpu, so we can't use our current ASID anymore. */
1798	\|\| pVCpu->hwaccm.s.cTLBFlushes != pCpu->cTLBFlushes)
1799	{
1800	/* Force a TLB flush on VM entry. */
1801	pVCpu->hwaccm.s.fForceTLBFlush = true;
1802	}
1803	else
1804	Assert(!pCpu->fFlushTLB);
1805
1806	pVCpu->hwaccm.s.idLastCpu = pCpu->idCpu;
1807
1808	/* Make sure we flush the TLB when required. Switch ASID to achieve the same thing, but without actually flushing the whole TLB (which is expensive). */
1809	if (pVCpu->hwaccm.s.fForceTLBFlush)
1810	{
1811	if ( ++pCpu->uCurrentASID >= pVM->hwaccm.s.uMaxASID
1812	\|\| pCpu->fFlushTLB)
1813	{
1814	pCpu->fFlushTLB = false;
1815	pCpu->uCurrentASID = 1; /* start at 1; host uses 0 */
1816	pCpu->cTLBFlushes++;
1817	}
1818	else
1819	{
1820	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatFlushASID);
1821	pVCpu->hwaccm.s.fForceTLBFlush = false;
1822	}
1823
1824	pVCpu->hwaccm.s.cTLBFlushes = pCpu->cTLBFlushes;
1825	pVCpu->hwaccm.s.uCurrentASID = pCpu->uCurrentASID;
1826	}
1827	else
1828	{
1829	Assert(!pCpu->fFlushTLB);
1830
1831	if (!pCpu->uCurrentASID \|\| !pVCpu->hwaccm.s.uCurrentASID)
1832	pVCpu->hwaccm.s.uCurrentASID = pCpu->uCurrentASID = 1;
1833	}
1834	AssertMsg(pVCpu->hwaccm.s.cTLBFlushes == pCpu->cTLBFlushes, ("Flush count mismatch for cpu %d (%x vs %x)\n", pCpu->idCpu, pVCpu->hwaccm.s.cTLBFlushes, pCpu->cTLBFlushes));
1835	AssertMsg(pCpu->uCurrentASID >= 1 && pCpu->uCurrentASID < pVM->hwaccm.s.uMaxASID, ("cpu%d uCurrentASID = %x\n", pCpu->idCpu, pCpu->uCurrentASID));
1836	AssertMsg(pVCpu->hwaccm.s.uCurrentASID >= 1 && pVCpu->hwaccm.s.uCurrentASID < pVM->hwaccm.s.uMaxASID, ("cpu%d VM uCurrentASID = %x\n", pCpu->idCpu, pVCpu->hwaccm.s.uCurrentASID));
1837
1838	int rc = VMXWriteVMCS(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hwaccm.s.uCurrentASID);
1839	AssertRC(rc);
1840
1841	if (pVCpu->hwaccm.s.fForceTLBFlush)
1842	vmxR0FlushVPID(pVM, pVCpu, pVM->hwaccm.s.vmx.enmFlushContext, 0);
1843
1844	#ifdef VBOX_WITH_STATISTICS
1845	if (pVCpu->hwaccm.s.fForceTLBFlush)
1846	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatFlushTLBWorldSwitch);
1847	else
1848	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatNoFlushTLBWorldSwitch);
1849	#endif
1850	}
1851	#endif /* HWACCM_VTX_WITH_VPID */
1852
1853	/**
1854	* Runs guest code in a VT-x VM.
1855	*
1856	* @returns VBox status code.
1857	* @param pVM The VM to operate on.
1858	* @param pVCpu The VMCPU to operate on.
1859	* @param pCtx Guest context
1860	*/
1861	VMMR0DECL(int) VMXR0RunGuestCode(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
1862	{
1863	int rc = VINF_SUCCESS;
1864	RTGCUINTREG val;
1865	RTGCUINTREG exitReason, instrError, cbInstr;
1866	RTGCUINTPTR exitQualification;
1867	RTGCUINTPTR intInfo = 0; /* shut up buggy gcc 4 */
1868	RTGCUINTPTR errCode, instrInfo;
1869	bool fSyncTPR = false;
1870	PHWACCM_CPUINFO pCpu = 0;
1871	unsigned cResume = 0;
1872	#ifdef VBOX_STRICT
1873	RTCPUID idCpuCheck;
1874	#endif
1875	#ifdef VBOX_WITH_STATISTICS
1876	bool fStatEntryStarted = true;
1877	bool fStatExit2Started = false;
1878	#endif
1879
1880	Log2(("\nE"));
1881
1882	STAM_PROFILE_ADV_START(&pVCpu->hwaccm.s.StatEntry, x);
1883
1884	#ifdef VBOX_STRICT
1885	{
1886	RTCCUINTREG val;
1887
1888	rc = VMXReadVMCS(VMX_VMCS_CTRL_PIN_EXEC_CONTROLS, &val);
1889	AssertRC(rc);
1890	Log2(("VMX_VMCS_CTRL_PIN_EXEC_CONTROLS = %08x\n", val));
1891
1892	/* allowed zero */
1893	if ((val & pVM->hwaccm.s.vmx.msr.vmx_pin_ctls.n.disallowed0) != pVM->hwaccm.s.vmx.msr.vmx_pin_ctls.n.disallowed0)
1894	Log(("Invalid VMX_VMCS_CTRL_PIN_EXEC_CONTROLS: zero\n"));
1895
1896	/* allowed one */
1897	if ((val & ~pVM->hwaccm.s.vmx.msr.vmx_pin_ctls.n.allowed1) != 0)
1898	Log(("Invalid VMX_VMCS_CTRL_PIN_EXEC_CONTROLS: one\n"));
1899
1900	rc = VMXReadVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, &val);
1901	AssertRC(rc);
1902	Log2(("VMX_VMCS_CTRL_PROC_EXEC_CONTROLS = %08x\n", val));
1903
1904	/* Must be set according to the MSR, but can be cleared in case of EPT. */
1905	if (pVM->hwaccm.s.fNestedPaging)
1906	val \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INVLPG_EXIT
1907	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
1908	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT;
1909
1910	/* allowed zero */
1911	if ((val & pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.disallowed0) != pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.disallowed0)
1912	Log(("Invalid VMX_VMCS_CTRL_PROC_EXEC_CONTROLS: zero\n"));
1913
1914	/* allowed one */
1915	if ((val & ~pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1) != 0)
1916	Log(("Invalid VMX_VMCS_CTRL_PROC_EXEC_CONTROLS: one\n"));
1917
1918	rc = VMXReadVMCS(VMX_VMCS_CTRL_ENTRY_CONTROLS, &val);
1919	AssertRC(rc);
1920	Log2(("VMX_VMCS_CTRL_ENTRY_CONTROLS = %08x\n", val));
1921
1922	/* allowed zero */
1923	if ((val & pVM->hwaccm.s.vmx.msr.vmx_entry.n.disallowed0) != pVM->hwaccm.s.vmx.msr.vmx_entry.n.disallowed0)
1924	Log(("Invalid VMX_VMCS_CTRL_ENTRY_CONTROLS: zero\n"));
1925
1926	/* allowed one */
1927	if ((val & ~pVM->hwaccm.s.vmx.msr.vmx_entry.n.allowed1) != 0)
1928	Log(("Invalid VMX_VMCS_CTRL_ENTRY_CONTROLS: one\n"));
1929
1930	rc = VMXReadVMCS(VMX_VMCS_CTRL_EXIT_CONTROLS, &val);
1931	AssertRC(rc);
1932	Log2(("VMX_VMCS_CTRL_EXIT_CONTROLS = %08x\n", val));
1933
1934	/* allowed zero */
1935	if ((val & pVM->hwaccm.s.vmx.msr.vmx_exit.n.disallowed0) != pVM->hwaccm.s.vmx.msr.vmx_exit.n.disallowed0)
1936	Log(("Invalid VMX_VMCS_CTRL_EXIT_CONTROLS: zero\n"));
1937
1938	/* allowed one */
1939	if ((val & ~pVM->hwaccm.s.vmx.msr.vmx_exit.n.allowed1) != 0)
1940	Log(("Invalid VMX_VMCS_CTRL_EXIT_CONTROLS: one\n"));
1941	}
1942	#endif
1943
1944	/* We can jump to this point to resume execution after determining that a VM-exit is innocent.
1945	*/
1946	ResumeExecution:
1947	STAM_STATS({
1948	if (fStatExit2Started) { STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2, y); fStatExit2Started = false; }
1949	if (!fStatEntryStarted) { STAM_PROFILE_ADV_START(&pVCpu->hwaccm.s.StatEntry, x); fStatEntryStarted = true; }
1950	});
1951	AssertMsg(pVCpu->hwaccm.s.idEnteredCpu == RTMpCpuId(),
1952	("Expected %d, I'm %d; cResume=%d exitReason=%RGv exitQualification=%RGv\n",
1953	(int)pVCpu->hwaccm.s.idEnteredCpu, (int)RTMpCpuId(), cResume, exitReason, exitQualification));
1954	Assert(!HWACCMR0SuspendPending());
1955
1956	/* Safety precaution; looping for too long here can have a very bad effect on the host */
1957	if (++cResume > HWACCM_MAX_RESUME_LOOPS)
1958	{
1959	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitMaxResume);
1960	rc = VINF_EM_RAW_INTERRUPT;
1961	goto end;
1962	}
1963
1964	/* Check for irq inhibition due to instruction fusing (sti, mov ss). */
1965	if (VM_FF_ISSET(pVM, VM_FF_INHIBIT_INTERRUPTS))
1966	{
1967	Log(("VM_FF_INHIBIT_INTERRUPTS at %RGv successor %RGv\n", (RTGCPTR)pCtx->rip, EMGetInhibitInterruptsPC(pVM)));
1968	if (pCtx->rip != EMGetInhibitInterruptsPC(pVM))
1969	{
1970	/* Note: we intentionally don't clear VM_FF_INHIBIT_INTERRUPTS here.
1971	* Before we are able to execute this instruction in raw mode (iret to guest code) an external interrupt might
1972	* force a world switch again. Possibly allowing a guest interrupt to be dispatched in the process. This could
1973	* break the guest. Sounds very unlikely, but such timing sensitive problems are not as rare as you might think.
1974	*/
1975	VM_FF_CLEAR(pVM, VM_FF_INHIBIT_INTERRUPTS);
1976	/* Irq inhibition is no longer active; clear the corresponding VMX state. */
1977	rc = VMXWriteVMCS(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, 0);
1978	AssertRC(rc);
1979	}
1980	}
1981	else
1982	{
1983	/* Irq inhibition is no longer active; clear the corresponding VMX state. */
1984	rc = VMXWriteVMCS(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, 0);
1985	AssertRC(rc);
1986	}
1987
1988	/* Check for pending actions that force us to go back to ring 3. */
1989	if (VM_FF_ISPENDING(pVM, VM_FF_TO_R3 \| VM_FF_TIMER))
1990	{
1991	VM_FF_CLEAR(pVM, VM_FF_TO_R3);
1992	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatSwitchToR3);
1993	rc = VINF_EM_RAW_TO_R3;
1994	goto end;
1995	}
1996	/* Pending request packets might contain actions that need immediate attention, such as pending hardware interrupts. */
1997	if (VM_FF_ISPENDING(pVM, VM_FF_REQUEST))
1998	{
1999	rc = VINF_EM_PENDING_REQUEST;
2000	goto end;
2001	}
2002
2003	/* When external interrupts are pending, we should exit the VM when IF is set. */
2004	/* Note! After VM_FF_INHIBIT_INTERRUPTS check!!! */
2005	rc = VMXR0CheckPendingInterrupt(pVM, pVCpu, pCtx);
2006	if (RT_FAILURE(rc))
2007	goto end;
2008
2009	/** @todo check timers?? */
2010
2011	/* TPR caching using CR8 is only available in 64 bits mode */
2012	/* Note the 32 bits exception for AMD (X86_CPUID_AMD_FEATURE_ECX_CR8L), but that appears missing in Intel CPUs */
2013	/* Note: we can't do this in LoadGuestState as PDMApicGetTPR can jump back to ring 3 (lock)!!!!! */
2014	/**
2015	* @todo reduce overhead
2016	*/
2017	if ( (pCtx->msrEFER & MSR_K6_EFER_LMA)
2018	&& pVM->hwaccm.s.vmx.pAPIC)
2019	{
2020	/* TPR caching in CR8 */
2021	uint8_t u8TPR;
2022	bool fPending;
2023
2024	int rc = PDMApicGetTPR(pVM, &u8TPR, &fPending);
2025	AssertRC(rc);
2026	/* The TPR can be found at offset 0x80 in the APIC mmio page. */
2027	pVM->hwaccm.s.vmx.pAPIC[0x80] = u8TPR << 4; /* bits 7-4 contain the task priority */
2028
2029	/* Two options here:
2030	* - external interrupt pending, but masked by the TPR value.
2031	* -> a CR8 update that lower the current TPR value should cause an exit
2032	* - no pending interrupts
2033	* -> We don't need to be explicitely notified. There are enough world switches for detecting pending interrupts.
2034	*/
2035	rc = VMXWriteVMCS(VMX_VMCS_CTRL_TPR_THRESHOLD, (fPending) ? u8TPR : 0);
2036	AssertRC(rc);
2037
2038	/* Always sync back the TPR; we should optimize this though / /* @todo optimize TPR sync. */
2039	fSyncTPR = true;
2040	}
2041
2042	#if defined(HWACCM_VTX_WITH_EPT) && defined(LOG_ENABLED)
2043	if ( pVM->hwaccm.s.fNestedPaging
2044	# ifdef HWACCM_VTX_WITH_VPID
2045	\|\| pVM->hwaccm.s.vmx.fVPID
2046	# endif /* HWACCM_VTX_WITH_VPID */
2047	)
2048	{
2049	pCpu = HWACCMR0GetCurrentCpu();
2050	if ( pVCpu->hwaccm.s.idLastCpu != pCpu->idCpu
2051	\|\| pVCpu->hwaccm.s.cTLBFlushes != pCpu->cTLBFlushes)
2052	{
2053	if (pVCpu->hwaccm.s.idLastCpu != pCpu->idCpu)
2054	Log(("Force TLB flush due to rescheduling to a different cpu (%d vs %d)\n", pVCpu->hwaccm.s.idLastCpu, pCpu->idCpu));
2055	else
2056	Log(("Force TLB flush due to changed TLB flush count (%x vs %x)\n", pVCpu->hwaccm.s.cTLBFlushes, pCpu->cTLBFlushes));
2057	}
2058	if (pCpu->fFlushTLB)
2059	Log(("Force TLB flush: first time cpu %d is used -> flush\n", pCpu->idCpu));
2060	else
2061	if (pVCpu->hwaccm.s.fForceTLBFlush)
2062	LogFlow(("Manual TLB flush\n"));
2063	}
2064	#endif
2065	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
2066	PGMDynMapFlushAutoSet(pVCpu);
2067	#endif
2068
2069	/*
2070	* NOTE: DO NOT DO ANYTHING AFTER THIS POINT THAT MIGHT JUMP BACK TO RING 3!
2071	* (until the actual world switch)
2072	*/
2073	#ifdef VBOX_STRICT
2074	idCpuCheck = RTMpCpuId();
2075	#endif
2076	#ifdef LOG_LOGGING
2077	VMMR0LogFlushDisable(pVCpu);
2078	#endif
2079	/* Save the host state first. */
2080	rc = VMXR0SaveHostState(pVM, pVCpu);
2081	if (rc != VINF_SUCCESS)
2082	goto end;
2083	/* Load the guest state */
2084	rc = VMXR0LoadGuestState(pVM, pVCpu, pCtx);
2085	if (rc != VINF_SUCCESS)
2086	goto end;
2087
2088	/* Deal with tagged TLB setup and invalidation. */
2089	pVM->hwaccm.s.vmx.pfnSetupTaggedTLB(pVM, pVCpu);
2090
2091	/* Non-register state Guest Context */
2092	/** @todo change me according to cpu state */
2093	rc = VMXWriteVMCS(VMX_VMCS32_GUEST_ACTIVITY_STATE, VMX_CMS_GUEST_ACTIVITY_ACTIVE);
2094	AssertRC(rc);
2095
2096	STAM_STATS({ STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatEntry, x); fStatEntryStarted = false; });
2097
2098	/* Manual save and restore:
2099	* - General purpose registers except RIP, RSP
2100	*
2101	* Trashed:
2102	* - CR2 (we don't care)
2103	* - LDTR (reset to 0)
2104	* - DRx (presumably not changed at all)
2105	* - DR7 (reset to 0x400)
2106	* - EFLAGS (reset to RT_BIT(1); not relevant)
2107	*
2108	*/
2109
2110	/* All done! Let's start VM execution. */
2111	STAM_PROFILE_ADV_START(&pVCpu->hwaccm.s.StatInGC, z);
2112	#ifdef VBOX_STRICT
2113	Assert(idCpuCheck == RTMpCpuId());
2114	#endif
2115	TMNotifyStartOfExecution(pVM);
2116	rc = pVCpu->hwaccm.s.vmx.pfnStartVM(pVCpu->hwaccm.s.fResumeVM, pCtx, &pVCpu->hwaccm.s.vmx.VMCSCache, pVM, pVCpu);
2117	TMNotifyEndOfExecution(pVM);
2118
2119	AssertMsg(!pVCpu->hwaccm.s.vmx.VMCSCache.Write.cValidEntries, ("pVCpu->hwaccm.s.vmx.VMCSCache.Write.cValidEntries=%d\n", pVCpu->hwaccm.s.vmx.VMCSCache.Write.cValidEntries));
2120
2121	/* In case we execute a goto ResumeExecution later on. */
2122	pVCpu->hwaccm.s.fResumeVM = true;
2123	pVCpu->hwaccm.s.fForceTLBFlush = false;
2124
2125	/*
2126	* !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
2127	* IMPORTANT: WE CAN'T DO ANY LOGGING OR OPERATIONS THAT CAN DO A LONGJMP BACK TO RING 3 BEFORE WE'VE SYNCED BACK (MOST OF) THE GUEST STATE
2128	* !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
2129	*/
2130
2131	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatInGC, z);
2132	STAM_PROFILE_ADV_START(&pVCpu->hwaccm.s.StatExit1, v);
2133
2134	if (rc != VINF_SUCCESS)
2135	{
2136	VMXR0ReportWorldSwitchError(pVM, pVCpu, rc, pCtx);
2137	goto end;
2138	}
2139	/* Success. Query the guest state and figure out what has happened. */
2140
2141	/* Investigate why there was a VM-exit. */
2142	rc = VMXReadCachedVMCS(VMX_VMCS32_RO_EXIT_REASON, &exitReason);
2143	STAM_COUNTER_INC(&pVCpu->hwaccm.s.paStatExitReasonR0[exitReason & MASK_EXITREASON_STAT]);
2144
2145	exitReason &= 0xffff; /* bit 0-15 contain the exit code. */
2146	rc \|= VMXReadCachedVMCS(VMX_VMCS32_RO_VM_INSTR_ERROR, &instrError);
2147	rc \|= VMXReadCachedVMCS(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &cbInstr);
2148	rc \|= VMXReadCachedVMCS(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &intInfo);
2149	/* might not be valid; depends on VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID. */
2150	rc \|= VMXReadCachedVMCS(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERRCODE, &errCode);
2151	rc \|= VMXReadCachedVMCS(VMX_VMCS32_RO_EXIT_INSTR_INFO, &instrInfo);
2152	rc \|= VMXReadCachedVMCS(VMX_VMCS_RO_EXIT_QUALIFICATION, &exitQualification);
2153	AssertRC(rc);
2154
2155	/* Sync back the guest state */
2156	rc = VMXR0SaveGuestState(pVM, pVCpu, pCtx);
2157	AssertRC(rc);
2158
2159	/* Note! NOW IT'S SAFE FOR LOGGING! */
2160	#ifdef LOG_LOGGING
2161	VMMR0LogFlushEnable(pVCpu);
2162	#endif
2163	Log2(("Raw exit reason %08x\n", exitReason));
2164
2165	/* Check if an injected event was interrupted prematurely. */
2166	rc = VMXReadCachedVMCS(VMX_VMCS32_RO_IDT_INFO, &val);
2167	AssertRC(rc);
2168	pVCpu->hwaccm.s.Event.intInfo = VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(val);
2169	if ( VMX_EXIT_INTERRUPTION_INFO_VALID(pVCpu->hwaccm.s.Event.intInfo)
2170	/* Ignore 'int xx' as they'll be restarted anyway. */
2171	&& VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hwaccm.s.Event.intInfo) != VMX_EXIT_INTERRUPTION_INFO_TYPE_SW
2172	/* Ignore software exceptions (such as int3) as they're reoccur when we restart the instruction anyway. */
2173	&& VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hwaccm.s.Event.intInfo) != VMX_EXIT_INTERRUPTION_INFO_TYPE_SWEXCPT)
2174	{
2175	pVCpu->hwaccm.s.Event.fPending = true;
2176	/* Error code present? */
2177	if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(pVCpu->hwaccm.s.Event.intInfo))
2178	{
2179	rc = VMXReadCachedVMCS(VMX_VMCS32_RO_IDT_ERRCODE, &val);
2180	AssertRC(rc);
2181	pVCpu->hwaccm.s.Event.errCode = val;
2182	Log(("Pending inject %RX64 at %RGv exit=%08x intInfo=%08x exitQualification=%RGv pending error=%RX64\n", pVCpu->hwaccm.s.Event.intInfo, (RTGCPTR)pCtx->rip, exitReason, intInfo, exitQualification, val));
2183	}
2184	else
2185	{
2186	Log(("Pending inject %RX64 at %RGv exit=%08x intInfo=%08x exitQualification=%RGv\n", pVCpu->hwaccm.s.Event.intInfo, (RTGCPTR)pCtx->rip, exitReason, intInfo, exitQualification));
2187	pVCpu->hwaccm.s.Event.errCode = 0;
2188	}
2189	}
2190	#ifdef VBOX_STRICT
2191	else
2192	if ( VMX_EXIT_INTERRUPTION_INFO_VALID(pVCpu->hwaccm.s.Event.intInfo)
2193	/* Ignore software exceptions (such as int3) as they're reoccur when we restart the instruction anyway. */
2194	&& VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hwaccm.s.Event.intInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_SWEXCPT)
2195	{
2196	Log(("Ignore pending inject %RX64 at %RGv exit=%08x intInfo=%08x exitQualification=%RGv\n", pVCpu->hwaccm.s.Event.intInfo, (RTGCPTR)pCtx->rip, exitReason, intInfo, exitQualification));
2197	}
2198
2199	if (exitReason == VMX_EXIT_ERR_INVALID_GUEST_STATE)
2200	HWACCMDumpRegs(pVM, pCtx);
2201	#endif
2202
2203	Log2(("E%d", exitReason));
2204	Log2(("Exit reason %d, exitQualification %RGv\n", (uint32_t)exitReason, exitQualification));
2205	Log2(("instrInfo=%d instrError=%d instr length=%d\n", (uint32_t)instrInfo, (uint32_t)instrError, (uint32_t)cbInstr));
2206	Log2(("Interruption error code %d\n", (uint32_t)errCode));
2207	Log2(("IntInfo = %08x\n", (uint32_t)intInfo));
2208	Log2(("New EIP=%RGv\n", (RTGCPTR)pCtx->rip));
2209
2210	if (fSyncTPR)
2211	{
2212	rc = PDMApicSetTPR(pVM, pVM->hwaccm.s.vmx.pAPIC[0x80] >> 4);
2213	AssertRC(rc);
2214	}
2215
2216	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit1, v);
2217	STAM_STATS({ STAM_PROFILE_ADV_START(&pVCpu->hwaccm.s.StatExit2, y); fStatExit2Started = true; });
2218
2219	/* Some cases don't need a complete resync of the guest CPU state; handle them here. */
2220	switch (exitReason)
2221	{
2222	case VMX_EXIT_EXCEPTION: /* 0 Exception or non-maskable interrupt (NMI). */
2223	case VMX_EXIT_EXTERNAL_IRQ: /* 1 External interrupt. */
2224	{
2225	uint32_t vector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(intInfo);
2226
2227	if (!VMX_EXIT_INTERRUPTION_INFO_VALID(intInfo))
2228	{
2229	Assert(exitReason == VMX_EXIT_EXTERNAL_IRQ);
2230	/* External interrupt; leave to allow it to be dispatched again. */
2231	rc = VINF_EM_RAW_INTERRUPT;
2232	break;
2233	}
2234	STAM_PROFILE_ADV_START(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
2235	switch (VMX_EXIT_INTERRUPTION_INFO_TYPE(intInfo))
2236	{
2237	case VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI: /* Non-maskable interrupt. */
2238	/* External interrupt; leave to allow it to be dispatched again. */
2239	rc = VINF_EM_RAW_INTERRUPT;
2240	break;
2241
2242	case VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT: /* External hardware interrupt. */
2243	AssertFailed(); /* can't come here; fails the first check. */
2244	break;
2245
2246	case VMX_EXIT_INTERRUPTION_INFO_TYPE_DBEXCPT: /* Unknown why we get this type for #DB */
2247	case VMX_EXIT_INTERRUPTION_INFO_TYPE_SWEXCPT: /* Software exception. (#BP or #OF) */
2248	Assert(vector == 1 \|\| vector == 3 \|\| vector == 4);
2249	/* no break */
2250	case VMX_EXIT_INTERRUPTION_INFO_TYPE_HWEXCPT: /* Hardware exception. */
2251	Log2(("Hardware/software interrupt %d\n", vector));
2252	switch (vector)
2253	{
2254	case X86_XCPT_NM:
2255	{
2256	Log(("#NM fault at %RGv error code %x\n", (RTGCPTR)pCtx->rip, errCode));
2257
2258	/** @todo don't intercept #NM exceptions anymore when we've activated the guest FPU state. */
2259	/* If we sync the FPU/XMM state on-demand, then we can continue execution as if nothing has happened. */
2260	rc = CPUMR0LoadGuestFPU(pVM, pVCpu, pCtx);
2261	if (rc == VINF_SUCCESS)
2262	{
2263	Assert(CPUMIsGuestFPUStateActive(pVCpu));
2264
2265	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitShadowNM);
2266
2267	/* Continue execution. */
2268	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_GUEST_CR0;
2269
2270	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
2271	goto ResumeExecution;
2272	}
2273
2274	Log(("Forward #NM fault to the guest\n"));
2275	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestNM);
2276	rc = VMXR0InjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), cbInstr, 0);
2277	AssertRC(rc);
2278	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
2279	goto ResumeExecution;
2280	}
2281
2282	case X86_XCPT_PF: /* Page fault */
2283	{
2284	#ifdef DEBUG
2285	if (pVM->hwaccm.s.fNestedPaging)
2286	{ /* A genuine pagefault.
2287	* Forward the trap to the guest by injecting the exception and resuming execution.
2288	*/
2289	Log(("Guest page fault at %RGv cr2=%RGv error code %x rsp=%RGv\n", (RTGCPTR)pCtx->rip, exitQualification, errCode, (RTGCPTR)pCtx->rsp));
2290
2291	Assert(CPUMIsGuestInPagedProtectedModeEx(pCtx));
2292
2293	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestPF);
2294
2295	/* Now we must update CR2. */
2296	pCtx->cr2 = exitQualification;
2297	rc = VMXR0InjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), cbInstr, errCode);
2298	AssertRC(rc);
2299
2300	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
2301	goto ResumeExecution;
2302	}
2303	#endif
2304	Assert(!pVM->hwaccm.s.fNestedPaging);
2305
2306	Log2(("Page fault at %RGv error code %x\n", exitQualification, errCode));
2307	/* Exit qualification contains the linear address of the page fault. */
2308	TRPMAssertTrap(pVM, X86_XCPT_PF, TRPM_TRAP);
2309	TRPMSetErrorCode(pVM, errCode);
2310	TRPMSetFaultAddress(pVM, exitQualification);
2311
2312	/* Forward it to our trap handler first, in case our shadow pages are out of sync. */
2313	rc = PGMTrap0eHandler(pVM, errCode, CPUMCTX2CORE(pCtx), (RTGCPTR)exitQualification);
2314	Log2(("PGMTrap0eHandler %RGv returned %Rrc\n", (RTGCPTR)pCtx->rip, rc));
2315	if (rc == VINF_SUCCESS)
2316	{ /* We've successfully synced our shadow pages, so let's just continue execution. */
2317	Log2(("Shadow page fault at %RGv cr2=%RGv error code %x\n", (RTGCPTR)pCtx->rip, exitQualification ,errCode));
2318	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitShadowPF);
2319
2320	TRPMResetTrap(pVM);
2321
2322	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
2323	goto ResumeExecution;
2324	}
2325	else
2326	if (rc == VINF_EM_RAW_GUEST_TRAP)
2327	{ /* A genuine pagefault.
2328	* Forward the trap to the guest by injecting the exception and resuming execution.
2329	*/
2330	Log2(("Forward page fault to the guest\n"));
2331
2332	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestPF);
2333	/* The error code might have been changed. */
2334	errCode = TRPMGetErrorCode(pVM);
2335
2336	TRPMResetTrap(pVM);
2337
2338	/* Now we must update CR2. */
2339	pCtx->cr2 = exitQualification;
2340	rc = VMXR0InjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), cbInstr, errCode);
2341	AssertRC(rc);
2342
2343	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
2344	goto ResumeExecution;
2345	}
2346	#ifdef VBOX_STRICT
2347	if (rc != VINF_EM_RAW_EMULATE_INSTR)
2348	Log2(("PGMTrap0eHandler failed with %d\n", rc));
2349	#endif
2350	/* Need to go back to the recompiler to emulate the instruction. */
2351	TRPMResetTrap(pVM);
2352	break;
2353	}
2354
2355	case X86_XCPT_MF: /* Floating point exception. */
2356	{
2357	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestMF);
2358	if (!(pCtx->cr0 & X86_CR0_NE))
2359	{
2360	/* old style FPU error reporting needs some extra work. */
2361	/** @todo don't fall back to the recompiler, but do it manually. */
2362	rc = VINF_EM_RAW_EMULATE_INSTR;
2363	break;
2364	}
2365	Log(("Trap %x at %04X:%RGv\n", vector, pCtx->cs, (RTGCPTR)pCtx->rip));
2366	rc = VMXR0InjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), cbInstr, errCode);
2367	AssertRC(rc);
2368
2369	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
2370	goto ResumeExecution;
2371	}
2372
2373	case X86_XCPT_DB: /* Debug exception. */
2374	{
2375	uint64_t uDR6;
2376
2377	/* DR6, DR7.GD and IA32_DEBUGCTL.LBR are not updated yet.
2378	*
2379	* Exit qualification bits:
2380	* 3:0 B0-B3 which breakpoint condition was met
2381	* 12:4 Reserved (0)
2382	* 13 BD - debug register access detected
2383	* 14 BS - single step execution or branch taken
2384	* 63:15 Reserved (0)
2385	*/
2386	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestDB);
2387
2388	/* Note that we don't support guest and host-initiated debugging at the same time. */
2389	Assert(DBGFIsStepping(pVM) \|\| CPUMIsGuestInRealModeEx(pCtx));
2390
2391	uDR6 = X86_DR6_INIT_VAL;
2392	uDR6 \|= (exitQualification & (X86_DR6_B0\|X86_DR6_B1\|X86_DR6_B2\|X86_DR6_B3\|X86_DR6_BD\|X86_DR6_BS));
2393	rc = DBGFR0Trap01Handler(pVM, CPUMCTX2CORE(pCtx), uDR6);
2394	if (rc == VINF_EM_RAW_GUEST_TRAP)
2395	{
2396	/** @todo this isn't working, but we'll never get here normally. */
2397
2398	/* Update DR6 here. */
2399	pCtx->dr[6] = uDR6;
2400
2401	/* X86_DR7_GD will be cleared if drx accesses should be trapped inside the guest. */
2402	pCtx->dr[7] &= ~X86_DR7_GD;
2403
2404	/* Paranoia. */
2405	pCtx->dr[7] &= 0xffffffff; /* upper 32 bits reserved */
2406	pCtx->dr[7] &= ~(RT_BIT(11) \| RT_BIT(12) \| RT_BIT(14) \| RT_BIT(15)); /* must be zero */
2407	pCtx->dr[7] \|= 0x400; /* must be one */
2408
2409	/* Resync DR7 */
2410	rc = VMXWriteVMCS64(VMX_VMCS64_GUEST_DR7, pCtx->dr[7]);
2411	AssertRC(rc);
2412
2413	Log(("Trap %x (debug) at %RGv exit qualification %RX64\n", vector, (RTGCPTR)pCtx->rip, exitQualification));
2414	rc = VMXR0InjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), cbInstr, errCode);
2415	AssertRC(rc);
2416
2417	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
2418	goto ResumeExecution;
2419	}
2420	/* Return to ring 3 to deal with the debug exit code. */
2421	break;
2422	}
2423
2424	case X86_XCPT_GP: /* General protection failure exception.*/
2425	{
2426	uint32_t cbSize;
2427
2428	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestGP);
2429	#ifdef VBOX_STRICT
2430	if (!CPUMIsGuestInRealModeEx(pCtx))
2431	{
2432	Log(("Trap %x at %04X:%RGv errorCode=%x\n", vector, pCtx->cs, (RTGCPTR)pCtx->rip, errCode));
2433	rc = VMXR0InjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), cbInstr, errCode);
2434	AssertRC(rc);
2435	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
2436	goto ResumeExecution;
2437	}
2438	#endif
2439	Assert(CPUMIsGuestInRealModeEx(pCtx));
2440
2441	LogFlow(("Real mode X86_XCPT_GP instruction emulation at %RGv\n", (RTGCPTR)pCtx->rip));
2442	rc = EMInterpretInstruction(pVM, CPUMCTX2CORE(pCtx), 0, &cbSize);
2443	if (rc == VINF_SUCCESS)
2444	{
2445	/* EIP has been updated already. */
2446
2447	/* lidt, lgdt can end up here. In the future crx changes as well. Just reload the whole context to be done with it. */
2448	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_ALL;
2449
2450	/* Only resume if successful. */
2451	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
2452	goto ResumeExecution;
2453	}
2454	AssertMsg(rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_EM_HALT, ("Unexpected rc=%Rrc\n", rc));
2455	break;
2456	}
2457
2458	#ifdef VBOX_STRICT
2459	case X86_XCPT_DE: /* Divide error. */
2460	case X86_XCPT_UD: /* Unknown opcode exception. */
2461	case X86_XCPT_SS: /* Stack segment exception. */
2462	case X86_XCPT_NP: /* Segment not present exception. */
2463	{
2464	switch(vector)
2465	{
2466	case X86_XCPT_DE:
2467	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestDE);
2468	break;
2469	case X86_XCPT_UD:
2470	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestUD);
2471	break;
2472	case X86_XCPT_SS:
2473	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestSS);
2474	break;
2475	case X86_XCPT_NP:
2476	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestNP);
2477	break;
2478	}
2479
2480	Log(("Trap %x at %04X:%RGv\n", vector, pCtx->cs, (RTGCPTR)pCtx->rip));
2481	rc = VMXR0InjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), cbInstr, errCode);
2482	AssertRC(rc);
2483
2484	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
2485	goto ResumeExecution;
2486	}
2487	#endif
2488	default:
2489	#ifdef HWACCM_VMX_EMULATE_REALMODE
2490	if (CPUMIsGuestInRealModeEx(pCtx))
2491	{
2492	Log(("Real Mode Trap %x at %04x:%04X error code %x\n", vector, pCtx->cs, pCtx->eip, errCode));
2493	rc = VMXR0InjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), cbInstr, errCode);
2494	AssertRC(rc);
2495
2496	/* Go back to ring 3 in case of a triple fault. */
2497	if ( vector == X86_XCPT_DF
2498	&& rc == VINF_EM_RESET)
2499	break;
2500
2501	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
2502	goto ResumeExecution;
2503	}
2504	#endif
2505	AssertMsgFailed(("Unexpected vm-exit caused by exception %x\n", vector));
2506	rc = VERR_VMX_UNEXPECTED_EXCEPTION;
2507	break;
2508	} /* switch (vector) */
2509
2510	break;
2511
2512	default:
2513	rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_CODE;
2514	AssertMsgFailed(("Unexpected interuption code %x\n", intInfo));
2515	break;
2516	}
2517
2518	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
2519	break;
2520	}
2521
2522	case VMX_EXIT_EPT_VIOLATION: /* 48 EPT violation. An attempt to access memory with a guest-physical address was disallowed by the configuration of the EPT paging structures. */
2523	{
2524	RTGCPHYS GCPhys;
2525
2526	Assert(pVM->hwaccm.s.fNestedPaging);
2527
2528	rc = VMXReadVMCS64(VMX_VMCS_EXIT_PHYS_ADDR_FULL, &GCPhys);
2529	AssertRC(rc);
2530	Assert(((exitQualification >> 7) & 3) != 2);
2531
2532	/* Determine the kind of violation. */
2533	errCode = 0;
2534	if (exitQualification & VMX_EXIT_QUALIFICATION_EPT_INSTR_FETCH)
2535	errCode \|= X86_TRAP_PF_ID;
2536
2537	if (exitQualification & VMX_EXIT_QUALIFICATION_EPT_DATA_WRITE)
2538	errCode \|= X86_TRAP_PF_RW;
2539
2540	/* If the page is present, then it's a page level protection fault. */
2541	if (exitQualification & VMX_EXIT_QUALIFICATION_EPT_ENTRY_PRESENT)
2542	errCode \|= X86_TRAP_PF_P;
2543
2544	Log(("EPT Page fault %x at %RGp error code %x\n", (uint32_t)exitQualification, GCPhys, errCode));
2545
2546	/* GCPhys contains the guest physical address of the page fault. */
2547	TRPMAssertTrap(pVM, X86_XCPT_PF, TRPM_TRAP);
2548	TRPMSetErrorCode(pVM, errCode);
2549	TRPMSetFaultAddress(pVM, GCPhys);
2550
2551	/* Handle the pagefault trap for the nested shadow table. */
2552	rc = PGMR0Trap0eHandlerNestedPaging(pVM, PGMMODE_EPT, errCode, CPUMCTX2CORE(pCtx), GCPhys);
2553	Log2(("PGMR0Trap0eHandlerNestedPaging %RGv returned %Rrc\n", (RTGCPTR)pCtx->rip, rc));
2554	if (rc == VINF_SUCCESS)
2555	{ /* We've successfully synced our shadow pages, so let's just continue execution. */
2556	Log2(("Shadow page fault at %RGv cr2=%RGp error code %x\n", (RTGCPTR)pCtx->rip, exitQualification , errCode));
2557	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitReasonNPF);
2558
2559	TRPMResetTrap(pVM);
2560
2561	goto ResumeExecution;
2562	}
2563
2564	#ifdef VBOX_STRICT
2565	if (rc != VINF_EM_RAW_EMULATE_INSTR)
2566	LogFlow(("PGMTrap0eHandlerNestedPaging failed with %d\n", rc));
2567	#endif
2568	/* Need to go back to the recompiler to emulate the instruction. */
2569	TRPMResetTrap(pVM);
2570	break;
2571	}
2572
2573	case VMX_EXIT_EPT_MISCONFIG:
2574	{
2575	RTGCPHYS GCPhys;
2576
2577	Assert(pVM->hwaccm.s.fNestedPaging);
2578
2579	rc = VMXReadVMCS64(VMX_VMCS_EXIT_PHYS_ADDR_FULL, &GCPhys);
2580	AssertRC(rc);
2581
2582	Log(("VMX_EXIT_EPT_MISCONFIG for %VGp\n", GCPhys));
2583	break;
2584	}
2585
2586	case VMX_EXIT_IRQ_WINDOW: /* 7 Interrupt window. */
2587	/* Clear VM-exit on IF=1 change. */
2588	LogFlow(("VMX_EXIT_IRQ_WINDOW %RGv pending=%d IF=%d\n", (RTGCPTR)pCtx->rip, VM_FF_ISPENDING(pVM, (VM_FF_INTERRUPT_APIC\|VM_FF_INTERRUPT_PIC)), pCtx->eflags.Bits.u1IF));
2589	pVCpu->hwaccm.s.vmx.proc_ctls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_IRQ_WINDOW_EXIT;
2590	rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
2591	AssertRC(rc);
2592	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitIrqWindow);
2593	goto ResumeExecution; /* we check for pending guest interrupts there */
2594
2595	case VMX_EXIT_WBINVD: /* 54 Guest software attempted to execute WBINVD. (conditional) */
2596	case VMX_EXIT_INVD: /* 13 Guest software attempted to execute INVD. (unconditional) */
2597	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitInvd);
2598	/* Skip instruction and continue directly. */
2599	pCtx->rip += cbInstr;
2600	/* Continue execution.*/
2601	goto ResumeExecution;
2602
2603	case VMX_EXIT_CPUID: /* 10 Guest software attempted to execute CPUID. */
2604	{
2605	Log2(("VMX: Cpuid %x\n", pCtx->eax));
2606	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitCpuid);
2607	rc = EMInterpretCpuId(pVM, CPUMCTX2CORE(pCtx));
2608	if (rc == VINF_SUCCESS)
2609	{
2610	/* Update EIP and continue execution. */
2611	Assert(cbInstr == 2);
2612	pCtx->rip += cbInstr;
2613	goto ResumeExecution;
2614	}
2615	AssertMsgFailed(("EMU: cpuid failed with %Rrc\n", rc));
2616	rc = VINF_EM_RAW_EMULATE_INSTR;
2617	break;
2618	}
2619
2620	case VMX_EXIT_RDTSC: /* 16 Guest software attempted to execute RDTSC. */
2621	{
2622	Log2(("VMX: Rdtsc\n"));
2623	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitRdtsc);
2624	rc = EMInterpretRdtsc(pVM, CPUMCTX2CORE(pCtx));
2625	if (rc == VINF_SUCCESS)
2626	{
2627	/* Update EIP and continue execution. */
2628	Assert(cbInstr == 2);
2629	pCtx->rip += cbInstr;
2630	goto ResumeExecution;
2631	}
2632	AssertMsgFailed(("EMU: rdtsc failed with %Rrc\n", rc));
2633	rc = VINF_EM_RAW_EMULATE_INSTR;
2634	break;
2635	}
2636
2637	case VMX_EXIT_INVPG: /* 14 Guest software attempted to execute INVPG. */
2638	{
2639	Log2(("VMX: invlpg\n"));
2640	Assert(!pVM->hwaccm.s.fNestedPaging);
2641
2642	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitInvpg);
2643	rc = EMInterpretInvlpg(pVM, CPUMCTX2CORE(pCtx), exitQualification);
2644	if (rc == VINF_SUCCESS)
2645	{
2646	/* Update EIP and continue execution. */
2647	pCtx->rip += cbInstr;
2648	goto ResumeExecution;
2649	}
2650	AssertMsg(rc == VERR_EM_INTERPRETER, ("EMU: invlpg %RGv failed with %Rrc\n", exitQualification, rc));
2651	break;
2652	}
2653
2654	case VMX_EXIT_RDMSR: /* 31 RDMSR. Guest software attempted to execute RDMSR. */
2655	case VMX_EXIT_WRMSR: /* 32 WRMSR. Guest software attempted to execute WRMSR. */
2656	{
2657	uint32_t cbSize;
2658
2659	/* Note: the intel manual claims there's a REX version of RDMSR that's slightly different, so we play safe by completely disassembling the instruction. */
2660	Log2(("VMX: %s\n", (exitReason == VMX_EXIT_RDMSR) ? "rdmsr" : "wrmsr"));
2661	rc = EMInterpretInstruction(pVM, CPUMCTX2CORE(pCtx), 0, &cbSize);
2662	if (rc == VINF_SUCCESS)
2663	{
2664	/* EIP has been updated already. */
2665
2666	/* Only resume if successful. */
2667	goto ResumeExecution;
2668	}
2669	AssertMsg(rc == VERR_EM_INTERPRETER, ("EMU: %s failed with %Rrc\n", (exitReason == VMX_EXIT_RDMSR) ? "rdmsr" : "wrmsr", rc));
2670	break;
2671	}
2672
2673	case VMX_EXIT_CRX_MOVE: /* 28 Control-register accesses. */
2674	{
2675	STAM_PROFILE_ADV_START(&pVCpu->hwaccm.s.StatExit2Sub2, y2);
2676
2677	switch (VMX_EXIT_QUALIFICATION_CRX_ACCESS(exitQualification))
2678	{
2679	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_WRITE:
2680	Log2(("VMX: %RGv mov cr%d, x\n", (RTGCPTR)pCtx->rip, VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification)));
2681	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitCRxWrite[VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification)]);
2682	rc = EMInterpretCRxWrite(pVM, CPUMCTX2CORE(pCtx),
2683	VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification),
2684	VMX_EXIT_QUALIFICATION_CRX_GENREG(exitQualification));
2685
2686	switch (VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification))
2687	{
2688	case 0:
2689	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_GUEST_CR0 \| HWACCM_CHANGED_GUEST_CR3;
2690	break;
2691	case 2:
2692	break;
2693	case 3:
2694	Assert(!pVM->hwaccm.s.fNestedPaging \|\| !CPUMIsGuestInPagedProtectedModeEx(pCtx));
2695	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_GUEST_CR3;
2696	break;
2697	case 4:
2698	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_GUEST_CR4;
2699	break;
2700	case 8:
2701	/* CR8 contains the APIC TPR */
2702	Assert(!(pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW));
2703	break;
2704
2705	default:
2706	AssertFailed();
2707	break;
2708	}
2709	/* Check if a sync operation is pending. */
2710	if ( rc == VINF_SUCCESS /* don't bother if we are going to ring 3 anyway */
2711	&& VM_FF_ISPENDING(pVM, VM_FF_PGM_SYNC_CR3 \| VM_FF_PGM_SYNC_CR3_NON_GLOBAL))
2712	{
2713	rc = PGMSyncCR3(pVM, CPUMGetGuestCR0(pVM), CPUMGetGuestCR3(pVM), CPUMGetGuestCR4(pVM), VM_FF_ISSET(pVM, VM_FF_PGM_SYNC_CR3));
2714	AssertRC(rc);
2715	}
2716	break;
2717
2718	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ:
2719	Log2(("VMX: mov x, crx\n"));
2720	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitCRxRead[VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification)]);
2721
2722	Assert(!pVM->hwaccm.s.fNestedPaging \|\| !CPUMIsGuestInPagedProtectedModeEx(pCtx) \|\| VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification) != USE_REG_CR3);
2723
2724	/* CR8 reads only cause an exit when the TPR shadow feature isn't present. */
2725	Assert(VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification) != 8 \|\| !(pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW));
2726
2727	rc = EMInterpretCRxRead(pVM, CPUMCTX2CORE(pCtx),
2728	VMX_EXIT_QUALIFICATION_CRX_GENREG(exitQualification),
2729	VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification));
2730	break;
2731
2732	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_CLTS:
2733	Log2(("VMX: clts\n"));
2734	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitCLTS);
2735	rc = EMInterpretCLTS(pVM);
2736	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_GUEST_CR0;
2737	break;
2738
2739	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_LMSW:
2740	Log2(("VMX: lmsw %x\n", VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(exitQualification)));
2741	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitLMSW);
2742	rc = EMInterpretLMSW(pVM, CPUMCTX2CORE(pCtx), VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(exitQualification));
2743	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_GUEST_CR0;
2744	break;
2745	}
2746
2747	/* Update EIP if no error occurred. */
2748	if (RT_SUCCESS(rc))
2749	pCtx->rip += cbInstr;
2750
2751	if (rc == VINF_SUCCESS)
2752	{
2753	/* Only resume if successful. */
2754	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub2, y2);
2755	goto ResumeExecution;
2756	}
2757	Assert(rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_PGM_SYNC_CR3);
2758	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub2, y2);
2759	break;
2760	}
2761
2762	case VMX_EXIT_DRX_MOVE: /* 29 Debug-register accesses. */
2763	{
2764	if (!DBGFIsStepping(pVM))
2765	{
2766	/* Disable drx move intercepts. */
2767	pVCpu->hwaccm.s.vmx.proc_ctls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT;
2768	rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
2769	AssertRC(rc);
2770
2771	/* Save the host and load the guest debug state. */
2772	rc = CPUMR0LoadGuestDebugState(pVM, pVCpu, pCtx, true /* include DR6 */);
2773	AssertRC(rc);
2774
2775	#ifdef VBOX_WITH_STATISTICS
2776	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatDRxContextSwitch);
2777	if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(exitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
2778	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitDRxWrite);
2779	else
2780	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitDRxRead);
2781	#endif
2782
2783	goto ResumeExecution;
2784	}
2785
2786	/** @todo clear VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT after the first time and restore drx registers afterwards */
2787	if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(exitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
2788	{
2789	Log2(("VMX: mov drx%d, genreg%d\n", VMX_EXIT_QUALIFICATION_DRX_REGISTER(exitQualification), VMX_EXIT_QUALIFICATION_DRX_GENREG(exitQualification)));
2790	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitDRxWrite);
2791	rc = EMInterpretDRxWrite(pVM, CPUMCTX2CORE(pCtx),
2792	VMX_EXIT_QUALIFICATION_DRX_REGISTER(exitQualification),
2793	VMX_EXIT_QUALIFICATION_DRX_GENREG(exitQualification));
2794	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_GUEST_DEBUG;
2795	Log2(("DR7=%08x\n", pCtx->dr[7]));
2796	}
2797	else
2798	{
2799	Log2(("VMX: mov x, drx\n"));
2800	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitDRxRead);
2801	rc = EMInterpretDRxRead(pVM, CPUMCTX2CORE(pCtx),
2802	VMX_EXIT_QUALIFICATION_DRX_GENREG(exitQualification),
2803	VMX_EXIT_QUALIFICATION_DRX_REGISTER(exitQualification));
2804	}
2805	/* Update EIP if no error occurred. */
2806	if (RT_SUCCESS(rc))
2807	pCtx->rip += cbInstr;
2808
2809	if (rc == VINF_SUCCESS)
2810	{
2811	/* Only resume if successful. */
2812	goto ResumeExecution;
2813	}
2814	Assert(rc == VERR_EM_INTERPRETER);
2815	break;
2816	}
2817
2818	/* Note: We'll get a #GP if the IO instruction isn't allowed (IOPL or TSS bitmap); no need to double check. */
2819	case VMX_EXIT_PORT_IO: /* 30 I/O instruction. */
2820	{
2821	STAM_PROFILE_ADV_START(&pVCpu->hwaccm.s.StatExit2Sub1, y1);
2822	uint32_t uIOWidth = VMX_EXIT_QUALIFICATION_IO_WIDTH(exitQualification);
2823	uint32_t uPort;
2824	bool fIOWrite = (VMX_EXIT_QUALIFICATION_IO_DIRECTION(exitQualification) == VMX_EXIT_QUALIFICATION_IO_DIRECTION_OUT);
2825
2826	/** @todo necessary to make the distinction? */
2827	if (VMX_EXIT_QUALIFICATION_IO_ENCODING(exitQualification) == VMX_EXIT_QUALIFICATION_IO_ENCODING_DX)
2828	{
2829	uPort = pCtx->edx & 0xffff;
2830	}
2831	else
2832	uPort = VMX_EXIT_QUALIFICATION_IO_PORT(exitQualification); /* Immediate encoding. */
2833
2834	/* paranoia */
2835	if (RT_UNLIKELY(uIOWidth == 2 \|\| uIOWidth >= 4))
2836	{
2837	rc = fIOWrite ? VINF_IOM_HC_IOPORT_WRITE : VINF_IOM_HC_IOPORT_READ;
2838	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub1, y1);
2839	break;
2840	}
2841
2842	uint32_t cbSize = g_aIOSize[uIOWidth];
2843
2844	if (VMX_EXIT_QUALIFICATION_IO_STRING(exitQualification))
2845	{
2846	/* ins/outs */
2847	DISCPUSTATE Cpu;
2848
2849	/* Disassemble manually to deal with segment prefixes. */
2850	/** @todo VMX_VMCS_EXIT_GUEST_LINEAR_ADDR contains the flat pointer operand of the instruction. */
2851	/** @todo VMX_VMCS32_RO_EXIT_INSTR_INFO also contains segment prefix info. */
2852	rc = EMInterpretDisasOne(pVM, CPUMCTX2CORE(pCtx), &Cpu, NULL);
2853	if (rc == VINF_SUCCESS)
2854	{
2855	if (fIOWrite)
2856	{
2857	Log2(("IOMInterpretOUTSEx %RGv %x size=%d\n", (RTGCPTR)pCtx->rip, uPort, cbSize));
2858	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitIOStringWrite);
2859	rc = IOMInterpretOUTSEx(pVM, CPUMCTX2CORE(pCtx), uPort, Cpu.prefix, cbSize);
2860	}
2861	else
2862	{
2863	Log2(("IOMInterpretINSEx %RGv %x size=%d\n", (RTGCPTR)pCtx->rip, uPort, cbSize));
2864	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitIOStringRead);
2865	rc = IOMInterpretINSEx(pVM, CPUMCTX2CORE(pCtx), uPort, Cpu.prefix, cbSize);
2866	}
2867	}
2868	else
2869	rc = VINF_EM_RAW_EMULATE_INSTR;
2870	}
2871	else
2872	{
2873	/* normal in/out */
2874	uint32_t uAndVal = g_aIOOpAnd[uIOWidth];
2875
2876	Assert(!VMX_EXIT_QUALIFICATION_IO_REP(exitQualification));
2877
2878	if (fIOWrite)
2879	{
2880	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitIOWrite);
2881	rc = IOMIOPortWrite(pVM, uPort, pCtx->eax & uAndVal, cbSize);
2882	}
2883	else
2884	{
2885	uint32_t u32Val = 0;
2886
2887	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitIORead);
2888	rc = IOMIOPortRead(pVM, uPort, &u32Val, cbSize);
2889	if (IOM_SUCCESS(rc))
2890	{
2891	/* Write back to the EAX register. */
2892	pCtx->eax = (pCtx->eax & ~uAndVal) \| (u32Val & uAndVal);
2893	}
2894	}
2895	}
2896	/*
2897	* Handled the I/O return codes.
2898	* (The unhandled cases end up with rc == VINF_EM_RAW_EMULATE_INSTR.)
2899	*/
2900	if (IOM_SUCCESS(rc))
2901	{
2902	/* Update EIP and continue execution. */
2903	pCtx->rip += cbInstr;
2904	if (RT_LIKELY(rc == VINF_SUCCESS))
2905	{
2906	/* If any IO breakpoints are armed, then we should check if a debug trap needs to be generated. */
2907	if (pCtx->dr[7] & X86_DR7_ENABLED_MASK)
2908	{
2909	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatDRxIOCheck);
2910	for (unsigned i=0;i<4;i++)
2911	{
2912	unsigned uBPLen = g_aIOSize[X86_DR7_GET_LEN(pCtx->dr[7], i)];
2913
2914	if ( (uPort >= pCtx->dr[i] && uPort < pCtx->dr[i] + uBPLen)
2915	&& (pCtx->dr[7] & (X86_DR7_L(i) \| X86_DR7_G(i)))
2916	&& (pCtx->dr[7] & X86_DR7_RW(i, X86_DR7_RW_IO)) == X86_DR7_RW(i, X86_DR7_RW_IO))
2917	{
2918	uint64_t uDR6;
2919
2920	Assert(CPUMIsGuestDebugStateActive(pVM));
2921
2922	uDR6 = ASMGetDR6();
2923
2924	/* Clear all breakpoint status flags and set the one we just hit. */
2925	uDR6 &= ~(X86_DR6_B0\|X86_DR6_B1\|X86_DR6_B2\|X86_DR6_B3);
2926	uDR6 \|= (uint64_t)RT_BIT(i);
2927
2928	/* Note: AMD64 Architecture Programmer's Manual 13.1:
2929	* Bits 15:13 of the DR6 register is never cleared by the processor and must be cleared by software after
2930	* the contents have been read.
2931	*/
2932	ASMSetDR6(uDR6);
2933
2934	/* X86_DR7_GD will be cleared if drx accesses should be trapped inside the guest. */
2935	pCtx->dr[7] &= ~X86_DR7_GD;
2936
2937	/* Paranoia. */
2938	pCtx->dr[7] &= 0xffffffff; /* upper 32 bits reserved */
2939	pCtx->dr[7] &= ~(RT_BIT(11) \| RT_BIT(12) \| RT_BIT(14) \| RT_BIT(15)); /* must be zero */
2940	pCtx->dr[7] \|= 0x400; /* must be one */
2941
2942	/* Resync DR7 */
2943	rc = VMXWriteVMCS64(VMX_VMCS64_GUEST_DR7, pCtx->dr[7]);
2944	AssertRC(rc);
2945
2946	/* Construct inject info. */
2947	intInfo = X86_XCPT_DB;
2948	intInfo \|= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
2949	intInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HWEXCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
2950
2951	Log(("Inject IO debug trap at %RGv\n", (RTGCPTR)pCtx->rip));
2952	rc = VMXR0InjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), 0, 0);
2953	AssertRC(rc);
2954
2955	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub1, y1);
2956	goto ResumeExecution;
2957	}
2958	}
2959	}
2960
2961	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub1, y1);
2962	goto ResumeExecution;
2963	}
2964	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub1, y1);
2965	break;
2966	}
2967
2968	#ifdef VBOX_STRICT
2969	if (rc == VINF_IOM_HC_IOPORT_READ)
2970	Assert(!fIOWrite);
2971	else if (rc == VINF_IOM_HC_IOPORT_WRITE)
2972	Assert(fIOWrite);
2973	else
2974	AssertMsg(RT_FAILURE(rc) \|\| rc == VINF_EM_RAW_EMULATE_INSTR \|\| rc == VINF_EM_RAW_GUEST_TRAP \|\| rc == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", rc));
2975	#endif
2976	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub1, y1);
2977	break;
2978	}
2979
2980	case VMX_EXIT_TPR: /* 43 TPR below threshold. Guest software executed MOV to CR8. */
2981	LogFlow(("VMX_EXIT_TPR\n"));
2982	/* RIP is already set to the next instruction and the TPR has been synced back. Just resume. */
2983	goto ResumeExecution;
2984
2985	default:
2986	/* The rest is handled after syncing the entire CPU state. */
2987	break;
2988	}
2989
2990	/* Note: the guest state isn't entirely synced back at this stage. */
2991
2992	/* Investigate why there was a VM-exit. (part 2) */
2993	switch (exitReason)
2994	{
2995	case VMX_EXIT_EXCEPTION: /* 0 Exception or non-maskable interrupt (NMI). */
2996	case VMX_EXIT_EXTERNAL_IRQ: /* 1 External interrupt. */
2997	case VMX_EXIT_EPT_VIOLATION:
2998	/* Already handled above. */
2999	break;
3000
3001	case VMX_EXIT_TRIPLE_FAULT: /* 2 Triple fault. */
3002	rc = VINF_EM_RESET; /* Triple fault equals a reset. */
3003	break;
3004
3005	case VMX_EXIT_INIT_SIGNAL: /* 3 INIT signal. */
3006	case VMX_EXIT_SIPI: /* 4 Start-up IPI (SIPI). */
3007	rc = VINF_EM_RAW_INTERRUPT;
3008	AssertFailed(); /* Can't happen. Yet. */
3009	break;
3010
3011	case VMX_EXIT_IO_SMI_IRQ: /* 5 I/O system-management interrupt (SMI). */
3012	case VMX_EXIT_SMI_IRQ: /* 6 Other SMI. */
3013	rc = VINF_EM_RAW_INTERRUPT;
3014	AssertFailed(); /* Can't happen afaik. */
3015	break;
3016
3017	case VMX_EXIT_TASK_SWITCH: /* 9 Task switch. */
3018	rc = VERR_EM_INTERPRETER;
3019	break;
3020
3021	case VMX_EXIT_HLT: /* 12 Guest software attempted to execute HLT. */
3022	/** Check if external interrupts are pending; if so, don't switch back. */
3023	pCtx->rip++; /* skip hlt */
3024	if ( pCtx->eflags.Bits.u1IF
3025	&& VM_FF_ISPENDING(pVM, (VM_FF_INTERRUPT_APIC\|VM_FF_INTERRUPT_PIC)))
3026	goto ResumeExecution;
3027
3028	rc = VINF_EM_HALT;
3029	break;
3030
3031	case VMX_EXIT_RSM: /* 17 Guest software attempted to execute RSM in SMM. */
3032	AssertFailed(); /* can't happen. */
3033	rc = VINF_EM_RAW_EXCEPTION_PRIVILEGED;
3034	break;
3035
3036	case VMX_EXIT_VMCALL: /* 18 Guest software executed VMCALL. */
3037	case VMX_EXIT_VMCLEAR: /* 19 Guest software executed VMCLEAR. */
3038	case VMX_EXIT_VMLAUNCH: /* 20 Guest software executed VMLAUNCH. */
3039	case VMX_EXIT_VMPTRLD: /* 21 Guest software executed VMPTRLD. */
3040	case VMX_EXIT_VMPTRST: /* 22 Guest software executed VMPTRST. */
3041	case VMX_EXIT_VMREAD: /* 23 Guest software executed VMREAD. */
3042	case VMX_EXIT_VMRESUME: /* 24 Guest software executed VMRESUME. */
3043	case VMX_EXIT_VMWRITE: /* 25 Guest software executed VMWRITE. */
3044	case VMX_EXIT_VMXOFF: /* 26 Guest software executed VMXOFF. */
3045	case VMX_EXIT_VMXON: /* 27 Guest software executed VMXON. */
3046	/** @todo inject #UD immediately */
3047	rc = VINF_EM_RAW_EXCEPTION_PRIVILEGED;
3048	break;
3049
3050	case VMX_EXIT_CPUID: /* 10 Guest software attempted to execute CPUID. */
3051	case VMX_EXIT_RDTSC: /* 16 Guest software attempted to execute RDTSC. */
3052	case VMX_EXIT_INVPG: /* 14 Guest software attempted to execute INVPG. */
3053	case VMX_EXIT_CRX_MOVE: /* 28 Control-register accesses. */
3054	case VMX_EXIT_DRX_MOVE: /* 29 Debug-register accesses. */
3055	case VMX_EXIT_PORT_IO: /* 30 I/O instruction. */
3056	/* already handled above */
3057	AssertMsg( rc == VINF_PGM_CHANGE_MODE
3058	\|\| rc == VINF_EM_RAW_INTERRUPT
3059	\|\| rc == VERR_EM_INTERPRETER
3060	\|\| rc == VINF_EM_RAW_EMULATE_INSTR
3061	\|\| rc == VINF_PGM_SYNC_CR3
3062	\|\| rc == VINF_IOM_HC_IOPORT_READ
3063	\|\| rc == VINF_IOM_HC_IOPORT_WRITE
3064	\|\| rc == VINF_EM_RAW_GUEST_TRAP
3065	\|\| rc == VINF_TRPM_XCPT_DISPATCHED
3066	\|\| rc == VINF_EM_RESCHEDULE_REM,
3067	("rc = %d\n", rc));
3068	break;
3069
3070	case VMX_EXIT_TPR: /* 43 TPR below threshold. Guest software executed MOV to CR8. */
3071	case VMX_EXIT_RDMSR: /* 31 RDMSR. Guest software attempted to execute RDMSR. */
3072	case VMX_EXIT_WRMSR: /* 32 WRMSR. Guest software attempted to execute WRMSR. */
3073	/* Note: If we decide to emulate them here, then we must sync the MSRs that could have been changed (sysenter, fs/gs base)!!! */
3074	rc = VERR_EM_INTERPRETER;
3075	break;
3076
3077	case VMX_EXIT_RDPMC: /* 15 Guest software attempted to execute RDPMC. */
3078	case VMX_EXIT_MWAIT: /* 36 Guest software executed MWAIT. */
3079	case VMX_EXIT_MONITOR: /* 39 Guest software attempted to execute MONITOR. */
3080	case VMX_EXIT_PAUSE: /* 40 Guest software attempted to execute PAUSE. */
3081	rc = VINF_EM_RAW_EXCEPTION_PRIVILEGED;
3082	break;
3083
3084	case VMX_EXIT_IRQ_WINDOW: /* 7 Interrupt window. */
3085	Assert(rc == VINF_EM_RAW_INTERRUPT);
3086	break;
3087
3088	case VMX_EXIT_ERR_INVALID_GUEST_STATE: /* 33 VM-entry failure due to invalid guest state. */
3089	{
3090	#ifdef VBOX_STRICT
3091	RTCCUINTREG val;
3092
3093	Log(("VMX_EXIT_ERR_INVALID_GUEST_STATE\n"));
3094
3095	VMXReadVMCS(VMX_VMCS64_GUEST_RIP, &val);
3096	Log(("Old eip %RGv new %RGv\n", (RTGCPTR)pCtx->rip, (RTGCPTR)val));
3097
3098	VMXReadVMCS(VMX_VMCS64_GUEST_CR0, &val);
3099	Log(("VMX_VMCS_GUEST_CR0 %RX64\n", val));
3100
3101	VMXReadVMCS(VMX_VMCS64_GUEST_CR3, &val);
3102	Log(("VMX_VMCS_GUEST_CR3 %RGp\n", val));
3103
3104	VMXReadVMCS(VMX_VMCS64_GUEST_CR4, &val);
3105	Log(("VMX_VMCS_GUEST_CR4 %RX64\n", val));
3106
3107	VMXReadVMCS(VMX_VMCS_GUEST_RFLAGS, &val);
3108	Log(("VMX_VMCS_GUEST_RFLAGS %08x\n", val));
3109
3110	VMX_LOG_SELREG(CS, "CS");
3111	VMX_LOG_SELREG(DS, "DS");
3112	VMX_LOG_SELREG(ES, "ES");
3113	VMX_LOG_SELREG(FS, "FS");
3114	VMX_LOG_SELREG(GS, "GS");
3115	VMX_LOG_SELREG(SS, "SS");
3116	VMX_LOG_SELREG(TR, "TR");
3117	VMX_LOG_SELREG(LDTR, "LDTR");
3118
3119	VMXReadVMCS(VMX_VMCS64_GUEST_GDTR_BASE, &val);
3120	Log(("VMX_VMCS_GUEST_GDTR_BASE %RGv\n", val));
3121	VMXReadVMCS(VMX_VMCS64_GUEST_IDTR_BASE, &val);
3122	Log(("VMX_VMCS_GUEST_IDTR_BASE %RGv\n", val));
3123	#endif /* VBOX_STRICT */
3124	rc = VERR_VMX_INVALID_GUEST_STATE;
3125	break;
3126	}
3127
3128	case VMX_EXIT_ERR_MSR_LOAD: /* 34 VM-entry failure due to MSR loading. */
3129	case VMX_EXIT_ERR_MACHINE_CHECK: /* 41 VM-entry failure due to machine-check. */
3130	default:
3131	rc = VERR_VMX_UNEXPECTED_EXIT_CODE;
3132	AssertMsgFailed(("Unexpected exit code %d\n", exitReason)); /* Can't happen. */
3133	break;
3134
3135	}
3136	end:
3137
3138	/* Signal changes for the recompiler. */
3139	CPUMSetChangedFlags(pVM, CPUM_CHANGED_SYSENTER_MSR \| CPUM_CHANGED_LDTR \| CPUM_CHANGED_GDTR \| CPUM_CHANGED_IDTR \| CPUM_CHANGED_TR \| CPUM_CHANGED_HIDDEN_SEL_REGS);
3140
3141	/* If we executed vmlaunch/vmresume and an external irq was pending, then we don't have to do a full sync the next time. */
3142	if ( exitReason == VMX_EXIT_EXTERNAL_IRQ
3143	&& !VMX_EXIT_INTERRUPTION_INFO_VALID(intInfo))
3144	{
3145	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatPendingHostIrq);
3146	/* On the next entry we'll only sync the host context. */
3147	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_HOST_CONTEXT;
3148	}
3149	else
3150	{
3151	/* On the next entry we'll sync everything. */
3152	/** @todo we can do better than this */
3153	/* Not in the VINF_PGM_CHANGE_MODE though! */
3154	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_ALL;
3155	}
3156
3157	/* translate into a less severe return code */
3158	if (rc == VERR_EM_INTERPRETER)
3159	rc = VINF_EM_RAW_EMULATE_INSTR;
3160	else
3161	/* Try to extract more information about what might have gone wrong here. */
3162	if (rc == VERR_VMX_INVALID_VMCS_PTR)
3163	{
3164	VMXGetActivateVMCS(&pVCpu->hwaccm.s.vmx.lasterror.u64VMCSPhys);
3165	pVCpu->hwaccm.s.vmx.lasterror.ulVMCSRevision = (uint32_t )pVCpu->hwaccm.s.vmx.pVMCS;
3166	pVCpu->hwaccm.s.vmx.lasterror.idEnteredCpu = pVCpu->hwaccm.s.idEnteredCpu;
3167	pVCpu->hwaccm.s.vmx.lasterror.idCurrentCpu = RTMpCpuId();
3168	}
3169
3170	STAM_STATS({
3171	if (fStatExit2Started) STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2, y);
3172	else if (fStatEntryStarted) STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatEntry, x);
3173	});
3174	Log2(("X"));
3175	return rc;
3176	}
3177
3178
3179	/**
3180	* Enters the VT-x session
3181	*
3182	* @returns VBox status code.
3183	* @param pVM The VM to operate on.
3184	* @param pVCpu The VMCPU to operate on.
3185	* @param pCpu CPU info struct
3186	*/
3187	VMMR0DECL(int) VMXR0Enter(PVM pVM, PVMCPU pVCpu, PHWACCM_CPUINFO pCpu)
3188	{
3189	Assert(pVM->hwaccm.s.vmx.fSupported);
3190
3191	unsigned cr4 = ASMGetCR4();
3192	if (!(cr4 & X86_CR4_VMXE))
3193	{
3194	AssertMsgFailed(("X86_CR4_VMXE should be set!\n"));
3195	return VERR_VMX_X86_CR4_VMXE_CLEARED;
3196	}
3197
3198	/* Activate the VM Control Structure. */
3199	int rc = VMXActivateVMCS(pVCpu->hwaccm.s.vmx.pVMCSPhys);
3200	if (RT_FAILURE(rc))
3201	return rc;
3202
3203	pVCpu->hwaccm.s.fResumeVM = false;
3204	return VINF_SUCCESS;
3205	}
3206
3207
3208	/**
3209	* Leaves the VT-x session
3210	*
3211	* @returns VBox status code.
3212	* @param pVM The VM to operate on.
3213	* @param pVCpu The VMCPU to operate on.
3214	* @param pCtx CPU context
3215	*/
3216	VMMR0DECL(int) VMXR0Leave(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
3217	{
3218	Assert(pVM->hwaccm.s.vmx.fSupported);
3219
3220	/* Save the guest debug state if necessary. */
3221	if (CPUMIsGuestDebugStateActive(pVM))
3222	{
3223	CPUMR0SaveGuestDebugState(pVM, pVCpu, pCtx, true /* save DR6 */);
3224
3225	/* Enable drx move intercepts again. */
3226	pVCpu->hwaccm.s.vmx.proc_ctls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT;
3227	int rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
3228	AssertRC(rc);
3229
3230	/* Resync the debug registers the next time. */
3231	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_GUEST_DEBUG;
3232	}
3233	else
3234	Assert(pVCpu->hwaccm.s.vmx.proc_ctls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT);
3235
3236	/* Clear VM Control Structure. Marking it inactive, clearing implementation specific data and writing back VMCS data to memory. */
3237	int rc = VMXClearVMCS(pVCpu->hwaccm.s.vmx.pVMCSPhys);
3238	AssertRC(rc);
3239
3240	return VINF_SUCCESS;
3241	}
3242
3243	/**
3244	* Flush the TLB (EPT)
3245	*
3246	* @returns VBox status code.
3247	* @param pVM The VM to operate on.
3248	* @param pVCpu The VM CPU to operate on.
3249	* @param enmFlush Type of flush
3250	* @param GCPhys Physical address of the page to flush
3251	*/
3252	static void vmxR0FlushEPT(PVM pVM, PVMCPU pVCpu, VMX_FLUSH enmFlush, RTGCPHYS GCPhys)
3253	{
3254	uint64_t descriptor[2];
3255
3256	LogFlow(("vmxR0FlushEPT %d %RGv\n", enmFlush, GCPhys));
3257	Assert(pVM->hwaccm.s.fNestedPaging);
3258	descriptor[0] = pVCpu->hwaccm.s.vmx.GCPhysEPTP;
3259	descriptor[1] = GCPhys;
3260	int rc = VMXR0InvEPT(enmFlush, &descriptor[0]);
3261	AssertRC(rc);
3262	}
3263
3264	#ifdef HWACCM_VTX_WITH_VPID
3265	/**
3266	* Flush the TLB (EPT)
3267	*
3268	* @returns VBox status code.
3269	* @param pVM The VM to operate on.
3270	* @param pVCpu The VM CPU to operate on.
3271	* @param enmFlush Type of flush
3272	* @param GCPtr Virtual address of the page to flush
3273	*/
3274	static void vmxR0FlushVPID(PVM pVM, PVMCPU pVCpu, VMX_FLUSH enmFlush, RTGCPTR GCPtr)
3275	{
3276	#if HC_ARCH_BITS == 32
3277	/* If we get a flush in 64 bits guest mode, then force a full TLB flush. Invvpid probably takes only 32 bits addresses. (@todo) */
3278	if ( CPUMIsGuestInLongMode(pVM)
3279	&& !VMX_IS_64BIT_HOST_MODE())
3280	{
3281	pVCpu->hwaccm.s.fForceTLBFlush = true;
3282	}
3283	else
3284	#endif
3285	{
3286	uint64_t descriptor[2];
3287
3288	Assert(pVM->hwaccm.s.vmx.fVPID);
3289	descriptor[0] = pVCpu->hwaccm.s.uCurrentASID;
3290	descriptor[1] = GCPtr;
3291	int rc = VMXR0InvVPID(enmFlush, &descriptor[0]);
3292	AssertRC(rc);
3293	}
3294	}
3295	#endif /* HWACCM_VTX_WITH_VPID */
3296
3297	/**
3298	* Invalidates a guest page
3299	*
3300	* @returns VBox status code.
3301	* @param pVM The VM to operate on.
3302	* @param pVCpu The VM CPU to operate on.
3303	* @param GCVirt Page to invalidate
3304	*/
3305	VMMR0DECL(int) VMXR0InvalidatePage(PVM pVM, PVMCPU pVCpu, RTGCPTR GCVirt)
3306	{
3307	bool fFlushPending = pVCpu->hwaccm.s.fForceTLBFlush;
3308
3309	LogFlow(("VMXR0InvalidatePage %RGv\n", GCVirt));
3310
3311	/* Only relevant if we want to use VPID.
3312	* In the nested paging case we still see such calls, but
3313	* can safely ignore them. (e.g. after cr3 updates)
3314	*/
3315	#ifdef HWACCM_VTX_WITH_VPID
3316	/* Skip it if a TLB flush is already pending. */
3317	if ( !fFlushPending
3318	&& pVM->hwaccm.s.vmx.fVPID)
3319	vmxR0FlushVPID(pVM, pVCpu, pVM->hwaccm.s.vmx.enmFlushPage, GCVirt);
3320	#endif /* HWACCM_VTX_WITH_VPID */
3321
3322	return VINF_SUCCESS;
3323	}
3324
3325	/**
3326	* Invalidates a guest page by physical address
3327	*
3328	* NOTE: Assumes the current instruction references this physical page though a virtual address!!
3329	*
3330	* @returns VBox status code.
3331	* @param pVM The VM to operate on.
3332	* @param pVCpu The VM CPU to operate on.
3333	* @param GCPhys Page to invalidate
3334	*/
3335	VMMR0DECL(int) VMXR0InvalidatePhysPage(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys)
3336	{
3337	bool fFlushPending = pVCpu->hwaccm.s.fForceTLBFlush;
3338
3339	Assert(pVM->hwaccm.s.fNestedPaging);
3340
3341	LogFlow(("VMXR0InvalidatePhysPage %RGp\n", GCPhys));
3342
3343	/* Skip it if a TLB flush is already pending. */
3344	if (!fFlushPending)
3345	vmxR0FlushEPT(pVM, pVCpu, pVM->hwaccm.s.vmx.enmFlushPage, GCPhys);
3346
3347	return VINF_SUCCESS;
3348	}
3349
3350	/**
3351	* Report world switch error and dump some useful debug info
3352	*
3353	* @param pVM The VM to operate on.
3354	* @param pVCpu The VMCPU to operate on.
3355	* @param rc Return code
3356	* @param pCtx Current CPU context (not updated)
3357	*/
3358	static void VMXR0ReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, int rc, PCPUMCTX pCtx)
3359	{
3360	switch (rc)
3361	{
3362	case VERR_VMX_INVALID_VMXON_PTR:
3363	AssertFailed();
3364	break;
3365
3366	case VERR_VMX_UNABLE_TO_START_VM:
3367	case VERR_VMX_UNABLE_TO_RESUME_VM:
3368	{
3369	int rc;
3370	RTCCUINTREG exitReason, instrError, val;
3371
3372	rc = VMXReadVMCS(VMX_VMCS32_RO_EXIT_REASON, &exitReason);
3373	rc \|= VMXReadVMCS(VMX_VMCS32_RO_VM_INSTR_ERROR, &instrError);
3374	AssertRC(rc);
3375	if (rc == VINF_SUCCESS)
3376	{
3377	Log(("Unable to start/resume VM for reason: %x. Instruction error %x\n", (uint32_t)exitReason, (uint32_t)instrError));
3378	Log(("Current stack %08x\n", &rc));
3379
3380	pVCpu->hwaccm.s.vmx.lasterror.ulInstrError = instrError;
3381	pVCpu->hwaccm.s.vmx.lasterror.ulExitReason = exitReason;
3382
3383	#ifdef VBOX_STRICT
3384	RTGDTR gdtr;
3385	PX86DESCHC pDesc;
3386
3387	ASMGetGDTR(&gdtr);
3388
3389	VMXReadVMCS(VMX_VMCS64_GUEST_RIP, &val);
3390	Log(("Old eip %RGv new %RGv\n", (RTGCPTR)pCtx->rip, (RTGCPTR)val));
3391	VMXReadVMCS(VMX_VMCS_CTRL_PIN_EXEC_CONTROLS, &val);
3392	Log(("VMX_VMCS_CTRL_PIN_EXEC_CONTROLS %08x\n", val));
3393	VMXReadVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, &val);
3394	Log(("VMX_VMCS_CTRL_PROC_EXEC_CONTROLS %08x\n", val));
3395	VMXReadVMCS(VMX_VMCS_CTRL_ENTRY_CONTROLS, &val);
3396	Log(("VMX_VMCS_CTRL_ENTRY_CONTROLS %08x\n", val));
3397	VMXReadVMCS(VMX_VMCS_CTRL_EXIT_CONTROLS, &val);
3398	Log(("VMX_VMCS_CTRL_EXIT_CONTROLS %08x\n", val));
3399
3400	VMXReadVMCS(VMX_VMCS_HOST_CR0, &val);
3401	Log(("VMX_VMCS_HOST_CR0 %08x\n", val));
3402
3403	VMXReadVMCS(VMX_VMCS_HOST_CR3, &val);
3404	Log(("VMX_VMCS_HOST_CR3 %08x\n", val));
3405
3406	VMXReadVMCS(VMX_VMCS_HOST_CR4, &val);
3407	Log(("VMX_VMCS_HOST_CR4 %08x\n", val));
3408
3409	VMXReadVMCS(VMX_VMCS16_HOST_FIELD_CS, &val);
3410	Log(("VMX_VMCS_HOST_FIELD_CS %08x\n", val));
3411
3412	VMXReadVMCS(VMX_VMCS_GUEST_RFLAGS, &val);
3413	Log(("VMX_VMCS_GUEST_RFLAGS %08x\n", val));
3414
3415	if (val < gdtr.cbGdt)
3416	{
3417	pDesc = &((PX86DESCHC)gdtr.pGdt)[val >> X86_SEL_SHIFT_HC];
3418	HWACCMR0DumpDescriptor(pDesc, val, "CS: ");
3419	}
3420
3421	VMXReadVMCS(VMX_VMCS16_HOST_FIELD_DS, &val);
3422	Log(("VMX_VMCS_HOST_FIELD_DS %08x\n", val));
3423	if (val < gdtr.cbGdt)
3424	{
3425	pDesc = &((PX86DESCHC)gdtr.pGdt)[val >> X86_SEL_SHIFT_HC];
3426	HWACCMR0DumpDescriptor(pDesc, val, "DS: ");
3427	}
3428
3429	VMXReadVMCS(VMX_VMCS16_HOST_FIELD_ES, &val);
3430	Log(("VMX_VMCS_HOST_FIELD_ES %08x\n", val));
3431	if (val < gdtr.cbGdt)
3432	{
3433	pDesc = &((PX86DESCHC)gdtr.pGdt)[val >> X86_SEL_SHIFT_HC];
3434	HWACCMR0DumpDescriptor(pDesc, val, "ES: ");
3435	}
3436
3437	VMXReadVMCS(VMX_VMCS16_HOST_FIELD_FS, &val);
3438	Log(("VMX_VMCS16_HOST_FIELD_FS %08x\n", val));
3439	if (val < gdtr.cbGdt)
3440	{
3441	pDesc = &((PX86DESCHC)gdtr.pGdt)[val >> X86_SEL_SHIFT_HC];
3442	HWACCMR0DumpDescriptor(pDesc, val, "FS: ");
3443	}
3444
3445	VMXReadVMCS(VMX_VMCS16_HOST_FIELD_GS, &val);
3446	Log(("VMX_VMCS16_HOST_FIELD_GS %08x\n", val));
3447	if (val < gdtr.cbGdt)
3448	{
3449	pDesc = &((PX86DESCHC)gdtr.pGdt)[val >> X86_SEL_SHIFT_HC];
3450	HWACCMR0DumpDescriptor(pDesc, val, "GS: ");
3451	}
3452
3453	VMXReadVMCS(VMX_VMCS16_HOST_FIELD_SS, &val);
3454	Log(("VMX_VMCS16_HOST_FIELD_SS %08x\n", val));
3455	if (val < gdtr.cbGdt)
3456	{
3457	pDesc = &((PX86DESCHC)gdtr.pGdt)[val >> X86_SEL_SHIFT_HC];
3458	HWACCMR0DumpDescriptor(pDesc, val, "SS: ");
3459	}
3460
3461	VMXReadVMCS(VMX_VMCS16_HOST_FIELD_TR, &val);
3462	Log(("VMX_VMCS16_HOST_FIELD_TR %08x\n", val));
3463	if (val < gdtr.cbGdt)
3464	{
3465	pDesc = &((PX86DESCHC)gdtr.pGdt)[val >> X86_SEL_SHIFT_HC];
3466	HWACCMR0DumpDescriptor(pDesc, val, "TR: ");
3467	}
3468
3469	VMXReadVMCS(VMX_VMCS_HOST_TR_BASE, &val);
3470	Log(("VMX_VMCS_HOST_TR_BASE %RHv\n", val));
3471
3472	VMXReadVMCS(VMX_VMCS_HOST_GDTR_BASE, &val);
3473	Log(("VMX_VMCS_HOST_GDTR_BASE %RHv\n", val));
3474	VMXReadVMCS(VMX_VMCS_HOST_IDTR_BASE, &val);
3475	Log(("VMX_VMCS_HOST_IDTR_BASE %RHv\n", val));
3476
3477	VMXReadVMCS(VMX_VMCS32_HOST_SYSENTER_CS, &val);
3478	Log(("VMX_VMCS_HOST_SYSENTER_CS %08x\n", val));
3479
3480	VMXReadVMCS(VMX_VMCS_HOST_SYSENTER_EIP, &val);
3481	Log(("VMX_VMCS_HOST_SYSENTER_EIP %RHv\n", val));
3482
3483	VMXReadVMCS(VMX_VMCS_HOST_SYSENTER_ESP, &val);
3484	Log(("VMX_VMCS_HOST_SYSENTER_ESP %RHv\n", val));
3485
3486	VMXReadVMCS(VMX_VMCS_HOST_RSP, &val);
3487	Log(("VMX_VMCS_HOST_RSP %RHv\n", val));
3488	VMXReadVMCS(VMX_VMCS_HOST_RIP, &val);
3489	Log(("VMX_VMCS_HOST_RIP %RHv\n", val));
3490
3491	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3492	if (VMX_IS_64BIT_HOST_MODE())
3493	{
3494	Log(("MSR_K6_EFER = %RX64\n", ASMRdMsr(MSR_K6_EFER)));
3495	Log(("MSR_K6_STAR = %RX64\n", ASMRdMsr(MSR_K6_STAR)));
3496	Log(("MSR_K8_LSTAR = %RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
3497	Log(("MSR_K8_CSTAR = %RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
3498	Log(("MSR_K8_SF_MASK = %RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
3499	}
3500	# endif
3501	#endif /* VBOX_STRICT */
3502	}
3503	break;
3504	}
3505
3506	default:
3507	/* impossible */
3508	AssertMsgFailed(("%Rrc (%#x)\n", rc, rc));
3509	break;
3510	}
3511	}
3512
3513	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3514	/**
3515	* Prepares for and executes VMLAUNCH (64 bits guest mode)
3516	*
3517	* @returns VBox status code
3518	* @param fResume vmlauch/vmresume
3519	* @param pCtx Guest context
3520	* @param pCache VMCS cache
3521	* @param pVM The VM to operate on.
3522	* @param pVCpu The VMCPU to operate on.
3523	*/
3524	DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
3525	{
3526	uint32_t aParam[6];
3527	PHWACCM_CPUINFO pCpu;
3528	RTHCPHYS pPageCpuPhys;
3529	int rc;
3530
3531	pCpu = HWACCMR0GetCurrentCpu();
3532	pPageCpuPhys = RTR0MemObjGetPagePhysAddr(pCpu->pMemObj, 0);
3533
3534	#ifdef DEBUG
3535	pCache->TestIn.pPageCpuPhys = 0;
3536	pCache->TestIn.pVMCSPhys = 0;
3537	pCache->TestIn.pCache = 0;
3538	pCache->TestOut.pVMCSPhys = 0;
3539	pCache->TestOut.pCache = 0;
3540	pCache->TestOut.pCtx = 0;
3541	#endif
3542
3543	aParam[0] = (uint32_t)(pPageCpuPhys); /* Param 1: VMXON physical address - Lo. */
3544	aParam[1] = (uint32_t)(pPageCpuPhys >> 32); /* Param 1: VMXON physical address - Hi. */
3545	aParam[2] = (uint32_t)(pVCpu->hwaccm.s.vmx.pVMCSPhys); /* Param 2: VMCS physical address - Lo. */
3546	aParam[3] = (uint32_t)(pVCpu->hwaccm.s.vmx.pVMCSPhys >> 32); /* Param 2: VMCS physical address - Hi. */
3547	aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hwaccm.s.vmx.VMCSCache);
3548	aParam[5] = 0;
3549
3550	rc = VMXR0Execute64BitsHandler(pVM, pVCpu, pCtx, pVM->hwaccm.s.pfnVMXGCStartVM64, 6, &aParam[0]);
3551
3552	#ifdef DEBUG
3553	AssertMsg(pCache->TestIn.pPageCpuPhys == pPageCpuPhys, ("%RHp vs %RHp\n", pCache->TestIn.pPageCpuPhys, pPageCpuPhys));
3554	AssertMsg(pCache->TestIn.pVMCSPhys == pVCpu->hwaccm.s.vmx.pVMCSPhys, ("%RHp vs %RHp\n", pCache->TestIn.pVMCSPhys, pVCpu->hwaccm.s.vmx.pVMCSPhys));
3555	AssertMsg(pCache->TestIn.pVMCSPhys == pCache->TestOut.pVMCSPhys, ("%RHp vs %RHp\n", pCache->TestIn.pVMCSPhys, pCache->TestOut.pVMCSPhys));
3556	AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache, pCache->TestOut.pCache));
3557	AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hwaccm.s.vmx.VMCSCache), ("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hwaccm.s.vmx.VMCSCache)));
3558	AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx, pCache->TestOut.pCtx));
3559	#endif
3560
3561	return rc;
3562	}
3563
3564	/**
3565	* Executes the specified handler in 64 mode
3566	*
3567	* @returns VBox status code.
3568	* @param pVM The VM to operate on.
3569	* @param pVCpu The VMCPU to operate on.
3570	* @param pCtx Guest context
3571	* @param pfnHandler RC handler
3572	* @param cbParam Number of parameters
3573	* @param paParam Array of 32 bits parameters
3574	*/
3575	VMMR0DECL(int) VMXR0Execute64BitsHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, RTRCPTR pfnHandler, uint32_t cbParam, uint32_t *paParam)
3576	{
3577	int rc, rc2;
3578	RTHCUINTREG uFlags;
3579	PHWACCM_CPUINFO pCpu;
3580	RTHCPHYS pPageCpuPhys;
3581
3582	/* @todo This code is not guest SMP safe (hyper context) */
3583	AssertReturn(pVM->cCPUs == 1, VERR_ACCESS_DENIED);
3584	AssertReturn(pVM->hwaccm.s.pfnHost32ToGuest64R0, VERR_INTERNAL_ERROR);
3585	Assert(pVCpu->hwaccm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hwaccm.s.vmx.VMCSCache.Write.aField));
3586	Assert(pVCpu->hwaccm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hwaccm.s.vmx.VMCSCache.Read.aField));
3587
3588	pCpu = HWACCMR0GetCurrentCpu();
3589	pPageCpuPhys = RTR0MemObjGetPagePhysAddr(pCpu->pMemObj, 0);
3590
3591	/* Clear VM Control Structure. Marking it inactive, clearing implementation specific data and writing back VMCS data to memory. */
3592	VMXClearVMCS(pVCpu->hwaccm.s.vmx.pVMCSPhys);
3593
3594	/* Leave VMX Root Mode. */
3595	VMXDisable();
3596
3597	uFlags = ASMIntDisableFlags();
3598
3599	CPUMSetHyperESP(pVM, VMMGetStackRC(pVM));
3600	CPUMSetHyperEIP(pVM, pfnHandler);
3601	for (int i=(int)cbParam-1;i>=0;i--)
3602	CPUMPushHyper(pVM, paParam[i]);
3603
3604	/* Call switcher. */
3605	rc = pVM->hwaccm.s.pfnHost32ToGuest64R0(pVM);
3606
3607	#ifdef VBOX_STRICT
3608	RTHCUINTREG uFlagsTest = ASMGetFlags();
3609	#endif
3610
3611	ASMSetFlags(uFlags);
3612
3613	/* Make sure the VMX instructions don't cause #UD faults. */
3614	ASMSetCR4(ASMGetCR4() \| X86_CR4_VMXE);
3615
3616	/* Enter VMX Root Mode */
3617	rc2 = VMXEnable(pPageCpuPhys);
3618	if (RT_FAILURE(rc2))
3619	{
3620	if (pVM)
3621	VMXR0CheckError(pVM, pVCpu, rc2);
3622	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
3623	return VERR_VMX_VMXON_FAILED;
3624	}
3625
3626	VMXActivateVMCS(pVCpu->hwaccm.s.vmx.pVMCSPhys);
3627	Assert(!(uFlagsTest & X86_EFL_IF));
3628
3629	return rc;
3630	}
3631
3632	#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL) */
3633
3634
3635	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
3636	/**
3637	* Executes VMWRITE
3638	*
3639	* @returns VBox status code
3640	* @param pVCpu The VMCPU to operate on.
3641	* @param idxField VMCS index
3642	* @param u64Val 16, 32 or 64 bits value
3643	*/
3644	VMMR0DECL(int) VMXWriteVMCS64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
3645	{
3646	int rc;
3647
3648	switch (idxField)
3649	{
3650	case VMX_VMCS_CTRL_TSC_OFFSET_FULL:
3651	case VMX_VMCS_CTRL_IO_BITMAP_A_FULL:
3652	case VMX_VMCS_CTRL_IO_BITMAP_B_FULL:
3653	case VMX_VMCS_CTRL_MSR_BITMAP_FULL:
3654	case VMX_VMCS_CTRL_VMEXIT_MSR_STORE_FULL:
3655	case VMX_VMCS_CTRL_VMEXIT_MSR_LOAD_FULL:
3656	case VMX_VMCS_CTRL_VMENTRY_MSR_LOAD_FULL:
3657	case VMX_VMCS_CTRL_VAPIC_PAGEADDR_FULL:
3658	case VMX_VMCS_GUEST_LINK_PTR_FULL:
3659	case VMX_VMCS_GUEST_PDPTR0_FULL:
3660	case VMX_VMCS_GUEST_PDPTR1_FULL:
3661	case VMX_VMCS_GUEST_PDPTR2_FULL:
3662	case VMX_VMCS_GUEST_PDPTR3_FULL:
3663	case VMX_VMCS_GUEST_DEBUGCTL_FULL:
3664	case VMX_VMCS_GUEST_EFER_FULL:
3665	case VMX_VMCS_CTRL_EPTP_FULL:
3666	/* These fields consist of two parts, which are both writable in 32 bits mode. */
3667	rc = VMXWriteVMCS32(idxField, u64Val);
3668	rc \|= VMXWriteVMCS32(idxField + 1, (uint32_t)(u64Val >> 32ULL));
3669	AssertRC(rc);
3670	return rc;
3671
3672	case VMX_VMCS64_GUEST_LDTR_BASE:
3673	case VMX_VMCS64_GUEST_TR_BASE:
3674	case VMX_VMCS64_GUEST_GDTR_BASE:
3675	case VMX_VMCS64_GUEST_IDTR_BASE:
3676	case VMX_VMCS64_GUEST_SYSENTER_EIP:
3677	case VMX_VMCS64_GUEST_SYSENTER_ESP:
3678	case VMX_VMCS64_GUEST_CR0:
3679	case VMX_VMCS64_GUEST_CR4:
3680	case VMX_VMCS64_GUEST_CR3:
3681	case VMX_VMCS64_GUEST_DR7:
3682	case VMX_VMCS64_GUEST_RIP:
3683	case VMX_VMCS64_GUEST_RSP:
3684	case VMX_VMCS64_GUEST_CS_BASE:
3685	case VMX_VMCS64_GUEST_DS_BASE:
3686	case VMX_VMCS64_GUEST_ES_BASE:
3687	case VMX_VMCS64_GUEST_FS_BASE:
3688	case VMX_VMCS64_GUEST_GS_BASE:
3689	case VMX_VMCS64_GUEST_SS_BASE:
3690	/* Queue a 64 bits value as we can't set it in 32 bits host mode. */
3691	if (u64Val >> 32ULL)
3692	rc = VMXWriteCachedVMCSEx(pVCpu, idxField, u64Val);
3693	else
3694	rc = VMXWriteVMCS32(idxField, (uint32_t)u64Val);
3695
3696	return rc;
3697
3698	default:
3699	AssertMsgFailed(("Unexpected field %x\n", idxField));
3700	return VERR_INVALID_PARAMETER;
3701	}
3702	}
3703
3704	/**
3705	* Cache VMCS writes for performance reasons (Darwin) and for running 64 bits guests on 32 bits hosts.
3706	*
3707	* @param pVCpu The VMCPU to operate on.
3708	* @param idxField VMCS field
3709	* @param u64Val Value
3710	*/
3711	VMMR0DECL(int) VMXWriteCachedVMCSEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
3712	{
3713	PVMCSCACHE pCache = &pVCpu->hwaccm.s.vmx.VMCSCache;
3714
3715	AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1, ("entries=%x\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
3716
3717	/* Make sure there are no duplicates. */
3718	for (unsigned i=0;i<pCache->Write.cValidEntries;i++)
3719	{
3720	if (pCache->Write.aField[i] == idxField)
3721	{
3722	pCache->Write.aFieldVal[i] = u64Val;
3723	return VINF_SUCCESS;
3724	}
3725	}
3726
3727	pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
3728	pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
3729	pCache->Write.cValidEntries++;
3730	return VINF_SUCCESS;
3731	}
3732
3733	#endif /* HC_ARCH_BITS == 32 && !VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0 */

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMR0/HWVMXR0.cpp@ 15440

Download in other formats: