HWVMXR0.cpp@ 15507

Last change on this file since 15507 was 15507, checked in by vboxsync, 16 years ago
VT-x: keep track of missed paging mode changes
Property svn:eol-style set to `native` Property svn:keywords set to `Id`
File size: 147.2 KB

Line
1	/* $Id: HWVMXR0.cpp 15507 2008-12-15 14:51:45Z vboxsync $ */
2	/** @file
3	* HWACCM VMX - Host Context Ring 0.
4	*/
5
6	/*
7	* Copyright (C) 2006-2007 Sun Microsystems, Inc.
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*
17	* Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa
18	* Clara, CA 95054 USA or visit http://www.sun.com if you need
19	* additional information or have any questions.
20	*/
21
22
23	/*******************************************************************************
24	* Header Files *
25	*******************************************************************************/
26	#define LOG_GROUP LOG_GROUP_HWACCM
27	#include <VBox/hwaccm.h>
28	#include "HWACCMInternal.h"
29	#include <VBox/vm.h>
30	#include <VBox/x86.h>
31	#include <VBox/pgm.h>
32	#include <VBox/pdm.h>
33	#include <VBox/err.h>
34	#include <VBox/log.h>
35	#include <VBox/selm.h>
36	#include <VBox/iom.h>
37	#include <iprt/param.h>
38	#include <iprt/assert.h>
39	#include <iprt/asm.h>
40	#include <iprt/string.h>
41	#include "HWVMXR0.h"
42
43	/*******************************************************************************
44	* Defined Constants And Macros *
45	*******************************************************************************/
46	#if defined(RT_ARCH_AMD64)
47	# define VMX_IS_64BIT_HOST_MODE() (true)
48	#elif defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
49	# define VMX_IS_64BIT_HOST_MODE() (g_fVMXIs64bitHost != 0)
50	#else
51	# define VMX_IS_64BIT_HOST_MODE() (false)
52	#endif
53
54	/*******************************************************************************
55	* Global Variables *
56	*******************************************************************************/
57	/* IO operation lookup arrays. */
58	static uint32_t const g_aIOSize[4] = {1, 2, 0, 4};
59	static uint32_t const g_aIOOpAnd[4] = {0xff, 0xffff, 0, 0xffffffff};
60
61	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
62	/** See HWACCMR0A.asm. */
63	extern "C" uint32_t g_fVMXIs64bitHost;
64	#endif
65
66	/*******************************************************************************
67	* Local Functions *
68	*******************************************************************************/
69	static void VMXR0ReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, int rc, PCPUMCTX pCtx);
70	static void vmxR0SetupTLBEPT(PVM pVM, PVMCPU pVCpu);
71	static void vmxR0SetupTLBVPID(PVM pVM, PVMCPU pVCpu);
72	static void vmxR0SetupTLBDummy(PVM pVM, PVMCPU pVCpu);
73	static void vmxR0FlushEPT(PVM pVM, PVMCPU pVCpu, VMX_FLUSH enmFlush, RTGCPHYS GCPhys);
74	static void vmxR0FlushVPID(PVM pVM, PVMCPU pVCpu, VMX_FLUSH enmFlush, RTGCPTR GCPtr);
75	static void vmxR0UpdateExceptionBitmap(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx);
76
77
78	static void VMXR0CheckError(PVM pVM, PVMCPU pVCpu, int rc)
79	{
80	if (rc == VERR_VMX_GENERIC)
81	{
82	RTCCUINTREG instrError;
83
84	VMXReadVMCS(VMX_VMCS32_RO_VM_INSTR_ERROR, &instrError);
85	pVCpu->hwaccm.s.vmx.lasterror.ulInstrError = instrError;
86	}
87	pVM->hwaccm.s.lLastError = rc;
88	}
89
90	/**
91	* Sets up and activates VT-x on the current CPU
92	*
93	* @returns VBox status code.
94	* @param pCpu CPU info struct
95	* @param pVM The VM to operate on. (can be NULL after a resume!!)
96	* @param pvPageCpu Pointer to the global cpu page
97	* @param pPageCpuPhys Physical address of the global cpu page
98	*/
99	VMMR0DECL(int) VMXR0EnableCpu(PHWACCM_CPUINFO pCpu, PVM pVM, void *pvPageCpu, RTHCPHYS pPageCpuPhys)
100	{
101	AssertReturn(pPageCpuPhys, VERR_INVALID_PARAMETER);
102	AssertReturn(pvPageCpu, VERR_INVALID_PARAMETER);
103
104	#ifdef LOG_ENABLED
105	SUPR0Printf("VMXR0EnableCpu cpu %d page (%x) %x\n", pCpu->idCpu, pvPageCpu, (uint32_t)pPageCpuPhys);
106	#endif
107	if (pVM)
108	{
109	/* Set revision dword at the beginning of the VMXON structure. */
110	(uint32_t )pvPageCpu = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hwaccm.s.vmx.msr.vmx_basic_info);
111	}
112
113	/** @todo we should unmap the two pages from the virtual address space in order to prevent accidental corruption.
114	* (which can have very bad consequences!!!)
115	*/
116
117	/* Make sure the VMX instructions don't cause #UD faults. */
118	ASMSetCR4(ASMGetCR4() \| X86_CR4_VMXE);
119
120	/* Enter VMX Root Mode */
121	int rc = VMXEnable(pPageCpuPhys);
122	if (RT_FAILURE(rc))
123	{
124	if (pVM)
125	VMXR0CheckError(pVM, &pVM->aCpus[0], rc);
126	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
127	return VERR_VMX_VMXON_FAILED;
128	}
129	return VINF_SUCCESS;
130	}
131
132	/**
133	* Deactivates VT-x on the current CPU
134	*
135	* @returns VBox status code.
136	* @param pCpu CPU info struct
137	* @param pvPageCpu Pointer to the global cpu page
138	* @param pPageCpuPhys Physical address of the global cpu page
139	*/
140	VMMR0DECL(int) VMXR0DisableCpu(PHWACCM_CPUINFO pCpu, void *pvPageCpu, RTHCPHYS pPageCpuPhys)
141	{
142	AssertReturn(pPageCpuPhys, VERR_INVALID_PARAMETER);
143	AssertReturn(pvPageCpu, VERR_INVALID_PARAMETER);
144
145	/* Leave VMX Root Mode. */
146	VMXDisable();
147
148	/* And clear the X86_CR4_VMXE bit */
149	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
150
151	#ifdef LOG_ENABLED
152	SUPR0Printf("VMXR0DisableCpu cpu %d\n", pCpu->idCpu);
153	#endif
154	return VINF_SUCCESS;
155	}
156
157	/**
158	* Does Ring-0 per VM VT-x init.
159	*
160	* @returns VBox status code.
161	* @param pVM The VM to operate on.
162	*/
163	VMMR0DECL(int) VMXR0InitVM(PVM pVM)
164	{
165	int rc;
166
167	#ifdef LOG_ENABLED
168	SUPR0Printf("VMXR0InitVM %x\n", pVM);
169	#endif
170
171	pVM->hwaccm.s.vmx.pMemObjAPIC = NIL_RTR0MEMOBJ;
172
173	if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
174	{
175	/* Allocate one page for the virtual APIC mmio cache. */
176	rc = RTR0MemObjAllocCont(&pVM->hwaccm.s.vmx.pMemObjAPIC, 1 << PAGE_SHIFT, true /* executable R0 mapping */);
177	AssertRC(rc);
178	if (RT_FAILURE(rc))
179	return rc;
180
181	pVM->hwaccm.s.vmx.pAPIC = (uint8_t *)RTR0MemObjAddress(pVM->hwaccm.s.vmx.pMemObjAPIC);
182	pVM->hwaccm.s.vmx.pAPICPhys = RTR0MemObjGetPagePhysAddr(pVM->hwaccm.s.vmx.pMemObjAPIC, 0);
183	ASMMemZero32(pVM->hwaccm.s.vmx.pAPIC, PAGE_SIZE);
184	}
185	else
186	{
187	pVM->hwaccm.s.vmx.pMemObjAPIC = 0;
188	pVM->hwaccm.s.vmx.pAPIC = 0;
189	pVM->hwaccm.s.vmx.pAPICPhys = 0;
190	}
191
192	/* Allocate the MSR bitmap if this feature is supported. */
193	if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
194	{
195	rc = RTR0MemObjAllocCont(&pVM->hwaccm.s.vmx.pMemObjMSRBitmap, 1 << PAGE_SHIFT, true /* executable R0 mapping */);
196	AssertRC(rc);
197	if (RT_FAILURE(rc))
198	return rc;
199
200	pVM->hwaccm.s.vmx.pMSRBitmap = (uint8_t *)RTR0MemObjAddress(pVM->hwaccm.s.vmx.pMemObjMSRBitmap);
201	pVM->hwaccm.s.vmx.pMSRBitmapPhys = RTR0MemObjGetPagePhysAddr(pVM->hwaccm.s.vmx.pMemObjMSRBitmap, 0);
202	memset(pVM->hwaccm.s.vmx.pMSRBitmap, 0xff, PAGE_SIZE);
203	}
204
205	/* Allocate VMCBs for all guest CPUs. */
206	for (unsigned i=0;i<pVM->cCPUs;i++)
207	{
208	PVMCPU pVCpu = &pVM->aCpus[i];
209
210	pVCpu->hwaccm.s.vmx.pMemObjVMCS = NIL_RTR0MEMOBJ;
211
212	/* Allocate one page for the VM control structure (VMCS). */
213	rc = RTR0MemObjAllocCont(&pVCpu->hwaccm.s.vmx.pMemObjVMCS, 1 << PAGE_SHIFT, true /* executable R0 mapping */);
214	AssertRC(rc);
215	if (RT_FAILURE(rc))
216	return rc;
217
218	pVCpu->hwaccm.s.vmx.pVMCS = RTR0MemObjAddress(pVCpu->hwaccm.s.vmx.pMemObjVMCS);
219	pVCpu->hwaccm.s.vmx.pVMCSPhys = RTR0MemObjGetPagePhysAddr(pVCpu->hwaccm.s.vmx.pMemObjVMCS, 0);
220	ASMMemZero32(pVCpu->hwaccm.s.vmx.pVMCS, PAGE_SIZE);
221
222	pVCpu->hwaccm.s.vmx.cr0_mask = 0;
223	pVCpu->hwaccm.s.vmx.cr4_mask = 0;
224
225	/* Current guest paging mode. */
226	pVCpu->hwaccm.s.vmx.enmLastSeenGuestMode = PGMMODE_REAL;
227
228	#ifdef LOG_ENABLED
229	SUPR0Printf("VMXR0InitVM %x VMCS=%x (%x)\n", pVM, pVCpu->hwaccm.s.vmx.pVMCS, (uint32_t)pVCpu->hwaccm.s.vmx.pVMCSPhys);
230	#endif
231	}
232
233	return VINF_SUCCESS;
234	}
235
236	/**
237	* Does Ring-0 per VM VT-x termination.
238	*
239	* @returns VBox status code.
240	* @param pVM The VM to operate on.
241	*/
242	VMMR0DECL(int) VMXR0TermVM(PVM pVM)
243	{
244	for (unsigned i=0;i<pVM->cCPUs;i++)
245	{
246	if (pVM->aCpus[i].hwaccm.s.vmx.pMemObjVMCS != NIL_RTR0MEMOBJ)
247	{
248	RTR0MemObjFree(pVM->aCpus[i].hwaccm.s.vmx.pMemObjVMCS, false);
249	pVM->aCpus[i].hwaccm.s.vmx.pMemObjVMCS = NIL_RTR0MEMOBJ;
250	pVM->aCpus[i].hwaccm.s.vmx.pVMCS = 0;
251	pVM->aCpus[i].hwaccm.s.vmx.pVMCSPhys = 0;
252	}
253	}
254	if (pVM->hwaccm.s.vmx.pMemObjAPIC != NIL_RTR0MEMOBJ)
255	{
256	RTR0MemObjFree(pVM->hwaccm.s.vmx.pMemObjAPIC, false);
257	pVM->hwaccm.s.vmx.pMemObjAPIC = NIL_RTR0MEMOBJ;
258	pVM->hwaccm.s.vmx.pAPIC = 0;
259	pVM->hwaccm.s.vmx.pAPICPhys = 0;
260	}
261	if (pVM->hwaccm.s.vmx.pMemObjMSRBitmap != NIL_RTR0MEMOBJ)
262	{
263	RTR0MemObjFree(pVM->hwaccm.s.vmx.pMemObjMSRBitmap, false);
264	pVM->hwaccm.s.vmx.pMemObjMSRBitmap = NIL_RTR0MEMOBJ;
265	pVM->hwaccm.s.vmx.pMSRBitmap = 0;
266	pVM->hwaccm.s.vmx.pMSRBitmapPhys = 0;
267	}
268	return VINF_SUCCESS;
269	}
270
271	/**
272	* Sets up VT-x for the specified VM
273	*
274	* @returns VBox status code.
275	* @param pVM The VM to operate on.
276	*/
277	VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
278	{
279	int rc = VINF_SUCCESS;
280	uint32_t val;
281
282	AssertReturn(pVM, VERR_INVALID_PARAMETER);
283
284	for (unsigned i=0;i<pVM->cCPUs;i++)
285	{
286	PVMCPU pVCpu = &pVM->aCpus[i];
287
288	Assert(pVCpu->hwaccm.s.vmx.pVMCS);
289
290	/* Set revision dword at the beginning of the VMCS structure. */
291	(uint32_t )pVCpu->hwaccm.s.vmx.pVMCS = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hwaccm.s.vmx.msr.vmx_basic_info);
292
293	/* Clear VM Control Structure. */
294	Log(("pVMCSPhys = %RHp\n", pVCpu->hwaccm.s.vmx.pVMCSPhys));
295	rc = VMXClearVMCS(pVCpu->hwaccm.s.vmx.pVMCSPhys);
296	if (RT_FAILURE(rc))
297	goto vmx_end;
298
299	/* Activate the VM Control Structure. */
300	rc = VMXActivateVMCS(pVCpu->hwaccm.s.vmx.pVMCSPhys);
301	if (RT_FAILURE(rc))
302	goto vmx_end;
303
304	/* VMX_VMCS_CTRL_PIN_EXEC_CONTROLS
305	* Set required bits to one and zero according to the MSR capabilities.
306	*/
307	val = pVM->hwaccm.s.vmx.msr.vmx_pin_ctls.n.disallowed0;
308	/* External and non-maskable interrupts cause VM-exits. */
309	val = val \| VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_EXT_INT_EXIT \| VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_NMI_EXIT;
310	val &= pVM->hwaccm.s.vmx.msr.vmx_pin_ctls.n.allowed1;
311
312	rc = VMXWriteVMCS(VMX_VMCS_CTRL_PIN_EXEC_CONTROLS, val);
313	AssertRC(rc);
314
315	/* VMX_VMCS_CTRL_PROC_EXEC_CONTROLS
316	* Set required bits to one and zero according to the MSR capabilities.
317	*/
318	val = pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.disallowed0;
319	/* Program which event cause VM-exits and which features we want to use. */
320	val = val \| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_HLT_EXIT
321	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_TSC_OFFSET
322	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT
323	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_UNCOND_IO_EXIT
324	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MWAIT_EXIT; /* don't execute mwait or else we'll idle inside the guest (host thinks the cpu load is high) */
325
326	/* Without nested paging we should intercept invlpg and cr3 mov instructions. */
327	if (!pVM->hwaccm.s.fNestedPaging)
328	val \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INVLPG_EXIT
329	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
330	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT;
331
332	/* Note: VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MWAIT_EXIT might cause a vmlaunch failure with an invalid control fields error. (combined with some other exit reasons) */
333	if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
334	{
335	/* CR8 reads from the APIC shadow page; writes cause an exit is they lower the TPR below the threshold */
336	val \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW;
337	Assert(pVM->hwaccm.s.vmx.pAPIC);
338	}
339	else
340	/* Exit on CR8 reads & writes in case the TPR shadow feature isn't present. */
341	val \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR8_STORE_EXIT \| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR8_LOAD_EXIT;
342
343	#ifdef VBOX_WITH_VTX_MSR_BITMAPS
344	if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
345	{
346	Assert(pVM->hwaccm.s.vmx.pMSRBitmapPhys);
347	val \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS;
348	}
349	#endif
350
351	/* We will use the secondary control if it's present. */
352	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
353
354	/* Mask away the bits that the CPU doesn't support */
355	/** @todo make sure they don't conflict with the above requirements. */
356	val &= pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1;
357	pVCpu->hwaccm.s.vmx.proc_ctls = val;
358
359	rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, val);
360	AssertRC(rc);
361
362	if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
363	{
364	/* VMX_VMCS_CTRL_PROC_EXEC_CONTROLS2
365	* Set required bits to one and zero according to the MSR capabilities.
366	*/
367	val = pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2.n.disallowed0;
368	val \|= VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT;
369
370	#ifdef HWACCM_VTX_WITH_EPT
371	if (pVM->hwaccm.s.fNestedPaging)
372	val \|= VMX_VMCS_CTRL_PROC_EXEC2_EPT;
373	#endif /* HWACCM_VTX_WITH_EPT */
374	#ifdef HWACCM_VTX_WITH_VPID
375	else
376	if (pVM->hwaccm.s.vmx.fVPID)
377	val \|= VMX_VMCS_CTRL_PROC_EXEC2_VPID;
378	#endif /* HWACCM_VTX_WITH_VPID */
379
380	/* Mask away the bits that the CPU doesn't support */
381	/** @todo make sure they don't conflict with the above requirements. */
382	val &= pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2.n.allowed1;
383
384	rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS2, val);
385	AssertRC(rc);
386	}
387
388	/* VMX_VMCS_CTRL_CR3_TARGET_COUNT
389	* Set required bits to one and zero according to the MSR capabilities.
390	*/
391	rc = VMXWriteVMCS(VMX_VMCS_CTRL_CR3_TARGET_COUNT, 0);
392	AssertRC(rc);
393
394	/* Forward all exception except #NM & #PF to the guest.
395	* We always need to check pagefaults since our shadow page table can be out of sync.
396	* And we always lazily sync the FPU & XMM state.
397	*/
398
399	/** @todo Possible optimization:
400	* Keep the FPU and XMM state current in the EM thread. That way there's no need to
401	* lazily sync anything, but the downside is that we can't use the FPU stack or XMM
402	* registers ourselves of course.
403	*
404	* Note: only possible if the current state is actually ours (X86_CR0_TS flag)
405	*/
406
407	/* Don't filter page faults; all of them should cause a switch. */
408	rc = VMXWriteVMCS(VMX_VMCS_CTRL_PAGEFAULT_ERROR_MASK, 0);
409	rc \|= VMXWriteVMCS(VMX_VMCS_CTRL_PAGEFAULT_ERROR_MATCH, 0);
410	AssertRC(rc);
411
412	/* Init TSC offset to zero. */
413	rc = VMXWriteVMCS64(VMX_VMCS_CTRL_TSC_OFFSET_FULL, 0);
414	AssertRC(rc);
415
416	rc = VMXWriteVMCS64(VMX_VMCS_CTRL_IO_BITMAP_A_FULL, 0);
417	AssertRC(rc);
418
419	rc = VMXWriteVMCS64(VMX_VMCS_CTRL_IO_BITMAP_B_FULL, 0);
420	AssertRC(rc);
421
422	/* Set the MSR bitmap address. */
423	if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
424	{
425	/* Optional */
426	rc = VMXWriteVMCS64(VMX_VMCS_CTRL_MSR_BITMAP_FULL, pVM->hwaccm.s.vmx.pMSRBitmapPhys);
427	AssertRC(rc);
428	}
429
430	/* Clear MSR controls. */
431	rc = VMXWriteVMCS64(VMX_VMCS_CTRL_VMEXIT_MSR_STORE_FULL, 0);
432	rc \|= VMXWriteVMCS64(VMX_VMCS_CTRL_VMEXIT_MSR_LOAD_FULL, 0);
433	rc \|= VMXWriteVMCS64(VMX_VMCS_CTRL_VMENTRY_MSR_LOAD_FULL, 0);
434	rc \|= VMXWriteVMCS(VMX_VMCS_CTRL_EXIT_MSR_STORE_COUNT, 0);
435	rc \|= VMXWriteVMCS(VMX_VMCS_CTRL_EXIT_MSR_LOAD_COUNT, 0);
436	AssertRC(rc);
437
438	if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
439	{
440	Assert(pVM->hwaccm.s.vmx.pMemObjAPIC);
441	/* Optional */
442	rc = VMXWriteVMCS(VMX_VMCS_CTRL_TPR_THRESHOLD, 0);
443	rc \|= VMXWriteVMCS64(VMX_VMCS_CTRL_VAPIC_PAGEADDR_FULL, pVM->hwaccm.s.vmx.pAPICPhys);
444	AssertRC(rc);
445	}
446
447	/* Set link pointer to -1. Not currently used. */
448	rc = VMXWriteVMCS64(VMX_VMCS_GUEST_LINK_PTR_FULL, 0xFFFFFFFFFFFFFFFFULL);
449	AssertRC(rc);
450
451	/* Clear VM Control Structure. Marking it inactive, clearing implementation specific data and writing back VMCS data to memory. */
452	rc = VMXClearVMCS(pVCpu->hwaccm.s.vmx.pVMCSPhys);
453	AssertRC(rc);
454
455	/* Configure the VMCS read cache. */
456	PVMCSCACHE pCache = &pVCpu->hwaccm.s.vmx.VMCSCache;
457
458	VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_RIP);
459	VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_RSP);
460	VMXSetupCachedReadVMCS(pCache, VMX_VMCS_GUEST_RFLAGS);
461	VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE);
462	VMXSetupCachedReadVMCS(pCache, VMX_VMCS_CTRL_CR0_READ_SHADOW);
463	VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_CR0);
464	VMXSetupCachedReadVMCS(pCache, VMX_VMCS_CTRL_CR4_READ_SHADOW);
465	VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_CR4);
466	VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_DR7);
467	VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_GUEST_SYSENTER_CS);
468	VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_SYSENTER_EIP);
469	VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_SYSENTER_ESP);
470	VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_GUEST_GDTR_LIMIT);
471	VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_GDTR_BASE);
472	VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_GUEST_IDTR_LIMIT);
473	VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_IDTR_BASE);
474
475	VMX_SETUP_SELREG(ES, pCache);
476	VMX_SETUP_SELREG(SS, pCache);
477	VMX_SETUP_SELREG(CS, pCache);
478	VMX_SETUP_SELREG(DS, pCache);
479	VMX_SETUP_SELREG(FS, pCache);
480	VMX_SETUP_SELREG(GS, pCache);
481	VMX_SETUP_SELREG(LDTR, pCache);
482	VMX_SETUP_SELREG(TR, pCache);
483
484	/* Status code VMCS reads. */
485	VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_EXIT_REASON);
486	VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_VM_INSTR_ERROR);
487	VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_EXIT_INSTR_LENGTH);
488	VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_EXIT_INTERRUPTION_ERRCODE);
489	VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO);
490	VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_EXIT_INSTR_INFO);
491	VMXSetupCachedReadVMCS(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
492	VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_IDT_INFO);
493	VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_IDT_ERRCODE);
494
495	if (pVM->hwaccm.s.fNestedPaging)
496	{
497	VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_CR3);
498	VMXSetupCachedReadVMCS(pCache, VMX_VMCS_EXIT_PHYS_ADDR_FULL);
499	pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
500	}
501	else
502	pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
503	} /* for each VMCPU */
504
505	/* Choose the right TLB setup function. */
506	if (pVM->hwaccm.s.fNestedPaging)
507	{
508	pVM->hwaccm.s.vmx.pfnSetupTaggedTLB = vmxR0SetupTLBEPT;
509
510	/* Default values for flushing. */
511	pVM->hwaccm.s.vmx.enmFlushPage = VMX_FLUSH_ALL_CONTEXTS;
512	pVM->hwaccm.s.vmx.enmFlushContext = VMX_FLUSH_ALL_CONTEXTS;
513
514	/* If the capabilities specify we can do more, then make use of it. */
515	if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVEPT_CAPS_INDIV)
516	pVM->hwaccm.s.vmx.enmFlushPage = VMX_FLUSH_PAGE;
517	else
518	if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVEPT_CAPS_CONTEXT)
519	pVM->hwaccm.s.vmx.enmFlushPage = VMX_FLUSH_SINGLE_CONTEXT;
520
521	if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVEPT_CAPS_CONTEXT)
522	pVM->hwaccm.s.vmx.enmFlushContext = VMX_FLUSH_SINGLE_CONTEXT;
523	}
524	#ifdef HWACCM_VTX_WITH_VPID
525	else
526	if (pVM->hwaccm.s.vmx.fVPID)
527	{
528	pVM->hwaccm.s.vmx.pfnSetupTaggedTLB = vmxR0SetupTLBVPID;
529
530	/* Default values for flushing. */
531	pVM->hwaccm.s.vmx.enmFlushPage = VMX_FLUSH_ALL_CONTEXTS;
532	pVM->hwaccm.s.vmx.enmFlushContext = VMX_FLUSH_ALL_CONTEXTS;
533
534	/* If the capabilities specify we can do more, then make use of it. */
535	if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVVPID_CAPS_INDIV)
536	pVM->hwaccm.s.vmx.enmFlushPage = VMX_FLUSH_PAGE;
537	else
538	if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVVPID_CAPS_CONTEXT)
539	pVM->hwaccm.s.vmx.enmFlushPage = VMX_FLUSH_SINGLE_CONTEXT;
540
541	if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVVPID_CAPS_CONTEXT)
542	pVM->hwaccm.s.vmx.enmFlushContext = VMX_FLUSH_SINGLE_CONTEXT;
543	}
544	#endif /* HWACCM_VTX_WITH_VPID */
545	else
546	pVM->hwaccm.s.vmx.pfnSetupTaggedTLB = vmxR0SetupTLBDummy;
547
548	vmx_end:
549	VMXR0CheckError(pVM, &pVM->aCpus[0], rc);
550	return rc;
551	}
552
553
554	/**
555	* Injects an event (trap or external interrupt)
556	*
557	* @returns VBox status code.
558	* @param pVM The VM to operate on.
559	* @param pVCpu The VMCPU to operate on.
560	* @param pCtx CPU Context
561	* @param intInfo VMX interrupt info
562	* @param cbInstr Opcode length of faulting instruction
563	* @param errCode Error code (optional)
564	*/
565	static int VMXR0InjectEvent(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, uint32_t intInfo, uint32_t cbInstr, uint32_t errCode)
566	{
567	int rc;
568	uint32_t iGate = VMX_EXIT_INTERRUPTION_INFO_VECTOR(intInfo);
569
570	#ifdef VBOX_STRICT
571	if (iGate == 0xE)
572	LogFlow(("VMXR0InjectEvent: Injecting interrupt %d at %RGv error code=%08x CR2=%08x intInfo=%08x\n", iGate, (RTGCPTR)pCtx->rip, errCode, pCtx->cr2, intInfo));
573	else
574	if (iGate < 0x20)
575	LogFlow(("VMXR0InjectEvent: Injecting interrupt %d at %RGv error code=%08x\n", iGate, (RTGCPTR)pCtx->rip, errCode));
576	else
577	{
578	LogFlow(("INJ-EI: %x at %RGv\n", iGate, (RTGCPTR)pCtx->rip));
579	Assert(!VM_FF_ISSET(pVM, VM_FF_INHIBIT_INTERRUPTS));
580	Assert(pCtx->eflags.u32 & X86_EFL_IF);
581	}
582	#endif
583
584	#ifdef HWACCM_VMX_EMULATE_REALMODE
585	if (CPUMIsGuestInRealModeEx(pCtx))
586	{
587	RTGCPHYS GCPhysHandler;
588	uint16_t offset, ip;
589	RTSEL sel;
590
591	/* Injecting events doesn't work right with real mode emulation.
592	* (#GP if we try to inject external hardware interrupts)
593	* Inject the interrupt or trap directly instead.
594	*/
595	Log(("Manual interrupt/trap '%x' inject (real mode)\n", iGate));
596
597	/* Check if the interrupt handler is present. */
598	if (iGate * 4 + 3 > pCtx->idtr.cbIdt)
599	{
600	Log(("IDT cbIdt violation\n"));
601	if (iGate != X86_XCPT_DF)
602	{
603	RTGCUINTPTR intInfo;
604
605	intInfo = (iGate == X86_XCPT_GP) ? (uint32_t)X86_XCPT_DF : iGate;
606	intInfo \|= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
607	intInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
608	intInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HWEXCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
609
610	return VMXR0InjectEvent(pVM, pVCpu, pCtx, intInfo, 0, 0 /* no error code according to the Intel docs */);
611	}
612	Log(("Triple fault -> reset the VM!\n"));
613	return VINF_EM_RESET;
614	}
615	if ( VMX_EXIT_INTERRUPTION_INFO_TYPE(intInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW
616	\|\| iGate == 3 /* Both #BP and #OF point to the instruction after. */
617	\|\| iGate == 4)
618	{
619	ip = pCtx->ip + cbInstr;
620	}
621	else
622	ip = pCtx->ip;
623
624	/* Read the selector:offset pair of the interrupt handler. */
625	GCPhysHandler = (RTGCPHYS)pCtx->idtr.pIdt + iGate * 4;
626	PGMPhysRead(pVM, GCPhysHandler, &offset, sizeof(offset));
627	PGMPhysRead(pVM, GCPhysHandler + 2, &sel, sizeof(sel));
628
629	LogFlow(("IDT handler %04X:%04X\n", sel, offset));
630
631	/* Construct the stack frame. */
632	/** @todo should check stack limit. */
633	pCtx->sp -= 2;
634	LogFlow(("ss:sp %04X:%04X eflags=%x\n", pCtx->ss, pCtx->sp, pCtx->eflags.u));
635	PGMPhysWrite(pVM, pCtx->ssHid.u64Base + pCtx->sp, &pCtx->eflags, sizeof(uint16_t));
636	pCtx->sp -= 2;
637	LogFlow(("ss:sp %04X:%04X cs=%x\n", pCtx->ss, pCtx->sp, pCtx->cs));
638	PGMPhysWrite(pVM, pCtx->ssHid.u64Base + pCtx->sp, &pCtx->cs, sizeof(uint16_t));
639	pCtx->sp -= 2;
640	LogFlow(("ss:sp %04X:%04X ip=%x\n", pCtx->ss, pCtx->sp, ip));
641	PGMPhysWrite(pVM, pCtx->ssHid.u64Base + pCtx->sp, &ip, sizeof(ip));
642
643	/* Update the CPU state for executing the handler. */
644	pCtx->rip = offset;
645	pCtx->cs = sel;
646	pCtx->csHid.u64Base = sel << 4;
647	pCtx->eflags.u &= ~(X86_EFL_IF\|X86_EFL_TF\|X86_EFL_RF\|X86_EFL_AC);
648
649	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_GUEST_SEGMENT_REGS;
650	return VINF_SUCCESS;
651	}
652	#endif /* HWACCM_VMX_EMULATE_REALMODE */
653
654	/* Set event injection state. */
655	rc = VMXWriteVMCS(VMX_VMCS_CTRL_ENTRY_IRQ_INFO, intInfo \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT));
656
657	rc \|= VMXWriteVMCS(VMX_VMCS_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
658	rc \|= VMXWriteVMCS(VMX_VMCS_CTRL_ENTRY_EXCEPTION_ERRCODE, errCode);
659
660	AssertRC(rc);
661	return rc;
662	}
663
664
665	/**
666	* Checks for pending guest interrupts and injects them
667	*
668	* @returns VBox status code.
669	* @param pVM The VM to operate on.
670	* @param pVCpu The VMCPU to operate on.
671	* @param pCtx CPU Context
672	*/
673	static int VMXR0CheckPendingInterrupt(PVM pVM, PVMCPU pVCpu, CPUMCTX *pCtx)
674	{
675	int rc;
676
677	/* Dispatch any pending interrupts. (injected before, but a VM exit occurred prematurely) */
678	if (pVCpu->hwaccm.s.Event.fPending)
679	{
680	Log(("Reinjecting event %RX64 %08x at %RGv cr2=%RX64\n", pVCpu->hwaccm.s.Event.intInfo, pVCpu->hwaccm.s.Event.errCode, (RTGCPTR)pCtx->rip, pCtx->cr2));
681	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatIntReinject);
682	rc = VMXR0InjectEvent(pVM, pVCpu, pCtx, pVCpu->hwaccm.s.Event.intInfo, 0, pVCpu->hwaccm.s.Event.errCode);
683	AssertRC(rc);
684
685	pVCpu->hwaccm.s.Event.fPending = false;
686	return VINF_SUCCESS;
687	}
688
689	if (pVM->hwaccm.s.fInjectNMI)
690	{
691	RTGCUINTPTR intInfo;
692
693	intInfo = X86_XCPT_NMI;
694	intInfo \|= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
695	intInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
696
697	rc = VMXR0InjectEvent(pVM, pVCpu, pCtx, intInfo, 0, 0);
698	AssertRC(rc);
699
700	pVM->hwaccm.s.fInjectNMI = false;
701	return VINF_SUCCESS;
702	}
703
704	/* When external interrupts are pending, we should exit the VM when IF is set. */
705	if ( !TRPMHasTrap(pVM)
706	&& VM_FF_ISPENDING(pVM, (VM_FF_INTERRUPT_APIC\|VM_FF_INTERRUPT_PIC)))
707	{
708	if (!(pCtx->eflags.u32 & X86_EFL_IF))
709	{
710	if (!(pVCpu->hwaccm.s.vmx.proc_ctls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_IRQ_WINDOW_EXIT))
711	{
712	LogFlow(("Enable irq window exit!\n"));
713	pVCpu->hwaccm.s.vmx.proc_ctls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_IRQ_WINDOW_EXIT;
714	rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
715	AssertRC(rc);
716	}
717	/* else nothing to do but wait */
718	}
719	else
720	if (!VM_FF_ISSET(pVM, VM_FF_INHIBIT_INTERRUPTS))
721	{
722	uint8_t u8Interrupt;
723
724	rc = PDMGetInterrupt(pVM, &u8Interrupt);
725	Log(("Dispatch interrupt: u8Interrupt=%x (%d) rc=%Rrc cs:rip=%04X:%RGv\n", u8Interrupt, u8Interrupt, rc, pCtx->cs, (RTGCPTR)pCtx->rip));
726	if (RT_SUCCESS(rc))
727	{
728	rc = TRPMAssertTrap(pVM, u8Interrupt, TRPM_HARDWARE_INT);
729	AssertRC(rc);
730	}
731	else
732	{
733	/* Can only happen in rare cases where a pending interrupt is cleared behind our back */
734	Assert(!VM_FF_ISPENDING(pVM, (VM_FF_INTERRUPT_APIC\|VM_FF_INTERRUPT_PIC)));
735	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatSwitchGuestIrq);
736	/* Just continue */
737	}
738	}
739	else
740	Log(("Pending interrupt blocked at %RGv by VM_FF_INHIBIT_INTERRUPTS!!\n", (RTGCPTR)pCtx->rip));
741	}
742
743	#ifdef VBOX_STRICT
744	if (TRPMHasTrap(pVM))
745	{
746	uint8_t u8Vector;
747	rc = TRPMQueryTrapAll(pVM, &u8Vector, 0, 0, 0);
748	AssertRC(rc);
749	}
750	#endif
751
752	if ( pCtx->eflags.u32 & X86_EFL_IF
753	&& (!VM_FF_ISSET(pVM, VM_FF_INHIBIT_INTERRUPTS))
754	&& TRPMHasTrap(pVM)
755	)
756	{
757	uint8_t u8Vector;
758	int rc;
759	TRPMEVENT enmType;
760	RTGCUINTPTR intInfo;
761	RTGCUINT errCode;
762
763	/* If a new event is pending, then dispatch it now. */
764	rc = TRPMQueryTrapAll(pVM, &u8Vector, &enmType, &errCode, 0);
765	AssertRC(rc);
766	Assert(pCtx->eflags.Bits.u1IF == 1 \|\| enmType == TRPM_TRAP);
767	Assert(enmType != TRPM_SOFTWARE_INT);
768
769	/* Clear the pending trap. */
770	rc = TRPMResetTrap(pVM);
771	AssertRC(rc);
772
773	intInfo = u8Vector;
774	intInfo \|= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
775
776	if (enmType == TRPM_TRAP)
777	{
778	switch (u8Vector) {
779	case 8:
780	case 10:
781	case 11:
782	case 12:
783	case 13:
784	case 14:
785	case 17:
786	/* Valid error codes. */
787	intInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
788	break;
789	default:
790	break;
791	}
792	if (u8Vector == X86_XCPT_BP \|\| u8Vector == X86_XCPT_OF)
793	intInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SWEXCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
794	else
795	intInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HWEXCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
796	}
797	else
798	intInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
799
800	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatIntInject);
801	rc = VMXR0InjectEvent(pVM, pVCpu, pCtx, intInfo, 0, errCode);
802	AssertRC(rc);
803	} /* if (interrupts can be dispatched) */
804
805	return VINF_SUCCESS;
806	}
807
808	/**
809	* Save the host state
810	*
811	* @returns VBox status code.
812	* @param pVM The VM to operate on.
813	* @param pVCpu The VMCPU to operate on.
814	*/
815	VMMR0DECL(int) VMXR0SaveHostState(PVM pVM, PVMCPU pVCpu)
816	{
817	int rc = VINF_SUCCESS;
818
819	/*
820	* Host CPU Context
821	*/
822	if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_HOST_CONTEXT)
823	{
824	RTIDTR idtr;
825	RTGDTR gdtr;
826	RTSEL SelTR;
827	PX86DESCHC pDesc;
828	uintptr_t trBase;
829	RTSEL cs;
830	RTSEL ss;
831	uint64_t cr3;
832
833	/* Control registers */
834	rc = VMXWriteVMCS(VMX_VMCS_HOST_CR0, ASMGetCR0());
835	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
836	if (VMX_IS_64BIT_HOST_MODE())
837	{
838	cr3 = hwaccmR0Get64bitCR3();
839	rc \|= VMXWriteVMCS64(VMX_VMCS_HOST_CR3, cr3);
840	}
841	else
842	#endif
843	{
844	cr3 = ASMGetCR3();
845	rc \|= VMXWriteVMCS(VMX_VMCS_HOST_CR3, cr3);
846	}
847	rc \|= VMXWriteVMCS(VMX_VMCS_HOST_CR4, ASMGetCR4());
848	AssertRC(rc);
849	Log2(("VMX_VMCS_HOST_CR0 %08x\n", ASMGetCR0()));
850	Log2(("VMX_VMCS_HOST_CR3 %08RX64\n", cr3));
851	Log2(("VMX_VMCS_HOST_CR4 %08x\n", ASMGetCR4()));
852
853	/* Selector registers. */
854	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
855	if (VMX_IS_64BIT_HOST_MODE())
856	{
857	cs = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelCS;
858	ss = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelSS;
859	}
860	else
861	{
862	/* sysenter loads LDT cs & ss, VMX doesn't like this. Load the GDT ones (safe). */
863	cs = (RTSEL)(uintptr_t)&SUPR0AbsKernelCS;
864	ss = (RTSEL)(uintptr_t)&SUPR0AbsKernelSS;
865	}
866	#else
867	cs = ASMGetCS();
868	ss = ASMGetSS();
869	#endif
870	Assert(!(cs & X86_SEL_LDT)); Assert((cs & X86_SEL_RPL) == 0);
871	Assert(!(ss & X86_SEL_LDT)); Assert((ss & X86_SEL_RPL) == 0);
872	rc = VMXWriteVMCS(VMX_VMCS16_HOST_FIELD_CS, cs);
873	/* Note: VMX is (again) very picky about the RPL of the selectors here; we'll restore them manually. */
874	rc \|= VMXWriteVMCS(VMX_VMCS16_HOST_FIELD_DS, 0);
875	rc \|= VMXWriteVMCS(VMX_VMCS16_HOST_FIELD_ES, 0);
876	#if HC_ARCH_BITS == 32
877	if (!VMX_IS_64BIT_HOST_MODE())
878	{
879	rc \|= VMXWriteVMCS(VMX_VMCS16_HOST_FIELD_FS, 0);
880	rc \|= VMXWriteVMCS(VMX_VMCS16_HOST_FIELD_GS, 0);
881	}
882	#endif
883	rc \|= VMXWriteVMCS(VMX_VMCS16_HOST_FIELD_SS, ss);
884	SelTR = ASMGetTR();
885	rc \|= VMXWriteVMCS(VMX_VMCS16_HOST_FIELD_TR, SelTR);
886	AssertRC(rc);
887	Log2(("VMX_VMCS_HOST_FIELD_CS %08x (%08x)\n", cs, ASMGetSS()));
888	Log2(("VMX_VMCS_HOST_FIELD_DS 00000000 (%08x)\n", ASMGetDS()));
889	Log2(("VMX_VMCS_HOST_FIELD_ES 00000000 (%08x)\n", ASMGetES()));
890	Log2(("VMX_VMCS_HOST_FIELD_FS 00000000 (%08x)\n", ASMGetFS()));
891	Log2(("VMX_VMCS_HOST_FIELD_GS 00000000 (%08x)\n", ASMGetGS()));
892	Log2(("VMX_VMCS_HOST_FIELD_SS %08x (%08x)\n", ss, ASMGetSS()));
893	Log2(("VMX_VMCS_HOST_FIELD_TR %08x\n", ASMGetTR()));
894
895	/* GDTR & IDTR */
896	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
897	if (VMX_IS_64BIT_HOST_MODE())
898	{
899	X86XDTR64 gdtr64, idtr64;
900	hwaccmR0Get64bitGDTRandIDTR(&gdtr64, &idtr64);
901	rc = VMXWriteVMCS64(VMX_VMCS_HOST_GDTR_BASE, gdtr64.uAddr);
902	rc \|= VMXWriteVMCS64(VMX_VMCS_HOST_IDTR_BASE, gdtr64.uAddr);
903	AssertRC(rc);
904	Log2(("VMX_VMCS_HOST_GDTR_BASE %RX64\n", gdtr64.uAddr));
905	Log2(("VMX_VMCS_HOST_IDTR_BASE %RX64\n", idtr64.uAddr));
906	gdtr.cbGdt = gdtr64.cb;
907	gdtr.pGdt = (uintptr_t)gdtr64.uAddr;
908	}
909	else
910	#endif
911	{
912	ASMGetGDTR(&gdtr);
913	rc = VMXWriteVMCS(VMX_VMCS_HOST_GDTR_BASE, gdtr.pGdt);
914	ASMGetIDTR(&idtr);
915	rc \|= VMXWriteVMCS(VMX_VMCS_HOST_IDTR_BASE, idtr.pIdt);
916	AssertRC(rc);
917	Log2(("VMX_VMCS_HOST_GDTR_BASE %RHv\n", gdtr.pGdt));
918	Log2(("VMX_VMCS_HOST_IDTR_BASE %RHv\n", idtr.pIdt));
919	}
920
921
922	/* Save the base address of the TR selector. */
923	if (SelTR > gdtr.cbGdt)
924	{
925	AssertMsgFailed(("Invalid TR selector %x. GDTR.cbGdt=%x\n", SelTR, gdtr.cbGdt));
926	return VERR_VMX_INVALID_HOST_STATE;
927	}
928
929	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
930	if (VMX_IS_64BIT_HOST_MODE())
931	{
932	pDesc = &((PX86DESCHC)gdtr.pGdt)[SelTR >> X86_SEL_SHIFT_HC]; /// ????
933	uint64_t trBase64 = X86DESC64_BASE(*(PX86DESC64)pDesc);
934	rc = VMXWriteVMCS64(VMX_VMCS_HOST_TR_BASE, trBase64);
935	Log2(("VMX_VMCS_HOST_TR_BASE %RX64\n", trBase64));
936	AssertRC(rc);
937	}
938	else
939	#endif
940	{
941	pDesc = &((PX86DESCHC)gdtr.pGdt)[SelTR >> X86_SEL_SHIFT_HC];
942	#if HC_ARCH_BITS == 64
943	trBase = X86DESC64_BASE(*pDesc);
944	#else
945	trBase = X86DESC_BASE(*pDesc);
946	#endif
947	rc = VMXWriteVMCS(VMX_VMCS_HOST_TR_BASE, trBase);
948	AssertRC(rc);
949	Log2(("VMX_VMCS_HOST_TR_BASE %RHv\n", trBase));
950	}
951
952	/* FS and GS base. */
953	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
954	if (VMX_IS_64BIT_HOST_MODE())
955	{
956	Log2(("MSR_K8_FS_BASE = %RX64\n", ASMRdMsr(MSR_K8_FS_BASE)));
957	Log2(("MSR_K8_GS_BASE = %RX64\n", ASMRdMsr(MSR_K8_GS_BASE)));
958	rc = VMXWriteVMCS64(VMX_VMCS_HOST_FS_BASE, ASMRdMsr(MSR_K8_FS_BASE));
959	rc \|= VMXWriteVMCS64(VMX_VMCS_HOST_GS_BASE, ASMRdMsr(MSR_K8_GS_BASE));
960	}
961	#endif
962	AssertRC(rc);
963
964	/* Sysenter MSRs. */
965	/** @todo expensive!! */
966	rc = VMXWriteVMCS(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
967	Log2(("VMX_VMCS_HOST_SYSENTER_CS %08x\n", ASMRdMsr_Low(MSR_IA32_SYSENTER_CS)));
968	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
969	if (VMX_IS_64BIT_HOST_MODE())
970	{
971	Log2(("VMX_VMCS_HOST_SYSENTER_EIP %RX64\n", ASMRdMsr(MSR_IA32_SYSENTER_EIP)));
972	Log2(("VMX_VMCS_HOST_SYSENTER_ESP %RX64\n", ASMRdMsr(MSR_IA32_SYSENTER_ESP)));
973	rc \|= VMXWriteVMCS64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
974	rc \|= VMXWriteVMCS64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
975	}
976	else
977	{
978	rc \|= VMXWriteVMCS(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
979	rc \|= VMXWriteVMCS(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
980	Log2(("VMX_VMCS_HOST_SYSENTER_EIP %RX32\n", ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP)));
981	Log2(("VMX_VMCS_HOST_SYSENTER_ESP %RX32\n", ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP)));
982	}
983	#elif HC_ARCH_BITS == 32
984	rc \|= VMXWriteVMCS(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
985	rc \|= VMXWriteVMCS(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
986	Log2(("VMX_VMCS_HOST_SYSENTER_EIP %RX32\n", ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP)));
987	Log2(("VMX_VMCS_HOST_SYSENTER_ESP %RX32\n", ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP)));
988	#else
989	Log2(("VMX_VMCS_HOST_SYSENTER_EIP %RX64\n", ASMRdMsr(MSR_IA32_SYSENTER_EIP)));
990	Log2(("VMX_VMCS_HOST_SYSENTER_ESP %RX64\n", ASMRdMsr(MSR_IA32_SYSENTER_ESP)));
991	rc \|= VMXWriteVMCS64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
992	rc \|= VMXWriteVMCS64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
993	#endif
994	AssertRC(rc);
995
996	#if 0 /* @todo deal with 32/64 */
997	/* Restore the host EFER - on CPUs that support it. */
998	if (pVM->hwaccm.s.vmx.msr.vmx_exit.n.allowed1 & VMX_VMCS_CTRL_EXIT_CONTROLS_LOAD_HOST_EFER_MSR)
999	{
1000	uint64_t msrEFER = ASMRdMsr(MSR_IA32_EFER);
1001	rc = VMXWriteVMCS64(VMX_VMCS_HOST_FIELD_EFER_FULL, msrEFER);
1002	AssertRC(rc);
1003	}
1004	#endif
1005	pVCpu->hwaccm.s.fContextUseFlags &= ~HWACCM_CHANGED_HOST_CONTEXT;
1006	}
1007	return rc;
1008	}
1009
1010	/**
1011	* Prefetch the 4 PDPT pointers (PAE and nested paging only)
1012	*
1013	* @param pVM The VM to operate on.
1014	* @param pVCpu The VMCPU to operate on.
1015	* @param pCtx Guest context
1016	*/
1017	static void vmxR0PrefetchPAEPdptrs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
1018	{
1019	if (CPUMIsGuestInPAEModeEx(pCtx))
1020	{
1021	X86PDPE Pdpe;
1022
1023	for (unsigned i=0;i<4;i++)
1024	{
1025	Pdpe = PGMGstGetPaePDPtr(pVM, i);
1026	int rc = VMXWriteVMCS64(VMX_VMCS_GUEST_PDPTR0_FULL + i*2, Pdpe.u);
1027	AssertRC(rc);
1028	}
1029	}
1030	}
1031
1032	/**
1033	* Update the exception bitmap according to the current CPU state
1034	*
1035	* @param pVM The VM to operate on.
1036	* @param pVCpu The VMCPU to operate on.
1037	* @param pCtx Guest context
1038	*/
1039	static void vmxR0UpdateExceptionBitmap(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
1040	{
1041	uint32_t u32TrapMask;
1042	Assert(pCtx);
1043
1044	u32TrapMask = HWACCM_VMX_TRAP_MASK;
1045	#ifndef DEBUG
1046	if (pVM->hwaccm.s.fNestedPaging)
1047	u32TrapMask &= ~RT_BIT(X86_XCPT_PF); /* no longer need to intercept #PF. */
1048	#endif
1049
1050	/* Also catch floating point exceptions as we need to report them to the guest in a different way. */
1051	if ( CPUMIsGuestFPUStateActive(pVCpu) == true
1052	&& !(pCtx->cr0 & X86_CR0_NE)
1053	&& !pVCpu->hwaccm.s.fFPUOldStyleOverride)
1054	{
1055	u32TrapMask \|= RT_BIT(X86_XCPT_MF);
1056	pVCpu->hwaccm.s.fFPUOldStyleOverride = true;
1057	}
1058
1059	#ifdef DEBUG
1060	/* Intercept X86_XCPT_DB if stepping is enabled */
1061	if (DBGFIsStepping(pVM))
1062	u32TrapMask \|= RT_BIT(X86_XCPT_DB);
1063	#endif
1064
1065	#ifdef VBOX_STRICT
1066	Assert(u32TrapMask & RT_BIT(X86_XCPT_GP));
1067	#endif
1068
1069	# ifdef HWACCM_VMX_EMULATE_REALMODE
1070	/* Intercept all exceptions in real mode as none of them can be injected directly (#GP otherwise). */
1071	if (CPUMIsGuestInRealModeEx(pCtx))
1072	u32TrapMask \|= HWACCM_VMX_TRAP_MASK_REALMODE;
1073	# endif /* HWACCM_VMX_EMULATE_REALMODE */
1074
1075	int rc = VMXWriteVMCS(VMX_VMCS_CTRL_EXCEPTION_BITMAP, u32TrapMask);
1076	AssertRC(rc);
1077	}
1078
1079	/**
1080	* Loads the guest state
1081	*
1082	* NOTE: Don't do anything here that can cause a jump back to ring 3!!!!!
1083	*
1084	* @returns VBox status code.
1085	* @param pVM The VM to operate on.
1086	* @param pVCpu The VMCPU to operate on.
1087	* @param pCtx Guest context
1088	*/
1089	VMMR0DECL(int) VMXR0LoadGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
1090	{
1091	int rc = VINF_SUCCESS;
1092	RTGCUINTPTR val;
1093	X86EFLAGS eflags;
1094
1095	/* VMX_VMCS_CTRL_ENTRY_CONTROLS
1096	* Set required bits to one and zero according to the MSR capabilities.
1097	*/
1098	val = pVM->hwaccm.s.vmx.msr.vmx_entry.n.disallowed0;
1099	/* Load guest debug controls (dr7 & IA32_DEBUGCTL_MSR) (forced to 1 on the 'first' VT-x capable CPUs; this actually includes the newest Nehalem CPUs) */
1100	val \|= VMX_VMCS_CTRL_ENTRY_CONTROLS_LOAD_DEBUG;
1101	#if 0 /* @todo deal with 32/64 */
1102	/* Required for the EFER write below, not supported on all CPUs. */
1103	val \|= VMX_VMCS_CTRL_ENTRY_CONTROLS_LOAD_GUEST_EFER_MSR;
1104	#endif
1105	/* 64 bits guest mode? */
1106	if (pCtx->msrEFER & MSR_K6_EFER_LMA)
1107	val \|= VMX_VMCS_CTRL_ENTRY_CONTROLS_IA64_MODE;
1108	/* else Must be zero when AMD64 is not available. */
1109
1110	/* Mask away the bits that the CPU doesn't support */
1111	val &= pVM->hwaccm.s.vmx.msr.vmx_entry.n.allowed1;
1112	rc = VMXWriteVMCS(VMX_VMCS_CTRL_ENTRY_CONTROLS, val);
1113	AssertRC(rc);
1114
1115	/* VMX_VMCS_CTRL_EXIT_CONTROLS
1116	* Set required bits to one and zero according to the MSR capabilities.
1117	*/
1118	val = pVM->hwaccm.s.vmx.msr.vmx_exit.n.disallowed0;
1119
1120	/* Save debug controls (dr7 & IA32_DEBUGCTL_MSR) (forced to 1 on the 'first' VT-x capable CPUs; this actually includes the newest Nehalem CPUs) */
1121	#if 0 /* @todo deal with 32/64 */
1122	val \|= VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_DEBUG \| VMX_VMCS_CTRL_EXIT_CONTROLS_LOAD_HOST_EFER_MSR;
1123	#else
1124	val \|= VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_DEBUG;
1125	#endif
1126
1127	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
1128	if (VMX_IS_64BIT_HOST_MODE())
1129	val \|= VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_AMD64;
1130	/* else: Must be zero when AMD64 is not available. */
1131	#elif HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
1132	if (pCtx->msrEFER & MSR_K6_EFER_LMA)
1133	val \|= VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_AMD64; /* our switcher goes to long mode */
1134	else
1135	Assert(!(val & VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_AMD64));
1136	#endif
1137	val &= pVM->hwaccm.s.vmx.msr.vmx_exit.n.allowed1;
1138	/* Don't acknowledge external interrupts on VM-exit. */
1139	rc = VMXWriteVMCS(VMX_VMCS_CTRL_EXIT_CONTROLS, val);
1140	AssertRC(rc);
1141
1142	/* Guest CPU context: ES, CS, SS, DS, FS, GS. */
1143	if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_SEGMENT_REGS)
1144	{
1145	#ifdef HWACCM_VMX_EMULATE_REALMODE
1146	PGMMODE enmGuestMode = PGMGetGuestMode(pVM);
1147	if (pVCpu->hwaccm.s.vmx.enmLastSeenGuestMode != enmGuestMode)
1148	{
1149	/* Correct weird requirements for switching to protected mode. */
1150	if ( pVCpu->hwaccm.s.vmx.enmLastSeenGuestMode == PGMMODE_REAL
1151	&& enmGuestMode >= PGMMODE_PROTECTED)
1152	{
1153	/* DPL of all hidden selector registers must match the current CPL (0). */
1154	pCtx->csHid.Attr.n.u2Dpl = 0;
1155	pCtx->csHid.Attr.n.u4Type = X86_SEL_TYPE_CODE \| X86_SEL_TYPE_RW_ACC;
1156
1157	pCtx->dsHid.Attr.n.u2Dpl = 0;
1158	pCtx->esHid.Attr.n.u2Dpl = 0;
1159	pCtx->fsHid.Attr.n.u2Dpl = 0;
1160	pCtx->gsHid.Attr.n.u2Dpl = 0;
1161	pCtx->ssHid.Attr.n.u2Dpl = 0;
1162	}
1163	else
1164	/* Switching from protected mode to real mode. */
1165	if ( pVCpu->hwaccm.s.vmx.enmLastSeenGuestMode >= PGMMODE_PROTECTED
1166	&& enmGuestMode == PGMMODE_REAL)
1167	{
1168	/* The limit must also be adjusted. */
1169	pCtx->csHid.u32Limit &= 0xffff;
1170	pCtx->dsHid.u32Limit &= 0xffff;
1171	pCtx->esHid.u32Limit &= 0xffff;
1172	pCtx->fsHid.u32Limit &= 0xffff;
1173	pCtx->gsHid.u32Limit &= 0xffff;
1174	pCtx->ssHid.u32Limit &= 0xffff;
1175
1176	Assert(pCtx->csHid.u64Base <= 0xfffff);
1177	Assert(pCtx->dsHid.u64Base <= 0xfffff);
1178	Assert(pCtx->esHid.u64Base <= 0xfffff);
1179	Assert(pCtx->fsHid.u64Base <= 0xfffff);
1180	Assert(pCtx->gsHid.u64Base <= 0xfffff);
1181	}
1182	pVCpu->hwaccm.s.vmx.enmLastSeenGuestMode = enmGuestMode;
1183	}
1184	else
1185	/* VT-x will fail with a guest invalid state otherwise... (CPU state after a reset) */
1186	if ( CPUMIsGuestInRealModeEx(pCtx)
1187	&& pCtx->csHid.u64Base == 0xffff0000)
1188	{
1189	pCtx->csHid.u64Base = 0xf0000;
1190	pCtx->cs = 0xf000;
1191	}
1192	#endif /* HWACCM_VMX_EMULATE_REALMODE */
1193
1194	VMX_WRITE_SELREG(ES, es);
1195	AssertRC(rc);
1196
1197	VMX_WRITE_SELREG(CS, cs);
1198	AssertRC(rc);
1199
1200	VMX_WRITE_SELREG(SS, ss);
1201	AssertRC(rc);
1202
1203	VMX_WRITE_SELREG(DS, ds);
1204	AssertRC(rc);
1205
1206	/* The base values in the hidden fs & gs registers are not in sync with the msrs; they are cut to 32 bits. */
1207	VMX_WRITE_SELREG(FS, fs);
1208	AssertRC(rc);
1209
1210	VMX_WRITE_SELREG(GS, gs);
1211	AssertRC(rc);
1212	}
1213
1214	/* Guest CPU context: LDTR. */
1215	if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_LDTR)
1216	{
1217	if (pCtx->ldtr == 0)
1218	{
1219	rc = VMXWriteVMCS(VMX_VMCS16_GUEST_FIELD_LDTR, 0);
1220	rc \|= VMXWriteVMCS(VMX_VMCS32_GUEST_LDTR_LIMIT, 0);
1221	rc \|= VMXWriteVMCS64(VMX_VMCS64_GUEST_LDTR_BASE, 0);
1222	/* Note: vmlaunch will fail with 0 or just 0x02. No idea why. */
1223	rc \|= VMXWriteVMCS(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, 0x82 /* present, LDT */);
1224	}
1225	else
1226	{
1227	rc = VMXWriteVMCS(VMX_VMCS16_GUEST_FIELD_LDTR, pCtx->ldtr);
1228	rc \|= VMXWriteVMCS(VMX_VMCS32_GUEST_LDTR_LIMIT, pCtx->ldtrHid.u32Limit);
1229	rc \|= VMXWriteVMCS64(VMX_VMCS64_GUEST_LDTR_BASE, pCtx->ldtrHid.u64Base);
1230	rc \|= VMXWriteVMCS(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, pCtx->ldtrHid.Attr.u);
1231	}
1232	AssertRC(rc);
1233	}
1234	/* Guest CPU context: TR. */
1235	if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_TR)
1236	{
1237	#ifdef HWACCM_VMX_EMULATE_REALMODE
1238	/* Real mode emulation using v86 mode with CR4.VME (interrupt redirection using the int bitmap in the TSS) */
1239	if (CPUMIsGuestInRealModeEx(pCtx))
1240	{
1241	RTGCPHYS GCPhys;
1242
1243	/* We convert it here every time as pci regions could be reconfigured. */
1244	rc = PDMVMMDevHeapR3ToGCPhys(pVM, pVM->hwaccm.s.vmx.pRealModeTSS, &GCPhys);
1245	AssertRC(rc);
1246
1247	rc = VMXWriteVMCS(VMX_VMCS16_GUEST_FIELD_TR, 0);
1248	rc \|= VMXWriteVMCS(VMX_VMCS32_GUEST_TR_LIMIT, HWACCM_VTX_TSS_SIZE);
1249	rc \|= VMXWriteVMCS64(VMX_VMCS64_GUEST_TR_BASE, GCPhys /* phys = virt in this mode */);
1250
1251	X86DESCATTR attr;
1252
1253	attr.u = 0;
1254	attr.n.u1Present = 1;
1255	attr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
1256	val = attr.u;
1257	}
1258	else
1259	#endif /* HWACCM_VMX_EMULATE_REALMODE */
1260	{
1261	rc = VMXWriteVMCS(VMX_VMCS16_GUEST_FIELD_TR, pCtx->tr);
1262	rc \|= VMXWriteVMCS(VMX_VMCS32_GUEST_TR_LIMIT, pCtx->trHid.u32Limit);
1263	rc \|= VMXWriteVMCS64(VMX_VMCS64_GUEST_TR_BASE, pCtx->trHid.u64Base);
1264
1265	val = pCtx->trHid.Attr.u;
1266
1267	/* The TSS selector must be busy. */
1268	if ((val & 0xF) == X86_SEL_TYPE_SYS_286_TSS_AVAIL)
1269	val = (val & ~0xF) \| X86_SEL_TYPE_SYS_286_TSS_BUSY;
1270	else
1271	/* Default even if no TR selector has been set (otherwise vmlaunch will fail!) */
1272	val = (val & ~0xF) \| X86_SEL_TYPE_SYS_386_TSS_BUSY;
1273
1274	}
1275	rc \|= VMXWriteVMCS(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, val);
1276	AssertRC(rc);
1277	}
1278	/* Guest CPU context: GDTR. */
1279	if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_GDTR)
1280	{
1281	rc = VMXWriteVMCS(VMX_VMCS32_GUEST_GDTR_LIMIT, pCtx->gdtr.cbGdt);
1282	rc \|= VMXWriteVMCS64(VMX_VMCS64_GUEST_GDTR_BASE, pCtx->gdtr.pGdt);
1283	AssertRC(rc);
1284	}
1285	/* Guest CPU context: IDTR. */
1286	if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_IDTR)
1287	{
1288	rc = VMXWriteVMCS(VMX_VMCS32_GUEST_IDTR_LIMIT, pCtx->idtr.cbIdt);
1289	rc \|= VMXWriteVMCS64(VMX_VMCS64_GUEST_IDTR_BASE, pCtx->idtr.pIdt);
1290	AssertRC(rc);
1291	}
1292
1293	/*
1294	* Sysenter MSRs (unconditional)
1295	*/
1296	rc = VMXWriteVMCS(VMX_VMCS32_GUEST_SYSENTER_CS, pCtx->SysEnter.cs);
1297	rc \|= VMXWriteVMCS64(VMX_VMCS64_GUEST_SYSENTER_EIP, pCtx->SysEnter.eip);
1298	rc \|= VMXWriteVMCS64(VMX_VMCS64_GUEST_SYSENTER_ESP, pCtx->SysEnter.esp);
1299	AssertRC(rc);
1300
1301	/* Control registers */
1302	if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_CR0)
1303	{
1304	val = pCtx->cr0;
1305	rc = VMXWriteVMCS(VMX_VMCS_CTRL_CR0_READ_SHADOW, val);
1306	Log2(("Guest CR0-shadow %08x\n", val));
1307	if (CPUMIsGuestFPUStateActive(pVCpu) == false)
1308	{
1309	/* Always use #NM exceptions to load the FPU/XMM state on demand. */
1310	val \|= X86_CR0_TS \| X86_CR0_ET \| X86_CR0_NE \| X86_CR0_MP;
1311	}
1312	else
1313	{
1314	/** @todo check if we support the old style mess correctly. */
1315	if (!(val & X86_CR0_NE))
1316	Log(("Forcing X86_CR0_NE!!!\n"));
1317
1318	val \|= X86_CR0_NE; /* always turn on the native mechanism to report FPU errors (old style uses interrupts) */
1319	}
1320	/* Note: protected mode & paging are always enabled; we use them for emulating real and protected mode without paging too. */
1321	val \|= X86_CR0_PE \| X86_CR0_PG;
1322	if (pVM->hwaccm.s.fNestedPaging)
1323	{
1324	if (CPUMIsGuestInPagedProtectedModeEx(pCtx))
1325	{
1326	/* Disable cr3 read/write monitoring as we don't need it for EPT. */
1327	pVCpu->hwaccm.s.vmx.proc_ctls &= ~( VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
1328	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT);
1329	}
1330	else
1331	{
1332	/* Reenable cr3 read/write monitoring as our identity mapped page table is active. */
1333	pVCpu->hwaccm.s.vmx.proc_ctls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
1334	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT;
1335	}
1336	rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
1337	AssertRC(rc);
1338	}
1339	else
1340	{
1341	/* Note: We must also set this as we rely on protecting various pages for which supervisor writes must be caught. */
1342	val \|= X86_CR0_WP;
1343	}
1344
1345	/* Always enable caching. */
1346	val &= ~(X86_CR0_CD\|X86_CR0_NW);
1347
1348	rc \|= VMXWriteVMCS64(VMX_VMCS64_GUEST_CR0, val);
1349	Log2(("Guest CR0 %08x\n", val));
1350	/* CR0 flags owned by the host; if the guests attempts to change them, then
1351	* the VM will exit.
1352	*/
1353	val = X86_CR0_PE /* Must monitor this bit (assumptions are made for real mode emulation) */
1354	\| X86_CR0_WP /* Must monitor this bit (it must always be enabled). */
1355	\| X86_CR0_PG /* Must monitor this bit (assumptions are made for real mode & protected mode without paging emulation) */
1356	\| X86_CR0_TS
1357	\| X86_CR0_ET /* Bit not restored during VM-exit! */
1358	\| X86_CR0_CD /* Bit not restored during VM-exit! */
1359	\| X86_CR0_NW /* Bit not restored during VM-exit! */
1360	\| X86_CR0_NE
1361	\| X86_CR0_MP;
1362	pVCpu->hwaccm.s.vmx.cr0_mask = val;
1363
1364	rc \|= VMXWriteVMCS(VMX_VMCS_CTRL_CR0_MASK, val);
1365	Log2(("Guest CR0-mask %08x\n", val));
1366	AssertRC(rc);
1367	}
1368	if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_CR4)
1369	{
1370	/* CR4 */
1371	rc = VMXWriteVMCS(VMX_VMCS_CTRL_CR4_READ_SHADOW, pCtx->cr4);
1372	Log2(("Guest CR4-shadow %08x\n", pCtx->cr4));
1373	/* Set the required bits in cr4 too (currently X86_CR4_VMXE). */
1374	val = pCtx->cr4 \| (uint32_t)pVM->hwaccm.s.vmx.msr.vmx_cr4_fixed0;
1375
1376	if (!pVM->hwaccm.s.fNestedPaging)
1377	{
1378	switch(pVCpu->hwaccm.s.enmShadowMode)
1379	{
1380	case PGMMODE_REAL: /* Real mode -> emulated using v86 mode */
1381	case PGMMODE_PROTECTED: /* Protected mode, no paging -> emulated using identity mapping. */
1382	case PGMMODE_32_BIT: /* 32-bit paging. */
1383	break;
1384
1385	case PGMMODE_PAE: /* PAE paging. */
1386	case PGMMODE_PAE_NX: /* PAE paging with NX enabled. */
1387	/** @todo use normal 32 bits paging */
1388	val \|= X86_CR4_PAE;
1389	break;
1390
1391	case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
1392	case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
1393	#ifdef VBOX_ENABLE_64_BITS_GUESTS
1394	break;
1395	#else
1396	AssertFailed();
1397	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
1398	#endif
1399	default: /* shut up gcc */
1400	AssertFailed();
1401	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
1402	}
1403	}
1404	else
1405	if (!CPUMIsGuestInPagedProtectedModeEx(pCtx))
1406	{
1407	/* We use 4 MB pages in our identity mapping page table for real and protected mode without paging. */
1408	val \|= X86_CR4_PSE;
1409	/* Our identity mapping is a 32 bits page directory. */
1410	val &= ~X86_CR4_PAE;
1411	}
1412
1413	#ifdef HWACCM_VMX_EMULATE_REALMODE
1414	/* Real mode emulation using v86 mode with CR4.VME (interrupt redirection using the int bitmap in the TSS) */
1415	if (CPUMIsGuestInRealModeEx(pCtx))
1416	val \|= X86_CR4_VME;
1417	#endif /* HWACCM_VMX_EMULATE_REALMODE */
1418
1419	rc \|= VMXWriteVMCS64(VMX_VMCS64_GUEST_CR4, val);
1420	Log2(("Guest CR4 %08x\n", val));
1421	/* CR4 flags owned by the host; if the guests attempts to change them, then
1422	* the VM will exit.
1423	*/
1424	val = 0
1425	#ifdef HWACCM_VMX_EMULATE_REALMODE
1426	\| X86_CR4_VME
1427	#endif
1428	\| X86_CR4_PAE
1429	\| X86_CR4_PGE
1430	\| X86_CR4_PSE
1431	\| X86_CR4_VMXE;
1432	pVCpu->hwaccm.s.vmx.cr4_mask = val;
1433
1434	rc \|= VMXWriteVMCS(VMX_VMCS_CTRL_CR4_MASK, val);
1435	Log2(("Guest CR4-mask %08x\n", val));
1436	AssertRC(rc);
1437	}
1438
1439	if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_CR3)
1440	{
1441	if (pVM->hwaccm.s.fNestedPaging)
1442	{
1443	AssertMsg( PGMGetEPTCR3(pVM) == PGMGetHyperCR3(pVM)
1444	\|\| VM_FF_ISPENDING(pVM, VM_FF_PGM_SYNC_CR3 \| VM_FF_PGM_SYNC_CR3_NON_GLOBAL),
1445	("%RHp vs %RHp\n", PGMGetEPTCR3(pVM), PGMGetHyperCR3(pVM)));
1446	pVCpu->hwaccm.s.vmx.GCPhysEPTP = PGMGetEPTCR3(pVM);
1447
1448	Assert(!(pVCpu->hwaccm.s.vmx.GCPhysEPTP & 0xfff));
1449	/** @todo Check the IA32_VMX_EPT_VPID_CAP MSR for other supported memory types. */
1450	pVCpu->hwaccm.s.vmx.GCPhysEPTP \|= VMX_EPT_MEMTYPE_WB
1451	\| (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
1452
1453	rc = VMXWriteVMCS64(VMX_VMCS_CTRL_EPTP_FULL, pVCpu->hwaccm.s.vmx.GCPhysEPTP);
1454	AssertRC(rc);
1455
1456	if (!CPUMIsGuestInPagedProtectedModeEx(pCtx))
1457	{
1458	RTGCPHYS GCPhys;
1459
1460	/* We convert it here every time as pci regions could be reconfigured. */
1461	rc = PDMVMMDevHeapR3ToGCPhys(pVM, pVM->hwaccm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
1462	AssertRC(rc);
1463
1464	/* We use our identity mapping page table here as we need to map guest virtual to guest physical addresses; EPT will
1465	* take care of the translation to host physical addresses.
1466	*/
1467	val = GCPhys;
1468	}
1469	else
1470	{
1471	/* Save the real guest CR3 in VMX_VMCS_GUEST_CR3 */
1472	val = pCtx->cr3;
1473	/* Prefetch the four PDPT entries in PAE mode. */
1474	vmxR0PrefetchPAEPdptrs(pVM, pVCpu, pCtx);
1475	}
1476	}
1477	else
1478	{
1479	val = PGMGetHyperCR3(pVM);
1480	Assert(val \|\| VM_FF_ISPENDING(pVM, VM_FF_PGM_SYNC_CR3 \| VM_FF_PGM_SYNC_CR3_NON_GLOBAL));
1481	}
1482
1483	/* Save our shadow CR3 register. */
1484	rc = VMXWriteVMCS64(VMX_VMCS64_GUEST_CR3, val);
1485	AssertRC(rc);
1486	}
1487
1488	/* Debug registers. */
1489	if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_DEBUG)
1490	{
1491	pCtx->dr[6] \|= X86_DR6_INIT_VAL; /* set all reserved bits to 1. */
1492	pCtx->dr[6] &= ~RT_BIT(12); /* must be zero. */
1493
1494	pCtx->dr[7] &= 0xffffffff; /* upper 32 bits reserved */
1495	pCtx->dr[7] &= ~(RT_BIT(11) \| RT_BIT(12) \| RT_BIT(14) \| RT_BIT(15)); /* must be zero */
1496	pCtx->dr[7] \|= 0x400; /* must be one */
1497
1498	/* Resync DR7 */
1499	rc = VMXWriteVMCS64(VMX_VMCS64_GUEST_DR7, pCtx->dr[7]);
1500	AssertRC(rc);
1501
1502	/* Sync the debug state now if any breakpoint is armed. */
1503	if ( (pCtx->dr[7] & (X86_DR7_ENABLED_MASK\|X86_DR7_GD))
1504	&& !CPUMIsGuestDebugStateActive(pVM)
1505	&& !DBGFIsStepping(pVM))
1506	{
1507	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatDRxArmed);
1508
1509	/* Disable drx move intercepts. */
1510	pVCpu->hwaccm.s.vmx.proc_ctls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT;
1511	rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
1512	AssertRC(rc);
1513
1514	/* Save the host and load the guest debug state. */
1515	rc = CPUMR0LoadGuestDebugState(pVM, pVCpu, pCtx, true /* include DR6 */);
1516	AssertRC(rc);
1517	}
1518
1519	/* IA32_DEBUGCTL MSR. */
1520	rc = VMXWriteVMCS64(VMX_VMCS_GUEST_DEBUGCTL_FULL, 0);
1521	AssertRC(rc);
1522
1523	/** @todo do we really ever need this? */
1524	rc \|= VMXWriteVMCS(VMX_VMCS_GUEST_DEBUG_EXCEPTIONS, 0);
1525	AssertRC(rc);
1526	}
1527
1528	/* EIP, ESP and EFLAGS */
1529	rc = VMXWriteVMCS64(VMX_VMCS64_GUEST_RIP, pCtx->rip);
1530	rc \|= VMXWriteVMCS64(VMX_VMCS64_GUEST_RSP, pCtx->rsp);
1531	AssertRC(rc);
1532
1533	/* Bits 22-31, 15, 5 & 3 must be zero. Bit 1 must be 1. */
1534	eflags = pCtx->eflags;
1535	eflags.u32 &= VMX_EFLAGS_RESERVED_0;
1536	eflags.u32 \|= VMX_EFLAGS_RESERVED_1;
1537
1538	#ifdef HWACCM_VMX_EMULATE_REALMODE
1539	/* Real mode emulation using v86 mode with CR4.VME (interrupt redirection using the int bitmap in the TSS) */
1540	if (CPUMIsGuestInRealModeEx(pCtx))
1541	{
1542	pVCpu->hwaccm.s.vmx.RealMode.eflags = eflags;
1543
1544	eflags.Bits.u1VM = 1;
1545	eflags.Bits.u2IOPL = 3;
1546	}
1547	#endif /* HWACCM_VMX_EMULATE_REALMODE */
1548	rc = VMXWriteVMCS(VMX_VMCS_GUEST_RFLAGS, eflags.u32);
1549	AssertRC(rc);
1550
1551	/* TSC offset. */
1552	uint64_t u64TSCOffset;
1553
1554	if (TMCpuTickCanUseRealTSC(pVM, &u64TSCOffset))
1555	{
1556	/* Note: VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT takes precedence over TSC_OFFSET */
1557	rc = VMXWriteVMCS64(VMX_VMCS_CTRL_TSC_OFFSET_FULL, u64TSCOffset);
1558	AssertRC(rc);
1559
1560	pVCpu->hwaccm.s.vmx.proc_ctls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT;
1561	rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
1562	AssertRC(rc);
1563	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatTSCOffset);
1564	}
1565	else
1566	{
1567	pVCpu->hwaccm.s.vmx.proc_ctls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT;
1568	rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
1569	AssertRC(rc);
1570	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatTSCIntercept);
1571	}
1572
1573	/* 64 bits guest mode? */
1574	if (pCtx->msrEFER & MSR_K6_EFER_LMA)
1575	{
1576	#if !defined(VBOX_ENABLE_64_BITS_GUESTS)
1577	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
1578	#elif HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
1579	pVCpu->hwaccm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
1580	#else
1581	# ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
1582	if (!pVM->hwaccm.s.fAllow64BitGuests)
1583	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
1584	# endif
1585	pVCpu->hwaccm.s.vmx.pfnStartVM = VMXR0StartVM64;
1586	#endif
1587	/* Unconditionally update these as wrmsr might have changed them. */
1588	rc = VMXWriteVMCS64(VMX_VMCS64_GUEST_FS_BASE, pCtx->fsHid.u64Base);
1589	AssertRC(rc);
1590	rc = VMXWriteVMCS64(VMX_VMCS64_GUEST_GS_BASE, pCtx->gsHid.u64Base);
1591	AssertRC(rc);
1592	}
1593	else
1594	{
1595	pVCpu->hwaccm.s.vmx.pfnStartVM = VMXR0StartVM32;
1596	}
1597
1598	#if 0 /* @todo deal with 32/64 */
1599	/* Unconditionally update the guest EFER - on CPUs that supports it. */
1600	if (pVM->hwaccm.s.vmx.msr.vmx_entry.n.allowed1 & VMX_VMCS_CTRL_ENTRY_CONTROLS_LOAD_GUEST_EFER_MSR)
1601	{
1602	rc = VMXWriteVMCS64(VMX_VMCS_GUEST_EFER_FULL, pCtx->msrEFER);
1603	AssertRC(rc);
1604	}
1605	#endif
1606
1607	vmxR0UpdateExceptionBitmap(pVM, pVCpu, pCtx);
1608
1609	/* Done. */
1610	pVCpu->hwaccm.s.fContextUseFlags &= ~HWACCM_CHANGED_ALL_GUEST;
1611
1612	return rc;
1613	}
1614
1615	/**
1616	* Syncs back the guest state
1617	*
1618	* @returns VBox status code.
1619	* @param pVM The VM to operate on.
1620	* @param pVCpu The VMCPU to operate on.
1621	* @param pCtx Guest context
1622	*/
1623	DECLINLINE(int) VMXR0SaveGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
1624	{
1625	RTGCUINTREG val, valShadow;
1626	RTGCUINTPTR uInterruptState;
1627	int rc;
1628
1629	/* Let's first sync back eip, esp, and eflags. */
1630	rc = VMXReadCachedVMCS(VMX_VMCS64_GUEST_RIP, &val);
1631	AssertRC(rc);
1632	pCtx->rip = val;
1633	rc = VMXReadCachedVMCS(VMX_VMCS64_GUEST_RSP, &val);
1634	AssertRC(rc);
1635	pCtx->rsp = val;
1636	rc = VMXReadCachedVMCS(VMX_VMCS_GUEST_RFLAGS, &val);
1637	AssertRC(rc);
1638	pCtx->eflags.u32 = val;
1639
1640	/* Take care of instruction fusing (sti, mov ss) */
1641	rc \|= VMXReadCachedVMCS(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &val);
1642	uInterruptState = val;
1643	if (uInterruptState != 0)
1644	{
1645	Assert(uInterruptState <= 2); /* only sti & mov ss */
1646	Log(("uInterruptState %x eip=%RGv\n", uInterruptState, pCtx->rip));
1647	EMSetInhibitInterruptsPC(pVM, pCtx->rip);
1648	}
1649	else
1650	VM_FF_CLEAR(pVM, VM_FF_INHIBIT_INTERRUPTS);
1651
1652	/* Control registers. */
1653	VMXReadCachedVMCS(VMX_VMCS_CTRL_CR0_READ_SHADOW, &valShadow);
1654	VMXReadCachedVMCS(VMX_VMCS64_GUEST_CR0, &val);
1655	val = (valShadow & pVCpu->hwaccm.s.vmx.cr0_mask) \| (val & ~pVCpu->hwaccm.s.vmx.cr0_mask);
1656	CPUMSetGuestCR0(pVM, val);
1657
1658	VMXReadCachedVMCS(VMX_VMCS_CTRL_CR4_READ_SHADOW, &valShadow);
1659	VMXReadCachedVMCS(VMX_VMCS64_GUEST_CR4, &val);
1660	val = (valShadow & pVCpu->hwaccm.s.vmx.cr4_mask) \| (val & ~pVCpu->hwaccm.s.vmx.cr4_mask);
1661	CPUMSetGuestCR4(pVM, val);
1662
1663	/* Note: no reason to sync back the CRx registers. They can't be changed by the guest. */
1664	/* Note: only in the nested paging case can CR3 & CR4 be changed by the guest. */
1665	if ( pVM->hwaccm.s.fNestedPaging
1666	&& CPUMIsGuestInPagedProtectedModeEx(pCtx))
1667	{
1668	PVMCSCACHE pCache = &pVCpu->hwaccm.s.vmx.VMCSCache;
1669
1670	/* Can be updated behind our back in the nested paging case. */
1671	CPUMSetGuestCR2(pVM, pCache->cr2);
1672
1673	VMXReadCachedVMCS(VMX_VMCS64_GUEST_CR3, &val);
1674
1675	if (val != pCtx->cr3)
1676	{
1677	CPUMSetGuestCR3(pVM, val);
1678	PGMUpdateCR3(pVM, val);
1679	}
1680	/* Prefetch the four PDPT entries in PAE mode. */
1681	vmxR0PrefetchPAEPdptrs(pVM, pVCpu, pCtx);
1682	}
1683
1684	/* Sync back DR7 here. */
1685	VMXReadCachedVMCS(VMX_VMCS64_GUEST_DR7, &val);
1686	pCtx->dr[7] = val;
1687
1688	/* Guest CPU context: ES, CS, SS, DS, FS, GS. */
1689	VMX_READ_SELREG(ES, es);
1690	VMX_READ_SELREG(SS, ss);
1691	VMX_READ_SELREG(CS, cs);
1692	VMX_READ_SELREG(DS, ds);
1693	VMX_READ_SELREG(FS, fs);
1694	VMX_READ_SELREG(GS, gs);
1695
1696	/*
1697	* System MSRs
1698	*/
1699	VMXReadCachedVMCS(VMX_VMCS32_GUEST_SYSENTER_CS, &val);
1700	pCtx->SysEnter.cs = val;
1701	VMXReadCachedVMCS(VMX_VMCS64_GUEST_SYSENTER_EIP, &val);
1702	pCtx->SysEnter.eip = val;
1703	VMXReadCachedVMCS(VMX_VMCS64_GUEST_SYSENTER_ESP, &val);
1704	pCtx->SysEnter.esp = val;
1705
1706	/* Misc. registers; must sync everything otherwise we can get out of sync when jumping to ring 3. */
1707	VMX_READ_SELREG(LDTR, ldtr);
1708
1709	VMXReadCachedVMCS(VMX_VMCS32_GUEST_GDTR_LIMIT, &val);
1710	pCtx->gdtr.cbGdt = val;
1711	VMXReadCachedVMCS(VMX_VMCS64_GUEST_GDTR_BASE, &val);
1712	pCtx->gdtr.pGdt = val;
1713
1714	VMXReadCachedVMCS(VMX_VMCS32_GUEST_IDTR_LIMIT, &val);
1715	pCtx->idtr.cbIdt = val;
1716	VMXReadCachedVMCS(VMX_VMCS64_GUEST_IDTR_BASE, &val);
1717	pCtx->idtr.pIdt = val;
1718
1719	#ifdef HWACCM_VMX_EMULATE_REALMODE
1720	/* Real mode emulation using v86 mode with CR4.VME (interrupt redirection using the int bitmap in the TSS) */
1721	if (CPUMIsGuestInRealModeEx(pCtx))
1722	{
1723	/* Hide our emulation flags */
1724	pCtx->eflags.Bits.u1VM = 0;
1725	pCtx->eflags.Bits.u2IOPL = pVCpu->hwaccm.s.vmx.RealMode.eflags.Bits.u2IOPL;
1726
1727	/* Force a TR resync every time in case we switch modes. */
1728	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_GUEST_TR;
1729	}
1730	else
1731	#endif /* HWACCM_VMX_EMULATE_REALMODE */
1732	{
1733	/* In real mode we have a fake TSS, so only sync it back when it's supposed to be valid. */
1734	VMX_READ_SELREG(TR, tr);
1735	}
1736	return VINF_SUCCESS;
1737	}
1738
1739	/**
1740	* Dummy placeholder
1741	*
1742	* @param pVM The VM to operate on.
1743	* @param pVCpu The VMCPU to operate on.
1744	*/
1745	static void vmxR0SetupTLBDummy(PVM pVM, PVMCPU pVCpu)
1746	{
1747	NOREF(pVM);
1748	NOREF(pVCpu);
1749	return;
1750	}
1751
1752	/**
1753	* Setup the tagged TLB for EPT
1754	*
1755	* @returns VBox status code.
1756	* @param pVM The VM to operate on.
1757	* @param pVCpu The VMCPU to operate on.
1758	*/
1759	static void vmxR0SetupTLBEPT(PVM pVM, PVMCPU pVCpu)
1760	{
1761	PHWACCM_CPUINFO pCpu;
1762
1763	Assert(pVM->hwaccm.s.fNestedPaging);
1764	Assert(!pVM->hwaccm.s.vmx.fVPID);
1765
1766	/* Deal with tagged TLBs if VPID or EPT is supported. */
1767	pCpu = HWACCMR0GetCurrentCpu();
1768	/* Force a TLB flush for the first world switch if the current cpu differs from the one we ran on last. */
1769	/* Note that this can happen both for start and resume due to long jumps back to ring 3. */
1770	if ( pVCpu->hwaccm.s.idLastCpu != pCpu->idCpu
1771	/* if the tlb flush count has changed, another VM has flushed the TLB of this cpu, so we can't use our current ASID anymore. */
1772	\|\| pVCpu->hwaccm.s.cTLBFlushes != pCpu->cTLBFlushes)
1773	{
1774	/* Force a TLB flush on VM entry. */
1775	pVCpu->hwaccm.s.fForceTLBFlush = true;
1776	}
1777	else
1778	Assert(!pCpu->fFlushTLB);
1779
1780	pVCpu->hwaccm.s.idLastCpu = pCpu->idCpu;
1781	pCpu->fFlushTLB = false;
1782
1783	if (pVCpu->hwaccm.s.fForceTLBFlush)
1784	vmxR0FlushEPT(pVM, pVCpu, pVM->hwaccm.s.vmx.enmFlushContext, 0);
1785
1786	#ifdef VBOX_WITH_STATISTICS
1787	if (pVCpu->hwaccm.s.fForceTLBFlush)
1788	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatFlushTLBWorldSwitch);
1789	else
1790	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatNoFlushTLBWorldSwitch);
1791	#endif
1792	}
1793
1794	#ifdef HWACCM_VTX_WITH_VPID
1795	/**
1796	* Setup the tagged TLB for VPID
1797	*
1798	* @returns VBox status code.
1799	* @param pVM The VM to operate on.
1800	* @param pVCpu The VMCPU to operate on.
1801	*/
1802	static void vmxR0SetupTLBVPID(PVM pVM, PVMCPU pVCpu)
1803	{
1804	PHWACCM_CPUINFO pCpu;
1805
1806	Assert(pVM->hwaccm.s.vmx.fVPID);
1807	Assert(!pVM->hwaccm.s.fNestedPaging);
1808
1809	/* Deal with tagged TLBs if VPID or EPT is supported. */
1810	pCpu = HWACCMR0GetCurrentCpu();
1811	/* Force a TLB flush for the first world switch if the current cpu differs from the one we ran on last. */
1812	/* Note that this can happen both for start and resume due to long jumps back to ring 3. */
1813	if ( pVCpu->hwaccm.s.idLastCpu != pCpu->idCpu
1814	/* if the tlb flush count has changed, another VM has flushed the TLB of this cpu, so we can't use our current ASID anymore. */
1815	\|\| pVCpu->hwaccm.s.cTLBFlushes != pCpu->cTLBFlushes)
1816	{
1817	/* Force a TLB flush on VM entry. */
1818	pVCpu->hwaccm.s.fForceTLBFlush = true;
1819	}
1820	else
1821	Assert(!pCpu->fFlushTLB);
1822
1823	pVCpu->hwaccm.s.idLastCpu = pCpu->idCpu;
1824
1825	/* Make sure we flush the TLB when required. Switch ASID to achieve the same thing, but without actually flushing the whole TLB (which is expensive). */
1826	if (pVCpu->hwaccm.s.fForceTLBFlush)
1827	{
1828	if ( ++pCpu->uCurrentASID >= pVM->hwaccm.s.uMaxASID
1829	\|\| pCpu->fFlushTLB)
1830	{
1831	pCpu->fFlushTLB = false;
1832	pCpu->uCurrentASID = 1; /* start at 1; host uses 0 */
1833	pCpu->cTLBFlushes++;
1834	}
1835	else
1836	{
1837	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatFlushASID);
1838	pVCpu->hwaccm.s.fForceTLBFlush = false;
1839	}
1840
1841	pVCpu->hwaccm.s.cTLBFlushes = pCpu->cTLBFlushes;
1842	pVCpu->hwaccm.s.uCurrentASID = pCpu->uCurrentASID;
1843	}
1844	else
1845	{
1846	Assert(!pCpu->fFlushTLB);
1847
1848	if (!pCpu->uCurrentASID \|\| !pVCpu->hwaccm.s.uCurrentASID)
1849	pVCpu->hwaccm.s.uCurrentASID = pCpu->uCurrentASID = 1;
1850	}
1851	AssertMsg(pVCpu->hwaccm.s.cTLBFlushes == pCpu->cTLBFlushes, ("Flush count mismatch for cpu %d (%x vs %x)\n", pCpu->idCpu, pVCpu->hwaccm.s.cTLBFlushes, pCpu->cTLBFlushes));
1852	AssertMsg(pCpu->uCurrentASID >= 1 && pCpu->uCurrentASID < pVM->hwaccm.s.uMaxASID, ("cpu%d uCurrentASID = %x\n", pCpu->idCpu, pCpu->uCurrentASID));
1853	AssertMsg(pVCpu->hwaccm.s.uCurrentASID >= 1 && pVCpu->hwaccm.s.uCurrentASID < pVM->hwaccm.s.uMaxASID, ("cpu%d VM uCurrentASID = %x\n", pCpu->idCpu, pVCpu->hwaccm.s.uCurrentASID));
1854
1855	int rc = VMXWriteVMCS(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hwaccm.s.uCurrentASID);
1856	AssertRC(rc);
1857
1858	if (pVCpu->hwaccm.s.fForceTLBFlush)
1859	vmxR0FlushVPID(pVM, pVCpu, pVM->hwaccm.s.vmx.enmFlushContext, 0);
1860
1861	#ifdef VBOX_WITH_STATISTICS
1862	if (pVCpu->hwaccm.s.fForceTLBFlush)
1863	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatFlushTLBWorldSwitch);
1864	else
1865	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatNoFlushTLBWorldSwitch);
1866	#endif
1867	}
1868	#endif /* HWACCM_VTX_WITH_VPID */
1869
1870	/**
1871	* Runs guest code in a VT-x VM.
1872	*
1873	* @returns VBox status code.
1874	* @param pVM The VM to operate on.
1875	* @param pVCpu The VMCPU to operate on.
1876	* @param pCtx Guest context
1877	*/
1878	VMMR0DECL(int) VMXR0RunGuestCode(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
1879	{
1880	int rc = VINF_SUCCESS;
1881	RTGCUINTREG val;
1882	RTGCUINTREG exitReason, instrError, cbInstr;
1883	RTGCUINTPTR exitQualification;
1884	RTGCUINTPTR intInfo = 0; /* shut up buggy gcc 4 */
1885	RTGCUINTPTR errCode, instrInfo;
1886	bool fSyncTPR = false;
1887	PHWACCM_CPUINFO pCpu = 0;
1888	unsigned cResume = 0;
1889	#ifdef VBOX_STRICT
1890	RTCPUID idCpuCheck;
1891	#endif
1892	#ifdef VBOX_WITH_STATISTICS
1893	bool fStatEntryStarted = true;
1894	bool fStatExit2Started = false;
1895	#endif
1896
1897	Log2(("\nE"));
1898
1899	STAM_PROFILE_ADV_START(&pVCpu->hwaccm.s.StatEntry, x);
1900
1901	#ifdef VBOX_STRICT
1902	{
1903	RTCCUINTREG val;
1904
1905	rc = VMXReadVMCS(VMX_VMCS_CTRL_PIN_EXEC_CONTROLS, &val);
1906	AssertRC(rc);
1907	Log2(("VMX_VMCS_CTRL_PIN_EXEC_CONTROLS = %08x\n", val));
1908
1909	/* allowed zero */
1910	if ((val & pVM->hwaccm.s.vmx.msr.vmx_pin_ctls.n.disallowed0) != pVM->hwaccm.s.vmx.msr.vmx_pin_ctls.n.disallowed0)
1911	Log(("Invalid VMX_VMCS_CTRL_PIN_EXEC_CONTROLS: zero\n"));
1912
1913	/* allowed one */
1914	if ((val & ~pVM->hwaccm.s.vmx.msr.vmx_pin_ctls.n.allowed1) != 0)
1915	Log(("Invalid VMX_VMCS_CTRL_PIN_EXEC_CONTROLS: one\n"));
1916
1917	rc = VMXReadVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, &val);
1918	AssertRC(rc);
1919	Log2(("VMX_VMCS_CTRL_PROC_EXEC_CONTROLS = %08x\n", val));
1920
1921	/* Must be set according to the MSR, but can be cleared in case of EPT. */
1922	if (pVM->hwaccm.s.fNestedPaging)
1923	val \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INVLPG_EXIT
1924	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
1925	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT;
1926
1927	/* allowed zero */
1928	if ((val & pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.disallowed0) != pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.disallowed0)
1929	Log(("Invalid VMX_VMCS_CTRL_PROC_EXEC_CONTROLS: zero\n"));
1930
1931	/* allowed one */
1932	if ((val & ~pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1) != 0)
1933	Log(("Invalid VMX_VMCS_CTRL_PROC_EXEC_CONTROLS: one\n"));
1934
1935	rc = VMXReadVMCS(VMX_VMCS_CTRL_ENTRY_CONTROLS, &val);
1936	AssertRC(rc);
1937	Log2(("VMX_VMCS_CTRL_ENTRY_CONTROLS = %08x\n", val));
1938
1939	/* allowed zero */
1940	if ((val & pVM->hwaccm.s.vmx.msr.vmx_entry.n.disallowed0) != pVM->hwaccm.s.vmx.msr.vmx_entry.n.disallowed0)
1941	Log(("Invalid VMX_VMCS_CTRL_ENTRY_CONTROLS: zero\n"));
1942
1943	/* allowed one */
1944	if ((val & ~pVM->hwaccm.s.vmx.msr.vmx_entry.n.allowed1) != 0)
1945	Log(("Invalid VMX_VMCS_CTRL_ENTRY_CONTROLS: one\n"));
1946
1947	rc = VMXReadVMCS(VMX_VMCS_CTRL_EXIT_CONTROLS, &val);
1948	AssertRC(rc);
1949	Log2(("VMX_VMCS_CTRL_EXIT_CONTROLS = %08x\n", val));
1950
1951	/* allowed zero */
1952	if ((val & pVM->hwaccm.s.vmx.msr.vmx_exit.n.disallowed0) != pVM->hwaccm.s.vmx.msr.vmx_exit.n.disallowed0)
1953	Log(("Invalid VMX_VMCS_CTRL_EXIT_CONTROLS: zero\n"));
1954
1955	/* allowed one */
1956	if ((val & ~pVM->hwaccm.s.vmx.msr.vmx_exit.n.allowed1) != 0)
1957	Log(("Invalid VMX_VMCS_CTRL_EXIT_CONTROLS: one\n"));
1958	}
1959	#endif
1960
1961	/* We can jump to this point to resume execution after determining that a VM-exit is innocent.
1962	*/
1963	ResumeExecution:
1964	STAM_STATS({
1965	if (fStatExit2Started) { STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2, y); fStatExit2Started = false; }
1966	if (!fStatEntryStarted) { STAM_PROFILE_ADV_START(&pVCpu->hwaccm.s.StatEntry, x); fStatEntryStarted = true; }
1967	});
1968	AssertMsg(pVCpu->hwaccm.s.idEnteredCpu == RTMpCpuId(),
1969	("Expected %d, I'm %d; cResume=%d exitReason=%RGv exitQualification=%RGv\n",
1970	(int)pVCpu->hwaccm.s.idEnteredCpu, (int)RTMpCpuId(), cResume, exitReason, exitQualification));
1971	Assert(!HWACCMR0SuspendPending());
1972
1973	/* Safety precaution; looping for too long here can have a very bad effect on the host */
1974	if (++cResume > HWACCM_MAX_RESUME_LOOPS)
1975	{
1976	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitMaxResume);
1977	rc = VINF_EM_RAW_INTERRUPT;
1978	goto end;
1979	}
1980
1981	/* Check for irq inhibition due to instruction fusing (sti, mov ss). */
1982	if (VM_FF_ISSET(pVM, VM_FF_INHIBIT_INTERRUPTS))
1983	{
1984	Log(("VM_FF_INHIBIT_INTERRUPTS at %RGv successor %RGv\n", (RTGCPTR)pCtx->rip, EMGetInhibitInterruptsPC(pVM)));
1985	if (pCtx->rip != EMGetInhibitInterruptsPC(pVM))
1986	{
1987	/* Note: we intentionally don't clear VM_FF_INHIBIT_INTERRUPTS here.
1988	* Before we are able to execute this instruction in raw mode (iret to guest code) an external interrupt might
1989	* force a world switch again. Possibly allowing a guest interrupt to be dispatched in the process. This could
1990	* break the guest. Sounds very unlikely, but such timing sensitive problems are not as rare as you might think.
1991	*/
1992	VM_FF_CLEAR(pVM, VM_FF_INHIBIT_INTERRUPTS);
1993	/* Irq inhibition is no longer active; clear the corresponding VMX state. */
1994	rc = VMXWriteVMCS(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, 0);
1995	AssertRC(rc);
1996	}
1997	}
1998	else
1999	{
2000	/* Irq inhibition is no longer active; clear the corresponding VMX state. */
2001	rc = VMXWriteVMCS(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, 0);
2002	AssertRC(rc);
2003	}
2004
2005	/* Check for pending actions that force us to go back to ring 3. */
2006	if (VM_FF_ISPENDING(pVM, VM_FF_TO_R3 \| VM_FF_TIMER))
2007	{
2008	VM_FF_CLEAR(pVM, VM_FF_TO_R3);
2009	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatSwitchToR3);
2010	rc = VINF_EM_RAW_TO_R3;
2011	goto end;
2012	}
2013	/* Pending request packets might contain actions that need immediate attention, such as pending hardware interrupts. */
2014	if (VM_FF_ISPENDING(pVM, VM_FF_REQUEST))
2015	{
2016	rc = VINF_EM_PENDING_REQUEST;
2017	goto end;
2018	}
2019
2020	/* When external interrupts are pending, we should exit the VM when IF is set. */
2021	/* Note! After VM_FF_INHIBIT_INTERRUPTS check!!! */
2022	rc = VMXR0CheckPendingInterrupt(pVM, pVCpu, pCtx);
2023	if (RT_FAILURE(rc))
2024	goto end;
2025
2026	/** @todo check timers?? */
2027
2028	/* TPR caching using CR8 is only available in 64 bits mode */
2029	/* Note the 32 bits exception for AMD (X86_CPUID_AMD_FEATURE_ECX_CR8L), but that appears missing in Intel CPUs */
2030	/* Note: we can't do this in LoadGuestState as PDMApicGetTPR can jump back to ring 3 (lock)!!!!! */
2031	/**
2032	* @todo reduce overhead
2033	*/
2034	if ( (pCtx->msrEFER & MSR_K6_EFER_LMA)
2035	&& pVM->hwaccm.s.vmx.pAPIC)
2036	{
2037	/* TPR caching in CR8 */
2038	uint8_t u8TPR;
2039	bool fPending;
2040
2041	int rc = PDMApicGetTPR(pVM, &u8TPR, &fPending);
2042	AssertRC(rc);
2043	/* The TPR can be found at offset 0x80 in the APIC mmio page. */
2044	pVM->hwaccm.s.vmx.pAPIC[0x80] = u8TPR << 4; /* bits 7-4 contain the task priority */
2045
2046	/* Two options here:
2047	* - external interrupt pending, but masked by the TPR value.
2048	* -> a CR8 update that lower the current TPR value should cause an exit
2049	* - no pending interrupts
2050	* -> We don't need to be explicitely notified. There are enough world switches for detecting pending interrupts.
2051	*/
2052	rc = VMXWriteVMCS(VMX_VMCS_CTRL_TPR_THRESHOLD, (fPending) ? u8TPR : 0);
2053	AssertRC(rc);
2054
2055	/* Always sync back the TPR; we should optimize this though / /* @todo optimize TPR sync. */
2056	fSyncTPR = true;
2057	}
2058
2059	#if defined(HWACCM_VTX_WITH_EPT) && defined(LOG_ENABLED)
2060	if ( pVM->hwaccm.s.fNestedPaging
2061	# ifdef HWACCM_VTX_WITH_VPID
2062	\|\| pVM->hwaccm.s.vmx.fVPID
2063	# endif /* HWACCM_VTX_WITH_VPID */
2064	)
2065	{
2066	pCpu = HWACCMR0GetCurrentCpu();
2067	if ( pVCpu->hwaccm.s.idLastCpu != pCpu->idCpu
2068	\|\| pVCpu->hwaccm.s.cTLBFlushes != pCpu->cTLBFlushes)
2069	{
2070	if (pVCpu->hwaccm.s.idLastCpu != pCpu->idCpu)
2071	Log(("Force TLB flush due to rescheduling to a different cpu (%d vs %d)\n", pVCpu->hwaccm.s.idLastCpu, pCpu->idCpu));
2072	else
2073	Log(("Force TLB flush due to changed TLB flush count (%x vs %x)\n", pVCpu->hwaccm.s.cTLBFlushes, pCpu->cTLBFlushes));
2074	}
2075	if (pCpu->fFlushTLB)
2076	Log(("Force TLB flush: first time cpu %d is used -> flush\n", pCpu->idCpu));
2077	else
2078	if (pVCpu->hwaccm.s.fForceTLBFlush)
2079	LogFlow(("Manual TLB flush\n"));
2080	}
2081	#endif
2082	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
2083	PGMDynMapFlushAutoSet(pVCpu);
2084	#endif
2085
2086	/*
2087	* NOTE: DO NOT DO ANYTHING AFTER THIS POINT THAT MIGHT JUMP BACK TO RING 3!
2088	* (until the actual world switch)
2089	*/
2090	#ifdef VBOX_STRICT
2091	idCpuCheck = RTMpCpuId();
2092	#endif
2093	#ifdef LOG_LOGGING
2094	VMMR0LogFlushDisable(pVCpu);
2095	#endif
2096	/* Save the host state first. */
2097	rc = VMXR0SaveHostState(pVM, pVCpu);
2098	if (rc != VINF_SUCCESS)
2099	goto end;
2100	/* Load the guest state */
2101	rc = VMXR0LoadGuestState(pVM, pVCpu, pCtx);
2102	if (rc != VINF_SUCCESS)
2103	goto end;
2104
2105	/* Deal with tagged TLB setup and invalidation. */
2106	pVM->hwaccm.s.vmx.pfnSetupTaggedTLB(pVM, pVCpu);
2107
2108	/* Non-register state Guest Context */
2109	/** @todo change me according to cpu state */
2110	rc = VMXWriteVMCS(VMX_VMCS32_GUEST_ACTIVITY_STATE, VMX_CMS_GUEST_ACTIVITY_ACTIVE);
2111	AssertRC(rc);
2112
2113	STAM_STATS({ STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatEntry, x); fStatEntryStarted = false; });
2114
2115	/* Manual save and restore:
2116	* - General purpose registers except RIP, RSP
2117	*
2118	* Trashed:
2119	* - CR2 (we don't care)
2120	* - LDTR (reset to 0)
2121	* - DRx (presumably not changed at all)
2122	* - DR7 (reset to 0x400)
2123	* - EFLAGS (reset to RT_BIT(1); not relevant)
2124	*
2125	*/
2126
2127	/* All done! Let's start VM execution. */
2128	STAM_PROFILE_ADV_START(&pVCpu->hwaccm.s.StatInGC, z);
2129	#ifdef VBOX_STRICT
2130	Assert(idCpuCheck == RTMpCpuId());
2131	#endif
2132	TMNotifyStartOfExecution(pVM);
2133	rc = pVCpu->hwaccm.s.vmx.pfnStartVM(pVCpu->hwaccm.s.fResumeVM, pCtx, &pVCpu->hwaccm.s.vmx.VMCSCache, pVM, pVCpu);
2134	TMNotifyEndOfExecution(pVM);
2135
2136	AssertMsg(!pVCpu->hwaccm.s.vmx.VMCSCache.Write.cValidEntries, ("pVCpu->hwaccm.s.vmx.VMCSCache.Write.cValidEntries=%d\n", pVCpu->hwaccm.s.vmx.VMCSCache.Write.cValidEntries));
2137
2138	/* In case we execute a goto ResumeExecution later on. */
2139	pVCpu->hwaccm.s.fResumeVM = true;
2140	pVCpu->hwaccm.s.fForceTLBFlush = false;
2141
2142	/*
2143	* !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
2144	* IMPORTANT: WE CAN'T DO ANY LOGGING OR OPERATIONS THAT CAN DO A LONGJMP BACK TO RING 3 BEFORE WE'VE SYNCED BACK (MOST OF) THE GUEST STATE
2145	* !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
2146	*/
2147
2148	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatInGC, z);
2149	STAM_PROFILE_ADV_START(&pVCpu->hwaccm.s.StatExit1, v);
2150
2151	if (rc != VINF_SUCCESS)
2152	{
2153	VMXR0ReportWorldSwitchError(pVM, pVCpu, rc, pCtx);
2154	goto end;
2155	}
2156	/* Success. Query the guest state and figure out what has happened. */
2157
2158	/* Investigate why there was a VM-exit. */
2159	rc = VMXReadCachedVMCS(VMX_VMCS32_RO_EXIT_REASON, &exitReason);
2160	STAM_COUNTER_INC(&pVCpu->hwaccm.s.paStatExitReasonR0[exitReason & MASK_EXITREASON_STAT]);
2161
2162	exitReason &= 0xffff; /* bit 0-15 contain the exit code. */
2163	rc \|= VMXReadCachedVMCS(VMX_VMCS32_RO_VM_INSTR_ERROR, &instrError);
2164	rc \|= VMXReadCachedVMCS(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &cbInstr);
2165	rc \|= VMXReadCachedVMCS(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &intInfo);
2166	/* might not be valid; depends on VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID. */
2167	rc \|= VMXReadCachedVMCS(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERRCODE, &errCode);
2168	rc \|= VMXReadCachedVMCS(VMX_VMCS32_RO_EXIT_INSTR_INFO, &instrInfo);
2169	rc \|= VMXReadCachedVMCS(VMX_VMCS_RO_EXIT_QUALIFICATION, &exitQualification);
2170	AssertRC(rc);
2171
2172	/* Sync back the guest state */
2173	rc = VMXR0SaveGuestState(pVM, pVCpu, pCtx);
2174	AssertRC(rc);
2175
2176	/* Note! NOW IT'S SAFE FOR LOGGING! */
2177	#ifdef LOG_LOGGING
2178	VMMR0LogFlushEnable(pVCpu);
2179	#endif
2180	Log2(("Raw exit reason %08x\n", exitReason));
2181
2182	/* Check if an injected event was interrupted prematurely. */
2183	rc = VMXReadCachedVMCS(VMX_VMCS32_RO_IDT_INFO, &val);
2184	AssertRC(rc);
2185	pVCpu->hwaccm.s.Event.intInfo = VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(val);
2186	if ( VMX_EXIT_INTERRUPTION_INFO_VALID(pVCpu->hwaccm.s.Event.intInfo)
2187	/* Ignore 'int xx' as they'll be restarted anyway. */
2188	&& VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hwaccm.s.Event.intInfo) != VMX_EXIT_INTERRUPTION_INFO_TYPE_SW
2189	/* Ignore software exceptions (such as int3) as they're reoccur when we restart the instruction anyway. */
2190	&& VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hwaccm.s.Event.intInfo) != VMX_EXIT_INTERRUPTION_INFO_TYPE_SWEXCPT)
2191	{
2192	pVCpu->hwaccm.s.Event.fPending = true;
2193	/* Error code present? */
2194	if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(pVCpu->hwaccm.s.Event.intInfo))
2195	{
2196	rc = VMXReadCachedVMCS(VMX_VMCS32_RO_IDT_ERRCODE, &val);
2197	AssertRC(rc);
2198	pVCpu->hwaccm.s.Event.errCode = val;
2199	Log(("Pending inject %RX64 at %RGv exit=%08x intInfo=%08x exitQualification=%RGv pending error=%RX64\n", pVCpu->hwaccm.s.Event.intInfo, (RTGCPTR)pCtx->rip, exitReason, intInfo, exitQualification, val));
2200	}
2201	else
2202	{
2203	Log(("Pending inject %RX64 at %RGv exit=%08x intInfo=%08x exitQualification=%RGv\n", pVCpu->hwaccm.s.Event.intInfo, (RTGCPTR)pCtx->rip, exitReason, intInfo, exitQualification));
2204	pVCpu->hwaccm.s.Event.errCode = 0;
2205	}
2206	}
2207	#ifdef VBOX_STRICT
2208	else
2209	if ( VMX_EXIT_INTERRUPTION_INFO_VALID(pVCpu->hwaccm.s.Event.intInfo)
2210	/* Ignore software exceptions (such as int3) as they're reoccur when we restart the instruction anyway. */
2211	&& VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hwaccm.s.Event.intInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_SWEXCPT)
2212	{
2213	Log(("Ignore pending inject %RX64 at %RGv exit=%08x intInfo=%08x exitQualification=%RGv\n", pVCpu->hwaccm.s.Event.intInfo, (RTGCPTR)pCtx->rip, exitReason, intInfo, exitQualification));
2214	}
2215
2216	if (exitReason == VMX_EXIT_ERR_INVALID_GUEST_STATE)
2217	HWACCMDumpRegs(pVM, pCtx);
2218	#endif
2219
2220	Log2(("E%d", exitReason));
2221	Log2(("Exit reason %d, exitQualification %RGv\n", (uint32_t)exitReason, exitQualification));
2222	Log2(("instrInfo=%d instrError=%d instr length=%d\n", (uint32_t)instrInfo, (uint32_t)instrError, (uint32_t)cbInstr));
2223	Log2(("Interruption error code %d\n", (uint32_t)errCode));
2224	Log2(("IntInfo = %08x\n", (uint32_t)intInfo));
2225	Log2(("New EIP=%RGv\n", (RTGCPTR)pCtx->rip));
2226
2227	if (fSyncTPR)
2228	{
2229	rc = PDMApicSetTPR(pVM, pVM->hwaccm.s.vmx.pAPIC[0x80] >> 4);
2230	AssertRC(rc);
2231	}
2232
2233	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit1, v);
2234	STAM_STATS({ STAM_PROFILE_ADV_START(&pVCpu->hwaccm.s.StatExit2, y); fStatExit2Started = true; });
2235
2236	/* Some cases don't need a complete resync of the guest CPU state; handle them here. */
2237	switch (exitReason)
2238	{
2239	case VMX_EXIT_EXCEPTION: /* 0 Exception or non-maskable interrupt (NMI). */
2240	case VMX_EXIT_EXTERNAL_IRQ: /* 1 External interrupt. */
2241	{
2242	uint32_t vector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(intInfo);
2243
2244	if (!VMX_EXIT_INTERRUPTION_INFO_VALID(intInfo))
2245	{
2246	Assert(exitReason == VMX_EXIT_EXTERNAL_IRQ);
2247	/* External interrupt; leave to allow it to be dispatched again. */
2248	rc = VINF_EM_RAW_INTERRUPT;
2249	break;
2250	}
2251	STAM_PROFILE_ADV_START(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
2252	switch (VMX_EXIT_INTERRUPTION_INFO_TYPE(intInfo))
2253	{
2254	case VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI: /* Non-maskable interrupt. */
2255	/* External interrupt; leave to allow it to be dispatched again. */
2256	rc = VINF_EM_RAW_INTERRUPT;
2257	break;
2258
2259	case VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT: /* External hardware interrupt. */
2260	AssertFailed(); /* can't come here; fails the first check. */
2261	break;
2262
2263	case VMX_EXIT_INTERRUPTION_INFO_TYPE_DBEXCPT: /* Unknown why we get this type for #DB */
2264	case VMX_EXIT_INTERRUPTION_INFO_TYPE_SWEXCPT: /* Software exception. (#BP or #OF) */
2265	Assert(vector == 1 \|\| vector == 3 \|\| vector == 4);
2266	/* no break */
2267	case VMX_EXIT_INTERRUPTION_INFO_TYPE_HWEXCPT: /* Hardware exception. */
2268	Log2(("Hardware/software interrupt %d\n", vector));
2269	switch (vector)
2270	{
2271	case X86_XCPT_NM:
2272	{
2273	Log(("#NM fault at %RGv error code %x\n", (RTGCPTR)pCtx->rip, errCode));
2274
2275	/** @todo don't intercept #NM exceptions anymore when we've activated the guest FPU state. */
2276	/* If we sync the FPU/XMM state on-demand, then we can continue execution as if nothing has happened. */
2277	rc = CPUMR0LoadGuestFPU(pVM, pVCpu, pCtx);
2278	if (rc == VINF_SUCCESS)
2279	{
2280	Assert(CPUMIsGuestFPUStateActive(pVCpu));
2281
2282	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitShadowNM);
2283
2284	/* Continue execution. */
2285	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_GUEST_CR0;
2286
2287	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
2288	goto ResumeExecution;
2289	}
2290
2291	Log(("Forward #NM fault to the guest\n"));
2292	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestNM);
2293	rc = VMXR0InjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), cbInstr, 0);
2294	AssertRC(rc);
2295	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
2296	goto ResumeExecution;
2297	}
2298
2299	case X86_XCPT_PF: /* Page fault */
2300	{
2301	#ifdef DEBUG
2302	if (pVM->hwaccm.s.fNestedPaging)
2303	{ /* A genuine pagefault.
2304	* Forward the trap to the guest by injecting the exception and resuming execution.
2305	*/
2306	Log(("Guest page fault at %RGv cr2=%RGv error code %x rsp=%RGv\n", (RTGCPTR)pCtx->rip, exitQualification, errCode, (RTGCPTR)pCtx->rsp));
2307
2308	Assert(CPUMIsGuestInPagedProtectedModeEx(pCtx));
2309
2310	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestPF);
2311
2312	/* Now we must update CR2. */
2313	pCtx->cr2 = exitQualification;
2314	rc = VMXR0InjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), cbInstr, errCode);
2315	AssertRC(rc);
2316
2317	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
2318	goto ResumeExecution;
2319	}
2320	#endif
2321	Assert(!pVM->hwaccm.s.fNestedPaging);
2322
2323	Log2(("Page fault at %RGv error code %x\n", exitQualification, errCode));
2324	/* Exit qualification contains the linear address of the page fault. */
2325	TRPMAssertTrap(pVM, X86_XCPT_PF, TRPM_TRAP);
2326	TRPMSetErrorCode(pVM, errCode);
2327	TRPMSetFaultAddress(pVM, exitQualification);
2328
2329	/* Forward it to our trap handler first, in case our shadow pages are out of sync. */
2330	rc = PGMTrap0eHandler(pVM, errCode, CPUMCTX2CORE(pCtx), (RTGCPTR)exitQualification);
2331	Log2(("PGMTrap0eHandler %RGv returned %Rrc\n", (RTGCPTR)pCtx->rip, rc));
2332	if (rc == VINF_SUCCESS)
2333	{ /* We've successfully synced our shadow pages, so let's just continue execution. */
2334	Log2(("Shadow page fault at %RGv cr2=%RGv error code %x\n", (RTGCPTR)pCtx->rip, exitQualification ,errCode));
2335	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitShadowPF);
2336
2337	TRPMResetTrap(pVM);
2338
2339	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
2340	goto ResumeExecution;
2341	}
2342	else
2343	if (rc == VINF_EM_RAW_GUEST_TRAP)
2344	{ /* A genuine pagefault.
2345	* Forward the trap to the guest by injecting the exception and resuming execution.
2346	*/
2347	Log2(("Forward page fault to the guest\n"));
2348
2349	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestPF);
2350	/* The error code might have been changed. */
2351	errCode = TRPMGetErrorCode(pVM);
2352
2353	TRPMResetTrap(pVM);
2354
2355	/* Now we must update CR2. */
2356	pCtx->cr2 = exitQualification;
2357	rc = VMXR0InjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), cbInstr, errCode);
2358	AssertRC(rc);
2359
2360	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
2361	goto ResumeExecution;
2362	}
2363	#ifdef VBOX_STRICT
2364	if (rc != VINF_EM_RAW_EMULATE_INSTR)
2365	Log2(("PGMTrap0eHandler failed with %d\n", rc));
2366	#endif
2367	/* Need to go back to the recompiler to emulate the instruction. */
2368	TRPMResetTrap(pVM);
2369	break;
2370	}
2371
2372	case X86_XCPT_MF: /* Floating point exception. */
2373	{
2374	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestMF);
2375	if (!(pCtx->cr0 & X86_CR0_NE))
2376	{
2377	/* old style FPU error reporting needs some extra work. */
2378	/** @todo don't fall back to the recompiler, but do it manually. */
2379	rc = VINF_EM_RAW_EMULATE_INSTR;
2380	break;
2381	}
2382	Log(("Trap %x at %04X:%RGv\n", vector, pCtx->cs, (RTGCPTR)pCtx->rip));
2383	rc = VMXR0InjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), cbInstr, errCode);
2384	AssertRC(rc);
2385
2386	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
2387	goto ResumeExecution;
2388	}
2389
2390	case X86_XCPT_DB: /* Debug exception. */
2391	{
2392	uint64_t uDR6;
2393
2394	/* DR6, DR7.GD and IA32_DEBUGCTL.LBR are not updated yet.
2395	*
2396	* Exit qualification bits:
2397	* 3:0 B0-B3 which breakpoint condition was met
2398	* 12:4 Reserved (0)
2399	* 13 BD - debug register access detected
2400	* 14 BS - single step execution or branch taken
2401	* 63:15 Reserved (0)
2402	*/
2403	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestDB);
2404
2405	/* Note that we don't support guest and host-initiated debugging at the same time. */
2406	Assert(DBGFIsStepping(pVM) \|\| CPUMIsGuestInRealModeEx(pCtx));
2407
2408	uDR6 = X86_DR6_INIT_VAL;
2409	uDR6 \|= (exitQualification & (X86_DR6_B0\|X86_DR6_B1\|X86_DR6_B2\|X86_DR6_B3\|X86_DR6_BD\|X86_DR6_BS));
2410	rc = DBGFR0Trap01Handler(pVM, CPUMCTX2CORE(pCtx), uDR6);
2411	if (rc == VINF_EM_RAW_GUEST_TRAP)
2412	{
2413	/** @todo this isn't working, but we'll never get here normally. */
2414
2415	/* Update DR6 here. */
2416	pCtx->dr[6] = uDR6;
2417
2418	/* X86_DR7_GD will be cleared if drx accesses should be trapped inside the guest. */
2419	pCtx->dr[7] &= ~X86_DR7_GD;
2420
2421	/* Paranoia. */
2422	pCtx->dr[7] &= 0xffffffff; /* upper 32 bits reserved */
2423	pCtx->dr[7] &= ~(RT_BIT(11) \| RT_BIT(12) \| RT_BIT(14) \| RT_BIT(15)); /* must be zero */
2424	pCtx->dr[7] \|= 0x400; /* must be one */
2425
2426	/* Resync DR7 */
2427	rc = VMXWriteVMCS64(VMX_VMCS64_GUEST_DR7, pCtx->dr[7]);
2428	AssertRC(rc);
2429
2430	Log(("Trap %x (debug) at %RGv exit qualification %RX64\n", vector, (RTGCPTR)pCtx->rip, exitQualification));
2431	rc = VMXR0InjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), cbInstr, errCode);
2432	AssertRC(rc);
2433
2434	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
2435	goto ResumeExecution;
2436	}
2437	/* Return to ring 3 to deal with the debug exit code. */
2438	break;
2439	}
2440
2441	case X86_XCPT_GP: /* General protection failure exception.*/
2442	{
2443	uint32_t cbSize;
2444
2445	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestGP);
2446	#ifdef VBOX_STRICT
2447	if (!CPUMIsGuestInRealModeEx(pCtx))
2448	{
2449	Log(("Trap %x at %04X:%RGv errorCode=%x\n", vector, pCtx->cs, (RTGCPTR)pCtx->rip, errCode));
2450	rc = VMXR0InjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), cbInstr, errCode);
2451	AssertRC(rc);
2452	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
2453	goto ResumeExecution;
2454	}
2455	#endif
2456	Assert(CPUMIsGuestInRealModeEx(pCtx));
2457
2458	LogFlow(("Real mode X86_XCPT_GP instruction emulation at %RGv\n", (RTGCPTR)pCtx->rip));
2459	rc = EMInterpretInstruction(pVM, CPUMCTX2CORE(pCtx), 0, &cbSize);
2460	if (rc == VINF_SUCCESS)
2461	{
2462	/* EIP has been updated already. */
2463
2464	/* lidt, lgdt can end up here. In the future crx changes as well. Just reload the whole context to be done with it. */
2465	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_ALL;
2466
2467	/* Only resume if successful. */
2468	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
2469	goto ResumeExecution;
2470	}
2471	AssertMsg(rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_EM_HALT, ("Unexpected rc=%Rrc\n", rc));
2472	break;
2473	}
2474
2475	#ifdef VBOX_STRICT
2476	case X86_XCPT_DE: /* Divide error. */
2477	case X86_XCPT_UD: /* Unknown opcode exception. */
2478	case X86_XCPT_SS: /* Stack segment exception. */
2479	case X86_XCPT_NP: /* Segment not present exception. */
2480	{
2481	switch(vector)
2482	{
2483	case X86_XCPT_DE:
2484	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestDE);
2485	break;
2486	case X86_XCPT_UD:
2487	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestUD);
2488	break;
2489	case X86_XCPT_SS:
2490	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestSS);
2491	break;
2492	case X86_XCPT_NP:
2493	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestNP);
2494	break;
2495	}
2496
2497	Log(("Trap %x at %04X:%RGv\n", vector, pCtx->cs, (RTGCPTR)pCtx->rip));
2498	rc = VMXR0InjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), cbInstr, errCode);
2499	AssertRC(rc);
2500
2501	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
2502	goto ResumeExecution;
2503	}
2504	#endif
2505	default:
2506	#ifdef HWACCM_VMX_EMULATE_REALMODE
2507	if (CPUMIsGuestInRealModeEx(pCtx))
2508	{
2509	Log(("Real Mode Trap %x at %04x:%04X error code %x\n", vector, pCtx->cs, pCtx->eip, errCode));
2510	rc = VMXR0InjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), cbInstr, errCode);
2511	AssertRC(rc);
2512
2513	/* Go back to ring 3 in case of a triple fault. */
2514	if ( vector == X86_XCPT_DF
2515	&& rc == VINF_EM_RESET)
2516	break;
2517
2518	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
2519	goto ResumeExecution;
2520	}
2521	#endif
2522	AssertMsgFailed(("Unexpected vm-exit caused by exception %x\n", vector));
2523	rc = VERR_VMX_UNEXPECTED_EXCEPTION;
2524	break;
2525	} /* switch (vector) */
2526
2527	break;
2528
2529	default:
2530	rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_CODE;
2531	AssertMsgFailed(("Unexpected interuption code %x\n", intInfo));
2532	break;
2533	}
2534
2535	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
2536	break;
2537	}
2538
2539	case VMX_EXIT_EPT_VIOLATION: /* 48 EPT violation. An attempt to access memory with a guest-physical address was disallowed by the configuration of the EPT paging structures. */
2540	{
2541	RTGCPHYS GCPhys;
2542
2543	Assert(pVM->hwaccm.s.fNestedPaging);
2544
2545	rc = VMXReadVMCS64(VMX_VMCS_EXIT_PHYS_ADDR_FULL, &GCPhys);
2546	AssertRC(rc);
2547	Assert(((exitQualification >> 7) & 3) != 2);
2548
2549	/* Determine the kind of violation. */
2550	errCode = 0;
2551	if (exitQualification & VMX_EXIT_QUALIFICATION_EPT_INSTR_FETCH)
2552	errCode \|= X86_TRAP_PF_ID;
2553
2554	if (exitQualification & VMX_EXIT_QUALIFICATION_EPT_DATA_WRITE)
2555	errCode \|= X86_TRAP_PF_RW;
2556
2557	/* If the page is present, then it's a page level protection fault. */
2558	if (exitQualification & VMX_EXIT_QUALIFICATION_EPT_ENTRY_PRESENT)
2559	errCode \|= X86_TRAP_PF_P;
2560
2561	Log(("EPT Page fault %x at %RGp error code %x\n", (uint32_t)exitQualification, GCPhys, errCode));
2562
2563	/* GCPhys contains the guest physical address of the page fault. */
2564	TRPMAssertTrap(pVM, X86_XCPT_PF, TRPM_TRAP);
2565	TRPMSetErrorCode(pVM, errCode);
2566	TRPMSetFaultAddress(pVM, GCPhys);
2567
2568	/* Handle the pagefault trap for the nested shadow table. */
2569	rc = PGMR0Trap0eHandlerNestedPaging(pVM, PGMMODE_EPT, errCode, CPUMCTX2CORE(pCtx), GCPhys);
2570	Log2(("PGMR0Trap0eHandlerNestedPaging %RGv returned %Rrc\n", (RTGCPTR)pCtx->rip, rc));
2571	if (rc == VINF_SUCCESS)
2572	{ /* We've successfully synced our shadow pages, so let's just continue execution. */
2573	Log2(("Shadow page fault at %RGv cr2=%RGp error code %x\n", (RTGCPTR)pCtx->rip, exitQualification , errCode));
2574	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitReasonNPF);
2575
2576	TRPMResetTrap(pVM);
2577
2578	goto ResumeExecution;
2579	}
2580
2581	#ifdef VBOX_STRICT
2582	if (rc != VINF_EM_RAW_EMULATE_INSTR)
2583	LogFlow(("PGMTrap0eHandlerNestedPaging failed with %d\n", rc));
2584	#endif
2585	/* Need to go back to the recompiler to emulate the instruction. */
2586	TRPMResetTrap(pVM);
2587	break;
2588	}
2589
2590	case VMX_EXIT_EPT_MISCONFIG:
2591	{
2592	RTGCPHYS GCPhys;
2593
2594	Assert(pVM->hwaccm.s.fNestedPaging);
2595
2596	rc = VMXReadVMCS64(VMX_VMCS_EXIT_PHYS_ADDR_FULL, &GCPhys);
2597	AssertRC(rc);
2598
2599	Log(("VMX_EXIT_EPT_MISCONFIG for %VGp\n", GCPhys));
2600	break;
2601	}
2602
2603	case VMX_EXIT_IRQ_WINDOW: /* 7 Interrupt window. */
2604	/* Clear VM-exit on IF=1 change. */
2605	LogFlow(("VMX_EXIT_IRQ_WINDOW %RGv pending=%d IF=%d\n", (RTGCPTR)pCtx->rip, VM_FF_ISPENDING(pVM, (VM_FF_INTERRUPT_APIC\|VM_FF_INTERRUPT_PIC)), pCtx->eflags.Bits.u1IF));
2606	pVCpu->hwaccm.s.vmx.proc_ctls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_IRQ_WINDOW_EXIT;
2607	rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
2608	AssertRC(rc);
2609	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitIrqWindow);
2610	goto ResumeExecution; /* we check for pending guest interrupts there */
2611
2612	case VMX_EXIT_WBINVD: /* 54 Guest software attempted to execute WBINVD. (conditional) */
2613	case VMX_EXIT_INVD: /* 13 Guest software attempted to execute INVD. (unconditional) */
2614	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitInvd);
2615	/* Skip instruction and continue directly. */
2616	pCtx->rip += cbInstr;
2617	/* Continue execution.*/
2618	goto ResumeExecution;
2619
2620	case VMX_EXIT_CPUID: /* 10 Guest software attempted to execute CPUID. */
2621	{
2622	Log2(("VMX: Cpuid %x\n", pCtx->eax));
2623	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitCpuid);
2624	rc = EMInterpretCpuId(pVM, CPUMCTX2CORE(pCtx));
2625	if (rc == VINF_SUCCESS)
2626	{
2627	/* Update EIP and continue execution. */
2628	Assert(cbInstr == 2);
2629	pCtx->rip += cbInstr;
2630	goto ResumeExecution;
2631	}
2632	AssertMsgFailed(("EMU: cpuid failed with %Rrc\n", rc));
2633	rc = VINF_EM_RAW_EMULATE_INSTR;
2634	break;
2635	}
2636
2637	case VMX_EXIT_RDTSC: /* 16 Guest software attempted to execute RDTSC. */
2638	{
2639	Log2(("VMX: Rdtsc\n"));
2640	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitRdtsc);
2641	rc = EMInterpretRdtsc(pVM, CPUMCTX2CORE(pCtx));
2642	if (rc == VINF_SUCCESS)
2643	{
2644	/* Update EIP and continue execution. */
2645	Assert(cbInstr == 2);
2646	pCtx->rip += cbInstr;
2647	goto ResumeExecution;
2648	}
2649	AssertMsgFailed(("EMU: rdtsc failed with %Rrc\n", rc));
2650	rc = VINF_EM_RAW_EMULATE_INSTR;
2651	break;
2652	}
2653
2654	case VMX_EXIT_INVPG: /* 14 Guest software attempted to execute INVPG. */
2655	{
2656	Log2(("VMX: invlpg\n"));
2657	Assert(!pVM->hwaccm.s.fNestedPaging);
2658
2659	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitInvpg);
2660	rc = EMInterpretInvlpg(pVM, CPUMCTX2CORE(pCtx), exitQualification);
2661	if (rc == VINF_SUCCESS)
2662	{
2663	/* Update EIP and continue execution. */
2664	pCtx->rip += cbInstr;
2665	goto ResumeExecution;
2666	}
2667	AssertMsg(rc == VERR_EM_INTERPRETER, ("EMU: invlpg %RGv failed with %Rrc\n", exitQualification, rc));
2668	break;
2669	}
2670
2671	case VMX_EXIT_RDMSR: /* 31 RDMSR. Guest software attempted to execute RDMSR. */
2672	case VMX_EXIT_WRMSR: /* 32 WRMSR. Guest software attempted to execute WRMSR. */
2673	{
2674	uint32_t cbSize;
2675
2676	/* Note: the intel manual claims there's a REX version of RDMSR that's slightly different, so we play safe by completely disassembling the instruction. */
2677	Log2(("VMX: %s\n", (exitReason == VMX_EXIT_RDMSR) ? "rdmsr" : "wrmsr"));
2678	rc = EMInterpretInstruction(pVM, CPUMCTX2CORE(pCtx), 0, &cbSize);
2679	if (rc == VINF_SUCCESS)
2680	{
2681	/* EIP has been updated already. */
2682
2683	/* Only resume if successful. */
2684	goto ResumeExecution;
2685	}
2686	AssertMsg(rc == VERR_EM_INTERPRETER, ("EMU: %s failed with %Rrc\n", (exitReason == VMX_EXIT_RDMSR) ? "rdmsr" : "wrmsr", rc));
2687	break;
2688	}
2689
2690	case VMX_EXIT_CRX_MOVE: /* 28 Control-register accesses. */
2691	{
2692	STAM_PROFILE_ADV_START(&pVCpu->hwaccm.s.StatExit2Sub2, y2);
2693
2694	switch (VMX_EXIT_QUALIFICATION_CRX_ACCESS(exitQualification))
2695	{
2696	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_WRITE:
2697	Log2(("VMX: %RGv mov cr%d, x\n", (RTGCPTR)pCtx->rip, VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification)));
2698	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitCRxWrite[VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification)]);
2699	rc = EMInterpretCRxWrite(pVM, CPUMCTX2CORE(pCtx),
2700	VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification),
2701	VMX_EXIT_QUALIFICATION_CRX_GENREG(exitQualification));
2702
2703	switch (VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification))
2704	{
2705	case 0:
2706	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_GUEST_CR0 \| HWACCM_CHANGED_GUEST_CR3;
2707	break;
2708	case 2:
2709	break;
2710	case 3:
2711	Assert(!pVM->hwaccm.s.fNestedPaging \|\| !CPUMIsGuestInPagedProtectedModeEx(pCtx));
2712	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_GUEST_CR3;
2713	break;
2714	case 4:
2715	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_GUEST_CR4;
2716	break;
2717	case 8:
2718	/* CR8 contains the APIC TPR */
2719	Assert(!(pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW));
2720	break;
2721
2722	default:
2723	AssertFailed();
2724	break;
2725	}
2726	/* Check if a sync operation is pending. */
2727	if ( rc == VINF_SUCCESS /* don't bother if we are going to ring 3 anyway */
2728	&& VM_FF_ISPENDING(pVM, VM_FF_PGM_SYNC_CR3 \| VM_FF_PGM_SYNC_CR3_NON_GLOBAL))
2729	{
2730	rc = PGMSyncCR3(pVM, CPUMGetGuestCR0(pVM), CPUMGetGuestCR3(pVM), CPUMGetGuestCR4(pVM), VM_FF_ISSET(pVM, VM_FF_PGM_SYNC_CR3));
2731	AssertRC(rc);
2732	}
2733	break;
2734
2735	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ:
2736	Log2(("VMX: mov x, crx\n"));
2737	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitCRxRead[VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification)]);
2738
2739	Assert(!pVM->hwaccm.s.fNestedPaging \|\| !CPUMIsGuestInPagedProtectedModeEx(pCtx) \|\| VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification) != USE_REG_CR3);
2740
2741	/* CR8 reads only cause an exit when the TPR shadow feature isn't present. */
2742	Assert(VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification) != 8 \|\| !(pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW));
2743
2744	rc = EMInterpretCRxRead(pVM, CPUMCTX2CORE(pCtx),
2745	VMX_EXIT_QUALIFICATION_CRX_GENREG(exitQualification),
2746	VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification));
2747	break;
2748
2749	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_CLTS:
2750	Log2(("VMX: clts\n"));
2751	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitCLTS);
2752	rc = EMInterpretCLTS(pVM);
2753	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_GUEST_CR0;
2754	break;
2755
2756	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_LMSW:
2757	Log2(("VMX: lmsw %x\n", VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(exitQualification)));
2758	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitLMSW);
2759	rc = EMInterpretLMSW(pVM, CPUMCTX2CORE(pCtx), VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(exitQualification));
2760	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_GUEST_CR0;
2761	break;
2762	}
2763
2764	/* Update EIP if no error occurred. */
2765	if (RT_SUCCESS(rc))
2766	pCtx->rip += cbInstr;
2767
2768	if (rc == VINF_SUCCESS)
2769	{
2770	/* Only resume if successful. */
2771	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub2, y2);
2772	goto ResumeExecution;
2773	}
2774	Assert(rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_PGM_SYNC_CR3);
2775	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub2, y2);
2776	break;
2777	}
2778
2779	case VMX_EXIT_DRX_MOVE: /* 29 Debug-register accesses. */
2780	{
2781	if (!DBGFIsStepping(pVM))
2782	{
2783	/* Disable drx move intercepts. */
2784	pVCpu->hwaccm.s.vmx.proc_ctls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT;
2785	rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
2786	AssertRC(rc);
2787
2788	/* Save the host and load the guest debug state. */
2789	rc = CPUMR0LoadGuestDebugState(pVM, pVCpu, pCtx, true /* include DR6 */);
2790	AssertRC(rc);
2791
2792	#ifdef VBOX_WITH_STATISTICS
2793	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatDRxContextSwitch);
2794	if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(exitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
2795	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitDRxWrite);
2796	else
2797	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitDRxRead);
2798	#endif
2799
2800	goto ResumeExecution;
2801	}
2802
2803	/** @todo clear VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT after the first time and restore drx registers afterwards */
2804	if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(exitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
2805	{
2806	Log2(("VMX: mov drx%d, genreg%d\n", VMX_EXIT_QUALIFICATION_DRX_REGISTER(exitQualification), VMX_EXIT_QUALIFICATION_DRX_GENREG(exitQualification)));
2807	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitDRxWrite);
2808	rc = EMInterpretDRxWrite(pVM, CPUMCTX2CORE(pCtx),
2809	VMX_EXIT_QUALIFICATION_DRX_REGISTER(exitQualification),
2810	VMX_EXIT_QUALIFICATION_DRX_GENREG(exitQualification));
2811	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_GUEST_DEBUG;
2812	Log2(("DR7=%08x\n", pCtx->dr[7]));
2813	}
2814	else
2815	{
2816	Log2(("VMX: mov x, drx\n"));
2817	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitDRxRead);
2818	rc = EMInterpretDRxRead(pVM, CPUMCTX2CORE(pCtx),
2819	VMX_EXIT_QUALIFICATION_DRX_GENREG(exitQualification),
2820	VMX_EXIT_QUALIFICATION_DRX_REGISTER(exitQualification));
2821	}
2822	/* Update EIP if no error occurred. */
2823	if (RT_SUCCESS(rc))
2824	pCtx->rip += cbInstr;
2825
2826	if (rc == VINF_SUCCESS)
2827	{
2828	/* Only resume if successful. */
2829	goto ResumeExecution;
2830	}
2831	Assert(rc == VERR_EM_INTERPRETER);
2832	break;
2833	}
2834
2835	/* Note: We'll get a #GP if the IO instruction isn't allowed (IOPL or TSS bitmap); no need to double check. */
2836	case VMX_EXIT_PORT_IO: /* 30 I/O instruction. */
2837	{
2838	STAM_PROFILE_ADV_START(&pVCpu->hwaccm.s.StatExit2Sub1, y1);
2839	uint32_t uIOWidth = VMX_EXIT_QUALIFICATION_IO_WIDTH(exitQualification);
2840	uint32_t uPort;
2841	bool fIOWrite = (VMX_EXIT_QUALIFICATION_IO_DIRECTION(exitQualification) == VMX_EXIT_QUALIFICATION_IO_DIRECTION_OUT);
2842
2843	/** @todo necessary to make the distinction? */
2844	if (VMX_EXIT_QUALIFICATION_IO_ENCODING(exitQualification) == VMX_EXIT_QUALIFICATION_IO_ENCODING_DX)
2845	{
2846	uPort = pCtx->edx & 0xffff;
2847	}
2848	else
2849	uPort = VMX_EXIT_QUALIFICATION_IO_PORT(exitQualification); /* Immediate encoding. */
2850
2851	/* paranoia */
2852	if (RT_UNLIKELY(uIOWidth == 2 \|\| uIOWidth >= 4))
2853	{
2854	rc = fIOWrite ? VINF_IOM_HC_IOPORT_WRITE : VINF_IOM_HC_IOPORT_READ;
2855	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub1, y1);
2856	break;
2857	}
2858
2859	uint32_t cbSize = g_aIOSize[uIOWidth];
2860
2861	if (VMX_EXIT_QUALIFICATION_IO_STRING(exitQualification))
2862	{
2863	/* ins/outs */
2864	DISCPUSTATE Cpu;
2865
2866	/* Disassemble manually to deal with segment prefixes. */
2867	/** @todo VMX_VMCS_EXIT_GUEST_LINEAR_ADDR contains the flat pointer operand of the instruction. */
2868	/** @todo VMX_VMCS32_RO_EXIT_INSTR_INFO also contains segment prefix info. */
2869	rc = EMInterpretDisasOne(pVM, CPUMCTX2CORE(pCtx), &Cpu, NULL);
2870	if (rc == VINF_SUCCESS)
2871	{
2872	if (fIOWrite)
2873	{
2874	Log2(("IOMInterpretOUTSEx %RGv %x size=%d\n", (RTGCPTR)pCtx->rip, uPort, cbSize));
2875	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitIOStringWrite);
2876	rc = IOMInterpretOUTSEx(pVM, CPUMCTX2CORE(pCtx), uPort, Cpu.prefix, cbSize);
2877	}
2878	else
2879	{
2880	Log2(("IOMInterpretINSEx %RGv %x size=%d\n", (RTGCPTR)pCtx->rip, uPort, cbSize));
2881	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitIOStringRead);
2882	rc = IOMInterpretINSEx(pVM, CPUMCTX2CORE(pCtx), uPort, Cpu.prefix, cbSize);
2883	}
2884	}
2885	else
2886	rc = VINF_EM_RAW_EMULATE_INSTR;
2887	}
2888	else
2889	{
2890	/* normal in/out */
2891	uint32_t uAndVal = g_aIOOpAnd[uIOWidth];
2892
2893	Assert(!VMX_EXIT_QUALIFICATION_IO_REP(exitQualification));
2894
2895	if (fIOWrite)
2896	{
2897	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitIOWrite);
2898	rc = IOMIOPortWrite(pVM, uPort, pCtx->eax & uAndVal, cbSize);
2899	}
2900	else
2901	{
2902	uint32_t u32Val = 0;
2903
2904	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitIORead);
2905	rc = IOMIOPortRead(pVM, uPort, &u32Val, cbSize);
2906	if (IOM_SUCCESS(rc))
2907	{
2908	/* Write back to the EAX register. */
2909	pCtx->eax = (pCtx->eax & ~uAndVal) \| (u32Val & uAndVal);
2910	}
2911	}
2912	}
2913	/*
2914	* Handled the I/O return codes.
2915	* (The unhandled cases end up with rc == VINF_EM_RAW_EMULATE_INSTR.)
2916	*/
2917	if (IOM_SUCCESS(rc))
2918	{
2919	/* Update EIP and continue execution. */
2920	pCtx->rip += cbInstr;
2921	if (RT_LIKELY(rc == VINF_SUCCESS))
2922	{
2923	/* If any IO breakpoints are armed, then we should check if a debug trap needs to be generated. */
2924	if (pCtx->dr[7] & X86_DR7_ENABLED_MASK)
2925	{
2926	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatDRxIOCheck);
2927	for (unsigned i=0;i<4;i++)
2928	{
2929	unsigned uBPLen = g_aIOSize[X86_DR7_GET_LEN(pCtx->dr[7], i)];
2930
2931	if ( (uPort >= pCtx->dr[i] && uPort < pCtx->dr[i] + uBPLen)
2932	&& (pCtx->dr[7] & (X86_DR7_L(i) \| X86_DR7_G(i)))
2933	&& (pCtx->dr[7] & X86_DR7_RW(i, X86_DR7_RW_IO)) == X86_DR7_RW(i, X86_DR7_RW_IO))
2934	{
2935	uint64_t uDR6;
2936
2937	Assert(CPUMIsGuestDebugStateActive(pVM));
2938
2939	uDR6 = ASMGetDR6();
2940
2941	/* Clear all breakpoint status flags and set the one we just hit. */
2942	uDR6 &= ~(X86_DR6_B0\|X86_DR6_B1\|X86_DR6_B2\|X86_DR6_B3);
2943	uDR6 \|= (uint64_t)RT_BIT(i);
2944
2945	/* Note: AMD64 Architecture Programmer's Manual 13.1:
2946	* Bits 15:13 of the DR6 register is never cleared by the processor and must be cleared by software after
2947	* the contents have been read.
2948	*/
2949	ASMSetDR6(uDR6);
2950
2951	/* X86_DR7_GD will be cleared if drx accesses should be trapped inside the guest. */
2952	pCtx->dr[7] &= ~X86_DR7_GD;
2953
2954	/* Paranoia. */
2955	pCtx->dr[7] &= 0xffffffff; /* upper 32 bits reserved */
2956	pCtx->dr[7] &= ~(RT_BIT(11) \| RT_BIT(12) \| RT_BIT(14) \| RT_BIT(15)); /* must be zero */
2957	pCtx->dr[7] \|= 0x400; /* must be one */
2958
2959	/* Resync DR7 */
2960	rc = VMXWriteVMCS64(VMX_VMCS64_GUEST_DR7, pCtx->dr[7]);
2961	AssertRC(rc);
2962
2963	/* Construct inject info. */
2964	intInfo = X86_XCPT_DB;
2965	intInfo \|= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
2966	intInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HWEXCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
2967
2968	Log(("Inject IO debug trap at %RGv\n", (RTGCPTR)pCtx->rip));
2969	rc = VMXR0InjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), 0, 0);
2970	AssertRC(rc);
2971
2972	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub1, y1);
2973	goto ResumeExecution;
2974	}
2975	}
2976	}
2977
2978	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub1, y1);
2979	goto ResumeExecution;
2980	}
2981	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub1, y1);
2982	break;
2983	}
2984
2985	#ifdef VBOX_STRICT
2986	if (rc == VINF_IOM_HC_IOPORT_READ)
2987	Assert(!fIOWrite);
2988	else if (rc == VINF_IOM_HC_IOPORT_WRITE)
2989	Assert(fIOWrite);
2990	else
2991	AssertMsg(RT_FAILURE(rc) \|\| rc == VINF_EM_RAW_EMULATE_INSTR \|\| rc == VINF_EM_RAW_GUEST_TRAP \|\| rc == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", rc));
2992	#endif
2993	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub1, y1);
2994	break;
2995	}
2996
2997	case VMX_EXIT_TPR: /* 43 TPR below threshold. Guest software executed MOV to CR8. */
2998	LogFlow(("VMX_EXIT_TPR\n"));
2999	/* RIP is already set to the next instruction and the TPR has been synced back. Just resume. */
3000	goto ResumeExecution;
3001
3002	default:
3003	/* The rest is handled after syncing the entire CPU state. */
3004	break;
3005	}
3006
3007	/* Note: the guest state isn't entirely synced back at this stage. */
3008
3009	/* Investigate why there was a VM-exit. (part 2) */
3010	switch (exitReason)
3011	{
3012	case VMX_EXIT_EXCEPTION: /* 0 Exception or non-maskable interrupt (NMI). */
3013	case VMX_EXIT_EXTERNAL_IRQ: /* 1 External interrupt. */
3014	case VMX_EXIT_EPT_VIOLATION:
3015	/* Already handled above. */
3016	break;
3017
3018	case VMX_EXIT_TRIPLE_FAULT: /* 2 Triple fault. */
3019	rc = VINF_EM_RESET; /* Triple fault equals a reset. */
3020	break;
3021
3022	case VMX_EXIT_INIT_SIGNAL: /* 3 INIT signal. */
3023	case VMX_EXIT_SIPI: /* 4 Start-up IPI (SIPI). */
3024	rc = VINF_EM_RAW_INTERRUPT;
3025	AssertFailed(); /* Can't happen. Yet. */
3026	break;
3027
3028	case VMX_EXIT_IO_SMI_IRQ: /* 5 I/O system-management interrupt (SMI). */
3029	case VMX_EXIT_SMI_IRQ: /* 6 Other SMI. */
3030	rc = VINF_EM_RAW_INTERRUPT;
3031	AssertFailed(); /* Can't happen afaik. */
3032	break;
3033
3034	case VMX_EXIT_TASK_SWITCH: /* 9 Task switch. */
3035	rc = VERR_EM_INTERPRETER;
3036	break;
3037
3038	case VMX_EXIT_HLT: /* 12 Guest software attempted to execute HLT. */
3039	/** Check if external interrupts are pending; if so, don't switch back. */
3040	pCtx->rip++; /* skip hlt */
3041	if ( pCtx->eflags.Bits.u1IF
3042	&& VM_FF_ISPENDING(pVM, (VM_FF_INTERRUPT_APIC\|VM_FF_INTERRUPT_PIC)))
3043	goto ResumeExecution;
3044
3045	rc = VINF_EM_HALT;
3046	break;
3047
3048	case VMX_EXIT_RSM: /* 17 Guest software attempted to execute RSM in SMM. */
3049	AssertFailed(); /* can't happen. */
3050	rc = VINF_EM_RAW_EXCEPTION_PRIVILEGED;
3051	break;
3052
3053	case VMX_EXIT_VMCALL: /* 18 Guest software executed VMCALL. */
3054	case VMX_EXIT_VMCLEAR: /* 19 Guest software executed VMCLEAR. */
3055	case VMX_EXIT_VMLAUNCH: /* 20 Guest software executed VMLAUNCH. */
3056	case VMX_EXIT_VMPTRLD: /* 21 Guest software executed VMPTRLD. */
3057	case VMX_EXIT_VMPTRST: /* 22 Guest software executed VMPTRST. */
3058	case VMX_EXIT_VMREAD: /* 23 Guest software executed VMREAD. */
3059	case VMX_EXIT_VMRESUME: /* 24 Guest software executed VMRESUME. */
3060	case VMX_EXIT_VMWRITE: /* 25 Guest software executed VMWRITE. */
3061	case VMX_EXIT_VMXOFF: /* 26 Guest software executed VMXOFF. */
3062	case VMX_EXIT_VMXON: /* 27 Guest software executed VMXON. */
3063	/** @todo inject #UD immediately */
3064	rc = VINF_EM_RAW_EXCEPTION_PRIVILEGED;
3065	break;
3066
3067	case VMX_EXIT_CPUID: /* 10 Guest software attempted to execute CPUID. */
3068	case VMX_EXIT_RDTSC: /* 16 Guest software attempted to execute RDTSC. */
3069	case VMX_EXIT_INVPG: /* 14 Guest software attempted to execute INVPG. */
3070	case VMX_EXIT_CRX_MOVE: /* 28 Control-register accesses. */
3071	case VMX_EXIT_DRX_MOVE: /* 29 Debug-register accesses. */
3072	case VMX_EXIT_PORT_IO: /* 30 I/O instruction. */
3073	/* already handled above */
3074	AssertMsg( rc == VINF_PGM_CHANGE_MODE
3075	\|\| rc == VINF_EM_RAW_INTERRUPT
3076	\|\| rc == VERR_EM_INTERPRETER
3077	\|\| rc == VINF_EM_RAW_EMULATE_INSTR
3078	\|\| rc == VINF_PGM_SYNC_CR3
3079	\|\| rc == VINF_IOM_HC_IOPORT_READ
3080	\|\| rc == VINF_IOM_HC_IOPORT_WRITE
3081	\|\| rc == VINF_EM_RAW_GUEST_TRAP
3082	\|\| rc == VINF_TRPM_XCPT_DISPATCHED
3083	\|\| rc == VINF_EM_RESCHEDULE_REM,
3084	("rc = %d\n", rc));
3085	break;
3086
3087	case VMX_EXIT_TPR: /* 43 TPR below threshold. Guest software executed MOV to CR8. */
3088	case VMX_EXIT_RDMSR: /* 31 RDMSR. Guest software attempted to execute RDMSR. */
3089	case VMX_EXIT_WRMSR: /* 32 WRMSR. Guest software attempted to execute WRMSR. */
3090	/* Note: If we decide to emulate them here, then we must sync the MSRs that could have been changed (sysenter, fs/gs base)!!! */
3091	rc = VERR_EM_INTERPRETER;
3092	break;
3093
3094	case VMX_EXIT_RDPMC: /* 15 Guest software attempted to execute RDPMC. */
3095	case VMX_EXIT_MWAIT: /* 36 Guest software executed MWAIT. */
3096	case VMX_EXIT_MONITOR: /* 39 Guest software attempted to execute MONITOR. */
3097	case VMX_EXIT_PAUSE: /* 40 Guest software attempted to execute PAUSE. */
3098	rc = VINF_EM_RAW_EXCEPTION_PRIVILEGED;
3099	break;
3100
3101	case VMX_EXIT_IRQ_WINDOW: /* 7 Interrupt window. */
3102	Assert(rc == VINF_EM_RAW_INTERRUPT);
3103	break;
3104
3105	case VMX_EXIT_ERR_INVALID_GUEST_STATE: /* 33 VM-entry failure due to invalid guest state. */
3106	{
3107	#ifdef VBOX_STRICT
3108	RTCCUINTREG val = 0;
3109
3110	Log(("VMX_EXIT_ERR_INVALID_GUEST_STATE\n"));
3111
3112	VMXReadVMCS(VMX_VMCS64_GUEST_RIP, &val);
3113	Log(("Old eip %RGv new %RGv\n", (RTGCPTR)pCtx->rip, (RTGCPTR)val));
3114
3115	VMXReadVMCS(VMX_VMCS64_GUEST_CR0, &val);
3116	Log(("VMX_VMCS_GUEST_CR0 %RX64\n", val));
3117
3118	VMXReadVMCS(VMX_VMCS64_GUEST_CR3, &val);
3119	Log(("VMX_VMCS_GUEST_CR3 %RGp\n", val));
3120
3121	VMXReadVMCS(VMX_VMCS64_GUEST_CR4, &val);
3122	Log(("VMX_VMCS_GUEST_CR4 %RX64\n", val));
3123
3124	VMXReadVMCS(VMX_VMCS_GUEST_RFLAGS, &val);
3125	Log(("VMX_VMCS_GUEST_RFLAGS %08x\n", val));
3126
3127	VMX_LOG_SELREG(CS, "CS");
3128	VMX_LOG_SELREG(DS, "DS");
3129	VMX_LOG_SELREG(ES, "ES");
3130	VMX_LOG_SELREG(FS, "FS");
3131	VMX_LOG_SELREG(GS, "GS");
3132	VMX_LOG_SELREG(SS, "SS");
3133	VMX_LOG_SELREG(TR, "TR");
3134	VMX_LOG_SELREG(LDTR, "LDTR");
3135
3136	VMXReadVMCS(VMX_VMCS64_GUEST_GDTR_BASE, &val);
3137	Log(("VMX_VMCS_GUEST_GDTR_BASE %RGv\n", val));
3138	VMXReadVMCS(VMX_VMCS64_GUEST_IDTR_BASE, &val);
3139	Log(("VMX_VMCS_GUEST_IDTR_BASE %RGv\n", val));
3140	#endif /* VBOX_STRICT */
3141	rc = VERR_VMX_INVALID_GUEST_STATE;
3142	break;
3143	}
3144
3145	case VMX_EXIT_ERR_MSR_LOAD: /* 34 VM-entry failure due to MSR loading. */
3146	case VMX_EXIT_ERR_MACHINE_CHECK: /* 41 VM-entry failure due to machine-check. */
3147	default:
3148	rc = VERR_VMX_UNEXPECTED_EXIT_CODE;
3149	AssertMsgFailed(("Unexpected exit code %d\n", exitReason)); /* Can't happen. */
3150	break;
3151
3152	}
3153	end:
3154
3155	/* Signal changes for the recompiler. */
3156	CPUMSetChangedFlags(pVM, CPUM_CHANGED_SYSENTER_MSR \| CPUM_CHANGED_LDTR \| CPUM_CHANGED_GDTR \| CPUM_CHANGED_IDTR \| CPUM_CHANGED_TR \| CPUM_CHANGED_HIDDEN_SEL_REGS);
3157
3158	/* If we executed vmlaunch/vmresume and an external irq was pending, then we don't have to do a full sync the next time. */
3159	if ( exitReason == VMX_EXIT_EXTERNAL_IRQ
3160	&& !VMX_EXIT_INTERRUPTION_INFO_VALID(intInfo))
3161	{
3162	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatPendingHostIrq);
3163	/* On the next entry we'll only sync the host context. */
3164	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_HOST_CONTEXT;
3165	}
3166	else
3167	{
3168	/* On the next entry we'll sync everything. */
3169	/** @todo we can do better than this */
3170	/* Not in the VINF_PGM_CHANGE_MODE though! */
3171	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_ALL;
3172	}
3173
3174	/* translate into a less severe return code */
3175	if (rc == VERR_EM_INTERPRETER)
3176	rc = VINF_EM_RAW_EMULATE_INSTR;
3177	else
3178	/* Try to extract more information about what might have gone wrong here. */
3179	if (rc == VERR_VMX_INVALID_VMCS_PTR)
3180	{
3181	VMXGetActivateVMCS(&pVCpu->hwaccm.s.vmx.lasterror.u64VMCSPhys);
3182	pVCpu->hwaccm.s.vmx.lasterror.ulVMCSRevision = (uint32_t )pVCpu->hwaccm.s.vmx.pVMCS;
3183	pVCpu->hwaccm.s.vmx.lasterror.idEnteredCpu = pVCpu->hwaccm.s.idEnteredCpu;
3184	pVCpu->hwaccm.s.vmx.lasterror.idCurrentCpu = RTMpCpuId();
3185	}
3186
3187	STAM_STATS({
3188	if (fStatExit2Started) STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2, y);
3189	else if (fStatEntryStarted) STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatEntry, x);
3190	});
3191	Log2(("X"));
3192	return rc;
3193	}
3194
3195
3196	/**
3197	* Enters the VT-x session
3198	*
3199	* @returns VBox status code.
3200	* @param pVM The VM to operate on.
3201	* @param pVCpu The VMCPU to operate on.
3202	* @param pCpu CPU info struct
3203	*/
3204	VMMR0DECL(int) VMXR0Enter(PVM pVM, PVMCPU pVCpu, PHWACCM_CPUINFO pCpu)
3205	{
3206	Assert(pVM->hwaccm.s.vmx.fSupported);
3207
3208	unsigned cr4 = ASMGetCR4();
3209	if (!(cr4 & X86_CR4_VMXE))
3210	{
3211	AssertMsgFailed(("X86_CR4_VMXE should be set!\n"));
3212	return VERR_VMX_X86_CR4_VMXE_CLEARED;
3213	}
3214
3215	/* Activate the VM Control Structure. */
3216	int rc = VMXActivateVMCS(pVCpu->hwaccm.s.vmx.pVMCSPhys);
3217	if (RT_FAILURE(rc))
3218	return rc;
3219
3220	pVCpu->hwaccm.s.fResumeVM = false;
3221	return VINF_SUCCESS;
3222	}
3223
3224
3225	/**
3226	* Leaves the VT-x session
3227	*
3228	* @returns VBox status code.
3229	* @param pVM The VM to operate on.
3230	* @param pVCpu The VMCPU to operate on.
3231	* @param pCtx CPU context
3232	*/
3233	VMMR0DECL(int) VMXR0Leave(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
3234	{
3235	Assert(pVM->hwaccm.s.vmx.fSupported);
3236
3237	/* Save the guest debug state if necessary. */
3238	if (CPUMIsGuestDebugStateActive(pVM))
3239	{
3240	CPUMR0SaveGuestDebugState(pVM, pVCpu, pCtx, true /* save DR6 */);
3241
3242	/* Enable drx move intercepts again. */
3243	pVCpu->hwaccm.s.vmx.proc_ctls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT;
3244	int rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
3245	AssertRC(rc);
3246
3247	/* Resync the debug registers the next time. */
3248	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_GUEST_DEBUG;
3249	}
3250	else
3251	Assert(pVCpu->hwaccm.s.vmx.proc_ctls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT);
3252
3253	/* Clear VM Control Structure. Marking it inactive, clearing implementation specific data and writing back VMCS data to memory. */
3254	int rc = VMXClearVMCS(pVCpu->hwaccm.s.vmx.pVMCSPhys);
3255	AssertRC(rc);
3256
3257	return VINF_SUCCESS;
3258	}
3259
3260	/**
3261	* Flush the TLB (EPT)
3262	*
3263	* @returns VBox status code.
3264	* @param pVM The VM to operate on.
3265	* @param pVCpu The VM CPU to operate on.
3266	* @param enmFlush Type of flush
3267	* @param GCPhys Physical address of the page to flush
3268	*/
3269	static void vmxR0FlushEPT(PVM pVM, PVMCPU pVCpu, VMX_FLUSH enmFlush, RTGCPHYS GCPhys)
3270	{
3271	uint64_t descriptor[2];
3272
3273	LogFlow(("vmxR0FlushEPT %d %RGv\n", enmFlush, GCPhys));
3274	Assert(pVM->hwaccm.s.fNestedPaging);
3275	descriptor[0] = pVCpu->hwaccm.s.vmx.GCPhysEPTP;
3276	descriptor[1] = GCPhys;
3277	int rc = VMXR0InvEPT(enmFlush, &descriptor[0]);
3278	AssertRC(rc);
3279	}
3280
3281	#ifdef HWACCM_VTX_WITH_VPID
3282	/**
3283	* Flush the TLB (EPT)
3284	*
3285	* @returns VBox status code.
3286	* @param pVM The VM to operate on.
3287	* @param pVCpu The VM CPU to operate on.
3288	* @param enmFlush Type of flush
3289	* @param GCPtr Virtual address of the page to flush
3290	*/
3291	static void vmxR0FlushVPID(PVM pVM, PVMCPU pVCpu, VMX_FLUSH enmFlush, RTGCPTR GCPtr)
3292	{
3293	#if HC_ARCH_BITS == 32
3294	/* If we get a flush in 64 bits guest mode, then force a full TLB flush. Invvpid probably takes only 32 bits addresses. (@todo) */
3295	if ( CPUMIsGuestInLongMode(pVM)
3296	&& !VMX_IS_64BIT_HOST_MODE())
3297	{
3298	pVCpu->hwaccm.s.fForceTLBFlush = true;
3299	}
3300	else
3301	#endif
3302	{
3303	uint64_t descriptor[2];
3304
3305	Assert(pVM->hwaccm.s.vmx.fVPID);
3306	descriptor[0] = pVCpu->hwaccm.s.uCurrentASID;
3307	descriptor[1] = GCPtr;
3308	int rc = VMXR0InvVPID(enmFlush, &descriptor[0]);
3309	AssertRC(rc);
3310	}
3311	}
3312	#endif /* HWACCM_VTX_WITH_VPID */
3313
3314	/**
3315	* Invalidates a guest page
3316	*
3317	* @returns VBox status code.
3318	* @param pVM The VM to operate on.
3319	* @param pVCpu The VM CPU to operate on.
3320	* @param GCVirt Page to invalidate
3321	*/
3322	VMMR0DECL(int) VMXR0InvalidatePage(PVM pVM, PVMCPU pVCpu, RTGCPTR GCVirt)
3323	{
3324	bool fFlushPending = pVCpu->hwaccm.s.fForceTLBFlush;
3325
3326	LogFlow(("VMXR0InvalidatePage %RGv\n", GCVirt));
3327
3328	/* Only relevant if we want to use VPID.
3329	* In the nested paging case we still see such calls, but
3330	* can safely ignore them. (e.g. after cr3 updates)
3331	*/
3332	#ifdef HWACCM_VTX_WITH_VPID
3333	/* Skip it if a TLB flush is already pending. */
3334	if ( !fFlushPending
3335	&& pVM->hwaccm.s.vmx.fVPID)
3336	vmxR0FlushVPID(pVM, pVCpu, pVM->hwaccm.s.vmx.enmFlushPage, GCVirt);
3337	#endif /* HWACCM_VTX_WITH_VPID */
3338
3339	return VINF_SUCCESS;
3340	}
3341
3342	/**
3343	* Invalidates a guest page by physical address
3344	*
3345	* NOTE: Assumes the current instruction references this physical page though a virtual address!!
3346	*
3347	* @returns VBox status code.
3348	* @param pVM The VM to operate on.
3349	* @param pVCpu The VM CPU to operate on.
3350	* @param GCPhys Page to invalidate
3351	*/
3352	VMMR0DECL(int) VMXR0InvalidatePhysPage(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys)
3353	{
3354	bool fFlushPending = pVCpu->hwaccm.s.fForceTLBFlush;
3355
3356	Assert(pVM->hwaccm.s.fNestedPaging);
3357
3358	LogFlow(("VMXR0InvalidatePhysPage %RGp\n", GCPhys));
3359
3360	/* Skip it if a TLB flush is already pending. */
3361	if (!fFlushPending)
3362	vmxR0FlushEPT(pVM, pVCpu, pVM->hwaccm.s.vmx.enmFlushPage, GCPhys);
3363
3364	return VINF_SUCCESS;
3365	}
3366
3367	/**
3368	* Report world switch error and dump some useful debug info
3369	*
3370	* @param pVM The VM to operate on.
3371	* @param pVCpu The VMCPU to operate on.
3372	* @param rc Return code
3373	* @param pCtx Current CPU context (not updated)
3374	*/
3375	static void VMXR0ReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, int rc, PCPUMCTX pCtx)
3376	{
3377	switch (rc)
3378	{
3379	case VERR_VMX_INVALID_VMXON_PTR:
3380	AssertFailed();
3381	break;
3382
3383	case VERR_VMX_UNABLE_TO_START_VM:
3384	case VERR_VMX_UNABLE_TO_RESUME_VM:
3385	{
3386	int rc;
3387	RTCCUINTREG exitReason, instrError, val;
3388
3389	rc = VMXReadVMCS(VMX_VMCS32_RO_EXIT_REASON, &exitReason);
3390	rc \|= VMXReadVMCS(VMX_VMCS32_RO_VM_INSTR_ERROR, &instrError);
3391	AssertRC(rc);
3392	if (rc == VINF_SUCCESS)
3393	{
3394	Log(("Unable to start/resume VM for reason: %x. Instruction error %x\n", (uint32_t)exitReason, (uint32_t)instrError));
3395	Log(("Current stack %08x\n", &rc));
3396
3397	pVCpu->hwaccm.s.vmx.lasterror.ulInstrError = instrError;
3398	pVCpu->hwaccm.s.vmx.lasterror.ulExitReason = exitReason;
3399
3400	#ifdef VBOX_STRICT
3401	RTGDTR gdtr;
3402	PX86DESCHC pDesc;
3403
3404	ASMGetGDTR(&gdtr);
3405
3406	VMXReadVMCS(VMX_VMCS64_GUEST_RIP, &val);
3407	Log(("Old eip %RGv new %RGv\n", (RTGCPTR)pCtx->rip, (RTGCPTR)val));
3408	VMXReadVMCS(VMX_VMCS_CTRL_PIN_EXEC_CONTROLS, &val);
3409	Log(("VMX_VMCS_CTRL_PIN_EXEC_CONTROLS %08x\n", val));
3410	VMXReadVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, &val);
3411	Log(("VMX_VMCS_CTRL_PROC_EXEC_CONTROLS %08x\n", val));
3412	VMXReadVMCS(VMX_VMCS_CTRL_ENTRY_CONTROLS, &val);
3413	Log(("VMX_VMCS_CTRL_ENTRY_CONTROLS %08x\n", val));
3414	VMXReadVMCS(VMX_VMCS_CTRL_EXIT_CONTROLS, &val);
3415	Log(("VMX_VMCS_CTRL_EXIT_CONTROLS %08x\n", val));
3416
3417	VMXReadVMCS(VMX_VMCS_HOST_CR0, &val);
3418	Log(("VMX_VMCS_HOST_CR0 %08x\n", val));
3419
3420	VMXReadVMCS(VMX_VMCS_HOST_CR3, &val);
3421	Log(("VMX_VMCS_HOST_CR3 %08x\n", val));
3422
3423	VMXReadVMCS(VMX_VMCS_HOST_CR4, &val);
3424	Log(("VMX_VMCS_HOST_CR4 %08x\n", val));
3425
3426	VMXReadVMCS(VMX_VMCS16_HOST_FIELD_CS, &val);
3427	Log(("VMX_VMCS_HOST_FIELD_CS %08x\n", val));
3428
3429	VMXReadVMCS(VMX_VMCS_GUEST_RFLAGS, &val);
3430	Log(("VMX_VMCS_GUEST_RFLAGS %08x\n", val));
3431
3432	if (val < gdtr.cbGdt)
3433	{
3434	pDesc = &((PX86DESCHC)gdtr.pGdt)[val >> X86_SEL_SHIFT_HC];
3435	HWACCMR0DumpDescriptor(pDesc, val, "CS: ");
3436	}
3437
3438	VMXReadVMCS(VMX_VMCS16_HOST_FIELD_DS, &val);
3439	Log(("VMX_VMCS_HOST_FIELD_DS %08x\n", val));
3440	if (val < gdtr.cbGdt)
3441	{
3442	pDesc = &((PX86DESCHC)gdtr.pGdt)[val >> X86_SEL_SHIFT_HC];
3443	HWACCMR0DumpDescriptor(pDesc, val, "DS: ");
3444	}
3445
3446	VMXReadVMCS(VMX_VMCS16_HOST_FIELD_ES, &val);
3447	Log(("VMX_VMCS_HOST_FIELD_ES %08x\n", val));
3448	if (val < gdtr.cbGdt)
3449	{
3450	pDesc = &((PX86DESCHC)gdtr.pGdt)[val >> X86_SEL_SHIFT_HC];
3451	HWACCMR0DumpDescriptor(pDesc, val, "ES: ");
3452	}
3453
3454	VMXReadVMCS(VMX_VMCS16_HOST_FIELD_FS, &val);
3455	Log(("VMX_VMCS16_HOST_FIELD_FS %08x\n", val));
3456	if (val < gdtr.cbGdt)
3457	{
3458	pDesc = &((PX86DESCHC)gdtr.pGdt)[val >> X86_SEL_SHIFT_HC];
3459	HWACCMR0DumpDescriptor(pDesc, val, "FS: ");
3460	}
3461
3462	VMXReadVMCS(VMX_VMCS16_HOST_FIELD_GS, &val);
3463	Log(("VMX_VMCS16_HOST_FIELD_GS %08x\n", val));
3464	if (val < gdtr.cbGdt)
3465	{
3466	pDesc = &((PX86DESCHC)gdtr.pGdt)[val >> X86_SEL_SHIFT_HC];
3467	HWACCMR0DumpDescriptor(pDesc, val, "GS: ");
3468	}
3469
3470	VMXReadVMCS(VMX_VMCS16_HOST_FIELD_SS, &val);
3471	Log(("VMX_VMCS16_HOST_FIELD_SS %08x\n", val));
3472	if (val < gdtr.cbGdt)
3473	{
3474	pDesc = &((PX86DESCHC)gdtr.pGdt)[val >> X86_SEL_SHIFT_HC];
3475	HWACCMR0DumpDescriptor(pDesc, val, "SS: ");
3476	}
3477
3478	VMXReadVMCS(VMX_VMCS16_HOST_FIELD_TR, &val);
3479	Log(("VMX_VMCS16_HOST_FIELD_TR %08x\n", val));
3480	if (val < gdtr.cbGdt)
3481	{
3482	pDesc = &((PX86DESCHC)gdtr.pGdt)[val >> X86_SEL_SHIFT_HC];
3483	HWACCMR0DumpDescriptor(pDesc, val, "TR: ");
3484	}
3485
3486	VMXReadVMCS(VMX_VMCS_HOST_TR_BASE, &val);
3487	Log(("VMX_VMCS_HOST_TR_BASE %RHv\n", val));
3488
3489	VMXReadVMCS(VMX_VMCS_HOST_GDTR_BASE, &val);
3490	Log(("VMX_VMCS_HOST_GDTR_BASE %RHv\n", val));
3491	VMXReadVMCS(VMX_VMCS_HOST_IDTR_BASE, &val);
3492	Log(("VMX_VMCS_HOST_IDTR_BASE %RHv\n", val));
3493
3494	VMXReadVMCS(VMX_VMCS32_HOST_SYSENTER_CS, &val);
3495	Log(("VMX_VMCS_HOST_SYSENTER_CS %08x\n", val));
3496
3497	VMXReadVMCS(VMX_VMCS_HOST_SYSENTER_EIP, &val);
3498	Log(("VMX_VMCS_HOST_SYSENTER_EIP %RHv\n", val));
3499
3500	VMXReadVMCS(VMX_VMCS_HOST_SYSENTER_ESP, &val);
3501	Log(("VMX_VMCS_HOST_SYSENTER_ESP %RHv\n", val));
3502
3503	VMXReadVMCS(VMX_VMCS_HOST_RSP, &val);
3504	Log(("VMX_VMCS_HOST_RSP %RHv\n", val));
3505	VMXReadVMCS(VMX_VMCS_HOST_RIP, &val);
3506	Log(("VMX_VMCS_HOST_RIP %RHv\n", val));
3507
3508	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3509	if (VMX_IS_64BIT_HOST_MODE())
3510	{
3511	Log(("MSR_K6_EFER = %RX64\n", ASMRdMsr(MSR_K6_EFER)));
3512	Log(("MSR_K6_STAR = %RX64\n", ASMRdMsr(MSR_K6_STAR)));
3513	Log(("MSR_K8_LSTAR = %RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
3514	Log(("MSR_K8_CSTAR = %RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
3515	Log(("MSR_K8_SF_MASK = %RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
3516	}
3517	# endif
3518	#endif /* VBOX_STRICT */
3519	}
3520	break;
3521	}
3522
3523	default:
3524	/* impossible */
3525	AssertMsgFailed(("%Rrc (%#x)\n", rc, rc));
3526	break;
3527	}
3528	}
3529
3530	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3531	/**
3532	* Prepares for and executes VMLAUNCH (64 bits guest mode)
3533	*
3534	* @returns VBox status code
3535	* @param fResume vmlauch/vmresume
3536	* @param pCtx Guest context
3537	* @param pCache VMCS cache
3538	* @param pVM The VM to operate on.
3539	* @param pVCpu The VMCPU to operate on.
3540	*/
3541	DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
3542	{
3543	uint32_t aParam[6];
3544	PHWACCM_CPUINFO pCpu;
3545	RTHCPHYS pPageCpuPhys;
3546	int rc;
3547
3548	pCpu = HWACCMR0GetCurrentCpu();
3549	pPageCpuPhys = RTR0MemObjGetPagePhysAddr(pCpu->pMemObj, 0);
3550
3551	#ifdef DEBUG
3552	pCache->TestIn.pPageCpuPhys = 0;
3553	pCache->TestIn.pVMCSPhys = 0;
3554	pCache->TestIn.pCache = 0;
3555	pCache->TestOut.pVMCSPhys = 0;
3556	pCache->TestOut.pCache = 0;
3557	pCache->TestOut.pCtx = 0;
3558	#endif
3559
3560	aParam[0] = (uint32_t)(pPageCpuPhys); /* Param 1: VMXON physical address - Lo. */
3561	aParam[1] = (uint32_t)(pPageCpuPhys >> 32); /* Param 1: VMXON physical address - Hi. */
3562	aParam[2] = (uint32_t)(pVCpu->hwaccm.s.vmx.pVMCSPhys); /* Param 2: VMCS physical address - Lo. */
3563	aParam[3] = (uint32_t)(pVCpu->hwaccm.s.vmx.pVMCSPhys >> 32); /* Param 2: VMCS physical address - Hi. */
3564	aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hwaccm.s.vmx.VMCSCache);
3565	aParam[5] = 0;
3566
3567	rc = VMXR0Execute64BitsHandler(pVM, pVCpu, pCtx, pVM->hwaccm.s.pfnVMXGCStartVM64, 6, &aParam[0]);
3568
3569	#ifdef DEBUG
3570	AssertMsg(pCache->TestIn.pPageCpuPhys == pPageCpuPhys, ("%RHp vs %RHp\n", pCache->TestIn.pPageCpuPhys, pPageCpuPhys));
3571	AssertMsg(pCache->TestIn.pVMCSPhys == pVCpu->hwaccm.s.vmx.pVMCSPhys, ("%RHp vs %RHp\n", pCache->TestIn.pVMCSPhys, pVCpu->hwaccm.s.vmx.pVMCSPhys));
3572	AssertMsg(pCache->TestIn.pVMCSPhys == pCache->TestOut.pVMCSPhys, ("%RHp vs %RHp\n", pCache->TestIn.pVMCSPhys, pCache->TestOut.pVMCSPhys));
3573	AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache, pCache->TestOut.pCache));
3574	AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hwaccm.s.vmx.VMCSCache), ("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hwaccm.s.vmx.VMCSCache)));
3575	AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx, pCache->TestOut.pCtx));
3576	#endif
3577
3578	return rc;
3579	}
3580
3581	/**
3582	* Executes the specified handler in 64 mode
3583	*
3584	* @returns VBox status code.
3585	* @param pVM The VM to operate on.
3586	* @param pVCpu The VMCPU to operate on.
3587	* @param pCtx Guest context
3588	* @param pfnHandler RC handler
3589	* @param cbParam Number of parameters
3590	* @param paParam Array of 32 bits parameters
3591	*/
3592	VMMR0DECL(int) VMXR0Execute64BitsHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, RTRCPTR pfnHandler, uint32_t cbParam, uint32_t *paParam)
3593	{
3594	int rc, rc2;
3595	RTHCUINTREG uFlags;
3596	PHWACCM_CPUINFO pCpu;
3597	RTHCPHYS pPageCpuPhys;
3598
3599	/* @todo This code is not guest SMP safe (hyper context) */
3600	AssertReturn(pVM->cCPUs == 1, VERR_ACCESS_DENIED);
3601	AssertReturn(pVM->hwaccm.s.pfnHost32ToGuest64R0, VERR_INTERNAL_ERROR);
3602	Assert(pVCpu->hwaccm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hwaccm.s.vmx.VMCSCache.Write.aField));
3603	Assert(pVCpu->hwaccm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hwaccm.s.vmx.VMCSCache.Read.aField));
3604
3605	pCpu = HWACCMR0GetCurrentCpu();
3606	pPageCpuPhys = RTR0MemObjGetPagePhysAddr(pCpu->pMemObj, 0);
3607
3608	/* Clear VM Control Structure. Marking it inactive, clearing implementation specific data and writing back VMCS data to memory. */
3609	VMXClearVMCS(pVCpu->hwaccm.s.vmx.pVMCSPhys);
3610
3611	/* Leave VMX Root Mode. */
3612	VMXDisable();
3613
3614	uFlags = ASMIntDisableFlags();
3615
3616	CPUMSetHyperESP(pVM, VMMGetStackRC(pVM));
3617	CPUMSetHyperEIP(pVM, pfnHandler);
3618	for (int i=(int)cbParam-1;i>=0;i--)
3619	CPUMPushHyper(pVM, paParam[i]);
3620
3621	/* Call switcher. */
3622	rc = pVM->hwaccm.s.pfnHost32ToGuest64R0(pVM);
3623
3624	#ifdef VBOX_STRICT
3625	RTHCUINTREG uFlagsTest = ASMGetFlags();
3626	#endif
3627
3628	ASMSetFlags(uFlags);
3629
3630	/* Make sure the VMX instructions don't cause #UD faults. */
3631	ASMSetCR4(ASMGetCR4() \| X86_CR4_VMXE);
3632
3633	/* Enter VMX Root Mode */
3634	rc2 = VMXEnable(pPageCpuPhys);
3635	if (RT_FAILURE(rc2))
3636	{
3637	if (pVM)
3638	VMXR0CheckError(pVM, pVCpu, rc2);
3639	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
3640	return VERR_VMX_VMXON_FAILED;
3641	}
3642
3643	VMXActivateVMCS(pVCpu->hwaccm.s.vmx.pVMCSPhys);
3644	Assert(!(uFlagsTest & X86_EFL_IF));
3645
3646	return rc;
3647	}
3648
3649	#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL) */
3650
3651
3652	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0)
3653	/**
3654	* Executes VMWRITE
3655	*
3656	* @returns VBox status code
3657	* @param pVCpu The VMCPU to operate on.
3658	* @param idxField VMCS index
3659	* @param u64Val 16, 32 or 64 bits value
3660	*/
3661	VMMR0DECL(int) VMXWriteVMCS64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
3662	{
3663	int rc;
3664
3665	switch (idxField)
3666	{
3667	case VMX_VMCS_CTRL_TSC_OFFSET_FULL:
3668	case VMX_VMCS_CTRL_IO_BITMAP_A_FULL:
3669	case VMX_VMCS_CTRL_IO_BITMAP_B_FULL:
3670	case VMX_VMCS_CTRL_MSR_BITMAP_FULL:
3671	case VMX_VMCS_CTRL_VMEXIT_MSR_STORE_FULL:
3672	case VMX_VMCS_CTRL_VMEXIT_MSR_LOAD_FULL:
3673	case VMX_VMCS_CTRL_VMENTRY_MSR_LOAD_FULL:
3674	case VMX_VMCS_CTRL_VAPIC_PAGEADDR_FULL:
3675	case VMX_VMCS_GUEST_LINK_PTR_FULL:
3676	case VMX_VMCS_GUEST_PDPTR0_FULL:
3677	case VMX_VMCS_GUEST_PDPTR1_FULL:
3678	case VMX_VMCS_GUEST_PDPTR2_FULL:
3679	case VMX_VMCS_GUEST_PDPTR3_FULL:
3680	case VMX_VMCS_GUEST_DEBUGCTL_FULL:
3681	case VMX_VMCS_GUEST_EFER_FULL:
3682	case VMX_VMCS_CTRL_EPTP_FULL:
3683	/* These fields consist of two parts, which are both writable in 32 bits mode. */
3684	rc = VMXWriteVMCS32(idxField, u64Val);
3685	rc \|= VMXWriteVMCS32(idxField + 1, (uint32_t)(u64Val >> 32ULL));
3686	AssertRC(rc);
3687	return rc;
3688
3689	case VMX_VMCS64_GUEST_LDTR_BASE:
3690	case VMX_VMCS64_GUEST_TR_BASE:
3691	case VMX_VMCS64_GUEST_GDTR_BASE:
3692	case VMX_VMCS64_GUEST_IDTR_BASE:
3693	case VMX_VMCS64_GUEST_SYSENTER_EIP:
3694	case VMX_VMCS64_GUEST_SYSENTER_ESP:
3695	case VMX_VMCS64_GUEST_CR0:
3696	case VMX_VMCS64_GUEST_CR4:
3697	case VMX_VMCS64_GUEST_CR3:
3698	case VMX_VMCS64_GUEST_DR7:
3699	case VMX_VMCS64_GUEST_RIP:
3700	case VMX_VMCS64_GUEST_RSP:
3701	case VMX_VMCS64_GUEST_CS_BASE:
3702	case VMX_VMCS64_GUEST_DS_BASE:
3703	case VMX_VMCS64_GUEST_ES_BASE:
3704	case VMX_VMCS64_GUEST_FS_BASE:
3705	case VMX_VMCS64_GUEST_GS_BASE:
3706	case VMX_VMCS64_GUEST_SS_BASE:
3707	/* Queue a 64 bits value as we can't set it in 32 bits host mode. */
3708	if (u64Val >> 32ULL)
3709	rc = VMXWriteCachedVMCSEx(pVCpu, idxField, u64Val);
3710	else
3711	rc = VMXWriteVMCS32(idxField, (uint32_t)u64Val);
3712
3713	return rc;
3714
3715	default:
3716	AssertMsgFailed(("Unexpected field %x\n", idxField));
3717	return VERR_INVALID_PARAMETER;
3718	}
3719	}
3720
3721	/**
3722	* Cache VMCS writes for performance reasons (Darwin) and for running 64 bits guests on 32 bits hosts.
3723	*
3724	* @param pVCpu The VMCPU to operate on.
3725	* @param idxField VMCS field
3726	* @param u64Val Value
3727	*/
3728	VMMR0DECL(int) VMXWriteCachedVMCSEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
3729	{
3730	PVMCSCACHE pCache = &pVCpu->hwaccm.s.vmx.VMCSCache;
3731
3732	AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1, ("entries=%x\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
3733
3734	/* Make sure there are no duplicates. */
3735	for (unsigned i=0;i<pCache->Write.cValidEntries;i++)
3736	{
3737	if (pCache->Write.aField[i] == idxField)
3738	{
3739	pCache->Write.aFieldVal[i] = u64Val;
3740	return VINF_SUCCESS;
3741	}
3742	}
3743
3744	pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
3745	pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
3746	pCache->Write.cValidEntries++;
3747	return VINF_SUCCESS;
3748	}
3749
3750	#endif /* HC_ARCH_BITS == 32 && !VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0 */

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMR0/HWVMXR0.cpp@ 15507

Download in other formats: