HWVMXR0.cpp@ 15371

Last change on this file since 15371 was 15371, checked in by vboxsync, 16 years ago
More paranoia
Property svn:eol-style set to `native` Property svn:keywords set to `Id`
File size: 143.2 KB

Line
1	/* $Id: HWVMXR0.cpp 15371 2008-12-12 14:41:28Z vboxsync $ */
2	/** @file
3	* HWACCM VMX - Host Context Ring 0.
4	*/
5
6	/*
7	* Copyright (C) 2006-2007 Sun Microsystems, Inc.
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*
17	* Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa
18	* Clara, CA 95054 USA or visit http://www.sun.com if you need
19	* additional information or have any questions.
20	*/
21
22
23	/*******************************************************************************
24	* Header Files *
25	*******************************************************************************/
26	#define LOG_GROUP LOG_GROUP_HWACCM
27	#include <VBox/hwaccm.h>
28	#include "HWACCMInternal.h"
29	#include <VBox/vm.h>
30	#include <VBox/x86.h>
31	#include <VBox/pgm.h>
32	#include <VBox/pdm.h>
33	#include <VBox/err.h>
34	#include <VBox/log.h>
35	#include <VBox/selm.h>
36	#include <VBox/iom.h>
37	#include <iprt/param.h>
38	#include <iprt/assert.h>
39	#include <iprt/asm.h>
40	#include <iprt/string.h>
41	#include "HWVMXR0.h"
42
43	/*******************************************************************************
44	* Defined Constants And Macros *
45	*******************************************************************************/
46	#if defined(RT_ARCH_AMD64)
47	# define VMX_IS_64BIT_HOST_MODE() (true)
48	#elif defined(VBOX_WITH_HYBIRD_32BIT_KERNEL)
49	# define VMX_IS_64BIT_HOST_MODE() (g_fVMXIs64bitHost != 0)
50	#else
51	# define VMX_IS_64BIT_HOST_MODE() (false)
52	#endif
53
54	/*******************************************************************************
55	* Global Variables *
56	*******************************************************************************/
57	/* IO operation lookup arrays. */
58	static uint32_t const g_aIOSize[4] = {1, 2, 0, 4};
59	static uint32_t const g_aIOOpAnd[4] = {0xff, 0xffff, 0, 0xffffffff};
60
61	#ifdef VBOX_WITH_HYBIRD_32BIT_KERNEL
62	/** See HWACCMR0A.asm. */
63	extern "C" uint32_t g_fVMXIs64bitHost;
64	#endif
65
66	/*******************************************************************************
67	* Local Functions *
68	*******************************************************************************/
69	static void VMXR0ReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, int rc, PCPUMCTX pCtx);
70	static void vmxR0SetupTLBEPT(PVM pVM, PVMCPU pVCpu);
71	static void vmxR0SetupTLBVPID(PVM pVM, PVMCPU pVCpu);
72	static void vmxR0SetupTLBDummy(PVM pVM, PVMCPU pVCpu);
73	static void vmxR0FlushEPT(PVM pVM, PVMCPU pVCpu, VMX_FLUSH enmFlush, RTGCPHYS GCPhys);
74	static void vmxR0FlushVPID(PVM pVM, PVMCPU pVCpu, VMX_FLUSH enmFlush, RTGCPTR GCPtr);
75	static void vmxR0UpdateExceptionBitmap(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx);
76
77
78	static void VMXR0CheckError(PVM pVM, PVMCPU pVCpu, int rc)
79	{
80	if (rc == VERR_VMX_GENERIC)
81	{
82	RTCCUINTREG instrError;
83
84	VMXReadVMCS(VMX_VMCS32_RO_VM_INSTR_ERROR, &instrError);
85	pVCpu->hwaccm.s.vmx.lasterror.ulInstrError = instrError;
86	}
87	pVM->hwaccm.s.lLastError = rc;
88	}
89
90	/**
91	* Sets up and activates VT-x on the current CPU
92	*
93	* @returns VBox status code.
94	* @param pCpu CPU info struct
95	* @param pVM The VM to operate on. (can be NULL after a resume!!)
96	* @param pvPageCpu Pointer to the global cpu page
97	* @param pPageCpuPhys Physical address of the global cpu page
98	*/
99	VMMR0DECL(int) VMXR0EnableCpu(PHWACCM_CPUINFO pCpu, PVM pVM, void *pvPageCpu, RTHCPHYS pPageCpuPhys)
100	{
101	AssertReturn(pPageCpuPhys, VERR_INVALID_PARAMETER);
102	AssertReturn(pvPageCpu, VERR_INVALID_PARAMETER);
103
104	#ifdef LOG_ENABLED
105	SUPR0Printf("VMXR0EnableCpu cpu %d page (%x) %x\n", pCpu->idCpu, pvPageCpu, (uint32_t)pPageCpuPhys);
106	#endif
107	if (pVM)
108	{
109	/* Set revision dword at the beginning of the VMXON structure. */
110	(uint32_t )pvPageCpu = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hwaccm.s.vmx.msr.vmx_basic_info);
111	}
112
113	/** @todo we should unmap the two pages from the virtual address space in order to prevent accidental corruption.
114	* (which can have very bad consequences!!!)
115	*/
116
117	/* Make sure the VMX instructions don't cause #UD faults. */
118	ASMSetCR4(ASMGetCR4() \| X86_CR4_VMXE);
119
120	/* Enter VMX Root Mode */
121	int rc = VMXEnable(pPageCpuPhys);
122	if (RT_FAILURE(rc))
123	{
124	if (pVM)
125	VMXR0CheckError(pVM, &pVM->aCpus[0], rc);
126	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
127	return VERR_VMX_VMXON_FAILED;
128	}
129	return VINF_SUCCESS;
130	}
131
132	/**
133	* Deactivates VT-x on the current CPU
134	*
135	* @returns VBox status code.
136	* @param pCpu CPU info struct
137	* @param pvPageCpu Pointer to the global cpu page
138	* @param pPageCpuPhys Physical address of the global cpu page
139	*/
140	VMMR0DECL(int) VMXR0DisableCpu(PHWACCM_CPUINFO pCpu, void *pvPageCpu, RTHCPHYS pPageCpuPhys)
141	{
142	AssertReturn(pPageCpuPhys, VERR_INVALID_PARAMETER);
143	AssertReturn(pvPageCpu, VERR_INVALID_PARAMETER);
144
145	/* Leave VMX Root Mode. */
146	VMXDisable();
147
148	/* And clear the X86_CR4_VMXE bit */
149	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
150
151	#ifdef LOG_ENABLED
152	SUPR0Printf("VMXR0DisableCpu cpu %d\n", pCpu->idCpu);
153	#endif
154	return VINF_SUCCESS;
155	}
156
157	/**
158	* Does Ring-0 per VM VT-x init.
159	*
160	* @returns VBox status code.
161	* @param pVM The VM to operate on.
162	*/
163	VMMR0DECL(int) VMXR0InitVM(PVM pVM)
164	{
165	int rc;
166
167	#ifdef LOG_ENABLED
168	SUPR0Printf("VMXR0InitVM %x\n", pVM);
169	#endif
170
171	pVM->hwaccm.s.vmx.pMemObjAPIC = NIL_RTR0MEMOBJ;
172
173	if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
174	{
175	/* Allocate one page for the virtual APIC mmio cache. */
176	rc = RTR0MemObjAllocCont(&pVM->hwaccm.s.vmx.pMemObjAPIC, 1 << PAGE_SHIFT, true /* executable R0 mapping */);
177	AssertRC(rc);
178	if (RT_FAILURE(rc))
179	return rc;
180
181	pVM->hwaccm.s.vmx.pAPIC = (uint8_t *)RTR0MemObjAddress(pVM->hwaccm.s.vmx.pMemObjAPIC);
182	pVM->hwaccm.s.vmx.pAPICPhys = RTR0MemObjGetPagePhysAddr(pVM->hwaccm.s.vmx.pMemObjAPIC, 0);
183	ASMMemZero32(pVM->hwaccm.s.vmx.pAPIC, PAGE_SIZE);
184	}
185	else
186	{
187	pVM->hwaccm.s.vmx.pMemObjAPIC = 0;
188	pVM->hwaccm.s.vmx.pAPIC = 0;
189	pVM->hwaccm.s.vmx.pAPICPhys = 0;
190	}
191
192	/* Allocate the MSR bitmap if this feature is supported. */
193	if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
194	{
195	rc = RTR0MemObjAllocCont(&pVM->hwaccm.s.vmx.pMemObjMSRBitmap, 1 << PAGE_SHIFT, true /* executable R0 mapping */);
196	AssertRC(rc);
197	if (RT_FAILURE(rc))
198	return rc;
199
200	pVM->hwaccm.s.vmx.pMSRBitmap = (uint8_t *)RTR0MemObjAddress(pVM->hwaccm.s.vmx.pMemObjMSRBitmap);
201	pVM->hwaccm.s.vmx.pMSRBitmapPhys = RTR0MemObjGetPagePhysAddr(pVM->hwaccm.s.vmx.pMemObjMSRBitmap, 0);
202	memset(pVM->hwaccm.s.vmx.pMSRBitmap, 0xff, PAGE_SIZE);
203	}
204
205	/* Allocate VMCBs for all guest CPUs. */
206	for (unsigned i=0;i<pVM->cCPUs;i++)
207	{
208	PVMCPU pVCpu = &pVM->aCpus[i];
209
210	pVCpu->hwaccm.s.vmx.pMemObjVMCS = NIL_RTR0MEMOBJ;
211
212	/* Allocate one page for the VM control structure (VMCS). */
213	rc = RTR0MemObjAllocCont(&pVCpu->hwaccm.s.vmx.pMemObjVMCS, 1 << PAGE_SHIFT, true /* executable R0 mapping */);
214	AssertRC(rc);
215	if (RT_FAILURE(rc))
216	return rc;
217
218	pVCpu->hwaccm.s.vmx.pVMCS = RTR0MemObjAddress(pVCpu->hwaccm.s.vmx.pMemObjVMCS);
219	pVCpu->hwaccm.s.vmx.pVMCSPhys = RTR0MemObjGetPagePhysAddr(pVCpu->hwaccm.s.vmx.pMemObjVMCS, 0);
220	ASMMemZero32(pVCpu->hwaccm.s.vmx.pVMCS, PAGE_SIZE);
221
222	pVCpu->hwaccm.s.vmx.cr0_mask = 0;
223	pVCpu->hwaccm.s.vmx.cr4_mask = 0;
224
225	/* Current guest paging mode. */
226	pVCpu->hwaccm.s.vmx.enmCurrGuestMode = PGMMODE_REAL;
227
228	#ifdef LOG_ENABLED
229	SUPR0Printf("VMXR0InitVM %x VMCS=%x (%x)\n", pVM, pVCpu->hwaccm.s.vmx.pVMCS, (uint32_t)pVCpu->hwaccm.s.vmx.pVMCSPhys);
230	#endif
231	}
232
233	return VINF_SUCCESS;
234	}
235
236	/**
237	* Does Ring-0 per VM VT-x termination.
238	*
239	* @returns VBox status code.
240	* @param pVM The VM to operate on.
241	*/
242	VMMR0DECL(int) VMXR0TermVM(PVM pVM)
243	{
244	for (unsigned i=0;i<pVM->cCPUs;i++)
245	{
246	if (pVM->aCpus[i].hwaccm.s.vmx.pMemObjVMCS != NIL_RTR0MEMOBJ)
247	{
248	RTR0MemObjFree(pVM->aCpus[i].hwaccm.s.vmx.pMemObjVMCS, false);
249	pVM->aCpus[i].hwaccm.s.vmx.pMemObjVMCS = NIL_RTR0MEMOBJ;
250	pVM->aCpus[i].hwaccm.s.vmx.pVMCS = 0;
251	pVM->aCpus[i].hwaccm.s.vmx.pVMCSPhys = 0;
252	}
253	}
254	if (pVM->hwaccm.s.vmx.pMemObjAPIC != NIL_RTR0MEMOBJ)
255	{
256	RTR0MemObjFree(pVM->hwaccm.s.vmx.pMemObjAPIC, false);
257	pVM->hwaccm.s.vmx.pMemObjAPIC = NIL_RTR0MEMOBJ;
258	pVM->hwaccm.s.vmx.pAPIC = 0;
259	pVM->hwaccm.s.vmx.pAPICPhys = 0;
260	}
261	if (pVM->hwaccm.s.vmx.pMemObjMSRBitmap != NIL_RTR0MEMOBJ)
262	{
263	RTR0MemObjFree(pVM->hwaccm.s.vmx.pMemObjMSRBitmap, false);
264	pVM->hwaccm.s.vmx.pMemObjMSRBitmap = NIL_RTR0MEMOBJ;
265	pVM->hwaccm.s.vmx.pMSRBitmap = 0;
266	pVM->hwaccm.s.vmx.pMSRBitmapPhys = 0;
267	}
268	return VINF_SUCCESS;
269	}
270
271	/**
272	* Sets up VT-x for the specified VM
273	*
274	* @returns VBox status code.
275	* @param pVM The VM to operate on.
276	*/
277	VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
278	{
279	int rc = VINF_SUCCESS;
280	uint32_t val;
281
282	AssertReturn(pVM, VERR_INVALID_PARAMETER);
283
284	for (unsigned i=0;i<pVM->cCPUs;i++)
285	{
286	PVMCPU pVCpu = &pVM->aCpus[i];
287
288	Assert(pVCpu->hwaccm.s.vmx.pVMCS);
289
290	/* Set revision dword at the beginning of the VMCS structure. */
291	(uint32_t )pVCpu->hwaccm.s.vmx.pVMCS = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hwaccm.s.vmx.msr.vmx_basic_info);
292
293	/* Clear VM Control Structure. */
294	Log(("pVMCSPhys = %RHp\n", pVCpu->hwaccm.s.vmx.pVMCSPhys));
295	rc = VMXClearVMCS(pVCpu->hwaccm.s.vmx.pVMCSPhys);
296	if (RT_FAILURE(rc))
297	goto vmx_end;
298
299	/* Activate the VM Control Structure. */
300	rc = VMXActivateVMCS(pVCpu->hwaccm.s.vmx.pVMCSPhys);
301	if (RT_FAILURE(rc))
302	goto vmx_end;
303
304	/* VMX_VMCS_CTRL_PIN_EXEC_CONTROLS
305	* Set required bits to one and zero according to the MSR capabilities.
306	*/
307	val = pVM->hwaccm.s.vmx.msr.vmx_pin_ctls.n.disallowed0;
308	/* External and non-maskable interrupts cause VM-exits. */
309	val = val \| VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_EXT_INT_EXIT \| VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_NMI_EXIT;
310	val &= pVM->hwaccm.s.vmx.msr.vmx_pin_ctls.n.allowed1;
311
312	rc = VMXWriteVMCS(VMX_VMCS_CTRL_PIN_EXEC_CONTROLS, val);
313	AssertRC(rc);
314
315	/* VMX_VMCS_CTRL_PROC_EXEC_CONTROLS
316	* Set required bits to one and zero according to the MSR capabilities.
317	*/
318	val = pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.disallowed0;
319	/* Program which event cause VM-exits and which features we want to use. */
320	val = val \| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_HLT_EXIT
321	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_TSC_OFFSET
322	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT
323	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_UNCOND_IO_EXIT
324	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MWAIT_EXIT; /* don't execute mwait or else we'll idle inside the guest (host thinks the cpu load is high) */
325
326	/* Without nested paging we should intercept invlpg and cr3 mov instructions. */
327	if (!pVM->hwaccm.s.fNestedPaging)
328	val \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INVLPG_EXIT
329	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
330	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT;
331
332	/* Note: VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MWAIT_EXIT might cause a vmlaunch failure with an invalid control fields error. (combined with some other exit reasons) */
333	if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
334	{
335	/* CR8 reads from the APIC shadow page; writes cause an exit is they lower the TPR below the threshold */
336	val \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW;
337	Assert(pVM->hwaccm.s.vmx.pAPIC);
338	}
339	else
340	/* Exit on CR8 reads & writes in case the TPR shadow feature isn't present. */
341	val \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR8_STORE_EXIT \| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR8_LOAD_EXIT;
342
343	#ifdef VBOX_WITH_VTX_MSR_BITMAPS
344	if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
345	{
346	Assert(pVM->hwaccm.s.vmx.pMSRBitmapPhys);
347	val \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS;
348	}
349	#endif
350
351	/* We will use the secondary control if it's present. */
352	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
353
354	/* Mask away the bits that the CPU doesn't support */
355	/** @todo make sure they don't conflict with the above requirements. */
356	val &= pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1;
357	pVCpu->hwaccm.s.vmx.proc_ctls = val;
358
359	rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, val);
360	AssertRC(rc);
361
362	if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
363	{
364	/* VMX_VMCS_CTRL_PROC_EXEC_CONTROLS2
365	* Set required bits to one and zero according to the MSR capabilities.
366	*/
367	val = pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2.n.disallowed0;
368	val \|= VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT;
369
370	#ifdef HWACCM_VTX_WITH_EPT
371	if (pVM->hwaccm.s.fNestedPaging)
372	val \|= VMX_VMCS_CTRL_PROC_EXEC2_EPT;
373	#endif /* HWACCM_VTX_WITH_EPT */
374	#ifdef HWACCM_VTX_WITH_VPID
375	else
376	if (pVM->hwaccm.s.vmx.fVPID)
377	val \|= VMX_VMCS_CTRL_PROC_EXEC2_VPID;
378	#endif /* HWACCM_VTX_WITH_VPID */
379
380	/* Mask away the bits that the CPU doesn't support */
381	/** @todo make sure they don't conflict with the above requirements. */
382	val &= pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2.n.allowed1;
383
384	rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS2, val);
385	AssertRC(rc);
386	}
387
388	/* VMX_VMCS_CTRL_CR3_TARGET_COUNT
389	* Set required bits to one and zero according to the MSR capabilities.
390	*/
391	rc = VMXWriteVMCS(VMX_VMCS_CTRL_CR3_TARGET_COUNT, 0);
392	AssertRC(rc);
393
394	/* VMX_VMCS_CTRL_EXIT_CONTROLS
395	* Set required bits to one and zero according to the MSR capabilities.
396	*/
397	val = pVM->hwaccm.s.vmx.msr.vmx_exit.n.disallowed0;
398
399	/* Save debug controls (dr7 & IA32_DEBUGCTL_MSR) (forced to 1 on the 'first' VT-x capable CPUs; this actually includes the newest Nehalem CPUs) */
400	val \|= VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_DEBUG;
401	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBIRD_32BIT_KERNEL)
402	if (VMX_IS_64BIT_HOST_MODE())
403	val \|= VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_AMD64;
404	/* else: Must be zero when AMD64 is not available. */
405	#endif
406	val &= pVM->hwaccm.s.vmx.msr.vmx_exit.n.allowed1;
407	/* Don't acknowledge external interrupts on VM-exit. */
408	rc = VMXWriteVMCS(VMX_VMCS_CTRL_EXIT_CONTROLS, val);
409	AssertRC(rc);
410
411	/* Forward all exception except #NM & #PF to the guest.
412	* We always need to check pagefaults since our shadow page table can be out of sync.
413	* And we always lazily sync the FPU & XMM state.
414	*/
415
416	/** @todo Possible optimization:
417	* Keep the FPU and XMM state current in the EM thread. That way there's no need to
418	* lazily sync anything, but the downside is that we can't use the FPU stack or XMM
419	* registers ourselves of course.
420	*
421	* Note: only possible if the current state is actually ours (X86_CR0_TS flag)
422	*/
423
424	/* Don't filter page faults; all of them should cause a switch. */
425	rc = VMXWriteVMCS(VMX_VMCS_CTRL_PAGEFAULT_ERROR_MASK, 0);
426	rc \|= VMXWriteVMCS(VMX_VMCS_CTRL_PAGEFAULT_ERROR_MATCH, 0);
427	AssertRC(rc);
428
429	/* Init TSC offset to zero. */
430	rc = VMXWriteVMCS64(VMX_VMCS_CTRL_TSC_OFFSET_FULL, 0);
431	AssertRC(rc);
432
433	rc = VMXWriteVMCS64(VMX_VMCS_CTRL_IO_BITMAP_A_FULL, 0);
434	AssertRC(rc);
435
436	rc = VMXWriteVMCS64(VMX_VMCS_CTRL_IO_BITMAP_B_FULL, 0);
437	AssertRC(rc);
438
439	/* Set the MSR bitmap address. */
440	if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
441	{
442	/* Optional */
443	rc = VMXWriteVMCS64(VMX_VMCS_CTRL_MSR_BITMAP_FULL, pVM->hwaccm.s.vmx.pMSRBitmapPhys);
444	AssertRC(rc);
445	}
446
447	/* Clear MSR controls. */
448	rc = VMXWriteVMCS64(VMX_VMCS_CTRL_VMEXIT_MSR_STORE_FULL, 0);
449	rc \|= VMXWriteVMCS64(VMX_VMCS_CTRL_VMEXIT_MSR_LOAD_FULL, 0);
450	rc \|= VMXWriteVMCS64(VMX_VMCS_CTRL_VMENTRY_MSR_LOAD_FULL, 0);
451	rc \|= VMXWriteVMCS(VMX_VMCS_CTRL_EXIT_MSR_STORE_COUNT, 0);
452	rc \|= VMXWriteVMCS(VMX_VMCS_CTRL_EXIT_MSR_LOAD_COUNT, 0);
453	AssertRC(rc);
454
455	if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
456	{
457	Assert(pVM->hwaccm.s.vmx.pMemObjAPIC);
458	/* Optional */
459	rc = VMXWriteVMCS(VMX_VMCS_CTRL_TPR_THRESHOLD, 0);
460	rc \|= VMXWriteVMCS64(VMX_VMCS_CTRL_VAPIC_PAGEADDR_FULL, pVM->hwaccm.s.vmx.pAPICPhys);
461	AssertRC(rc);
462	}
463
464	/* Set link pointer to -1. Not currently used. */
465	rc = VMXWriteVMCS64(VMX_VMCS_GUEST_LINK_PTR_FULL, 0xFFFFFFFFFFFFFFFFULL);
466	AssertRC(rc);
467
468	/* Clear VM Control Structure. Marking it inactive, clearing implementation specific data and writing back VMCS data to memory. */
469	rc = VMXClearVMCS(pVCpu->hwaccm.s.vmx.pVMCSPhys);
470	AssertRC(rc);
471
472	/* Configure the VMCS read cache. */
473	PVMCSCACHE pCache = &pVCpu->hwaccm.s.vmx.VMCSCache;
474
475	VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_RIP);
476	VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_RSP);
477	VMXSetupCachedReadVMCS(pCache, VMX_VMCS_GUEST_RFLAGS);
478	VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE);
479	VMXSetupCachedReadVMCS(pCache, VMX_VMCS_CTRL_CR0_READ_SHADOW);
480	VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_CR0);
481	VMXSetupCachedReadVMCS(pCache, VMX_VMCS_CTRL_CR4_READ_SHADOW);
482	VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_CR4);
483	VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_DR7);
484	VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_GUEST_SYSENTER_CS);
485	VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_SYSENTER_EIP);
486	VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_SYSENTER_ESP);
487	VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_GUEST_GDTR_LIMIT);
488	VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_GDTR_BASE);
489	VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_GUEST_IDTR_LIMIT);
490	VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_IDTR_BASE);
491
492	VMX_SETUP_SELREG(ES, pCache);
493	VMX_SETUP_SELREG(SS, pCache);
494	VMX_SETUP_SELREG(CS, pCache);
495	VMX_SETUP_SELREG(DS, pCache);
496	VMX_SETUP_SELREG(FS, pCache);
497	VMX_SETUP_SELREG(GS, pCache);
498	VMX_SETUP_SELREG(LDTR, pCache);
499	VMX_SETUP_SELREG(TR, pCache);
500
501	/* Status code VMCS reads. */
502	VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_EXIT_REASON);
503	VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_VM_INSTR_ERROR);
504	VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_EXIT_INSTR_LENGTH);
505	VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_EXIT_INTERRUPTION_ERRCODE);
506	VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO);
507	VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_EXIT_INSTR_INFO);
508	VMXSetupCachedReadVMCS(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
509	VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_IDT_INFO);
510	VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_IDT_ERRCODE);
511
512	if (pVM->hwaccm.s.fNestedPaging)
513	{
514	VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_CR3);
515	VMXSetupCachedReadVMCS(pCache, VMX_VMCS_EXIT_PHYS_ADDR_FULL);
516	pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
517	}
518	else
519	pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
520	} /* for each VMCPU */
521
522	/* Choose the right TLB setup function. */
523	if (pVM->hwaccm.s.fNestedPaging)
524	{
525	pVM->hwaccm.s.vmx.pfnSetupTaggedTLB = vmxR0SetupTLBEPT;
526
527	/* Default values for flushing. */
528	pVM->hwaccm.s.vmx.enmFlushPage = VMX_FLUSH_ALL_CONTEXTS;
529	pVM->hwaccm.s.vmx.enmFlushContext = VMX_FLUSH_ALL_CONTEXTS;
530
531	/* If the capabilities specify we can do more, then make use of it. */
532	if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVEPT_CAPS_INDIV)
533	pVM->hwaccm.s.vmx.enmFlushPage = VMX_FLUSH_PAGE;
534	else
535	if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVEPT_CAPS_CONTEXT)
536	pVM->hwaccm.s.vmx.enmFlushPage = VMX_FLUSH_SINGLE_CONTEXT;
537
538	if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVEPT_CAPS_CONTEXT)
539	pVM->hwaccm.s.vmx.enmFlushContext = VMX_FLUSH_SINGLE_CONTEXT;
540	}
541	#ifdef HWACCM_VTX_WITH_VPID
542	else
543	if (pVM->hwaccm.s.vmx.fVPID)
544	{
545	pVM->hwaccm.s.vmx.pfnSetupTaggedTLB = vmxR0SetupTLBVPID;
546
547	/* Default values for flushing. */
548	pVM->hwaccm.s.vmx.enmFlushPage = VMX_FLUSH_ALL_CONTEXTS;
549	pVM->hwaccm.s.vmx.enmFlushContext = VMX_FLUSH_ALL_CONTEXTS;
550
551	/* If the capabilities specify we can do more, then make use of it. */
552	if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVVPID_CAPS_INDIV)
553	pVM->hwaccm.s.vmx.enmFlushPage = VMX_FLUSH_PAGE;
554	else
555	if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVVPID_CAPS_CONTEXT)
556	pVM->hwaccm.s.vmx.enmFlushPage = VMX_FLUSH_SINGLE_CONTEXT;
557
558	if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVVPID_CAPS_CONTEXT)
559	pVM->hwaccm.s.vmx.enmFlushContext = VMX_FLUSH_SINGLE_CONTEXT;
560	}
561	#endif /* HWACCM_VTX_WITH_VPID */
562	else
563	pVM->hwaccm.s.vmx.pfnSetupTaggedTLB = vmxR0SetupTLBDummy;
564
565	vmx_end:
566	VMXR0CheckError(pVM, &pVM->aCpus[0], rc);
567	return rc;
568	}
569
570
571	/**
572	* Injects an event (trap or external interrupt)
573	*
574	* @returns VBox status code.
575	* @param pVM The VM to operate on.
576	* @param pVCpu The VMCPU to operate on.
577	* @param pCtx CPU Context
578	* @param intInfo VMX interrupt info
579	* @param cbInstr Opcode length of faulting instruction
580	* @param errCode Error code (optional)
581	*/
582	static int VMXR0InjectEvent(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, uint32_t intInfo, uint32_t cbInstr, uint32_t errCode)
583	{
584	int rc;
585	uint32_t iGate = VMX_EXIT_INTERRUPTION_INFO_VECTOR(intInfo);
586
587	#ifdef VBOX_STRICT
588	if (iGate == 0xE)
589	LogFlow(("VMXR0InjectEvent: Injecting interrupt %d at %RGv error code=%08x CR2=%08x intInfo=%08x\n", iGate, (RTGCPTR)pCtx->rip, errCode, pCtx->cr2, intInfo));
590	else
591	if (iGate < 0x20)
592	LogFlow(("VMXR0InjectEvent: Injecting interrupt %d at %RGv error code=%08x\n", iGate, (RTGCPTR)pCtx->rip, errCode));
593	else
594	{
595	LogFlow(("INJ-EI: %x at %RGv\n", iGate, (RTGCPTR)pCtx->rip));
596	Assert(!VM_FF_ISSET(pVM, VM_FF_INHIBIT_INTERRUPTS));
597	Assert(pCtx->eflags.u32 & X86_EFL_IF);
598	}
599	#endif
600
601	#ifdef HWACCM_VMX_EMULATE_REALMODE
602	if (CPUMIsGuestInRealModeEx(pCtx))
603	{
604	RTGCPHYS GCPhysHandler;
605	uint16_t offset, ip;
606	RTSEL sel;
607
608	/* Injecting events doesn't work right with real mode emulation.
609	* (#GP if we try to inject external hardware interrupts)
610	* Inject the interrupt or trap directly instead.
611	*/
612	Log(("Manual interrupt/trap '%x' inject (real mode)\n", iGate));
613
614	/* Check if the interrupt handler is present. */
615	if (iGate * 4 + 3 > pCtx->idtr.cbIdt)
616	{
617	Log(("IDT cbIdt violation\n"));
618	if (iGate != X86_XCPT_DF)
619	{
620	RTGCUINTPTR intInfo;
621
622	intInfo = (iGate == X86_XCPT_GP) ? (uint32_t)X86_XCPT_DF : iGate;
623	intInfo \|= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
624	intInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
625	intInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HWEXCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
626
627	return VMXR0InjectEvent(pVM, pVCpu, pCtx, intInfo, 0, 0 /* no error code according to the Intel docs */);
628	}
629	Log(("Triple fault -> reset the VM!\n"));
630	return VINF_EM_RESET;
631	}
632	if ( VMX_EXIT_INTERRUPTION_INFO_TYPE(intInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW
633	\|\| iGate == 3 /* Both #BP and #OF point to the instruction after. */
634	\|\| iGate == 4)
635	{
636	ip = pCtx->ip + cbInstr;
637	}
638	else
639	ip = pCtx->ip;
640
641	/* Read the selector:offset pair of the interrupt handler. */
642	GCPhysHandler = (RTGCPHYS)pCtx->idtr.pIdt + iGate * 4;
643	PGMPhysRead(pVM, GCPhysHandler, &offset, sizeof(offset));
644	PGMPhysRead(pVM, GCPhysHandler + 2, &sel, sizeof(sel));
645
646	LogFlow(("IDT handler %04X:%04X\n", sel, offset));
647
648	/* Construct the stack frame. */
649	/** @todo should check stack limit. */
650	pCtx->sp -= 2;
651	LogFlow(("ss:sp %04X:%04X eflags=%x\n", pCtx->ss, pCtx->sp, pCtx->eflags.u));
652	PGMPhysWrite(pVM, pCtx->ssHid.u64Base + pCtx->sp, &pCtx->eflags, sizeof(uint16_t));
653	pCtx->sp -= 2;
654	LogFlow(("ss:sp %04X:%04X cs=%x\n", pCtx->ss, pCtx->sp, pCtx->cs));
655	PGMPhysWrite(pVM, pCtx->ssHid.u64Base + pCtx->sp, &pCtx->cs, sizeof(uint16_t));
656	pCtx->sp -= 2;
657	LogFlow(("ss:sp %04X:%04X ip=%x\n", pCtx->ss, pCtx->sp, ip));
658	PGMPhysWrite(pVM, pCtx->ssHid.u64Base + pCtx->sp, &ip, sizeof(ip));
659
660	/* Update the CPU state for executing the handler. */
661	pCtx->rip = offset;
662	pCtx->cs = sel;
663	pCtx->csHid.u64Base = sel << 4;
664	pCtx->eflags.u &= ~(X86_EFL_IF\|X86_EFL_TF\|X86_EFL_RF\|X86_EFL_AC);
665
666	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_GUEST_SEGMENT_REGS;
667	return VINF_SUCCESS;
668	}
669	#endif /* HWACCM_VMX_EMULATE_REALMODE */
670
671	/* Set event injection state. */
672	rc = VMXWriteCachedVMCS(VMX_VMCS_CTRL_ENTRY_IRQ_INFO, intInfo \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT));
673
674	rc \|= VMXWriteCachedVMCS(VMX_VMCS_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
675	rc \|= VMXWriteCachedVMCS(VMX_VMCS_CTRL_ENTRY_EXCEPTION_ERRCODE, errCode);
676
677	AssertRC(rc);
678	return rc;
679	}
680
681
682	/**
683	* Checks for pending guest interrupts and injects them
684	*
685	* @returns VBox status code.
686	* @param pVM The VM to operate on.
687	* @param pVCpu The VMCPU to operate on.
688	* @param pCtx CPU Context
689	*/
690	static int VMXR0CheckPendingInterrupt(PVM pVM, PVMCPU pVCpu, CPUMCTX *pCtx)
691	{
692	int rc;
693
694	/* Dispatch any pending interrupts. (injected before, but a VM exit occurred prematurely) */
695	if (pVCpu->hwaccm.s.Event.fPending)
696	{
697	Log(("Reinjecting event %RX64 %08x at %RGv cr2=%RX64\n", pVCpu->hwaccm.s.Event.intInfo, pVCpu->hwaccm.s.Event.errCode, (RTGCPTR)pCtx->rip, pCtx->cr2));
698	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatIntReinject);
699	rc = VMXR0InjectEvent(pVM, pVCpu, pCtx, pVCpu->hwaccm.s.Event.intInfo, 0, pVCpu->hwaccm.s.Event.errCode);
700	AssertRC(rc);
701
702	pVCpu->hwaccm.s.Event.fPending = false;
703	return VINF_SUCCESS;
704	}
705
706	if (pVM->hwaccm.s.fInjectNMI)
707	{
708	RTGCUINTPTR intInfo;
709
710	intInfo = X86_XCPT_NMI;
711	intInfo \|= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
712	intInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
713
714	rc = VMXR0InjectEvent(pVM, pVCpu, pCtx, intInfo, 0, 0);
715	AssertRC(rc);
716
717	pVM->hwaccm.s.fInjectNMI = false;
718	return VINF_SUCCESS;
719	}
720
721	/* When external interrupts are pending, we should exit the VM when IF is set. */
722	if ( !TRPMHasTrap(pVM)
723	&& VM_FF_ISPENDING(pVM, (VM_FF_INTERRUPT_APIC\|VM_FF_INTERRUPT_PIC)))
724	{
725	if (!(pCtx->eflags.u32 & X86_EFL_IF))
726	{
727	if (!(pVCpu->hwaccm.s.vmx.proc_ctls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_IRQ_WINDOW_EXIT))
728	{
729	LogFlow(("Enable irq window exit!\n"));
730	pVCpu->hwaccm.s.vmx.proc_ctls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_IRQ_WINDOW_EXIT;
731	rc = VMXWriteCachedVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
732	AssertRC(rc);
733	}
734	/* else nothing to do but wait */
735	}
736	else
737	if (!VM_FF_ISSET(pVM, VM_FF_INHIBIT_INTERRUPTS))
738	{
739	uint8_t u8Interrupt;
740
741	rc = PDMGetInterrupt(pVM, &u8Interrupt);
742	Log(("Dispatch interrupt: u8Interrupt=%x (%d) rc=%Rrc cs:rip=%04X:%RGv\n", u8Interrupt, u8Interrupt, rc, pCtx->cs, (RTGCPTR)pCtx->rip));
743	if (RT_SUCCESS(rc))
744	{
745	rc = TRPMAssertTrap(pVM, u8Interrupt, TRPM_HARDWARE_INT);
746	AssertRC(rc);
747	}
748	else
749	{
750	/* Can only happen in rare cases where a pending interrupt is cleared behind our back */
751	Assert(!VM_FF_ISPENDING(pVM, (VM_FF_INTERRUPT_APIC\|VM_FF_INTERRUPT_PIC)));
752	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatSwitchGuestIrq);
753	/* Just continue */
754	}
755	}
756	else
757	Log(("Pending interrupt blocked at %RGv by VM_FF_INHIBIT_INTERRUPTS!!\n", (RTGCPTR)pCtx->rip));
758	}
759
760	#ifdef VBOX_STRICT
761	if (TRPMHasTrap(pVM))
762	{
763	uint8_t u8Vector;
764	rc = TRPMQueryTrapAll(pVM, &u8Vector, 0, 0, 0);
765	AssertRC(rc);
766	}
767	#endif
768
769	if ( pCtx->eflags.u32 & X86_EFL_IF
770	&& (!VM_FF_ISSET(pVM, VM_FF_INHIBIT_INTERRUPTS))
771	&& TRPMHasTrap(pVM)
772	)
773	{
774	uint8_t u8Vector;
775	int rc;
776	TRPMEVENT enmType;
777	RTGCUINTPTR intInfo;
778	RTGCUINT errCode;
779
780	/* If a new event is pending, then dispatch it now. */
781	rc = TRPMQueryTrapAll(pVM, &u8Vector, &enmType, &errCode, 0);
782	AssertRC(rc);
783	Assert(pCtx->eflags.Bits.u1IF == 1 \|\| enmType == TRPM_TRAP);
784	Assert(enmType != TRPM_SOFTWARE_INT);
785
786	/* Clear the pending trap. */
787	rc = TRPMResetTrap(pVM);
788	AssertRC(rc);
789
790	intInfo = u8Vector;
791	intInfo \|= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
792
793	if (enmType == TRPM_TRAP)
794	{
795	switch (u8Vector) {
796	case 8:
797	case 10:
798	case 11:
799	case 12:
800	case 13:
801	case 14:
802	case 17:
803	/* Valid error codes. */
804	intInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
805	break;
806	default:
807	break;
808	}
809	if (u8Vector == X86_XCPT_BP \|\| u8Vector == X86_XCPT_OF)
810	intInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SWEXCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
811	else
812	intInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HWEXCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
813	}
814	else
815	intInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
816
817	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatIntInject);
818	rc = VMXR0InjectEvent(pVM, pVCpu, pCtx, intInfo, 0, errCode);
819	AssertRC(rc);
820	} /* if (interrupts can be dispatched) */
821
822	return VINF_SUCCESS;
823	}
824
825	/**
826	* Save the host state
827	*
828	* @returns VBox status code.
829	* @param pVM The VM to operate on.
830	* @param pVCpu The VMCPU to operate on.
831	*/
832	VMMR0DECL(int) VMXR0SaveHostState(PVM pVM, PVMCPU pVCpu)
833	{
834	int rc = VINF_SUCCESS;
835
836	/*
837	* Host CPU Context
838	*/
839	if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_HOST_CONTEXT)
840	{
841	RTIDTR idtr;
842	RTGDTR gdtr;
843	RTSEL SelTR;
844	PX86DESCHC pDesc;
845	uintptr_t trBase;
846	RTSEL cs;
847	RTSEL ss;
848	uint64_t cr3;
849
850	/* Control registers */
851	rc = VMXWriteCachedVMCS(VMX_VMCS_HOST_CR0, ASMGetCR0());
852	#ifdef VBOX_WITH_HYBIRD_32BIT_KERNEL
853	if (VMX_IS_64BIT_HOST_MODE())
854	{
855	cr3 = hwaccmR0Get64bitCR3();
856	rc \|= VMXWriteCachedVMCS64(VMX_VMCS_HOST_CR3, cr3);
857	}
858	else
859	#endif
860	{
861	cr3 = ASMGetCR3();
862	rc \|= VMXWriteCachedVMCS(VMX_VMCS_HOST_CR3, cr3);
863	}
864	rc \|= VMXWriteCachedVMCS(VMX_VMCS_HOST_CR4, ASMGetCR4());
865	AssertRC(rc);
866	Log2(("VMX_VMCS_HOST_CR0 %08x\n", ASMGetCR0()));
867	Log2(("VMX_VMCS_HOST_CR3 %08RX64\n", cr3));
868	Log2(("VMX_VMCS_HOST_CR4 %08x\n", ASMGetCR4()));
869
870	/* Selector registers. */
871	#ifdef VBOX_WITH_HYBIRD_32BIT_KERNEL
872	if (VMX_IS_64BIT_HOST_MODE())
873	{
874	cs = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelCS;
875	ss = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelSS;
876	}
877	else
878	{
879	/* sysenter loads LDT cs & ss, VMX doesn't like this. Load the GDT ones (safe). */
880	cs = (RTSEL)(uintptr_t)&SUPR0AbsKernelCS;
881	ss = (RTSEL)(uintptr_t)&SUPR0AbsKernelSS;
882	}
883	#else
884	cs = ASMGetCS();
885	ss = ASMGetSS();
886	#endif
887	Assert(!(cs & X86_SEL_LDT)); Assert((cs & X86_SEL_RPL) == 0);
888	Assert(!(ss & X86_SEL_LDT)); Assert((ss & X86_SEL_RPL) == 0);
889	rc = VMXWriteCachedVMCS(VMX_VMCS16_HOST_FIELD_CS, cs);
890	/* Note: VMX is (again) very picky about the RPL of the selectors here; we'll restore them manually. */
891	rc \|= VMXWriteCachedVMCS(VMX_VMCS16_HOST_FIELD_DS, 0);
892	rc \|= VMXWriteCachedVMCS(VMX_VMCS16_HOST_FIELD_ES, 0);
893	#if HC_ARCH_BITS == 32
894	if (!VMX_IS_64BIT_HOST_MODE())
895	{
896	rc \|= VMXWriteCachedVMCS(VMX_VMCS16_HOST_FIELD_FS, 0);
897	rc \|= VMXWriteCachedVMCS(VMX_VMCS16_HOST_FIELD_GS, 0);
898	}
899	#endif
900	rc \|= VMXWriteCachedVMCS(VMX_VMCS16_HOST_FIELD_SS, ss);
901	SelTR = ASMGetTR();
902	rc \|= VMXWriteCachedVMCS(VMX_VMCS16_HOST_FIELD_TR, SelTR);
903	AssertRC(rc);
904	Log2(("VMX_VMCS_HOST_FIELD_CS %08x (%08x)\n", cs, ASMGetSS()));
905	Log2(("VMX_VMCS_HOST_FIELD_DS 00000000 (%08x)\n", ASMGetDS()));
906	Log2(("VMX_VMCS_HOST_FIELD_ES 00000000 (%08x)\n", ASMGetES()));
907	Log2(("VMX_VMCS_HOST_FIELD_FS 00000000 (%08x)\n", ASMGetFS()));
908	Log2(("VMX_VMCS_HOST_FIELD_GS 00000000 (%08x)\n", ASMGetGS()));
909	Log2(("VMX_VMCS_HOST_FIELD_SS %08x (%08x)\n", ss, ASMGetSS()));
910	Log2(("VMX_VMCS_HOST_FIELD_TR %08x\n", ASMGetTR()));
911
912	/* GDTR & IDTR */
913	#ifdef VBOX_WITH_HYBIRD_32BIT_KERNEL
914	if (VMX_IS_64BIT_HOST_MODE())
915	{
916	X86XDTR64 gdtr64, idtr64;
917	hwaccmR0Get64bitGDTRandIDTR(&gdtr64, &idtr64);
918	rc = VMXWriteCachedVMCS64(VMX_VMCS_HOST_GDTR_BASE, gdtr64.uAddr);
919	rc \|= VMXWriteCachedVMCS64(VMX_VMCS_HOST_IDTR_BASE, gdtr64.uAddr);
920	AssertRC(rc);
921	Log2(("VMX_VMCS_HOST_GDTR_BASE %RX64\n", gdtr64.uAddr));
922	Log2(("VMX_VMCS_HOST_IDTR_BASE %RX64\n", idtr64.uAddr));
923	gdtr.cbGdt = gdtr64.cb;
924	gdtr.pGdt = (uintptr_t)gdtr64.uAddr;
925	}
926	else
927	#endif
928	{
929	ASMGetGDTR(&gdtr);
930	rc = VMXWriteCachedVMCS(VMX_VMCS_HOST_GDTR_BASE, gdtr.pGdt);
931	ASMGetIDTR(&idtr);
932	rc \|= VMXWriteCachedVMCS(VMX_VMCS_HOST_IDTR_BASE, idtr.pIdt);
933	AssertRC(rc);
934	Log2(("VMX_VMCS_HOST_GDTR_BASE %RHv\n", gdtr.pGdt));
935	Log2(("VMX_VMCS_HOST_IDTR_BASE %RHv\n", idtr.pIdt));
936	}
937
938
939	/* Save the base address of the TR selector. */
940	if (SelTR > gdtr.cbGdt)
941	{
942	AssertMsgFailed(("Invalid TR selector %x. GDTR.cbGdt=%x\n", SelTR, gdtr.cbGdt));
943	return VERR_VMX_INVALID_HOST_STATE;
944	}
945
946	#ifdef VBOX_WITH_HYBIRD_32BIT_KERNEL
947	if (VMX_IS_64BIT_HOST_MODE())
948	{
949	pDesc = &((PX86DESCHC)gdtr.pGdt)[SelTR >> X86_SEL_SHIFT_HC]; /// ????
950	uint64_t trBase64 = X86DESC64_BASE(*(PX86DESC64)pDesc);
951	rc = VMXWriteCachedVMCS64(VMX_VMCS_HOST_TR_BASE, trBase64);
952	Log2(("VMX_VMCS_HOST_TR_BASE %RX64\n", trBase64));
953	AssertRC(rc);
954	}
955	else
956	#endif
957	{
958	pDesc = &((PX86DESCHC)gdtr.pGdt)[SelTR >> X86_SEL_SHIFT_HC];
959	#if HC_ARCH_BITS == 64
960	trBase = X86DESC64_BASE(*pDesc);
961	#else
962	trBase = X86DESC_BASE(*pDesc);
963	#endif
964	rc = VMXWriteCachedVMCS(VMX_VMCS_HOST_TR_BASE, trBase);
965	AssertRC(rc);
966	Log2(("VMX_VMCS_HOST_TR_BASE %RHv\n", trBase));
967	}
968
969	/* FS and GS base. */
970	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBIRD_32BIT_KERNEL)
971	if (VMX_IS_64BIT_HOST_MODE())
972	{
973	Log2(("MSR_K8_FS_BASE = %RX64\n", ASMRdMsr(MSR_K8_FS_BASE)));
974	Log2(("MSR_K8_GS_BASE = %RX64\n", ASMRdMsr(MSR_K8_GS_BASE)));
975	rc = VMXWriteCachedVMCS64(VMX_VMCS_HOST_FS_BASE, ASMRdMsr(MSR_K8_FS_BASE));
976	rc \|= VMXWriteCachedVMCS64(VMX_VMCS_HOST_GS_BASE, ASMRdMsr(MSR_K8_GS_BASE));
977	}
978	#endif
979	AssertRC(rc);
980
981	/* Sysenter MSRs. */
982	/** @todo expensive!! */
983	rc = VMXWriteCachedVMCS(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
984	Log2(("VMX_VMCS_HOST_SYSENTER_CS %08x\n", ASMRdMsr_Low(MSR_IA32_SYSENTER_CS)));
985	#ifdef VBOX_WITH_HYBIRD_32BIT_KERNEL
986	if (VMX_IS_64BIT_HOST_MODE())
987	{
988	Log2(("VMX_VMCS_HOST_SYSENTER_EIP %RX64\n", ASMRdMsr(MSR_IA32_SYSENTER_EIP)));
989	Log2(("VMX_VMCS_HOST_SYSENTER_ESP %RX64\n", ASMRdMsr(MSR_IA32_SYSENTER_ESP)));
990	rc \|= VMXWriteCachedVMCS64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
991	rc \|= VMXWriteCachedVMCS64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
992	}
993	else
994	{
995	rc \|= VMXWriteCachedVMCS(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
996	rc \|= VMXWriteCachedVMCS(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
997	Log2(("VMX_VMCS_HOST_SYSENTER_EIP %RX32\n", ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP)));
998	Log2(("VMX_VMCS_HOST_SYSENTER_ESP %RX32\n", ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP)));
999	}
1000	#elif HC_ARCH_BITS == 32
1001	rc \|= VMXWriteCachedVMCS(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
1002	rc \|= VMXWriteCachedVMCS(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
1003	Log2(("VMX_VMCS_HOST_SYSENTER_EIP %RX32\n", ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP)));
1004	Log2(("VMX_VMCS_HOST_SYSENTER_ESP %RX32\n", ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP)));
1005	#else
1006	Log2(("VMX_VMCS_HOST_SYSENTER_EIP %RX64\n", ASMRdMsr(MSR_IA32_SYSENTER_EIP)));
1007	Log2(("VMX_VMCS_HOST_SYSENTER_ESP %RX64\n", ASMRdMsr(MSR_IA32_SYSENTER_ESP)));
1008	rc \|= VMXWriteCachedVMCS64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
1009	rc \|= VMXWriteCachedVMCS64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
1010	#endif
1011	AssertRC(rc);
1012
1013	pVCpu->hwaccm.s.fContextUseFlags &= ~HWACCM_CHANGED_HOST_CONTEXT;
1014	}
1015	return rc;
1016	}
1017
1018	/**
1019	* Prefetch the 4 PDPT pointers (PAE and nested paging only)
1020	*
1021	* @param pVM The VM to operate on.
1022	* @param pVCpu The VMCPU to operate on.
1023	* @param pCtx Guest context
1024	*/
1025	static void vmxR0PrefetchPAEPdptrs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
1026	{
1027	if (CPUMIsGuestInPAEModeEx(pCtx))
1028	{
1029	X86PDPE Pdpe;
1030
1031	for (unsigned i=0;i<4;i++)
1032	{
1033	Pdpe = PGMGstGetPaePDPtr(pVM, i);
1034	int rc = VMXWriteCachedVMCS64(VMX_VMCS_GUEST_PDPTR0_FULL + i*2, Pdpe.u);
1035	AssertRC(rc);
1036	}
1037	}
1038	}
1039
1040	/**
1041	* Update the exception bitmap according to the current CPU state
1042	*
1043	* @param pVM The VM to operate on.
1044	* @param pVCpu The VMCPU to operate on.
1045	* @param pCtx Guest context
1046	*/
1047	static void vmxR0UpdateExceptionBitmap(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
1048	{
1049	uint32_t u32TrapMask;
1050	Assert(pCtx);
1051
1052	u32TrapMask = HWACCM_VMX_TRAP_MASK;
1053	#ifndef DEBUG
1054	if (pVM->hwaccm.s.fNestedPaging)
1055	u32TrapMask &= ~RT_BIT(X86_XCPT_PF); /* no longer need to intercept #PF. */
1056	#endif
1057
1058	/* Also catch floating point exceptions as we need to report them to the guest in a different way. */
1059	if ( CPUMIsGuestFPUStateActive(pVCpu) == true
1060	&& !(pCtx->cr0 & X86_CR0_NE)
1061	&& !pVCpu->hwaccm.s.fFPUOldStyleOverride)
1062	{
1063	u32TrapMask \|= RT_BIT(X86_XCPT_MF);
1064	pVCpu->hwaccm.s.fFPUOldStyleOverride = true;
1065	}
1066
1067	#ifdef DEBUG
1068	/* Intercept X86_XCPT_DB if stepping is enabled */
1069	if (DBGFIsStepping(pVM))
1070	u32TrapMask \|= RT_BIT(X86_XCPT_DB);
1071	#endif
1072
1073	#ifdef VBOX_STRICT
1074	Assert(u32TrapMask & RT_BIT(X86_XCPT_GP));
1075	#endif
1076
1077	# ifdef HWACCM_VMX_EMULATE_REALMODE
1078	/* Intercept all exceptions in real mode as none of them can be injected directly (#GP otherwise). */
1079	if (CPUMIsGuestInRealModeEx(pCtx))
1080	u32TrapMask \|= HWACCM_VMX_TRAP_MASK_REALMODE;
1081	# endif /* HWACCM_VMX_EMULATE_REALMODE */
1082
1083	int rc = VMXWriteCachedVMCS(VMX_VMCS_CTRL_EXCEPTION_BITMAP, u32TrapMask);
1084	AssertRC(rc);
1085	}
1086
1087	/**
1088	* Loads the guest state
1089	*
1090	* NOTE: Don't do anything here that can cause a jump back to ring 3!!!!!
1091	*
1092	* @returns VBox status code.
1093	* @param pVM The VM to operate on.
1094	* @param pVCpu The VMCPU to operate on.
1095	* @param pCtx Guest context
1096	*/
1097	VMMR0DECL(int) VMXR0LoadGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
1098	{
1099	int rc = VINF_SUCCESS;
1100	RTGCUINTPTR val;
1101	X86EFLAGS eflags;
1102
1103	/* Guest CPU context: ES, CS, SS, DS, FS, GS. */
1104	if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_SEGMENT_REGS)
1105	{
1106	#ifdef HWACCM_VMX_EMULATE_REALMODE
1107	PGMMODE enmGuestMode = PGMGetGuestMode(pVM);
1108	if (pVCpu->hwaccm.s.vmx.enmCurrGuestMode != enmGuestMode)
1109	{
1110	/* Correct weird requirements for switching to protected mode. */
1111	if ( pVCpu->hwaccm.s.vmx.enmCurrGuestMode == PGMMODE_REAL
1112	&& enmGuestMode >= PGMMODE_PROTECTED)
1113	{
1114	/* DPL of all hidden selector registers must match the current CPL (0). */
1115	pCtx->csHid.Attr.n.u2Dpl = 0;
1116	pCtx->csHid.Attr.n.u4Type = X86_SEL_TYPE_CODE \| X86_SEL_TYPE_RW_ACC;
1117
1118	pCtx->dsHid.Attr.n.u2Dpl = 0;
1119	pCtx->esHid.Attr.n.u2Dpl = 0;
1120	pCtx->fsHid.Attr.n.u2Dpl = 0;
1121	pCtx->gsHid.Attr.n.u2Dpl = 0;
1122	pCtx->ssHid.Attr.n.u2Dpl = 0;
1123	}
1124	else
1125	/* Switching from protected mode to real mode. */
1126	if ( pVCpu->hwaccm.s.vmx.enmCurrGuestMode >= PGMMODE_PROTECTED
1127	&& enmGuestMode == PGMMODE_REAL)
1128	{
1129	/* The limit must also be adjusted. */
1130	pCtx->csHid.u32Limit &= 0xffff;
1131	pCtx->dsHid.u32Limit &= 0xffff;
1132	pCtx->esHid.u32Limit &= 0xffff;
1133	pCtx->fsHid.u32Limit &= 0xffff;
1134	pCtx->gsHid.u32Limit &= 0xffff;
1135	pCtx->ssHid.u32Limit &= 0xffff;
1136
1137	Assert(pCtx->csHid.u64Base <= 0xfffff);
1138	Assert(pCtx->dsHid.u64Base <= 0xfffff);
1139	Assert(pCtx->esHid.u64Base <= 0xfffff);
1140	Assert(pCtx->fsHid.u64Base <= 0xfffff);
1141	Assert(pCtx->gsHid.u64Base <= 0xfffff);
1142	}
1143	pVCpu->hwaccm.s.vmx.enmCurrGuestMode = enmGuestMode;
1144	}
1145	else
1146	/* VT-x will fail with a guest invalid state otherwise... (CPU state after a reset) */
1147	if ( CPUMIsGuestInRealModeEx(pCtx)
1148	&& pCtx->csHid.u64Base == 0xffff0000)
1149	{
1150	pCtx->csHid.u64Base = 0xf0000;
1151	pCtx->cs = 0xf000;
1152	}
1153	#endif /* HWACCM_VMX_EMULATE_REALMODE */
1154
1155	VMX_WRITE_SELREG(ES, es);
1156	AssertRC(rc);
1157
1158	VMX_WRITE_SELREG(CS, cs);
1159	AssertRC(rc);
1160
1161	VMX_WRITE_SELREG(SS, ss);
1162	AssertRC(rc);
1163
1164	VMX_WRITE_SELREG(DS, ds);
1165	AssertRC(rc);
1166
1167	/* The base values in the hidden fs & gs registers are not in sync with the msrs; they are cut to 32 bits. */
1168	VMX_WRITE_SELREG(FS, fs);
1169	AssertRC(rc);
1170
1171	VMX_WRITE_SELREG(GS, gs);
1172	AssertRC(rc);
1173	}
1174
1175	/* Guest CPU context: LDTR. */
1176	if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_LDTR)
1177	{
1178	if (pCtx->ldtr == 0)
1179	{
1180	rc = VMXWriteCachedVMCS(VMX_VMCS16_GUEST_FIELD_LDTR, 0);
1181	rc \|= VMXWriteCachedVMCS(VMX_VMCS32_GUEST_LDTR_LIMIT, 0);
1182	rc \|= VMXWriteCachedVMCS(VMX_VMCS64_GUEST_LDTR_BASE, 0);
1183	/* Note: vmlaunch will fail with 0 or just 0x02. No idea why. */
1184	rc \|= VMXWriteCachedVMCS(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, 0x82 /* present, LDT */);
1185	}
1186	else
1187	{
1188	rc = VMXWriteCachedVMCS(VMX_VMCS16_GUEST_FIELD_LDTR, pCtx->ldtr);
1189	rc \|= VMXWriteCachedVMCS(VMX_VMCS32_GUEST_LDTR_LIMIT, pCtx->ldtrHid.u32Limit);
1190	rc \|= VMXWriteCachedVMCS(VMX_VMCS64_GUEST_LDTR_BASE, pCtx->ldtrHid.u64Base);
1191	rc \|= VMXWriteCachedVMCS(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, pCtx->ldtrHid.Attr.u);
1192	}
1193	AssertRC(rc);
1194	}
1195	/* Guest CPU context: TR. */
1196	if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_TR)
1197	{
1198	#ifdef HWACCM_VMX_EMULATE_REALMODE
1199	/* Real mode emulation using v86 mode with CR4.VME (interrupt redirection using the int bitmap in the TSS) */
1200	if (CPUMIsGuestInRealModeEx(pCtx))
1201	{
1202	RTGCPHYS GCPhys;
1203
1204	/* We convert it here every time as pci regions could be reconfigured. */
1205	rc = PDMVMMDevHeapR3ToGCPhys(pVM, pVM->hwaccm.s.vmx.pRealModeTSS, &GCPhys);
1206	AssertRC(rc);
1207
1208	rc = VMXWriteCachedVMCS(VMX_VMCS16_GUEST_FIELD_TR, 0);
1209	rc \|= VMXWriteCachedVMCS(VMX_VMCS32_GUEST_TR_LIMIT, HWACCM_VTX_TSS_SIZE);
1210	rc \|= VMXWriteCachedVMCS(VMX_VMCS64_GUEST_TR_BASE, GCPhys /* phys = virt in this mode */);
1211
1212	X86DESCATTR attr;
1213
1214	attr.u = 0;
1215	attr.n.u1Present = 1;
1216	attr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
1217	val = attr.u;
1218	}
1219	else
1220	#endif /* HWACCM_VMX_EMULATE_REALMODE */
1221	{
1222	rc = VMXWriteCachedVMCS(VMX_VMCS16_GUEST_FIELD_TR, pCtx->tr);
1223	rc \|= VMXWriteCachedVMCS(VMX_VMCS32_GUEST_TR_LIMIT, pCtx->trHid.u32Limit);
1224	rc \|= VMXWriteCachedVMCS(VMX_VMCS64_GUEST_TR_BASE, pCtx->trHid.u64Base);
1225
1226	val = pCtx->trHid.Attr.u;
1227
1228	/* The TSS selector must be busy. */
1229	if ((val & 0xF) == X86_SEL_TYPE_SYS_286_TSS_AVAIL)
1230	val = (val & ~0xF) \| X86_SEL_TYPE_SYS_286_TSS_BUSY;
1231	else
1232	/* Default even if no TR selector has been set (otherwise vmlaunch will fail!) */
1233	val = (val & ~0xF) \| X86_SEL_TYPE_SYS_386_TSS_BUSY;
1234
1235	}
1236	rc \|= VMXWriteCachedVMCS(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, val);
1237	AssertRC(rc);
1238	}
1239	/* Guest CPU context: GDTR. */
1240	if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_GDTR)
1241	{
1242	rc = VMXWriteCachedVMCS(VMX_VMCS32_GUEST_GDTR_LIMIT, pCtx->gdtr.cbGdt);
1243	rc \|= VMXWriteCachedVMCS(VMX_VMCS64_GUEST_GDTR_BASE, pCtx->gdtr.pGdt);
1244	AssertRC(rc);
1245	}
1246	/* Guest CPU context: IDTR. */
1247	if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_IDTR)
1248	{
1249	rc = VMXWriteCachedVMCS(VMX_VMCS32_GUEST_IDTR_LIMIT, pCtx->idtr.cbIdt);
1250	rc \|= VMXWriteCachedVMCS(VMX_VMCS64_GUEST_IDTR_BASE, pCtx->idtr.pIdt);
1251	AssertRC(rc);
1252	}
1253
1254	/*
1255	* Sysenter MSRs (unconditional)
1256	*/
1257	rc = VMXWriteCachedVMCS(VMX_VMCS32_GUEST_SYSENTER_CS, pCtx->SysEnter.cs);
1258	rc \|= VMXWriteCachedVMCS(VMX_VMCS64_GUEST_SYSENTER_EIP, pCtx->SysEnter.eip);
1259	rc \|= VMXWriteCachedVMCS(VMX_VMCS64_GUEST_SYSENTER_ESP, pCtx->SysEnter.esp);
1260	AssertRC(rc);
1261
1262	/* Control registers */
1263	if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_CR0)
1264	{
1265	val = pCtx->cr0;
1266	rc = VMXWriteCachedVMCS(VMX_VMCS_CTRL_CR0_READ_SHADOW, val);
1267	Log2(("Guest CR0-shadow %08x\n", val));
1268	if (CPUMIsGuestFPUStateActive(pVCpu) == false)
1269	{
1270	/* Always use #NM exceptions to load the FPU/XMM state on demand. */
1271	val \|= X86_CR0_TS \| X86_CR0_ET \| X86_CR0_NE \| X86_CR0_MP;
1272	}
1273	else
1274	{
1275	/** @todo check if we support the old style mess correctly. */
1276	if (!(val & X86_CR0_NE))
1277	Log(("Forcing X86_CR0_NE!!!\n"));
1278
1279	val \|= X86_CR0_NE; /* always turn on the native mechanism to report FPU errors (old style uses interrupts) */
1280	}
1281	/* Note: protected mode & paging are always enabled; we use them for emulating real and protected mode without paging too. */
1282	val \|= X86_CR0_PE \| X86_CR0_PG;
1283	if (pVM->hwaccm.s.fNestedPaging)
1284	{
1285	if (CPUMIsGuestInPagedProtectedModeEx(pCtx))
1286	{
1287	/* Disable cr3 read/write monitoring as we don't need it for EPT. */
1288	pVCpu->hwaccm.s.vmx.proc_ctls &= ~( VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
1289	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT);
1290	}
1291	else
1292	{
1293	/* Reenable cr3 read/write monitoring as our identity mapped page table is active. */
1294	pVCpu->hwaccm.s.vmx.proc_ctls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
1295	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT;
1296	}
1297	rc = VMXWriteCachedVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
1298	AssertRC(rc);
1299	}
1300	else
1301	{
1302	/* Note: We must also set this as we rely on protecting various pages for which supervisor writes must be caught. */
1303	val \|= X86_CR0_WP;
1304	}
1305
1306	/* Always enable caching. */
1307	val &= ~(X86_CR0_CD\|X86_CR0_NW);
1308
1309	rc \|= VMXWriteCachedVMCS(VMX_VMCS64_GUEST_CR0, val);
1310	Log2(("Guest CR0 %08x\n", val));
1311	/* CR0 flags owned by the host; if the guests attempts to change them, then
1312	* the VM will exit.
1313	*/
1314	val = X86_CR0_PE /* Must monitor this bit (assumptions are made for real mode emulation) */
1315	\| X86_CR0_WP /* Must monitor this bit (it must always be enabled). */
1316	\| X86_CR0_PG /* Must monitor this bit (assumptions are made for real mode & protected mode without paging emulation) */
1317	\| X86_CR0_TS
1318	\| X86_CR0_ET /* Bit not restored during VM-exit! */
1319	\| X86_CR0_CD /* Bit not restored during VM-exit! */
1320	\| X86_CR0_NW /* Bit not restored during VM-exit! */
1321	\| X86_CR0_NE
1322	\| X86_CR0_MP;
1323	pVCpu->hwaccm.s.vmx.cr0_mask = val;
1324
1325	rc \|= VMXWriteCachedVMCS(VMX_VMCS_CTRL_CR0_MASK, val);
1326	Log2(("Guest CR0-mask %08x\n", val));
1327	AssertRC(rc);
1328	}
1329	if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_CR4)
1330	{
1331	/* CR4 */
1332	rc = VMXWriteCachedVMCS(VMX_VMCS_CTRL_CR4_READ_SHADOW, pCtx->cr4);
1333	Log2(("Guest CR4-shadow %08x\n", pCtx->cr4));
1334	/* Set the required bits in cr4 too (currently X86_CR4_VMXE). */
1335	val = pCtx->cr4 \| (uint32_t)pVM->hwaccm.s.vmx.msr.vmx_cr4_fixed0;
1336
1337	if (!pVM->hwaccm.s.fNestedPaging)
1338	{
1339	switch(pVCpu->hwaccm.s.enmShadowMode)
1340	{
1341	case PGMMODE_REAL: /* Real mode -> emulated using v86 mode */
1342	case PGMMODE_PROTECTED: /* Protected mode, no paging -> emulated using identity mapping. */
1343	case PGMMODE_32_BIT: /* 32-bit paging. */
1344	break;
1345
1346	case PGMMODE_PAE: /* PAE paging. */
1347	case PGMMODE_PAE_NX: /* PAE paging with NX enabled. */
1348	/** @todo use normal 32 bits paging */
1349	val \|= X86_CR4_PAE;
1350	break;
1351
1352	case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
1353	case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
1354	#ifdef VBOX_ENABLE_64_BITS_GUESTS
1355	break;
1356	#else
1357	AssertFailed();
1358	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
1359	#endif
1360	default: /* shut up gcc */
1361	AssertFailed();
1362	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
1363	}
1364	}
1365	else
1366	if (!CPUMIsGuestInPagedProtectedModeEx(pCtx))
1367	{
1368	/* We use 4 MB pages in our identity mapping page table for real and protected mode without paging. */
1369	val \|= X86_CR4_PSE;
1370	/* Our identity mapping is a 32 bits page directory. */
1371	val &= ~X86_CR4_PAE;
1372	}
1373
1374	#ifdef HWACCM_VMX_EMULATE_REALMODE
1375	/* Real mode emulation using v86 mode with CR4.VME (interrupt redirection using the int bitmap in the TSS) */
1376	if (CPUMIsGuestInRealModeEx(pCtx))
1377	val \|= X86_CR4_VME;
1378	#endif /* HWACCM_VMX_EMULATE_REALMODE */
1379
1380	rc \|= VMXWriteCachedVMCS(VMX_VMCS64_GUEST_CR4, val);
1381	Log2(("Guest CR4 %08x\n", val));
1382	/* CR4 flags owned by the host; if the guests attempts to change them, then
1383	* the VM will exit.
1384	*/
1385	val = 0
1386	#ifdef HWACCM_VMX_EMULATE_REALMODE
1387	\| X86_CR4_VME
1388	#endif
1389	\| X86_CR4_PAE
1390	\| X86_CR4_PGE
1391	\| X86_CR4_PSE
1392	\| X86_CR4_VMXE;
1393	pVCpu->hwaccm.s.vmx.cr4_mask = val;
1394
1395	rc \|= VMXWriteCachedVMCS(VMX_VMCS_CTRL_CR4_MASK, val);
1396	Log2(("Guest CR4-mask %08x\n", val));
1397	AssertRC(rc);
1398	}
1399
1400	if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_CR3)
1401	{
1402	if (pVM->hwaccm.s.fNestedPaging)
1403	{
1404	AssertMsg(PGMGetEPTCR3(pVM) == PGMGetHyperCR3(pVM), ("%RHp vs %RHp\n", PGMGetEPTCR3(pVM), PGMGetHyperCR3(pVM)));
1405	pVCpu->hwaccm.s.vmx.GCPhysEPTP = PGMGetEPTCR3(pVM);
1406
1407	Assert(!(pVCpu->hwaccm.s.vmx.GCPhysEPTP & 0xfff));
1408	/** @todo Check the IA32_VMX_EPT_VPID_CAP MSR for other supported memory types. */
1409	pVCpu->hwaccm.s.vmx.GCPhysEPTP \|= VMX_EPT_MEMTYPE_WB
1410	\| (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
1411
1412	rc = VMXWriteCachedVMCS64(VMX_VMCS_CTRL_EPTP_FULL, pVCpu->hwaccm.s.vmx.GCPhysEPTP);
1413	AssertRC(rc);
1414
1415	if (!CPUMIsGuestInPagedProtectedModeEx(pCtx))
1416	{
1417	RTGCPHYS GCPhys;
1418
1419	/* We convert it here every time as pci regions could be reconfigured. */
1420	rc = PDMVMMDevHeapR3ToGCPhys(pVM, pVM->hwaccm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
1421	AssertRC(rc);
1422
1423	/* We use our identity mapping page table here as we need to map guest virtual to guest physical addresses; EPT will
1424	* take care of the translation to host physical addresses.
1425	*/
1426	val = GCPhys;
1427	}
1428	else
1429	{
1430	/* Save the real guest CR3 in VMX_VMCS_GUEST_CR3 */
1431	val = pCtx->cr3;
1432	/* Prefetch the four PDPT entries in PAE mode. */
1433	vmxR0PrefetchPAEPdptrs(pVM, pVCpu, pCtx);
1434	}
1435	}
1436	else
1437	{
1438	val = PGMGetHyperCR3(pVM);
1439	Assert(val);
1440	}
1441
1442	/* Save our shadow CR3 register. */
1443	rc = VMXWriteCachedVMCS(VMX_VMCS64_GUEST_CR3, val);
1444	AssertRC(rc);
1445	}
1446
1447	/* Debug registers. */
1448	if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_DEBUG)
1449	{
1450	pCtx->dr[6] \|= X86_DR6_INIT_VAL; /* set all reserved bits to 1. */
1451	pCtx->dr[6] &= ~RT_BIT(12); /* must be zero. */
1452
1453	pCtx->dr[7] &= 0xffffffff; /* upper 32 bits reserved */
1454	pCtx->dr[7] &= ~(RT_BIT(11) \| RT_BIT(12) \| RT_BIT(14) \| RT_BIT(15)); /* must be zero */
1455	pCtx->dr[7] \|= 0x400; /* must be one */
1456
1457	/* Resync DR7 */
1458	rc = VMXWriteCachedVMCS(VMX_VMCS64_GUEST_DR7, pCtx->dr[7]);
1459	AssertRC(rc);
1460
1461	/* Sync the debug state now if any breakpoint is armed. */
1462	if ( (pCtx->dr[7] & (X86_DR7_ENABLED_MASK\|X86_DR7_GD))
1463	&& !CPUMIsGuestDebugStateActive(pVM)
1464	&& !DBGFIsStepping(pVM))
1465	{
1466	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatDRxArmed);
1467
1468	/* Disable drx move intercepts. */
1469	pVCpu->hwaccm.s.vmx.proc_ctls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT;
1470	rc = VMXWriteCachedVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
1471	AssertRC(rc);
1472
1473	/* Save the host and load the guest debug state. */
1474	rc = CPUMR0LoadGuestDebugState(pVM, pVCpu, pCtx, true /* include DR6 */);
1475	AssertRC(rc);
1476	}
1477
1478	/* IA32_DEBUGCTL MSR. */
1479	rc = VMXWriteCachedVMCS64(VMX_VMCS_GUEST_DEBUGCTL_FULL, 0);
1480	AssertRC(rc);
1481
1482	/** @todo do we really ever need this? */
1483	rc \|= VMXWriteCachedVMCS(VMX_VMCS_GUEST_DEBUG_EXCEPTIONS, 0);
1484	AssertRC(rc);
1485	}
1486
1487	/* EIP, ESP and EFLAGS */
1488	rc = VMXWriteCachedVMCS(VMX_VMCS64_GUEST_RIP, pCtx->rip);
1489	rc \|= VMXWriteCachedVMCS(VMX_VMCS64_GUEST_RSP, pCtx->rsp);
1490	AssertRC(rc);
1491
1492	/* Bits 22-31, 15, 5 & 3 must be zero. Bit 1 must be 1. */
1493	eflags = pCtx->eflags;
1494	eflags.u32 &= VMX_EFLAGS_RESERVED_0;
1495	eflags.u32 \|= VMX_EFLAGS_RESERVED_1;
1496
1497	#ifdef HWACCM_VMX_EMULATE_REALMODE
1498	/* Real mode emulation using v86 mode with CR4.VME (interrupt redirection using the int bitmap in the TSS) */
1499	if (CPUMIsGuestInRealModeEx(pCtx))
1500	{
1501	pVCpu->hwaccm.s.vmx.RealMode.eflags = eflags;
1502
1503	eflags.Bits.u1VM = 1;
1504	eflags.Bits.u2IOPL = 3;
1505	}
1506	#endif /* HWACCM_VMX_EMULATE_REALMODE */
1507	rc = VMXWriteCachedVMCS(VMX_VMCS_GUEST_RFLAGS, eflags.u32);
1508	AssertRC(rc);
1509
1510	/* TSC offset. */
1511	uint64_t u64TSCOffset;
1512
1513	if (TMCpuTickCanUseRealTSC(pVM, &u64TSCOffset))
1514	{
1515	/* Note: VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT takes precedence over TSC_OFFSET */
1516	rc = VMXWriteCachedVMCS64(VMX_VMCS_CTRL_TSC_OFFSET_FULL, u64TSCOffset);
1517	AssertRC(rc);
1518
1519	pVCpu->hwaccm.s.vmx.proc_ctls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT;
1520	rc = VMXWriteCachedVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
1521	AssertRC(rc);
1522	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatTSCOffset);
1523	}
1524	else
1525	{
1526	pVCpu->hwaccm.s.vmx.proc_ctls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT;
1527	rc = VMXWriteCachedVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
1528	AssertRC(rc);
1529	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatTSCIntercept);
1530	}
1531
1532	/* VMX_VMCS_CTRL_ENTRY_CONTROLS
1533	* Set required bits to one and zero according to the MSR capabilities.
1534	*/
1535	val = pVM->hwaccm.s.vmx.msr.vmx_entry.n.disallowed0;
1536	/* Load guest debug controls (dr7 & IA32_DEBUGCTL_MSR) (forced to 1 on the 'first' VT-x capable CPUs; this actually includes the newest Nehalem CPUs) */
1537	val \|= VMX_VMCS_CTRL_ENTRY_CONTROLS_LOAD_DEBUG;
1538
1539	/* 64 bits guest mode? */
1540	if (pCtx->msrEFER & MSR_K6_EFER_LMA)
1541	val \|= VMX_VMCS_CTRL_ENTRY_CONTROLS_IA64_MODE;
1542	/* else Must be zero when AMD64 is not available. */
1543
1544	/* Mask away the bits that the CPU doesn't support */
1545	val &= pVM->hwaccm.s.vmx.msr.vmx_entry.n.allowed1;
1546	rc = VMXWriteCachedVMCS(VMX_VMCS_CTRL_ENTRY_CONTROLS, val);
1547	AssertRC(rc);
1548
1549	/* 64 bits guest mode? */
1550	if (pCtx->msrEFER & MSR_K6_EFER_LMA)
1551	{
1552	#if !defined(VBOX_ENABLE_64_BITS_GUESTS)
1553	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
1554	#elif HC_ARCH_BITS == 32 && !defined(RT_OS_DARWIN)
1555	pVCpu->hwaccm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
1556	#else
1557	pVCpu->hwaccm.s.vmx.pfnStartVM = VMXR0StartVM64;
1558	#endif
1559	/* Unconditionally update these as wrmsr might have changed them. */
1560	rc = VMXWriteCachedVMCS(VMX_VMCS64_GUEST_FS_BASE, pCtx->fsHid.u64Base);
1561	AssertRC(rc);
1562	rc = VMXWriteCachedVMCS(VMX_VMCS64_GUEST_GS_BASE, pCtx->gsHid.u64Base);
1563	AssertRC(rc);
1564	}
1565	else
1566	{
1567	pVCpu->hwaccm.s.vmx.pfnStartVM = VMXR0StartVM32;
1568	}
1569
1570	vmxR0UpdateExceptionBitmap(pVM, pVCpu, pCtx);
1571
1572	/* Done. */
1573	pVCpu->hwaccm.s.fContextUseFlags &= ~HWACCM_CHANGED_ALL_GUEST;
1574
1575	return rc;
1576	}
1577
1578	/**
1579	* Syncs back the guest state
1580	*
1581	* @returns VBox status code.
1582	* @param pVM The VM to operate on.
1583	* @param pVCpu The VMCPU to operate on.
1584	* @param pCtx Guest context
1585	*/
1586	DECLINLINE(int) VMXR0SaveGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
1587	{
1588	RTGCUINTREG val, valShadow;
1589	RTGCUINTPTR uInterruptState;
1590	int rc;
1591
1592	/* Let's first sync back eip, esp, and eflags. */
1593	rc = VMXReadCachedVMCS(VMX_VMCS64_GUEST_RIP, &val);
1594	AssertRC(rc);
1595	pCtx->rip = val;
1596	rc = VMXReadCachedVMCS(VMX_VMCS64_GUEST_RSP, &val);
1597	AssertRC(rc);
1598	pCtx->rsp = val;
1599	rc = VMXReadCachedVMCS(VMX_VMCS_GUEST_RFLAGS, &val);
1600	AssertRC(rc);
1601	pCtx->eflags.u32 = val;
1602
1603	/* Take care of instruction fusing (sti, mov ss) */
1604	rc \|= VMXReadCachedVMCS(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &val);
1605	uInterruptState = val;
1606	if (uInterruptState != 0)
1607	{
1608	Assert(uInterruptState <= 2); /* only sti & mov ss */
1609	Log(("uInterruptState %x eip=%RGv\n", uInterruptState, pCtx->rip));
1610	EMSetInhibitInterruptsPC(pVM, pCtx->rip);
1611	}
1612	else
1613	VM_FF_CLEAR(pVM, VM_FF_INHIBIT_INTERRUPTS);
1614
1615	/* Control registers. */
1616	VMXReadCachedVMCS(VMX_VMCS_CTRL_CR0_READ_SHADOW, &valShadow);
1617	VMXReadCachedVMCS(VMX_VMCS64_GUEST_CR0, &val);
1618	val = (valShadow & pVCpu->hwaccm.s.vmx.cr0_mask) \| (val & ~pVCpu->hwaccm.s.vmx.cr0_mask);
1619	CPUMSetGuestCR0(pVM, val);
1620
1621	VMXReadCachedVMCS(VMX_VMCS_CTRL_CR4_READ_SHADOW, &valShadow);
1622	VMXReadCachedVMCS(VMX_VMCS64_GUEST_CR4, &val);
1623	val = (valShadow & pVCpu->hwaccm.s.vmx.cr4_mask) \| (val & ~pVCpu->hwaccm.s.vmx.cr4_mask);
1624	CPUMSetGuestCR4(pVM, val);
1625
1626	/* Note: no reason to sync back the CRx registers. They can't be changed by the guest. */
1627	/* Note: only in the nested paging case can CR3 & CR4 be changed by the guest. */
1628	if ( pVM->hwaccm.s.fNestedPaging
1629	&& CPUMIsGuestInPagedProtectedModeEx(pCtx))
1630	{
1631	/* Can be updated behind our back in the nested paging case. */
1632	CPUMSetGuestCR2(pVM, ASMGetCR2());
1633
1634	VMXReadCachedVMCS(VMX_VMCS64_GUEST_CR3, &val);
1635
1636	if (val != pCtx->cr3)
1637	{
1638	CPUMSetGuestCR3(pVM, val);
1639	PGMUpdateCR3(pVM, val);
1640	}
1641	/* Prefetch the four PDPT entries in PAE mode. */
1642	vmxR0PrefetchPAEPdptrs(pVM, pVCpu, pCtx);
1643	}
1644
1645	/* Sync back DR7 here. */
1646	VMXReadCachedVMCS(VMX_VMCS64_GUEST_DR7, &val);
1647	pCtx->dr[7] = val;
1648
1649	/* Guest CPU context: ES, CS, SS, DS, FS, GS. */
1650	VMX_READ_SELREG(ES, es);
1651	VMX_READ_SELREG(SS, ss);
1652	VMX_READ_SELREG(CS, cs);
1653	VMX_READ_SELREG(DS, ds);
1654	VMX_READ_SELREG(FS, fs);
1655	VMX_READ_SELREG(GS, gs);
1656
1657	/*
1658	* System MSRs
1659	*/
1660	VMXReadCachedVMCS(VMX_VMCS32_GUEST_SYSENTER_CS, &val);
1661	pCtx->SysEnter.cs = val;
1662	VMXReadCachedVMCS(VMX_VMCS64_GUEST_SYSENTER_EIP, &val);
1663	pCtx->SysEnter.eip = val;
1664	VMXReadCachedVMCS(VMX_VMCS64_GUEST_SYSENTER_ESP, &val);
1665	pCtx->SysEnter.esp = val;
1666
1667	/* Misc. registers; must sync everything otherwise we can get out of sync when jumping to ring 3. */
1668	VMX_READ_SELREG(LDTR, ldtr);
1669
1670	VMXReadCachedVMCS(VMX_VMCS32_GUEST_GDTR_LIMIT, &val);
1671	pCtx->gdtr.cbGdt = val;
1672	VMXReadCachedVMCS(VMX_VMCS64_GUEST_GDTR_BASE, &val);
1673	pCtx->gdtr.pGdt = val;
1674
1675	VMXReadCachedVMCS(VMX_VMCS32_GUEST_IDTR_LIMIT, &val);
1676	pCtx->idtr.cbIdt = val;
1677	VMXReadCachedVMCS(VMX_VMCS64_GUEST_IDTR_BASE, &val);
1678	pCtx->idtr.pIdt = val;
1679
1680	#ifdef HWACCM_VMX_EMULATE_REALMODE
1681	/* Real mode emulation using v86 mode with CR4.VME (interrupt redirection using the int bitmap in the TSS) */
1682	if (CPUMIsGuestInRealModeEx(pCtx))
1683	{
1684	/* Hide our emulation flags */
1685	pCtx->eflags.Bits.u1VM = 0;
1686	pCtx->eflags.Bits.u2IOPL = pVCpu->hwaccm.s.vmx.RealMode.eflags.Bits.u2IOPL;
1687
1688	/* Force a TR resync every time in case we switch modes. */
1689	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_GUEST_TR;
1690	}
1691	else
1692	#endif /* HWACCM_VMX_EMULATE_REALMODE */
1693	{
1694	/* In real mode we have a fake TSS, so only sync it back when it's supposed to be valid. */
1695	VMX_READ_SELREG(TR, tr);
1696	}
1697	return VINF_SUCCESS;
1698	}
1699
1700	/**
1701	* Dummy placeholder
1702	*
1703	* @param pVM The VM to operate on.
1704	* @param pVCpu The VMCPU to operate on.
1705	*/
1706	static void vmxR0SetupTLBDummy(PVM pVM, PVMCPU pVCpu)
1707	{
1708	NOREF(pVM);
1709	NOREF(pVCpu);
1710	return;
1711	}
1712
1713	/**
1714	* Setup the tagged TLB for EPT
1715	*
1716	* @returns VBox status code.
1717	* @param pVM The VM to operate on.
1718	* @param pVCpu The VMCPU to operate on.
1719	*/
1720	static void vmxR0SetupTLBEPT(PVM pVM, PVMCPU pVCpu)
1721	{
1722	PHWACCM_CPUINFO pCpu;
1723
1724	Assert(pVM->hwaccm.s.fNestedPaging);
1725	Assert(!pVM->hwaccm.s.vmx.fVPID);
1726
1727	/* Deal with tagged TLBs if VPID or EPT is supported. */
1728	pCpu = HWACCMR0GetCurrentCpu();
1729	/* Force a TLB flush for the first world switch if the current cpu differs from the one we ran on last. */
1730	/* Note that this can happen both for start and resume due to long jumps back to ring 3. */
1731	if ( pVCpu->hwaccm.s.idLastCpu != pCpu->idCpu
1732	/* if the tlb flush count has changed, another VM has flushed the TLB of this cpu, so we can't use our current ASID anymore. */
1733	\|\| pVCpu->hwaccm.s.cTLBFlushes != pCpu->cTLBFlushes)
1734	{
1735	/* Force a TLB flush on VM entry. */
1736	pVCpu->hwaccm.s.fForceTLBFlush = true;
1737	}
1738	else
1739	Assert(!pCpu->fFlushTLB);
1740
1741	pVCpu->hwaccm.s.idLastCpu = pCpu->idCpu;
1742	pCpu->fFlushTLB = false;
1743
1744	if (pVCpu->hwaccm.s.fForceTLBFlush)
1745	vmxR0FlushEPT(pVM, pVCpu, pVM->hwaccm.s.vmx.enmFlushContext, 0);
1746
1747	#ifdef VBOX_WITH_STATISTICS
1748	if (pVCpu->hwaccm.s.fForceTLBFlush)
1749	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatFlushTLBWorldSwitch);
1750	else
1751	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatNoFlushTLBWorldSwitch);
1752	#endif
1753	}
1754
1755	#ifdef HWACCM_VTX_WITH_VPID
1756	/**
1757	* Setup the tagged TLB for VPID
1758	*
1759	* @returns VBox status code.
1760	* @param pVM The VM to operate on.
1761	* @param pVCpu The VMCPU to operate on.
1762	*/
1763	static void vmxR0SetupTLBVPID(PVM pVM, PVMCPU pVCpu)
1764	{
1765	PHWACCM_CPUINFO pCpu;
1766
1767	Assert(pVM->hwaccm.s.vmx.fVPID);
1768	Assert(!pVM->hwaccm.s.fNestedPaging);
1769
1770	/* Deal with tagged TLBs if VPID or EPT is supported. */
1771	pCpu = HWACCMR0GetCurrentCpu();
1772	/* Force a TLB flush for the first world switch if the current cpu differs from the one we ran on last. */
1773	/* Note that this can happen both for start and resume due to long jumps back to ring 3. */
1774	if ( pVCpu->hwaccm.s.idLastCpu != pCpu->idCpu
1775	/* if the tlb flush count has changed, another VM has flushed the TLB of this cpu, so we can't use our current ASID anymore. */
1776	\|\| pVCpu->hwaccm.s.cTLBFlushes != pCpu->cTLBFlushes)
1777	{
1778	/* Force a TLB flush on VM entry. */
1779	pVCpu->hwaccm.s.fForceTLBFlush = true;
1780	}
1781	else
1782	Assert(!pCpu->fFlushTLB);
1783
1784	pVCpu->hwaccm.s.idLastCpu = pCpu->idCpu;
1785
1786	/* Make sure we flush the TLB when required. Switch ASID to achieve the same thing, but without actually flushing the whole TLB (which is expensive). */
1787	if (pVCpu->hwaccm.s.fForceTLBFlush)
1788	{
1789	if ( ++pCpu->uCurrentASID >= pVM->hwaccm.s.uMaxASID
1790	\|\| pCpu->fFlushTLB)
1791	{
1792	pCpu->fFlushTLB = false;
1793	pCpu->uCurrentASID = 1; /* start at 1; host uses 0 */
1794	pCpu->cTLBFlushes++;
1795	}
1796	else
1797	{
1798	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatFlushASID);
1799	pVCpu->hwaccm.s.fForceTLBFlush = false;
1800	}
1801
1802	pVCpu->hwaccm.s.cTLBFlushes = pCpu->cTLBFlushes;
1803	pVCpu->hwaccm.s.uCurrentASID = pCpu->uCurrentASID;
1804	}
1805	else
1806	{
1807	Assert(!pCpu->fFlushTLB);
1808
1809	if (!pCpu->uCurrentASID \|\| !pVCpu->hwaccm.s.uCurrentASID)
1810	pVCpu->hwaccm.s.uCurrentASID = pCpu->uCurrentASID = 1;
1811	}
1812	AssertMsg(pVCpu->hwaccm.s.cTLBFlushes == pCpu->cTLBFlushes, ("Flush count mismatch for cpu %d (%x vs %x)\n", pCpu->idCpu, pVCpu->hwaccm.s.cTLBFlushes, pCpu->cTLBFlushes));
1813	AssertMsg(pCpu->uCurrentASID >= 1 && pCpu->uCurrentASID < pVM->hwaccm.s.uMaxASID, ("cpu%d uCurrentASID = %x\n", pCpu->idCpu, pCpu->uCurrentASID));
1814	AssertMsg(pVCpu->hwaccm.s.uCurrentASID >= 1 && pVCpu->hwaccm.s.uCurrentASID < pVM->hwaccm.s.uMaxASID, ("cpu%d VM uCurrentASID = %x\n", pCpu->idCpu, pVCpu->hwaccm.s.uCurrentASID));
1815
1816	int rc = VMXWriteCachedVMCS(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hwaccm.s.uCurrentASID);
1817	AssertRC(rc);
1818
1819	if (pVCpu->hwaccm.s.fForceTLBFlush)
1820	vmxR0FlushVPID(pVM, pVCpu, pVM->hwaccm.s.vmx.enmFlushContext, 0);
1821
1822	#ifdef VBOX_WITH_STATISTICS
1823	if (pVCpu->hwaccm.s.fForceTLBFlush)
1824	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatFlushTLBWorldSwitch);
1825	else
1826	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatNoFlushTLBWorldSwitch);
1827	#endif
1828	}
1829	#endif /* HWACCM_VTX_WITH_VPID */
1830
1831	/**
1832	* Runs guest code in a VT-x VM.
1833	*
1834	* @returns VBox status code.
1835	* @param pVM The VM to operate on.
1836	* @param pVCpu The VMCPU to operate on.
1837	* @param pCtx Guest context
1838	*/
1839	VMMR0DECL(int) VMXR0RunGuestCode(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
1840	{
1841	int rc = VINF_SUCCESS;
1842	RTGCUINTREG val;
1843	RTGCUINTREG exitReason, instrError, cbInstr;
1844	RTGCUINTPTR exitQualification;
1845	RTGCUINTPTR intInfo = 0; /* shut up buggy gcc 4 */
1846	RTGCUINTPTR errCode, instrInfo;
1847	bool fSyncTPR = false;
1848	PHWACCM_CPUINFO pCpu = 0;
1849	unsigned cResume = 0;
1850	#ifdef VBOX_STRICT
1851	RTCPUID idCpuCheck;
1852	#endif
1853	#ifdef VBOX_WITH_STATISTICS
1854	bool fStatEntryStarted = true;
1855	bool fStatExit2Started = false;
1856	#endif
1857
1858	Log2(("\nE"));
1859
1860	STAM_PROFILE_ADV_START(&pVCpu->hwaccm.s.StatEntry, x);
1861
1862	#ifdef VBOX_STRICT
1863	{
1864	RTCCUINTREG val;
1865	VMXFlushWriteCache(pVCpu);
1866
1867	rc = VMXReadVMCS(VMX_VMCS_CTRL_PIN_EXEC_CONTROLS, &val);
1868	AssertRC(rc);
1869	Log2(("VMX_VMCS_CTRL_PIN_EXEC_CONTROLS = %08x\n", val));
1870
1871	/* allowed zero */
1872	if ((val & pVM->hwaccm.s.vmx.msr.vmx_pin_ctls.n.disallowed0) != pVM->hwaccm.s.vmx.msr.vmx_pin_ctls.n.disallowed0)
1873	Log(("Invalid VMX_VMCS_CTRL_PIN_EXEC_CONTROLS: zero\n"));
1874
1875	/* allowed one */
1876	if ((val & ~pVM->hwaccm.s.vmx.msr.vmx_pin_ctls.n.allowed1) != 0)
1877	Log(("Invalid VMX_VMCS_CTRL_PIN_EXEC_CONTROLS: one\n"));
1878
1879	rc = VMXReadVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, &val);
1880	AssertRC(rc);
1881	Log2(("VMX_VMCS_CTRL_PROC_EXEC_CONTROLS = %08x\n", val));
1882
1883	/* Must be set according to the MSR, but can be cleared in case of EPT. */
1884	if (pVM->hwaccm.s.fNestedPaging)
1885	val \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INVLPG_EXIT
1886	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
1887	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT;
1888
1889	/* allowed zero */
1890	if ((val & pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.disallowed0) != pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.disallowed0)
1891	Log(("Invalid VMX_VMCS_CTRL_PROC_EXEC_CONTROLS: zero\n"));
1892
1893	/* allowed one */
1894	if ((val & ~pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1) != 0)
1895	Log(("Invalid VMX_VMCS_CTRL_PROC_EXEC_CONTROLS: one\n"));
1896
1897	rc = VMXReadVMCS(VMX_VMCS_CTRL_ENTRY_CONTROLS, &val);
1898	AssertRC(rc);
1899	Log2(("VMX_VMCS_CTRL_ENTRY_CONTROLS = %08x\n", val));
1900
1901	/* allowed zero */
1902	if ((val & pVM->hwaccm.s.vmx.msr.vmx_entry.n.disallowed0) != pVM->hwaccm.s.vmx.msr.vmx_entry.n.disallowed0)
1903	Log(("Invalid VMX_VMCS_CTRL_ENTRY_CONTROLS: zero\n"));
1904
1905	/* allowed one */
1906	if ((val & ~pVM->hwaccm.s.vmx.msr.vmx_entry.n.allowed1) != 0)
1907	Log(("Invalid VMX_VMCS_CTRL_ENTRY_CONTROLS: one\n"));
1908
1909	rc = VMXReadVMCS(VMX_VMCS_CTRL_EXIT_CONTROLS, &val);
1910	AssertRC(rc);
1911	Log2(("VMX_VMCS_CTRL_EXIT_CONTROLS = %08x\n", val));
1912
1913	/* allowed zero */
1914	if ((val & pVM->hwaccm.s.vmx.msr.vmx_exit.n.disallowed0) != pVM->hwaccm.s.vmx.msr.vmx_exit.n.disallowed0)
1915	Log(("Invalid VMX_VMCS_CTRL_EXIT_CONTROLS: zero\n"));
1916
1917	/* allowed one */
1918	if ((val & ~pVM->hwaccm.s.vmx.msr.vmx_exit.n.allowed1) != 0)
1919	Log(("Invalid VMX_VMCS_CTRL_EXIT_CONTROLS: one\n"));
1920	}
1921	#endif
1922
1923	/* We can jump to this point to resume execution after determining that a VM-exit is innocent.
1924	*/
1925	ResumeExecution:
1926	STAM_STATS({
1927	if (fStatExit2Started) { STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2, y); fStatExit2Started = false; }
1928	if (!fStatEntryStarted) { STAM_PROFILE_ADV_START(&pVCpu->hwaccm.s.StatEntry, x); fStatEntryStarted = true; }
1929	});
1930	AssertMsg(pVCpu->hwaccm.s.idEnteredCpu == RTMpCpuId(),
1931	("Expected %d, I'm %d; cResume=%d exitReason=%RGv exitQualification=%RGv\n",
1932	(int)pVCpu->hwaccm.s.idEnteredCpu, (int)RTMpCpuId(), cResume, exitReason, exitQualification));
1933	Assert(!HWACCMR0SuspendPending());
1934
1935	/* Safety precaution; looping for too long here can have a very bad effect on the host */
1936	if (++cResume > HWACCM_MAX_RESUME_LOOPS)
1937	{
1938	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitMaxResume);
1939	rc = VINF_EM_RAW_INTERRUPT;
1940	goto end;
1941	}
1942
1943	/* Check for irq inhibition due to instruction fusing (sti, mov ss). */
1944	if (VM_FF_ISSET(pVM, VM_FF_INHIBIT_INTERRUPTS))
1945	{
1946	Log(("VM_FF_INHIBIT_INTERRUPTS at %RGv successor %RGv\n", (RTGCPTR)pCtx->rip, EMGetInhibitInterruptsPC(pVM)));
1947	if (pCtx->rip != EMGetInhibitInterruptsPC(pVM))
1948	{
1949	/* Note: we intentionally don't clear VM_FF_INHIBIT_INTERRUPTS here.
1950	* Before we are able to execute this instruction in raw mode (iret to guest code) an external interrupt might
1951	* force a world switch again. Possibly allowing a guest interrupt to be dispatched in the process. This could
1952	* break the guest. Sounds very unlikely, but such timing sensitive problems are not as rare as you might think.
1953	*/
1954	VM_FF_CLEAR(pVM, VM_FF_INHIBIT_INTERRUPTS);
1955	/* Irq inhibition is no longer active; clear the corresponding VMX state. */
1956	rc = VMXWriteCachedVMCS(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, 0);
1957	AssertRC(rc);
1958	}
1959	}
1960	else
1961	{
1962	/* Irq inhibition is no longer active; clear the corresponding VMX state. */
1963	rc = VMXWriteCachedVMCS(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, 0);
1964	AssertRC(rc);
1965	}
1966
1967	/* Check for pending actions that force us to go back to ring 3. */
1968	if (VM_FF_ISPENDING(pVM, VM_FF_TO_R3 \| VM_FF_TIMER))
1969	{
1970	VM_FF_CLEAR(pVM, VM_FF_TO_R3);
1971	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatSwitchToR3);
1972	rc = VINF_EM_RAW_TO_R3;
1973	goto end;
1974	}
1975	/* Pending request packets might contain actions that need immediate attention, such as pending hardware interrupts. */
1976	if (VM_FF_ISPENDING(pVM, VM_FF_REQUEST))
1977	{
1978	rc = VINF_EM_PENDING_REQUEST;
1979	goto end;
1980	}
1981
1982	/* When external interrupts are pending, we should exit the VM when IF is set. */
1983	/* Note! After VM_FF_INHIBIT_INTERRUPTS check!!! */
1984	rc = VMXR0CheckPendingInterrupt(pVM, pVCpu, pCtx);
1985	if (RT_FAILURE(rc))
1986	goto end;
1987
1988	/** @todo check timers?? */
1989
1990	/* TPR caching using CR8 is only available in 64 bits mode */
1991	/* Note the 32 bits exception for AMD (X86_CPUID_AMD_FEATURE_ECX_CR8L), but that appears missing in Intel CPUs */
1992	/* Note: we can't do this in LoadGuestState as PDMApicGetTPR can jump back to ring 3 (lock)!!!!! */
1993	/**
1994	* @todo reduce overhead
1995	*/
1996	if ( (pCtx->msrEFER & MSR_K6_EFER_LMA)
1997	&& pVM->hwaccm.s.vmx.pAPIC)
1998	{
1999	/* TPR caching in CR8 */
2000	uint8_t u8TPR;
2001	bool fPending;
2002
2003	int rc = PDMApicGetTPR(pVM, &u8TPR, &fPending);
2004	AssertRC(rc);
2005	/* The TPR can be found at offset 0x80 in the APIC mmio page. */
2006	pVM->hwaccm.s.vmx.pAPIC[0x80] = u8TPR << 4; /* bits 7-4 contain the task priority */
2007
2008	/* Two options here:
2009	* - external interrupt pending, but masked by the TPR value.
2010	* -> a CR8 update that lower the current TPR value should cause an exit
2011	* - no pending interrupts
2012	* -> We don't need to be explicitely notified. There are enough world switches for detecting pending interrupts.
2013	*/
2014	rc = VMXWriteCachedVMCS(VMX_VMCS_CTRL_TPR_THRESHOLD, (fPending) ? u8TPR : 0);
2015	AssertRC(rc);
2016
2017	/* Always sync back the TPR; we should optimize this though / /* @todo optimize TPR sync. */
2018	fSyncTPR = true;
2019	}
2020
2021	#if defined(HWACCM_VTX_WITH_EPT) && defined(LOG_ENABLED)
2022	if ( pVM->hwaccm.s.fNestedPaging
2023	# ifdef HWACCM_VTX_WITH_VPID
2024	\|\| pVM->hwaccm.s.vmx.fVPID
2025	# endif /* HWACCM_VTX_WITH_VPID */
2026	)
2027	{
2028	pCpu = HWACCMR0GetCurrentCpu();
2029	if ( pVCpu->hwaccm.s.idLastCpu != pCpu->idCpu
2030	\|\| pVCpu->hwaccm.s.cTLBFlushes != pCpu->cTLBFlushes)
2031	{
2032	if (pVCpu->hwaccm.s.idLastCpu != pCpu->idCpu)
2033	Log(("Force TLB flush due to rescheduling to a different cpu (%d vs %d)\n", pVCpu->hwaccm.s.idLastCpu, pCpu->idCpu));
2034	else
2035	Log(("Force TLB flush due to changed TLB flush count (%x vs %x)\n", pVCpu->hwaccm.s.cTLBFlushes, pCpu->cTLBFlushes));
2036	}
2037	if (pCpu->fFlushTLB)
2038	Log(("Force TLB flush: first time cpu %d is used -> flush\n", pCpu->idCpu));
2039	else
2040	if (pVCpu->hwaccm.s.fForceTLBFlush)
2041	LogFlow(("Manual TLB flush\n"));
2042	}
2043	#endif
2044	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
2045	PGMDynMapFlushAutoSet(pVCpu);
2046	#endif
2047
2048	/*
2049	* NOTE: DO NOT DO ANYTHING AFTER THIS POINT THAT MIGHT JUMP BACK TO RING 3!
2050	* (until the actual world switch)
2051	*/
2052	#ifdef VBOX_STRICT
2053	idCpuCheck = RTMpCpuId();
2054	#endif
2055	#ifdef LOG_LOGGING
2056	VMMR0LogFlushDisable(pVCpu);
2057	#endif
2058	/* Save the host state first. */
2059	rc = VMXR0SaveHostState(pVM, pVCpu);
2060	if (rc != VINF_SUCCESS)
2061	goto end;
2062	/* Load the guest state */
2063	rc = VMXR0LoadGuestState(pVM, pVCpu, pCtx);
2064	if (rc != VINF_SUCCESS)
2065	goto end;
2066
2067	/* Deal with tagged TLB setup and invalidation. */
2068	pVM->hwaccm.s.vmx.pfnSetupTaggedTLB(pVM, pVCpu);
2069
2070	/* Non-register state Guest Context */
2071	/** @todo change me according to cpu state */
2072	rc = VMXWriteCachedVMCS(VMX_VMCS32_GUEST_ACTIVITY_STATE, VMX_CMS_GUEST_ACTIVITY_ACTIVE);
2073	AssertRC(rc);
2074
2075	STAM_STATS({ STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatEntry, x); fStatEntryStarted = false; });
2076
2077	/* Manual save and restore:
2078	* - General purpose registers except RIP, RSP
2079	*
2080	* Trashed:
2081	* - CR2 (we don't care)
2082	* - LDTR (reset to 0)
2083	* - DRx (presumably not changed at all)
2084	* - DR7 (reset to 0x400)
2085	* - EFLAGS (reset to RT_BIT(1); not relevant)
2086	*
2087	*/
2088
2089	/* All done! Let's start VM execution. */
2090	STAM_PROFILE_ADV_START(&pVCpu->hwaccm.s.StatInGC, z);
2091	#ifdef VBOX_STRICT
2092	Assert(idCpuCheck == RTMpCpuId());
2093	#endif
2094	TMNotifyStartOfExecution(pVM);
2095	rc = pVCpu->hwaccm.s.vmx.pfnStartVM(pVCpu->hwaccm.s.fResumeVM, pCtx, &pVCpu->hwaccm.s.vmx.VMCSCache, pVM, pVCpu);
2096	TMNotifyEndOfExecution(pVM);
2097
2098	AssertMsg(!pVCpu->hwaccm.s.vmx.VMCSCache.Write.cValidEntries, ("pVCpu->hwaccm.s.vmx.VMCSCache.Write.cValidEntries=%d\n", pVCpu->hwaccm.s.vmx.VMCSCache.Write.cValidEntries));
2099
2100	/* In case we execute a goto ResumeExecution later on. */
2101	pVCpu->hwaccm.s.fResumeVM = true;
2102	pVCpu->hwaccm.s.fForceTLBFlush = false;
2103
2104	/*
2105	* !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
2106	* IMPORTANT: WE CAN'T DO ANY LOGGING OR OPERATIONS THAT CAN DO A LONGJMP BACK TO RING 3 BEFORE WE'VE SYNCED BACK (MOST OF) THE GUEST STATE
2107	* !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
2108	*/
2109
2110	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatInGC, z);
2111	STAM_PROFILE_ADV_START(&pVCpu->hwaccm.s.StatExit1, v);
2112
2113	if (rc != VINF_SUCCESS)
2114	{
2115	VMXR0ReportWorldSwitchError(pVM, pVCpu, rc, pCtx);
2116	goto end;
2117	}
2118	/* Success. Query the guest state and figure out what has happened. */
2119
2120	/* Investigate why there was a VM-exit. */
2121	rc = VMXReadCachedVMCS(VMX_VMCS32_RO_EXIT_REASON, &exitReason);
2122	STAM_COUNTER_INC(&pVCpu->hwaccm.s.paStatExitReasonR0[exitReason & MASK_EXITREASON_STAT]);
2123
2124	exitReason &= 0xffff; /* bit 0-15 contain the exit code. */
2125	rc \|= VMXReadCachedVMCS(VMX_VMCS32_RO_VM_INSTR_ERROR, &instrError);
2126	rc \|= VMXReadCachedVMCS(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &cbInstr);
2127	rc \|= VMXReadCachedVMCS(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &intInfo);
2128	/* might not be valid; depends on VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID. */
2129	rc \|= VMXReadCachedVMCS(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERRCODE, &errCode);
2130	rc \|= VMXReadCachedVMCS(VMX_VMCS32_RO_EXIT_INSTR_INFO, &instrInfo);
2131	rc \|= VMXReadCachedVMCS(VMX_VMCS_RO_EXIT_QUALIFICATION, &exitQualification);
2132	AssertRC(rc);
2133
2134	/* Sync back the guest state */
2135	rc = VMXR0SaveGuestState(pVM, pVCpu, pCtx);
2136	AssertRC(rc);
2137
2138	/* Note! NOW IT'S SAFE FOR LOGGING! */
2139	#ifdef LOG_LOGGING
2140	VMMR0LogFlushEnable(pVCpu);
2141	#endif
2142	Log2(("Raw exit reason %08x\n", exitReason));
2143
2144	/* Check if an injected event was interrupted prematurely. */
2145	rc = VMXReadCachedVMCS(VMX_VMCS32_RO_IDT_INFO, &val);
2146	AssertRC(rc);
2147	pVCpu->hwaccm.s.Event.intInfo = VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(val);
2148	if ( VMX_EXIT_INTERRUPTION_INFO_VALID(pVCpu->hwaccm.s.Event.intInfo)
2149	/* Ignore 'int xx' as they'll be restarted anyway. */
2150	&& VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hwaccm.s.Event.intInfo) != VMX_EXIT_INTERRUPTION_INFO_TYPE_SW
2151	/* Ignore software exceptions (such as int3) as they're reoccur when we restart the instruction anyway. */
2152	&& VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hwaccm.s.Event.intInfo) != VMX_EXIT_INTERRUPTION_INFO_TYPE_SWEXCPT)
2153	{
2154	pVCpu->hwaccm.s.Event.fPending = true;
2155	/* Error code present? */
2156	if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(pVCpu->hwaccm.s.Event.intInfo))
2157	{
2158	rc = VMXReadCachedVMCS(VMX_VMCS32_RO_IDT_ERRCODE, &val);
2159	AssertRC(rc);
2160	pVCpu->hwaccm.s.Event.errCode = val;
2161	Log(("Pending inject %RX64 at %RGv exit=%08x intInfo=%08x exitQualification=%RGv pending error=%RX64\n", pVCpu->hwaccm.s.Event.intInfo, (RTGCPTR)pCtx->rip, exitReason, intInfo, exitQualification, val));
2162	}
2163	else
2164	{
2165	Log(("Pending inject %RX64 at %RGv exit=%08x intInfo=%08x exitQualification=%RGv\n", pVCpu->hwaccm.s.Event.intInfo, (RTGCPTR)pCtx->rip, exitReason, intInfo, exitQualification));
2166	pVCpu->hwaccm.s.Event.errCode = 0;
2167	}
2168	}
2169	#ifdef VBOX_STRICT
2170	else
2171	if ( VMX_EXIT_INTERRUPTION_INFO_VALID(pVCpu->hwaccm.s.Event.intInfo)
2172	/* Ignore software exceptions (such as int3) as they're reoccur when we restart the instruction anyway. */
2173	&& VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hwaccm.s.Event.intInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_SWEXCPT)
2174	{
2175	Log(("Ignore pending inject %RX64 at %RGv exit=%08x intInfo=%08x exitQualification=%RGv\n", pVCpu->hwaccm.s.Event.intInfo, (RTGCPTR)pCtx->rip, exitReason, intInfo, exitQualification));
2176	}
2177
2178	if (exitReason == VMX_EXIT_ERR_INVALID_GUEST_STATE)
2179	HWACCMDumpRegs(pVM, pCtx);
2180	#endif
2181
2182	Log2(("E%d", exitReason));
2183	Log2(("Exit reason %d, exitQualification %RGv\n", (uint32_t)exitReason, exitQualification));
2184	Log2(("instrInfo=%d instrError=%d instr length=%d\n", (uint32_t)instrInfo, (uint32_t)instrError, (uint32_t)cbInstr));
2185	Log2(("Interruption error code %d\n", (uint32_t)errCode));
2186	Log2(("IntInfo = %08x\n", (uint32_t)intInfo));
2187	Log2(("New EIP=%RGv\n", (RTGCPTR)pCtx->rip));
2188
2189	if (fSyncTPR)
2190	{
2191	rc = PDMApicSetTPR(pVM, pVM->hwaccm.s.vmx.pAPIC[0x80] >> 4);
2192	AssertRC(rc);
2193	}
2194
2195	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit1, v);
2196	STAM_STATS({ STAM_PROFILE_ADV_START(&pVCpu->hwaccm.s.StatExit2, y); fStatExit2Started = true; });
2197
2198	/* Some cases don't need a complete resync of the guest CPU state; handle them here. */
2199	switch (exitReason)
2200	{
2201	case VMX_EXIT_EXCEPTION: /* 0 Exception or non-maskable interrupt (NMI). */
2202	case VMX_EXIT_EXTERNAL_IRQ: /* 1 External interrupt. */
2203	{
2204	uint32_t vector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(intInfo);
2205
2206	if (!VMX_EXIT_INTERRUPTION_INFO_VALID(intInfo))
2207	{
2208	Assert(exitReason == VMX_EXIT_EXTERNAL_IRQ);
2209	/* External interrupt; leave to allow it to be dispatched again. */
2210	rc = VINF_EM_RAW_INTERRUPT;
2211	break;
2212	}
2213	STAM_PROFILE_ADV_START(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
2214	switch (VMX_EXIT_INTERRUPTION_INFO_TYPE(intInfo))
2215	{
2216	case VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI: /* Non-maskable interrupt. */
2217	/* External interrupt; leave to allow it to be dispatched again. */
2218	rc = VINF_EM_RAW_INTERRUPT;
2219	break;
2220
2221	case VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT: /* External hardware interrupt. */
2222	AssertFailed(); /* can't come here; fails the first check. */
2223	break;
2224
2225	case VMX_EXIT_INTERRUPTION_INFO_TYPE_DBEXCPT: /* Unknown why we get this type for #DB */
2226	case VMX_EXIT_INTERRUPTION_INFO_TYPE_SWEXCPT: /* Software exception. (#BP or #OF) */
2227	Assert(vector == 1 \|\| vector == 3 \|\| vector == 4);
2228	/* no break */
2229	case VMX_EXIT_INTERRUPTION_INFO_TYPE_HWEXCPT: /* Hardware exception. */
2230	Log2(("Hardware/software interrupt %d\n", vector));
2231	switch (vector)
2232	{
2233	case X86_XCPT_NM:
2234	{
2235	Log(("#NM fault at %RGv error code %x\n", (RTGCPTR)pCtx->rip, errCode));
2236
2237	/** @todo don't intercept #NM exceptions anymore when we've activated the guest FPU state. */
2238	/* If we sync the FPU/XMM state on-demand, then we can continue execution as if nothing has happened. */
2239	rc = CPUMR0LoadGuestFPU(pVM, pVCpu, pCtx);
2240	if (rc == VINF_SUCCESS)
2241	{
2242	Assert(CPUMIsGuestFPUStateActive(pVCpu));
2243
2244	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitShadowNM);
2245
2246	/* Continue execution. */
2247	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_GUEST_CR0;
2248
2249	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
2250	goto ResumeExecution;
2251	}
2252
2253	Log(("Forward #NM fault to the guest\n"));
2254	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestNM);
2255	rc = VMXR0InjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), cbInstr, 0);
2256	AssertRC(rc);
2257	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
2258	goto ResumeExecution;
2259	}
2260
2261	case X86_XCPT_PF: /* Page fault */
2262	{
2263	#ifdef DEBUG
2264	if (pVM->hwaccm.s.fNestedPaging)
2265	{ /* A genuine pagefault.
2266	* Forward the trap to the guest by injecting the exception and resuming execution.
2267	*/
2268	Log(("Guest page fault at %RGv cr2=%RGv error code %x rsp=%RGv\n", (RTGCPTR)pCtx->rip, exitQualification, errCode, (RTGCPTR)pCtx->rsp));
2269
2270	Assert(CPUMIsGuestInPagedProtectedModeEx(pCtx));
2271
2272	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestPF);
2273
2274	/* Now we must update CR2. */
2275	pCtx->cr2 = exitQualification;
2276	rc = VMXR0InjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), cbInstr, errCode);
2277	AssertRC(rc);
2278
2279	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
2280	goto ResumeExecution;
2281	}
2282	#endif
2283	Assert(!pVM->hwaccm.s.fNestedPaging);
2284
2285	Log2(("Page fault at %RGv error code %x\n", exitQualification, errCode));
2286	/* Exit qualification contains the linear address of the page fault. */
2287	TRPMAssertTrap(pVM, X86_XCPT_PF, TRPM_TRAP);
2288	TRPMSetErrorCode(pVM, errCode);
2289	TRPMSetFaultAddress(pVM, exitQualification);
2290
2291	/* Forward it to our trap handler first, in case our shadow pages are out of sync. */
2292	rc = PGMTrap0eHandler(pVM, errCode, CPUMCTX2CORE(pCtx), (RTGCPTR)exitQualification);
2293	Log2(("PGMTrap0eHandler %RGv returned %Rrc\n", (RTGCPTR)pCtx->rip, rc));
2294	if (rc == VINF_SUCCESS)
2295	{ /* We've successfully synced our shadow pages, so let's just continue execution. */
2296	Log2(("Shadow page fault at %RGv cr2=%RGv error code %x\n", (RTGCPTR)pCtx->rip, exitQualification ,errCode));
2297	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitShadowPF);
2298
2299	TRPMResetTrap(pVM);
2300
2301	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
2302	goto ResumeExecution;
2303	}
2304	else
2305	if (rc == VINF_EM_RAW_GUEST_TRAP)
2306	{ /* A genuine pagefault.
2307	* Forward the trap to the guest by injecting the exception and resuming execution.
2308	*/
2309	Log2(("Forward page fault to the guest\n"));
2310
2311	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestPF);
2312	/* The error code might have been changed. */
2313	errCode = TRPMGetErrorCode(pVM);
2314
2315	TRPMResetTrap(pVM);
2316
2317	/* Now we must update CR2. */
2318	pCtx->cr2 = exitQualification;
2319	rc = VMXR0InjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), cbInstr, errCode);
2320	AssertRC(rc);
2321
2322	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
2323	goto ResumeExecution;
2324	}
2325	#ifdef VBOX_STRICT
2326	if (rc != VINF_EM_RAW_EMULATE_INSTR)
2327	Log2(("PGMTrap0eHandler failed with %d\n", rc));
2328	#endif
2329	/* Need to go back to the recompiler to emulate the instruction. */
2330	TRPMResetTrap(pVM);
2331	break;
2332	}
2333
2334	case X86_XCPT_MF: /* Floating point exception. */
2335	{
2336	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestMF);
2337	if (!(pCtx->cr0 & X86_CR0_NE))
2338	{
2339	/* old style FPU error reporting needs some extra work. */
2340	/** @todo don't fall back to the recompiler, but do it manually. */
2341	rc = VINF_EM_RAW_EMULATE_INSTR;
2342	break;
2343	}
2344	Log(("Trap %x at %04X:%RGv\n", vector, pCtx->cs, (RTGCPTR)pCtx->rip));
2345	rc = VMXR0InjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), cbInstr, errCode);
2346	AssertRC(rc);
2347
2348	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
2349	goto ResumeExecution;
2350	}
2351
2352	case X86_XCPT_DB: /* Debug exception. */
2353	{
2354	uint64_t uDR6;
2355
2356	/* DR6, DR7.GD and IA32_DEBUGCTL.LBR are not updated yet.
2357	*
2358	* Exit qualification bits:
2359	* 3:0 B0-B3 which breakpoint condition was met
2360	* 12:4 Reserved (0)
2361	* 13 BD - debug register access detected
2362	* 14 BS - single step execution or branch taken
2363	* 63:15 Reserved (0)
2364	*/
2365	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestDB);
2366
2367	/* Note that we don't support guest and host-initiated debugging at the same time. */
2368	Assert(DBGFIsStepping(pVM) \|\| CPUMIsGuestInRealModeEx(pCtx));
2369
2370	uDR6 = X86_DR6_INIT_VAL;
2371	uDR6 \|= (exitQualification & (X86_DR6_B0\|X86_DR6_B1\|X86_DR6_B2\|X86_DR6_B3\|X86_DR6_BD\|X86_DR6_BS));
2372	rc = DBGFR0Trap01Handler(pVM, CPUMCTX2CORE(pCtx), uDR6);
2373	if (rc == VINF_EM_RAW_GUEST_TRAP)
2374	{
2375	/** @todo this isn't working, but we'll never get here normally. */
2376
2377	/* Update DR6 here. */
2378	pCtx->dr[6] = uDR6;
2379
2380	/* X86_DR7_GD will be cleared if drx accesses should be trapped inside the guest. */
2381	pCtx->dr[7] &= ~X86_DR7_GD;
2382
2383	/* Paranoia. */
2384	pCtx->dr[7] &= 0xffffffff; /* upper 32 bits reserved */
2385	pCtx->dr[7] &= ~(RT_BIT(11) \| RT_BIT(12) \| RT_BIT(14) \| RT_BIT(15)); /* must be zero */
2386	pCtx->dr[7] \|= 0x400; /* must be one */
2387
2388	/* Resync DR7 */
2389	rc = VMXWriteCachedVMCS(VMX_VMCS64_GUEST_DR7, pCtx->dr[7]);
2390	AssertRC(rc);
2391
2392	Log(("Trap %x (debug) at %RGv exit qualification %RX64\n", vector, (RTGCPTR)pCtx->rip, exitQualification));
2393	rc = VMXR0InjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), cbInstr, errCode);
2394	AssertRC(rc);
2395
2396	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
2397	goto ResumeExecution;
2398	}
2399	/* Return to ring 3 to deal with the debug exit code. */
2400	break;
2401	}
2402
2403	case X86_XCPT_GP: /* General protection failure exception.*/
2404	{
2405	uint32_t cbSize;
2406
2407	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestGP);
2408	#ifdef VBOX_STRICT
2409	if (!CPUMIsGuestInRealModeEx(pCtx))
2410	{
2411	Log(("Trap %x at %04X:%RGv errorCode=%x\n", vector, pCtx->cs, (RTGCPTR)pCtx->rip, errCode));
2412	rc = VMXR0InjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), cbInstr, errCode);
2413	AssertRC(rc);
2414	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
2415	goto ResumeExecution;
2416	}
2417	#endif
2418	Assert(CPUMIsGuestInRealModeEx(pCtx));
2419
2420	LogFlow(("Real mode X86_XCPT_GP instruction emulation at %RGv\n", (RTGCPTR)pCtx->rip));
2421	rc = EMInterpretInstruction(pVM, CPUMCTX2CORE(pCtx), 0, &cbSize);
2422	if (rc == VINF_SUCCESS)
2423	{
2424	/* EIP has been updated already. */
2425
2426	/* lidt, lgdt can end up here. In the future crx changes as well. Just reload the whole context to be done with it. */
2427	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_ALL;
2428
2429	/* Only resume if successful. */
2430	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
2431	goto ResumeExecution;
2432	}
2433	AssertMsg(rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_EM_HALT, ("Unexpected rc=%Rrc\n", rc));
2434	break;
2435	}
2436
2437	#ifdef VBOX_STRICT
2438	case X86_XCPT_DE: /* Divide error. */
2439	case X86_XCPT_UD: /* Unknown opcode exception. */
2440	case X86_XCPT_SS: /* Stack segment exception. */
2441	case X86_XCPT_NP: /* Segment not present exception. */
2442	{
2443	switch(vector)
2444	{
2445	case X86_XCPT_DE:
2446	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestDE);
2447	break;
2448	case X86_XCPT_UD:
2449	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestUD);
2450	break;
2451	case X86_XCPT_SS:
2452	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestSS);
2453	break;
2454	case X86_XCPT_NP:
2455	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestNP);
2456	break;
2457	}
2458
2459	Log(("Trap %x at %04X:%RGv\n", vector, pCtx->cs, (RTGCPTR)pCtx->rip));
2460	rc = VMXR0InjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), cbInstr, errCode);
2461	AssertRC(rc);
2462
2463	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
2464	goto ResumeExecution;
2465	}
2466	#endif
2467	default:
2468	#ifdef HWACCM_VMX_EMULATE_REALMODE
2469	if (CPUMIsGuestInRealModeEx(pCtx))
2470	{
2471	Log(("Real Mode Trap %x at %04x:%04X error code %x\n", vector, pCtx->cs, pCtx->eip, errCode));
2472	rc = VMXR0InjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), cbInstr, errCode);
2473	AssertRC(rc);
2474
2475	/* Go back to ring 3 in case of a triple fault. */
2476	if ( vector == X86_XCPT_DF
2477	&& rc == VINF_EM_RESET)
2478	break;
2479
2480	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
2481	goto ResumeExecution;
2482	}
2483	#endif
2484	AssertMsgFailed(("Unexpected vm-exit caused by exception %x\n", vector));
2485	rc = VERR_VMX_UNEXPECTED_EXCEPTION;
2486	break;
2487	} /* switch (vector) */
2488
2489	break;
2490
2491	default:
2492	rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_CODE;
2493	AssertMsgFailed(("Unexpected interuption code %x\n", intInfo));
2494	break;
2495	}
2496
2497	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
2498	break;
2499	}
2500
2501	case VMX_EXIT_EPT_VIOLATION: /* 48 EPT violation. An attempt to access memory with a guest-physical address was disallowed by the configuration of the EPT paging structures. */
2502	{
2503	RTGCPHYS GCPhys;
2504
2505	Assert(pVM->hwaccm.s.fNestedPaging);
2506
2507	rc = VMXReadVMCS64(VMX_VMCS_EXIT_PHYS_ADDR_FULL, &GCPhys);
2508	AssertRC(rc);
2509	Assert(((exitQualification >> 7) & 3) != 2);
2510
2511	/* Determine the kind of violation. */
2512	errCode = 0;
2513	if (exitQualification & VMX_EXIT_QUALIFICATION_EPT_INSTR_FETCH)
2514	errCode \|= X86_TRAP_PF_ID;
2515
2516	if (exitQualification & VMX_EXIT_QUALIFICATION_EPT_DATA_WRITE)
2517	errCode \|= X86_TRAP_PF_RW;
2518
2519	/* If the page is present, then it's a page level protection fault. */
2520	if (exitQualification & VMX_EXIT_QUALIFICATION_EPT_ENTRY_PRESENT)
2521	errCode \|= X86_TRAP_PF_P;
2522
2523	Log(("EPT Page fault %x at %RGp error code %x\n", (uint32_t)exitQualification, GCPhys, errCode));
2524
2525	/* GCPhys contains the guest physical address of the page fault. */
2526	TRPMAssertTrap(pVM, X86_XCPT_PF, TRPM_TRAP);
2527	TRPMSetErrorCode(pVM, errCode);
2528	TRPMSetFaultAddress(pVM, GCPhys);
2529
2530	/* Handle the pagefault trap for the nested shadow table. */
2531	rc = PGMR0Trap0eHandlerNestedPaging(pVM, PGMMODE_EPT, errCode, CPUMCTX2CORE(pCtx), GCPhys);
2532	Log2(("PGMR0Trap0eHandlerNestedPaging %RGv returned %Rrc\n", (RTGCPTR)pCtx->rip, rc));
2533	if (rc == VINF_SUCCESS)
2534	{ /* We've successfully synced our shadow pages, so let's just continue execution. */
2535	Log2(("Shadow page fault at %RGv cr2=%RGp error code %x\n", (RTGCPTR)pCtx->rip, exitQualification , errCode));
2536	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitReasonNPF);
2537
2538	TRPMResetTrap(pVM);
2539
2540	goto ResumeExecution;
2541	}
2542
2543	#ifdef VBOX_STRICT
2544	if (rc != VINF_EM_RAW_EMULATE_INSTR)
2545	LogFlow(("PGMTrap0eHandlerNestedPaging failed with %d\n", rc));
2546	#endif
2547	/* Need to go back to the recompiler to emulate the instruction. */
2548	TRPMResetTrap(pVM);
2549	break;
2550	}
2551
2552	case VMX_EXIT_EPT_MISCONFIG:
2553	{
2554	RTGCPHYS GCPhys;
2555
2556	Assert(pVM->hwaccm.s.fNestedPaging);
2557
2558	rc = VMXReadVMCS64(VMX_VMCS_EXIT_PHYS_ADDR_FULL, &GCPhys);
2559	AssertRC(rc);
2560
2561	Log(("VMX_EXIT_EPT_MISCONFIG for %VGp\n", GCPhys));
2562	break;
2563	}
2564
2565	case VMX_EXIT_IRQ_WINDOW: /* 7 Interrupt window. */
2566	/* Clear VM-exit on IF=1 change. */
2567	LogFlow(("VMX_EXIT_IRQ_WINDOW %RGv pending=%d IF=%d\n", (RTGCPTR)pCtx->rip, VM_FF_ISPENDING(pVM, (VM_FF_INTERRUPT_APIC\|VM_FF_INTERRUPT_PIC)), pCtx->eflags.Bits.u1IF));
2568	pVCpu->hwaccm.s.vmx.proc_ctls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_IRQ_WINDOW_EXIT;
2569	rc = VMXWriteCachedVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
2570	AssertRC(rc);
2571	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitIrqWindow);
2572	goto ResumeExecution; /* we check for pending guest interrupts there */
2573
2574	case VMX_EXIT_WBINVD: /* 54 Guest software attempted to execute WBINVD. (conditional) */
2575	case VMX_EXIT_INVD: /* 13 Guest software attempted to execute INVD. (unconditional) */
2576	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitInvd);
2577	/* Skip instruction and continue directly. */
2578	pCtx->rip += cbInstr;
2579	/* Continue execution.*/
2580	goto ResumeExecution;
2581
2582	case VMX_EXIT_CPUID: /* 10 Guest software attempted to execute CPUID. */
2583	{
2584	Log2(("VMX: Cpuid %x\n", pCtx->eax));
2585	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitCpuid);
2586	rc = EMInterpretCpuId(pVM, CPUMCTX2CORE(pCtx));
2587	if (rc == VINF_SUCCESS)
2588	{
2589	/* Update EIP and continue execution. */
2590	Assert(cbInstr == 2);
2591	pCtx->rip += cbInstr;
2592	goto ResumeExecution;
2593	}
2594	AssertMsgFailed(("EMU: cpuid failed with %Rrc\n", rc));
2595	rc = VINF_EM_RAW_EMULATE_INSTR;
2596	break;
2597	}
2598
2599	case VMX_EXIT_RDTSC: /* 16 Guest software attempted to execute RDTSC. */
2600	{
2601	Log2(("VMX: Rdtsc\n"));
2602	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitRdtsc);
2603	rc = EMInterpretRdtsc(pVM, CPUMCTX2CORE(pCtx));
2604	if (rc == VINF_SUCCESS)
2605	{
2606	/* Update EIP and continue execution. */
2607	Assert(cbInstr == 2);
2608	pCtx->rip += cbInstr;
2609	goto ResumeExecution;
2610	}
2611	AssertMsgFailed(("EMU: rdtsc failed with %Rrc\n", rc));
2612	rc = VINF_EM_RAW_EMULATE_INSTR;
2613	break;
2614	}
2615
2616	case VMX_EXIT_INVPG: /* 14 Guest software attempted to execute INVPG. */
2617	{
2618	Log2(("VMX: invlpg\n"));
2619	Assert(!pVM->hwaccm.s.fNestedPaging);
2620
2621	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitInvpg);
2622	rc = EMInterpretInvlpg(pVM, CPUMCTX2CORE(pCtx), exitQualification);
2623	if (rc == VINF_SUCCESS)
2624	{
2625	/* Update EIP and continue execution. */
2626	pCtx->rip += cbInstr;
2627	goto ResumeExecution;
2628	}
2629	AssertMsg(rc == VERR_EM_INTERPRETER, ("EMU: invlpg %RGv failed with %Rrc\n", exitQualification, rc));
2630	break;
2631	}
2632
2633	case VMX_EXIT_RDMSR: /* 31 RDMSR. Guest software attempted to execute RDMSR. */
2634	case VMX_EXIT_WRMSR: /* 32 WRMSR. Guest software attempted to execute WRMSR. */
2635	{
2636	uint32_t cbSize;
2637
2638	/* Note: the intel manual claims there's a REX version of RDMSR that's slightly different, so we play safe by completely disassembling the instruction. */
2639	Log2(("VMX: %s\n", (exitReason == VMX_EXIT_RDMSR) ? "rdmsr" : "wrmsr"));
2640	rc = EMInterpretInstruction(pVM, CPUMCTX2CORE(pCtx), 0, &cbSize);
2641	if (rc == VINF_SUCCESS)
2642	{
2643	/* EIP has been updated already. */
2644
2645	/* Only resume if successful. */
2646	goto ResumeExecution;
2647	}
2648	AssertMsg(rc == VERR_EM_INTERPRETER, ("EMU: %s failed with %Rrc\n", (exitReason == VMX_EXIT_RDMSR) ? "rdmsr" : "wrmsr", rc));
2649	break;
2650	}
2651
2652	case VMX_EXIT_CRX_MOVE: /* 28 Control-register accesses. */
2653	{
2654	STAM_PROFILE_ADV_START(&pVCpu->hwaccm.s.StatExit2Sub2, y2);
2655
2656	switch (VMX_EXIT_QUALIFICATION_CRX_ACCESS(exitQualification))
2657	{
2658	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_WRITE:
2659	Log2(("VMX: %RGv mov cr%d, x\n", (RTGCPTR)pCtx->rip, VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification)));
2660	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitCRxWrite[VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification)]);
2661	rc = EMInterpretCRxWrite(pVM, CPUMCTX2CORE(pCtx),
2662	VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification),
2663	VMX_EXIT_QUALIFICATION_CRX_GENREG(exitQualification));
2664
2665	switch (VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification))
2666	{
2667	case 0:
2668	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_GUEST_CR0 \| HWACCM_CHANGED_GUEST_CR3;
2669	break;
2670	case 2:
2671	break;
2672	case 3:
2673	Assert(!pVM->hwaccm.s.fNestedPaging \|\| !CPUMIsGuestInPagedProtectedModeEx(pCtx));
2674	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_GUEST_CR3;
2675	break;
2676	case 4:
2677	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_GUEST_CR4;
2678	break;
2679	case 8:
2680	/* CR8 contains the APIC TPR */
2681	Assert(!(pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW));
2682	break;
2683
2684	default:
2685	AssertFailed();
2686	break;
2687	}
2688	/* Check if a sync operation is pending. */
2689	if ( rc == VINF_SUCCESS /* don't bother if we are going to ring 3 anyway */
2690	&& VM_FF_ISPENDING(pVM, VM_FF_PGM_SYNC_CR3 \| VM_FF_PGM_SYNC_CR3_NON_GLOBAL))
2691	{
2692	rc = PGMSyncCR3(pVM, CPUMGetGuestCR0(pVM), CPUMGetGuestCR3(pVM), CPUMGetGuestCR4(pVM), VM_FF_ISSET(pVM, VM_FF_PGM_SYNC_CR3));
2693	AssertRC(rc);
2694	}
2695	break;
2696
2697	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ:
2698	Log2(("VMX: mov x, crx\n"));
2699	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitCRxRead[VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification)]);
2700
2701	Assert(!pVM->hwaccm.s.fNestedPaging \|\| !CPUMIsGuestInPagedProtectedModeEx(pCtx) \|\| VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification) != USE_REG_CR3);
2702
2703	/* CR8 reads only cause an exit when the TPR shadow feature isn't present. */
2704	Assert(VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification) != 8 \|\| !(pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW));
2705
2706	rc = EMInterpretCRxRead(pVM, CPUMCTX2CORE(pCtx),
2707	VMX_EXIT_QUALIFICATION_CRX_GENREG(exitQualification),
2708	VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification));
2709	break;
2710
2711	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_CLTS:
2712	Log2(("VMX: clts\n"));
2713	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitCLTS);
2714	rc = EMInterpretCLTS(pVM);
2715	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_GUEST_CR0;
2716	break;
2717
2718	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_LMSW:
2719	Log2(("VMX: lmsw %x\n", VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(exitQualification)));
2720	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitLMSW);
2721	rc = EMInterpretLMSW(pVM, CPUMCTX2CORE(pCtx), VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(exitQualification));
2722	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_GUEST_CR0;
2723	break;
2724	}
2725
2726	/* Update EIP if no error occurred. */
2727	if (RT_SUCCESS(rc))
2728	pCtx->rip += cbInstr;
2729
2730	if (rc == VINF_SUCCESS)
2731	{
2732	/* Only resume if successful. */
2733	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub2, y2);
2734	goto ResumeExecution;
2735	}
2736	Assert(rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_PGM_SYNC_CR3);
2737	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub2, y2);
2738	break;
2739	}
2740
2741	case VMX_EXIT_DRX_MOVE: /* 29 Debug-register accesses. */
2742	{
2743	if (!DBGFIsStepping(pVM))
2744	{
2745	/* Disable drx move intercepts. */
2746	pVCpu->hwaccm.s.vmx.proc_ctls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT;
2747	rc = VMXWriteCachedVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
2748	AssertRC(rc);
2749
2750	/* Save the host and load the guest debug state. */
2751	rc = CPUMR0LoadGuestDebugState(pVM, pVCpu, pCtx, true /* include DR6 */);
2752	AssertRC(rc);
2753
2754	#ifdef VBOX_WITH_STATISTICS
2755	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatDRxContextSwitch);
2756	if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(exitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
2757	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitDRxWrite);
2758	else
2759	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitDRxRead);
2760	#endif
2761
2762	goto ResumeExecution;
2763	}
2764
2765	/** @todo clear VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT after the first time and restore drx registers afterwards */
2766	if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(exitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
2767	{
2768	Log2(("VMX: mov drx%d, genreg%d\n", VMX_EXIT_QUALIFICATION_DRX_REGISTER(exitQualification), VMX_EXIT_QUALIFICATION_DRX_GENREG(exitQualification)));
2769	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitDRxWrite);
2770	rc = EMInterpretDRxWrite(pVM, CPUMCTX2CORE(pCtx),
2771	VMX_EXIT_QUALIFICATION_DRX_REGISTER(exitQualification),
2772	VMX_EXIT_QUALIFICATION_DRX_GENREG(exitQualification));
2773	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_GUEST_DEBUG;
2774	Log2(("DR7=%08x\n", pCtx->dr[7]));
2775	}
2776	else
2777	{
2778	Log2(("VMX: mov x, drx\n"));
2779	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitDRxRead);
2780	rc = EMInterpretDRxRead(pVM, CPUMCTX2CORE(pCtx),
2781	VMX_EXIT_QUALIFICATION_DRX_GENREG(exitQualification),
2782	VMX_EXIT_QUALIFICATION_DRX_REGISTER(exitQualification));
2783	}
2784	/* Update EIP if no error occurred. */
2785	if (RT_SUCCESS(rc))
2786	pCtx->rip += cbInstr;
2787
2788	if (rc == VINF_SUCCESS)
2789	{
2790	/* Only resume if successful. */
2791	goto ResumeExecution;
2792	}
2793	Assert(rc == VERR_EM_INTERPRETER);
2794	break;
2795	}
2796
2797	/* Note: We'll get a #GP if the IO instruction isn't allowed (IOPL or TSS bitmap); no need to double check. */
2798	case VMX_EXIT_PORT_IO: /* 30 I/O instruction. */
2799	{
2800	STAM_PROFILE_ADV_START(&pVCpu->hwaccm.s.StatExit2Sub1, y1);
2801	uint32_t uIOWidth = VMX_EXIT_QUALIFICATION_IO_WIDTH(exitQualification);
2802	uint32_t uPort;
2803	bool fIOWrite = (VMX_EXIT_QUALIFICATION_IO_DIRECTION(exitQualification) == VMX_EXIT_QUALIFICATION_IO_DIRECTION_OUT);
2804
2805	/** @todo necessary to make the distinction? */
2806	if (VMX_EXIT_QUALIFICATION_IO_ENCODING(exitQualification) == VMX_EXIT_QUALIFICATION_IO_ENCODING_DX)
2807	{
2808	uPort = pCtx->edx & 0xffff;
2809	}
2810	else
2811	uPort = VMX_EXIT_QUALIFICATION_IO_PORT(exitQualification); /* Immediate encoding. */
2812
2813	/* paranoia */
2814	if (RT_UNLIKELY(uIOWidth == 2 \|\| uIOWidth >= 4))
2815	{
2816	rc = fIOWrite ? VINF_IOM_HC_IOPORT_WRITE : VINF_IOM_HC_IOPORT_READ;
2817	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub1, y1);
2818	break;
2819	}
2820
2821	uint32_t cbSize = g_aIOSize[uIOWidth];
2822
2823	if (VMX_EXIT_QUALIFICATION_IO_STRING(exitQualification))
2824	{
2825	/* ins/outs */
2826	DISCPUSTATE Cpu;
2827
2828	/* Disassemble manually to deal with segment prefixes. */
2829	/** @todo VMX_VMCS_EXIT_GUEST_LINEAR_ADDR contains the flat pointer operand of the instruction. */
2830	/** @todo VMX_VMCS32_RO_EXIT_INSTR_INFO also contains segment prefix info. */
2831	rc = EMInterpretDisasOne(pVM, CPUMCTX2CORE(pCtx), &Cpu, NULL);
2832	if (rc == VINF_SUCCESS)
2833	{
2834	if (fIOWrite)
2835	{
2836	Log2(("IOMInterpretOUTSEx %RGv %x size=%d\n", (RTGCPTR)pCtx->rip, uPort, cbSize));
2837	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitIOStringWrite);
2838	rc = IOMInterpretOUTSEx(pVM, CPUMCTX2CORE(pCtx), uPort, Cpu.prefix, cbSize);
2839	}
2840	else
2841	{
2842	Log2(("IOMInterpretINSEx %RGv %x size=%d\n", (RTGCPTR)pCtx->rip, uPort, cbSize));
2843	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitIOStringRead);
2844	rc = IOMInterpretINSEx(pVM, CPUMCTX2CORE(pCtx), uPort, Cpu.prefix, cbSize);
2845	}
2846	}
2847	else
2848	rc = VINF_EM_RAW_EMULATE_INSTR;
2849	}
2850	else
2851	{
2852	/* normal in/out */
2853	uint32_t uAndVal = g_aIOOpAnd[uIOWidth];
2854
2855	Assert(!VMX_EXIT_QUALIFICATION_IO_REP(exitQualification));
2856
2857	if (fIOWrite)
2858	{
2859	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitIOWrite);
2860	rc = IOMIOPortWrite(pVM, uPort, pCtx->eax & uAndVal, cbSize);
2861	}
2862	else
2863	{
2864	uint32_t u32Val = 0;
2865
2866	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitIORead);
2867	rc = IOMIOPortRead(pVM, uPort, &u32Val, cbSize);
2868	if (IOM_SUCCESS(rc))
2869	{
2870	/* Write back to the EAX register. */
2871	pCtx->eax = (pCtx->eax & ~uAndVal) \| (u32Val & uAndVal);
2872	}
2873	}
2874	}
2875	/*
2876	* Handled the I/O return codes.
2877	* (The unhandled cases end up with rc == VINF_EM_RAW_EMULATE_INSTR.)
2878	*/
2879	if (IOM_SUCCESS(rc))
2880	{
2881	/* Update EIP and continue execution. */
2882	pCtx->rip += cbInstr;
2883	if (RT_LIKELY(rc == VINF_SUCCESS))
2884	{
2885	/* If any IO breakpoints are armed, then we should check if a debug trap needs to be generated. */
2886	if (pCtx->dr[7] & X86_DR7_ENABLED_MASK)
2887	{
2888	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatDRxIOCheck);
2889	for (unsigned i=0;i<4;i++)
2890	{
2891	unsigned uBPLen = g_aIOSize[X86_DR7_GET_LEN(pCtx->dr[7], i)];
2892
2893	if ( (uPort >= pCtx->dr[i] && uPort < pCtx->dr[i] + uBPLen)
2894	&& (pCtx->dr[7] & (X86_DR7_L(i) \| X86_DR7_G(i)))
2895	&& (pCtx->dr[7] & X86_DR7_RW(i, X86_DR7_RW_IO)) == X86_DR7_RW(i, X86_DR7_RW_IO))
2896	{
2897	uint64_t uDR6;
2898
2899	Assert(CPUMIsGuestDebugStateActive(pVM));
2900
2901	uDR6 = ASMGetDR6();
2902
2903	/* Clear all breakpoint status flags and set the one we just hit. */
2904	uDR6 &= ~(X86_DR6_B0\|X86_DR6_B1\|X86_DR6_B2\|X86_DR6_B3);
2905	uDR6 \|= (uint64_t)RT_BIT(i);
2906
2907	/* Note: AMD64 Architecture Programmer's Manual 13.1:
2908	* Bits 15:13 of the DR6 register is never cleared by the processor and must be cleared by software after
2909	* the contents have been read.
2910	*/
2911	ASMSetDR6(uDR6);
2912
2913	/* X86_DR7_GD will be cleared if drx accesses should be trapped inside the guest. */
2914	pCtx->dr[7] &= ~X86_DR7_GD;
2915
2916	/* Paranoia. */
2917	pCtx->dr[7] &= 0xffffffff; /* upper 32 bits reserved */
2918	pCtx->dr[7] &= ~(RT_BIT(11) \| RT_BIT(12) \| RT_BIT(14) \| RT_BIT(15)); /* must be zero */
2919	pCtx->dr[7] \|= 0x400; /* must be one */
2920
2921	/* Resync DR7 */
2922	rc = VMXWriteCachedVMCS(VMX_VMCS64_GUEST_DR7, pCtx->dr[7]);
2923	AssertRC(rc);
2924
2925	/* Construct inject info. */
2926	intInfo = X86_XCPT_DB;
2927	intInfo \|= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
2928	intInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HWEXCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
2929
2930	Log(("Inject IO debug trap at %RGv\n", (RTGCPTR)pCtx->rip));
2931	rc = VMXR0InjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), 0, 0);
2932	AssertRC(rc);
2933
2934	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub1, y1);
2935	goto ResumeExecution;
2936	}
2937	}
2938	}
2939
2940	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub1, y1);
2941	goto ResumeExecution;
2942	}
2943	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub1, y1);
2944	break;
2945	}
2946
2947	#ifdef VBOX_STRICT
2948	if (rc == VINF_IOM_HC_IOPORT_READ)
2949	Assert(!fIOWrite);
2950	else if (rc == VINF_IOM_HC_IOPORT_WRITE)
2951	Assert(fIOWrite);
2952	else
2953	AssertMsg(RT_FAILURE(rc) \|\| rc == VINF_EM_RAW_EMULATE_INSTR \|\| rc == VINF_EM_RAW_GUEST_TRAP \|\| rc == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", rc));
2954	#endif
2955	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub1, y1);
2956	break;
2957	}
2958
2959	case VMX_EXIT_TPR: /* 43 TPR below threshold. Guest software executed MOV to CR8. */
2960	LogFlow(("VMX_EXIT_TPR\n"));
2961	/* RIP is already set to the next instruction and the TPR has been synced back. Just resume. */
2962	goto ResumeExecution;
2963
2964	default:
2965	/* The rest is handled after syncing the entire CPU state. */
2966	break;
2967	}
2968
2969	/* Note: the guest state isn't entirely synced back at this stage. */
2970
2971	/* Investigate why there was a VM-exit. (part 2) */
2972	switch (exitReason)
2973	{
2974	case VMX_EXIT_EXCEPTION: /* 0 Exception or non-maskable interrupt (NMI). */
2975	case VMX_EXIT_EXTERNAL_IRQ: /* 1 External interrupt. */
2976	case VMX_EXIT_EPT_VIOLATION:
2977	/* Already handled above. */
2978	break;
2979
2980	case VMX_EXIT_TRIPLE_FAULT: /* 2 Triple fault. */
2981	rc = VINF_EM_RESET; /* Triple fault equals a reset. */
2982	break;
2983
2984	case VMX_EXIT_INIT_SIGNAL: /* 3 INIT signal. */
2985	case VMX_EXIT_SIPI: /* 4 Start-up IPI (SIPI). */
2986	rc = VINF_EM_RAW_INTERRUPT;
2987	AssertFailed(); /* Can't happen. Yet. */
2988	break;
2989
2990	case VMX_EXIT_IO_SMI_IRQ: /* 5 I/O system-management interrupt (SMI). */
2991	case VMX_EXIT_SMI_IRQ: /* 6 Other SMI. */
2992	rc = VINF_EM_RAW_INTERRUPT;
2993	AssertFailed(); /* Can't happen afaik. */
2994	break;
2995
2996	case VMX_EXIT_TASK_SWITCH: /* 9 Task switch. */
2997	rc = VERR_EM_INTERPRETER;
2998	break;
2999
3000	case VMX_EXIT_HLT: /* 12 Guest software attempted to execute HLT. */
3001	/** Check if external interrupts are pending; if so, don't switch back. */
3002	pCtx->rip++; /* skip hlt */
3003	if ( pCtx->eflags.Bits.u1IF
3004	&& VM_FF_ISPENDING(pVM, (VM_FF_INTERRUPT_APIC\|VM_FF_INTERRUPT_PIC)))
3005	goto ResumeExecution;
3006
3007	rc = VINF_EM_HALT;
3008	break;
3009
3010	case VMX_EXIT_RSM: /* 17 Guest software attempted to execute RSM in SMM. */
3011	AssertFailed(); /* can't happen. */
3012	rc = VINF_EM_RAW_EXCEPTION_PRIVILEGED;
3013	break;
3014
3015	case VMX_EXIT_VMCALL: /* 18 Guest software executed VMCALL. */
3016	case VMX_EXIT_VMCLEAR: /* 19 Guest software executed VMCLEAR. */
3017	case VMX_EXIT_VMLAUNCH: /* 20 Guest software executed VMLAUNCH. */
3018	case VMX_EXIT_VMPTRLD: /* 21 Guest software executed VMPTRLD. */
3019	case VMX_EXIT_VMPTRST: /* 22 Guest software executed VMPTRST. */
3020	case VMX_EXIT_VMREAD: /* 23 Guest software executed VMREAD. */
3021	case VMX_EXIT_VMRESUME: /* 24 Guest software executed VMRESUME. */
3022	case VMX_EXIT_VMWRITE: /* 25 Guest software executed VMWRITE. */
3023	case VMX_EXIT_VMXOFF: /* 26 Guest software executed VMXOFF. */
3024	case VMX_EXIT_VMXON: /* 27 Guest software executed VMXON. */
3025	/** @todo inject #UD immediately */
3026	rc = VINF_EM_RAW_EXCEPTION_PRIVILEGED;
3027	break;
3028
3029	case VMX_EXIT_CPUID: /* 10 Guest software attempted to execute CPUID. */
3030	case VMX_EXIT_RDTSC: /* 16 Guest software attempted to execute RDTSC. */
3031	case VMX_EXIT_INVPG: /* 14 Guest software attempted to execute INVPG. */
3032	case VMX_EXIT_CRX_MOVE: /* 28 Control-register accesses. */
3033	case VMX_EXIT_DRX_MOVE: /* 29 Debug-register accesses. */
3034	case VMX_EXIT_PORT_IO: /* 30 I/O instruction. */
3035	/* already handled above */
3036	AssertMsg( rc == VINF_PGM_CHANGE_MODE
3037	\|\| rc == VINF_EM_RAW_INTERRUPT
3038	\|\| rc == VERR_EM_INTERPRETER
3039	\|\| rc == VINF_EM_RAW_EMULATE_INSTR
3040	\|\| rc == VINF_PGM_SYNC_CR3
3041	\|\| rc == VINF_IOM_HC_IOPORT_READ
3042	\|\| rc == VINF_IOM_HC_IOPORT_WRITE
3043	\|\| rc == VINF_EM_RAW_GUEST_TRAP
3044	\|\| rc == VINF_TRPM_XCPT_DISPATCHED
3045	\|\| rc == VINF_EM_RESCHEDULE_REM,
3046	("rc = %d\n", rc));
3047	break;
3048
3049	case VMX_EXIT_TPR: /* 43 TPR below threshold. Guest software executed MOV to CR8. */
3050	case VMX_EXIT_RDMSR: /* 31 RDMSR. Guest software attempted to execute RDMSR. */
3051	case VMX_EXIT_WRMSR: /* 32 WRMSR. Guest software attempted to execute WRMSR. */
3052	/* Note: If we decide to emulate them here, then we must sync the MSRs that could have been changed (sysenter, fs/gs base)!!! */
3053	rc = VERR_EM_INTERPRETER;
3054	break;
3055
3056	case VMX_EXIT_RDPMC: /* 15 Guest software attempted to execute RDPMC. */
3057	case VMX_EXIT_MWAIT: /* 36 Guest software executed MWAIT. */
3058	case VMX_EXIT_MONITOR: /* 39 Guest software attempted to execute MONITOR. */
3059	case VMX_EXIT_PAUSE: /* 40 Guest software attempted to execute PAUSE. */
3060	rc = VINF_EM_RAW_EXCEPTION_PRIVILEGED;
3061	break;
3062
3063	case VMX_EXIT_IRQ_WINDOW: /* 7 Interrupt window. */
3064	Assert(rc == VINF_EM_RAW_INTERRUPT);
3065	break;
3066
3067	case VMX_EXIT_ERR_INVALID_GUEST_STATE: /* 33 VM-entry failure due to invalid guest state. */
3068	{
3069	#ifdef VBOX_STRICT
3070	RTCCUINTREG val;
3071
3072	Log(("VMX_EXIT_ERR_INVALID_GUEST_STATE\n"));
3073
3074	VMXReadVMCS(VMX_VMCS64_GUEST_RIP, &val);
3075	Log(("Old eip %RGv new %RGv\n", (RTGCPTR)pCtx->rip, (RTGCPTR)val));
3076
3077	VMXReadVMCS(VMX_VMCS64_GUEST_CR0, &val);
3078	Log(("VMX_VMCS_GUEST_CR0 %RX64\n", val));
3079
3080	VMXReadVMCS(VMX_VMCS64_GUEST_CR3, &val);
3081	Log(("VMX_VMCS_GUEST_CR3 %RGp\n", val));
3082
3083	VMXReadVMCS(VMX_VMCS64_GUEST_CR4, &val);
3084	Log(("VMX_VMCS_GUEST_CR4 %RX64\n", val));
3085
3086	VMXReadVMCS(VMX_VMCS_GUEST_RFLAGS, &val);
3087	Log(("VMX_VMCS_GUEST_RFLAGS %08x\n", val));
3088
3089	VMX_LOG_SELREG(CS, "CS");
3090	VMX_LOG_SELREG(DS, "DS");
3091	VMX_LOG_SELREG(ES, "ES");
3092	VMX_LOG_SELREG(FS, "FS");
3093	VMX_LOG_SELREG(GS, "GS");
3094	VMX_LOG_SELREG(SS, "SS");
3095	VMX_LOG_SELREG(TR, "TR");
3096	VMX_LOG_SELREG(LDTR, "LDTR");
3097
3098	VMXReadVMCS(VMX_VMCS64_GUEST_GDTR_BASE, &val);
3099	Log(("VMX_VMCS_GUEST_GDTR_BASE %RGv\n", val));
3100	VMXReadVMCS(VMX_VMCS64_GUEST_IDTR_BASE, &val);
3101	Log(("VMX_VMCS_GUEST_IDTR_BASE %RGv\n", val));
3102	#endif /* VBOX_STRICT */
3103	rc = VERR_VMX_INVALID_GUEST_STATE;
3104	break;
3105	}
3106
3107	case VMX_EXIT_ERR_MSR_LOAD: /* 34 VM-entry failure due to MSR loading. */
3108	case VMX_EXIT_ERR_MACHINE_CHECK: /* 41 VM-entry failure due to machine-check. */
3109	default:
3110	rc = VERR_VMX_UNEXPECTED_EXIT_CODE;
3111	AssertMsgFailed(("Unexpected exit code %d\n", exitReason)); /* Can't happen. */
3112	break;
3113
3114	}
3115	end:
3116
3117	/* Signal changes for the recompiler. */
3118	CPUMSetChangedFlags(pVM, CPUM_CHANGED_SYSENTER_MSR \| CPUM_CHANGED_LDTR \| CPUM_CHANGED_GDTR \| CPUM_CHANGED_IDTR \| CPUM_CHANGED_TR \| CPUM_CHANGED_HIDDEN_SEL_REGS);
3119
3120	/* If we executed vmlaunch/vmresume and an external irq was pending, then we don't have to do a full sync the next time. */
3121	if ( exitReason == VMX_EXIT_EXTERNAL_IRQ
3122	&& !VMX_EXIT_INTERRUPTION_INFO_VALID(intInfo))
3123	{
3124	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatPendingHostIrq);
3125	/* On the next entry we'll only sync the host context. */
3126	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_HOST_CONTEXT;
3127	}
3128	else
3129	{
3130	/* On the next entry we'll sync everything. */
3131	/** @todo we can do better than this */
3132	/* Not in the VINF_PGM_CHANGE_MODE though! */
3133	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_ALL;
3134	}
3135
3136	/* translate into a less severe return code */
3137	if (rc == VERR_EM_INTERPRETER)
3138	rc = VINF_EM_RAW_EMULATE_INSTR;
3139	else
3140	/* Try to extract more information about what might have gone wrong here. */
3141	if (rc == VERR_VMX_INVALID_VMCS_PTR)
3142	{
3143	VMXGetActivateVMCS(&pVCpu->hwaccm.s.vmx.lasterror.u64VMCSPhys);
3144	pVCpu->hwaccm.s.vmx.lasterror.ulVMCSRevision = (uint32_t )pVCpu->hwaccm.s.vmx.pVMCS;
3145	pVCpu->hwaccm.s.vmx.lasterror.idEnteredCpu = pVCpu->hwaccm.s.idEnteredCpu;
3146	pVCpu->hwaccm.s.vmx.lasterror.idCurrentCpu = RTMpCpuId();
3147	}
3148
3149	STAM_STATS({
3150	if (fStatExit2Started) STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2, y);
3151	else if (fStatEntryStarted) STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatEntry, x);
3152	});
3153	Log2(("X"));
3154	return rc;
3155	}
3156
3157
3158	/**
3159	* Enters the VT-x session
3160	*
3161	* @returns VBox status code.
3162	* @param pVM The VM to operate on.
3163	* @param pVCpu The VMCPU to operate on.
3164	* @param pCpu CPU info struct
3165	*/
3166	VMMR0DECL(int) VMXR0Enter(PVM pVM, PVMCPU pVCpu, PHWACCM_CPUINFO pCpu)
3167	{
3168	Assert(pVM->hwaccm.s.vmx.fSupported);
3169
3170	unsigned cr4 = ASMGetCR4();
3171	if (!(cr4 & X86_CR4_VMXE))
3172	{
3173	AssertMsgFailed(("X86_CR4_VMXE should be set!\n"));
3174	return VERR_VMX_X86_CR4_VMXE_CLEARED;
3175	}
3176
3177	/* Activate the VM Control Structure. */
3178	int rc = VMXActivateVMCS(pVCpu->hwaccm.s.vmx.pVMCSPhys);
3179	if (RT_FAILURE(rc))
3180	return rc;
3181
3182	pVCpu->hwaccm.s.fResumeVM = false;
3183	return VINF_SUCCESS;
3184	}
3185
3186
3187	/**
3188	* Leaves the VT-x session
3189	*
3190	* @returns VBox status code.
3191	* @param pVM The VM to operate on.
3192	* @param pVCpu The VMCPU to operate on.
3193	* @param pCtx CPU context
3194	*/
3195	VMMR0DECL(int) VMXR0Leave(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
3196	{
3197	Assert(pVM->hwaccm.s.vmx.fSupported);
3198
3199	/* Save the guest debug state if necessary. */
3200	if (CPUMIsGuestDebugStateActive(pVM))
3201	{
3202	CPUMR0SaveGuestDebugState(pVM, pVCpu, pCtx, true /* save DR6 */);
3203
3204	/* Enable drx move intercepts again. */
3205	pVCpu->hwaccm.s.vmx.proc_ctls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT;
3206	int rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
3207	AssertRC(rc);
3208
3209	/* Resync the debug registers the next time. */
3210	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_GUEST_DEBUG;
3211	}
3212	else
3213	Assert(pVCpu->hwaccm.s.vmx.proc_ctls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT);
3214
3215	/* Flush all pending VMCS writes. */
3216	VMXFlushWriteCache(pVCpu);
3217
3218	/* Clear VM Control Structure. Marking it inactive, clearing implementation specific data and writing back VMCS data to memory. */
3219	int rc = VMXClearVMCS(pVCpu->hwaccm.s.vmx.pVMCSPhys);
3220	AssertRC(rc);
3221
3222	return VINF_SUCCESS;
3223	}
3224
3225	/**
3226	* Flush the TLB (EPT)
3227	*
3228	* @returns VBox status code.
3229	* @param pVM The VM to operate on.
3230	* @param pVCpu The VM CPU to operate on.
3231	* @param enmFlush Type of flush
3232	* @param GCPhys Physical address of the page to flush
3233	*/
3234	static void vmxR0FlushEPT(PVM pVM, PVMCPU pVCpu, VMX_FLUSH enmFlush, RTGCPHYS GCPhys)
3235	{
3236	uint64_t descriptor[2];
3237
3238	LogFlow(("vmxR0FlushEPT %d %RGv\n", enmFlush, GCPhys));
3239	Assert(pVM->hwaccm.s.fNestedPaging);
3240	descriptor[0] = pVCpu->hwaccm.s.vmx.GCPhysEPTP;
3241	descriptor[1] = GCPhys;
3242	int rc = VMXR0InvEPT(enmFlush, &descriptor[0]);
3243	AssertRC(rc);
3244	}
3245
3246	#ifdef HWACCM_VTX_WITH_VPID
3247	/**
3248	* Flush the TLB (EPT)
3249	*
3250	* @returns VBox status code.
3251	* @param pVM The VM to operate on.
3252	* @param pVCpu The VM CPU to operate on.
3253	* @param enmFlush Type of flush
3254	* @param GCPtr Virtual address of the page to flush
3255	*/
3256	static void vmxR0FlushVPID(PVM pVM, PVMCPU pVCpu, VMX_FLUSH enmFlush, RTGCPTR GCPtr)
3257	{
3258	#if HC_ARCH_BITS == 32
3259	/* If we get a flush in 64 bits guest mode, then force a full TLB flush. Invvpid probably takes only 32 bits addresses. (@todo) */
3260	if (CPUMIsGuestInLongMode(pVM))
3261	{
3262	pVCpu->hwaccm.s.fForceTLBFlush = true;
3263	}
3264	else
3265	#endif
3266	{
3267	uint64_t descriptor[2];
3268
3269	Assert(pVM->hwaccm.s.vmx.fVPID);
3270	descriptor[0] = pVCpu->hwaccm.s.uCurrentASID;
3271	descriptor[1] = GCPtr;
3272	int rc = VMXR0InvVPID(enmFlush, &descriptor[0]);
3273	AssertRC(rc);
3274	}
3275	}
3276	#endif /* HWACCM_VTX_WITH_VPID */
3277
3278	/**
3279	* Invalidates a guest page
3280	*
3281	* @returns VBox status code.
3282	* @param pVM The VM to operate on.
3283	* @param pVCpu The VM CPU to operate on.
3284	* @param GCVirt Page to invalidate
3285	*/
3286	VMMR0DECL(int) VMXR0InvalidatePage(PVM pVM, PVMCPU pVCpu, RTGCPTR GCVirt)
3287	{
3288	bool fFlushPending = pVCpu->hwaccm.s.fForceTLBFlush;
3289
3290	LogFlow(("VMXR0InvalidatePage %RGv\n", GCVirt));
3291
3292	/* Only relevant if we want to use VPID.
3293	* In the nested paging case we still see such calls, but
3294	* can safely ignore them. (e.g. after cr3 updates)
3295	*/
3296	#ifdef HWACCM_VTX_WITH_VPID
3297	/* Skip it if a TLB flush is already pending. */
3298	if ( !fFlushPending
3299	&& pVM->hwaccm.s.vmx.fVPID)
3300	vmxR0FlushVPID(pVM, pVCpu, pVM->hwaccm.s.vmx.enmFlushPage, GCVirt);
3301	#endif /* HWACCM_VTX_WITH_VPID */
3302
3303	return VINF_SUCCESS;
3304	}
3305
3306	/**
3307	* Invalidates a guest page by physical address
3308	*
3309	* NOTE: Assumes the current instruction references this physical page though a virtual address!!
3310	*
3311	* @returns VBox status code.
3312	* @param pVM The VM to operate on.
3313	* @param pVCpu The VM CPU to operate on.
3314	* @param GCPhys Page to invalidate
3315	*/
3316	VMMR0DECL(int) VMXR0InvalidatePhysPage(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys)
3317	{
3318	bool fFlushPending = pVCpu->hwaccm.s.fForceTLBFlush;
3319
3320	Assert(pVM->hwaccm.s.fNestedPaging);
3321
3322	LogFlow(("VMXR0InvalidatePhysPage %RGp\n", GCPhys));
3323
3324	/* Skip it if a TLB flush is already pending. */
3325	if (!fFlushPending)
3326	vmxR0FlushEPT(pVM, pVCpu, pVM->hwaccm.s.vmx.enmFlushPage, GCPhys);
3327
3328	return VINF_SUCCESS;
3329	}
3330
3331	/**
3332	* Report world switch error and dump some useful debug info
3333	*
3334	* @param pVM The VM to operate on.
3335	* @param pVCpu The VMCPU to operate on.
3336	* @param rc Return code
3337	* @param pCtx Current CPU context (not updated)
3338	*/
3339	static void VMXR0ReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, int rc, PCPUMCTX pCtx)
3340	{
3341	switch (rc)
3342	{
3343	case VERR_VMX_INVALID_VMXON_PTR:
3344	AssertFailed();
3345	break;
3346
3347	case VERR_VMX_UNABLE_TO_START_VM:
3348	case VERR_VMX_UNABLE_TO_RESUME_VM:
3349	{
3350	int rc;
3351	RTCCUINTREG exitReason, instrError, val;
3352
3353	rc = VMXReadVMCS(VMX_VMCS32_RO_EXIT_REASON, &exitReason);
3354	rc \|= VMXReadVMCS(VMX_VMCS32_RO_VM_INSTR_ERROR, &instrError);
3355	AssertRC(rc);
3356	if (rc == VINF_SUCCESS)
3357	{
3358	Log(("Unable to start/resume VM for reason: %x. Instruction error %x\n", (uint32_t)exitReason, (uint32_t)instrError));
3359	Log(("Current stack %08x\n", &rc));
3360
3361	pVCpu->hwaccm.s.vmx.lasterror.ulInstrError = instrError;
3362	pVCpu->hwaccm.s.vmx.lasterror.ulExitReason = exitReason;
3363
3364	#ifdef VBOX_STRICT
3365	RTGDTR gdtr;
3366	PX86DESCHC pDesc;
3367
3368	ASMGetGDTR(&gdtr);
3369
3370	VMXReadVMCS(VMX_VMCS64_GUEST_RIP, &val);
3371	Log(("Old eip %RGv new %RGv\n", (RTGCPTR)pCtx->rip, (RTGCPTR)val));
3372	VMXReadVMCS(VMX_VMCS_CTRL_PIN_EXEC_CONTROLS, &val);
3373	Log(("VMX_VMCS_CTRL_PIN_EXEC_CONTROLS %08x\n", val));
3374	VMXReadVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, &val);
3375	Log(("VMX_VMCS_CTRL_PROC_EXEC_CONTROLS %08x\n", val));
3376	VMXReadVMCS(VMX_VMCS_CTRL_ENTRY_CONTROLS, &val);
3377	Log(("VMX_VMCS_CTRL_ENTRY_CONTROLS %08x\n", val));
3378	VMXReadVMCS(VMX_VMCS_CTRL_EXIT_CONTROLS, &val);
3379	Log(("VMX_VMCS_CTRL_EXIT_CONTROLS %08x\n", val));
3380
3381	VMXReadVMCS(VMX_VMCS_HOST_CR0, &val);
3382	Log(("VMX_VMCS_HOST_CR0 %08x\n", val));
3383
3384	VMXReadVMCS(VMX_VMCS_HOST_CR3, &val);
3385	Log(("VMX_VMCS_HOST_CR3 %08x\n", val));
3386
3387	VMXReadVMCS(VMX_VMCS_HOST_CR4, &val);
3388	Log(("VMX_VMCS_HOST_CR4 %08x\n", val));
3389
3390	VMXReadVMCS(VMX_VMCS16_HOST_FIELD_CS, &val);
3391	Log(("VMX_VMCS_HOST_FIELD_CS %08x\n", val));
3392
3393	VMXReadVMCS(VMX_VMCS_GUEST_RFLAGS, &val);
3394	Log(("VMX_VMCS_GUEST_RFLAGS %08x\n", val));
3395
3396	if (val < gdtr.cbGdt)
3397	{
3398	pDesc = &((PX86DESCHC)gdtr.pGdt)[val >> X86_SEL_SHIFT_HC];
3399	HWACCMR0DumpDescriptor(pDesc, val, "CS: ");
3400	}
3401
3402	VMXReadVMCS(VMX_VMCS16_HOST_FIELD_DS, &val);
3403	Log(("VMX_VMCS_HOST_FIELD_DS %08x\n", val));
3404	if (val < gdtr.cbGdt)
3405	{
3406	pDesc = &((PX86DESCHC)gdtr.pGdt)[val >> X86_SEL_SHIFT_HC];
3407	HWACCMR0DumpDescriptor(pDesc, val, "DS: ");
3408	}
3409
3410	VMXReadVMCS(VMX_VMCS16_HOST_FIELD_ES, &val);
3411	Log(("VMX_VMCS_HOST_FIELD_ES %08x\n", val));
3412	if (val < gdtr.cbGdt)
3413	{
3414	pDesc = &((PX86DESCHC)gdtr.pGdt)[val >> X86_SEL_SHIFT_HC];
3415	HWACCMR0DumpDescriptor(pDesc, val, "ES: ");
3416	}
3417
3418	VMXReadVMCS(VMX_VMCS16_HOST_FIELD_FS, &val);
3419	Log(("VMX_VMCS16_HOST_FIELD_FS %08x\n", val));
3420	if (val < gdtr.cbGdt)
3421	{
3422	pDesc = &((PX86DESCHC)gdtr.pGdt)[val >> X86_SEL_SHIFT_HC];
3423	HWACCMR0DumpDescriptor(pDesc, val, "FS: ");
3424	}
3425
3426	VMXReadVMCS(VMX_VMCS16_HOST_FIELD_GS, &val);
3427	Log(("VMX_VMCS16_HOST_FIELD_GS %08x\n", val));
3428	if (val < gdtr.cbGdt)
3429	{
3430	pDesc = &((PX86DESCHC)gdtr.pGdt)[val >> X86_SEL_SHIFT_HC];
3431	HWACCMR0DumpDescriptor(pDesc, val, "GS: ");
3432	}
3433
3434	VMXReadVMCS(VMX_VMCS16_HOST_FIELD_SS, &val);
3435	Log(("VMX_VMCS16_HOST_FIELD_SS %08x\n", val));
3436	if (val < gdtr.cbGdt)
3437	{
3438	pDesc = &((PX86DESCHC)gdtr.pGdt)[val >> X86_SEL_SHIFT_HC];
3439	HWACCMR0DumpDescriptor(pDesc, val, "SS: ");
3440	}
3441
3442	VMXReadVMCS(VMX_VMCS16_HOST_FIELD_TR, &val);
3443	Log(("VMX_VMCS16_HOST_FIELD_TR %08x\n", val));
3444	if (val < gdtr.cbGdt)
3445	{
3446	pDesc = &((PX86DESCHC)gdtr.pGdt)[val >> X86_SEL_SHIFT_HC];
3447	HWACCMR0DumpDescriptor(pDesc, val, "TR: ");
3448	}
3449
3450	VMXReadVMCS(VMX_VMCS_HOST_TR_BASE, &val);
3451	Log(("VMX_VMCS_HOST_TR_BASE %RHv\n", val));
3452
3453	VMXReadVMCS(VMX_VMCS_HOST_GDTR_BASE, &val);
3454	Log(("VMX_VMCS_HOST_GDTR_BASE %RHv\n", val));
3455	VMXReadVMCS(VMX_VMCS_HOST_IDTR_BASE, &val);
3456	Log(("VMX_VMCS_HOST_IDTR_BASE %RHv\n", val));
3457
3458	VMXReadVMCS(VMX_VMCS32_HOST_SYSENTER_CS, &val);
3459	Log(("VMX_VMCS_HOST_SYSENTER_CS %08x\n", val));
3460
3461	VMXReadVMCS(VMX_VMCS_HOST_SYSENTER_EIP, &val);
3462	Log(("VMX_VMCS_HOST_SYSENTER_EIP %RHv\n", val));
3463
3464	VMXReadVMCS(VMX_VMCS_HOST_SYSENTER_ESP, &val);
3465	Log(("VMX_VMCS_HOST_SYSENTER_ESP %RHv\n", val));
3466
3467	VMXReadVMCS(VMX_VMCS_HOST_RSP, &val);
3468	Log(("VMX_VMCS_HOST_RSP %RHv\n", val));
3469	VMXReadVMCS(VMX_VMCS_HOST_RIP, &val);
3470	Log(("VMX_VMCS_HOST_RIP %RHv\n", val));
3471
3472	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBIRD_32BIT_KERNEL)
3473	if (VMX_IS_64BIT_HOST_MODE())
3474	{
3475	Log(("MSR_K6_EFER = %RX64\n", ASMRdMsr(MSR_K6_EFER)));
3476	Log(("MSR_K6_STAR = %RX64\n", ASMRdMsr(MSR_K6_STAR)));
3477	Log(("MSR_K8_LSTAR = %RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
3478	Log(("MSR_K8_CSTAR = %RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
3479	Log(("MSR_K8_SF_MASK = %RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
3480	}
3481	# endif
3482	#endif /* VBOX_STRICT */
3483	}
3484	break;
3485	}
3486
3487	default:
3488	/* impossible */
3489	AssertMsgFailed(("%Rrc (%#x)\n", rc, rc));
3490	break;
3491	}
3492	}
3493
3494	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
3495	/**
3496	* Prepares for and executes VMLAUNCH (64 bits guest mode)
3497	*
3498	* @returns VBox status code
3499	* @param fResume vmlauch/vmresume
3500	* @param pCtx Guest context
3501	* @param pCache VMCS cache
3502	* @param pVM The VM to operate on.
3503	* @param pVCpu The VMCPU to operate on.
3504	*/
3505	DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
3506	{
3507	uint32_t aParam[6];
3508	PHWACCM_CPUINFO pCpu;
3509	RTHCPHYS pPageCpuPhys;
3510	int rc;
3511
3512	pCpu = HWACCMR0GetCurrentCpu();
3513	pPageCpuPhys = RTR0MemObjGetPagePhysAddr(pCpu->pMemObj, 0);
3514
3515
3516	#ifdef DEBUG
3517	pCache->TestIn.pPageCpuPhys = 0;
3518	pCache->TestIn.pVMCSPhys = 0;
3519	pCache->TestIn.pCache = 0;
3520	pCache->TestOut.pVMCSPhys = 0;
3521	pCache->TestOut.pCache = 0;
3522	pCache->TestOut.pCtx = 0;
3523	#endif
3524
3525	aParam[0] = (uint32_t)(pPageCpuPhys); /* Param 1: VMXON physical address - Lo. */
3526	aParam[1] = (uint32_t)(pPageCpuPhys >> 32); /* Param 1: VMXON physical address - Hi. */
3527	aParam[2] = (uint32_t)(pVCpu->hwaccm.s.vmx.pVMCSPhys); /* Param 2: VMCS physical address - Lo. */
3528	aParam[3] = (uint32_t)(pVCpu->hwaccm.s.vmx.pVMCSPhys >> 32); /* Param 2: VMCS physical address - Hi. */
3529	aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hwaccm.s.vmx.VMCSCache);
3530	aParam[5] = 0;
3531
3532	rc = VMXR0Execute64BitsHandler(pVM, pVCpu, pCtx, pVM->hwaccm.s.pfnVMXGCStartVM64, 6, &aParam[0]);
3533
3534	#ifdef DEBUG
3535	AssertMsg(pCache->TestIn.pPageCpuPhys == pPageCpuPhys, ("%RHp vs %RHp\n", pCache->TestIn.pPageCpuPhys, pPageCpuPhys));
3536	AssertMsg(pCache->TestIn.pVMCSPhys == pVCpu->hwaccm.s.vmx.pVMCSPhys, ("%RHp vs %RHp\n", pCache->TestIn.pVMCSPhys, pVCpu->hwaccm.s.vmx.pVMCSPhys));
3537	AssertMsg(pCache->TestIn.pVMCSPhys == pCache->TestOut.pVMCSPhys, ("%RHp vs %RHp\n", pCache->TestIn.pVMCSPhys, pCache->TestOut.pVMCSPhys));
3538	AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache, pCache->TestOut.pCache));
3539	AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hwaccm.s.vmx.VMCSCache), ("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hwaccm.s.vmx.VMCSCache)));
3540	AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx, pCache->TestOut.pCtx));
3541	#endif
3542
3543	return rc;
3544	}
3545
3546	/**
3547	* Executes the specified handler in 64 mode
3548	*
3549	* @returns VBox status code.
3550	* @param pVM The VM to operate on.
3551	* @param pVCpu The VMCPU to operate on.
3552	* @param pCtx Guest context
3553	* @param pfnHandler RC handler
3554	* @param cbParam Number of parameters
3555	* @param paParam Array of 32 bits parameters
3556	*/
3557	VMMR0DECL(int) VMXR0Execute64BitsHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, RTRCPTR pfnHandler, uint32_t cbParam, uint32_t *paParam)
3558	{
3559	int rc, rc2;
3560	RTHCUINTREG uFlags;
3561	PHWACCM_CPUINFO pCpu;
3562	RTHCPHYS pPageCpuPhys;
3563
3564	/* @todo This code is not guest SMP safe (hyper context) */
3565	AssertReturn(pVM->cCPUs == 1, VERR_ACCESS_DENIED);
3566	AssertReturn(pVM->hwaccm.s.pfnHost32ToGuest64R0, VERR_INTERNAL_ERROR);
3567	Assert(pVCpu->hwaccm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hwaccm.s.vmx.VMCSCache.Write.aField));
3568	Assert(pVCpu->hwaccm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hwaccm.s.vmx.VMCSCache.Read.aField));
3569
3570	pCpu = HWACCMR0GetCurrentCpu();
3571	pPageCpuPhys = RTR0MemObjGetPagePhysAddr(pCpu->pMemObj, 0);
3572
3573	/* Clear VM Control Structure. Marking it inactive, clearing implementation specific data and writing back VMCS data to memory. */
3574	VMXClearVMCS(pVCpu->hwaccm.s.vmx.pVMCSPhys);
3575
3576	/* Leave VMX Root Mode. */
3577	VMXDisable();
3578
3579	uFlags = ASMIntDisableFlags();
3580
3581	CPUMSetHyperESP(pVM, VMMGetStackRC(pVM));
3582	CPUMSetHyperEIP(pVM, pfnHandler);
3583	for (int i=(int)cbParam-1;i>=0;i--)
3584	CPUMPushHyper(pVM, paParam[i]);
3585
3586	/* Call switcher. */
3587	rc = pVM->hwaccm.s.pfnHost32ToGuest64R0(pVM);
3588
3589	#ifdef VBOX_STRICT
3590	RTHCUINTREG uFlagsTest = ASMGetFlags();
3591	#endif
3592
3593	ASMSetFlags(uFlags);
3594
3595	/* Make sure the VMX instructions don't cause #UD faults. */
3596	ASMSetCR4(ASMGetCR4() \| X86_CR4_VMXE);
3597
3598	/* Enter VMX Root Mode */
3599	rc2 = VMXEnable(pPageCpuPhys);
3600	if (RT_FAILURE(rc2))
3601	{
3602	if (pVM)
3603	VMXR0CheckError(pVM, pVCpu, rc2);
3604	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
3605	return VERR_VMX_VMXON_FAILED;
3606	}
3607
3608	VMXActivateVMCS(pVCpu->hwaccm.s.vmx.pVMCSPhys);
3609	Assert(!(uFlagsTest & X86_EFL_IF));
3610
3611	return rc;
3612	}
3613
3614	#endif /* HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) */
3615
3616
3617	#ifdef VMX_USE_CACHED_VMCS_ACCESSES
3618	/**
3619	* Flush the write cache in order not to overflow it with frequent ring switches.
3620	*
3621	* @param pVCpu The VMCPU to operate on.
3622	*/
3623	VMMR0DECL(void) VMXFlushWriteCache(PVMCPU pVCpu)
3624	{
3625	PVMCSCACHE pCache = &pVCpu->hwaccm.s.vmx.VMCSCache;
3626	/* Flush the queued writes first. */
3627	for (unsigned i=0;i<pCache->Write.cValidEntries;i++)
3628	VMXWriteVMCS(pCache->Write.aField[i], pCache->Write.aFieldVal[i]);
3629
3630	pCache->Write.cValidEntries = 0;
3631	}
3632	#endif

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMR0/HWVMXR0.cpp@ 15371

Download in other formats: