HWVMXR0.cpp@ 15291

Last change on this file since 15291 was 15291, checked in by vboxsync, 16 years ago
Flush pending writes when leaving ring 0.
Property svn:eol-style set to `native` Property svn:keywords set to `Id`
File size: 141.5 KB

Line
1	/* $Id: HWVMXR0.cpp 15291 2008-12-11 08:22:18Z vboxsync $ */
2	/** @file
3	* HWACCM VMX - Host Context Ring 0.
4	*/
5
6	/*
7	* Copyright (C) 2006-2007 Sun Microsystems, Inc.
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*
17	* Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa
18	* Clara, CA 95054 USA or visit http://www.sun.com if you need
19	* additional information or have any questions.
20	*/
21
22
23	/*******************************************************************************
24	* Header Files *
25	*******************************************************************************/
26	#define LOG_GROUP LOG_GROUP_HWACCM
27	#include <VBox/hwaccm.h>
28	#include "HWACCMInternal.h"
29	#include <VBox/vm.h>
30	#include <VBox/x86.h>
31	#include <VBox/pgm.h>
32	#include <VBox/pdm.h>
33	#include <VBox/err.h>
34	#include <VBox/log.h>
35	#include <VBox/selm.h>
36	#include <VBox/iom.h>
37	#include <iprt/param.h>
38	#include <iprt/assert.h>
39	#include <iprt/asm.h>
40	#include <iprt/string.h>
41	#include "HWVMXR0.h"
42
43	/*******************************************************************************
44	* Defined Constants And Macros *
45	*******************************************************************************/
46	#if defined(RT_ARCH_AMD64)
47	# define VMX_IS_64BIT_HOST_MODE() (true)
48	#elif defined(VBOX_WITH_HYBIRD_32BIT_KERNEL)
49	# define VMX_IS_64BIT_HOST_MODE() (g_fVMXIs64bitHost != 0)
50	#else
51	# define VMX_IS_64BIT_HOST_MODE() (false)
52	#endif
53
54	/*******************************************************************************
55	* Global Variables *
56	*******************************************************************************/
57	/* IO operation lookup arrays. */
58	static uint32_t const g_aIOSize[4] = {1, 2, 0, 4};
59	static uint32_t const g_aIOOpAnd[4] = {0xff, 0xffff, 0, 0xffffffff};
60
61	#ifdef VBOX_WITH_HYBIRD_32BIT_KERNEL
62	/** See HWACCMR0A.asm. */
63	extern "C" uint32_t g_fVMXIs64bitHost;
64	#endif
65
66	/*******************************************************************************
67	* Local Functions *
68	*******************************************************************************/
69	static void VMXR0ReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, int rc, PCPUMCTX pCtx);
70	static void vmxR0SetupTLBEPT(PVM pVM, PVMCPU pVCpu);
71	static void vmxR0SetupTLBVPID(PVM pVM, PVMCPU pVCpu);
72	static void vmxR0SetupTLBDummy(PVM pVM, PVMCPU pVCpu);
73	static void vmxR0FlushEPT(PVM pVM, PVMCPU pVCpu, VMX_FLUSH enmFlush, RTGCPHYS GCPhys);
74	static void vmxR0FlushVPID(PVM pVM, PVMCPU pVCpu, VMX_FLUSH enmFlush, RTGCPTR GCPtr);
75	static void vmxR0UpdateExceptionBitmap(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx);
76
77
78	static void VMXR0CheckError(PVM pVM, PVMCPU pVCpu, int rc)
79	{
80	if (rc == VERR_VMX_GENERIC)
81	{
82	RTCCUINTREG instrError;
83
84	VMXReadVMCS(VMX_VMCS32_RO_VM_INSTR_ERROR, &instrError);
85	pVCpu->hwaccm.s.vmx.lasterror.ulInstrError = instrError;
86	}
87	pVM->hwaccm.s.lLastError = rc;
88	}
89
90	/**
91	* Sets up and activates VT-x on the current CPU
92	*
93	* @returns VBox status code.
94	* @param pCpu CPU info struct
95	* @param pVM The VM to operate on. (can be NULL after a resume!!)
96	* @param pvPageCpu Pointer to the global cpu page
97	* @param pPageCpuPhys Physical address of the global cpu page
98	*/
99	VMMR0DECL(int) VMXR0EnableCpu(PHWACCM_CPUINFO pCpu, PVM pVM, void *pvPageCpu, RTHCPHYS pPageCpuPhys)
100	{
101	AssertReturn(pPageCpuPhys, VERR_INVALID_PARAMETER);
102	AssertReturn(pvPageCpu, VERR_INVALID_PARAMETER);
103
104	#ifdef LOG_ENABLED
105	SUPR0Printf("VMXR0EnableCpu cpu %d page (%x) %x\n", pCpu->idCpu, pvPageCpu, (uint32_t)pPageCpuPhys);
106	#endif
107	if (pVM)
108	{
109	/* Set revision dword at the beginning of the VMXON structure. */
110	(uint32_t )pvPageCpu = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hwaccm.s.vmx.msr.vmx_basic_info);
111	}
112
113	/** @todo we should unmap the two pages from the virtual address space in order to prevent accidental corruption.
114	* (which can have very bad consequences!!!)
115	*/
116
117	/* Make sure the VMX instructions don't cause #UD faults. */
118	ASMSetCR4(ASMGetCR4() \| X86_CR4_VMXE);
119
120	/* Enter VMX Root Mode */
121	int rc = VMXEnable(pPageCpuPhys);
122	if (RT_FAILURE(rc))
123	{
124	if (pVM)
125	VMXR0CheckError(pVM, &pVM->aCpus[0], rc);
126	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
127	return VERR_VMX_VMXON_FAILED;
128	}
129	return VINF_SUCCESS;
130	}
131
132	/**
133	* Deactivates VT-x on the current CPU
134	*
135	* @returns VBox status code.
136	* @param pCpu CPU info struct
137	* @param pvPageCpu Pointer to the global cpu page
138	* @param pPageCpuPhys Physical address of the global cpu page
139	*/
140	VMMR0DECL(int) VMXR0DisableCpu(PHWACCM_CPUINFO pCpu, void *pvPageCpu, RTHCPHYS pPageCpuPhys)
141	{
142	AssertReturn(pPageCpuPhys, VERR_INVALID_PARAMETER);
143	AssertReturn(pvPageCpu, VERR_INVALID_PARAMETER);
144
145	/* Leave VMX Root Mode. */
146	VMXDisable();
147
148	/* And clear the X86_CR4_VMXE bit */
149	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
150
151	#ifdef LOG_ENABLED
152	SUPR0Printf("VMXR0DisableCpu cpu %d\n", pCpu->idCpu);
153	#endif
154	return VINF_SUCCESS;
155	}
156
157	/**
158	* Does Ring-0 per VM VT-x init.
159	*
160	* @returns VBox status code.
161	* @param pVM The VM to operate on.
162	*/
163	VMMR0DECL(int) VMXR0InitVM(PVM pVM)
164	{
165	int rc;
166
167	#ifdef LOG_ENABLED
168	SUPR0Printf("VMXR0InitVM %x\n", pVM);
169	#endif
170
171	pVM->hwaccm.s.vmx.pMemObjAPIC = NIL_RTR0MEMOBJ;
172
173	if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
174	{
175	/* Allocate one page for the virtual APIC mmio cache. */
176	rc = RTR0MemObjAllocCont(&pVM->hwaccm.s.vmx.pMemObjAPIC, 1 << PAGE_SHIFT, true /* executable R0 mapping */);
177	AssertRC(rc);
178	if (RT_FAILURE(rc))
179	return rc;
180
181	pVM->hwaccm.s.vmx.pAPIC = (uint8_t *)RTR0MemObjAddress(pVM->hwaccm.s.vmx.pMemObjAPIC);
182	pVM->hwaccm.s.vmx.pAPICPhys = RTR0MemObjGetPagePhysAddr(pVM->hwaccm.s.vmx.pMemObjAPIC, 0);
183	ASMMemZero32(pVM->hwaccm.s.vmx.pAPIC, PAGE_SIZE);
184	}
185	else
186	{
187	pVM->hwaccm.s.vmx.pMemObjAPIC = 0;
188	pVM->hwaccm.s.vmx.pAPIC = 0;
189	pVM->hwaccm.s.vmx.pAPICPhys = 0;
190	}
191
192	/* Allocate the MSR bitmap if this feature is supported. */
193	if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
194	{
195	rc = RTR0MemObjAllocCont(&pVM->hwaccm.s.vmx.pMemObjMSRBitmap, 1 << PAGE_SHIFT, true /* executable R0 mapping */);
196	AssertRC(rc);
197	if (RT_FAILURE(rc))
198	return rc;
199
200	pVM->hwaccm.s.vmx.pMSRBitmap = (uint8_t *)RTR0MemObjAddress(pVM->hwaccm.s.vmx.pMemObjMSRBitmap);
201	pVM->hwaccm.s.vmx.pMSRBitmapPhys = RTR0MemObjGetPagePhysAddr(pVM->hwaccm.s.vmx.pMemObjMSRBitmap, 0);
202	memset(pVM->hwaccm.s.vmx.pMSRBitmap, 0xff, PAGE_SIZE);
203	}
204
205	/* Allocate VMCBs for all guest CPUs. */
206	for (unsigned i=0;i<pVM->cCPUs;i++)
207	{
208	PVMCPU pVCpu = &pVM->aCpus[i];
209
210	pVCpu->hwaccm.s.vmx.pMemObjVMCS = NIL_RTR0MEMOBJ;
211
212	/* Allocate one page for the VM control structure (VMCS). */
213	rc = RTR0MemObjAllocCont(&pVCpu->hwaccm.s.vmx.pMemObjVMCS, 1 << PAGE_SHIFT, true /* executable R0 mapping */);
214	AssertRC(rc);
215	if (RT_FAILURE(rc))
216	return rc;
217
218	pVCpu->hwaccm.s.vmx.pVMCS = RTR0MemObjAddress(pVCpu->hwaccm.s.vmx.pMemObjVMCS);
219	pVCpu->hwaccm.s.vmx.pVMCSPhys = RTR0MemObjGetPagePhysAddr(pVCpu->hwaccm.s.vmx.pMemObjVMCS, 0);
220	ASMMemZero32(pVCpu->hwaccm.s.vmx.pVMCS, PAGE_SIZE);
221
222	pVCpu->hwaccm.s.vmx.cr0_mask = 0;
223	pVCpu->hwaccm.s.vmx.cr4_mask = 0;
224
225	/* Current guest paging mode. */
226	pVCpu->hwaccm.s.vmx.enmCurrGuestMode = PGMMODE_REAL;
227
228	#ifdef LOG_ENABLED
229	SUPR0Printf("VMXR0InitVM %x VMCS=%x (%x)\n", pVM, pVCpu->hwaccm.s.vmx.pVMCS, (uint32_t)pVCpu->hwaccm.s.vmx.pVMCSPhys);
230	#endif
231	}
232
233	return VINF_SUCCESS;
234	}
235
236	/**
237	* Does Ring-0 per VM VT-x termination.
238	*
239	* @returns VBox status code.
240	* @param pVM The VM to operate on.
241	*/
242	VMMR0DECL(int) VMXR0TermVM(PVM pVM)
243	{
244	for (unsigned i=0;i<pVM->cCPUs;i++)
245	{
246	if (pVM->aCpus[i].hwaccm.s.vmx.pMemObjVMCS != NIL_RTR0MEMOBJ)
247	{
248	RTR0MemObjFree(pVM->aCpus[i].hwaccm.s.vmx.pMemObjVMCS, false);
249	pVM->aCpus[i].hwaccm.s.vmx.pMemObjVMCS = NIL_RTR0MEMOBJ;
250	pVM->aCpus[i].hwaccm.s.vmx.pVMCS = 0;
251	pVM->aCpus[i].hwaccm.s.vmx.pVMCSPhys = 0;
252	}
253	}
254	if (pVM->hwaccm.s.vmx.pMemObjAPIC != NIL_RTR0MEMOBJ)
255	{
256	RTR0MemObjFree(pVM->hwaccm.s.vmx.pMemObjAPIC, false);
257	pVM->hwaccm.s.vmx.pMemObjAPIC = NIL_RTR0MEMOBJ;
258	pVM->hwaccm.s.vmx.pAPIC = 0;
259	pVM->hwaccm.s.vmx.pAPICPhys = 0;
260	}
261	if (pVM->hwaccm.s.vmx.pMemObjMSRBitmap != NIL_RTR0MEMOBJ)
262	{
263	RTR0MemObjFree(pVM->hwaccm.s.vmx.pMemObjMSRBitmap, false);
264	pVM->hwaccm.s.vmx.pMemObjMSRBitmap = NIL_RTR0MEMOBJ;
265	pVM->hwaccm.s.vmx.pMSRBitmap = 0;
266	pVM->hwaccm.s.vmx.pMSRBitmapPhys = 0;
267	}
268	return VINF_SUCCESS;
269	}
270
271	/**
272	* Sets up VT-x for the specified VM
273	*
274	* @returns VBox status code.
275	* @param pVM The VM to operate on.
276	*/
277	VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
278	{
279	int rc = VINF_SUCCESS;
280	uint32_t val;
281
282	AssertReturn(pVM, VERR_INVALID_PARAMETER);
283
284	for (unsigned i=0;i<pVM->cCPUs;i++)
285	{
286	PVMCPU pVCpu = &pVM->aCpus[i];
287
288	Assert(pVCpu->hwaccm.s.vmx.pVMCS);
289
290	/* Set revision dword at the beginning of the VMCS structure. */
291	(uint32_t )pVCpu->hwaccm.s.vmx.pVMCS = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hwaccm.s.vmx.msr.vmx_basic_info);
292
293	/* Clear VM Control Structure. */
294	Log(("pVMCSPhys = %RHp\n", pVCpu->hwaccm.s.vmx.pVMCSPhys));
295	rc = VMXClearVMCS(pVCpu->hwaccm.s.vmx.pVMCSPhys);
296	if (RT_FAILURE(rc))
297	goto vmx_end;
298
299	/* Activate the VM Control Structure. */
300	rc = VMXActivateVMCS(pVCpu->hwaccm.s.vmx.pVMCSPhys);
301	if (RT_FAILURE(rc))
302	goto vmx_end;
303
304	/* VMX_VMCS_CTRL_PIN_EXEC_CONTROLS
305	* Set required bits to one and zero according to the MSR capabilities.
306	*/
307	val = pVM->hwaccm.s.vmx.msr.vmx_pin_ctls.n.disallowed0;
308	/* External and non-maskable interrupts cause VM-exits. */
309	val = val \| VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_EXT_INT_EXIT \| VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_NMI_EXIT;
310	val &= pVM->hwaccm.s.vmx.msr.vmx_pin_ctls.n.allowed1;
311
312	rc = VMXWriteVMCS(VMX_VMCS_CTRL_PIN_EXEC_CONTROLS, val);
313	AssertRC(rc);
314
315	/* VMX_VMCS_CTRL_PROC_EXEC_CONTROLS
316	* Set required bits to one and zero according to the MSR capabilities.
317	*/
318	val = pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.disallowed0;
319	/* Program which event cause VM-exits and which features we want to use. */
320	val = val \| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_HLT_EXIT
321	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_TSC_OFFSET
322	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT
323	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_UNCOND_IO_EXIT
324	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MWAIT_EXIT; /* don't execute mwait or else we'll idle inside the guest (host thinks the cpu load is high) */
325
326	/* Without nested paging we should intercept invlpg and cr3 mov instructions. */
327	if (!pVM->hwaccm.s.fNestedPaging)
328	val \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INVLPG_EXIT
329	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
330	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT;
331
332	/* Note: VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MWAIT_EXIT might cause a vmlaunch failure with an invalid control fields error. (combined with some other exit reasons) */
333	if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
334	{
335	/* CR8 reads from the APIC shadow page; writes cause an exit is they lower the TPR below the threshold */
336	val \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW;
337	Assert(pVM->hwaccm.s.vmx.pAPIC);
338	}
339	else
340	/* Exit on CR8 reads & writes in case the TPR shadow feature isn't present. */
341	val \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR8_STORE_EXIT \| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR8_LOAD_EXIT;
342
343	#ifdef VBOX_WITH_VTX_MSR_BITMAPS
344	if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
345	{
346	Assert(pVM->hwaccm.s.vmx.pMSRBitmapPhys);
347	val \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS;
348	}
349	#endif
350
351	/* We will use the secondary control if it's present. */
352	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
353
354	/* Mask away the bits that the CPU doesn't support */
355	/** @todo make sure they don't conflict with the above requirements. */
356	val &= pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1;
357	pVCpu->hwaccm.s.vmx.proc_ctls = val;
358
359	rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, val);
360	AssertRC(rc);
361
362	if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
363	{
364	/* VMX_VMCS_CTRL_PROC_EXEC_CONTROLS2
365	* Set required bits to one and zero according to the MSR capabilities.
366	*/
367	val = pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2.n.disallowed0;
368	val \|= VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT;
369
370	#ifdef HWACCM_VTX_WITH_EPT
371	if (pVM->hwaccm.s.fNestedPaging)
372	val \|= VMX_VMCS_CTRL_PROC_EXEC2_EPT;
373	#endif /* HWACCM_VTX_WITH_EPT */
374	#ifdef HWACCM_VTX_WITH_VPID
375	else
376	if (pVM->hwaccm.s.vmx.fVPID)
377	val \|= VMX_VMCS_CTRL_PROC_EXEC2_VPID;
378	#endif /* HWACCM_VTX_WITH_VPID */
379
380	/* Mask away the bits that the CPU doesn't support */
381	/** @todo make sure they don't conflict with the above requirements. */
382	val &= pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2.n.allowed1;
383
384	rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS2, val);
385	AssertRC(rc);
386	}
387
388	/* VMX_VMCS_CTRL_CR3_TARGET_COUNT
389	* Set required bits to one and zero according to the MSR capabilities.
390	*/
391	rc = VMXWriteVMCS(VMX_VMCS_CTRL_CR3_TARGET_COUNT, 0);
392	AssertRC(rc);
393
394	/* VMX_VMCS_CTRL_EXIT_CONTROLS
395	* Set required bits to one and zero according to the MSR capabilities.
396	*/
397	val = pVM->hwaccm.s.vmx.msr.vmx_exit.n.disallowed0;
398
399	/* Save debug controls (dr7 & IA32_DEBUGCTL_MSR) (forced to 1 on the 'first' VT-x capable CPUs; this actually includes the newest Nehalem CPUs) */
400	val \|= VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_DEBUG;
401	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBIRD_32BIT_KERNEL)
402	if (VMX_IS_64BIT_HOST_MODE())
403	val \|= VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_AMD64;
404	/* else: Must be zero when AMD64 is not available. */
405	#endif
406	val &= pVM->hwaccm.s.vmx.msr.vmx_exit.n.allowed1;
407	/* Don't acknowledge external interrupts on VM-exit. */
408	rc = VMXWriteVMCS(VMX_VMCS_CTRL_EXIT_CONTROLS, val);
409	AssertRC(rc);
410
411	/* Forward all exception except #NM & #PF to the guest.
412	* We always need to check pagefaults since our shadow page table can be out of sync.
413	* And we always lazily sync the FPU & XMM state.
414	*/
415
416	/** @todo Possible optimization:
417	* Keep the FPU and XMM state current in the EM thread. That way there's no need to
418	* lazily sync anything, but the downside is that we can't use the FPU stack or XMM
419	* registers ourselves of course.
420	*
421	* Note: only possible if the current state is actually ours (X86_CR0_TS flag)
422	*/
423
424	/* Don't filter page faults; all of them should cause a switch. */
425	rc = VMXWriteVMCS(VMX_VMCS_CTRL_PAGEFAULT_ERROR_MASK, 0);
426	rc \|= VMXWriteVMCS(VMX_VMCS_CTRL_PAGEFAULT_ERROR_MATCH, 0);
427	AssertRC(rc);
428
429	/* Init TSC offset to zero. */
430	rc = VMXWriteVMCS64(VMX_VMCS_CTRL_TSC_OFFSET_FULL, 0);
431	AssertRC(rc);
432
433	rc = VMXWriteVMCS64(VMX_VMCS_CTRL_IO_BITMAP_A_FULL, 0);
434	AssertRC(rc);
435
436	rc = VMXWriteVMCS64(VMX_VMCS_CTRL_IO_BITMAP_B_FULL, 0);
437	AssertRC(rc);
438
439	/* Set the MSR bitmap address. */
440	if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
441	{
442	/* Optional */
443	rc = VMXWriteVMCS64(VMX_VMCS_CTRL_MSR_BITMAP_FULL, pVM->hwaccm.s.vmx.pMSRBitmapPhys);
444	AssertRC(rc);
445	}
446
447	/* Clear MSR controls. */
448	rc = VMXWriteVMCS64(VMX_VMCS_CTRL_VMEXIT_MSR_STORE_FULL, 0);
449	rc \|= VMXWriteVMCS64(VMX_VMCS_CTRL_VMEXIT_MSR_LOAD_FULL, 0);
450	rc \|= VMXWriteVMCS64(VMX_VMCS_CTRL_VMENTRY_MSR_LOAD_FULL, 0);
451	rc \|= VMXWriteVMCS(VMX_VMCS_CTRL_EXIT_MSR_STORE_COUNT, 0);
452	rc \|= VMXWriteVMCS(VMX_VMCS_CTRL_EXIT_MSR_LOAD_COUNT, 0);
453	AssertRC(rc);
454
455	if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
456	{
457	Assert(pVM->hwaccm.s.vmx.pMemObjAPIC);
458	/* Optional */
459	rc = VMXWriteVMCS(VMX_VMCS_CTRL_TPR_THRESHOLD, 0);
460	rc \|= VMXWriteVMCS64(VMX_VMCS_CTRL_VAPIC_PAGEADDR_FULL, pVM->hwaccm.s.vmx.pAPICPhys);
461	AssertRC(rc);
462	}
463
464	/* Set link pointer to -1. Not currently used. */
465	rc = VMXWriteVMCS64(VMX_VMCS_GUEST_LINK_PTR_FULL, 0xFFFFFFFFFFFFFFFFULL);
466	AssertRC(rc);
467
468	/* Clear VM Control Structure. Marking it inactive, clearing implementation specific data and writing back VMCS data to memory. */
469	rc = VMXClearVMCS(pVCpu->hwaccm.s.vmx.pVMCSPhys);
470	AssertRC(rc);
471
472	/* Configure the VMCS read cache. */
473	PVMCSCACHE pCache = &pVCpu->hwaccm.s.vmx.VMCSCache;
474
475	VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_RIP);
476	VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_RSP);
477	VMXSetupCachedReadVMCS(pCache, VMX_VMCS_GUEST_RFLAGS);
478	VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE);
479	VMXSetupCachedReadVMCS(pCache, VMX_VMCS_CTRL_CR0_READ_SHADOW);
480	VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_CR0);
481	VMXSetupCachedReadVMCS(pCache, VMX_VMCS_CTRL_CR4_READ_SHADOW);
482	VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_CR4);
483	VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_DR7);
484	VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_GUEST_SYSENTER_CS);
485	VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_SYSENTER_EIP);
486	VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_SYSENTER_ESP);
487	VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_GUEST_GDTR_LIMIT);
488	VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_GDTR_BASE);
489	VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_GUEST_IDTR_LIMIT);
490	VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_IDTR_BASE);
491
492	VMX_SETUP_SELREG(ES, pCache);
493	VMX_SETUP_SELREG(SS, pCache);
494	VMX_SETUP_SELREG(CS, pCache);
495	VMX_SETUP_SELREG(DS, pCache);
496	VMX_SETUP_SELREG(FS, pCache);
497	VMX_SETUP_SELREG(GS, pCache);
498	VMX_SETUP_SELREG(LDTR, pCache);
499	VMX_SETUP_SELREG(TR, pCache);
500
501	/* Status code VMCS reads. */
502	VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_EXIT_REASON);
503	VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_VM_INSTR_ERROR);
504	VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_EXIT_INSTR_LENGTH);
505	VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_EXIT_INTERRUPTION_ERRCODE);
506	VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO);
507	VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_EXIT_INSTR_INFO);
508	VMXSetupCachedReadVMCS(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
509	VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_IDT_INFO);
510	VMXSetupCachedReadVMCS(pCache, VMX_VMCS32_RO_IDT_ERRCODE);
511
512	if (pVM->hwaccm.s.fNestedPaging)
513	{
514	VMXSetupCachedReadVMCS(pCache, VMX_VMCS64_GUEST_CR3);
515	VMXSetupCachedReadVMCS(pCache, VMX_VMCS_EXIT_PHYS_ADDR_FULL);
516	pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
517	}
518	else
519	pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
520	} /* for each VMCPU */
521
522	/* Choose the right TLB setup function. */
523	if (pVM->hwaccm.s.fNestedPaging)
524	{
525	pVM->hwaccm.s.vmx.pfnSetupTaggedTLB = vmxR0SetupTLBEPT;
526
527	/* Default values for flushing. */
528	pVM->hwaccm.s.vmx.enmFlushPage = VMX_FLUSH_ALL_CONTEXTS;
529	pVM->hwaccm.s.vmx.enmFlushContext = VMX_FLUSH_ALL_CONTEXTS;
530
531	/* If the capabilities specify we can do more, then make use of it. */
532	if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVEPT_CAPS_INDIV)
533	pVM->hwaccm.s.vmx.enmFlushPage = VMX_FLUSH_PAGE;
534	else
535	if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVEPT_CAPS_CONTEXT)
536	pVM->hwaccm.s.vmx.enmFlushPage = VMX_FLUSH_SINGLE_CONTEXT;
537
538	if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVEPT_CAPS_CONTEXT)
539	pVM->hwaccm.s.vmx.enmFlushContext = VMX_FLUSH_SINGLE_CONTEXT;
540	}
541	#ifdef HWACCM_VTX_WITH_VPID
542	else
543	if (pVM->hwaccm.s.vmx.fVPID)
544	{
545	pVM->hwaccm.s.vmx.pfnSetupTaggedTLB = vmxR0SetupTLBVPID;
546
547	/* Default values for flushing. */
548	pVM->hwaccm.s.vmx.enmFlushPage = VMX_FLUSH_ALL_CONTEXTS;
549	pVM->hwaccm.s.vmx.enmFlushContext = VMX_FLUSH_ALL_CONTEXTS;
550
551	/* If the capabilities specify we can do more, then make use of it. */
552	if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVVPID_CAPS_INDIV)
553	pVM->hwaccm.s.vmx.enmFlushPage = VMX_FLUSH_PAGE;
554	else
555	if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVVPID_CAPS_CONTEXT)
556	pVM->hwaccm.s.vmx.enmFlushPage = VMX_FLUSH_SINGLE_CONTEXT;
557
558	if (pVM->hwaccm.s.vmx.msr.vmx_eptcaps & MSR_IA32_VMX_EPT_CAPS_INVVPID_CAPS_CONTEXT)
559	pVM->hwaccm.s.vmx.enmFlushContext = VMX_FLUSH_SINGLE_CONTEXT;
560	}
561	#endif /* HWACCM_VTX_WITH_VPID */
562	else
563	pVM->hwaccm.s.vmx.pfnSetupTaggedTLB = vmxR0SetupTLBDummy;
564
565	vmx_end:
566	VMXR0CheckError(pVM, &pVM->aCpus[0], rc);
567	return rc;
568	}
569
570
571	/**
572	* Injects an event (trap or external interrupt)
573	*
574	* @returns VBox status code.
575	* @param pVM The VM to operate on.
576	* @param pVCpu The VMCPU to operate on.
577	* @param pCtx CPU Context
578	* @param intInfo VMX interrupt info
579	* @param cbInstr Opcode length of faulting instruction
580	* @param errCode Error code (optional)
581	*/
582	static int VMXR0InjectEvent(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, uint32_t intInfo, uint32_t cbInstr, uint32_t errCode)
583	{
584	int rc;
585	uint32_t iGate = VMX_EXIT_INTERRUPTION_INFO_VECTOR(intInfo);
586
587	#ifdef VBOX_STRICT
588	if (iGate == 0xE)
589	LogFlow(("VMXR0InjectEvent: Injecting interrupt %d at %RGv error code=%08x CR2=%08x intInfo=%08x\n", iGate, (RTGCPTR)pCtx->rip, errCode, pCtx->cr2, intInfo));
590	else
591	if (iGate < 0x20)
592	LogFlow(("VMXR0InjectEvent: Injecting interrupt %d at %RGv error code=%08x\n", iGate, (RTGCPTR)pCtx->rip, errCode));
593	else
594	{
595	LogFlow(("INJ-EI: %x at %RGv\n", iGate, (RTGCPTR)pCtx->rip));
596	Assert(!VM_FF_ISSET(pVM, VM_FF_INHIBIT_INTERRUPTS));
597	Assert(pCtx->eflags.u32 & X86_EFL_IF);
598	}
599	#endif
600
601	#ifdef HWACCM_VMX_EMULATE_REALMODE
602	if (CPUMIsGuestInRealModeEx(pCtx))
603	{
604	RTGCPHYS GCPhysHandler;
605	uint16_t offset, ip;
606	RTSEL sel;
607
608	/* Injecting events doesn't work right with real mode emulation.
609	* (#GP if we try to inject external hardware interrupts)
610	* Inject the interrupt or trap directly instead.
611	*/
612	Log(("Manual interrupt/trap '%x' inject (real mode)\n", iGate));
613
614	/* Check if the interrupt handler is present. */
615	if (iGate * 4 + 3 > pCtx->idtr.cbIdt)
616	{
617	Log(("IDT cbIdt violation\n"));
618	if (iGate != X86_XCPT_DF)
619	{
620	RTGCUINTPTR intInfo;
621
622	intInfo = (iGate == X86_XCPT_GP) ? (uint32_t)X86_XCPT_DF : iGate;
623	intInfo \|= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
624	intInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
625	intInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HWEXCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
626
627	return VMXR0InjectEvent(pVM, pVCpu, pCtx, intInfo, 0, 0 /* no error code according to the Intel docs */);
628	}
629	Log(("Triple fault -> reset the VM!\n"));
630	return VINF_EM_RESET;
631	}
632	if ( VMX_EXIT_INTERRUPTION_INFO_TYPE(intInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW
633	\|\| iGate == 3 /* Both #BP and #OF point to the instruction after. */
634	\|\| iGate == 4)
635	{
636	ip = pCtx->ip + cbInstr;
637	}
638	else
639	ip = pCtx->ip;
640
641	/* Read the selector:offset pair of the interrupt handler. */
642	GCPhysHandler = (RTGCPHYS)pCtx->idtr.pIdt + iGate * 4;
643	PGMPhysRead(pVM, GCPhysHandler, &offset, sizeof(offset));
644	PGMPhysRead(pVM, GCPhysHandler + 2, &sel, sizeof(sel));
645
646	LogFlow(("IDT handler %04X:%04X\n", sel, offset));
647
648	/* Construct the stack frame. */
649	/** @todo should check stack limit. */
650	pCtx->sp -= 2;
651	LogFlow(("ss:sp %04X:%04X eflags=%x\n", pCtx->ss, pCtx->sp, pCtx->eflags.u));
652	PGMPhysWrite(pVM, pCtx->ssHid.u64Base + pCtx->sp, &pCtx->eflags, sizeof(uint16_t));
653	pCtx->sp -= 2;
654	LogFlow(("ss:sp %04X:%04X cs=%x\n", pCtx->ss, pCtx->sp, pCtx->cs));
655	PGMPhysWrite(pVM, pCtx->ssHid.u64Base + pCtx->sp, &pCtx->cs, sizeof(uint16_t));
656	pCtx->sp -= 2;
657	LogFlow(("ss:sp %04X:%04X ip=%x\n", pCtx->ss, pCtx->sp, ip));
658	PGMPhysWrite(pVM, pCtx->ssHid.u64Base + pCtx->sp, &ip, sizeof(ip));
659
660	/* Update the CPU state for executing the handler. */
661	pCtx->rip = offset;
662	pCtx->cs = sel;
663	pCtx->csHid.u64Base = sel << 4;
664	pCtx->eflags.u &= ~(X86_EFL_IF\|X86_EFL_TF\|X86_EFL_RF\|X86_EFL_AC);
665
666	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_GUEST_SEGMENT_REGS;
667	return VINF_SUCCESS;
668	}
669	#endif /* HWACCM_VMX_EMULATE_REALMODE */
670
671	/* Set event injection state. */
672	rc = VMXWriteCachedVMCS(VMX_VMCS_CTRL_ENTRY_IRQ_INFO, intInfo \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT));
673
674	rc \|= VMXWriteCachedVMCS(VMX_VMCS_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
675	rc \|= VMXWriteCachedVMCS(VMX_VMCS_CTRL_ENTRY_EXCEPTION_ERRCODE, errCode);
676
677	AssertRC(rc);
678	return rc;
679	}
680
681
682	/**
683	* Checks for pending guest interrupts and injects them
684	*
685	* @returns VBox status code.
686	* @param pVM The VM to operate on.
687	* @param pVCpu The VMCPU to operate on.
688	* @param pCtx CPU Context
689	*/
690	static int VMXR0CheckPendingInterrupt(PVM pVM, PVMCPU pVCpu, CPUMCTX *pCtx)
691	{
692	int rc;
693
694	/* Dispatch any pending interrupts. (injected before, but a VM exit occurred prematurely) */
695	if (pVCpu->hwaccm.s.Event.fPending)
696	{
697	Log(("Reinjecting event %RX64 %08x at %RGv cr2=%RX64\n", pVCpu->hwaccm.s.Event.intInfo, pVCpu->hwaccm.s.Event.errCode, (RTGCPTR)pCtx->rip, pCtx->cr2));
698	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatIntReinject);
699	rc = VMXR0InjectEvent(pVM, pVCpu, pCtx, pVCpu->hwaccm.s.Event.intInfo, 0, pVCpu->hwaccm.s.Event.errCode);
700	AssertRC(rc);
701
702	pVCpu->hwaccm.s.Event.fPending = false;
703	return VINF_SUCCESS;
704	}
705
706	if (pVM->hwaccm.s.fInjectNMI)
707	{
708	RTGCUINTPTR intInfo;
709
710	intInfo = X86_XCPT_NMI;
711	intInfo \|= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
712	intInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
713
714	rc = VMXR0InjectEvent(pVM, pVCpu, pCtx, intInfo, 0, 0);
715	AssertRC(rc);
716
717	pVM->hwaccm.s.fInjectNMI = false;
718	return VINF_SUCCESS;
719	}
720
721	/* When external interrupts are pending, we should exit the VM when IF is set. */
722	if ( !TRPMHasTrap(pVM)
723	&& VM_FF_ISPENDING(pVM, (VM_FF_INTERRUPT_APIC\|VM_FF_INTERRUPT_PIC)))
724	{
725	if (!(pCtx->eflags.u32 & X86_EFL_IF))
726	{
727	if (!(pVCpu->hwaccm.s.vmx.proc_ctls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_IRQ_WINDOW_EXIT))
728	{
729	LogFlow(("Enable irq window exit!\n"));
730	pVCpu->hwaccm.s.vmx.proc_ctls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_IRQ_WINDOW_EXIT;
731	rc = VMXWriteCachedVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
732	AssertRC(rc);
733	}
734	/* else nothing to do but wait */
735	}
736	else
737	if (!VM_FF_ISSET(pVM, VM_FF_INHIBIT_INTERRUPTS))
738	{
739	uint8_t u8Interrupt;
740
741	rc = PDMGetInterrupt(pVM, &u8Interrupt);
742	Log(("Dispatch interrupt: u8Interrupt=%x (%d) rc=%Rrc cs:rip=%04X:%RGv\n", u8Interrupt, u8Interrupt, rc, pCtx->cs, (RTGCPTR)pCtx->rip));
743	if (RT_SUCCESS(rc))
744	{
745	rc = TRPMAssertTrap(pVM, u8Interrupt, TRPM_HARDWARE_INT);
746	AssertRC(rc);
747	}
748	else
749	{
750	/* Can only happen in rare cases where a pending interrupt is cleared behind our back */
751	Assert(!VM_FF_ISPENDING(pVM, (VM_FF_INTERRUPT_APIC\|VM_FF_INTERRUPT_PIC)));
752	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatSwitchGuestIrq);
753	/* Just continue */
754	}
755	}
756	else
757	Log(("Pending interrupt blocked at %RGv by VM_FF_INHIBIT_INTERRUPTS!!\n", (RTGCPTR)pCtx->rip));
758	}
759
760	#ifdef VBOX_STRICT
761	if (TRPMHasTrap(pVM))
762	{
763	uint8_t u8Vector;
764	rc = TRPMQueryTrapAll(pVM, &u8Vector, 0, 0, 0);
765	AssertRC(rc);
766	}
767	#endif
768
769	if ( pCtx->eflags.u32 & X86_EFL_IF
770	&& (!VM_FF_ISSET(pVM, VM_FF_INHIBIT_INTERRUPTS))
771	&& TRPMHasTrap(pVM)
772	)
773	{
774	uint8_t u8Vector;
775	int rc;
776	TRPMEVENT enmType;
777	RTGCUINTPTR intInfo;
778	RTGCUINT errCode;
779
780	/* If a new event is pending, then dispatch it now. */
781	rc = TRPMQueryTrapAll(pVM, &u8Vector, &enmType, &errCode, 0);
782	AssertRC(rc);
783	Assert(pCtx->eflags.Bits.u1IF == 1 \|\| enmType == TRPM_TRAP);
784	Assert(enmType != TRPM_SOFTWARE_INT);
785
786	/* Clear the pending trap. */
787	rc = TRPMResetTrap(pVM);
788	AssertRC(rc);
789
790	intInfo = u8Vector;
791	intInfo \|= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
792
793	if (enmType == TRPM_TRAP)
794	{
795	switch (u8Vector) {
796	case 8:
797	case 10:
798	case 11:
799	case 12:
800	case 13:
801	case 14:
802	case 17:
803	/* Valid error codes. */
804	intInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
805	break;
806	default:
807	break;
808	}
809	if (u8Vector == X86_XCPT_BP \|\| u8Vector == X86_XCPT_OF)
810	intInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SWEXCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
811	else
812	intInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HWEXCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
813	}
814	else
815	intInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
816
817	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatIntInject);
818	rc = VMXR0InjectEvent(pVM, pVCpu, pCtx, intInfo, 0, errCode);
819	AssertRC(rc);
820	} /* if (interrupts can be dispatched) */
821
822	return VINF_SUCCESS;
823	}
824
825	/**
826	* Save the host state
827	*
828	* @returns VBox status code.
829	* @param pVM The VM to operate on.
830	* @param pVCpu The VMCPU to operate on.
831	*/
832	VMMR0DECL(int) VMXR0SaveHostState(PVM pVM, PVMCPU pVCpu)
833	{
834	int rc = VINF_SUCCESS;
835
836	/*
837	* Host CPU Context
838	*/
839	if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_HOST_CONTEXT)
840	{
841	RTIDTR idtr;
842	RTGDTR gdtr;
843	RTSEL SelTR;
844	PX86DESCHC pDesc;
845	uintptr_t trBase;
846	RTSEL cs;
847	RTSEL ss;
848	uint64_t cr3;
849
850	/* Control registers */
851	rc = VMXWriteCachedVMCS(VMX_VMCS_HOST_CR0, ASMGetCR0());
852	#ifdef VBOX_WITH_HYBIRD_32BIT_KERNEL
853	if (VMX_IS_64BIT_HOST_MODE())
854	{
855	cr3 = hwaccmR0Get64bitCR3();
856	rc \|= VMXWriteCachedVMCS64(VMX_VMCS_HOST_CR3, cr3);
857	}
858	else
859	#endif
860	{
861	cr3 = ASMGetCR3();
862	rc \|= VMXWriteCachedVMCS(VMX_VMCS_HOST_CR3, cr3);
863	}
864	rc \|= VMXWriteCachedVMCS(VMX_VMCS_HOST_CR4, ASMGetCR4());
865	AssertRC(rc);
866	Log2(("VMX_VMCS_HOST_CR0 %08x\n", ASMGetCR0()));
867	Log2(("VMX_VMCS_HOST_CR3 %08RX64\n", cr3));
868	Log2(("VMX_VMCS_HOST_CR4 %08x\n", ASMGetCR4()));
869
870	/* Selector registers. */
871	#ifdef VBOX_WITH_HYBIRD_32BIT_KERNEL
872	if (VMX_IS_64BIT_HOST_MODE())
873	{
874	cs = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelCS;
875	ss = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelSS;
876	}
877	else
878	{
879	/* sysenter loads LDT cs & ss, VMX doesn't like this. Load the GDT ones (safe). */
880	cs = (RTSEL)(uintptr_t)&SUPR0AbsKernelCS;
881	ss = (RTSEL)(uintptr_t)&SUPR0AbsKernelSS;
882	}
883	#else
884	cs = ASMGetCS();
885	ss = ASMGetSS();
886	#endif
887	Assert(!(cs & X86_SEL_LDT)); Assert((cs & X86_SEL_RPL) == 0);
888	Assert(!(ss & X86_SEL_LDT)); Assert((ss & X86_SEL_RPL) == 0);
889	rc = VMXWriteCachedVMCS(VMX_VMCS16_HOST_FIELD_CS, cs);
890	/* Note: VMX is (again) very picky about the RPL of the selectors here; we'll restore them manually. */
891	rc \|= VMXWriteCachedVMCS(VMX_VMCS16_HOST_FIELD_DS, 0);
892	rc \|= VMXWriteCachedVMCS(VMX_VMCS16_HOST_FIELD_ES, 0);
893	#if HC_ARCH_BITS == 32
894	if (!VMX_IS_64BIT_HOST_MODE())
895	{
896	rc \|= VMXWriteCachedVMCS(VMX_VMCS16_HOST_FIELD_FS, 0);
897	rc \|= VMXWriteCachedVMCS(VMX_VMCS16_HOST_FIELD_GS, 0);
898	}
899	#endif
900	rc \|= VMXWriteCachedVMCS(VMX_VMCS16_HOST_FIELD_SS, ss);
901	SelTR = ASMGetTR();
902	rc \|= VMXWriteCachedVMCS(VMX_VMCS16_HOST_FIELD_TR, SelTR);
903	AssertRC(rc);
904	Log2(("VMX_VMCS_HOST_FIELD_CS %08x (%08x)\n", cs, ASMGetSS()));
905	Log2(("VMX_VMCS_HOST_FIELD_DS 00000000 (%08x)\n", ASMGetDS()));
906	Log2(("VMX_VMCS_HOST_FIELD_ES 00000000 (%08x)\n", ASMGetES()));
907	Log2(("VMX_VMCS_HOST_FIELD_FS 00000000 (%08x)\n", ASMGetFS()));
908	Log2(("VMX_VMCS_HOST_FIELD_GS 00000000 (%08x)\n", ASMGetGS()));
909	Log2(("VMX_VMCS_HOST_FIELD_SS %08x (%08x)\n", ss, ASMGetSS()));
910	Log2(("VMX_VMCS_HOST_FIELD_TR %08x\n", ASMGetTR()));
911
912	/* GDTR & IDTR */
913	#ifdef VBOX_WITH_HYBIRD_32BIT_KERNEL
914	if (VMX_IS_64BIT_HOST_MODE())
915	{
916	X86XDTR64 gdtr64, idtr64;
917	hwaccmR0Get64bitGDTRandIDTR(&gdtr64, &idtr64);
918	rc = VMXWriteCachedVMCS64(VMX_VMCS_HOST_GDTR_BASE, gdtr64.uAddr);
919	rc \|= VMXWriteCachedVMCS64(VMX_VMCS_HOST_IDTR_BASE, gdtr64.uAddr);
920	AssertRC(rc);
921	Log2(("VMX_VMCS_HOST_GDTR_BASE %RX64\n", gdtr64.uAddr));
922	Log2(("VMX_VMCS_HOST_IDTR_BASE %RX64\n", idtr64.uAddr));
923	gdtr.cbGdt = gdtr64.cb;
924	gdtr.pGdt = (uintptr_t)gdtr64.uAddr;
925	}
926	else
927	#endif
928	{
929	ASMGetGDTR(&gdtr);
930	rc = VMXWriteCachedVMCS(VMX_VMCS_HOST_GDTR_BASE, gdtr.pGdt);
931	ASMGetIDTR(&idtr);
932	rc \|= VMXWriteCachedVMCS(VMX_VMCS_HOST_IDTR_BASE, idtr.pIdt);
933	AssertRC(rc);
934	Log2(("VMX_VMCS_HOST_GDTR_BASE %RHv\n", gdtr.pGdt));
935	Log2(("VMX_VMCS_HOST_IDTR_BASE %RHv\n", idtr.pIdt));
936	}
937
938
939	/* Save the base address of the TR selector. */
940	if (SelTR > gdtr.cbGdt)
941	{
942	AssertMsgFailed(("Invalid TR selector %x. GDTR.cbGdt=%x\n", SelTR, gdtr.cbGdt));
943	return VERR_VMX_INVALID_HOST_STATE;
944	}
945
946	#ifdef VBOX_WITH_HYBIRD_32BIT_KERNEL
947	if (VMX_IS_64BIT_HOST_MODE())
948	{
949	pDesc = &((PX86DESCHC)gdtr.pGdt)[SelTR >> X86_SEL_SHIFT_HC]; /// ????
950	uint64_t trBase64 = X86DESC64_BASE(*(PX86DESC64)pDesc);
951	rc = VMXWriteCachedVMCS64(VMX_VMCS_HOST_TR_BASE, trBase64);
952	Log2(("VMX_VMCS_HOST_TR_BASE %RX64\n", trBase64));
953	AssertRC(rc);
954	}
955	else
956	#endif
957	{
958	pDesc = &((PX86DESCHC)gdtr.pGdt)[SelTR >> X86_SEL_SHIFT_HC];
959	#if HC_ARCH_BITS == 64
960	trBase = X86DESC64_BASE(*pDesc);
961	#else
962	trBase = X86DESC_BASE(*pDesc);
963	#endif
964	rc = VMXWriteCachedVMCS(VMX_VMCS_HOST_TR_BASE, trBase);
965	AssertRC(rc);
966	Log2(("VMX_VMCS_HOST_TR_BASE %RHv\n", trBase));
967	}
968
969	/* FS and GS base. */
970	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBIRD_32BIT_KERNEL)
971	if (VMX_IS_64BIT_HOST_MODE())
972	{
973	Log2(("MSR_K8_FS_BASE = %RX64\n", ASMRdMsr(MSR_K8_FS_BASE)));
974	Log2(("MSR_K8_GS_BASE = %RX64\n", ASMRdMsr(MSR_K8_GS_BASE)));
975	rc = VMXWriteCachedVMCS64(VMX_VMCS_HOST_FS_BASE, ASMRdMsr(MSR_K8_FS_BASE));
976	rc \|= VMXWriteCachedVMCS64(VMX_VMCS_HOST_GS_BASE, ASMRdMsr(MSR_K8_GS_BASE));
977	}
978	#endif
979	AssertRC(rc);
980
981	/* Sysenter MSRs. */
982	/** @todo expensive!! */
983	rc = VMXWriteCachedVMCS(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
984	Log2(("VMX_VMCS_HOST_SYSENTER_CS %08x\n", ASMRdMsr_Low(MSR_IA32_SYSENTER_CS)));
985	#ifdef VBOX_WITH_HYBIRD_32BIT_KERNEL
986	if (VMX_IS_64BIT_HOST_MODE())
987	{
988	Log2(("VMX_VMCS_HOST_SYSENTER_EIP %RX64\n", ASMRdMsr(MSR_IA32_SYSENTER_EIP)));
989	Log2(("VMX_VMCS_HOST_SYSENTER_ESP %RX64\n", ASMRdMsr(MSR_IA32_SYSENTER_ESP)));
990	rc \|= VMXWriteCachedVMCS64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
991	rc \|= VMXWriteCachedVMCS64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
992	}
993	else
994	{
995	rc \|= VMXWriteCachedVMCS(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
996	rc \|= VMXWriteCachedVMCS(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
997	Log2(("VMX_VMCS_HOST_SYSENTER_EIP %RX32\n", ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP)));
998	Log2(("VMX_VMCS_HOST_SYSENTER_ESP %RX32\n", ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP)));
999	}
1000	#elif HC_ARCH_BITS == 32
1001	rc \|= VMXWriteCachedVMCS(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
1002	rc \|= VMXWriteCachedVMCS(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
1003	Log2(("VMX_VMCS_HOST_SYSENTER_EIP %RX32\n", ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP)));
1004	Log2(("VMX_VMCS_HOST_SYSENTER_ESP %RX32\n", ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP)));
1005	#else
1006	Log2(("VMX_VMCS_HOST_SYSENTER_EIP %RX64\n", ASMRdMsr(MSR_IA32_SYSENTER_EIP)));
1007	Log2(("VMX_VMCS_HOST_SYSENTER_ESP %RX64\n", ASMRdMsr(MSR_IA32_SYSENTER_ESP)));
1008	rc \|= VMXWriteCachedVMCS64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
1009	rc \|= VMXWriteCachedVMCS64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
1010	#endif
1011	AssertRC(rc);
1012
1013	pVCpu->hwaccm.s.fContextUseFlags &= ~HWACCM_CHANGED_HOST_CONTEXT;
1014	}
1015	return rc;
1016	}
1017
1018	/**
1019	* Prefetch the 4 PDPT pointers (PAE and nested paging only)
1020	*
1021	* @param pVM The VM to operate on.
1022	* @param pVCpu The VMCPU to operate on.
1023	* @param pCtx Guest context
1024	*/
1025	static void vmxR0PrefetchPAEPdptrs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
1026	{
1027	if (CPUMIsGuestInPAEModeEx(pCtx))
1028	{
1029	X86PDPE Pdpe;
1030
1031	for (unsigned i=0;i<4;i++)
1032	{
1033	Pdpe = PGMGstGetPaePDPtr(pVM, i);
1034	int rc = VMXWriteCachedVMCS64(VMX_VMCS_GUEST_PDPTR0_FULL + i*2, Pdpe.u);
1035	AssertRC(rc);
1036	}
1037	}
1038	}
1039
1040	/**
1041	* Update the exception bitmap according to the current CPU state
1042	*
1043	* @param pVM The VM to operate on.
1044	* @param pVCpu The VMCPU to operate on.
1045	* @param pCtx Guest context
1046	*/
1047	static void vmxR0UpdateExceptionBitmap(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
1048	{
1049	uint32_t u32TrapMask;
1050	Assert(pCtx);
1051
1052	u32TrapMask = HWACCM_VMX_TRAP_MASK;
1053	#ifndef DEBUG
1054	if (pVM->hwaccm.s.fNestedPaging)
1055	u32TrapMask &= ~RT_BIT(X86_XCPT_PF); /* no longer need to intercept #PF. */
1056	#endif
1057
1058	/* Also catch floating point exceptions as we need to report them to the guest in a different way. */
1059	if ( CPUMIsGuestFPUStateActive(pVCpu) == true
1060	&& !(pCtx->cr0 & X86_CR0_NE)
1061	&& !pVCpu->hwaccm.s.fFPUOldStyleOverride)
1062	{
1063	u32TrapMask \|= RT_BIT(X86_XCPT_MF);
1064	pVCpu->hwaccm.s.fFPUOldStyleOverride = true;
1065	}
1066
1067	#ifdef DEBUG
1068	/* Intercept X86_XCPT_DB if stepping is enabled */
1069	if (DBGFIsStepping(pVM))
1070	u32TrapMask \|= RT_BIT(X86_XCPT_DB);
1071	#endif
1072
1073	#ifdef VBOX_STRICT
1074	Assert(u32TrapMask & RT_BIT(X86_XCPT_GP));
1075	#endif
1076
1077	# ifdef HWACCM_VMX_EMULATE_REALMODE
1078	/* Intercept all exceptions in real mode as none of them can be injected directly (#GP otherwise). */
1079	if (CPUMIsGuestInRealModeEx(pCtx))
1080	u32TrapMask \|= HWACCM_VMX_TRAP_MASK_REALMODE;
1081	# endif /* HWACCM_VMX_EMULATE_REALMODE */
1082
1083	int rc = VMXWriteCachedVMCS(VMX_VMCS_CTRL_EXCEPTION_BITMAP, u32TrapMask);
1084	AssertRC(rc);
1085	}
1086
1087	/**
1088	* Loads the guest state
1089	*
1090	* NOTE: Don't do anything here that can cause a jump back to ring 3!!!!!
1091	*
1092	* @returns VBox status code.
1093	* @param pVM The VM to operate on.
1094	* @param pVCpu The VMCPU to operate on.
1095	* @param pCtx Guest context
1096	*/
1097	VMMR0DECL(int) VMXR0LoadGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
1098	{
1099	int rc = VINF_SUCCESS;
1100	RTGCUINTPTR val;
1101	X86EFLAGS eflags;
1102
1103	/* Guest CPU context: ES, CS, SS, DS, FS, GS. */
1104	if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_SEGMENT_REGS)
1105	{
1106	#ifdef HWACCM_VMX_EMULATE_REALMODE
1107	PGMMODE enmGuestMode = PGMGetGuestMode(pVM);
1108	if (pVCpu->hwaccm.s.vmx.enmCurrGuestMode != enmGuestMode)
1109	{
1110	/* Correct weird requirements for switching to protected mode. */
1111	if ( pVCpu->hwaccm.s.vmx.enmCurrGuestMode == PGMMODE_REAL
1112	&& enmGuestMode >= PGMMODE_PROTECTED)
1113	{
1114	/* DPL of all hidden selector registers must match the current CPL (0). */
1115	pCtx->csHid.Attr.n.u2Dpl = 0;
1116	pCtx->csHid.Attr.n.u4Type = X86_SEL_TYPE_CODE \| X86_SEL_TYPE_RW_ACC;
1117
1118	pCtx->dsHid.Attr.n.u2Dpl = 0;
1119	pCtx->esHid.Attr.n.u2Dpl = 0;
1120	pCtx->fsHid.Attr.n.u2Dpl = 0;
1121	pCtx->gsHid.Attr.n.u2Dpl = 0;
1122	pCtx->ssHid.Attr.n.u2Dpl = 0;
1123	}
1124	else
1125	/* Switching from protected mode to real mode. */
1126	if ( pVCpu->hwaccm.s.vmx.enmCurrGuestMode >= PGMMODE_PROTECTED
1127	&& enmGuestMode == PGMMODE_REAL)
1128	{
1129	/* The limit must also be adjusted. */
1130	pCtx->csHid.u32Limit &= 0xffff;
1131	pCtx->dsHid.u32Limit &= 0xffff;
1132	pCtx->esHid.u32Limit &= 0xffff;
1133	pCtx->fsHid.u32Limit &= 0xffff;
1134	pCtx->gsHid.u32Limit &= 0xffff;
1135	pCtx->ssHid.u32Limit &= 0xffff;
1136
1137	Assert(pCtx->csHid.u64Base <= 0xfffff);
1138	Assert(pCtx->dsHid.u64Base <= 0xfffff);
1139	Assert(pCtx->esHid.u64Base <= 0xfffff);
1140	Assert(pCtx->fsHid.u64Base <= 0xfffff);
1141	Assert(pCtx->gsHid.u64Base <= 0xfffff);
1142	}
1143	pVCpu->hwaccm.s.vmx.enmCurrGuestMode = enmGuestMode;
1144	}
1145	else
1146	/* VT-x will fail with a guest invalid state otherwise... (CPU state after a reset) */
1147	if ( CPUMIsGuestInRealModeEx(pCtx)
1148	&& pCtx->csHid.u64Base == 0xffff0000)
1149	{
1150	pCtx->csHid.u64Base = 0xf0000;
1151	pCtx->cs = 0xf000;
1152	}
1153	#endif /* HWACCM_VMX_EMULATE_REALMODE */
1154
1155	VMX_WRITE_SELREG(ES, es);
1156	AssertRC(rc);
1157
1158	VMX_WRITE_SELREG(CS, cs);
1159	AssertRC(rc);
1160
1161	VMX_WRITE_SELREG(SS, ss);
1162	AssertRC(rc);
1163
1164	VMX_WRITE_SELREG(DS, ds);
1165	AssertRC(rc);
1166
1167	/* The base values in the hidden fs & gs registers are not in sync with the msrs; they are cut to 32 bits. */
1168	VMX_WRITE_SELREG(FS, fs);
1169	AssertRC(rc);
1170
1171	VMX_WRITE_SELREG(GS, gs);
1172	AssertRC(rc);
1173	}
1174
1175	/* Guest CPU context: LDTR. */
1176	if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_LDTR)
1177	{
1178	if (pCtx->ldtr == 0)
1179	{
1180	rc = VMXWriteCachedVMCS(VMX_VMCS16_GUEST_FIELD_LDTR, 0);
1181	rc \|= VMXWriteCachedVMCS(VMX_VMCS32_GUEST_LDTR_LIMIT, 0);
1182	rc \|= VMXWriteCachedVMCS(VMX_VMCS64_GUEST_LDTR_BASE, 0);
1183	/* Note: vmlaunch will fail with 0 or just 0x02. No idea why. */
1184	rc \|= VMXWriteCachedVMCS(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, 0x82 /* present, LDT */);
1185	}
1186	else
1187	{
1188	rc = VMXWriteCachedVMCS(VMX_VMCS16_GUEST_FIELD_LDTR, pCtx->ldtr);
1189	rc \|= VMXWriteCachedVMCS(VMX_VMCS32_GUEST_LDTR_LIMIT, pCtx->ldtrHid.u32Limit);
1190	rc \|= VMXWriteCachedVMCS(VMX_VMCS64_GUEST_LDTR_BASE, pCtx->ldtrHid.u64Base);
1191	rc \|= VMXWriteCachedVMCS(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, pCtx->ldtrHid.Attr.u);
1192	}
1193	AssertRC(rc);
1194	}
1195	/* Guest CPU context: TR. */
1196	if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_TR)
1197	{
1198	#ifdef HWACCM_VMX_EMULATE_REALMODE
1199	/* Real mode emulation using v86 mode with CR4.VME (interrupt redirection using the int bitmap in the TSS) */
1200	if (CPUMIsGuestInRealModeEx(pCtx))
1201	{
1202	RTGCPHYS GCPhys;
1203
1204	/* We convert it here every time as pci regions could be reconfigured. */
1205	rc = PDMVMMDevHeapR3ToGCPhys(pVM, pVM->hwaccm.s.vmx.pRealModeTSS, &GCPhys);
1206	AssertRC(rc);
1207
1208	rc = VMXWriteCachedVMCS(VMX_VMCS16_GUEST_FIELD_TR, 0);
1209	rc \|= VMXWriteCachedVMCS(VMX_VMCS32_GUEST_TR_LIMIT, HWACCM_VTX_TSS_SIZE);
1210	rc \|= VMXWriteCachedVMCS(VMX_VMCS64_GUEST_TR_BASE, GCPhys /* phys = virt in this mode */);
1211
1212	X86DESCATTR attr;
1213
1214	attr.u = 0;
1215	attr.n.u1Present = 1;
1216	attr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
1217	val = attr.u;
1218	}
1219	else
1220	#endif /* HWACCM_VMX_EMULATE_REALMODE */
1221	{
1222	rc = VMXWriteCachedVMCS(VMX_VMCS16_GUEST_FIELD_TR, pCtx->tr);
1223	rc \|= VMXWriteCachedVMCS(VMX_VMCS32_GUEST_TR_LIMIT, pCtx->trHid.u32Limit);
1224	rc \|= VMXWriteCachedVMCS(VMX_VMCS64_GUEST_TR_BASE, pCtx->trHid.u64Base);
1225
1226	val = pCtx->trHid.Attr.u;
1227
1228	/* The TSS selector must be busy. */
1229	if ((val & 0xF) == X86_SEL_TYPE_SYS_286_TSS_AVAIL)
1230	val = (val & ~0xF) \| X86_SEL_TYPE_SYS_286_TSS_BUSY;
1231	else
1232	/* Default even if no TR selector has been set (otherwise vmlaunch will fail!) */
1233	val = (val & ~0xF) \| X86_SEL_TYPE_SYS_386_TSS_BUSY;
1234
1235	}
1236	rc \|= VMXWriteCachedVMCS(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, val);
1237	AssertRC(rc);
1238	}
1239	/* Guest CPU context: GDTR. */
1240	if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_GDTR)
1241	{
1242	rc = VMXWriteCachedVMCS(VMX_VMCS32_GUEST_GDTR_LIMIT, pCtx->gdtr.cbGdt);
1243	rc \|= VMXWriteCachedVMCS(VMX_VMCS64_GUEST_GDTR_BASE, pCtx->gdtr.pGdt);
1244	AssertRC(rc);
1245	}
1246	/* Guest CPU context: IDTR. */
1247	if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_IDTR)
1248	{
1249	rc = VMXWriteCachedVMCS(VMX_VMCS32_GUEST_IDTR_LIMIT, pCtx->idtr.cbIdt);
1250	rc \|= VMXWriteCachedVMCS(VMX_VMCS64_GUEST_IDTR_BASE, pCtx->idtr.pIdt);
1251	AssertRC(rc);
1252	}
1253
1254	/*
1255	* Sysenter MSRs (unconditional)
1256	*/
1257	rc = VMXWriteCachedVMCS(VMX_VMCS32_GUEST_SYSENTER_CS, pCtx->SysEnter.cs);
1258	rc \|= VMXWriteCachedVMCS(VMX_VMCS64_GUEST_SYSENTER_EIP, pCtx->SysEnter.eip);
1259	rc \|= VMXWriteCachedVMCS(VMX_VMCS64_GUEST_SYSENTER_ESP, pCtx->SysEnter.esp);
1260	AssertRC(rc);
1261
1262	/* Control registers */
1263	if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_CR0)
1264	{
1265	val = pCtx->cr0;
1266	rc = VMXWriteCachedVMCS(VMX_VMCS_CTRL_CR0_READ_SHADOW, val);
1267	Log2(("Guest CR0-shadow %08x\n", val));
1268	if (CPUMIsGuestFPUStateActive(pVCpu) == false)
1269	{
1270	/* Always use #NM exceptions to load the FPU/XMM state on demand. */
1271	val \|= X86_CR0_TS \| X86_CR0_ET \| X86_CR0_NE \| X86_CR0_MP;
1272	}
1273	else
1274	{
1275	/** @todo check if we support the old style mess correctly. */
1276	if (!(val & X86_CR0_NE))
1277	Log(("Forcing X86_CR0_NE!!!\n"));
1278
1279	val \|= X86_CR0_NE; /* always turn on the native mechanism to report FPU errors (old style uses interrupts) */
1280	}
1281	/* Note: protected mode & paging are always enabled; we use them for emulating real and protected mode without paging too. */
1282	val \|= X86_CR0_PE \| X86_CR0_PG;
1283	if (pVM->hwaccm.s.fNestedPaging)
1284	{
1285	if (CPUMIsGuestInPagedProtectedModeEx(pCtx))
1286	{
1287	/* Disable cr3 read/write monitoring as we don't need it for EPT. */
1288	pVCpu->hwaccm.s.vmx.proc_ctls &= ~( VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
1289	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT);
1290	}
1291	else
1292	{
1293	/* Reenable cr3 read/write monitoring as our identity mapped page table is active. */
1294	pVCpu->hwaccm.s.vmx.proc_ctls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
1295	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT;
1296	}
1297	rc = VMXWriteCachedVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
1298	AssertRC(rc);
1299	}
1300	else
1301	{
1302	/* Note: We must also set this as we rely on protecting various pages for which supervisor writes must be caught. */
1303	val \|= X86_CR0_WP;
1304	}
1305
1306	/* Always enable caching. */
1307	val &= ~(X86_CR0_CD\|X86_CR0_NW);
1308
1309	rc \|= VMXWriteCachedVMCS(VMX_VMCS64_GUEST_CR0, val);
1310	Log2(("Guest CR0 %08x\n", val));
1311	/* CR0 flags owned by the host; if the guests attempts to change them, then
1312	* the VM will exit.
1313	*/
1314	val = X86_CR0_PE /* Must monitor this bit (assumptions are made for real mode emulation) */
1315	\| X86_CR0_WP /* Must monitor this bit (it must always be enabled). */
1316	\| X86_CR0_PG /* Must monitor this bit (assumptions are made for real mode & protected mode without paging emulation) */
1317	\| X86_CR0_TS
1318	\| X86_CR0_ET /* Bit not restored during VM-exit! */
1319	\| X86_CR0_CD /* Bit not restored during VM-exit! */
1320	\| X86_CR0_NW /* Bit not restored during VM-exit! */
1321	\| X86_CR0_NE
1322	\| X86_CR0_MP;
1323	pVCpu->hwaccm.s.vmx.cr0_mask = val;
1324
1325	rc \|= VMXWriteCachedVMCS(VMX_VMCS_CTRL_CR0_MASK, val);
1326	Log2(("Guest CR0-mask %08x\n", val));
1327	AssertRC(rc);
1328	}
1329	if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_CR4)
1330	{
1331	/* CR4 */
1332	rc = VMXWriteCachedVMCS(VMX_VMCS_CTRL_CR4_READ_SHADOW, pCtx->cr4);
1333	Log2(("Guest CR4-shadow %08x\n", pCtx->cr4));
1334	/* Set the required bits in cr4 too (currently X86_CR4_VMXE). */
1335	val = pCtx->cr4 \| (uint32_t)pVM->hwaccm.s.vmx.msr.vmx_cr4_fixed0;
1336
1337	if (!pVM->hwaccm.s.fNestedPaging)
1338	{
1339	switch(pVCpu->hwaccm.s.enmShadowMode)
1340	{
1341	case PGMMODE_REAL: /* Real mode -> emulated using v86 mode */
1342	case PGMMODE_PROTECTED: /* Protected mode, no paging -> emulated using identity mapping. */
1343	case PGMMODE_32_BIT: /* 32-bit paging. */
1344	break;
1345
1346	case PGMMODE_PAE: /* PAE paging. */
1347	case PGMMODE_PAE_NX: /* PAE paging with NX enabled. */
1348	/** @todo use normal 32 bits paging */
1349	val \|= X86_CR4_PAE;
1350	break;
1351
1352	case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
1353	case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
1354	#ifdef VBOX_ENABLE_64_BITS_GUESTS
1355	break;
1356	#else
1357	AssertFailed();
1358	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
1359	#endif
1360	default: /* shut up gcc */
1361	AssertFailed();
1362	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
1363	}
1364	}
1365	else
1366	if (!CPUMIsGuestInPagedProtectedModeEx(pCtx))
1367	{
1368	/* We use 4 MB pages in our identity mapping page table for real and protected mode without paging. */
1369	val \|= X86_CR4_PSE;
1370	/* Our identity mapping is a 32 bits page directory. */
1371	val &= ~X86_CR4_PAE;
1372	}
1373
1374	#ifdef HWACCM_VMX_EMULATE_REALMODE
1375	/* Real mode emulation using v86 mode with CR4.VME (interrupt redirection using the int bitmap in the TSS) */
1376	if (CPUMIsGuestInRealModeEx(pCtx))
1377	val \|= X86_CR4_VME;
1378	#endif /* HWACCM_VMX_EMULATE_REALMODE */
1379
1380	rc \|= VMXWriteCachedVMCS(VMX_VMCS64_GUEST_CR4, val);
1381	Log2(("Guest CR4 %08x\n", val));
1382	/* CR4 flags owned by the host; if the guests attempts to change them, then
1383	* the VM will exit.
1384	*/
1385	val = 0
1386	#ifdef HWACCM_VMX_EMULATE_REALMODE
1387	\| X86_CR4_VME
1388	#endif
1389	\| X86_CR4_PAE
1390	\| X86_CR4_PGE
1391	\| X86_CR4_PSE
1392	\| X86_CR4_VMXE;
1393	pVCpu->hwaccm.s.vmx.cr4_mask = val;
1394
1395	rc \|= VMXWriteCachedVMCS(VMX_VMCS_CTRL_CR4_MASK, val);
1396	Log2(("Guest CR4-mask %08x\n", val));
1397	AssertRC(rc);
1398	}
1399
1400	if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_CR3)
1401	{
1402	if (pVM->hwaccm.s.fNestedPaging)
1403	{
1404	AssertMsg(PGMGetEPTCR3(pVM) == PGMGetHyperCR3(pVM), ("%RHp vs %RHp\n", PGMGetEPTCR3(pVM), PGMGetHyperCR3(pVM)));
1405	pVCpu->hwaccm.s.vmx.GCPhysEPTP = PGMGetEPTCR3(pVM);
1406
1407	Assert(!(pVCpu->hwaccm.s.vmx.GCPhysEPTP & 0xfff));
1408	/** @todo Check the IA32_VMX_EPT_VPID_CAP MSR for other supported memory types. */
1409	pVCpu->hwaccm.s.vmx.GCPhysEPTP \|= VMX_EPT_MEMTYPE_WB
1410	\| (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
1411
1412	rc = VMXWriteCachedVMCS64(VMX_VMCS_CTRL_EPTP_FULL, pVCpu->hwaccm.s.vmx.GCPhysEPTP);
1413	AssertRC(rc);
1414
1415	if (!CPUMIsGuestInPagedProtectedModeEx(pCtx))
1416	{
1417	RTGCPHYS GCPhys;
1418
1419	/* We convert it here every time as pci regions could be reconfigured. */
1420	rc = PDMVMMDevHeapR3ToGCPhys(pVM, pVM->hwaccm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
1421	AssertRC(rc);
1422
1423	/* We use our identity mapping page table here as we need to map guest virtual to guest physical addresses; EPT will
1424	* take care of the translation to host physical addresses.
1425	*/
1426	val = GCPhys;
1427	}
1428	else
1429	{
1430	/* Save the real guest CR3 in VMX_VMCS_GUEST_CR3 */
1431	val = pCtx->cr3;
1432	/* Prefetch the four PDPT entries in PAE mode. */
1433	vmxR0PrefetchPAEPdptrs(pVM, pVCpu, pCtx);
1434	}
1435	}
1436	else
1437	{
1438	val = PGMGetHyperCR3(pVM);
1439	Assert(val);
1440	}
1441
1442	/* Save our shadow CR3 register. */
1443	rc = VMXWriteCachedVMCS(VMX_VMCS64_GUEST_CR3, val);
1444	AssertRC(rc);
1445	}
1446
1447	/* Debug registers. */
1448	if (pVCpu->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_DEBUG)
1449	{
1450	pCtx->dr[6] \|= X86_DR6_INIT_VAL; /* set all reserved bits to 1. */
1451	pCtx->dr[6] &= ~RT_BIT(12); /* must be zero. */
1452
1453	pCtx->dr[7] &= 0xffffffff; /* upper 32 bits reserved */
1454	pCtx->dr[7] &= ~(RT_BIT(11) \| RT_BIT(12) \| RT_BIT(14) \| RT_BIT(15)); /* must be zero */
1455	pCtx->dr[7] \|= 0x400; /* must be one */
1456
1457	/* Resync DR7 */
1458	rc = VMXWriteCachedVMCS(VMX_VMCS64_GUEST_DR7, pCtx->dr[7]);
1459	AssertRC(rc);
1460
1461	/* Sync the debug state now if any breakpoint is armed. */
1462	if ( (pCtx->dr[7] & (X86_DR7_ENABLED_MASK\|X86_DR7_GD))
1463	&& !CPUMIsGuestDebugStateActive(pVM)
1464	&& !DBGFIsStepping(pVM))
1465	{
1466	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatDRxArmed);
1467
1468	/* Disable drx move intercepts. */
1469	pVCpu->hwaccm.s.vmx.proc_ctls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT;
1470	rc = VMXWriteCachedVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
1471	AssertRC(rc);
1472
1473	/* Save the host and load the guest debug state. */
1474	rc = CPUMR0LoadGuestDebugState(pVM, pVCpu, pCtx, true /* include DR6 */);
1475	AssertRC(rc);
1476	}
1477
1478	/* IA32_DEBUGCTL MSR. */
1479	rc = VMXWriteCachedVMCS64(VMX_VMCS_GUEST_DEBUGCTL_FULL, 0);
1480	AssertRC(rc);
1481
1482	/** @todo do we really ever need this? */
1483	rc \|= VMXWriteCachedVMCS(VMX_VMCS_GUEST_DEBUG_EXCEPTIONS, 0);
1484	AssertRC(rc);
1485	}
1486
1487	/* EIP, ESP and EFLAGS */
1488	rc = VMXWriteCachedVMCS(VMX_VMCS64_GUEST_RIP, pCtx->rip);
1489	rc \|= VMXWriteCachedVMCS(VMX_VMCS64_GUEST_RSP, pCtx->rsp);
1490	AssertRC(rc);
1491
1492	/* Bits 22-31, 15, 5 & 3 must be zero. Bit 1 must be 1. */
1493	eflags = pCtx->eflags;
1494	eflags.u32 &= VMX_EFLAGS_RESERVED_0;
1495	eflags.u32 \|= VMX_EFLAGS_RESERVED_1;
1496
1497	#ifdef HWACCM_VMX_EMULATE_REALMODE
1498	/* Real mode emulation using v86 mode with CR4.VME (interrupt redirection using the int bitmap in the TSS) */
1499	if (CPUMIsGuestInRealModeEx(pCtx))
1500	{
1501	pVCpu->hwaccm.s.vmx.RealMode.eflags = eflags;
1502
1503	eflags.Bits.u1VM = 1;
1504	eflags.Bits.u2IOPL = 3;
1505	}
1506	#endif /* HWACCM_VMX_EMULATE_REALMODE */
1507	rc = VMXWriteCachedVMCS(VMX_VMCS_GUEST_RFLAGS, eflags.u32);
1508	AssertRC(rc);
1509
1510	/* TSC offset. */
1511	uint64_t u64TSCOffset;
1512
1513	if (TMCpuTickCanUseRealTSC(pVM, &u64TSCOffset))
1514	{
1515	/* Note: VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT takes precedence over TSC_OFFSET */
1516	rc = VMXWriteCachedVMCS64(VMX_VMCS_CTRL_TSC_OFFSET_FULL, u64TSCOffset);
1517	AssertRC(rc);
1518
1519	pVCpu->hwaccm.s.vmx.proc_ctls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT;
1520	rc = VMXWriteCachedVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
1521	AssertRC(rc);
1522	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatTSCOffset);
1523	}
1524	else
1525	{
1526	pVCpu->hwaccm.s.vmx.proc_ctls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT;
1527	rc = VMXWriteCachedVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
1528	AssertRC(rc);
1529	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatTSCIntercept);
1530	}
1531
1532	/* VMX_VMCS_CTRL_ENTRY_CONTROLS
1533	* Set required bits to one and zero according to the MSR capabilities.
1534	*/
1535	val = pVM->hwaccm.s.vmx.msr.vmx_entry.n.disallowed0;
1536	/* Load guest debug controls (dr7 & IA32_DEBUGCTL_MSR) (forced to 1 on the 'first' VT-x capable CPUs; this actually includes the newest Nehalem CPUs) */
1537	val \|= VMX_VMCS_CTRL_ENTRY_CONTROLS_LOAD_DEBUG;
1538
1539	/* 64 bits guest mode? */
1540	if (pCtx->msrEFER & MSR_K6_EFER_LMA)
1541	val \|= VMX_VMCS_CTRL_ENTRY_CONTROLS_IA64_MODE;
1542	/* else Must be zero when AMD64 is not available. */
1543
1544	/* Mask away the bits that the CPU doesn't support */
1545	val &= pVM->hwaccm.s.vmx.msr.vmx_entry.n.allowed1;
1546	rc = VMXWriteCachedVMCS(VMX_VMCS_CTRL_ENTRY_CONTROLS, val);
1547	AssertRC(rc);
1548
1549	/* 64 bits guest mode? */
1550	if (pCtx->msrEFER & MSR_K6_EFER_LMA)
1551	{
1552	#if !defined(VBOX_ENABLE_64_BITS_GUESTS)
1553	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
1554	#elif HC_ARCH_BITS == 32 && !defined(RT_OS_DARWIN)
1555	pVCpu->hwaccm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
1556	#else
1557	pVCpu->hwaccm.s.vmx.pfnStartVM = VMXR0StartVM64;
1558	#endif
1559	/* Unconditionally update these as wrmsr might have changed them. */
1560	rc = VMXWriteCachedVMCS(VMX_VMCS64_GUEST_FS_BASE, pCtx->fsHid.u64Base);
1561	AssertRC(rc);
1562	rc = VMXWriteCachedVMCS(VMX_VMCS64_GUEST_GS_BASE, pCtx->gsHid.u64Base);
1563	AssertRC(rc);
1564	}
1565	else
1566	{
1567	pVCpu->hwaccm.s.vmx.pfnStartVM = VMXR0StartVM32;
1568	}
1569
1570	vmxR0UpdateExceptionBitmap(pVM, pVCpu, pCtx);
1571
1572	/* Done. */
1573	pVCpu->hwaccm.s.fContextUseFlags &= ~HWACCM_CHANGED_ALL_GUEST;
1574
1575	return rc;
1576	}
1577
1578	/**
1579	* Syncs back the guest state
1580	*
1581	* @returns VBox status code.
1582	* @param pVM The VM to operate on.
1583	* @param pVCpu The VMCPU to operate on.
1584	* @param pCtx Guest context
1585	*/
1586	DECLINLINE(int) VMXR0SaveGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
1587	{
1588	RTCCUINTREG val, valShadow;
1589	RTGCUINTPTR uInterruptState;
1590	int rc;
1591
1592	/* Let's first sync back eip, esp, and eflags. */
1593	rc = VMXReadCachedVMCS(VMX_VMCS64_GUEST_RIP, &val);
1594	AssertRC(rc);
1595	pCtx->rip = val;
1596	rc = VMXReadCachedVMCS(VMX_VMCS64_GUEST_RSP, &val);
1597	AssertRC(rc);
1598	pCtx->rsp = val;
1599	rc = VMXReadCachedVMCS(VMX_VMCS_GUEST_RFLAGS, &val);
1600	AssertRC(rc);
1601	pCtx->eflags.u32 = val;
1602
1603	/* Take care of instruction fusing (sti, mov ss) */
1604	rc \|= VMXReadCachedVMCS(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &val);
1605	uInterruptState = val;
1606	if (uInterruptState != 0)
1607	{
1608	Assert(uInterruptState <= 2); /* only sti & mov ss */
1609	Log(("uInterruptState %x eip=%RGv\n", uInterruptState, pCtx->rip));
1610	EMSetInhibitInterruptsPC(pVM, pCtx->rip);
1611	}
1612	else
1613	VM_FF_CLEAR(pVM, VM_FF_INHIBIT_INTERRUPTS);
1614
1615	/* Control registers. */
1616	VMXReadCachedVMCS(VMX_VMCS_CTRL_CR0_READ_SHADOW, &valShadow);
1617	VMXReadCachedVMCS(VMX_VMCS64_GUEST_CR0, &val);
1618	val = (valShadow & pVCpu->hwaccm.s.vmx.cr0_mask) \| (val & ~pVCpu->hwaccm.s.vmx.cr0_mask);
1619	CPUMSetGuestCR0(pVM, val);
1620
1621	VMXReadCachedVMCS(VMX_VMCS_CTRL_CR4_READ_SHADOW, &valShadow);
1622	VMXReadCachedVMCS(VMX_VMCS64_GUEST_CR4, &val);
1623	val = (valShadow & pVCpu->hwaccm.s.vmx.cr4_mask) \| (val & ~pVCpu->hwaccm.s.vmx.cr4_mask);
1624	CPUMSetGuestCR4(pVM, val);
1625
1626	/* Note: no reason to sync back the CRx registers. They can't be changed by the guest. */
1627	/* Note: only in the nested paging case can CR3 & CR4 be changed by the guest. */
1628	if ( pVM->hwaccm.s.fNestedPaging
1629	&& CPUMIsGuestInPagedProtectedModeEx(pCtx))
1630	{
1631	/* Can be updated behind our back in the nested paging case. */
1632	CPUMSetGuestCR2(pVM, ASMGetCR2());
1633
1634	VMXReadCachedVMCS(VMX_VMCS64_GUEST_CR3, &val);
1635
1636	if (val != pCtx->cr3)
1637	{
1638	CPUMSetGuestCR3(pVM, val);
1639	PGMUpdateCR3(pVM, val);
1640	}
1641	/* Prefetch the four PDPT entries in PAE mode. */
1642	vmxR0PrefetchPAEPdptrs(pVM, pVCpu, pCtx);
1643	}
1644
1645	/* Sync back DR7 here. */
1646	VMXReadCachedVMCS(VMX_VMCS64_GUEST_DR7, &val);
1647	pCtx->dr[7] = val;
1648
1649	/* Guest CPU context: ES, CS, SS, DS, FS, GS. */
1650	VMX_READ_SELREG(ES, es);
1651	VMX_READ_SELREG(SS, ss);
1652	VMX_READ_SELREG(CS, cs);
1653	VMX_READ_SELREG(DS, ds);
1654	VMX_READ_SELREG(FS, fs);
1655	VMX_READ_SELREG(GS, gs);
1656
1657	/*
1658	* System MSRs
1659	*/
1660	VMXReadCachedVMCS(VMX_VMCS32_GUEST_SYSENTER_CS, &val);
1661	pCtx->SysEnter.cs = val;
1662	VMXReadCachedVMCS(VMX_VMCS64_GUEST_SYSENTER_EIP, &val);
1663	pCtx->SysEnter.eip = val;
1664	VMXReadCachedVMCS(VMX_VMCS64_GUEST_SYSENTER_ESP, &val);
1665	pCtx->SysEnter.esp = val;
1666
1667	/* Misc. registers; must sync everything otherwise we can get out of sync when jumping to ring 3. */
1668	VMX_READ_SELREG(LDTR, ldtr);
1669
1670	VMXReadCachedVMCS(VMX_VMCS32_GUEST_GDTR_LIMIT, &val);
1671	pCtx->gdtr.cbGdt = val;
1672	VMXReadCachedVMCS(VMX_VMCS64_GUEST_GDTR_BASE, &val);
1673	pCtx->gdtr.pGdt = val;
1674
1675	VMXReadCachedVMCS(VMX_VMCS32_GUEST_IDTR_LIMIT, &val);
1676	pCtx->idtr.cbIdt = val;
1677	VMXReadCachedVMCS(VMX_VMCS64_GUEST_IDTR_BASE, &val);
1678	pCtx->idtr.pIdt = val;
1679
1680	#ifdef HWACCM_VMX_EMULATE_REALMODE
1681	/* Real mode emulation using v86 mode with CR4.VME (interrupt redirection using the int bitmap in the TSS) */
1682	if (CPUMIsGuestInRealModeEx(pCtx))
1683	{
1684	/* Hide our emulation flags */
1685	pCtx->eflags.Bits.u1VM = 0;
1686	pCtx->eflags.Bits.u2IOPL = pVCpu->hwaccm.s.vmx.RealMode.eflags.Bits.u2IOPL;
1687
1688	/* Force a TR resync every time in case we switch modes. */
1689	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_GUEST_TR;
1690	}
1691	else
1692	#endif /* HWACCM_VMX_EMULATE_REALMODE */
1693	{
1694	/* In real mode we have a fake TSS, so only sync it back when it's supposed to be valid. */
1695	VMX_READ_SELREG(TR, tr);
1696	}
1697	return VINF_SUCCESS;
1698	}
1699
1700	/**
1701	* Dummy placeholder
1702	*
1703	* @param pVM The VM to operate on.
1704	* @param pVCpu The VMCPU to operate on.
1705	*/
1706	static void vmxR0SetupTLBDummy(PVM pVM, PVMCPU pVCpu)
1707	{
1708	NOREF(pVM);
1709	NOREF(pVCpu);
1710	return;
1711	}
1712
1713	/**
1714	* Setup the tagged TLB for EPT
1715	*
1716	* @returns VBox status code.
1717	* @param pVM The VM to operate on.
1718	* @param pVCpu The VMCPU to operate on.
1719	*/
1720	static void vmxR0SetupTLBEPT(PVM pVM, PVMCPU pVCpu)
1721	{
1722	PHWACCM_CPUINFO pCpu;
1723
1724	Assert(pVM->hwaccm.s.fNestedPaging);
1725	Assert(!pVM->hwaccm.s.vmx.fVPID);
1726
1727	/* Deal with tagged TLBs if VPID or EPT is supported. */
1728	pCpu = HWACCMR0GetCurrentCpu();
1729	/* Force a TLB flush for the first world switch if the current cpu differs from the one we ran on last. */
1730	/* Note that this can happen both for start and resume due to long jumps back to ring 3. */
1731	if ( pVCpu->hwaccm.s.idLastCpu != pCpu->idCpu
1732	/* if the tlb flush count has changed, another VM has flushed the TLB of this cpu, so we can't use our current ASID anymore. */
1733	\|\| pVCpu->hwaccm.s.cTLBFlushes != pCpu->cTLBFlushes)
1734	{
1735	/* Force a TLB flush on VM entry. */
1736	pVCpu->hwaccm.s.fForceTLBFlush = true;
1737	}
1738	else
1739	Assert(!pCpu->fFlushTLB);
1740
1741	pVCpu->hwaccm.s.idLastCpu = pCpu->idCpu;
1742	pCpu->fFlushTLB = false;
1743
1744	if (pVCpu->hwaccm.s.fForceTLBFlush)
1745	vmxR0FlushEPT(pVM, pVCpu, pVM->hwaccm.s.vmx.enmFlushContext, 0);
1746
1747	#ifdef VBOX_WITH_STATISTICS
1748	if (pVCpu->hwaccm.s.fForceTLBFlush)
1749	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatFlushTLBWorldSwitch);
1750	else
1751	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatNoFlushTLBWorldSwitch);
1752	#endif
1753	}
1754
1755	#ifdef HWACCM_VTX_WITH_VPID
1756	/**
1757	* Setup the tagged TLB for VPID
1758	*
1759	* @returns VBox status code.
1760	* @param pVM The VM to operate on.
1761	* @param pVCpu The VMCPU to operate on.
1762	*/
1763	static void vmxR0SetupTLBVPID(PVM pVM, PVMCPU pVCpu)
1764	{
1765	PHWACCM_CPUINFO pCpu;
1766
1767	Assert(pVM->hwaccm.s.vmx.fVPID);
1768	Assert(!pVM->hwaccm.s.fNestedPaging);
1769
1770	/* Deal with tagged TLBs if VPID or EPT is supported. */
1771	pCpu = HWACCMR0GetCurrentCpu();
1772	/* Force a TLB flush for the first world switch if the current cpu differs from the one we ran on last. */
1773	/* Note that this can happen both for start and resume due to long jumps back to ring 3. */
1774	if ( pVCpu->hwaccm.s.idLastCpu != pCpu->idCpu
1775	/* if the tlb flush count has changed, another VM has flushed the TLB of this cpu, so we can't use our current ASID anymore. */
1776	\|\| pVCpu->hwaccm.s.cTLBFlushes != pCpu->cTLBFlushes)
1777	{
1778	/* Force a TLB flush on VM entry. */
1779	pVCpu->hwaccm.s.fForceTLBFlush = true;
1780	}
1781	else
1782	Assert(!pCpu->fFlushTLB);
1783
1784	pVCpu->hwaccm.s.idLastCpu = pCpu->idCpu;
1785
1786	/* Make sure we flush the TLB when required. Switch ASID to achieve the same thing, but without actually flushing the whole TLB (which is expensive). */
1787	if (pVCpu->hwaccm.s.fForceTLBFlush)
1788	{
1789	if ( ++pCpu->uCurrentASID >= pVM->hwaccm.s.uMaxASID
1790	\|\| pCpu->fFlushTLB)
1791	{
1792	pCpu->fFlushTLB = false;
1793	pCpu->uCurrentASID = 1; /* start at 1; host uses 0 */
1794	pCpu->cTLBFlushes++;
1795	}
1796	else
1797	{
1798	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatFlushASID);
1799	pVCpu->hwaccm.s.fForceTLBFlush = false;
1800	}
1801
1802	pVCpu->hwaccm.s.cTLBFlushes = pCpu->cTLBFlushes;
1803	pVCpu->hwaccm.s.uCurrentASID = pCpu->uCurrentASID;
1804	}
1805	else
1806	{
1807	Assert(!pCpu->fFlushTLB);
1808
1809	if (!pCpu->uCurrentASID \|\| !pVCpu->hwaccm.s.uCurrentASID)
1810	pVCpu->hwaccm.s.uCurrentASID = pCpu->uCurrentASID = 1;
1811	}
1812	AssertMsg(pVCpu->hwaccm.s.cTLBFlushes == pCpu->cTLBFlushes, ("Flush count mismatch for cpu %d (%x vs %x)\n", pCpu->idCpu, pVCpu->hwaccm.s.cTLBFlushes, pCpu->cTLBFlushes));
1813	AssertMsg(pCpu->uCurrentASID >= 1 && pCpu->uCurrentASID < pVM->hwaccm.s.uMaxASID, ("cpu%d uCurrentASID = %x\n", pCpu->idCpu, pCpu->uCurrentASID));
1814	AssertMsg(pVCpu->hwaccm.s.uCurrentASID >= 1 && pVCpu->hwaccm.s.uCurrentASID < pVM->hwaccm.s.uMaxASID, ("cpu%d VM uCurrentASID = %x\n", pCpu->idCpu, pVCpu->hwaccm.s.uCurrentASID));
1815
1816	int rc = VMXWriteCachedVMCS(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hwaccm.s.uCurrentASID);
1817	AssertRC(rc);
1818
1819	if (pVCpu->hwaccm.s.fForceTLBFlush)
1820	vmxR0FlushVPID(pVM, pVCpu, pVM->hwaccm.s.vmx.enmFlushContext, 0);
1821
1822	#ifdef VBOX_WITH_STATISTICS
1823	if (pVCpu->hwaccm.s.fForceTLBFlush)
1824	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatFlushTLBWorldSwitch);
1825	else
1826	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatNoFlushTLBWorldSwitch);
1827	#endif
1828	}
1829	#endif /* HWACCM_VTX_WITH_VPID */
1830
1831	/**
1832	* Runs guest code in a VT-x VM.
1833	*
1834	* @returns VBox status code.
1835	* @param pVM The VM to operate on.
1836	* @param pVCpu The VMCPU to operate on.
1837	* @param pCtx Guest context
1838	*/
1839	VMMR0DECL(int) VMXR0RunGuestCode(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
1840	{
1841	int rc = VINF_SUCCESS;
1842	RTCCUINTREG val;
1843	RTCCUINTREG exitReason, instrError, cbInstr;
1844	RTGCUINTPTR exitQualification;
1845	RTGCUINTPTR intInfo = 0; /* shut up buggy gcc 4 */
1846	RTGCUINTPTR errCode, instrInfo;
1847	bool fSyncTPR = false;
1848	PHWACCM_CPUINFO pCpu = 0;
1849	unsigned cResume = 0;
1850	#ifdef VBOX_STRICT
1851	RTCPUID idCpuCheck;
1852	#endif
1853	#ifdef VBOX_WITH_STATISTICS
1854	bool fStatEntryStarted = true;
1855	bool fStatExit2Started = false;
1856	#endif
1857
1858	Log2(("\nE"));
1859
1860	STAM_PROFILE_ADV_START(&pVCpu->hwaccm.s.StatEntry, x);
1861
1862	#ifdef VBOX_STRICT
1863	VMXFlushWriteCache(pVCpu);
1864
1865	rc = VMXReadVMCS(VMX_VMCS_CTRL_PIN_EXEC_CONTROLS, &val);
1866	AssertRC(rc);
1867	Log2(("VMX_VMCS_CTRL_PIN_EXEC_CONTROLS = %08x\n", val));
1868
1869	/* allowed zero */
1870	if ((val & pVM->hwaccm.s.vmx.msr.vmx_pin_ctls.n.disallowed0) != pVM->hwaccm.s.vmx.msr.vmx_pin_ctls.n.disallowed0)
1871	Log(("Invalid VMX_VMCS_CTRL_PIN_EXEC_CONTROLS: zero\n"));
1872
1873	/* allowed one */
1874	if ((val & ~pVM->hwaccm.s.vmx.msr.vmx_pin_ctls.n.allowed1) != 0)
1875	Log(("Invalid VMX_VMCS_CTRL_PIN_EXEC_CONTROLS: one\n"));
1876
1877	rc = VMXReadVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, &val);
1878	AssertRC(rc);
1879	Log2(("VMX_VMCS_CTRL_PROC_EXEC_CONTROLS = %08x\n", val));
1880
1881	/* Must be set according to the MSR, but can be cleared in case of EPT. */
1882	if (pVM->hwaccm.s.fNestedPaging)
1883	val \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INVLPG_EXIT
1884	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
1885	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT;
1886
1887	/* allowed zero */
1888	if ((val & pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.disallowed0) != pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.disallowed0)
1889	Log(("Invalid VMX_VMCS_CTRL_PROC_EXEC_CONTROLS: zero\n"));
1890
1891	/* allowed one */
1892	if ((val & ~pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1) != 0)
1893	Log(("Invalid VMX_VMCS_CTRL_PROC_EXEC_CONTROLS: one\n"));
1894
1895	rc = VMXReadVMCS(VMX_VMCS_CTRL_ENTRY_CONTROLS, &val);
1896	AssertRC(rc);
1897	Log2(("VMX_VMCS_CTRL_ENTRY_CONTROLS = %08x\n", val));
1898
1899	/* allowed zero */
1900	if ((val & pVM->hwaccm.s.vmx.msr.vmx_entry.n.disallowed0) != pVM->hwaccm.s.vmx.msr.vmx_entry.n.disallowed0)
1901	Log(("Invalid VMX_VMCS_CTRL_ENTRY_CONTROLS: zero\n"));
1902
1903	/* allowed one */
1904	if ((val & ~pVM->hwaccm.s.vmx.msr.vmx_entry.n.allowed1) != 0)
1905	Log(("Invalid VMX_VMCS_CTRL_ENTRY_CONTROLS: one\n"));
1906
1907	rc = VMXReadVMCS(VMX_VMCS_CTRL_EXIT_CONTROLS, &val);
1908	AssertRC(rc);
1909	Log2(("VMX_VMCS_CTRL_EXIT_CONTROLS = %08x\n", val));
1910
1911	/* allowed zero */
1912	if ((val & pVM->hwaccm.s.vmx.msr.vmx_exit.n.disallowed0) != pVM->hwaccm.s.vmx.msr.vmx_exit.n.disallowed0)
1913	Log(("Invalid VMX_VMCS_CTRL_EXIT_CONTROLS: zero\n"));
1914
1915	/* allowed one */
1916	if ((val & ~pVM->hwaccm.s.vmx.msr.vmx_exit.n.allowed1) != 0)
1917	Log(("Invalid VMX_VMCS_CTRL_EXIT_CONTROLS: one\n"));
1918	#endif
1919
1920	/* We can jump to this point to resume execution after determining that a VM-exit is innocent.
1921	*/
1922	ResumeExecution:
1923	STAM_STATS({
1924	if (fStatExit2Started) { STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2, y); fStatExit2Started = false; }
1925	if (!fStatEntryStarted) { STAM_PROFILE_ADV_START(&pVCpu->hwaccm.s.StatEntry, x); fStatEntryStarted = true; }
1926	});
1927	AssertMsg(pVCpu->hwaccm.s.idEnteredCpu == RTMpCpuId(),
1928	("Expected %d, I'm %d; cResume=%d exitReason=%RTreg exitQualification=%RTreg\n",
1929	(int)pVCpu->hwaccm.s.idEnteredCpu, (int)RTMpCpuId(), cResume, exitReason, exitQualification));
1930	Assert(!HWACCMR0SuspendPending());
1931
1932	/* Safety precaution; looping for too long here can have a very bad effect on the host */
1933	if (++cResume > HWACCM_MAX_RESUME_LOOPS)
1934	{
1935	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitMaxResume);
1936	rc = VINF_EM_RAW_INTERRUPT;
1937	goto end;
1938	}
1939
1940	/* Check for irq inhibition due to instruction fusing (sti, mov ss). */
1941	if (VM_FF_ISSET(pVM, VM_FF_INHIBIT_INTERRUPTS))
1942	{
1943	Log(("VM_FF_INHIBIT_INTERRUPTS at %RGv successor %RGv\n", (RTGCPTR)pCtx->rip, EMGetInhibitInterruptsPC(pVM)));
1944	if (pCtx->rip != EMGetInhibitInterruptsPC(pVM))
1945	{
1946	/* Note: we intentionally don't clear VM_FF_INHIBIT_INTERRUPTS here.
1947	* Before we are able to execute this instruction in raw mode (iret to guest code) an external interrupt might
1948	* force a world switch again. Possibly allowing a guest interrupt to be dispatched in the process. This could
1949	* break the guest. Sounds very unlikely, but such timing sensitive problems are not as rare as you might think.
1950	*/
1951	VM_FF_CLEAR(pVM, VM_FF_INHIBIT_INTERRUPTS);
1952	/* Irq inhibition is no longer active; clear the corresponding VMX state. */
1953	rc = VMXWriteCachedVMCS(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, 0);
1954	AssertRC(rc);
1955	}
1956	}
1957	else
1958	{
1959	/* Irq inhibition is no longer active; clear the corresponding VMX state. */
1960	rc = VMXWriteCachedVMCS(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, 0);
1961	AssertRC(rc);
1962	}
1963
1964	/* Check for pending actions that force us to go back to ring 3. */
1965	if (VM_FF_ISPENDING(pVM, VM_FF_TO_R3 \| VM_FF_TIMER))
1966	{
1967	VM_FF_CLEAR(pVM, VM_FF_TO_R3);
1968	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatSwitchToR3);
1969	rc = VINF_EM_RAW_TO_R3;
1970	goto end;
1971	}
1972	/* Pending request packets might contain actions that need immediate attention, such as pending hardware interrupts. */
1973	if (VM_FF_ISPENDING(pVM, VM_FF_REQUEST))
1974	{
1975	rc = VINF_EM_PENDING_REQUEST;
1976	goto end;
1977	}
1978
1979	/* When external interrupts are pending, we should exit the VM when IF is set. */
1980	/* Note! After VM_FF_INHIBIT_INTERRUPTS check!!! */
1981	rc = VMXR0CheckPendingInterrupt(pVM, pVCpu, pCtx);
1982	if (RT_FAILURE(rc))
1983	goto end;
1984
1985	/** @todo check timers?? */
1986
1987	/* TPR caching using CR8 is only available in 64 bits mode */
1988	/* Note the 32 bits exception for AMD (X86_CPUID_AMD_FEATURE_ECX_CR8L), but that appears missing in Intel CPUs */
1989	/* Note: we can't do this in LoadGuestState as PDMApicGetTPR can jump back to ring 3 (lock)!!!!! */
1990	/**
1991	* @todo reduce overhead
1992	*/
1993	if ( (pCtx->msrEFER & MSR_K6_EFER_LMA)
1994	&& pVM->hwaccm.s.vmx.pAPIC)
1995	{
1996	/* TPR caching in CR8 */
1997	uint8_t u8TPR;
1998	bool fPending;
1999
2000	int rc = PDMApicGetTPR(pVM, &u8TPR, &fPending);
2001	AssertRC(rc);
2002	/* The TPR can be found at offset 0x80 in the APIC mmio page. */
2003	pVM->hwaccm.s.vmx.pAPIC[0x80] = u8TPR << 4; /* bits 7-4 contain the task priority */
2004
2005	/* Two options here:
2006	* - external interrupt pending, but masked by the TPR value.
2007	* -> a CR8 update that lower the current TPR value should cause an exit
2008	* - no pending interrupts
2009	* -> We don't need to be explicitely notified. There are enough world switches for detecting pending interrupts.
2010	*/
2011	rc = VMXWriteCachedVMCS(VMX_VMCS_CTRL_TPR_THRESHOLD, (fPending) ? u8TPR : 0);
2012	AssertRC(rc);
2013
2014	/* Always sync back the TPR; we should optimize this though / /* @todo optimize TPR sync. */
2015	fSyncTPR = true;
2016	}
2017
2018	#if defined(HWACCM_VTX_WITH_EPT) && defined(LOG_ENABLED)
2019	if ( pVM->hwaccm.s.fNestedPaging
2020	# ifdef HWACCM_VTX_WITH_VPID
2021	\|\| pVM->hwaccm.s.vmx.fVPID
2022	# endif /* HWACCM_VTX_WITH_VPID */
2023	)
2024	{
2025	pCpu = HWACCMR0GetCurrentCpu();
2026	if ( pVCpu->hwaccm.s.idLastCpu != pCpu->idCpu
2027	\|\| pVCpu->hwaccm.s.cTLBFlushes != pCpu->cTLBFlushes)
2028	{
2029	if (pVCpu->hwaccm.s.idLastCpu != pCpu->idCpu)
2030	Log(("Force TLB flush due to rescheduling to a different cpu (%d vs %d)\n", pVCpu->hwaccm.s.idLastCpu, pCpu->idCpu));
2031	else
2032	Log(("Force TLB flush due to changed TLB flush count (%x vs %x)\n", pVCpu->hwaccm.s.cTLBFlushes, pCpu->cTLBFlushes));
2033	}
2034	if (pCpu->fFlushTLB)
2035	Log(("Force TLB flush: first time cpu %d is used -> flush\n", pCpu->idCpu));
2036	else
2037	if (pVCpu->hwaccm.s.fForceTLBFlush)
2038	LogFlow(("Manual TLB flush\n"));
2039	}
2040	#endif
2041	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
2042	PGMDynMapReleaseAutoSet(pVCpu);
2043	#endif
2044
2045	/*
2046	* NOTE: DO NOT DO ANYTHING AFTER THIS POINT THAT MIGHT JUMP BACK TO RING 3!
2047	* (until the actual world switch)
2048	*/
2049	#ifdef VBOX_STRICT
2050	idCpuCheck = RTMpCpuId();
2051	#endif
2052	#ifdef LOG_LOGGING
2053	VMMR0LogFlushDisable(pVCpu);
2054	#endif
2055	/* Save the host state first. */
2056	rc = VMXR0SaveHostState(pVM, pVCpu);
2057	if (rc != VINF_SUCCESS)
2058	goto end;
2059	/* Load the guest state */
2060	rc = VMXR0LoadGuestState(pVM, pVCpu, pCtx);
2061	if (rc != VINF_SUCCESS)
2062	goto end;
2063
2064	/* Deal with tagged TLB setup and invalidation. */
2065	pVM->hwaccm.s.vmx.pfnSetupTaggedTLB(pVM, pVCpu);
2066
2067	/* Non-register state Guest Context */
2068	/** @todo change me according to cpu state */
2069	rc = VMXWriteCachedVMCS(VMX_VMCS32_GUEST_ACTIVITY_STATE, VMX_CMS_GUEST_ACTIVITY_ACTIVE);
2070	AssertRC(rc);
2071
2072	STAM_STATS({ STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatEntry, x); fStatEntryStarted = false; });
2073
2074	/* Manual save and restore:
2075	* - General purpose registers except RIP, RSP
2076	*
2077	* Trashed:
2078	* - CR2 (we don't care)
2079	* - LDTR (reset to 0)
2080	* - DRx (presumably not changed at all)
2081	* - DR7 (reset to 0x400)
2082	* - EFLAGS (reset to RT_BIT(1); not relevant)
2083	*
2084	*/
2085
2086	/* All done! Let's start VM execution. */
2087	STAM_PROFILE_ADV_START(&pVCpu->hwaccm.s.StatInGC, z);
2088	#ifdef VBOX_STRICT
2089	Assert(idCpuCheck == RTMpCpuId());
2090	#endif
2091	TMNotifyStartOfExecution(pVM);
2092	rc = pVCpu->hwaccm.s.vmx.pfnStartVM(pVCpu->hwaccm.s.fResumeVM, pCtx, &pVCpu->hwaccm.s.vmx.VMCSCache, pVM, pVCpu);
2093	TMNotifyEndOfExecution(pVM);
2094
2095	AssertMsg(!pVCpu->hwaccm.s.vmx.VMCSCache.Write.cValidEntries, ("pVCpu->hwaccm.s.vmx.VMCSCache.Write.cValidEntries=%d\n", pVCpu->hwaccm.s.vmx.VMCSCache.Write.cValidEntries));
2096
2097	/* In case we execute a goto ResumeExecution later on. */
2098	pVCpu->hwaccm.s.fResumeVM = true;
2099	pVCpu->hwaccm.s.fForceTLBFlush = false;
2100
2101	/*
2102	* !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
2103	* IMPORTANT: WE CAN'T DO ANY LOGGING OR OPERATIONS THAT CAN DO A LONGJMP BACK TO RING 3 BEFORE WE'VE SYNCED BACK (MOST OF) THE GUEST STATE
2104	* !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
2105	*/
2106
2107	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatInGC, z);
2108	STAM_PROFILE_ADV_START(&pVCpu->hwaccm.s.StatExit1, v);
2109
2110	if (rc != VINF_SUCCESS)
2111	{
2112	VMXR0ReportWorldSwitchError(pVM, pVCpu, rc, pCtx);
2113	goto end;
2114	}
2115	/* Success. Query the guest state and figure out what has happened. */
2116
2117	/* Investigate why there was a VM-exit. */
2118	rc = VMXReadCachedVMCS(VMX_VMCS32_RO_EXIT_REASON, &exitReason);
2119	STAM_COUNTER_INC(&pVCpu->hwaccm.s.paStatExitReasonR0[exitReason & MASK_EXITREASON_STAT]);
2120
2121	exitReason &= 0xffff; /* bit 0-15 contain the exit code. */
2122	rc \|= VMXReadCachedVMCS(VMX_VMCS32_RO_VM_INSTR_ERROR, &instrError);
2123	rc \|= VMXReadCachedVMCS(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &cbInstr);
2124	rc \|= VMXReadCachedVMCS(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &val);
2125	intInfo = val;
2126	rc \|= VMXReadCachedVMCS(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERRCODE, &val);
2127	errCode = val; /* might not be valid; depends on VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID. */
2128	rc \|= VMXReadCachedVMCS(VMX_VMCS32_RO_EXIT_INSTR_INFO, &val);
2129	instrInfo = val;
2130	rc \|= VMXReadCachedVMCS(VMX_VMCS_RO_EXIT_QUALIFICATION, &val);
2131	exitQualification = val;
2132	AssertRC(rc);
2133
2134	/* Sync back the guest state */
2135	rc = VMXR0SaveGuestState(pVM, pVCpu, pCtx);
2136	AssertRC(rc);
2137
2138	/* Note! NOW IT'S SAFE FOR LOGGING! */
2139	#ifdef LOG_LOGGING
2140	VMMR0LogFlushEnable(pVCpu);
2141	#endif
2142	Log2(("Raw exit reason %08x\n", exitReason));
2143
2144	/* Check if an injected event was interrupted prematurely. */
2145	rc = VMXReadCachedVMCS(VMX_VMCS32_RO_IDT_INFO, &val);
2146	AssertRC(rc);
2147	pVCpu->hwaccm.s.Event.intInfo = VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(val);
2148	if ( VMX_EXIT_INTERRUPTION_INFO_VALID(pVCpu->hwaccm.s.Event.intInfo)
2149	/* Ignore 'int xx' as they'll be restarted anyway. */
2150	&& VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hwaccm.s.Event.intInfo) != VMX_EXIT_INTERRUPTION_INFO_TYPE_SW
2151	/* Ignore software exceptions (such as int3) as they're reoccur when we restart the instruction anyway. */
2152	&& VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hwaccm.s.Event.intInfo) != VMX_EXIT_INTERRUPTION_INFO_TYPE_SWEXCPT)
2153	{
2154	pVCpu->hwaccm.s.Event.fPending = true;
2155	/* Error code present? */
2156	if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(pVCpu->hwaccm.s.Event.intInfo))
2157	{
2158	rc = VMXReadCachedVMCS(VMX_VMCS32_RO_IDT_ERRCODE, &val);
2159	AssertRC(rc);
2160	pVCpu->hwaccm.s.Event.errCode = val;
2161	Log(("Pending inject %RX64 at %RGv exit=%08x intInfo=%08x exitQualification=%08x pending error=%RX64\n", pVCpu->hwaccm.s.Event.intInfo, (RTGCPTR)pCtx->rip, exitReason, intInfo, exitQualification, val));
2162	}
2163	else
2164	{
2165	Log(("Pending inject %RX64 at %RGv exit=%08x intInfo=%08x exitQualification=%08x\n", pVCpu->hwaccm.s.Event.intInfo, (RTGCPTR)pCtx->rip, exitReason, intInfo, exitQualification));
2166	pVCpu->hwaccm.s.Event.errCode = 0;
2167	}
2168	}
2169	#ifdef VBOX_STRICT
2170	else
2171	if ( VMX_EXIT_INTERRUPTION_INFO_VALID(pVCpu->hwaccm.s.Event.intInfo)
2172	/* Ignore software exceptions (such as int3) as they're reoccur when we restart the instruction anyway. */
2173	&& VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hwaccm.s.Event.intInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_SWEXCPT)
2174	{
2175	Log(("Ignore pending inject %RX64 at %RGv exit=%08x intInfo=%08x exitQualification=%08x\n", pVCpu->hwaccm.s.Event.intInfo, (RTGCPTR)pCtx->rip, exitReason, intInfo, exitQualification));
2176	}
2177
2178	if (exitReason == VMX_EXIT_ERR_INVALID_GUEST_STATE)
2179	HWACCMDumpRegs(pVM, pCtx);
2180	#endif
2181
2182	Log2(("E%d", exitReason));
2183	Log2(("Exit reason %d, exitQualification %08x\n", exitReason, exitQualification));
2184	Log2(("instrInfo=%d instrError=%d instr length=%d\n", instrInfo, instrError, cbInstr));
2185	Log2(("Interruption error code %d\n", errCode));
2186	Log2(("IntInfo = %08x\n", intInfo));
2187	Log2(("New EIP=%RGv\n", (RTGCPTR)pCtx->rip));
2188
2189	if (fSyncTPR)
2190	{
2191	rc = PDMApicSetTPR(pVM, pVM->hwaccm.s.vmx.pAPIC[0x80] >> 4);
2192	AssertRC(rc);
2193	}
2194
2195	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit1, v);
2196	STAM_STATS({ STAM_PROFILE_ADV_START(&pVCpu->hwaccm.s.StatExit2, y); fStatExit2Started = true; });
2197
2198	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
2199	PGMDynMapStartAutoSet(pVCpu);
2200	#endif
2201
2202	/* Some cases don't need a complete resync of the guest CPU state; handle them here. */
2203	switch (exitReason)
2204	{
2205	case VMX_EXIT_EXCEPTION: /* 0 Exception or non-maskable interrupt (NMI). */
2206	case VMX_EXIT_EXTERNAL_IRQ: /* 1 External interrupt. */
2207	{
2208	uint32_t vector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(intInfo);
2209
2210	if (!VMX_EXIT_INTERRUPTION_INFO_VALID(intInfo))
2211	{
2212	Assert(exitReason == VMX_EXIT_EXTERNAL_IRQ);
2213	/* External interrupt; leave to allow it to be dispatched again. */
2214	rc = VINF_EM_RAW_INTERRUPT;
2215	break;
2216	}
2217	STAM_PROFILE_ADV_START(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
2218	switch (VMX_EXIT_INTERRUPTION_INFO_TYPE(intInfo))
2219	{
2220	case VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI: /* Non-maskable interrupt. */
2221	/* External interrupt; leave to allow it to be dispatched again. */
2222	rc = VINF_EM_RAW_INTERRUPT;
2223	break;
2224
2225	case VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT: /* External hardware interrupt. */
2226	AssertFailed(); /* can't come here; fails the first check. */
2227	break;
2228
2229	case VMX_EXIT_INTERRUPTION_INFO_TYPE_DBEXCPT: /* Unknown why we get this type for #DB */
2230	case VMX_EXIT_INTERRUPTION_INFO_TYPE_SWEXCPT: /* Software exception. (#BP or #OF) */
2231	Assert(vector == 1 \|\| vector == 3 \|\| vector == 4);
2232	/* no break */
2233	case VMX_EXIT_INTERRUPTION_INFO_TYPE_HWEXCPT: /* Hardware exception. */
2234	Log2(("Hardware/software interrupt %d\n", vector));
2235	switch (vector)
2236	{
2237	case X86_XCPT_NM:
2238	{
2239	Log(("#NM fault at %RGv error code %x\n", (RTGCPTR)pCtx->rip, errCode));
2240
2241	/** @todo don't intercept #NM exceptions anymore when we've activated the guest FPU state. */
2242	/* If we sync the FPU/XMM state on-demand, then we can continue execution as if nothing has happened. */
2243	rc = CPUMR0LoadGuestFPU(pVM, pVCpu, pCtx);
2244	if (rc == VINF_SUCCESS)
2245	{
2246	Assert(CPUMIsGuestFPUStateActive(pVCpu));
2247
2248	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitShadowNM);
2249
2250	/* Continue execution. */
2251	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_GUEST_CR0;
2252
2253	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
2254	goto ResumeExecution;
2255	}
2256
2257	Log(("Forward #NM fault to the guest\n"));
2258	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestNM);
2259	rc = VMXR0InjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), cbInstr, 0);
2260	AssertRC(rc);
2261	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
2262	goto ResumeExecution;
2263	}
2264
2265	case X86_XCPT_PF: /* Page fault */
2266	{
2267	#ifdef DEBUG
2268	if (pVM->hwaccm.s.fNestedPaging)
2269	{ /* A genuine pagefault.
2270	* Forward the trap to the guest by injecting the exception and resuming execution.
2271	*/
2272	Log(("Guest page fault at %RGv cr2=%RGv error code %x rsp=%RGv\n", (RTGCPTR)pCtx->rip, exitQualification, errCode, (RTGCPTR)pCtx->rsp));
2273
2274	Assert(CPUMIsGuestInPagedProtectedModeEx(pCtx));
2275
2276	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestPF);
2277
2278	/* Now we must update CR2. */
2279	pCtx->cr2 = exitQualification;
2280	rc = VMXR0InjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), cbInstr, errCode);
2281	AssertRC(rc);
2282
2283	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
2284	goto ResumeExecution;
2285	}
2286	#endif
2287	Assert(!pVM->hwaccm.s.fNestedPaging);
2288
2289	Log2(("Page fault at %RGv error code %x\n", exitQualification, errCode));
2290	/* Exit qualification contains the linear address of the page fault. */
2291	TRPMAssertTrap(pVM, X86_XCPT_PF, TRPM_TRAP);
2292	TRPMSetErrorCode(pVM, errCode);
2293	TRPMSetFaultAddress(pVM, exitQualification);
2294
2295	/* Forward it to our trap handler first, in case our shadow pages are out of sync. */
2296	rc = PGMTrap0eHandler(pVM, errCode, CPUMCTX2CORE(pCtx), (RTGCPTR)exitQualification);
2297	Log2(("PGMTrap0eHandler %RGv returned %Rrc\n", (RTGCPTR)pCtx->rip, rc));
2298	if (rc == VINF_SUCCESS)
2299	{ /* We've successfully synced our shadow pages, so let's just continue execution. */
2300	Log2(("Shadow page fault at %RGv cr2=%RGv error code %x\n", (RTGCPTR)pCtx->rip, exitQualification ,errCode));
2301	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitShadowPF);
2302
2303	TRPMResetTrap(pVM);
2304
2305	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
2306	goto ResumeExecution;
2307	}
2308	else
2309	if (rc == VINF_EM_RAW_GUEST_TRAP)
2310	{ /* A genuine pagefault.
2311	* Forward the trap to the guest by injecting the exception and resuming execution.
2312	*/
2313	Log2(("Forward page fault to the guest\n"));
2314
2315	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestPF);
2316	/* The error code might have been changed. */
2317	errCode = TRPMGetErrorCode(pVM);
2318
2319	TRPMResetTrap(pVM);
2320
2321	/* Now we must update CR2. */
2322	pCtx->cr2 = exitQualification;
2323	rc = VMXR0InjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), cbInstr, errCode);
2324	AssertRC(rc);
2325
2326	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
2327	goto ResumeExecution;
2328	}
2329	#ifdef VBOX_STRICT
2330	if (rc != VINF_EM_RAW_EMULATE_INSTR)
2331	Log2(("PGMTrap0eHandler failed with %d\n", rc));
2332	#endif
2333	/* Need to go back to the recompiler to emulate the instruction. */
2334	TRPMResetTrap(pVM);
2335	break;
2336	}
2337
2338	case X86_XCPT_MF: /* Floating point exception. */
2339	{
2340	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestMF);
2341	if (!(pCtx->cr0 & X86_CR0_NE))
2342	{
2343	/* old style FPU error reporting needs some extra work. */
2344	/** @todo don't fall back to the recompiler, but do it manually. */
2345	rc = VINF_EM_RAW_EMULATE_INSTR;
2346	break;
2347	}
2348	Log(("Trap %x at %04X:%RGv\n", vector, pCtx->cs, (RTGCPTR)pCtx->rip));
2349	rc = VMXR0InjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), cbInstr, errCode);
2350	AssertRC(rc);
2351
2352	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
2353	goto ResumeExecution;
2354	}
2355
2356	case X86_XCPT_DB: /* Debug exception. */
2357	{
2358	uint64_t uDR6;
2359
2360	/* DR6, DR7.GD and IA32_DEBUGCTL.LBR are not updated yet.
2361	*
2362	* Exit qualification bits:
2363	* 3:0 B0-B3 which breakpoint condition was met
2364	* 12:4 Reserved (0)
2365	* 13 BD - debug register access detected
2366	* 14 BS - single step execution or branch taken
2367	* 63:15 Reserved (0)
2368	*/
2369	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestDB);
2370
2371	/* Note that we don't support guest and host-initiated debugging at the same time. */
2372	Assert(DBGFIsStepping(pVM) \|\| CPUMIsGuestInRealModeEx(pCtx));
2373
2374	uDR6 = X86_DR6_INIT_VAL;
2375	uDR6 \|= (exitQualification & (X86_DR6_B0\|X86_DR6_B1\|X86_DR6_B2\|X86_DR6_B3\|X86_DR6_BD\|X86_DR6_BS));
2376	rc = DBGFR0Trap01Handler(pVM, CPUMCTX2CORE(pCtx), uDR6);
2377	if (rc == VINF_EM_RAW_GUEST_TRAP)
2378	{
2379	/** @todo this isn't working, but we'll never get here normally. */
2380
2381	/* Update DR6 here. */
2382	pCtx->dr[6] = uDR6;
2383
2384	/* X86_DR7_GD will be cleared if drx accesses should be trapped inside the guest. */
2385	pCtx->dr[7] &= ~X86_DR7_GD;
2386
2387	/* Paranoia. */
2388	pCtx->dr[7] &= 0xffffffff; /* upper 32 bits reserved */
2389	pCtx->dr[7] &= ~(RT_BIT(11) \| RT_BIT(12) \| RT_BIT(14) \| RT_BIT(15)); /* must be zero */
2390	pCtx->dr[7] \|= 0x400; /* must be one */
2391
2392	/* Resync DR7 */
2393	rc = VMXWriteCachedVMCS(VMX_VMCS64_GUEST_DR7, pCtx->dr[7]);
2394	AssertRC(rc);
2395
2396	Log(("Trap %x (debug) at %RGv exit qualification %RX64\n", vector, (RTGCPTR)pCtx->rip, exitQualification));
2397	rc = VMXR0InjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), cbInstr, errCode);
2398	AssertRC(rc);
2399
2400	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
2401	goto ResumeExecution;
2402	}
2403	/* Return to ring 3 to deal with the debug exit code. */
2404	break;
2405	}
2406
2407	case X86_XCPT_GP: /* General protection failure exception.*/
2408	{
2409	uint32_t cbSize;
2410
2411	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestGP);
2412	#ifdef VBOX_STRICT
2413	if (!CPUMIsGuestInRealModeEx(pCtx))
2414	{
2415	Log(("Trap %x at %04X:%RGv errorCode=%x\n", vector, pCtx->cs, (RTGCPTR)pCtx->rip, errCode));
2416	rc = VMXR0InjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), cbInstr, errCode);
2417	AssertRC(rc);
2418	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
2419	goto ResumeExecution;
2420	}
2421	#endif
2422	Assert(CPUMIsGuestInRealModeEx(pCtx));
2423
2424	LogFlow(("Real mode X86_XCPT_GP instruction emulation at %RGv\n", (RTGCPTR)pCtx->rip));
2425	rc = EMInterpretInstruction(pVM, CPUMCTX2CORE(pCtx), 0, &cbSize);
2426	if (rc == VINF_SUCCESS)
2427	{
2428	/* EIP has been updated already. */
2429
2430	/* lidt, lgdt can end up here. In the future crx changes as well. Just reload the whole context to be done with it. */
2431	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_ALL;
2432
2433	/* Only resume if successful. */
2434	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
2435	goto ResumeExecution;
2436	}
2437	AssertMsg(rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_EM_HALT, ("Unexpected rc=%Rrc\n", rc));
2438	break;
2439	}
2440
2441	#ifdef VBOX_STRICT
2442	case X86_XCPT_DE: /* Divide error. */
2443	case X86_XCPT_UD: /* Unknown opcode exception. */
2444	case X86_XCPT_SS: /* Stack segment exception. */
2445	case X86_XCPT_NP: /* Segment not present exception. */
2446	{
2447	switch(vector)
2448	{
2449	case X86_XCPT_DE:
2450	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestDE);
2451	break;
2452	case X86_XCPT_UD:
2453	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestUD);
2454	break;
2455	case X86_XCPT_SS:
2456	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestSS);
2457	break;
2458	case X86_XCPT_NP:
2459	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitGuestNP);
2460	break;
2461	}
2462
2463	Log(("Trap %x at %04X:%RGv\n", vector, pCtx->cs, (RTGCPTR)pCtx->rip));
2464	rc = VMXR0InjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), cbInstr, errCode);
2465	AssertRC(rc);
2466
2467	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
2468	goto ResumeExecution;
2469	}
2470	#endif
2471	default:
2472	#ifdef HWACCM_VMX_EMULATE_REALMODE
2473	if (CPUMIsGuestInRealModeEx(pCtx))
2474	{
2475	Log(("Real Mode Trap %x at %04x:%04X error code %x\n", vector, pCtx->cs, pCtx->eip, errCode));
2476	rc = VMXR0InjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), cbInstr, errCode);
2477	AssertRC(rc);
2478
2479	/* Go back to ring 3 in case of a triple fault. */
2480	if ( vector == X86_XCPT_DF
2481	&& rc == VINF_EM_RESET)
2482	break;
2483
2484	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
2485	goto ResumeExecution;
2486	}
2487	#endif
2488	AssertMsgFailed(("Unexpected vm-exit caused by exception %x\n", vector));
2489	rc = VERR_VMX_UNEXPECTED_EXCEPTION;
2490	break;
2491	} /* switch (vector) */
2492
2493	break;
2494
2495	default:
2496	rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_CODE;
2497	AssertMsgFailed(("Unexpected interuption code %x\n", intInfo));
2498	break;
2499	}
2500
2501	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub3, y3);
2502	break;
2503	}
2504
2505	case VMX_EXIT_EPT_VIOLATION: /* 48 EPT violation. An attempt to access memory with a guest-physical address was disallowed by the configuration of the EPT paging structures. */
2506	{
2507	RTGCPHYS GCPhys;
2508
2509	Assert(pVM->hwaccm.s.fNestedPaging);
2510
2511	rc = VMXReadVMCS64(VMX_VMCS_EXIT_PHYS_ADDR_FULL, &GCPhys);
2512	AssertRC(rc);
2513	Assert(((exitQualification >> 7) & 3) != 2);
2514
2515	/* Determine the kind of violation. */
2516	errCode = 0;
2517	if (exitQualification & VMX_EXIT_QUALIFICATION_EPT_INSTR_FETCH)
2518	errCode \|= X86_TRAP_PF_ID;
2519
2520	if (exitQualification & VMX_EXIT_QUALIFICATION_EPT_DATA_WRITE)
2521	errCode \|= X86_TRAP_PF_RW;
2522
2523	/* If the page is present, then it's a page level protection fault. */
2524	if (exitQualification & VMX_EXIT_QUALIFICATION_EPT_ENTRY_PRESENT)
2525	errCode \|= X86_TRAP_PF_P;
2526
2527	Log(("EPT Page fault %x at %RGp error code %x\n", (uint32_t)exitQualification, GCPhys, errCode));
2528
2529	/* GCPhys contains the guest physical address of the page fault. */
2530	TRPMAssertTrap(pVM, X86_XCPT_PF, TRPM_TRAP);
2531	TRPMSetErrorCode(pVM, errCode);
2532	TRPMSetFaultAddress(pVM, GCPhys);
2533
2534	/* Handle the pagefault trap for the nested shadow table. */
2535	rc = PGMR0Trap0eHandlerNestedPaging(pVM, PGMMODE_EPT, errCode, CPUMCTX2CORE(pCtx), GCPhys);
2536	Log2(("PGMR0Trap0eHandlerNestedPaging %RGv returned %Rrc\n", (RTGCPTR)pCtx->rip, rc));
2537	if (rc == VINF_SUCCESS)
2538	{ /* We've successfully synced our shadow pages, so let's just continue execution. */
2539	Log2(("Shadow page fault at %RGv cr2=%RGp error code %x\n", (RTGCPTR)pCtx->rip, exitQualification , errCode));
2540	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitReasonNPF);
2541
2542	TRPMResetTrap(pVM);
2543
2544	goto ResumeExecution;
2545	}
2546
2547	#ifdef VBOX_STRICT
2548	if (rc != VINF_EM_RAW_EMULATE_INSTR)
2549	LogFlow(("PGMTrap0eHandlerNestedPaging failed with %d\n", rc));
2550	#endif
2551	/* Need to go back to the recompiler to emulate the instruction. */
2552	TRPMResetTrap(pVM);
2553	break;
2554	}
2555
2556	case VMX_EXIT_EPT_MISCONFIG:
2557	{
2558	RTGCPHYS GCPhys;
2559
2560	Assert(pVM->hwaccm.s.fNestedPaging);
2561
2562	rc = VMXReadVMCS64(VMX_VMCS_EXIT_PHYS_ADDR_FULL, &GCPhys);
2563	AssertRC(rc);
2564
2565	Log(("VMX_EXIT_EPT_MISCONFIG for %VGp\n", GCPhys));
2566	break;
2567	}
2568
2569	case VMX_EXIT_IRQ_WINDOW: /* 7 Interrupt window. */
2570	/* Clear VM-exit on IF=1 change. */
2571	LogFlow(("VMX_EXIT_IRQ_WINDOW %RGv pending=%d IF=%d\n", (RTGCPTR)pCtx->rip, VM_FF_ISPENDING(pVM, (VM_FF_INTERRUPT_APIC\|VM_FF_INTERRUPT_PIC)), pCtx->eflags.Bits.u1IF));
2572	pVCpu->hwaccm.s.vmx.proc_ctls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_IRQ_WINDOW_EXIT;
2573	rc = VMXWriteCachedVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
2574	AssertRC(rc);
2575	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitIrqWindow);
2576	goto ResumeExecution; /* we check for pending guest interrupts there */
2577
2578	case VMX_EXIT_WBINVD: /* 54 Guest software attempted to execute WBINVD. (conditional) */
2579	case VMX_EXIT_INVD: /* 13 Guest software attempted to execute INVD. (unconditional) */
2580	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitInvd);
2581	/* Skip instruction and continue directly. */
2582	pCtx->rip += cbInstr;
2583	/* Continue execution.*/
2584	goto ResumeExecution;
2585
2586	case VMX_EXIT_CPUID: /* 10 Guest software attempted to execute CPUID. */
2587	{
2588	Log2(("VMX: Cpuid %x\n", pCtx->eax));
2589	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitCpuid);
2590	rc = EMInterpretCpuId(pVM, CPUMCTX2CORE(pCtx));
2591	if (rc == VINF_SUCCESS)
2592	{
2593	/* Update EIP and continue execution. */
2594	Assert(cbInstr == 2);
2595	pCtx->rip += cbInstr;
2596	goto ResumeExecution;
2597	}
2598	AssertMsgFailed(("EMU: cpuid failed with %Rrc\n", rc));
2599	rc = VINF_EM_RAW_EMULATE_INSTR;
2600	break;
2601	}
2602
2603	case VMX_EXIT_RDTSC: /* 16 Guest software attempted to execute RDTSC. */
2604	{
2605	Log2(("VMX: Rdtsc\n"));
2606	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitRdtsc);
2607	rc = EMInterpretRdtsc(pVM, CPUMCTX2CORE(pCtx));
2608	if (rc == VINF_SUCCESS)
2609	{
2610	/* Update EIP and continue execution. */
2611	Assert(cbInstr == 2);
2612	pCtx->rip += cbInstr;
2613	goto ResumeExecution;
2614	}
2615	AssertMsgFailed(("EMU: rdtsc failed with %Rrc\n", rc));
2616	rc = VINF_EM_RAW_EMULATE_INSTR;
2617	break;
2618	}
2619
2620	case VMX_EXIT_INVPG: /* 14 Guest software attempted to execute INVPG. */
2621	{
2622	Log2(("VMX: invlpg\n"));
2623	Assert(!pVM->hwaccm.s.fNestedPaging);
2624
2625	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitInvpg);
2626	rc = EMInterpretInvlpg(pVM, CPUMCTX2CORE(pCtx), exitQualification);
2627	if (rc == VINF_SUCCESS)
2628	{
2629	/* Update EIP and continue execution. */
2630	pCtx->rip += cbInstr;
2631	goto ResumeExecution;
2632	}
2633	AssertMsg(rc == VERR_EM_INTERPRETER, ("EMU: invlpg %RGv failed with %Rrc\n", exitQualification, rc));
2634	break;
2635	}
2636
2637	case VMX_EXIT_RDMSR: /* 31 RDMSR. Guest software attempted to execute RDMSR. */
2638	case VMX_EXIT_WRMSR: /* 32 WRMSR. Guest software attempted to execute WRMSR. */
2639	{
2640	uint32_t cbSize;
2641
2642	/* Note: the intel manual claims there's a REX version of RDMSR that's slightly different, so we play safe by completely disassembling the instruction. */
2643	Log2(("VMX: %s\n", (exitReason == VMX_EXIT_RDMSR) ? "rdmsr" : "wrmsr"));
2644	rc = EMInterpretInstruction(pVM, CPUMCTX2CORE(pCtx), 0, &cbSize);
2645	if (rc == VINF_SUCCESS)
2646	{
2647	/* EIP has been updated already. */
2648
2649	/* Only resume if successful. */
2650	goto ResumeExecution;
2651	}
2652	AssertMsg(rc == VERR_EM_INTERPRETER, ("EMU: %s failed with %Rrc\n", (exitReason == VMX_EXIT_RDMSR) ? "rdmsr" : "wrmsr", rc));
2653	break;
2654	}
2655
2656	case VMX_EXIT_CRX_MOVE: /* 28 Control-register accesses. */
2657	{
2658	STAM_PROFILE_ADV_START(&pVCpu->hwaccm.s.StatExit2Sub2, y2);
2659
2660	switch (VMX_EXIT_QUALIFICATION_CRX_ACCESS(exitQualification))
2661	{
2662	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_WRITE:
2663	Log2(("VMX: %RGv mov cr%d, x\n", (RTGCPTR)pCtx->rip, VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification)));
2664	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitCRxWrite[VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification)]);
2665	rc = EMInterpretCRxWrite(pVM, CPUMCTX2CORE(pCtx),
2666	VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification),
2667	VMX_EXIT_QUALIFICATION_CRX_GENREG(exitQualification));
2668
2669	switch (VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification))
2670	{
2671	case 0:
2672	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_GUEST_CR0 \| HWACCM_CHANGED_GUEST_CR3;
2673	break;
2674	case 2:
2675	break;
2676	case 3:
2677	Assert(!pVM->hwaccm.s.fNestedPaging \|\| !CPUMIsGuestInPagedProtectedModeEx(pCtx));
2678	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_GUEST_CR3;
2679	break;
2680	case 4:
2681	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_GUEST_CR4;
2682	break;
2683	case 8:
2684	/* CR8 contains the APIC TPR */
2685	Assert(!(pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW));
2686	break;
2687
2688	default:
2689	AssertFailed();
2690	break;
2691	}
2692	/* Check if a sync operation is pending. */
2693	if ( rc == VINF_SUCCESS /* don't bother if we are going to ring 3 anyway */
2694	&& VM_FF_ISPENDING(pVM, VM_FF_PGM_SYNC_CR3 \| VM_FF_PGM_SYNC_CR3_NON_GLOBAL))
2695	{
2696	rc = PGMSyncCR3(pVM, CPUMGetGuestCR0(pVM), CPUMGetGuestCR3(pVM), CPUMGetGuestCR4(pVM), VM_FF_ISSET(pVM, VM_FF_PGM_SYNC_CR3));
2697	AssertRC(rc);
2698	}
2699	break;
2700
2701	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ:
2702	Log2(("VMX: mov x, crx\n"));
2703	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitCRxRead[VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification)]);
2704
2705	Assert(!pVM->hwaccm.s.fNestedPaging \|\| !CPUMIsGuestInPagedProtectedModeEx(pCtx) \|\| VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification) != USE_REG_CR3);
2706
2707	/* CR8 reads only cause an exit when the TPR shadow feature isn't present. */
2708	Assert(VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification) != 8 \|\| !(pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW));
2709
2710	rc = EMInterpretCRxRead(pVM, CPUMCTX2CORE(pCtx),
2711	VMX_EXIT_QUALIFICATION_CRX_GENREG(exitQualification),
2712	VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification));
2713	break;
2714
2715	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_CLTS:
2716	Log2(("VMX: clts\n"));
2717	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitCLTS);
2718	rc = EMInterpretCLTS(pVM);
2719	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_GUEST_CR0;
2720	break;
2721
2722	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_LMSW:
2723	Log2(("VMX: lmsw %x\n", VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(exitQualification)));
2724	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitLMSW);
2725	rc = EMInterpretLMSW(pVM, CPUMCTX2CORE(pCtx), VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(exitQualification));
2726	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_GUEST_CR0;
2727	break;
2728	}
2729
2730	/* Update EIP if no error occurred. */
2731	if (RT_SUCCESS(rc))
2732	pCtx->rip += cbInstr;
2733
2734	if (rc == VINF_SUCCESS)
2735	{
2736	/* Only resume if successful. */
2737	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub2, y2);
2738	goto ResumeExecution;
2739	}
2740	Assert(rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_PGM_SYNC_CR3);
2741	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub2, y2);
2742	break;
2743	}
2744
2745	case VMX_EXIT_DRX_MOVE: /* 29 Debug-register accesses. */
2746	{
2747	if (!DBGFIsStepping(pVM))
2748	{
2749	/* Disable drx move intercepts. */
2750	pVCpu->hwaccm.s.vmx.proc_ctls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT;
2751	rc = VMXWriteCachedVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
2752	AssertRC(rc);
2753
2754	/* Save the host and load the guest debug state. */
2755	rc = CPUMR0LoadGuestDebugState(pVM, pVCpu, pCtx, true /* include DR6 */);
2756	AssertRC(rc);
2757
2758	#ifdef VBOX_WITH_STATISTICS
2759	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatDRxContextSwitch);
2760	if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(exitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
2761	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitDRxWrite);
2762	else
2763	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitDRxRead);
2764	#endif
2765
2766	goto ResumeExecution;
2767	}
2768
2769	/** @todo clear VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT after the first time and restore drx registers afterwards */
2770	if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(exitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
2771	{
2772	Log2(("VMX: mov drx%d, genreg%d\n", VMX_EXIT_QUALIFICATION_DRX_REGISTER(exitQualification), VMX_EXIT_QUALIFICATION_DRX_GENREG(exitQualification)));
2773	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitDRxWrite);
2774	rc = EMInterpretDRxWrite(pVM, CPUMCTX2CORE(pCtx),
2775	VMX_EXIT_QUALIFICATION_DRX_REGISTER(exitQualification),
2776	VMX_EXIT_QUALIFICATION_DRX_GENREG(exitQualification));
2777	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_GUEST_DEBUG;
2778	Log2(("DR7=%08x\n", pCtx->dr[7]));
2779	}
2780	else
2781	{
2782	Log2(("VMX: mov x, drx\n"));
2783	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitDRxRead);
2784	rc = EMInterpretDRxRead(pVM, CPUMCTX2CORE(pCtx),
2785	VMX_EXIT_QUALIFICATION_DRX_GENREG(exitQualification),
2786	VMX_EXIT_QUALIFICATION_DRX_REGISTER(exitQualification));
2787	}
2788	/* Update EIP if no error occurred. */
2789	if (RT_SUCCESS(rc))
2790	pCtx->rip += cbInstr;
2791
2792	if (rc == VINF_SUCCESS)
2793	{
2794	/* Only resume if successful. */
2795	goto ResumeExecution;
2796	}
2797	Assert(rc == VERR_EM_INTERPRETER);
2798	break;
2799	}
2800
2801	/* Note: We'll get a #GP if the IO instruction isn't allowed (IOPL or TSS bitmap); no need to double check. */
2802	case VMX_EXIT_PORT_IO: /* 30 I/O instruction. */
2803	{
2804	STAM_PROFILE_ADV_START(&pVCpu->hwaccm.s.StatExit2Sub1, y1);
2805	uint32_t uIOWidth = VMX_EXIT_QUALIFICATION_IO_WIDTH(exitQualification);
2806	uint32_t uPort;
2807	bool fIOWrite = (VMX_EXIT_QUALIFICATION_IO_DIRECTION(exitQualification) == VMX_EXIT_QUALIFICATION_IO_DIRECTION_OUT);
2808
2809	/** @todo necessary to make the distinction? */
2810	if (VMX_EXIT_QUALIFICATION_IO_ENCODING(exitQualification) == VMX_EXIT_QUALIFICATION_IO_ENCODING_DX)
2811	{
2812	uPort = pCtx->edx & 0xffff;
2813	}
2814	else
2815	uPort = VMX_EXIT_QUALIFICATION_IO_PORT(exitQualification); /* Immediate encoding. */
2816
2817	/* paranoia */
2818	if (RT_UNLIKELY(uIOWidth == 2 \|\| uIOWidth >= 4))
2819	{
2820	rc = fIOWrite ? VINF_IOM_HC_IOPORT_WRITE : VINF_IOM_HC_IOPORT_READ;
2821	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub1, y1);
2822	break;
2823	}
2824
2825	uint32_t cbSize = g_aIOSize[uIOWidth];
2826
2827	if (VMX_EXIT_QUALIFICATION_IO_STRING(exitQualification))
2828	{
2829	/* ins/outs */
2830	DISCPUSTATE Cpu;
2831
2832	/* Disassemble manually to deal with segment prefixes. */
2833	/** @todo VMX_VMCS_EXIT_GUEST_LINEAR_ADDR contains the flat pointer operand of the instruction. */
2834	/** @todo VMX_VMCS32_RO_EXIT_INSTR_INFO also contains segment prefix info. */
2835	rc = EMInterpretDisasOne(pVM, CPUMCTX2CORE(pCtx), &Cpu, NULL);
2836	if (rc == VINF_SUCCESS)
2837	{
2838	if (fIOWrite)
2839	{
2840	Log2(("IOMInterpretOUTSEx %RGv %x size=%d\n", (RTGCPTR)pCtx->rip, uPort, cbSize));
2841	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitIOStringWrite);
2842	rc = IOMInterpretOUTSEx(pVM, CPUMCTX2CORE(pCtx), uPort, Cpu.prefix, cbSize);
2843	}
2844	else
2845	{
2846	Log2(("IOMInterpretINSEx %RGv %x size=%d\n", (RTGCPTR)pCtx->rip, uPort, cbSize));
2847	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitIOStringRead);
2848	rc = IOMInterpretINSEx(pVM, CPUMCTX2CORE(pCtx), uPort, Cpu.prefix, cbSize);
2849	}
2850	}
2851	else
2852	rc = VINF_EM_RAW_EMULATE_INSTR;
2853	}
2854	else
2855	{
2856	/* normal in/out */
2857	uint32_t uAndVal = g_aIOOpAnd[uIOWidth];
2858
2859	Assert(!VMX_EXIT_QUALIFICATION_IO_REP(exitQualification));
2860
2861	if (fIOWrite)
2862	{
2863	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitIOWrite);
2864	rc = IOMIOPortWrite(pVM, uPort, pCtx->eax & uAndVal, cbSize);
2865	}
2866	else
2867	{
2868	uint32_t u32Val = 0;
2869
2870	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatExitIORead);
2871	rc = IOMIOPortRead(pVM, uPort, &u32Val, cbSize);
2872	if (IOM_SUCCESS(rc))
2873	{
2874	/* Write back to the EAX register. */
2875	pCtx->eax = (pCtx->eax & ~uAndVal) \| (u32Val & uAndVal);
2876	}
2877	}
2878	}
2879	/*
2880	* Handled the I/O return codes.
2881	* (The unhandled cases end up with rc == VINF_EM_RAW_EMULATE_INSTR.)
2882	*/
2883	if (IOM_SUCCESS(rc))
2884	{
2885	/* Update EIP and continue execution. */
2886	pCtx->rip += cbInstr;
2887	if (RT_LIKELY(rc == VINF_SUCCESS))
2888	{
2889	/* If any IO breakpoints are armed, then we should check if a debug trap needs to be generated. */
2890	if (pCtx->dr[7] & X86_DR7_ENABLED_MASK)
2891	{
2892	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatDRxIOCheck);
2893	for (unsigned i=0;i<4;i++)
2894	{
2895	unsigned uBPLen = g_aIOSize[X86_DR7_GET_LEN(pCtx->dr[7], i)];
2896
2897	if ( (uPort >= pCtx->dr[i] && uPort < pCtx->dr[i] + uBPLen)
2898	&& (pCtx->dr[7] & (X86_DR7_L(i) \| X86_DR7_G(i)))
2899	&& (pCtx->dr[7] & X86_DR7_RW(i, X86_DR7_RW_IO)) == X86_DR7_RW(i, X86_DR7_RW_IO))
2900	{
2901	uint64_t uDR6;
2902
2903	Assert(CPUMIsGuestDebugStateActive(pVM));
2904
2905	uDR6 = ASMGetDR6();
2906
2907	/* Clear all breakpoint status flags and set the one we just hit. */
2908	uDR6 &= ~(X86_DR6_B0\|X86_DR6_B1\|X86_DR6_B2\|X86_DR6_B3);
2909	uDR6 \|= (uint64_t)RT_BIT(i);
2910
2911	/* Note: AMD64 Architecture Programmer's Manual 13.1:
2912	* Bits 15:13 of the DR6 register is never cleared by the processor and must be cleared by software after
2913	* the contents have been read.
2914	*/
2915	ASMSetDR6(uDR6);
2916
2917	/* X86_DR7_GD will be cleared if drx accesses should be trapped inside the guest. */
2918	pCtx->dr[7] &= ~X86_DR7_GD;
2919
2920	/* Paranoia. */
2921	pCtx->dr[7] &= 0xffffffff; /* upper 32 bits reserved */
2922	pCtx->dr[7] &= ~(RT_BIT(11) \| RT_BIT(12) \| RT_BIT(14) \| RT_BIT(15)); /* must be zero */
2923	pCtx->dr[7] \|= 0x400; /* must be one */
2924
2925	/* Resync DR7 */
2926	rc = VMXWriteCachedVMCS(VMX_VMCS64_GUEST_DR7, pCtx->dr[7]);
2927	AssertRC(rc);
2928
2929	/* Construct inject info. */
2930	intInfo = X86_XCPT_DB;
2931	intInfo \|= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
2932	intInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HWEXCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
2933
2934	Log(("Inject IO debug trap at %RGv\n", (RTGCPTR)pCtx->rip));
2935	rc = VMXR0InjectEvent(pVM, pVCpu, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), 0, 0);
2936	AssertRC(rc);
2937
2938	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub1, y1);
2939	goto ResumeExecution;
2940	}
2941	}
2942	}
2943
2944	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub1, y1);
2945	goto ResumeExecution;
2946	}
2947	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub1, y1);
2948	break;
2949	}
2950
2951	#ifdef VBOX_STRICT
2952	if (rc == VINF_IOM_HC_IOPORT_READ)
2953	Assert(!fIOWrite);
2954	else if (rc == VINF_IOM_HC_IOPORT_WRITE)
2955	Assert(fIOWrite);
2956	else
2957	AssertMsg(RT_FAILURE(rc) \|\| rc == VINF_EM_RAW_EMULATE_INSTR \|\| rc == VINF_EM_RAW_GUEST_TRAP \|\| rc == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", rc));
2958	#endif
2959	STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2Sub1, y1);
2960	break;
2961	}
2962
2963	case VMX_EXIT_TPR: /* 43 TPR below threshold. Guest software executed MOV to CR8. */
2964	LogFlow(("VMX_EXIT_TPR\n"));
2965	/* RIP is already set to the next instruction and the TPR has been synced back. Just resume. */
2966	goto ResumeExecution;
2967
2968	default:
2969	/* The rest is handled after syncing the entire CPU state. */
2970	break;
2971	}
2972
2973	/* Note: the guest state isn't entirely synced back at this stage. */
2974
2975	/* Investigate why there was a VM-exit. (part 2) */
2976	switch (exitReason)
2977	{
2978	case VMX_EXIT_EXCEPTION: /* 0 Exception or non-maskable interrupt (NMI). */
2979	case VMX_EXIT_EXTERNAL_IRQ: /* 1 External interrupt. */
2980	case VMX_EXIT_EPT_VIOLATION:
2981	/* Already handled above. */
2982	break;
2983
2984	case VMX_EXIT_TRIPLE_FAULT: /* 2 Triple fault. */
2985	rc = VINF_EM_RESET; /* Triple fault equals a reset. */
2986	break;
2987
2988	case VMX_EXIT_INIT_SIGNAL: /* 3 INIT signal. */
2989	case VMX_EXIT_SIPI: /* 4 Start-up IPI (SIPI). */
2990	rc = VINF_EM_RAW_INTERRUPT;
2991	AssertFailed(); /* Can't happen. Yet. */
2992	break;
2993
2994	case VMX_EXIT_IO_SMI_IRQ: /* 5 I/O system-management interrupt (SMI). */
2995	case VMX_EXIT_SMI_IRQ: /* 6 Other SMI. */
2996	rc = VINF_EM_RAW_INTERRUPT;
2997	AssertFailed(); /* Can't happen afaik. */
2998	break;
2999
3000	case VMX_EXIT_TASK_SWITCH: /* 9 Task switch. */
3001	rc = VERR_EM_INTERPRETER;
3002	break;
3003
3004	case VMX_EXIT_HLT: /* 12 Guest software attempted to execute HLT. */
3005	/** Check if external interrupts are pending; if so, don't switch back. */
3006	pCtx->rip++; /* skip hlt */
3007	if ( pCtx->eflags.Bits.u1IF
3008	&& VM_FF_ISPENDING(pVM, (VM_FF_INTERRUPT_APIC\|VM_FF_INTERRUPT_PIC)))
3009	goto ResumeExecution;
3010
3011	rc = VINF_EM_HALT;
3012	break;
3013
3014	case VMX_EXIT_RSM: /* 17 Guest software attempted to execute RSM in SMM. */
3015	AssertFailed(); /* can't happen. */
3016	rc = VINF_EM_RAW_EXCEPTION_PRIVILEGED;
3017	break;
3018
3019	case VMX_EXIT_VMCALL: /* 18 Guest software executed VMCALL. */
3020	case VMX_EXIT_VMCLEAR: /* 19 Guest software executed VMCLEAR. */
3021	case VMX_EXIT_VMLAUNCH: /* 20 Guest software executed VMLAUNCH. */
3022	case VMX_EXIT_VMPTRLD: /* 21 Guest software executed VMPTRLD. */
3023	case VMX_EXIT_VMPTRST: /* 22 Guest software executed VMPTRST. */
3024	case VMX_EXIT_VMREAD: /* 23 Guest software executed VMREAD. */
3025	case VMX_EXIT_VMRESUME: /* 24 Guest software executed VMRESUME. */
3026	case VMX_EXIT_VMWRITE: /* 25 Guest software executed VMWRITE. */
3027	case VMX_EXIT_VMXOFF: /* 26 Guest software executed VMXOFF. */
3028	case VMX_EXIT_VMXON: /* 27 Guest software executed VMXON. */
3029	/** @todo inject #UD immediately */
3030	rc = VINF_EM_RAW_EXCEPTION_PRIVILEGED;
3031	break;
3032
3033	case VMX_EXIT_CPUID: /* 10 Guest software attempted to execute CPUID. */
3034	case VMX_EXIT_RDTSC: /* 16 Guest software attempted to execute RDTSC. */
3035	case VMX_EXIT_INVPG: /* 14 Guest software attempted to execute INVPG. */
3036	case VMX_EXIT_CRX_MOVE: /* 28 Control-register accesses. */
3037	case VMX_EXIT_DRX_MOVE: /* 29 Debug-register accesses. */
3038	case VMX_EXIT_PORT_IO: /* 30 I/O instruction. */
3039	/* already handled above */
3040	AssertMsg( rc == VINF_PGM_CHANGE_MODE
3041	\|\| rc == VINF_EM_RAW_INTERRUPT
3042	\|\| rc == VERR_EM_INTERPRETER
3043	\|\| rc == VINF_EM_RAW_EMULATE_INSTR
3044	\|\| rc == VINF_PGM_SYNC_CR3
3045	\|\| rc == VINF_IOM_HC_IOPORT_READ
3046	\|\| rc == VINF_IOM_HC_IOPORT_WRITE
3047	\|\| rc == VINF_EM_RAW_GUEST_TRAP
3048	\|\| rc == VINF_TRPM_XCPT_DISPATCHED
3049	\|\| rc == VINF_EM_RESCHEDULE_REM,
3050	("rc = %d\n", rc));
3051	break;
3052
3053	case VMX_EXIT_TPR: /* 43 TPR below threshold. Guest software executed MOV to CR8. */
3054	case VMX_EXIT_RDMSR: /* 31 RDMSR. Guest software attempted to execute RDMSR. */
3055	case VMX_EXIT_WRMSR: /* 32 WRMSR. Guest software attempted to execute WRMSR. */
3056	/* Note: If we decide to emulate them here, then we must sync the MSRs that could have been changed (sysenter, fs/gs base)!!! */
3057	rc = VERR_EM_INTERPRETER;
3058	break;
3059
3060	case VMX_EXIT_RDPMC: /* 15 Guest software attempted to execute RDPMC. */
3061	case VMX_EXIT_MWAIT: /* 36 Guest software executed MWAIT. */
3062	case VMX_EXIT_MONITOR: /* 39 Guest software attempted to execute MONITOR. */
3063	case VMX_EXIT_PAUSE: /* 40 Guest software attempted to execute PAUSE. */
3064	rc = VINF_EM_RAW_EXCEPTION_PRIVILEGED;
3065	break;
3066
3067	case VMX_EXIT_IRQ_WINDOW: /* 7 Interrupt window. */
3068	Assert(rc == VINF_EM_RAW_INTERRUPT);
3069	break;
3070
3071	case VMX_EXIT_ERR_INVALID_GUEST_STATE: /* 33 VM-entry failure due to invalid guest state. */
3072	{
3073	#ifdef VBOX_STRICT
3074	Log(("VMX_EXIT_ERR_INVALID_GUEST_STATE\n"));
3075
3076	VMXReadVMCS(VMX_VMCS64_GUEST_RIP, &val);
3077	Log(("Old eip %RGv new %RGv\n", (RTGCPTR)pCtx->rip, (RTGCPTR)val));
3078
3079	VMXReadVMCS(VMX_VMCS64_GUEST_CR0, &val);
3080	Log(("VMX_VMCS_GUEST_CR0 %RX64\n", val));
3081
3082	VMXReadVMCS(VMX_VMCS64_GUEST_CR3, &val);
3083	Log(("VMX_VMCS_GUEST_CR3 %RGp\n", val));
3084
3085	VMXReadVMCS(VMX_VMCS64_GUEST_CR4, &val);
3086	Log(("VMX_VMCS_GUEST_CR4 %RX64\n", val));
3087
3088	VMXReadVMCS(VMX_VMCS_GUEST_RFLAGS, &val);
3089	Log(("VMX_VMCS_GUEST_RFLAGS %08x\n", val));
3090
3091	VMX_LOG_SELREG(CS, "CS");
3092	VMX_LOG_SELREG(DS, "DS");
3093	VMX_LOG_SELREG(ES, "ES");
3094	VMX_LOG_SELREG(FS, "FS");
3095	VMX_LOG_SELREG(GS, "GS");
3096	VMX_LOG_SELREG(SS, "SS");
3097	VMX_LOG_SELREG(TR, "TR");
3098	VMX_LOG_SELREG(LDTR, "LDTR");
3099
3100	VMXReadVMCS(VMX_VMCS64_GUEST_GDTR_BASE, &val);
3101	Log(("VMX_VMCS_GUEST_GDTR_BASE %RGv\n", val));
3102	VMXReadVMCS(VMX_VMCS64_GUEST_IDTR_BASE, &val);
3103	Log(("VMX_VMCS_GUEST_IDTR_BASE %RGv\n", val));
3104	#endif /* VBOX_STRICT */
3105	rc = VERR_VMX_INVALID_GUEST_STATE;
3106	break;
3107	}
3108
3109	case VMX_EXIT_ERR_MSR_LOAD: /* 34 VM-entry failure due to MSR loading. */
3110	case VMX_EXIT_ERR_MACHINE_CHECK: /* 41 VM-entry failure due to machine-check. */
3111	default:
3112	rc = VERR_VMX_UNEXPECTED_EXIT_CODE;
3113	AssertMsgFailed(("Unexpected exit code %d\n", exitReason)); /* Can't happen. */
3114	break;
3115
3116	}
3117	end:
3118
3119	/* Signal changes for the recompiler. */
3120	CPUMSetChangedFlags(pVM, CPUM_CHANGED_SYSENTER_MSR \| CPUM_CHANGED_LDTR \| CPUM_CHANGED_GDTR \| CPUM_CHANGED_IDTR \| CPUM_CHANGED_TR \| CPUM_CHANGED_HIDDEN_SEL_REGS);
3121
3122	/* If we executed vmlaunch/vmresume and an external irq was pending, then we don't have to do a full sync the next time. */
3123	if ( exitReason == VMX_EXIT_EXTERNAL_IRQ
3124	&& !VMX_EXIT_INTERRUPTION_INFO_VALID(intInfo))
3125	{
3126	STAM_COUNTER_INC(&pVCpu->hwaccm.s.StatPendingHostIrq);
3127	/* On the next entry we'll only sync the host context. */
3128	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_HOST_CONTEXT;
3129	}
3130	else
3131	{
3132	/* On the next entry we'll sync everything. */
3133	/** @todo we can do better than this */
3134	/* Not in the VINF_PGM_CHANGE_MODE though! */
3135	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_ALL;
3136	}
3137
3138	/* translate into a less severe return code */
3139	if (rc == VERR_EM_INTERPRETER)
3140	rc = VINF_EM_RAW_EMULATE_INSTR;
3141	else
3142	/* Try to extract more information about what might have gone wrong here. */
3143	if (rc == VERR_VMX_INVALID_VMCS_PTR)
3144	{
3145	VMXGetActivateVMCS(&pVCpu->hwaccm.s.vmx.lasterror.u64VMCSPhys);
3146	pVCpu->hwaccm.s.vmx.lasterror.ulVMCSRevision = (uint32_t )pVCpu->hwaccm.s.vmx.pVMCS;
3147	pVCpu->hwaccm.s.vmx.lasterror.idEnteredCpu = pVCpu->hwaccm.s.idEnteredCpu;
3148	pVCpu->hwaccm.s.vmx.lasterror.idCurrentCpu = RTMpCpuId();
3149	}
3150
3151	STAM_STATS({
3152	if (fStatExit2Started) STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatExit2, y);
3153	else if (fStatEntryStarted) STAM_PROFILE_ADV_STOP(&pVCpu->hwaccm.s.StatEntry, x);
3154	});
3155	Log2(("X"));
3156	return rc;
3157	}
3158
3159
3160	/**
3161	* Enters the VT-x session
3162	*
3163	* @returns VBox status code.
3164	* @param pVM The VM to operate on.
3165	* @param pVCpu The VMCPU to operate on.
3166	* @param pCpu CPU info struct
3167	*/
3168	VMMR0DECL(int) VMXR0Enter(PVM pVM, PVMCPU pVCpu, PHWACCM_CPUINFO pCpu)
3169	{
3170	Assert(pVM->hwaccm.s.vmx.fSupported);
3171
3172	unsigned cr4 = ASMGetCR4();
3173	if (!(cr4 & X86_CR4_VMXE))
3174	{
3175	AssertMsgFailed(("X86_CR4_VMXE should be set!\n"));
3176	return VERR_VMX_X86_CR4_VMXE_CLEARED;
3177	}
3178
3179	/* Activate the VM Control Structure. */
3180	int rc = VMXActivateVMCS(pVCpu->hwaccm.s.vmx.pVMCSPhys);
3181	if (RT_FAILURE(rc))
3182	return rc;
3183
3184	pVCpu->hwaccm.s.fResumeVM = false;
3185	return VINF_SUCCESS;
3186	}
3187
3188
3189	/**
3190	* Leaves the VT-x session
3191	*
3192	* @returns VBox status code.
3193	* @param pVM The VM to operate on.
3194	* @param pVCpu The VMCPU to operate on.
3195	* @param pCtx CPU context
3196	*/
3197	VMMR0DECL(int) VMXR0Leave(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
3198	{
3199	Assert(pVM->hwaccm.s.vmx.fSupported);
3200
3201	/* Save the guest debug state if necessary. */
3202	if (CPUMIsGuestDebugStateActive(pVM))
3203	{
3204	CPUMR0SaveGuestDebugState(pVM, pVCpu, pCtx, true /* save DR6 */);
3205
3206	/* Enable drx move intercepts again. */
3207	pVCpu->hwaccm.s.vmx.proc_ctls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT;
3208	int rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVCpu->hwaccm.s.vmx.proc_ctls);
3209	AssertRC(rc);
3210
3211	/* Resync the debug registers the next time. */
3212	pVCpu->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_GUEST_DEBUG;
3213	}
3214	else
3215	Assert(pVCpu->hwaccm.s.vmx.proc_ctls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT);
3216
3217	/* Flush all pending VMCS writes. */
3218	VMXFlushWriteCache(pVCpu);
3219
3220	/* Clear VM Control Structure. Marking it inactive, clearing implementation specific data and writing back VMCS data to memory. */
3221	int rc = VMXClearVMCS(pVCpu->hwaccm.s.vmx.pVMCSPhys);
3222	AssertRC(rc);
3223
3224	return VINF_SUCCESS;
3225	}
3226
3227	/**
3228	* Flush the TLB (EPT)
3229	*
3230	* @returns VBox status code.
3231	* @param pVM The VM to operate on.
3232	* @param pVCpu The VM CPU to operate on.
3233	* @param enmFlush Type of flush
3234	* @param GCPhys Physical address of the page to flush
3235	*/
3236	static void vmxR0FlushEPT(PVM pVM, PVMCPU pVCpu, VMX_FLUSH enmFlush, RTGCPHYS GCPhys)
3237	{
3238	uint64_t descriptor[2];
3239
3240	LogFlow(("vmxR0FlushEPT %d %RGv\n", enmFlush, GCPhys));
3241	Assert(pVM->hwaccm.s.fNestedPaging);
3242	descriptor[0] = pVCpu->hwaccm.s.vmx.GCPhysEPTP;
3243	descriptor[1] = GCPhys;
3244	int rc = VMXR0InvEPT(enmFlush, &descriptor[0]);
3245	AssertRC(rc);
3246	}
3247
3248	#ifdef HWACCM_VTX_WITH_VPID
3249	/**
3250	* Flush the TLB (EPT)
3251	*
3252	* @returns VBox status code.
3253	* @param pVM The VM to operate on.
3254	* @param pVCpu The VM CPU to operate on.
3255	* @param enmFlush Type of flush
3256	* @param GCPtr Virtual address of the page to flush
3257	*/
3258	static void vmxR0FlushVPID(PVM pVM, PVMCPU pVCpu, VMX_FLUSH enmFlush, RTGCPTR GCPtr)
3259	{
3260	#if HC_ARCH_BITS == 32
3261	/* If we get a flush in 64 bits guest mode, then force a full TLB flush. Invvpid probably takes only 32 bits addresses. (@todo) */
3262	if (CPUMIsGuestInLongMode(pVM))
3263	{
3264	pVCpu->hwaccm.s.fForceTLBFlush = true;
3265	}
3266	else
3267	#endif
3268	{
3269	uint64_t descriptor[2];
3270
3271	Assert(pVM->hwaccm.s.vmx.fVPID);
3272	descriptor[0] = pVCpu->hwaccm.s.uCurrentASID;
3273	descriptor[1] = GCPtr;
3274	int rc = VMXR0InvVPID(enmFlush, &descriptor[0]);
3275	AssertRC(rc);
3276	}
3277	}
3278	#endif /* HWACCM_VTX_WITH_VPID */
3279
3280	/**
3281	* Invalidates a guest page
3282	*
3283	* @returns VBox status code.
3284	* @param pVM The VM to operate on.
3285	* @param pVCpu The VM CPU to operate on.
3286	* @param GCVirt Page to invalidate
3287	*/
3288	VMMR0DECL(int) VMXR0InvalidatePage(PVM pVM, PVMCPU pVCpu, RTGCPTR GCVirt)
3289	{
3290	bool fFlushPending = pVCpu->hwaccm.s.fForceTLBFlush;
3291
3292	LogFlow(("VMXR0InvalidatePage %RGv\n", GCVirt));
3293
3294	/* Only relevant if we want to use VPID.
3295	* In the nested paging case we still see such calls, but
3296	* can safely ignore them. (e.g. after cr3 updates)
3297	*/
3298	#ifdef HWACCM_VTX_WITH_VPID
3299	/* Skip it if a TLB flush is already pending. */
3300	if ( !fFlushPending
3301	&& pVM->hwaccm.s.vmx.fVPID)
3302	vmxR0FlushVPID(pVM, pVCpu, pVM->hwaccm.s.vmx.enmFlushPage, GCVirt);
3303	#endif /* HWACCM_VTX_WITH_VPID */
3304
3305	return VINF_SUCCESS;
3306	}
3307
3308	/**
3309	* Invalidates a guest page by physical address
3310	*
3311	* NOTE: Assumes the current instruction references this physical page though a virtual address!!
3312	*
3313	* @returns VBox status code.
3314	* @param pVM The VM to operate on.
3315	* @param pVCpu The VM CPU to operate on.
3316	* @param GCPhys Page to invalidate
3317	*/
3318	VMMR0DECL(int) VMXR0InvalidatePhysPage(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys)
3319	{
3320	bool fFlushPending = pVCpu->hwaccm.s.fForceTLBFlush;
3321
3322	Assert(pVM->hwaccm.s.fNestedPaging);
3323
3324	LogFlow(("VMXR0InvalidatePhysPage %RGp\n", GCPhys));
3325
3326	/* Skip it if a TLB flush is already pending. */
3327	if (!fFlushPending)
3328	vmxR0FlushEPT(pVM, pVCpu, pVM->hwaccm.s.vmx.enmFlushPage, GCPhys);
3329
3330	return VINF_SUCCESS;
3331	}
3332
3333	/**
3334	* Report world switch error and dump some useful debug info
3335	*
3336	* @param pVM The VM to operate on.
3337	* @param pVCpu The VMCPU to operate on.
3338	* @param rc Return code
3339	* @param pCtx Current CPU context (not updated)
3340	*/
3341	static void VMXR0ReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, int rc, PCPUMCTX pCtx)
3342	{
3343	switch (rc)
3344	{
3345	case VERR_VMX_INVALID_VMXON_PTR:
3346	AssertFailed();
3347	break;
3348
3349	case VERR_VMX_UNABLE_TO_START_VM:
3350	case VERR_VMX_UNABLE_TO_RESUME_VM:
3351	{
3352	int rc;
3353	RTCCUINTREG exitReason, instrError, val;
3354
3355	rc = VMXReadVMCS(VMX_VMCS32_RO_EXIT_REASON, &exitReason);
3356	rc \|= VMXReadVMCS(VMX_VMCS32_RO_VM_INSTR_ERROR, &instrError);
3357	AssertRC(rc);
3358	if (rc == VINF_SUCCESS)
3359	{
3360	Log(("Unable to start/resume VM for reason: %x. Instruction error %x\n", (uint32_t)exitReason, (uint32_t)instrError));
3361	Log(("Current stack %08x\n", &rc));
3362
3363	pVCpu->hwaccm.s.vmx.lasterror.ulInstrError = instrError;
3364	pVCpu->hwaccm.s.vmx.lasterror.ulExitReason = exitReason;
3365
3366	#ifdef VBOX_STRICT
3367	RTGDTR gdtr;
3368	PX86DESCHC pDesc;
3369
3370	ASMGetGDTR(&gdtr);
3371
3372	VMXReadVMCS(VMX_VMCS64_GUEST_RIP, &val);
3373	Log(("Old eip %RGv new %RGv\n", (RTGCPTR)pCtx->rip, (RTGCPTR)val));
3374	VMXReadVMCS(VMX_VMCS_CTRL_PIN_EXEC_CONTROLS, &val);
3375	Log(("VMX_VMCS_CTRL_PIN_EXEC_CONTROLS %08x\n", val));
3376	VMXReadVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, &val);
3377	Log(("VMX_VMCS_CTRL_PROC_EXEC_CONTROLS %08x\n", val));
3378	VMXReadVMCS(VMX_VMCS_CTRL_ENTRY_CONTROLS, &val);
3379	Log(("VMX_VMCS_CTRL_ENTRY_CONTROLS %08x\n", val));
3380	VMXReadVMCS(VMX_VMCS_CTRL_EXIT_CONTROLS, &val);
3381	Log(("VMX_VMCS_CTRL_EXIT_CONTROLS %08x\n", val));
3382
3383	VMXReadVMCS(VMX_VMCS_HOST_CR0, &val);
3384	Log(("VMX_VMCS_HOST_CR0 %08x\n", val));
3385
3386	VMXReadVMCS(VMX_VMCS_HOST_CR3, &val);
3387	Log(("VMX_VMCS_HOST_CR3 %08x\n", val));
3388
3389	VMXReadVMCS(VMX_VMCS_HOST_CR4, &val);
3390	Log(("VMX_VMCS_HOST_CR4 %08x\n", val));
3391
3392	VMXReadVMCS(VMX_VMCS16_HOST_FIELD_CS, &val);
3393	Log(("VMX_VMCS_HOST_FIELD_CS %08x\n", val));
3394
3395	VMXReadVMCS(VMX_VMCS_GUEST_RFLAGS, &val);
3396	Log(("VMX_VMCS_GUEST_RFLAGS %08x\n", val));
3397
3398	if (val < gdtr.cbGdt)
3399	{
3400	pDesc = &((PX86DESCHC)gdtr.pGdt)[val >> X86_SEL_SHIFT_HC];
3401	HWACCMR0DumpDescriptor(pDesc, val, "CS: ");
3402	}
3403
3404	VMXReadVMCS(VMX_VMCS16_HOST_FIELD_DS, &val);
3405	Log(("VMX_VMCS_HOST_FIELD_DS %08x\n", val));
3406	if (val < gdtr.cbGdt)
3407	{
3408	pDesc = &((PX86DESCHC)gdtr.pGdt)[val >> X86_SEL_SHIFT_HC];
3409	HWACCMR0DumpDescriptor(pDesc, val, "DS: ");
3410	}
3411
3412	VMXReadVMCS(VMX_VMCS16_HOST_FIELD_ES, &val);
3413	Log(("VMX_VMCS_HOST_FIELD_ES %08x\n", val));
3414	if (val < gdtr.cbGdt)
3415	{
3416	pDesc = &((PX86DESCHC)gdtr.pGdt)[val >> X86_SEL_SHIFT_HC];
3417	HWACCMR0DumpDescriptor(pDesc, val, "ES: ");
3418	}
3419
3420	VMXReadVMCS(VMX_VMCS16_HOST_FIELD_FS, &val);
3421	Log(("VMX_VMCS16_HOST_FIELD_FS %08x\n", val));
3422	if (val < gdtr.cbGdt)
3423	{
3424	pDesc = &((PX86DESCHC)gdtr.pGdt)[val >> X86_SEL_SHIFT_HC];
3425	HWACCMR0DumpDescriptor(pDesc, val, "FS: ");
3426	}
3427
3428	VMXReadVMCS(VMX_VMCS16_HOST_FIELD_GS, &val);
3429	Log(("VMX_VMCS16_HOST_FIELD_GS %08x\n", val));
3430	if (val < gdtr.cbGdt)
3431	{
3432	pDesc = &((PX86DESCHC)gdtr.pGdt)[val >> X86_SEL_SHIFT_HC];
3433	HWACCMR0DumpDescriptor(pDesc, val, "GS: ");
3434	}
3435
3436	VMXReadVMCS(VMX_VMCS16_HOST_FIELD_SS, &val);
3437	Log(("VMX_VMCS16_HOST_FIELD_SS %08x\n", val));
3438	if (val < gdtr.cbGdt)
3439	{
3440	pDesc = &((PX86DESCHC)gdtr.pGdt)[val >> X86_SEL_SHIFT_HC];
3441	HWACCMR0DumpDescriptor(pDesc, val, "SS: ");
3442	}
3443
3444	VMXReadVMCS(VMX_VMCS16_HOST_FIELD_TR, &val);
3445	Log(("VMX_VMCS16_HOST_FIELD_TR %08x\n", val));
3446	if (val < gdtr.cbGdt)
3447	{
3448	pDesc = &((PX86DESCHC)gdtr.pGdt)[val >> X86_SEL_SHIFT_HC];
3449	HWACCMR0DumpDescriptor(pDesc, val, "TR: ");
3450	}
3451
3452	VMXReadVMCS(VMX_VMCS_HOST_TR_BASE, &val);
3453	Log(("VMX_VMCS_HOST_TR_BASE %RHv\n", val));
3454
3455	VMXReadVMCS(VMX_VMCS_HOST_GDTR_BASE, &val);
3456	Log(("VMX_VMCS_HOST_GDTR_BASE %RHv\n", val));
3457	VMXReadVMCS(VMX_VMCS_HOST_IDTR_BASE, &val);
3458	Log(("VMX_VMCS_HOST_IDTR_BASE %RHv\n", val));
3459
3460	VMXReadVMCS(VMX_VMCS32_HOST_SYSENTER_CS, &val);
3461	Log(("VMX_VMCS_HOST_SYSENTER_CS %08x\n", val));
3462
3463	VMXReadVMCS(VMX_VMCS_HOST_SYSENTER_EIP, &val);
3464	Log(("VMX_VMCS_HOST_SYSENTER_EIP %RHv\n", val));
3465
3466	VMXReadVMCS(VMX_VMCS_HOST_SYSENTER_ESP, &val);
3467	Log(("VMX_VMCS_HOST_SYSENTER_ESP %RHv\n", val));
3468
3469	VMXReadVMCS(VMX_VMCS_HOST_RSP, &val);
3470	Log(("VMX_VMCS_HOST_RSP %RHv\n", val));
3471	VMXReadVMCS(VMX_VMCS_HOST_RIP, &val);
3472	Log(("VMX_VMCS_HOST_RIP %RHv\n", val));
3473
3474	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBIRD_32BIT_KERNEL)
3475	if (VMX_IS_64BIT_HOST_MODE())
3476	{
3477	Log(("MSR_K6_EFER = %RX64\n", ASMRdMsr(MSR_K6_EFER)));
3478	Log(("MSR_K6_STAR = %RX64\n", ASMRdMsr(MSR_K6_STAR)));
3479	Log(("MSR_K8_LSTAR = %RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
3480	Log(("MSR_K8_CSTAR = %RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
3481	Log(("MSR_K8_SF_MASK = %RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
3482	}
3483	# endif
3484	#endif /* VBOX_STRICT */
3485	}
3486	break;
3487	}
3488
3489	default:
3490	/* impossible */
3491	AssertMsgFailed(("%Rrc (%#x)\n", rc, rc));
3492	break;
3493	}
3494	}
3495
3496	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
3497	/**
3498	* Prepares for and executes VMLAUNCH (64 bits guest mode)
3499	*
3500	* @returns VBox status code
3501	* @param fResume vmlauch/vmresume
3502	* @param pCtx Guest context
3503	* @param pCache VMCS cache
3504	* @param pVM The VM to operate on.
3505	* @param pVCpu The VMCPU to operate on.
3506	*/
3507	DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
3508	{
3509	uint32_t aParam[4];
3510	PHWACCM_CPUINFO pCpu;
3511	RTHCPHYS pPageCpuPhys;
3512
3513	pCpu = HWACCMR0GetCurrentCpuEx(pVCpu->idCpu);
3514	pPageCpuPhys = RTR0MemObjGetPagePhysAddr(pCpu->pMemObj, 0);
3515
3516	aParam[0] = (uint32_t)(pPageCpuPhys); /* Param 1: VMXON physical address - Lo. */
3517	aParam[1] = (uint32_t)(pPageCpuPhys >> 32); /* Param 1: VMXON physical address - Hi. */
3518	aParam[2] = (uint32_t)(pVCpu->hwaccm.s.vmx.pVMCSPhys); /* Param 2: VMCS physical address - Lo. */
3519	aParam[3] = (uint32_t)(pVCpu->hwaccm.s.vmx.pVMCSPhys >> 32); /* Param 2: VMCS physical address - Hi. */
3520
3521	return VMXR0Execute64BitsHandler(pVM, pVCpu, pCtx, pVM->hwaccm.s.pfnVMXGCStartVM64, 4, &aParam[0]);
3522	}
3523
3524	/**
3525	* Executes the specified handler in 64 mode
3526	*
3527	* @returns VBox status code.
3528	* @param pVM The VM to operate on.
3529	* @param pVCpu The VMCPU to operate on.
3530	* @param pCtx Guest context
3531	* @param pfnHandler RC handler
3532	* @param cbParam Number of parameters
3533	* @param paParam Array of 32 bits parameters
3534	*/
3535	VMMR0DECL(int) VMXR0Execute64BitsHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, RTRCPTR pfnHandler, uint32_t cbParam, uint32_t *paParam)
3536	{
3537	int rc, rc2;
3538	RTCCUINTREG uFlags;
3539	PHWACCM_CPUINFO pCpu;
3540	RTHCPHYS pPageCpuPhys;
3541
3542	/* @todo This code is not guest SMP safe (hyper context) */
3543	AssertReturn(pVM->cCPUs == 1, VERR_ACCESS_DENIED);
3544	AssertReturn(pVM->hwaccm.s.pfnHost32ToGuest64R0, VERR_INTERNAL_ERROR);
3545
3546	pCpu = HWACCMR0GetCurrentCpuEx(pVCpu->idCpu);
3547	pPageCpuPhys = RTR0MemObjGetPagePhysAddr(pCpu->pMemObj, 0);
3548
3549	/* Clear VM Control Structure. Marking it inactive, clearing implementation specific data and writing back VMCS data to memory. */
3550	VMXClearVMCS(pVCpu->hwaccm.s.vmx.pVMCSPhys);
3551
3552	/* Leave VMX Root Mode. */
3553	VMXDisable();
3554
3555	uFlags = ASMIntDisableFlags();
3556
3557	CPUMSetHyperESP(pVM, VMMGetStackRC(pVM));
3558	CPUMSetHyperEIP(pVM, pfnHandler);
3559	for (int i=(int)cbParam-1;i>=0;i--)
3560	CPUMPushHyper(pVM, paParam[i]);
3561
3562	/* Call switcher. */
3563	rc = pVM->hwaccm.s.pfnHost32ToGuest64R0(pVM);
3564
3565	ASMSetFlags(uFlags);
3566
3567	/* Make sure the VMX instructions don't cause #UD faults. */
3568	ASMSetCR4(ASMGetCR4() \| X86_CR4_VMXE);
3569
3570	/* Enter VMX Root Mode */
3571	rc2 = VMXEnable(pPageCpuPhys);
3572	if (RT_FAILURE(rc2))
3573	{
3574	if (pVM)
3575	VMXR0CheckError(pVM, pVCpu, rc2);
3576	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
3577	return VERR_VMX_VMXON_FAILED;
3578	}
3579
3580	VMXActivateVMCS(pVCpu->hwaccm.s.vmx.pVMCSPhys);
3581	return rc;
3582	}
3583
3584	#endif /* HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) */
3585
3586
3587	#ifdef VMX_USE_CACHED_VMCS_ACCESSES
3588	/**
3589	* Flush the write cache in order not to overflow it with frequent ring switches.
3590	*
3591	* @param pVCpu The VMCPU to operate on.
3592	*/
3593	VMMR0DECL(void) VMXFlushWriteCache(PVMCPU pVCpu)
3594	{
3595	PVMCSCACHE pCache = &pVCpu->hwaccm.s.vmx.VMCSCache;
3596	/* Flush the queued writes first. */
3597	for (unsigned i=0;i<pCache->Write.cValidEntries;i++)
3598	VMXWriteVMCS(pCache->Write.aField[i], pCache->Write.aFieldVal[i]);
3599
3600	pCache->Write.cValidEntries = 0;
3601	}
3602	#endif

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMR0/HWVMXR0.cpp@ 15291

Download in other formats: