HWVMXR0.cpp@ 13085

Last change on this file since 13085 was 13077, checked in by vboxsync, 16 years ago
Corrected assertion.
Property svn:eol-style set to `native` Property svn:keywords set to `Id`
File size: 117.9 KB

Line
1	/* $Id: HWVMXR0.cpp 13077 2008-10-08 12:53:34Z vboxsync $ */
2	/** @file
3	* HWACCM VMX - Host Context Ring 0.
4	*/
5
6	/*
7	* Copyright (C) 2006-2007 Sun Microsystems, Inc.
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*
17	* Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa
18	* Clara, CA 95054 USA or visit http://www.sun.com if you need
19	* additional information or have any questions.
20	*/
21
22
23	/*******************************************************************************
24	* Header Files *
25	*******************************************************************************/
26	#define LOG_GROUP LOG_GROUP_HWACCM
27	#include <VBox/hwaccm.h>
28	#include "HWACCMInternal.h"
29	#include <VBox/vm.h>
30	#include <VBox/x86.h>
31	#include <VBox/pgm.h>
32	#include <VBox/pdm.h>
33	#include <VBox/err.h>
34	#include <VBox/log.h>
35	#include <VBox/selm.h>
36	#include <VBox/iom.h>
37	#include <iprt/param.h>
38	#include <iprt/assert.h>
39	#include <iprt/asm.h>
40	#include <iprt/string.h>
41	#include "HWVMXR0.h"
42
43	/*******************************************************************************
44	* Global Variables *
45	*******************************************************************************/
46	/* IO operation lookup arrays. */
47	static uint32_t const g_aIOSize[4] = {1, 2, 0, 4};
48	static uint32_t const g_aIOOpAnd[4] = {0xff, 0xffff, 0, 0xffffffff};
49
50	/*******************************************************************************
51	* Local Functions *
52	*******************************************************************************/
53	#ifdef VBOX_STRICT
54	static void VMXR0ReportWorldSwitchError(PVM pVM, int rc, PCPUMCTX pCtx);
55	#else
56	#define VMXR0ReportWorldSwitchError(a, b, c) do { } while (0);
57	#endif /* VBOX_STRICT */
58
59
60	static void VMXR0CheckError(PVM pVM, int rc)
61	{
62	if (rc == VERR_VMX_GENERIC)
63	{
64	RTCCUINTREG instrError;
65
66	VMXReadVMCS(VMX_VMCS_RO_VM_INSTR_ERROR, &instrError);
67	pVM->hwaccm.s.vmx.ulLastInstrError = instrError;
68	}
69	pVM->hwaccm.s.lLastError = rc;
70	}
71
72	/**
73	* Sets up and activates VT-x on the current CPU
74	*
75	* @returns VBox status code.
76	* @param pCpu CPU info struct
77	* @param pVM The VM to operate on.
78	* @param pvPageCpu Pointer to the global cpu page
79	* @param pPageCpuPhys Physical address of the global cpu page
80	*/
81	VMMR0DECL(int) VMXR0EnableCpu(PHWACCM_CPUINFO pCpu, PVM pVM, void *pvPageCpu, RTHCPHYS pPageCpuPhys)
82	{
83	AssertReturn(pPageCpuPhys, VERR_INVALID_PARAMETER);
84	AssertReturn(pVM, VERR_INVALID_PARAMETER);
85	AssertReturn(pvPageCpu, VERR_INVALID_PARAMETER);
86
87	/* Setup Intel VMX. */
88	Assert(pVM->hwaccm.s.vmx.fSupported);
89
90	#ifdef LOG_ENABLED
91	SUPR0Printf("VMXR0EnableCpu cpu %d page (%x) %x\n", pCpu->idCpu, pvPageCpu, (uint32_t)pPageCpuPhys);
92	#endif
93	/* Set revision dword at the beginning of the VMXON structure. */
94	(uint32_t )pvPageCpu = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hwaccm.s.vmx.msr.vmx_basic_info);
95
96	/** @todo we should unmap the two pages from the virtual address space in order to prevent accidental corruption.
97	* (which can have very bad consequences!!!)
98	*/
99
100	/* Make sure the VMX instructions don't cause #UD faults. */
101	ASMSetCR4(ASMGetCR4() \| X86_CR4_VMXE);
102
103	/* Enter VMX Root Mode */
104	int rc = VMXEnable(pPageCpuPhys);
105	if (VBOX_FAILURE(rc))
106	{
107	VMXR0CheckError(pVM, rc);
108	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
109	return VERR_VMX_VMXON_FAILED;
110	}
111	return VINF_SUCCESS;
112	}
113
114	/**
115	* Deactivates VT-x on the current CPU
116	*
117	* @returns VBox status code.
118	* @param pCpu CPU info struct
119	* @param pvPageCpu Pointer to the global cpu page
120	* @param pPageCpuPhys Physical address of the global cpu page
121	*/
122	VMMR0DECL(int) VMXR0DisableCpu(PHWACCM_CPUINFO pCpu, void *pvPageCpu, RTHCPHYS pPageCpuPhys)
123	{
124	AssertReturn(pPageCpuPhys, VERR_INVALID_PARAMETER);
125	AssertReturn(pvPageCpu, VERR_INVALID_PARAMETER);
126
127	/* Leave VMX Root Mode. */
128	VMXDisable();
129
130	/* And clear the X86_CR4_VMXE bit */
131	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
132
133	#ifdef LOG_ENABLED
134	SUPR0Printf("VMXR0DisableCpu cpu %d\n", pCpu->idCpu);
135	#endif
136	return VINF_SUCCESS;
137	}
138
139	/**
140	* Does Ring-0 per VM VT-x init.
141	*
142	* @returns VBox status code.
143	* @param pVM The VM to operate on.
144	*/
145	VMMR0DECL(int) VMXR0InitVM(PVM pVM)
146	{
147	int rc;
148
149	#ifdef LOG_ENABLED
150	SUPR0Printf("VMXR0InitVM %x\n", pVM);
151	#endif
152	pVM->hwaccm.s.vmx.pMemObjVMCS = NIL_RTR0MEMOBJ;
153	pVM->hwaccm.s.vmx.pMemObjAPIC = NIL_RTR0MEMOBJ;
154
155
156	/* Allocate one page for the VM control structure (VMCS). */
157	rc = RTR0MemObjAllocCont(&pVM->hwaccm.s.vmx.pMemObjVMCS, 1 << PAGE_SHIFT, true /* executable R0 mapping */);
158	AssertRC(rc);
159	if (RT_FAILURE(rc))
160	return rc;
161
162	pVM->hwaccm.s.vmx.pVMCS = RTR0MemObjAddress(pVM->hwaccm.s.vmx.pMemObjVMCS);
163	pVM->hwaccm.s.vmx.pVMCSPhys = RTR0MemObjGetPagePhysAddr(pVM->hwaccm.s.vmx.pMemObjVMCS, 0);
164	ASMMemZero32(pVM->hwaccm.s.vmx.pVMCS, PAGE_SIZE);
165
166	if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
167	{
168	/* Allocate one page for the virtual APIC mmio cache. */
169	rc = RTR0MemObjAllocCont(&pVM->hwaccm.s.vmx.pMemObjAPIC, 1 << PAGE_SHIFT, true /* executable R0 mapping */);
170	AssertRC(rc);
171	if (RT_FAILURE(rc))
172	return rc;
173
174	pVM->hwaccm.s.vmx.pAPIC = (uint8_t *)RTR0MemObjAddress(pVM->hwaccm.s.vmx.pMemObjAPIC);
175	pVM->hwaccm.s.vmx.pAPICPhys = RTR0MemObjGetPagePhysAddr(pVM->hwaccm.s.vmx.pMemObjAPIC, 0);
176	ASMMemZero32(pVM->hwaccm.s.vmx.pAPIC, PAGE_SIZE);
177	}
178	else
179	{
180	pVM->hwaccm.s.vmx.pMemObjAPIC = 0;
181	pVM->hwaccm.s.vmx.pAPIC = 0;
182	pVM->hwaccm.s.vmx.pAPICPhys = 0;
183	}
184
185	/* Allocate the MSR bitmap if this feature is supported. */
186	if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
187	{
188	rc = RTR0MemObjAllocCont(&pVM->hwaccm.s.vmx.pMemObjMSRBitmap, 1 << PAGE_SHIFT, true /* executable R0 mapping */);
189	AssertRC(rc);
190	if (RT_FAILURE(rc))
191	return rc;
192
193	pVM->hwaccm.s.vmx.pMSRBitmap = (uint8_t *)RTR0MemObjAddress(pVM->hwaccm.s.vmx.pMemObjMSRBitmap);
194	pVM->hwaccm.s.vmx.pMSRBitmapPhys = RTR0MemObjGetPagePhysAddr(pVM->hwaccm.s.vmx.pMemObjMSRBitmap, 0);
195	memset(pVM->hwaccm.s.vmx.pMSRBitmap, 0xff, PAGE_SIZE);
196	}
197
198	/* Current guest paging mode. */
199	pVM->hwaccm.s.vmx.enmCurrGuestMode = PGMMODE_REAL;
200
201	#ifdef LOG_ENABLED
202	SUPR0Printf("VMXR0InitVM %x VMCS=%x (%x)\n", pVM, pVM->hwaccm.s.vmx.pVMCS, (uint32_t)pVM->hwaccm.s.vmx.pVMCSPhys);
203	#endif
204	return VINF_SUCCESS;
205	}
206
207	/**
208	* Does Ring-0 per VM VT-x termination.
209	*
210	* @returns VBox status code.
211	* @param pVM The VM to operate on.
212	*/
213	VMMR0DECL(int) VMXR0TermVM(PVM pVM)
214	{
215	if (pVM->hwaccm.s.vmx.pMemObjVMCS != NIL_RTR0MEMOBJ)
216	{
217	RTR0MemObjFree(pVM->hwaccm.s.vmx.pMemObjVMCS, false);
218	pVM->hwaccm.s.vmx.pMemObjVMCS = NIL_RTR0MEMOBJ;
219	pVM->hwaccm.s.vmx.pVMCS = 0;
220	pVM->hwaccm.s.vmx.pVMCSPhys = 0;
221	}
222	if (pVM->hwaccm.s.vmx.pMemObjAPIC != NIL_RTR0MEMOBJ)
223	{
224	RTR0MemObjFree(pVM->hwaccm.s.vmx.pMemObjAPIC, false);
225	pVM->hwaccm.s.vmx.pMemObjAPIC = NIL_RTR0MEMOBJ;
226	pVM->hwaccm.s.vmx.pAPIC = 0;
227	pVM->hwaccm.s.vmx.pAPICPhys = 0;
228	}
229	if (pVM->hwaccm.s.vmx.pMemObjMSRBitmap != NIL_RTR0MEMOBJ)
230	{
231	RTR0MemObjFree(pVM->hwaccm.s.vmx.pMemObjMSRBitmap, false);
232	pVM->hwaccm.s.vmx.pMemObjMSRBitmap = NIL_RTR0MEMOBJ;
233	pVM->hwaccm.s.vmx.pMSRBitmap = 0;
234	pVM->hwaccm.s.vmx.pMSRBitmapPhys = 0;
235	}
236	return VINF_SUCCESS;
237	}
238
239	/**
240	* Sets up VT-x for the specified VM
241	*
242	* @returns VBox status code.
243	* @param pVM The VM to operate on.
244	*/
245	VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
246	{
247	int rc = VINF_SUCCESS;
248	uint32_t val;
249
250	AssertReturn(pVM, VERR_INVALID_PARAMETER);
251	Assert(pVM->hwaccm.s.vmx.pVMCS);
252
253	/* Set revision dword at the beginning of the VMCS structure. */
254	(uint32_t )pVM->hwaccm.s.vmx.pVMCS = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hwaccm.s.vmx.msr.vmx_basic_info);
255
256	/* Clear VM Control Structure. */
257	Log(("pVMCSPhys = %VHp\n", pVM->hwaccm.s.vmx.pVMCSPhys));
258	rc = VMXClearVMCS(pVM->hwaccm.s.vmx.pVMCSPhys);
259	if (VBOX_FAILURE(rc))
260	goto vmx_end;
261
262	/* Activate the VM Control Structure. */
263	rc = VMXActivateVMCS(pVM->hwaccm.s.vmx.pVMCSPhys);
264	if (VBOX_FAILURE(rc))
265	goto vmx_end;
266
267	/* VMX_VMCS_CTRL_PIN_EXEC_CONTROLS
268	* Set required bits to one and zero according to the MSR capabilities.
269	*/
270	val = pVM->hwaccm.s.vmx.msr.vmx_pin_ctls.n.disallowed0;
271	/* External and non-maskable interrupts cause VM-exits. */
272	val = val \| VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_EXT_INT_EXIT \| VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_NMI_EXIT;
273	val &= pVM->hwaccm.s.vmx.msr.vmx_pin_ctls.n.allowed1;
274
275	rc = VMXWriteVMCS(VMX_VMCS_CTRL_PIN_EXEC_CONTROLS, val);
276	AssertRC(rc);
277
278	/* VMX_VMCS_CTRL_PROC_EXEC_CONTROLS
279	* Set required bits to one and zero according to the MSR capabilities.
280	*/
281	val = pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.disallowed0;
282	/* Program which event cause VM-exits and which features we want to use. */
283	val = val \| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_HLT_EXIT
284	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_TSC_OFFSET
285	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT
286	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_UNCOND_IO_EXIT
287	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MWAIT_EXIT; /* don't execute mwait or else we'll idle inside the guest (host thinks the cpu load is high) */
288
289	/* Without nested paging we should intercept invlpg and cr3 mov instructions. */
290	if (!pVM->hwaccm.s.fNestedPaging)
291	val \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INVLPG_EXIT
292	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
293	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT;
294
295	/* Note: VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MWAIT_EXIT might cause a vmlaunch failure with an invalid control fields error. (combined with some other exit reasons) */
296
297	#if HC_ARCH_BITS == 64
298	if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
299	{
300	/* CR8 reads from the APIC shadow page; writes cause an exit is they lower the TPR below the threshold */
301	val \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW;
302	Assert(pVM->hwaccm.s.vmx.pAPIC);
303	}
304	else
305	/* Exit on CR8 reads & writes in case the TPR shadow feature isn't present. */
306	val \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR8_STORE_EXIT \| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR8_LOAD_EXIT;
307	#endif
308
309	#ifdef VBOX_WITH_VTX_MSR_BITMAPS
310	if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
311	{
312	Assert(pVM->hwaccm.s.vmx.pMSRBitmapPhys);
313	val \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS;
314	}
315	#endif
316
317	/* We will use the secondary control if it's present. */
318	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
319
320	/* Mask away the bits that the CPU doesn't support */
321	/** @todo make sure they don't conflict with the above requirements. */
322	val &= pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1;
323	pVM->hwaccm.s.vmx.proc_ctls = val;
324
325	rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, val);
326	AssertRC(rc);
327
328	if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
329	{
330	/* VMX_VMCS_CTRL_PROC_EXEC_CONTROLS2
331	* Set required bits to one and zero according to the MSR capabilities.
332	*/
333	val = pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2.n.disallowed0;
334	val \|= VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT;
335
336	#ifdef HWACCM_VTX_WITH_EPT
337	if (pVM->hwaccm.s.fNestedPaging)
338	val \|= VMX_VMCS_CTRL_PROC_EXEC2_EPT;
339	#endif
340	/* Mask away the bits that the CPU doesn't support */
341	/** @todo make sure they don't conflict with the above requirements. */
342	val &= pVM->hwaccm.s.vmx.msr.vmx_proc_ctls2.n.allowed1;
343
344	rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS2, val);
345	AssertRC(rc);
346	}
347
348	/* VMX_VMCS_CTRL_CR3_TARGET_COUNT
349	* Set required bits to one and zero according to the MSR capabilities.
350	*/
351	rc = VMXWriteVMCS(VMX_VMCS_CTRL_CR3_TARGET_COUNT, 0);
352	AssertRC(rc);
353
354	/* VMX_VMCS_CTRL_EXIT_CONTROLS
355	* Set required bits to one and zero according to the MSR capabilities.
356	*/
357	val = pVM->hwaccm.s.vmx.msr.vmx_exit.n.disallowed0;
358
359	/* Save debug controls (dr7 & IA32_DEBUGCTL_MSR) (forced to 1 on the 'first' VT-x capable CPUs; this actually includes the newest Nehalem CPUs) */
360	val \|= VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_DEBUG;
361	#if HC_ARCH_BITS == 64
362	val \|= VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_AMD64;
363	#else
364	/* else Must be zero when AMD64 is not available. */
365	#endif
366	val &= pVM->hwaccm.s.vmx.msr.vmx_exit.n.allowed1;
367	/* Don't acknowledge external interrupts on VM-exit. */
368	rc = VMXWriteVMCS(VMX_VMCS_CTRL_EXIT_CONTROLS, val);
369	AssertRC(rc);
370
371	/* Forward all exception except #NM & #PF to the guest.
372	* We always need to check pagefaults since our shadow page table can be out of sync.
373	* And we always lazily sync the FPU & XMM state.
374	*/
375
376	/** @todo Possible optimization:
377	* Keep the FPU and XMM state current in the EM thread. That way there's no need to
378	* lazily sync anything, but the downside is that we can't use the FPU stack or XMM
379	* registers ourselves of course.
380	*
381	* Note: only possible if the current state is actually ours (X86_CR0_TS flag)
382	*/
383	pVM->hwaccm.s.vmx.u32TrapMask = HWACCM_VMX_TRAP_MASK;
384	#ifndef DEBUG
385	if (pVM->hwaccm.s.fNestedPaging)
386	pVM->hwaccm.s.vmx.u32TrapMask &= ~RT_BIT(X86_XCPT_PF); /* no longer need to intercept #PF. */
387	#endif
388	rc = VMXWriteVMCS(VMX_VMCS_CTRL_EXCEPTION_BITMAP, pVM->hwaccm.s.vmx.u32TrapMask);
389	AssertRC(rc);
390
391	/* Don't filter page faults; all of them should cause a switch. */
392	rc = VMXWriteVMCS(VMX_VMCS_CTRL_PAGEFAULT_ERROR_MASK, 0);
393	rc \|= VMXWriteVMCS(VMX_VMCS_CTRL_PAGEFAULT_ERROR_MATCH, 0);
394	AssertRC(rc);
395
396	/* Init TSC offset to zero. */
397	rc = VMXWriteVMCS(VMX_VMCS_CTRL_TSC_OFFSET_FULL, 0);
398	#if HC_ARCH_BITS == 32
399	rc \|= VMXWriteVMCS(VMX_VMCS_CTRL_TSC_OFFSET_HIGH, 0);
400	#endif
401	AssertRC(rc);
402
403	rc = VMXWriteVMCS(VMX_VMCS_CTRL_IO_BITMAP_A_FULL, 0);
404	#if HC_ARCH_BITS == 32
405	rc \|= VMXWriteVMCS(VMX_VMCS_CTRL_IO_BITMAP_A_HIGH, 0);
406	#endif
407	AssertRC(rc);
408
409	rc = VMXWriteVMCS(VMX_VMCS_CTRL_IO_BITMAP_B_FULL, 0);
410	#if HC_ARCH_BITS == 32
411	rc \|= VMXWriteVMCS(VMX_VMCS_CTRL_IO_BITMAP_B_HIGH, 0);
412	#endif
413	AssertRC(rc);
414
415	/* Set the MSR bitmap address. */
416	if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
417	{
418	/* Optional */
419	rc = VMXWriteVMCS(VMX_VMCS_CTRL_MSR_BITMAP_FULL, pVM->hwaccm.s.vmx.pMSRBitmapPhys);
420	#if HC_ARCH_BITS == 32
421	rc \|= VMXWriteVMCS(VMX_VMCS_CTRL_MSR_BITMAP_HIGH, pVM->hwaccm.s.vmx.pMSRBitmapPhys >> 32);
422	#endif
423	AssertRC(rc);
424	}
425
426	/* Clear MSR controls. */
427	rc = VMXWriteVMCS(VMX_VMCS_CTRL_VMEXIT_MSR_STORE_FULL, 0);
428	rc \|= VMXWriteVMCS(VMX_VMCS_CTRL_VMEXIT_MSR_LOAD_FULL, 0);
429	rc \|= VMXWriteVMCS(VMX_VMCS_CTRL_VMENTRY_MSR_LOAD_FULL, 0);
430	#if HC_ARCH_BITS == 32
431	rc \|= VMXWriteVMCS(VMX_VMCS_CTRL_VMEXIT_MSR_STORE_HIGH, 0);
432	rc \|= VMXWriteVMCS(VMX_VMCS_CTRL_VMEXIT_MSR_LOAD_HIGH, 0);
433	rc \|= VMXWriteVMCS(VMX_VMCS_CTRL_VMEXIT_MSR_LOAD_HIGH, 0);
434	#endif
435	rc \|= VMXWriteVMCS(VMX_VMCS_CTRL_EXIT_MSR_STORE_COUNT, 0);
436	rc \|= VMXWriteVMCS(VMX_VMCS_CTRL_EXIT_MSR_LOAD_COUNT, 0);
437	AssertRC(rc);
438
439	if (pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
440	{
441	Assert(pVM->hwaccm.s.vmx.pMemObjAPIC);
442	/* Optional */
443	rc = VMXWriteVMCS(VMX_VMCS_CTRL_TPR_THRESHOLD, 0);
444	rc \|= VMXWriteVMCS(VMX_VMCS_CTRL_VAPIC_PAGEADDR_FULL, pVM->hwaccm.s.vmx.pAPICPhys);
445	#if HC_ARCH_BITS == 32
446	rc \|= VMXWriteVMCS(VMX_VMCS_CTRL_VAPIC_PAGEADDR_HIGH, pVM->hwaccm.s.vmx.pAPICPhys >> 32);
447	#endif
448	AssertRC(rc);
449	}
450
451	/* Set link pointer to -1. Not currently used. */
452	#if HC_ARCH_BITS == 32
453	rc = VMXWriteVMCS(VMX_VMCS_GUEST_LINK_PTR_FULL, 0xFFFFFFFF);
454	rc \|= VMXWriteVMCS(VMX_VMCS_GUEST_LINK_PTR_HIGH, 0xFFFFFFFF);
455	#else
456	rc = VMXWriteVMCS(VMX_VMCS_GUEST_LINK_PTR_FULL, 0xFFFFFFFFFFFFFFFF);
457	#endif
458	AssertRC(rc);
459
460	/* Clear VM Control Structure. Marking it inactive, clearing implementation specific data and writing back VMCS data to memory. */
461	rc = VMXClearVMCS(pVM->hwaccm.s.vmx.pVMCSPhys);
462	AssertRC(rc);
463
464	vmx_end:
465	VMXR0CheckError(pVM, rc);
466	return rc;
467	}
468
469
470	/**
471	* Injects an event (trap or external interrupt)
472	*
473	* @returns VBox status code.
474	* @param pVM The VM to operate on.
475	* @param pCtx CPU Context
476	* @param intInfo VMX interrupt info
477	* @param cbInstr Opcode length of faulting instruction
478	* @param errCode Error code (optional)
479	*/
480	static int VMXR0InjectEvent(PVM pVM, CPUMCTX *pCtx, uint32_t intInfo, uint32_t cbInstr, uint32_t errCode)
481	{
482	int rc;
483
484	#ifdef VBOX_STRICT
485	uint32_t iGate = VMX_EXIT_INTERRUPTION_INFO_VECTOR(intInfo);
486	if (iGate == 0xE)
487	LogFlow(("VMXR0InjectEvent: Injecting interrupt %d at %VGv error code=%08x CR2=%08x intInfo=%08x\n", iGate, pCtx->rip, errCode, pCtx->cr2, intInfo));
488	else
489	if (iGate < 0x20)
490	LogFlow(("VMXR0InjectEvent: Injecting interrupt %d at %VGv error code=%08x\n", iGate, pCtx->rip, errCode));
491	else
492	{
493	LogFlow(("INJ-EI: %x at %VGv\n", iGate, pCtx->rip));
494	Assert(!VM_FF_ISSET(pVM, VM_FF_INHIBIT_INTERRUPTS));
495	Assert(pCtx->eflags.u32 & X86_EFL_IF);
496	}
497	#endif
498
499	#ifdef HWACCM_VMX_EMULATE_REALMODE
500	if (CPUMIsGuestInRealModeEx(pCtx))
501	{
502	/* Injecting events doesn't work right with real mode emulation.
503	* (#GP if we try to inject external hardware interrupts)
504	* Fake an 'int x' instruction. Note that we need to take special precautions when
505	* the inject is interrupted as the normal pending event method seems to be broken in this case.
506	*/
507	LogFlow(("Fake 'int %x' inject (real mode)\n", iGate));
508	/* Make sure the return address is set to the current IP. (ugly hack alert) */
509	pCtx->rip--;
510	cbInstr = 1;
511	intInfo = VMX_EXIT_INTERRUPTION_INFO_VECTOR(intInfo) \| (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
512
513	pVM->hwaccm.s.vmx.RealMode.Event.intInfo = intInfo;
514	pVM->hwaccm.s.vmx.RealMode.Event.fPending = true;
515	pVM->hwaccm.s.vmx.RealMode.eip = pCtx->eip;
516	}
517	#endif /* HWACCM_VMX_EMULATE_REALMODE */
518
519	/* Set event injection state. */
520	rc = VMXWriteVMCS(VMX_VMCS_CTRL_ENTRY_IRQ_INFO, intInfo \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT));
521
522	rc \|= VMXWriteVMCS(VMX_VMCS_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
523	rc \|= VMXWriteVMCS(VMX_VMCS_CTRL_ENTRY_EXCEPTION_ERRCODE, errCode);
524
525	AssertRC(rc);
526	return rc;
527	}
528
529
530	/**
531	* Checks for pending guest interrupts and injects them
532	*
533	* @returns VBox status code.
534	* @param pVM The VM to operate on.
535	* @param pCtx CPU Context
536	*/
537	static int VMXR0CheckPendingInterrupt(PVM pVM, CPUMCTX *pCtx)
538	{
539	int rc;
540
541	/* Dispatch any pending interrupts. (injected before, but a VM exit occurred prematurely) */
542	if (pVM->hwaccm.s.Event.fPending)
543	{
544	Log(("Reinjecting event %VX64 %08x at %VGv cr2=%RX64\n", pVM->hwaccm.s.Event.intInfo, pVM->hwaccm.s.Event.errCode, pCtx->rip, pCtx->cr2));
545	STAM_COUNTER_INC(&pVM->hwaccm.s.StatIntReinject);
546	rc = VMXR0InjectEvent(pVM, pCtx, pVM->hwaccm.s.Event.intInfo, 0, pVM->hwaccm.s.Event.errCode);
547	AssertRC(rc);
548
549	pVM->hwaccm.s.Event.fPending = false;
550	return VINF_SUCCESS;
551	}
552
553	/* When external interrupts are pending, we should exit the VM when IF is set. */
554	if ( !TRPMHasTrap(pVM)
555	&& VM_FF_ISPENDING(pVM, (VM_FF_INTERRUPT_APIC\|VM_FF_INTERRUPT_PIC)))
556	{
557	if (!(pCtx->eflags.u32 & X86_EFL_IF))
558	{
559	if (!(pVM->hwaccm.s.vmx.proc_ctls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_IRQ_WINDOW_EXIT))
560	{
561	LogFlow(("Enable irq window exit!\n"));
562	pVM->hwaccm.s.vmx.proc_ctls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_IRQ_WINDOW_EXIT;
563	rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVM->hwaccm.s.vmx.proc_ctls);
564	AssertRC(rc);
565	}
566	/* else nothing to do but wait */
567	}
568	else
569	if (!VM_FF_ISSET(pVM, VM_FF_INHIBIT_INTERRUPTS))
570	{
571	uint8_t u8Interrupt;
572
573	rc = PDMGetInterrupt(pVM, &u8Interrupt);
574	Log(("Dispatch interrupt: u8Interrupt=%x (%d) rc=%Vrc cs:eip=%04X:%VGv\n", u8Interrupt, u8Interrupt, rc, pCtx->cs, pCtx->rip));
575	if (VBOX_SUCCESS(rc))
576	{
577	rc = TRPMAssertTrap(pVM, u8Interrupt, TRPM_HARDWARE_INT);
578	AssertRC(rc);
579	}
580	else
581	{
582	/* Can only happen in rare cases where a pending interrupt is cleared behind our back */
583	Assert(!VM_FF_ISPENDING(pVM, (VM_FF_INTERRUPT_APIC\|VM_FF_INTERRUPT_PIC)));
584	STAM_COUNTER_INC(&pVM->hwaccm.s.StatSwitchGuestIrq);
585	/* Just continue */
586	}
587	}
588	else
589	Log(("Pending interrupt blocked at %VGv by VM_FF_INHIBIT_INTERRUPTS!!\n", pCtx->rip));
590	}
591
592	#ifdef VBOX_STRICT
593	if (TRPMHasTrap(pVM))
594	{
595	uint8_t u8Vector;
596	rc = TRPMQueryTrapAll(pVM, &u8Vector, 0, 0, 0);
597	AssertRC(rc);
598	}
599	#endif
600
601	if ( pCtx->eflags.u32 & X86_EFL_IF
602	&& (!VM_FF_ISSET(pVM, VM_FF_INHIBIT_INTERRUPTS))
603	&& TRPMHasTrap(pVM)
604	)
605	{
606	uint8_t u8Vector;
607	int rc;
608	TRPMEVENT enmType;
609	RTGCUINTPTR intInfo;
610	RTGCUINT errCode;
611
612	/* If a new event is pending, then dispatch it now. */
613	rc = TRPMQueryTrapAll(pVM, &u8Vector, &enmType, &errCode, 0);
614	AssertRC(rc);
615	Assert(pCtx->eflags.Bits.u1IF == 1 \|\| enmType == TRPM_TRAP);
616	Assert(enmType != TRPM_SOFTWARE_INT);
617
618	/* Clear the pending trap. */
619	rc = TRPMResetTrap(pVM);
620	AssertRC(rc);
621
622	intInfo = u8Vector;
623	intInfo \|= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
624
625	if (enmType == TRPM_TRAP)
626	{
627	switch (u8Vector) {
628	case 8:
629	case 10:
630	case 11:
631	case 12:
632	case 13:
633	case 14:
634	case 17:
635	/* Valid error codes. */
636	intInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
637	break;
638	default:
639	break;
640	}
641	if (u8Vector == X86_XCPT_BP \|\| u8Vector == X86_XCPT_OF)
642	intInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SWEXCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
643	else
644	intInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HWEXCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
645	}
646	else
647	intInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
648
649	STAM_COUNTER_INC(&pVM->hwaccm.s.StatIntInject);
650	rc = VMXR0InjectEvent(pVM, pCtx, intInfo, 0, errCode);
651	AssertRC(rc);
652	} /* if (interrupts can be dispatched) */
653
654	return VINF_SUCCESS;
655	}
656
657	/**
658	* Save the host state
659	*
660	* @returns VBox status code.
661	* @param pVM The VM to operate on.
662	*/
663	VMMR0DECL(int) VMXR0SaveHostState(PVM pVM)
664	{
665	int rc = VINF_SUCCESS;
666
667	/*
668	* Host CPU Context
669	*/
670	if (pVM->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_HOST_CONTEXT)
671	{
672	RTIDTR idtr;
673	RTGDTR gdtr;
674	RTSEL SelTR;
675	PX86DESCHC pDesc;
676	uintptr_t trBase;
677
678	/* Control registers */
679	rc = VMXWriteVMCS(VMX_VMCS_HOST_CR0, ASMGetCR0());
680	rc \|= VMXWriteVMCS(VMX_VMCS_HOST_CR3, ASMGetCR3());
681	rc \|= VMXWriteVMCS(VMX_VMCS_HOST_CR4, ASMGetCR4());
682	AssertRC(rc);
683	Log2(("VMX_VMCS_HOST_CR0 %08x\n", ASMGetCR0()));
684	Log2(("VMX_VMCS_HOST_CR3 %VHp\n", ASMGetCR3()));
685	Log2(("VMX_VMCS_HOST_CR4 %08x\n", ASMGetCR4()));
686
687	/* Selector registers. */
688	rc = VMXWriteVMCS(VMX_VMCS_HOST_FIELD_CS, ASMGetCS());
689	/* Note: VMX is (again) very picky about the RPL of the selectors here; we'll restore them manually. */
690	rc \|= VMXWriteVMCS(VMX_VMCS_HOST_FIELD_DS, 0);
691	rc \|= VMXWriteVMCS(VMX_VMCS_HOST_FIELD_ES, 0);
692	#if HC_ARCH_BITS == 32
693	rc \|= VMXWriteVMCS(VMX_VMCS_HOST_FIELD_FS, 0);
694	rc \|= VMXWriteVMCS(VMX_VMCS_HOST_FIELD_GS, 0);
695	#endif
696	rc \|= VMXWriteVMCS(VMX_VMCS_HOST_FIELD_SS, ASMGetSS());
697	SelTR = ASMGetTR();
698	rc \|= VMXWriteVMCS(VMX_VMCS_HOST_FIELD_TR, SelTR);
699	AssertRC(rc);
700	Log2(("VMX_VMCS_HOST_FIELD_CS %08x\n", ASMGetCS()));
701	Log2(("VMX_VMCS_HOST_FIELD_DS %08x\n", ASMGetDS()));
702	Log2(("VMX_VMCS_HOST_FIELD_ES %08x\n", ASMGetES()));
703	Log2(("VMX_VMCS_HOST_FIELD_FS %08x\n", ASMGetFS()));
704	Log2(("VMX_VMCS_HOST_FIELD_GS %08x\n", ASMGetGS()));
705	Log2(("VMX_VMCS_HOST_FIELD_SS %08x\n", ASMGetSS()));
706	Log2(("VMX_VMCS_HOST_FIELD_TR %08x\n", ASMGetTR()));
707
708	/* GDTR & IDTR */
709	ASMGetGDTR(&gdtr);
710	rc = VMXWriteVMCS(VMX_VMCS_HOST_GDTR_BASE, gdtr.pGdt);
711	ASMGetIDTR(&idtr);
712	rc \|= VMXWriteVMCS(VMX_VMCS_HOST_IDTR_BASE, idtr.pIdt);
713	AssertRC(rc);
714	Log2(("VMX_VMCS_HOST_GDTR_BASE %VHv\n", gdtr.pGdt));
715	Log2(("VMX_VMCS_HOST_IDTR_BASE %VHv\n", idtr.pIdt));
716
717	/* Save the base address of the TR selector. */
718	if (SelTR > gdtr.cbGdt)
719	{
720	AssertMsgFailed(("Invalid TR selector %x. GDTR.cbGdt=%x\n", SelTR, gdtr.cbGdt));
721	return VERR_VMX_INVALID_HOST_STATE;
722	}
723
724	pDesc = &((PX86DESCHC)gdtr.pGdt)[SelTR >> X86_SEL_SHIFT_HC];
725	#if HC_ARCH_BITS == 64
726	trBase = X86DESC64_BASE(*pDesc);
727	#else
728	trBase = X86DESC_BASE(*pDesc);
729	#endif
730	rc = VMXWriteVMCS(VMX_VMCS_HOST_TR_BASE, trBase);
731	AssertRC(rc);
732	Log2(("VMX_VMCS_HOST_TR_BASE %VHv\n", trBase));
733
734	/* FS and GS base. */
735	#if HC_ARCH_BITS == 64
736	Log2(("MSR_K8_FS_BASE = %VHv\n", ASMRdMsr(MSR_K8_FS_BASE)));
737	Log2(("MSR_K8_GS_BASE = %VHv\n", ASMRdMsr(MSR_K8_GS_BASE)));
738	rc = VMXWriteVMCS64(VMX_VMCS_HOST_FS_BASE, ASMRdMsr(MSR_K8_FS_BASE));
739	rc \|= VMXWriteVMCS64(VMX_VMCS_HOST_GS_BASE, ASMRdMsr(MSR_K8_GS_BASE));
740	#endif
741	AssertRC(rc);
742
743	/* Sysenter MSRs. */
744	/** @todo expensive!! */
745	rc = VMXWriteVMCS(VMX_VMCS_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
746	Log2(("VMX_VMCS_HOST_SYSENTER_CS %08x\n", ASMRdMsr_Low(MSR_IA32_SYSENTER_CS)));
747	#if HC_ARCH_BITS == 32
748	rc \|= VMXWriteVMCS(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
749	rc \|= VMXWriteVMCS(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
750	Log2(("VMX_VMCS_HOST_SYSENTER_EIP %VHv\n", ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP)));
751	Log2(("VMX_VMCS_HOST_SYSENTER_ESP %VHv\n", ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP)));
752	#else
753	Log2(("VMX_VMCS_HOST_SYSENTER_EIP %VHv\n", ASMRdMsr(MSR_IA32_SYSENTER_EIP)));
754	Log2(("VMX_VMCS_HOST_SYSENTER_ESP %VHv\n", ASMRdMsr(MSR_IA32_SYSENTER_ESP)));
755	rc \|= VMXWriteVMCS64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
756	rc \|= VMXWriteVMCS64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
757	#endif
758	AssertRC(rc);
759
760	pVM->hwaccm.s.fContextUseFlags &= ~HWACCM_CHANGED_HOST_CONTEXT;
761	}
762	return rc;
763	}
764
765
766	/**
767	* Loads the guest state
768	*
769	* NOTE: Don't do anything here that can cause a jump back to ring 3!!!!!
770	*
771	* @returns VBox status code.
772	* @param pVM The VM to operate on.
773	* @param pCtx Guest context
774	*/
775	VMMR0DECL(int) VMXR0LoadGuestState(PVM pVM, CPUMCTX *pCtx)
776	{
777	int rc = VINF_SUCCESS;
778	RTGCUINTPTR val;
779	X86EFLAGS eflags;
780
781	/* Guest CPU context: ES, CS, SS, DS, FS, GS. */
782	if (pVM->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_SEGMENT_REGS)
783	{
784	#ifdef HWACCM_VMX_EMULATE_REALMODE
785	PGMMODE enmGuestMode = PGMGetGuestMode(pVM);
786	if (pVM->hwaccm.s.vmx.enmCurrGuestMode != enmGuestMode)
787	{
788	# define VTX_CORRECT_PROT_SEL(reg) \
789	{ \
790	if ( pCtx->reg##Hid.u64Base == (pVM->hwaccm.s.vmx.RealMode.reg##Hid.u64Base & 0xfffff) \
791	&& pCtx->reg == ((pVM->hwaccm.s.vmx.RealMode.reg##Hid.u64Base >> 4) & ~X86_SEL_RPL)) \
792	{ \
793	pCtx->reg##Hid = pVM->hwaccm.s.vmx.RealMode.reg##Hid; \
794	pCtx->reg = pVM->hwaccm.s.vmx.RealMode.reg; \
795	} \
796	}
797
798	/* Correct weird requirements for switching to protected mode. */
799	if ( pVM->hwaccm.s.vmx.enmCurrGuestMode == PGMMODE_REAL
800	&& enmGuestMode >= PGMMODE_PROTECTED)
801	{
802	/* DPL of all hidden selector registers must match the current CPL (0). */
803	pCtx->csHid.Attr.n.u2Dpl = 0;
804	pCtx->csHid.Attr.n.u4Type = X86_SEL_TYPE_CODE \| X86_SEL_TYPE_RW_ACC;
805
806	pCtx->dsHid.Attr.n.u2Dpl = 0;
807	pCtx->esHid.Attr.n.u2Dpl = 0;
808	pCtx->fsHid.Attr.n.u2Dpl = 0;
809	pCtx->gsHid.Attr.n.u2Dpl = 0;
810	pCtx->ssHid.Attr.n.u2Dpl = 0;
811
812	/* RPL of all selectors must match the current CPL (0). */
813	pCtx->cs &= ~X86_SEL_RPL;
814	pCtx->ds &= ~X86_SEL_RPL;
815	pCtx->es &= ~X86_SEL_RPL;
816	pCtx->fs &= ~X86_SEL_RPL;
817	pCtx->gs &= ~X86_SEL_RPL;
818	pCtx->ss &= ~X86_SEL_RPL;
819
820	if (pVM->hwaccm.s.vmx.RealMode.fValid)
821	{
822	VTX_CORRECT_PROT_SEL(ds);
823	VTX_CORRECT_PROT_SEL(es);
824	VTX_CORRECT_PROT_SEL(fs);
825	VTX_CORRECT_PROT_SEL(gs);
826	VTX_CORRECT_PROT_SEL(ss);
827	pVM->hwaccm.s.vmx.RealMode.fValid = false;
828	}
829	}
830	else
831	/* Switching from protected mode to real mode. */
832	if ( pVM->hwaccm.s.vmx.enmCurrGuestMode >= PGMMODE_PROTECTED
833	&& enmGuestMode == PGMMODE_REAL)
834	{
835	/* Save the original hidden selectors in case we need to restore them later on. */
836	pVM->hwaccm.s.vmx.RealMode.ds = pCtx->ds;
837	pVM->hwaccm.s.vmx.RealMode.dsHid = pCtx->dsHid;
838	pVM->hwaccm.s.vmx.RealMode.es = pCtx->es;
839	pVM->hwaccm.s.vmx.RealMode.esHid = pCtx->esHid;
840	pVM->hwaccm.s.vmx.RealMode.fs = pCtx->fs;
841	pVM->hwaccm.s.vmx.RealMode.fsHid = pCtx->fsHid;
842	pVM->hwaccm.s.vmx.RealMode.gs = pCtx->gs;
843	pVM->hwaccm.s.vmx.RealMode.gsHid = pCtx->gsHid;
844	pVM->hwaccm.s.vmx.RealMode.ss = pCtx->ss;
845	pVM->hwaccm.s.vmx.RealMode.ssHid = pCtx->ssHid;
846	pVM->hwaccm.s.vmx.RealMode.fValid = true;
847
848	/* The selector value & base must be adjusted or else... */
849	pCtx->cs = pCtx->csHid.u64Base >> 4;
850	pCtx->ds = pCtx->dsHid.u64Base >> 4;
851	pCtx->es = pCtx->esHid.u64Base >> 4;
852	pCtx->fs = pCtx->fsHid.u64Base >> 4;
853	pCtx->gs = pCtx->gsHid.u64Base >> 4;
854	pCtx->ss = pCtx->ssHid.u64Base >> 4;
855
856	/* The limit must also be adjusted. */
857	pCtx->csHid.u32Limit &= 0xffff;
858	pCtx->dsHid.u32Limit &= 0xffff;
859	pCtx->esHid.u32Limit &= 0xffff;
860	pCtx->fsHid.u32Limit &= 0xffff;
861	pCtx->gsHid.u32Limit &= 0xffff;
862	pCtx->ssHid.u32Limit &= 0xffff;
863
864	Assert(pCtx->dsHid.u64Base <= 0xfffff);
865	Assert(pCtx->esHid.u64Base <= 0xfffff);
866	Assert(pCtx->fsHid.u64Base <= 0xfffff);
867	Assert(pCtx->gsHid.u64Base <= 0xfffff);
868	}
869	pVM->hwaccm.s.vmx.enmCurrGuestMode = enmGuestMode;
870	}
871	else
872	/* VT-x will fail with a guest invalid state otherwise... (CPU state after a reset) */
873	if ( CPUMIsGuestInRealModeEx(pCtx)
874	&& pCtx->csHid.u64Base == 0xffff0000)
875	{
876	pCtx->csHid.u64Base = 0xf0000;
877	pCtx->cs = 0xf000;
878	}
879	#endif /* HWACCM_VMX_EMULATE_REALMODE */
880
881	VMX_WRITE_SELREG(ES, es);
882	AssertRC(rc);
883
884	VMX_WRITE_SELREG(CS, cs);
885	AssertRC(rc);
886
887	VMX_WRITE_SELREG(SS, ss);
888	AssertRC(rc);
889
890	VMX_WRITE_SELREG(DS, ds);
891	AssertRC(rc);
892
893	/* The base values in the hidden fs & gs registers are not in sync with the msrs; they are cut to 32 bits. */
894	VMX_WRITE_SELREG(FS, fs);
895	AssertRC(rc);
896
897	VMX_WRITE_SELREG(GS, gs);
898	AssertRC(rc);
899	}
900
901	/* Guest CPU context: LDTR. */
902	if (pVM->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_LDTR)
903	{
904	if (pCtx->ldtr == 0)
905	{
906	rc = VMXWriteVMCS(VMX_VMCS_GUEST_FIELD_LDTR, 0);
907	rc \|= VMXWriteVMCS(VMX_VMCS_GUEST_LDTR_LIMIT, 0);
908	rc \|= VMXWriteVMCS(VMX_VMCS_GUEST_LDTR_BASE, 0);
909	/* Note: vmlaunch will fail with 0 or just 0x02. No idea why. */
910	rc \|= VMXWriteVMCS(VMX_VMCS_GUEST_LDTR_ACCESS_RIGHTS, 0x82 /* present, LDT */);
911	}
912	else
913	{
914	rc = VMXWriteVMCS(VMX_VMCS_GUEST_FIELD_LDTR, pCtx->ldtr);
915	rc \|= VMXWriteVMCS(VMX_VMCS_GUEST_LDTR_LIMIT, pCtx->ldtrHid.u32Limit);
916	rc \|= VMXWriteVMCS(VMX_VMCS_GUEST_LDTR_BASE, pCtx->ldtrHid.u64Base);
917	rc \|= VMXWriteVMCS(VMX_VMCS_GUEST_LDTR_ACCESS_RIGHTS, pCtx->ldtrHid.Attr.u);
918	}
919	AssertRC(rc);
920	}
921	/* Guest CPU context: TR. */
922	if (pVM->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_TR)
923	{
924	#ifdef HWACCM_VMX_EMULATE_REALMODE
925	/* Real mode emulation using v86 mode with CR4.VME (interrupt redirection using the int bitmap in the TSS) */
926	if (CPUMIsGuestInRealModeEx(pCtx))
927	{
928	RTGCPHYS GCPhys;
929
930	/* We convert it here every time as pci regions could be reconfigured. */
931	rc = PDMVMMDevHeapR3ToGCPhys(pVM, pVM->hwaccm.s.vmx.pRealModeTSS, &GCPhys);
932	AssertRC(rc);
933
934	rc = VMXWriteVMCS(VMX_VMCS_GUEST_FIELD_TR, 0);
935	rc \|= VMXWriteVMCS(VMX_VMCS_GUEST_TR_LIMIT, HWACCM_VTX_TSS_SIZE);
936	rc \|= VMXWriteVMCS(VMX_VMCS_GUEST_TR_BASE, GCPhys /* phys = virt in this mode */);
937
938	X86DESCATTR attr;
939
940	attr.u = 0;
941	attr.n.u1Present = 1;
942	attr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
943	val = attr.u;
944	}
945	else
946	#endif /* HWACCM_VMX_EMULATE_REALMODE */
947	{
948	rc = VMXWriteVMCS(VMX_VMCS_GUEST_FIELD_TR, pCtx->tr);
949	rc \|= VMXWriteVMCS(VMX_VMCS_GUEST_TR_LIMIT, pCtx->trHid.u32Limit);
950	rc \|= VMXWriteVMCS(VMX_VMCS_GUEST_TR_BASE, pCtx->trHid.u64Base);
951
952	val = pCtx->trHid.Attr.u;
953
954	/* The TSS selector must be busy. */
955	if ((val & 0xF) == X86_SEL_TYPE_SYS_286_TSS_AVAIL)
956	val = (val & ~0xF) \| X86_SEL_TYPE_SYS_286_TSS_BUSY;
957	else
958	/* Default even if no TR selector has been set (otherwise vmlaunch will fail!) */
959	val = (val & ~0xF) \| X86_SEL_TYPE_SYS_386_TSS_BUSY;
960
961	}
962	rc \|= VMXWriteVMCS(VMX_VMCS_GUEST_TR_ACCESS_RIGHTS, val);
963	AssertRC(rc);
964	}
965	/* Guest CPU context: GDTR. */
966	if (pVM->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_GDTR)
967	{
968	rc = VMXWriteVMCS(VMX_VMCS_GUEST_GDTR_LIMIT, pCtx->gdtr.cbGdt);
969	rc \|= VMXWriteVMCS(VMX_VMCS_GUEST_GDTR_BASE, pCtx->gdtr.pGdt);
970	AssertRC(rc);
971	}
972	/* Guest CPU context: IDTR. */
973	if (pVM->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_IDTR)
974	{
975	rc = VMXWriteVMCS(VMX_VMCS_GUEST_IDTR_LIMIT, pCtx->idtr.cbIdt);
976	rc \|= VMXWriteVMCS(VMX_VMCS_GUEST_IDTR_BASE, pCtx->idtr.pIdt);
977	AssertRC(rc);
978	}
979
980	/*
981	* Sysenter MSRs (unconditional)
982	*/
983	rc = VMXWriteVMCS(VMX_VMCS_GUEST_SYSENTER_CS, pCtx->SysEnter.cs);
984	rc \|= VMXWriteVMCS(VMX_VMCS_GUEST_SYSENTER_EIP, pCtx->SysEnter.eip);
985	rc \|= VMXWriteVMCS(VMX_VMCS_GUEST_SYSENTER_ESP, pCtx->SysEnter.esp);
986	AssertRC(rc);
987
988	/* Control registers */
989	if (pVM->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_CR0)
990	{
991	val = pCtx->cr0;
992	rc = VMXWriteVMCS(VMX_VMCS_CTRL_CR0_READ_SHADOW, val);
993	Log2(("Guest CR0-shadow %08x\n", val));
994	if (CPUMIsGuestFPUStateActive(pVM) == false)
995	{
996	/* Always use #NM exceptions to load the FPU/XMM state on demand. */
997	val \|= X86_CR0_TS \| X86_CR0_ET \| X86_CR0_NE \| X86_CR0_MP;
998	}
999	else
1000	{
1001	/** @todo check if we support the old style mess correctly. */
1002	if (!(val & X86_CR0_NE))
1003	{
1004	Log(("Forcing X86_CR0_NE!!!\n"));
1005
1006	/* Also catch floating point exceptions as we need to report them to the guest in a different way. */
1007	if (!pVM->hwaccm.s.fFPUOldStyleOverride)
1008	{
1009	pVM->hwaccm.s.vmx.u32TrapMask \|= RT_BIT(X86_XCPT_MF);
1010	rc = VMXWriteVMCS(VMX_VMCS_CTRL_EXCEPTION_BITMAP, pVM->hwaccm.s.vmx.u32TrapMask);
1011	AssertRC(rc);
1012	pVM->hwaccm.s.fFPUOldStyleOverride = true;
1013	}
1014	}
1015
1016	val \|= X86_CR0_NE; /* always turn on the native mechanism to report FPU errors (old style uses interrupts) */
1017	}
1018	/* Note: protected mode & paging are always enabled; we use them for emulating real and protected mode without paging too. */
1019	val \|= X86_CR0_PE \| X86_CR0_PG;
1020	if (pVM->hwaccm.s.fNestedPaging)
1021	{
1022	if (!(pCtx->cr0 & X86_CR0_PG))
1023	{
1024	/* Reenable cr3 read/write monitoring as our identity mapped page table is active. */
1025	pVM->hwaccm.s.vmx.proc_ctls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
1026	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT;
1027	}
1028	else
1029	{
1030	/* Disable cr3 read/write monitoring as we don't need it for EPT. */
1031	pVM->hwaccm.s.vmx.proc_ctls &= ~( VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
1032	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT);
1033	}
1034	rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVM->hwaccm.s.vmx.proc_ctls);
1035	AssertRC(rc);
1036	}
1037	else
1038	{
1039	/* Note: We must also set this as we rely on protecting various pages for which supervisor writes must be caught. */
1040	val \|= X86_CR0_WP;
1041	}
1042
1043	/* Always enable caching. */
1044	val &= ~(X86_CR0_CD\|X86_CR0_NW);
1045
1046	rc \|= VMXWriteVMCS(VMX_VMCS_GUEST_CR0, val);
1047	Log2(("Guest CR0 %08x\n", val));
1048	/* CR0 flags owned by the host; if the guests attempts to change them, then
1049	* the VM will exit.
1050	*/
1051	val = X86_CR0_PE /* Must monitor this bit (assumptions are made for real mode emulation) */
1052	\| X86_CR0_WP /* Must monitor this bit (it must always be enabled). */
1053	\| X86_CR0_PG /* Must monitor this bit (assumptions are made for real mode & protected mode without paging emulation) */
1054	\| X86_CR0_TS
1055	\| X86_CR0_ET /* Bit not restored during VM-exit! */
1056	\| X86_CR0_CD /* Bit not restored during VM-exit! */
1057	\| X86_CR0_NW /* Bit not restored during VM-exit! */
1058	\| X86_CR0_NE
1059	\| X86_CR0_MP;
1060	pVM->hwaccm.s.vmx.cr0_mask = val;
1061
1062	rc \|= VMXWriteVMCS(VMX_VMCS_CTRL_CR0_MASK, val);
1063	Log2(("Guest CR0-mask %08x\n", val));
1064	AssertRC(rc);
1065	}
1066	if (pVM->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_CR4)
1067	{
1068	/* CR4 */
1069	rc = VMXWriteVMCS(VMX_VMCS_CTRL_CR4_READ_SHADOW, pCtx->cr4);
1070	Log2(("Guest CR4-shadow %08x\n", pCtx->cr4));
1071	/* Set the required bits in cr4 too (currently X86_CR4_VMXE). */
1072	val = pCtx->cr4 \| (uint32_t)pVM->hwaccm.s.vmx.msr.vmx_cr4_fixed0;
1073
1074	if (!pVM->hwaccm.s.fNestedPaging)
1075	{
1076	switch(pVM->hwaccm.s.enmShadowMode)
1077	{
1078	case PGMMODE_REAL: /* Real mode -> emulated using v86 mode */
1079	case PGMMODE_PROTECTED: /* Protected mode, no paging -> emulated using identity mapping. */
1080	case PGMMODE_32_BIT: /* 32-bit paging. */
1081	break;
1082
1083	case PGMMODE_PAE: /* PAE paging. */
1084	case PGMMODE_PAE_NX: /* PAE paging with NX enabled. */
1085	/** @todo use normal 32 bits paging */
1086	val \|= X86_CR4_PAE;
1087	break;
1088
1089	case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
1090	case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
1091	#ifdef VBOX_ENABLE_64_BITS_GUESTS
1092	break;
1093	#else
1094	AssertFailed();
1095	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
1096	#endif
1097	default: /* shut up gcc */
1098	AssertFailed();
1099	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
1100	}
1101	}
1102	else
1103	if (!(pCtx->cr0 & X86_CR0_PG))
1104	{
1105	/* We use 4 MB pages in our identity mapping page table for real and protected mode without paging. */
1106	val \|= X86_CR4_PSE;
1107	}
1108
1109	#ifdef HWACCM_VMX_EMULATE_REALMODE
1110	/* Real mode emulation using v86 mode with CR4.VME (interrupt redirection using the int bitmap in the TSS) */
1111	if (CPUMIsGuestInRealModeEx(pCtx))
1112	val \|= X86_CR4_VME;
1113	#endif /* HWACCM_VMX_EMULATE_REALMODE */
1114
1115	rc \|= VMXWriteVMCS(VMX_VMCS_GUEST_CR4, val);
1116	Log2(("Guest CR4 %08x\n", val));
1117	/* CR4 flags owned by the host; if the guests attempts to change them, then
1118	* the VM will exit.
1119	*/
1120	val = 0
1121	#ifdef HWACCM_VMX_EMULATE_REALMODE
1122	\| X86_CR4_VME
1123	#endif
1124	\| X86_CR4_PAE
1125	\| X86_CR4_PGE
1126	\| X86_CR4_PSE
1127	\| X86_CR4_VMXE;
1128	pVM->hwaccm.s.vmx.cr4_mask = val;
1129
1130	rc \|= VMXWriteVMCS(VMX_VMCS_CTRL_CR4_MASK, val);
1131	Log2(("Guest CR4-mask %08x\n", val));
1132	AssertRC(rc);
1133	}
1134
1135	if (pVM->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_CR3)
1136	{
1137	val = PGMGetHyperCR3(pVM);
1138	Assert(val);
1139	if (pVM->hwaccm.s.fNestedPaging)
1140	{
1141	Assert(!(val & 0xfff));
1142	/** @todo Check the IA32_VMX_EPT_VPID_CAP MSR for other supported memory types. */
1143	val \|= VMX_EPT_MEMTYPE_WB
1144	\| (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
1145	#if HC_ARCH_BITS == 64
1146	rc = VMXWriteVMCS(VMX_VMCS_CTRL_EPTP_FULL, val);
1147	#else
1148	rc = VMXWriteVMCS(VMX_VMCS_CTRL_EPTP_FULL, val);
1149	rc = VMXWriteVMCS(VMX_VMCS_CTRL_EPTP_HIGH, val);
1150	#endif
1151	AssertRC(rc);
1152
1153	if (!(pCtx->cr0 & X86_CR0_PG))
1154	{
1155	RTGCPHYS GCPhys;
1156
1157	/* We convert it here every time as pci regions could be reconfigured. */
1158	rc = PDMVMMDevHeapR3ToGCPhys(pVM, pVM->hwaccm.s.vmx.pRealModeEPTPageTable, &GCPhys);
1159	AssertRC(rc);
1160
1161	/* We use our identity mapping page table here as we need to map guest virtual to guest physical addresses; EPT will
1162	* take care of the translation to host physical addresses.
1163	*/
1164	val = GCPhys;
1165	}
1166	else
1167	{
1168	/* Save the real guest CR3 in VMX_VMCS_GUEST_CR3 */
1169	val = pCtx->cr3;
1170	}
1171	}
1172	/* Save our shadow CR3 register. */
1173	rc = VMXWriteVMCS(VMX_VMCS_GUEST_CR3, val);
1174	AssertRC(rc);
1175	}
1176
1177	/* Debug registers. */
1178	if (pVM->hwaccm.s.fContextUseFlags & HWACCM_CHANGED_GUEST_DEBUG)
1179	{
1180	pCtx->dr[6] \|= X86_DR6_INIT_VAL; /* set all reserved bits to 1. */
1181	pCtx->dr[6] &= ~RT_BIT(12); /* must be zero. */
1182
1183	pCtx->dr[7] &= 0xffffffff; /* upper 32 bits reserved */
1184	pCtx->dr[7] &= ~(RT_BIT(11) \| RT_BIT(12) \| RT_BIT(14) \| RT_BIT(15)); /* must be zero */
1185	pCtx->dr[7] \|= 0x400; /* must be one */
1186
1187	/* Resync DR7 */
1188	rc = VMXWriteVMCS(VMX_VMCS_GUEST_DR7, pCtx->dr[7]);
1189	AssertRC(rc);
1190
1191	/* Sync the debug state now if any breakpoint is armed. */
1192	if ( (pCtx->dr[7] & (X86_DR7_ENABLED_MASK\|X86_DR7_GD))
1193	&& !CPUMIsGuestDebugStateActive(pVM)
1194	&& !DBGFIsStepping(pVM))
1195	{
1196	STAM_COUNTER_INC(&pVM->hwaccm.s.StatDRxArmed);
1197
1198	/* Disable drx move intercepts. */
1199	pVM->hwaccm.s.vmx.proc_ctls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT;
1200	rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVM->hwaccm.s.vmx.proc_ctls);
1201	AssertRC(rc);
1202
1203	/* Save the host and load the guest debug state. */
1204	rc = CPUMR0LoadGuestDebugState(pVM, pCtx, true /* include DR6 */);
1205	AssertRC(rc);
1206	}
1207
1208	/* IA32_DEBUGCTL MSR. */
1209	rc = VMXWriteVMCS(VMX_VMCS_GUEST_DEBUGCTL_FULL, 0);
1210	rc \|= VMXWriteVMCS(VMX_VMCS_GUEST_DEBUGCTL_HIGH, 0);
1211	AssertRC(rc);
1212
1213	/** @todo do we really ever need this? */
1214	rc \|= VMXWriteVMCS(VMX_VMCS_GUEST_DEBUG_EXCEPTIONS, 0);
1215	AssertRC(rc);
1216	}
1217
1218	/* EIP, ESP and EFLAGS */
1219	rc = VMXWriteVMCS(VMX_VMCS_GUEST_RIP, pCtx->rip);
1220	rc \|= VMXWriteVMCS(VMX_VMCS_GUEST_RSP, pCtx->rsp);
1221	AssertRC(rc);
1222
1223	/* Bits 22-31, 15, 5 & 3 must be zero. Bit 1 must be 1. */
1224	eflags = pCtx->eflags;
1225	eflags.u32 &= VMX_EFLAGS_RESERVED_0;
1226	eflags.u32 \|= VMX_EFLAGS_RESERVED_1;
1227
1228	#ifdef HWACCM_VMX_EMULATE_REALMODE
1229	/* Real mode emulation using v86 mode with CR4.VME (interrupt redirection using the int bitmap in the TSS) */
1230	if (CPUMIsGuestInRealModeEx(pCtx))
1231	{
1232	eflags.Bits.u1VM = 1;
1233	eflags.Bits.u2IOPL = 3;
1234	}
1235	#endif /* HWACCM_VMX_EMULATE_REALMODE */
1236	rc = VMXWriteVMCS(VMX_VMCS_GUEST_RFLAGS, eflags.u32);
1237	AssertRC(rc);
1238
1239	/* TSC offset. */
1240	uint64_t u64TSCOffset;
1241
1242	if (TMCpuTickCanUseRealTSC(pVM, &u64TSCOffset))
1243	{
1244	/* Note: VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT takes precedence over TSC_OFFSET */
1245	#if HC_ARCH_BITS == 64
1246	rc = VMXWriteVMCS(VMX_VMCS_CTRL_TSC_OFFSET_FULL, u64TSCOffset);
1247	#else
1248	rc = VMXWriteVMCS(VMX_VMCS_CTRL_TSC_OFFSET_FULL, (uint32_t)u64TSCOffset);
1249	rc \|= VMXWriteVMCS(VMX_VMCS_CTRL_TSC_OFFSET_HIGH, (uint32_t)(u64TSCOffset >> 32ULL));
1250	#endif
1251	AssertRC(rc);
1252
1253	pVM->hwaccm.s.vmx.proc_ctls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT;
1254	rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVM->hwaccm.s.vmx.proc_ctls);
1255	AssertRC(rc);
1256	STAM_COUNTER_INC(&pVM->hwaccm.s.StatTSCOffset);
1257	}
1258	else
1259	{
1260	pVM->hwaccm.s.vmx.proc_ctls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT;
1261	rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVM->hwaccm.s.vmx.proc_ctls);
1262	AssertRC(rc);
1263	STAM_COUNTER_INC(&pVM->hwaccm.s.StatTSCIntercept);
1264	}
1265
1266	/* VMX_VMCS_CTRL_ENTRY_CONTROLS
1267	* Set required bits to one and zero according to the MSR capabilities.
1268	*/
1269	val = pVM->hwaccm.s.vmx.msr.vmx_entry.n.disallowed0;
1270	/* Load guest debug controls (dr7 & IA32_DEBUGCTL_MSR) (forced to 1 on the 'first' VT-x capable CPUs; this actually includes the newest Nehalem CPUs) */
1271	val \|= VMX_VMCS_CTRL_ENTRY_CONTROLS_LOAD_DEBUG;
1272
1273	/* 64 bits guest mode? */
1274	if (pCtx->msrEFER & MSR_K6_EFER_LMA)
1275	val \|= VMX_VMCS_CTRL_ENTRY_CONTROLS_IA64_MODE;
1276	/* else Must be zero when AMD64 is not available. */
1277
1278	/* Mask away the bits that the CPU doesn't support */
1279	val &= pVM->hwaccm.s.vmx.msr.vmx_entry.n.allowed1;
1280	rc = VMXWriteVMCS(VMX_VMCS_CTRL_ENTRY_CONTROLS, val);
1281	AssertRC(rc);
1282
1283	/* 64 bits guest mode? */
1284	if (pCtx->msrEFER & MSR_K6_EFER_LMA)
1285	{
1286	#if !defined(VBOX_WITH_64_BITS_GUESTS) \|\| HC_ARCH_BITS != 64
1287	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
1288	#else
1289	pVM->hwaccm.s.vmx.pfnStartVM = VMXR0StartVM64;
1290	#endif
1291	/* Unconditionally update these as wrmsr might have changed them. */
1292	rc = VMXWriteVMCS(VMX_VMCS_GUEST_FS_BASE, pCtx->fsHid.u64Base);
1293	AssertRC(rc);
1294	rc = VMXWriteVMCS(VMX_VMCS_GUEST_GS_BASE, pCtx->gsHid.u64Base);
1295	AssertRC(rc);
1296	}
1297	else
1298	{
1299	pVM->hwaccm.s.vmx.pfnStartVM = VMXR0StartVM32;
1300	}
1301
1302	#ifdef DEBUG
1303	/* Intercept X86_XCPT_DB if stepping is enabled */
1304	if (DBGFIsStepping(pVM))
1305	pVM->hwaccm.s.vmx.u32TrapMask \|= RT_BIT(X86_XCPT_DB);
1306	else
1307	pVM->hwaccm.s.vmx.u32TrapMask &= ~RT_BIT(X86_XCPT_DB);
1308
1309	rc = VMXWriteVMCS(VMX_VMCS_CTRL_EXCEPTION_BITMAP, pVM->hwaccm.s.vmx.u32TrapMask);
1310	#endif
1311
1312	#ifdef VBOX_STRICT
1313	Assert(pVM->hwaccm.s.vmx.u32TrapMask & RT_BIT(X86_XCPT_GP));
1314	#else
1315	# ifdef HWACCM_VMX_EMULATE_REALMODE
1316	/* Intercept #GP faults in real mode to handle privileged instructions. */
1317	if (CPUMIsGuestInRealModeEx(pCtx))
1318	pVM->hwaccm.s.vmx.u32TrapMask \|= RT_BIT(X86_XCPT_GP);
1319	else
1320	pVM->hwaccm.s.vmx.u32TrapMask &= ~RT_BIT(X86_XCPT_GP);
1321	# endif /* HWACCM_VMX_EMULATE_REALMODE */
1322	rc = VMXWriteVMCS(VMX_VMCS_CTRL_EXCEPTION_BITMAP, pVM->hwaccm.s.vmx.u32TrapMask);
1323	AssertRC(rc);
1324	#endif
1325
1326	/* Done. */
1327	pVM->hwaccm.s.fContextUseFlags &= ~HWACCM_CHANGED_ALL_GUEST;
1328
1329	return rc;
1330	}
1331
1332	/**
1333	* Syncs back the guest state
1334	*
1335	* @returns VBox status code.
1336	* @param pVM The VM to operate on.
1337	* @param pCtx Guest context
1338	*/
1339	DECLINLINE(int) VMXR0SaveGuestState(PVM pVM, CPUMCTX *pCtx)
1340	{
1341	RTCCUINTREG val, valShadow;
1342	RTGCUINTPTR uInterruptState;
1343	int rc;
1344
1345	/* Let's first sync back eip, esp, and eflags. */
1346	rc = VMXReadVMCS(VMX_VMCS_GUEST_RIP, &val);
1347	AssertRC(rc);
1348	pCtx->rip = val;
1349	rc = VMXReadVMCS(VMX_VMCS_GUEST_RSP, &val);
1350	AssertRC(rc);
1351	pCtx->rsp = val;
1352	rc = VMXReadVMCS(VMX_VMCS_GUEST_RFLAGS, &val);
1353	AssertRC(rc);
1354	pCtx->eflags.u32 = val;
1355
1356	/* Take care of instruction fusing (sti, mov ss) */
1357	rc \|= VMXReadVMCS(VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE, &val);
1358	uInterruptState = val;
1359	if (uInterruptState != 0)
1360	{
1361	Assert(uInterruptState <= 2); /* only sti & mov ss */
1362	Log(("uInterruptState %x eip=%VGv\n", uInterruptState, pCtx->rip));
1363	EMSetInhibitInterruptsPC(pVM, pCtx->rip);
1364	}
1365	else
1366	VM_FF_CLEAR(pVM, VM_FF_INHIBIT_INTERRUPTS);
1367
1368	/* Control registers. */
1369	VMXReadVMCS(VMX_VMCS_CTRL_CR0_READ_SHADOW, &valShadow);
1370	VMXReadVMCS(VMX_VMCS_GUEST_CR0, &val);
1371	val = (valShadow & pVM->hwaccm.s.vmx.cr0_mask) \| (val & ~pVM->hwaccm.s.vmx.cr0_mask);
1372	CPUMSetGuestCR0(pVM, val);
1373
1374	VMXReadVMCS(VMX_VMCS_CTRL_CR4_READ_SHADOW, &valShadow);
1375	VMXReadVMCS(VMX_VMCS_GUEST_CR4, &val);
1376	val = (valShadow & pVM->hwaccm.s.vmx.cr4_mask) \| (val & ~pVM->hwaccm.s.vmx.cr4_mask);
1377	CPUMSetGuestCR4(pVM, val);
1378
1379	/* Can be updated behind our back in the nested paging case. */
1380	CPUMSetGuestCR2(pVM, ASMGetCR2());
1381
1382	/* Note: no reason to sync back the CRx registers. They can't be changed by the guest. */
1383	/* Note: only in the nested paging case can CR3 & CR4 be changed by the guest. */
1384	if (pVM->hwaccm.s.fNestedPaging)
1385	{
1386	VMXReadVMCS(VMX_VMCS_GUEST_CR3, &val);
1387
1388	if (val != pCtx->cr3)
1389	{
1390	CPUMSetGuestCR3(pVM, val);
1391	PGMUpdateCR3(pVM, val);
1392	}
1393	}
1394
1395	/* Sync back DR7 here. */
1396	VMXReadVMCS(VMX_VMCS_GUEST_DR7, &val);
1397	pCtx->dr[7] = val;
1398
1399	/* Guest CPU context: ES, CS, SS, DS, FS, GS. */
1400	VMX_READ_SELREG(ES, es);
1401	VMX_READ_SELREG(SS, ss);
1402	VMX_READ_SELREG(CS, cs);
1403	VMX_READ_SELREG(DS, ds);
1404	VMX_READ_SELREG(FS, fs);
1405	VMX_READ_SELREG(GS, gs);
1406
1407	/*
1408	* System MSRs
1409	*/
1410	VMXReadVMCS(VMX_VMCS_GUEST_SYSENTER_CS, &val);
1411	pCtx->SysEnter.cs = val;
1412	VMXReadVMCS(VMX_VMCS_GUEST_SYSENTER_EIP, &val);
1413	pCtx->SysEnter.eip = val;
1414	VMXReadVMCS(VMX_VMCS_GUEST_SYSENTER_ESP, &val);
1415	pCtx->SysEnter.esp = val;
1416
1417	/* Misc. registers; must sync everything otherwise we can get out of sync when jumping to ring 3. */
1418	VMX_READ_SELREG(LDTR, ldtr);
1419
1420	VMXReadVMCS(VMX_VMCS_GUEST_GDTR_LIMIT, &val);
1421	pCtx->gdtr.cbGdt = val;
1422	VMXReadVMCS(VMX_VMCS_GUEST_GDTR_BASE, &val);
1423	pCtx->gdtr.pGdt = val;
1424
1425	VMXReadVMCS(VMX_VMCS_GUEST_IDTR_LIMIT, &val);
1426	pCtx->idtr.cbIdt = val;
1427	VMXReadVMCS(VMX_VMCS_GUEST_IDTR_BASE, &val);
1428	pCtx->idtr.pIdt = val;
1429
1430	#ifdef HWACCM_VMX_EMULATE_REALMODE
1431	/* Real mode emulation using v86 mode with CR4.VME (interrupt redirection using the int bitmap in the TSS) */
1432	if (CPUMIsGuestInRealModeEx(pCtx))
1433	{
1434	/* Hide our emulation flags */
1435	pCtx->eflags.Bits.u1VM = 0;
1436	pCtx->eflags.Bits.u2IOPL = 0;
1437
1438	/* Force a TR resync every time in case we switch modes. */
1439	pVM->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_GUEST_TR;
1440	}
1441	else
1442	#endif /* HWACCM_VMX_EMULATE_REALMODE */
1443	{
1444	/* In real mode we have a fake TSS, so only sync it back when it's supposed to be valid. */
1445	VMX_READ_SELREG(TR, tr);
1446	}
1447	return VINF_SUCCESS;
1448	}
1449
1450	/**
1451	* Runs guest code in a VT-x VM.
1452	*
1453	* @returns VBox status code.
1454	* @param pVM The VM to operate on.
1455	* @param pCtx Guest context
1456	*/
1457	VMMR0DECL(int) VMXR0RunGuestCode(PVM pVM, CPUMCTX *pCtx)
1458	{
1459	int rc = VINF_SUCCESS;
1460	RTCCUINTREG val;
1461	RTCCUINTREG exitReason, instrError, cbInstr;
1462	RTGCUINTPTR exitQualification;
1463	RTGCUINTPTR intInfo = 0; /* shut up buggy gcc 4 */
1464	RTGCUINTPTR errCode, instrInfo;
1465	bool fSyncTPR = false;
1466	PHWACCM_CPUINFO pCpu = 0;
1467	unsigned cResume = 0;
1468	#ifdef VBOX_STRICT
1469	RTCPUID idCpuCheck;
1470	#endif
1471
1472	Log2(("\nE"));
1473
1474	STAM_PROFILE_ADV_START(&pVM->hwaccm.s.StatEntry, x);
1475
1476	#ifdef VBOX_STRICT
1477	rc = VMXReadVMCS(VMX_VMCS_CTRL_PIN_EXEC_CONTROLS, &val);
1478	AssertRC(rc);
1479	Log2(("VMX_VMCS_CTRL_PIN_EXEC_CONTROLS = %08x\n", val));
1480
1481	/* allowed zero */
1482	if ((val & pVM->hwaccm.s.vmx.msr.vmx_pin_ctls.n.disallowed0) != pVM->hwaccm.s.vmx.msr.vmx_pin_ctls.n.disallowed0)
1483	Log(("Invalid VMX_VMCS_CTRL_PIN_EXEC_CONTROLS: zero\n"));
1484
1485	/* allowed one */
1486	if ((val & ~pVM->hwaccm.s.vmx.msr.vmx_pin_ctls.n.allowed1) != 0)
1487	Log(("Invalid VMX_VMCS_CTRL_PIN_EXEC_CONTROLS: one\n"));
1488
1489	rc = VMXReadVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, &val);
1490	AssertRC(rc);
1491	Log2(("VMX_VMCS_CTRL_PROC_EXEC_CONTROLS = %08x\n", val));
1492
1493	/* allowed zero */
1494	if ((val & pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.disallowed0) != pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.disallowed0)
1495	Log(("Invalid VMX_VMCS_CTRL_PROC_EXEC_CONTROLS: zero\n"));
1496
1497	/* allowed one */
1498	if ((val & ~pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1) != 0)
1499	Log(("Invalid VMX_VMCS_CTRL_PROC_EXEC_CONTROLS: one\n"));
1500
1501	rc = VMXReadVMCS(VMX_VMCS_CTRL_ENTRY_CONTROLS, &val);
1502	AssertRC(rc);
1503	Log2(("VMX_VMCS_CTRL_ENTRY_CONTROLS = %08x\n", val));
1504
1505	/* allowed zero */
1506	if ((val & pVM->hwaccm.s.vmx.msr.vmx_entry.n.disallowed0) != pVM->hwaccm.s.vmx.msr.vmx_entry.n.disallowed0)
1507	Log(("Invalid VMX_VMCS_CTRL_ENTRY_CONTROLS: zero\n"));
1508
1509	/* allowed one */
1510	if ((val & ~pVM->hwaccm.s.vmx.msr.vmx_entry.n.allowed1) != 0)
1511	Log(("Invalid VMX_VMCS_CTRL_ENTRY_CONTROLS: one\n"));
1512
1513	rc = VMXReadVMCS(VMX_VMCS_CTRL_EXIT_CONTROLS, &val);
1514	AssertRC(rc);
1515	Log2(("VMX_VMCS_CTRL_EXIT_CONTROLS = %08x\n", val));
1516
1517	/* allowed zero */
1518	if ((val & pVM->hwaccm.s.vmx.msr.vmx_exit.n.disallowed0) != pVM->hwaccm.s.vmx.msr.vmx_exit.n.disallowed0)
1519	Log(("Invalid VMX_VMCS_CTRL_EXIT_CONTROLS: zero\n"));
1520
1521	/* allowed one */
1522	if ((val & ~pVM->hwaccm.s.vmx.msr.vmx_exit.n.allowed1) != 0)
1523	Log(("Invalid VMX_VMCS_CTRL_EXIT_CONTROLS: one\n"));
1524	#endif
1525
1526	/* We can jump to this point to resume execution after determining that a VM-exit is innocent.
1527	*/
1528	ResumeExecution:
1529	AssertMsg(pVM->hwaccm.s.idEnteredCpu == RTMpCpuId(),
1530	("Expected %d, I'm %d; cResume=%d exitReason=%RTreg exitQualification=%RTreg\n",
1531	(int)pVM->hwaccm.s.idEnteredCpu, (int)RTMpCpuId(), cResume, exitReason, exitQualification));
1532
1533	/* Safety precaution; looping for too long here can have a very bad effect on the host */
1534	if (++cResume > HWACCM_MAX_RESUME_LOOPS)
1535	{
1536	STAM_COUNTER_INC(&pVM->hwaccm.s.StatExitMaxResume);
1537	rc = VINF_EM_RAW_INTERRUPT;
1538	goto end;
1539	}
1540
1541	/* Check for irq inhibition due to instruction fusing (sti, mov ss). */
1542	if (VM_FF_ISSET(pVM, VM_FF_INHIBIT_INTERRUPTS))
1543	{
1544	Log(("VM_FF_INHIBIT_INTERRUPTS at %VGv successor %VGv\n", pCtx->rip, EMGetInhibitInterruptsPC(pVM)));
1545	if (pCtx->rip != EMGetInhibitInterruptsPC(pVM))
1546	{
1547	/* Note: we intentionally don't clear VM_FF_INHIBIT_INTERRUPTS here.
1548	* Before we are able to execute this instruction in raw mode (iret to guest code) an external interrupt might
1549	* force a world switch again. Possibly allowing a guest interrupt to be dispatched in the process. This could
1550	* break the guest. Sounds very unlikely, but such timing sensitive problem are not as rare as you might think.
1551	*/
1552	VM_FF_CLEAR(pVM, VM_FF_INHIBIT_INTERRUPTS);
1553	/* Irq inhibition is no longer active; clear the corresponding VMX state. */
1554	rc = VMXWriteVMCS(VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE, 0);
1555	AssertRC(rc);
1556	}
1557	}
1558	else
1559	{
1560	/* Irq inhibition is no longer active; clear the corresponding VMX state. */
1561	rc = VMXWriteVMCS(VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE, 0);
1562	AssertRC(rc);
1563	}
1564
1565	/* Check for pending actions that force us to go back to ring 3. */
1566	if (VM_FF_ISPENDING(pVM, VM_FF_TO_R3 \| VM_FF_TIMER))
1567	{
1568	VM_FF_CLEAR(pVM, VM_FF_TO_R3);
1569	STAM_COUNTER_INC(&pVM->hwaccm.s.StatSwitchToR3);
1570	STAM_PROFILE_ADV_STOP(&pVM->hwaccm.s.StatEntry, x);
1571	rc = VINF_EM_RAW_TO_R3;
1572	goto end;
1573	}
1574	/* Pending request packets might contain actions that need immediate attention, such as pending hardware interrupts. */
1575	if (VM_FF_ISPENDING(pVM, VM_FF_REQUEST))
1576	{
1577	STAM_PROFILE_ADV_STOP(&pVM->hwaccm.s.StatEntry, x);
1578	rc = VINF_EM_PENDING_REQUEST;
1579	goto end;
1580	}
1581
1582	/* When external interrupts are pending, we should exit the VM when IF is set. */
1583	/* Note! After VM_FF_INHIBIT_INTERRUPTS check!!! */
1584	rc = VMXR0CheckPendingInterrupt(pVM, pCtx);
1585	if (VBOX_FAILURE(rc))
1586	{
1587	STAM_PROFILE_ADV_STOP(&pVM->hwaccm.s.StatEntry, x);
1588	goto end;
1589	}
1590
1591	/** @todo check timers?? */
1592
1593	/* TPR caching using CR8 is only available in 64 bits mode */
1594	/* Note the 32 bits exception for AMD (X86_CPUID_AMD_FEATURE_ECX_CR8L), but that appears missing in Intel CPUs */
1595	/* Note: we can't do this in LoadGuestState as PDMApicGetTPR can jump back to ring 3 (lock)!!!!! */
1596	/**
1597	* @todo reduce overhead
1598	*/
1599	if ( (pCtx->msrEFER & MSR_K6_EFER_LMA)
1600	&& pVM->hwaccm.s.vmx.pAPIC)
1601	{
1602	/* TPR caching in CR8 */
1603	uint8_t u8TPR;
1604	bool fPending;
1605
1606	int rc = PDMApicGetTPR(pVM, &u8TPR, &fPending);
1607	AssertRC(rc);
1608	/* The TPR can be found at offset 0x80 in the APIC mmio page. */
1609	pVM->hwaccm.s.vmx.pAPIC[0x80] = u8TPR << 4; /* bits 7-4 contain the task priority */
1610
1611	/* Two options here:
1612	* - external interrupt pending, but masked by the TPR value.
1613	* -> a CR8 update that lower the current TPR value should cause an exit
1614	* - no pending interrupts
1615	* -> We don't need to be explicitely notified. There are enough world switches for detecting pending interrupts.
1616	*/
1617	rc = VMXWriteVMCS(VMX_VMCS_CTRL_TPR_THRESHOLD, (fPending) ? u8TPR : 0);
1618	AssertRC(rc);
1619
1620	/* Always sync back the TPR; we should optimize this though / /* @todo optimize TPR sync. */
1621	fSyncTPR = true;
1622	}
1623
1624	#if defined(HWACCM_VTX_WITH_EPT) && defined(LOG_ENABLED)
1625	pCpu = HWACCMR0GetCurrentCpu();
1626	if ( pVM->hwaccm.s.idLastCpu != pCpu->idCpu
1627	\|\| pVM->hwaccm.s.cTLBFlushes != pCpu->cTLBFlushes)
1628	{
1629	if (pVM->hwaccm.s.idLastCpu != pCpu->idCpu)
1630	Log(("Force TLB flush due to rescheduling to a different cpu (%d vs %d)\n", pVM->hwaccm.s.idLastCpu, pCpu->idCpu));
1631	else
1632	Log(("Force TLB flush due to changed TLB flush count (%x vs %x)\n", pVM->hwaccm.s.cTLBFlushes, pCpu->cTLBFlushes));
1633	}
1634	if (pCpu->fFlushTLB)
1635	Log(("Force TLB flush: first time cpu %d is used -> flush\n", pCpu->idCpu));
1636	#endif
1637
1638	/*
1639	* NOTE: DO NOT DO ANYTHING AFTER THIS POINT THAT MIGHT JUMP BACK TO RING 3!
1640	* (until the actual world switch)
1641	*/
1642	#ifdef VBOX_STRICT
1643	idCpuCheck = RTMpCpuId();
1644	#endif
1645	/* Save the host state first. */
1646	rc = VMXR0SaveHostState(pVM);
1647	if (rc != VINF_SUCCESS)
1648	{
1649	STAM_PROFILE_ADV_STOP(&pVM->hwaccm.s.StatEntry, x);
1650	goto end;
1651	}
1652	/* Load the guest state */
1653	rc = VMXR0LoadGuestState(pVM, pCtx);
1654	if (rc != VINF_SUCCESS)
1655	{
1656	STAM_PROFILE_ADV_STOP(&pVM->hwaccm.s.StatEntry, x);
1657	goto end;
1658	}
1659
1660	#ifdef HWACCM_VTX_WITH_EPT
1661	/* Deal with tagged TLBs if VPID or EPT is supported. */
1662	if ( pVM->hwaccm.s.fNestedPaging
1663	\|\| pVM->hwaccm.s.vmx.fVPID)
1664	{
1665	pCpu = HWACCMR0GetCurrentCpu();
1666	/* Force a TLB flush for the first world switch if the current cpu differs from the one we ran on last. */
1667	/* Note that this can happen both for start and resume due to long jumps back to ring 3. */
1668	if ( pVM->hwaccm.s.idLastCpu != pCpu->idCpu
1669	/* if the tlb flush count has changed, another VM has flushed the TLB of this cpu, so we can't use our current ASID anymore. */
1670	\|\| pVM->hwaccm.s.cTLBFlushes != pCpu->cTLBFlushes)
1671	{
1672	/* Force a TLB flush on VM entry. */
1673	pVM->hwaccm.s.fForceTLBFlush = true;
1674	}
1675	else
1676	Assert(!pCpu->fFlushTLB);
1677
1678	pVM->hwaccm.s.idLastCpu = pCpu->idCpu;
1679
1680	/* Make sure we flush the TLB when required. Switch ASID to achieve the same thing, but without actually flushing the whole TLB (which is expensive). */
1681	if (pVM->hwaccm.s.fForceTLBFlush)
1682	{
1683	if ( ++pCpu->uCurrentASID >= pVM->hwaccm.s.uMaxASID
1684	\|\| pCpu->fFlushTLB)
1685	{
1686	pCpu->fFlushTLB = false;
1687	pCpu->uCurrentASID = 1; /* start at 1; host uses 0 */
1688	pCpu->cTLBFlushes++;
1689	}
1690	else
1691	{
1692	STAM_COUNTER_INC(&pVM->hwaccm.s.StatFlushASID);
1693	pVM->hwaccm.s.fForceTLBFlush = false;
1694	}
1695
1696	pVM->hwaccm.s.cTLBFlushes = pCpu->cTLBFlushes;
1697	pVM->hwaccm.s.uCurrentASID = pCpu->uCurrentASID;
1698	}
1699	else
1700	{
1701	Assert(!pCpu->fFlushTLB);
1702
1703	if (!pCpu->uCurrentASID \|\| !pVM->hwaccm.s.uCurrentASID)
1704	pVM->hwaccm.s.uCurrentASID = pCpu->uCurrentASID = 1;
1705	}
1706	AssertMsg(pVM->hwaccm.s.cTLBFlushes == pCpu->cTLBFlushes, ("Flush count mismatch for cpu %d (%x vs %x)\n", pCpu->idCpu, pVM->hwaccm.s.cTLBFlushes, pCpu->cTLBFlushes));
1707	AssertMsg(pCpu->uCurrentASID >= 1 && pCpu->uCurrentASID < pVM->hwaccm.s.uMaxASID, ("cpu%d uCurrentASID = %x\n", pCpu->idCpu, pCpu->uCurrentASID));
1708	AssertMsg(pVM->hwaccm.s.uCurrentASID >= 1 && pVM->hwaccm.s.uCurrentASID < pVM->hwaccm.s.uMaxASID, ("cpu%d VM uCurrentASID = %x\n", pCpu->idCpu, pVM->hwaccm.s.uCurrentASID));
1709
1710	if (pVM->hwaccm.s.fVPID)
1711	{
1712	rc = VMXWriteVMCS(VMX_VMCS_GUEST_FIELD_VPID, pVM->hwaccm.s.uCurrentASID);
1713	AssertRC(rc);
1714	}
1715
1716	if (pVM->hwaccm.s.fForceTLBFlush)
1717	{
1718
1719	}
1720
1721	#ifdef VBOX_WITH_STATISTICS
1722	if (pVM->hwaccm.s.fForceTLBFlush)
1723	STAM_COUNTER_INC(&pVM->hwaccm.s.StatFlushTLBWorldSwitch);
1724	else
1725	STAM_COUNTER_INC(&pVM->hwaccm.s.StatNoFlushTLBWorldSwitch);
1726	#endif
1727	}
1728	#endif /* HWACCM_VTX_WITH_EPT */
1729
1730	/* Non-register state Guest Context */
1731	/** @todo change me according to cpu state */
1732	rc = VMXWriteVMCS(VMX_VMCS_GUEST_ACTIVITY_STATE, VMX_CMS_GUEST_ACTIVITY_ACTIVE);
1733	AssertRC(rc);
1734
1735	STAM_PROFILE_ADV_STOP(&pVM->hwaccm.s.StatEntry, x);
1736
1737	/* Manual save and restore:
1738	* - General purpose registers except RIP, RSP
1739	*
1740	* Trashed:
1741	* - CR2 (we don't care)
1742	* - LDTR (reset to 0)
1743	* - DRx (presumably not changed at all)
1744	* - DR7 (reset to 0x400)
1745	* - EFLAGS (reset to RT_BIT(1); not relevant)
1746	*
1747	*/
1748
1749	/* All done! Let's start VM execution. */
1750	STAM_PROFILE_ADV_START(&pVM->hwaccm.s.StatInGC, x);
1751	#ifdef VBOX_STRICT
1752	Assert(idCpuCheck == RTMpCpuId());
1753	#endif
1754	TMNotifyStartOfExecution(pVM);
1755	rc = pVM->hwaccm.s.vmx.pfnStartVM(pVM->hwaccm.s.vmx.fResumeVM, pCtx);
1756	TMNotifyEndOfExecution(pVM);
1757
1758	/* In case we execute a goto ResumeExecution later on. */
1759	pVM->hwaccm.s.vmx.fResumeVM = true;
1760	pVM->hwaccm.s.fForceTLBFlush = false;
1761
1762	/*
1763	* !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
1764	* IMPORTANT: WE CAN'T DO ANY LOGGING OR OPERATIONS THAT CAN DO A LONGJMP BACK TO RING 3 BEFORE WE'VE SYNCED BACK (MOST OF) THE GUEST STATE
1765	* !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
1766	*/
1767
1768	STAM_PROFILE_ADV_STOP(&pVM->hwaccm.s.StatInGC, x);
1769	STAM_PROFILE_ADV_START(&pVM->hwaccm.s.StatExit, x);
1770
1771	if (rc != VINF_SUCCESS)
1772	{
1773	VMXR0ReportWorldSwitchError(pVM, rc, pCtx);
1774	goto end;
1775	}
1776	/* Success. Query the guest state and figure out what has happened. */
1777
1778	/* Investigate why there was a VM-exit. */
1779	rc = VMXReadVMCS(VMX_VMCS_RO_EXIT_REASON, &exitReason);
1780	STAM_COUNTER_INC(&pVM->hwaccm.s.paStatExitReasonR0[exitReason & MASK_EXITREASON_STAT]);
1781
1782	exitReason &= 0xffff; /* bit 0-15 contain the exit code. */
1783	rc \|= VMXReadVMCS(VMX_VMCS_RO_VM_INSTR_ERROR, &instrError);
1784	rc \|= VMXReadVMCS(VMX_VMCS_RO_EXIT_INSTR_LENGTH, &cbInstr);
1785	rc \|= VMXReadVMCS(VMX_VMCS_RO_EXIT_INTERRUPTION_INFO, &val);
1786	intInfo = val;
1787	rc \|= VMXReadVMCS(VMX_VMCS_RO_EXIT_INTERRUPTION_ERRCODE, &val);
1788	errCode = val; /* might not be valid; depends on VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID. */
1789	rc \|= VMXReadVMCS(VMX_VMCS_RO_EXIT_INSTR_INFO, &val);
1790	instrInfo = val;
1791	rc \|= VMXReadVMCS(VMX_VMCS_RO_EXIT_QUALIFICATION, &val);
1792	exitQualification = val;
1793	AssertRC(rc);
1794
1795	/* Sync back the guest state */
1796	rc = VMXR0SaveGuestState(pVM, pCtx);
1797	AssertRC(rc);
1798
1799	/* Note! NOW IT'S SAFE FOR LOGGING! */
1800	Log2(("Raw exit reason %08x\n", exitReason));
1801
1802	/* Check if an injected event was interrupted prematurely. */
1803	rc = VMXReadVMCS(VMX_VMCS_RO_IDT_INFO, &val);
1804	AssertRC(rc);
1805	#ifdef HWACCM_VMX_EMULATE_REALMODE
1806	/* For some reason injected software interrupts are ignored (not signalled as pending) when e.g. a shadow page fault occurs. */
1807	if ( CPUMIsGuestInRealModeEx(pCtx)
1808	&& pVM->hwaccm.s.vmx.RealMode.eip == pCtx->eip
1809	&& pVM->hwaccm.s.vmx.RealMode.Event.fPending)
1810	{
1811	Assert(!pVM->hwaccm.s.Event.fPending);
1812
1813	Log(("Pending real-mode inject %VX64 at %VGv\n", pVM->hwaccm.s.vmx.RealMode.Event.intInfo, pCtx->rip));
1814
1815	/* We faked an 'int x' instruction and messed with IP, so correct it here. */
1816	pCtx->rip++;
1817	pVM->hwaccm.s.Event.intInfo = pVM->hwaccm.s.vmx.RealMode.Event.intInfo;
1818	pVM->hwaccm.s.Event.fPending = true;
1819	}
1820	else
1821	#endif /* HWACCM_VMX_EMULATE_REALMODE */
1822	{
1823	pVM->hwaccm.s.Event.intInfo = VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(val);
1824	if ( VMX_EXIT_INTERRUPTION_INFO_VALID(pVM->hwaccm.s.Event.intInfo)
1825	&& VMX_EXIT_INTERRUPTION_INFO_TYPE(pVM->hwaccm.s.Event.intInfo) != VMX_EXIT_INTERRUPTION_INFO_TYPE_SW)
1826	{
1827	pVM->hwaccm.s.Event.fPending = true;
1828	/* Error code present? */
1829	if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(pVM->hwaccm.s.Event.intInfo))
1830	{
1831	rc = VMXReadVMCS(VMX_VMCS_RO_IDT_ERRCODE, &val);
1832	AssertRC(rc);
1833	pVM->hwaccm.s.Event.errCode = val;
1834	Log(("Pending inject %VX64 at %VGv exit=%08x intInfo=%08x exitQualification=%08x pending error=%RX64\n", pVM->hwaccm.s.Event.intInfo, pCtx->rip, exitReason, intInfo, exitQualification, val));
1835	}
1836	else
1837	{
1838	Log(("Pending inject %VX64 at %VGv exit=%08x intInfo=%08x exitQualification=%08x\n", pVM->hwaccm.s.Event.intInfo, pCtx->rip, exitReason, intInfo, exitQualification));
1839	pVM->hwaccm.s.Event.errCode = 0;
1840	}
1841	}
1842	}
1843	pVM->hwaccm.s.vmx.RealMode.Event.fPending = false;
1844
1845	#ifdef VBOX_STRICT
1846	if (exitReason == VMX_EXIT_ERR_INVALID_GUEST_STATE)
1847	HWACCMDumpRegs(pVM, pCtx);
1848	#endif
1849
1850	Log2(("E%d", exitReason));
1851	Log2(("Exit reason %d, exitQualification %08x\n", exitReason, exitQualification));
1852	Log2(("instrInfo=%d instrError=%d instr length=%d\n", instrInfo, instrError, cbInstr));
1853	Log2(("Interruption error code %d\n", errCode));
1854	Log2(("IntInfo = %08x\n", intInfo));
1855	Log2(("New EIP=%VGv\n", pCtx->rip));
1856
1857	if (fSyncTPR)
1858	{
1859	rc = PDMApicSetTPR(pVM, pVM->hwaccm.s.vmx.pAPIC[0x80] >> 4);
1860	AssertRC(rc);
1861	}
1862
1863	/* Some cases don't need a complete resync of the guest CPU state; handle them here. */
1864	switch (exitReason)
1865	{
1866	case VMX_EXIT_EXCEPTION: /* 0 Exception or non-maskable interrupt (NMI). */
1867	case VMX_EXIT_EXTERNAL_IRQ: /* 1 External interrupt. */
1868	{
1869	uint32_t vector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(intInfo);
1870
1871	if (!VMX_EXIT_INTERRUPTION_INFO_VALID(intInfo))
1872	{
1873	Assert(exitReason == VMX_EXIT_EXTERNAL_IRQ);
1874	/* External interrupt; leave to allow it to be dispatched again. */
1875	rc = VINF_EM_RAW_INTERRUPT;
1876	break;
1877	}
1878	switch (VMX_EXIT_INTERRUPTION_INFO_TYPE(intInfo))
1879	{
1880	case VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI: /* Non-maskable interrupt. */
1881	/* External interrupt; leave to allow it to be dispatched again. */
1882	rc = VINF_EM_RAW_INTERRUPT;
1883	break;
1884
1885	case VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT: /* External hardware interrupt. */
1886	AssertFailed(); /* can't come here; fails the first check. */
1887	break;
1888
1889	case VMX_EXIT_INTERRUPTION_INFO_TYPE_SWEXCPT: /* Software exception. (#BP or #OF) */
1890	Assert(vector == 3 \|\| vector == 4);
1891	/* no break */
1892	case VMX_EXIT_INTERRUPTION_INFO_TYPE_HWEXCPT: /* Hardware exception. */
1893	Log2(("Hardware/software interrupt %d\n", vector));
1894	switch (vector)
1895	{
1896	case X86_XCPT_NM:
1897	{
1898	Log(("#NM fault at %VGv error code %x\n", pCtx->rip, errCode));
1899
1900	/** @todo don't intercept #NM exceptions anymore when we've activated the guest FPU state. */
1901	/* If we sync the FPU/XMM state on-demand, then we can continue execution as if nothing has happened. */
1902	rc = CPUMR0LoadGuestFPU(pVM, pCtx);
1903	if (rc == VINF_SUCCESS)
1904	{
1905	Assert(CPUMIsGuestFPUStateActive(pVM));
1906
1907	STAM_COUNTER_INC(&pVM->hwaccm.s.StatExitShadowNM);
1908
1909	/* Continue execution. */
1910	STAM_PROFILE_ADV_STOP(&pVM->hwaccm.s.StatExit, x);
1911	pVM->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_GUEST_CR0;
1912
1913	goto ResumeExecution;
1914	}
1915
1916	Log(("Forward #NM fault to the guest\n"));
1917	STAM_COUNTER_INC(&pVM->hwaccm.s.StatExitGuestNM);
1918	rc = VMXR0InjectEvent(pVM, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), cbInstr, 0);
1919	AssertRC(rc);
1920	STAM_PROFILE_ADV_STOP(&pVM->hwaccm.s.StatExit, x);
1921	goto ResumeExecution;
1922	}
1923
1924	case X86_XCPT_PF: /* Page fault */
1925	{
1926	#ifdef DEBUG
1927	if (pVM->hwaccm.s.fNestedPaging)
1928	{ /* A genuine pagefault.
1929	* Forward the trap to the guest by injecting the exception and resuming execution.
1930	*/
1931	Log(("Guest page fault at %VGv cr2=%VGv error code %x rsp=%VGv\n", (RTGCPTR)pCtx->rip, exitQualification, errCode, (RTGCPTR)pCtx->rsp));
1932
1933	STAM_COUNTER_INC(&pVM->hwaccm.s.StatExitGuestPF);
1934	/* The error code might have been changed. */
1935	errCode = TRPMGetErrorCode(pVM);
1936
1937	TRPMResetTrap(pVM);
1938
1939	/* Now we must update CR2. */
1940	pCtx->cr2 = exitQualification;
1941	rc = VMXR0InjectEvent(pVM, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), cbInstr, errCode);
1942	AssertRC(rc);
1943
1944	STAM_PROFILE_ADV_STOP(&pVM->hwaccm.s.StatExit, x);
1945	goto ResumeExecution;
1946	}
1947	#endif
1948	Assert(!pVM->hwaccm.s.fNestedPaging);
1949
1950	Log2(("Page fault at %VGv error code %x\n", exitQualification, errCode));
1951	/* Exit qualification contains the linear address of the page fault. */
1952	TRPMAssertTrap(pVM, X86_XCPT_PF, TRPM_TRAP);
1953	TRPMSetErrorCode(pVM, errCode);
1954	TRPMSetFaultAddress(pVM, exitQualification);
1955
1956	/* Forward it to our trap handler first, in case our shadow pages are out of sync. */
1957	rc = PGMTrap0eHandler(pVM, errCode, CPUMCTX2CORE(pCtx), (RTGCPTR)exitQualification);
1958	Log2(("PGMTrap0eHandler %VGv returned %Vrc\n", pCtx->rip, rc));
1959	if (rc == VINF_SUCCESS)
1960	{ /* We've successfully synced our shadow pages, so let's just continue execution. */
1961	Log2(("Shadow page fault at %VGv cr2=%VGv error code %x\n", pCtx->rip, exitQualification ,errCode));
1962	STAM_COUNTER_INC(&pVM->hwaccm.s.StatExitShadowPF);
1963
1964	TRPMResetTrap(pVM);
1965
1966	STAM_PROFILE_ADV_STOP(&pVM->hwaccm.s.StatExit, x);
1967	goto ResumeExecution;
1968	}
1969	else
1970	if (rc == VINF_EM_RAW_GUEST_TRAP)
1971	{ /* A genuine pagefault.
1972	* Forward the trap to the guest by injecting the exception and resuming execution.
1973	*/
1974	Log2(("Forward page fault to the guest\n"));
1975
1976	STAM_COUNTER_INC(&pVM->hwaccm.s.StatExitGuestPF);
1977	/* The error code might have been changed. */
1978	errCode = TRPMGetErrorCode(pVM);
1979
1980	TRPMResetTrap(pVM);
1981
1982	/* Now we must update CR2. */
1983	pCtx->cr2 = exitQualification;
1984	rc = VMXR0InjectEvent(pVM, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), cbInstr, errCode);
1985	AssertRC(rc);
1986
1987	STAM_PROFILE_ADV_STOP(&pVM->hwaccm.s.StatExit, x);
1988	goto ResumeExecution;
1989	}
1990	#ifdef VBOX_STRICT
1991	if (rc != VINF_EM_RAW_EMULATE_INSTR)
1992	Log2(("PGMTrap0eHandler failed with %d\n", rc));
1993	#endif
1994	/* Need to go back to the recompiler to emulate the instruction. */
1995	TRPMResetTrap(pVM);
1996	break;
1997	}
1998
1999	case X86_XCPT_MF: /* Floating point exception. */
2000	{
2001	STAM_COUNTER_INC(&pVM->hwaccm.s.StatExitGuestMF);
2002	if (!(pCtx->cr0 & X86_CR0_NE))
2003	{
2004	/* old style FPU error reporting needs some extra work. */
2005	/** @todo don't fall back to the recompiler, but do it manually. */
2006	rc = VINF_EM_RAW_EMULATE_INSTR;
2007	break;
2008	}
2009	Log(("Trap %x at %VGv\n", vector, pCtx->rip));
2010	rc = VMXR0InjectEvent(pVM, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), cbInstr, errCode);
2011	AssertRC(rc);
2012
2013	STAM_PROFILE_ADV_STOP(&pVM->hwaccm.s.StatExit, x);
2014	goto ResumeExecution;
2015	}
2016
2017	case X86_XCPT_DB: /* Debug exception. */
2018	{
2019	uint64_t uDR6;
2020
2021	/* DR6, DR7.GD and IA32_DEBUGCTL.LBR are not updated yet.
2022	*
2023	* Exit qualification bits:
2024	* 3:0 B0-B3 which breakpoint condition was met
2025	* 12:4 Reserved (0)
2026	* 13 BD - debug register access detected
2027	* 14 BS - single step execution or branch taken
2028	* 63:15 Reserved (0)
2029	*/
2030	STAM_COUNTER_INC(&pVM->hwaccm.s.StatExitGuestDB);
2031
2032	/* Note that we don't support guest and host-initiated debugging at the same time. */
2033	Assert(DBGFIsStepping(pVM));
2034
2035	uDR6 = X86_DR6_INIT_VAL;
2036	uDR6 \|= (exitQualification & (X86_DR6_B0\|X86_DR6_B1\|X86_DR6_B2\|X86_DR6_B3\|X86_DR6_BD\|X86_DR6_BS));
2037	rc = DBGFR0Trap01Handler(pVM, CPUMCTX2CORE(pCtx), uDR6);
2038	if (rc == VINF_EM_RAW_GUEST_TRAP)
2039	{
2040	/** @todo this isn't working, but we'll never get here normally. */
2041
2042	/* Update DR6 here. */
2043	pCtx->dr[6] = uDR6;
2044
2045	/* X86_DR7_GD will be cleared if drx accesses should be trapped inside the guest. */
2046	pCtx->dr[7] &= ~X86_DR7_GD;
2047
2048	/* Paranoia. */
2049	pCtx->dr[7] &= 0xffffffff; /* upper 32 bits reserved */
2050	pCtx->dr[7] &= ~(RT_BIT(11) \| RT_BIT(12) \| RT_BIT(14) \| RT_BIT(15)); /* must be zero */
2051	pCtx->dr[7] \|= 0x400; /* must be one */
2052
2053	/* Resync DR7 */
2054	rc = VMXWriteVMCS(VMX_VMCS_GUEST_DR7, pCtx->dr[7]);
2055	AssertRC(rc);
2056
2057	Log(("Trap %x (debug) at %VGv exit qualification %VX64\n", vector, pCtx->rip, exitQualification));
2058	rc = VMXR0InjectEvent(pVM, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), cbInstr, errCode);
2059	AssertRC(rc);
2060
2061	STAM_PROFILE_ADV_STOP(&pVM->hwaccm.s.StatExit, x);
2062	goto ResumeExecution;
2063	}
2064	/* Return to ring 3 to deal with the debug exit code. */
2065	break;
2066	}
2067
2068	case X86_XCPT_GP: /* General protection failure exception.*/
2069	{
2070	uint32_t cbSize;
2071
2072	STAM_COUNTER_INC(&pVM->hwaccm.s.StatExitGuestGP);
2073	#ifdef VBOX_STRICT
2074	if (!CPUMIsGuestInRealModeEx(pCtx))
2075	{
2076	Log(("Trap %x at %VGv error code %x\n", vector, pCtx->rip, errCode));
2077	rc = VMXR0InjectEvent(pVM, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), cbInstr, errCode);
2078	AssertRC(rc);
2079	STAM_PROFILE_ADV_STOP(&pVM->hwaccm.s.StatExit, x);
2080	goto ResumeExecution;
2081	}
2082	#endif
2083	Assert(CPUMIsGuestInRealModeEx(pCtx));
2084
2085	LogFlow(("Real mode X86_XCPT_GP instruction emulation at %VGv\n", pCtx->rip));
2086	rc = EMInterpretInstruction(pVM, CPUMCTX2CORE(pCtx), 0, &cbSize);
2087	if (rc == VINF_SUCCESS)
2088	{
2089	/* EIP has been updated already. */
2090
2091	/* lidt, lgdt can end up here. In the future crx changes as well. Just reload the whole context to be done with it. */
2092	pVM->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_ALL;
2093
2094	/* Only resume if successful. */
2095	STAM_PROFILE_ADV_STOP(&pVM->hwaccm.s.StatExit, x);
2096	goto ResumeExecution;
2097	}
2098	AssertMsg(rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_EM_HALT, ("Unexpected rc=%Vrc\n", rc));
2099	break;
2100	}
2101
2102	#ifdef VBOX_STRICT
2103	case X86_XCPT_DE: /* Divide error. */
2104	case X86_XCPT_UD: /* Unknown opcode exception. */
2105	case X86_XCPT_SS: /* Stack segment exception. */
2106	case X86_XCPT_NP: /* Segment not present exception. */
2107	{
2108	switch(vector)
2109	{
2110	case X86_XCPT_DE:
2111	STAM_COUNTER_INC(&pVM->hwaccm.s.StatExitGuestDE);
2112	break;
2113	case X86_XCPT_UD:
2114	STAM_COUNTER_INC(&pVM->hwaccm.s.StatExitGuestUD);
2115	break;
2116	case X86_XCPT_SS:
2117	STAM_COUNTER_INC(&pVM->hwaccm.s.StatExitGuestSS);
2118	break;
2119	case X86_XCPT_NP:
2120	STAM_COUNTER_INC(&pVM->hwaccm.s.StatExitGuestNP);
2121	break;
2122	}
2123
2124	Log(("Trap %x at %VGv error code %x\n", vector, pCtx->rip, errCode));
2125	rc = VMXR0InjectEvent(pVM, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), cbInstr, errCode);
2126	AssertRC(rc);
2127
2128	STAM_PROFILE_ADV_STOP(&pVM->hwaccm.s.StatExit, x);
2129	goto ResumeExecution;
2130	}
2131	#endif
2132	default:
2133	AssertMsgFailed(("Unexpected vm-exit caused by exception %x\n", vector));
2134	rc = VERR_VMX_UNEXPECTED_EXCEPTION;
2135	break;
2136	} /* switch (vector) */
2137
2138	break;
2139
2140	default:
2141	rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_CODE;
2142	AssertFailed();
2143	break;
2144	}
2145
2146	break;
2147	}
2148
2149	case VMX_EXIT_EPT_VIOLATION: /* 48 EPT violation. An attempt to access memory with a guest-physical address was disallowed by the configuration of the EPT paging structures. */
2150	{
2151	Log2(("EPT Page fault at %VGv error code %x\n", exitQualification ,errCode));
2152	Assert(pVM->hwaccm.s.fNestedPaging);
2153
2154	/* Exit qualification contains the linear address of the page fault. */
2155	TRPMAssertTrap(pVM, X86_XCPT_PF, TRPM_TRAP);
2156	TRPMSetErrorCode(pVM, errCode);
2157	TRPMSetFaultAddress(pVM, exitQualification );
2158
2159	/* Handle the pagefault trap for the nested shadow table. */
2160	rc = PGMR0Trap0eHandlerNestedPaging(pVM, PGMMODE_EPT, errCode, CPUMCTX2CORE(pCtx), exitQualification );
2161	Log2(("PGMR0Trap0eHandlerNestedPaging %VGv returned %Vrc\n", pCtx->rip, rc));
2162	if (rc == VINF_SUCCESS)
2163	{ /* We've successfully synced our shadow pages, so let's just continue execution. */
2164	Log2(("Shadow page fault at %VGv cr2=%VGp error code %x\n", pCtx->rip, exitQualification , errCode));
2165	STAM_COUNTER_INC(&pVM->hwaccm.s.StatExitShadowPF);
2166
2167	TRPMResetTrap(pVM);
2168
2169	STAM_PROFILE_ADV_STOP(&pVM->hwaccm.s.StatExit, x);
2170	goto ResumeExecution;
2171	}
2172
2173	#ifdef VBOX_STRICT
2174	if (rc != VINF_EM_RAW_EMULATE_INSTR)
2175	LogFlow(("PGMTrap0eHandlerNestedPaging failed with %d\n", rc));
2176	#endif
2177	/* Need to go back to the recompiler to emulate the instruction. */
2178	TRPMResetTrap(pVM);
2179	break;
2180	}
2181
2182	case VMX_EXIT_IRQ_WINDOW: /* 7 Interrupt window. */
2183	/* Clear VM-exit on IF=1 change. */
2184	LogFlow(("VMX_EXIT_IRQ_WINDOW %VGv pending=%d IF=%d\n", pCtx->rip, VM_FF_ISPENDING(pVM, (VM_FF_INTERRUPT_APIC\|VM_FF_INTERRUPT_PIC)), pCtx->eflags.Bits.u1IF));
2185	pVM->hwaccm.s.vmx.proc_ctls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_IRQ_WINDOW_EXIT;
2186	rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVM->hwaccm.s.vmx.proc_ctls);
2187	AssertRC(rc);
2188	STAM_COUNTER_INC(&pVM->hwaccm.s.StatExitIrqWindow);
2189	goto ResumeExecution; /* we check for pending guest interrupts there */
2190
2191	case VMX_EXIT_WBINVD: /* 54 Guest software attempted to execute WBINVD. (conditional) */
2192	case VMX_EXIT_INVD: /* 13 Guest software attempted to execute INVD. (unconditional) */
2193	STAM_COUNTER_INC(&pVM->hwaccm.s.StatExitInvd);
2194	/* Skip instruction and continue directly. */
2195	pCtx->rip += cbInstr;
2196	/* Continue execution.*/
2197	STAM_PROFILE_ADV_STOP(&pVM->hwaccm.s.StatExit, x);
2198	goto ResumeExecution;
2199
2200	case VMX_EXIT_CPUID: /* 10 Guest software attempted to execute CPUID. */
2201	{
2202	Log2(("VMX: Cpuid %x\n", pCtx->eax));
2203	STAM_COUNTER_INC(&pVM->hwaccm.s.StatExitCpuid);
2204	rc = EMInterpretCpuId(pVM, CPUMCTX2CORE(pCtx));
2205	if (rc == VINF_SUCCESS)
2206	{
2207	/* Update EIP and continue execution. */
2208	Assert(cbInstr == 2);
2209	pCtx->rip += cbInstr;
2210	STAM_PROFILE_ADV_STOP(&pVM->hwaccm.s.StatExit, x);
2211	goto ResumeExecution;
2212	}
2213	AssertMsgFailed(("EMU: cpuid failed with %Vrc\n", rc));
2214	rc = VINF_EM_RAW_EMULATE_INSTR;
2215	break;
2216	}
2217
2218	case VMX_EXIT_RDTSC: /* 16 Guest software attempted to execute RDTSC. */
2219	{
2220	Log2(("VMX: Rdtsc\n"));
2221	STAM_COUNTER_INC(&pVM->hwaccm.s.StatExitRdtsc);
2222	rc = EMInterpretRdtsc(pVM, CPUMCTX2CORE(pCtx));
2223	if (rc == VINF_SUCCESS)
2224	{
2225	/* Update EIP and continue execution. */
2226	Assert(cbInstr == 2);
2227	pCtx->rip += cbInstr;
2228	STAM_PROFILE_ADV_STOP(&pVM->hwaccm.s.StatExit, x);
2229	goto ResumeExecution;
2230	}
2231	AssertMsgFailed(("EMU: rdtsc failed with %Vrc\n", rc));
2232	rc = VINF_EM_RAW_EMULATE_INSTR;
2233	break;
2234	}
2235
2236	case VMX_EXIT_INVPG: /* 14 Guest software attempted to execute INVPG. */
2237	{
2238	Log2(("VMX: invlpg\n"));
2239	Assert(!pVM->hwaccm.s.fNestedPaging);
2240
2241	STAM_COUNTER_INC(&pVM->hwaccm.s.StatExitInvpg);
2242	rc = EMInterpretInvlpg(pVM, CPUMCTX2CORE(pCtx), exitQualification);
2243	if (rc == VINF_SUCCESS)
2244	{
2245	/* Update EIP and continue execution. */
2246	pCtx->rip += cbInstr;
2247	STAM_PROFILE_ADV_STOP(&pVM->hwaccm.s.StatExit, x);
2248	goto ResumeExecution;
2249	}
2250	AssertMsg(rc == VERR_EM_INTERPRETER, ("EMU: invlpg %VGv failed with %Vrc\n", exitQualification, rc));
2251	break;
2252	}
2253
2254	case VMX_EXIT_RDMSR: /* 31 RDMSR. Guest software attempted to execute RDMSR. */
2255	case VMX_EXIT_WRMSR: /* 32 WRMSR. Guest software attempted to execute WRMSR. */
2256	{
2257	uint32_t cbSize;
2258
2259	/* Note: the intel manual claims there's a REX version of RDMSR that's slightly different, so we play safe by completely disassembling the instruction. */
2260	Log2(("VMX: %s\n", (exitReason == VMX_EXIT_RDMSR) ? "rdmsr" : "wrmsr"));
2261	rc = EMInterpretInstruction(pVM, CPUMCTX2CORE(pCtx), 0, &cbSize);
2262	if (rc == VINF_SUCCESS)
2263	{
2264	/* EIP has been updated already. */
2265
2266	/* Only resume if successful. */
2267	STAM_PROFILE_ADV_STOP(&pVM->hwaccm.s.StatExit, x);
2268	goto ResumeExecution;
2269	}
2270	AssertMsg(rc == VERR_EM_INTERPRETER, ("EMU: %s failed with %Vrc\n", (exitReason == VMX_EXIT_RDMSR) ? "rdmsr" : "wrmsr", rc));
2271	break;
2272	}
2273
2274	case VMX_EXIT_CRX_MOVE: /* 28 Control-register accesses. */
2275	{
2276	switch (VMX_EXIT_QUALIFICATION_CRX_ACCESS(exitQualification))
2277	{
2278	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_WRITE:
2279	Log2(("VMX: %VGv mov cr%d, x\n", pCtx->rip, VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification)));
2280	STAM_COUNTER_INC(&pVM->hwaccm.s.StatExitCRxWrite);
2281	rc = EMInterpretCRxWrite(pVM, CPUMCTX2CORE(pCtx),
2282	VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification),
2283	VMX_EXIT_QUALIFICATION_CRX_GENREG(exitQualification));
2284
2285	switch (VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification))
2286	{
2287	case 0:
2288	pVM->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_GUEST_CR0;
2289	break;
2290	case 2:
2291	break;
2292	case 3:
2293	Assert(!pVM->hwaccm.s.fNestedPaging \|\| !(pCtx->cr0 & X86_CR0_PG));
2294	pVM->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_GUEST_CR3;
2295	break;
2296	case 4:
2297	pVM->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_GUEST_CR4;
2298	break;
2299	case 8:
2300	/* CR8 contains the APIC TPR */
2301	Assert(!(pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW));
2302	break;
2303
2304	default:
2305	AssertFailed();
2306	break;
2307	}
2308	/* Check if a sync operation is pending. */
2309	if ( rc == VINF_SUCCESS /* don't bother if we are going to ring 3 anyway */
2310	&& VM_FF_ISPENDING(pVM, VM_FF_PGM_SYNC_CR3 \| VM_FF_PGM_SYNC_CR3_NON_GLOBAL))
2311	{
2312	rc = PGMSyncCR3(pVM, CPUMGetGuestCR0(pVM), CPUMGetGuestCR3(pVM), CPUMGetGuestCR4(pVM), VM_FF_ISSET(pVM, VM_FF_PGM_SYNC_CR3));
2313	AssertRC(rc);
2314	}
2315	break;
2316
2317	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ:
2318	Log2(("VMX: mov x, crx\n"));
2319	STAM_COUNTER_INC(&pVM->hwaccm.s.StatExitCRxRead);
2320
2321	Assert(!pVM->hwaccm.s.fNestedPaging \|\| !(pCtx->cr0 & X86_CR0_PG) \|\| VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification) != USE_REG_CR3);
2322
2323	/* CR8 reads only cause an exit when the TPR shadow feature isn't present. */
2324	Assert(VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification) != 8 \|\| !(pVM->hwaccm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW));
2325
2326	rc = EMInterpretCRxRead(pVM, CPUMCTX2CORE(pCtx),
2327	VMX_EXIT_QUALIFICATION_CRX_GENREG(exitQualification),
2328	VMX_EXIT_QUALIFICATION_CRX_REGISTER(exitQualification));
2329	break;
2330
2331	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_CLTS:
2332	Log2(("VMX: clts\n"));
2333	STAM_COUNTER_INC(&pVM->hwaccm.s.StatExitCLTS);
2334	rc = EMInterpretCLTS(pVM);
2335	pVM->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_GUEST_CR0;
2336	break;
2337
2338	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_LMSW:
2339	Log2(("VMX: lmsw %x\n", VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(exitQualification)));
2340	STAM_COUNTER_INC(&pVM->hwaccm.s.StatExitLMSW);
2341	rc = EMInterpretLMSW(pVM, VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(exitQualification));
2342	pVM->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_GUEST_CR0;
2343	break;
2344	}
2345
2346	/* Update EIP if no error occurred. */
2347	if (VBOX_SUCCESS(rc))
2348	pCtx->rip += cbInstr;
2349
2350	if (rc == VINF_SUCCESS)
2351	{
2352	/* Only resume if successful. */
2353	STAM_PROFILE_ADV_STOP(&pVM->hwaccm.s.StatExit, x);
2354	goto ResumeExecution;
2355	}
2356	Assert(rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_PGM_SYNC_CR3);
2357	break;
2358	}
2359
2360	case VMX_EXIT_DRX_MOVE: /* 29 Debug-register accesses. */
2361	{
2362	if (!DBGFIsStepping(pVM))
2363	{
2364	/* Disable drx move intercepts. */
2365	pVM->hwaccm.s.vmx.proc_ctls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT;
2366	rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVM->hwaccm.s.vmx.proc_ctls);
2367	AssertRC(rc);
2368
2369	/* Save the host and load the guest debug state. */
2370	rc = CPUMR0LoadGuestDebugState(pVM, pCtx, true /* include DR6 */);
2371	AssertRC(rc);
2372
2373	#ifdef VBOX_WITH_STATISTICS
2374	STAM_COUNTER_INC(&pVM->hwaccm.s.StatDRxContextSwitch);
2375	if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(exitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
2376	STAM_COUNTER_INC(&pVM->hwaccm.s.StatExitDRxWrite);
2377	else
2378	STAM_COUNTER_INC(&pVM->hwaccm.s.StatExitDRxRead);
2379	#endif
2380
2381	STAM_PROFILE_ADV_STOP(&pVM->hwaccm.s.StatExit, x);
2382	goto ResumeExecution;
2383	}
2384
2385	/** @todo clear VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT after the first time and restore drx registers afterwards */
2386	if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(exitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
2387	{
2388	Log2(("VMX: mov drx%d, genreg%d\n", VMX_EXIT_QUALIFICATION_DRX_REGISTER(exitQualification), VMX_EXIT_QUALIFICATION_DRX_GENREG(exitQualification)));
2389	STAM_COUNTER_INC(&pVM->hwaccm.s.StatExitDRxWrite);
2390	rc = EMInterpretDRxWrite(pVM, CPUMCTX2CORE(pCtx),
2391	VMX_EXIT_QUALIFICATION_DRX_REGISTER(exitQualification),
2392	VMX_EXIT_QUALIFICATION_DRX_GENREG(exitQualification));
2393	pVM->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_GUEST_DEBUG;
2394	Log2(("DR7=%08x\n", pCtx->dr[7]));
2395	}
2396	else
2397	{
2398	Log2(("VMX: mov x, drx\n"));
2399	STAM_COUNTER_INC(&pVM->hwaccm.s.StatExitDRxRead);
2400	rc = EMInterpretDRxRead(pVM, CPUMCTX2CORE(pCtx),
2401	VMX_EXIT_QUALIFICATION_DRX_GENREG(exitQualification),
2402	VMX_EXIT_QUALIFICATION_DRX_REGISTER(exitQualification));
2403	}
2404	/* Update EIP if no error occurred. */
2405	if (VBOX_SUCCESS(rc))
2406	pCtx->rip += cbInstr;
2407
2408	if (rc == VINF_SUCCESS)
2409	{
2410	/* Only resume if successful. */
2411	STAM_PROFILE_ADV_STOP(&pVM->hwaccm.s.StatExit, x);
2412	goto ResumeExecution;
2413	}
2414	Assert(rc == VERR_EM_INTERPRETER);
2415	break;
2416	}
2417
2418	/* Note: We'll get a #GP if the IO instruction isn't allowed (IOPL or TSS bitmap); no need to double check. */
2419	case VMX_EXIT_PORT_IO: /* 30 I/O instruction. */
2420	{
2421	uint32_t uIOWidth = VMX_EXIT_QUALIFICATION_IO_WIDTH(exitQualification);
2422	uint32_t uPort;
2423	bool fIOWrite = (VMX_EXIT_QUALIFICATION_IO_DIRECTION(exitQualification) == VMX_EXIT_QUALIFICATION_IO_DIRECTION_OUT);
2424
2425	/** @todo necessary to make the distinction? */
2426	if (VMX_EXIT_QUALIFICATION_IO_ENCODING(exitQualification) == VMX_EXIT_QUALIFICATION_IO_ENCODING_DX)
2427	{
2428	uPort = pCtx->edx & 0xffff;
2429	}
2430	else
2431	uPort = VMX_EXIT_QUALIFICATION_IO_PORT(exitQualification); /* Immediate encoding. */
2432
2433	/* paranoia */
2434	if (RT_UNLIKELY(uIOWidth == 2 \|\| uIOWidth >= 4))
2435	{
2436	rc = fIOWrite ? VINF_IOM_HC_IOPORT_WRITE : VINF_IOM_HC_IOPORT_READ;
2437	break;
2438	}
2439
2440	uint32_t cbSize = g_aIOSize[uIOWidth];
2441
2442	if (VMX_EXIT_QUALIFICATION_IO_STRING(exitQualification))
2443	{
2444	/* ins/outs */
2445	uint32_t prefix = 0;
2446	if (VMX_EXIT_QUALIFICATION_IO_REP(exitQualification))
2447	prefix \|= PREFIX_REP;
2448
2449	if (fIOWrite)
2450	{
2451	Log2(("IOMInterpretOUTSEx %VGv %x size=%d\n", pCtx->rip, uPort, cbSize));
2452	STAM_COUNTER_INC(&pVM->hwaccm.s.StatExitIOStringWrite);
2453	rc = IOMInterpretOUTSEx(pVM, CPUMCTX2CORE(pCtx), uPort, prefix, cbSize);
2454	}
2455	else
2456	{
2457	Log2(("IOMInterpretINSEx %VGv %x size=%d\n", pCtx->rip, uPort, cbSize));
2458	STAM_COUNTER_INC(&pVM->hwaccm.s.StatExitIOStringRead);
2459	rc = IOMInterpretINSEx(pVM, CPUMCTX2CORE(pCtx), uPort, prefix, cbSize);
2460	}
2461	}
2462	else
2463	{
2464	/* normal in/out */
2465	uint32_t uAndVal = g_aIOOpAnd[uIOWidth];
2466
2467	Assert(!VMX_EXIT_QUALIFICATION_IO_REP(exitQualification));
2468
2469	if (fIOWrite)
2470	{
2471	STAM_COUNTER_INC(&pVM->hwaccm.s.StatExitIOWrite);
2472	rc = IOMIOPortWrite(pVM, uPort, pCtx->eax & uAndVal, cbSize);
2473	}
2474	else
2475	{
2476	uint32_t u32Val = 0;
2477
2478	STAM_COUNTER_INC(&pVM->hwaccm.s.StatExitIORead);
2479	rc = IOMIOPortRead(pVM, uPort, &u32Val, cbSize);
2480	if (IOM_SUCCESS(rc))
2481	{
2482	/* Write back to the EAX register. */
2483	pCtx->eax = (pCtx->eax & ~uAndVal) \| (u32Val & uAndVal);
2484	}
2485	}
2486	}
2487	/*
2488	* Handled the I/O return codes.
2489	* (The unhandled cases end up with rc == VINF_EM_RAW_EMULATE_INSTR.)
2490	*/
2491	if (IOM_SUCCESS(rc))
2492	{
2493	/* Update EIP and continue execution. */
2494	pCtx->rip += cbInstr;
2495	if (RT_LIKELY(rc == VINF_SUCCESS))
2496	{
2497	/* If any IO breakpoints are armed, then we should check if a debug trap needs to be generated. */
2498	if (pCtx->dr[7] & X86_DR7_ENABLED_MASK)
2499	{
2500	STAM_COUNTER_INC(&pVM->hwaccm.s.StatDRxIOCheck);
2501	for (unsigned i=0;i<4;i++)
2502	{
2503	unsigned uBPLen = g_aIOSize[X86_DR7_GET_LEN(pCtx->dr[7], i)];
2504
2505	if ( (uPort >= pCtx->dr[i] && uPort < pCtx->dr[i] + uBPLen)
2506	&& (pCtx->dr[7] & (X86_DR7_L(i) \| X86_DR7_G(i)))
2507	&& (pCtx->dr[7] & X86_DR7_RW(i, X86_DR7_RW_IO)) == X86_DR7_RW(i, X86_DR7_RW_IO))
2508	{
2509	uint64_t uDR6;
2510
2511	Assert(CPUMIsGuestDebugStateActive(pVM));
2512
2513	uDR6 = ASMGetDR6();
2514
2515	/* Clear all breakpoint status flags and set the one we just hit. */
2516	uDR6 &= ~(X86_DR6_B0\|X86_DR6_B1\|X86_DR6_B2\|X86_DR6_B3);
2517	uDR6 \|= RT_BIT(i);
2518
2519	/* Note: AMD64 Architecture Programmer's Manual 13.1:
2520	* Bits 15:13 of the DR6 register is never cleared by the processor and must be cleared by software after
2521	* the contents have been read.
2522	*/
2523	ASMSetDR6(uDR6);
2524
2525	/* X86_DR7_GD will be cleared if drx accesses should be trapped inside the guest. */
2526	pCtx->dr[7] &= ~X86_DR7_GD;
2527
2528	/* Paranoia. */
2529	pCtx->dr[7] &= 0xffffffff; /* upper 32 bits reserved */
2530	pCtx->dr[7] &= ~(RT_BIT(11) \| RT_BIT(12) \| RT_BIT(14) \| RT_BIT(15)); /* must be zero */
2531	pCtx->dr[7] \|= 0x400; /* must be one */
2532
2533	/* Resync DR7 */
2534	rc = VMXWriteVMCS(VMX_VMCS_GUEST_DR7, pCtx->dr[7]);
2535	AssertRC(rc);
2536
2537	/* Construct inject info. */
2538	intInfo = X86_XCPT_DB;
2539	intInfo \|= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
2540	intInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HWEXCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
2541
2542	Log(("Inject IO debug trap at %VGv\n", pCtx->rip));
2543	rc = VMXR0InjectEvent(pVM, pCtx, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(intInfo), 0, 0);
2544	AssertRC(rc);
2545
2546	STAM_PROFILE_ADV_STOP(&pVM->hwaccm.s.StatExit, x);
2547	goto ResumeExecution;
2548	}
2549	}
2550	}
2551
2552	STAM_PROFILE_ADV_STOP(&pVM->hwaccm.s.StatExit, x);
2553	goto ResumeExecution;
2554	}
2555	break;
2556	}
2557
2558	#ifdef VBOX_STRICT
2559	if (rc == VINF_IOM_HC_IOPORT_READ)
2560	Assert(!fIOWrite);
2561	else if (rc == VINF_IOM_HC_IOPORT_WRITE)
2562	Assert(fIOWrite);
2563	else
2564	AssertMsg(VBOX_FAILURE(rc) \|\| rc == VINF_EM_RAW_EMULATE_INSTR \|\| rc == VINF_EM_RAW_GUEST_TRAP \|\| rc == VINF_TRPM_XCPT_DISPATCHED, ("%Vrc\n", rc));
2565	#endif
2566	break;
2567	}
2568
2569	case VMX_EXIT_TPR: /* 43 TPR below threshold. Guest software executed MOV to CR8. */
2570	LogFlow(("VMX_EXIT_TPR\n"));
2571	/* RIP is already set to the next instruction and the TPR has been synced back. Just resume. */
2572	goto ResumeExecution;
2573
2574	default:
2575	/* The rest is handled after syncing the entire CPU state. */
2576	break;
2577	}
2578
2579	/* Note: the guest state isn't entirely synced back at this stage. */
2580
2581	/* Investigate why there was a VM-exit. (part 2) */
2582	switch (exitReason)
2583	{
2584	case VMX_EXIT_EXCEPTION: /* 0 Exception or non-maskable interrupt (NMI). */
2585	case VMX_EXIT_EXTERNAL_IRQ: /* 1 External interrupt. */
2586	/* Already handled above. */
2587	break;
2588
2589	case VMX_EXIT_TRIPLE_FAULT: /* 2 Triple fault. */
2590	rc = VINF_EM_RESET; /* Triple fault equals a reset. */
2591	break;
2592
2593	case VMX_EXIT_INIT_SIGNAL: /* 3 INIT signal. */
2594	case VMX_EXIT_SIPI: /* 4 Start-up IPI (SIPI). */
2595	rc = VINF_EM_RAW_INTERRUPT;
2596	AssertFailed(); /* Can't happen. Yet. */
2597	break;
2598
2599	case VMX_EXIT_IO_SMI_IRQ: /* 5 I/O system-management interrupt (SMI). */
2600	case VMX_EXIT_SMI_IRQ: /* 6 Other SMI. */
2601	rc = VINF_EM_RAW_INTERRUPT;
2602	AssertFailed(); /* Can't happen afaik. */
2603	break;
2604
2605	case VMX_EXIT_TASK_SWITCH: /* 9 Task switch. */
2606	rc = VERR_EM_INTERPRETER;
2607	break;
2608
2609	case VMX_EXIT_HLT: /* 12 Guest software attempted to execute HLT. */
2610	/** Check if external interrupts are pending; if so, don't switch back. */
2611	pCtx->rip++; /* skip hlt */
2612	if ( pCtx->eflags.Bits.u1IF
2613	&& VM_FF_ISPENDING(pVM, (VM_FF_INTERRUPT_APIC\|VM_FF_INTERRUPT_PIC)))
2614	goto ResumeExecution;
2615
2616	rc = VINF_EM_HALT;
2617	break;
2618
2619	case VMX_EXIT_RSM: /* 17 Guest software attempted to execute RSM in SMM. */
2620	AssertFailed(); /* can't happen. */
2621	rc = VINF_EM_RAW_EXCEPTION_PRIVILEGED;
2622	break;
2623
2624	case VMX_EXIT_VMCALL: /* 18 Guest software executed VMCALL. */
2625	case VMX_EXIT_VMCLEAR: /* 19 Guest software executed VMCLEAR. */
2626	case VMX_EXIT_VMLAUNCH: /* 20 Guest software executed VMLAUNCH. */
2627	case VMX_EXIT_VMPTRLD: /* 21 Guest software executed VMPTRLD. */
2628	case VMX_EXIT_VMPTRST: /* 22 Guest software executed VMPTRST. */
2629	case VMX_EXIT_VMREAD: /* 23 Guest software executed VMREAD. */
2630	case VMX_EXIT_VMRESUME: /* 24 Guest software executed VMRESUME. */
2631	case VMX_EXIT_VMWRITE: /* 25 Guest software executed VMWRITE. */
2632	case VMX_EXIT_VMXOFF: /* 26 Guest software executed VMXOFF. */
2633	case VMX_EXIT_VMXON: /* 27 Guest software executed VMXON. */
2634	/** @todo inject #UD immediately */
2635	rc = VINF_EM_RAW_EXCEPTION_PRIVILEGED;
2636	break;
2637
2638	case VMX_EXIT_CPUID: /* 10 Guest software attempted to execute CPUID. */
2639	case VMX_EXIT_RDTSC: /* 16 Guest software attempted to execute RDTSC. */
2640	case VMX_EXIT_INVPG: /* 14 Guest software attempted to execute INVPG. */
2641	case VMX_EXIT_CRX_MOVE: /* 28 Control-register accesses. */
2642	case VMX_EXIT_DRX_MOVE: /* 29 Debug-register accesses. */
2643	case VMX_EXIT_PORT_IO: /* 30 I/O instruction. */
2644	/* already handled above */
2645	AssertMsg( rc == VINF_PGM_CHANGE_MODE
2646	\|\| rc == VINF_EM_RAW_INTERRUPT
2647	\|\| rc == VERR_EM_INTERPRETER
2648	\|\| rc == VINF_EM_RAW_EMULATE_INSTR
2649	\|\| rc == VINF_PGM_SYNC_CR3
2650	\|\| rc == VINF_IOM_HC_IOPORT_READ
2651	\|\| rc == VINF_IOM_HC_IOPORT_WRITE
2652	\|\| rc == VINF_EM_RAW_GUEST_TRAP
2653	\|\| rc == VINF_TRPM_XCPT_DISPATCHED
2654	\|\| rc == VINF_EM_RESCHEDULE_REM,
2655	("rc = %d\n", rc));
2656	break;
2657
2658	case VMX_EXIT_TPR: /* 43 TPR below threshold. Guest software executed MOV to CR8. */
2659	case VMX_EXIT_RDMSR: /* 31 RDMSR. Guest software attempted to execute RDMSR. */
2660	case VMX_EXIT_WRMSR: /* 32 WRMSR. Guest software attempted to execute WRMSR. */
2661	/* Note: If we decide to emulate them here, then we must sync the MSRs that could have been changed (sysenter, fs/gs base)!!! */
2662	rc = VERR_EM_INTERPRETER;
2663	break;
2664
2665	case VMX_EXIT_RDPMC: /* 15 Guest software attempted to execute RDPMC. */
2666	case VMX_EXIT_MWAIT: /* 36 Guest software executed MWAIT. */
2667	case VMX_EXIT_MONITOR: /* 39 Guest software attempted to execute MONITOR. */
2668	case VMX_EXIT_PAUSE: /* 40 Guest software attempted to execute PAUSE. */
2669	rc = VINF_EM_RAW_EXCEPTION_PRIVILEGED;
2670	break;
2671
2672	case VMX_EXIT_IRQ_WINDOW: /* 7 Interrupt window. */
2673	Assert(rc == VINF_EM_RAW_INTERRUPT);
2674	break;
2675
2676	case VMX_EXIT_ERR_INVALID_GUEST_STATE: /* 33 VM-entry failure due to invalid guest state. */
2677	{
2678	#ifdef VBOX_STRICT
2679	Log(("VMX_EXIT_ERR_INVALID_GUEST_STATE\n"));
2680
2681	VMXReadVMCS(VMX_VMCS_GUEST_RIP, &val);
2682	Log(("Old eip %VGv new %VGv\n", pCtx->rip, (RTGCPTR)val));
2683
2684	VMXReadVMCS(VMX_VMCS_GUEST_CR0, &val);
2685	Log(("VMX_VMCS_GUEST_CR0 %RX64\n", val));
2686
2687	VMXReadVMCS(VMX_VMCS_GUEST_CR3, &val);
2688	Log(("VMX_VMCS_HOST_CR3 %VGp\n", val));
2689
2690	VMXReadVMCS(VMX_VMCS_GUEST_CR4, &val);
2691	Log(("VMX_VMCS_GUEST_CR4 %RX64\n", val));
2692
2693	VMXReadVMCS(VMX_VMCS_GUEST_RFLAGS, &val);
2694	Log(("VMX_VMCS_GUEST_RFLAGS %08x\n", val));
2695
2696	VMX_LOG_SELREG(CS, "CS");
2697	VMX_LOG_SELREG(DS, "DS");
2698	VMX_LOG_SELREG(ES, "ES");
2699	VMX_LOG_SELREG(FS, "FS");
2700	VMX_LOG_SELREG(GS, "GS");
2701	VMX_LOG_SELREG(SS, "SS");
2702	VMX_LOG_SELREG(TR, "TR");
2703	VMX_LOG_SELREG(LDTR, "LDTR");
2704
2705	VMXReadVMCS(VMX_VMCS_GUEST_GDTR_BASE, &val);
2706	Log(("VMX_VMCS_GUEST_GDTR_BASE %VGv\n", val));
2707	VMXReadVMCS(VMX_VMCS_GUEST_IDTR_BASE, &val);
2708	Log(("VMX_VMCS_GUEST_IDTR_BASE %VGv\n", val));
2709	#endif /* VBOX_STRICT */
2710	rc = VERR_VMX_INVALID_GUEST_STATE;
2711	break;
2712	}
2713
2714	case VMX_EXIT_ERR_MSR_LOAD: /* 34 VM-entry failure due to MSR loading. */
2715	case VMX_EXIT_ERR_MACHINE_CHECK: /* 41 VM-entry failure due to machine-check. */
2716	default:
2717	rc = VERR_VMX_UNEXPECTED_EXIT_CODE;
2718	AssertMsgFailed(("Unexpected exit code %d\n", exitReason)); /* Can't happen. */
2719	break;
2720
2721	}
2722	end:
2723
2724	/* Signal changes for the recompiler. */
2725	CPUMSetChangedFlags(pVM, CPUM_CHANGED_SYSENTER_MSR \| CPUM_CHANGED_LDTR \| CPUM_CHANGED_GDTR \| CPUM_CHANGED_IDTR \| CPUM_CHANGED_TR \| CPUM_CHANGED_HIDDEN_SEL_REGS);
2726
2727	/* If we executed vmlaunch/vmresume and an external irq was pending, then we don't have to do a full sync the next time. */
2728	if ( exitReason == VMX_EXIT_EXTERNAL_IRQ
2729	&& !VMX_EXIT_INTERRUPTION_INFO_VALID(intInfo))
2730	{
2731	STAM_COUNTER_INC(&pVM->hwaccm.s.StatPendingHostIrq);
2732	/* On the next entry we'll only sync the host context. */
2733	pVM->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_HOST_CONTEXT;
2734	}
2735	else
2736	{
2737	/* On the next entry we'll sync everything. */
2738	/** @todo we can do better than this */
2739	/* Not in the VINF_PGM_CHANGE_MODE though! */
2740	pVM->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_ALL;
2741	}
2742
2743	/* translate into a less severe return code */
2744	if (rc == VERR_EM_INTERPRETER)
2745	rc = VINF_EM_RAW_EMULATE_INSTR;
2746	else
2747	/* Try to extract more information about what might have gone wrong here. */
2748	if (rc == VERR_VMX_INVALID_VMCS_PTR)
2749	{
2750	VMXGetActivateVMCS(&pVM->hwaccm.s.vmx.lasterror.u64VMCSPhys);
2751	pVM->hwaccm.s.vmx.lasterror.ulVMCSRevision = (uint32_t )pVM->hwaccm.s.vmx.pVMCS;
2752	}
2753
2754	STAM_PROFILE_ADV_STOP(&pVM->hwaccm.s.StatExit, x);
2755
2756	Log2(("X"));
2757	return rc;
2758	}
2759
2760
2761	/**
2762	* Enters the VT-x session
2763	*
2764	* @returns VBox status code.
2765	* @param pVM The VM to operate on.
2766	* @param pCpu CPU info struct
2767	*/
2768	VMMR0DECL(int) VMXR0Enter(PVM pVM, PHWACCM_CPUINFO pCpu)
2769	{
2770	Assert(pVM->hwaccm.s.vmx.fSupported);
2771
2772	unsigned cr4 = ASMGetCR4();
2773	if (!(cr4 & X86_CR4_VMXE))
2774	{
2775	AssertMsgFailed(("X86_CR4_VMXE should be set!\n"));
2776	return VERR_VMX_X86_CR4_VMXE_CLEARED;
2777	}
2778
2779	/* Activate the VM Control Structure. */
2780	int rc = VMXActivateVMCS(pVM->hwaccm.s.vmx.pVMCSPhys);
2781	if (VBOX_FAILURE(rc))
2782	return rc;
2783
2784	pVM->hwaccm.s.vmx.fResumeVM = false;
2785	return VINF_SUCCESS;
2786	}
2787
2788
2789	/**
2790	* Leaves the VT-x session
2791	*
2792	* @returns VBox status code.
2793	* @param pVM The VM to operate on.
2794	* @param pCtx CPU context
2795	*/
2796	VMMR0DECL(int) VMXR0Leave(PVM pVM, PCPUMCTX pCtx)
2797	{
2798	Assert(pVM->hwaccm.s.vmx.fSupported);
2799
2800	/* Save the guest debug state if necessary. */
2801	if (CPUMIsGuestDebugStateActive(pVM))
2802	{
2803	CPUMR0SaveGuestDebugState(pVM, pCtx, true /* save DR6 */);
2804
2805	/* Enable drx move intercepts again. */
2806	pVM->hwaccm.s.vmx.proc_ctls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT;
2807	int rc = VMXWriteVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, pVM->hwaccm.s.vmx.proc_ctls);
2808	AssertRC(rc);
2809
2810	/* Resync the debug registers the next time. */
2811	pVM->hwaccm.s.fContextUseFlags \|= HWACCM_CHANGED_GUEST_DEBUG;
2812	}
2813	else
2814	Assert(pVM->hwaccm.s.vmx.proc_ctls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT);
2815
2816	/* Clear VM Control Structure. Marking it inactive, clearing implementation specific data and writing back VMCS data to memory. */
2817	int rc = VMXClearVMCS(pVM->hwaccm.s.vmx.pVMCSPhys);
2818	AssertRC(rc);
2819
2820	return VINF_SUCCESS;
2821	}
2822
2823	/**
2824	* Invalidates a guest page
2825	*
2826	* @returns VBox status code.
2827	* @param pVM The VM to operate on.
2828	* @param GCVirt Page to invalidate
2829	*/
2830	VMMR0DECL(int) VMXR0InvalidatePage(PVM pVM, RTGCPTR GCVirt)
2831	{
2832	bool fFlushPending = pVM->hwaccm.s.fForceTLBFlush;
2833
2834	/* @todo Only relevant if we want to use VPID. */
2835
2836	/* Skip it if a TLB flush is already pending. */
2837	if (!fFlushPending)
2838	{
2839	}
2840	return VINF_SUCCESS;
2841	}
2842
2843	/**
2844	* Invalidates a guest page by physical address
2845	*
2846	* NOTE: Assumes the current instruction references this physical page though a virtual address!!
2847	*
2848	* @returns VBox status code.
2849	* @param pVM The VM to operate on.
2850	* @param GCPhys Page to invalidate
2851	*/
2852	VMMR0DECL(int) VMXR0InvalidatePhysPage(PVM pVM, RTGCPHYS GCPhys)
2853	{
2854	bool fFlushPending = pVM->hwaccm.s.fForceTLBFlush;
2855
2856	Assert(pVM->hwaccm.s.fNestedPaging);
2857
2858	/* Skip it if a TLB flush is already pending. */
2859	if (!fFlushPending)
2860	{
2861	}
2862	return VINF_SUCCESS;
2863	}
2864
2865	#ifdef VBOX_STRICT
2866	/**
2867	* Report world switch error and dump some useful debug info
2868	*
2869	* @param pVM The VM to operate on.
2870	* @param rc Return code
2871	* @param pCtx Current CPU context (not updated)
2872	*/
2873	static void VMXR0ReportWorldSwitchError(PVM pVM, int rc, PCPUMCTX pCtx)
2874	{
2875	switch (rc)
2876	{
2877	case VERR_VMX_INVALID_VMXON_PTR:
2878	AssertFailed();
2879	break;
2880
2881	case VERR_VMX_UNABLE_TO_START_VM:
2882	case VERR_VMX_UNABLE_TO_RESUME_VM:
2883	{
2884	int rc;
2885	RTCCUINTREG exitReason, instrError, val;
2886
2887	rc = VMXReadVMCS(VMX_VMCS_RO_EXIT_REASON, &exitReason);
2888	rc \|= VMXReadVMCS(VMX_VMCS_RO_VM_INSTR_ERROR, &instrError);
2889	AssertRC(rc);
2890	if (rc == VINF_SUCCESS)
2891	{
2892	RTGDTR gdtr;
2893	PX86DESCHC pDesc;
2894
2895	ASMGetGDTR(&gdtr);
2896
2897	Log(("Unable to start/resume VM for reason: %x. Instruction error %x\n", (uint32_t)exitReason, (uint32_t)instrError));
2898	Log(("Current stack %08x\n", &rc));
2899
2900
2901	VMXReadVMCS(VMX_VMCS_GUEST_RIP, &val);
2902	Log(("Old eip %VGv new %VGv\n", pCtx->rip, (RTGCPTR)val));
2903	VMXReadVMCS(VMX_VMCS_CTRL_PIN_EXEC_CONTROLS, &val);
2904	Log(("VMX_VMCS_CTRL_PIN_EXEC_CONTROLS %08x\n", val));
2905	VMXReadVMCS(VMX_VMCS_CTRL_PROC_EXEC_CONTROLS, &val);
2906	Log(("VMX_VMCS_CTRL_PROC_EXEC_CONTROLS %08x\n", val));
2907	VMXReadVMCS(VMX_VMCS_CTRL_ENTRY_CONTROLS, &val);
2908	Log(("VMX_VMCS_CTRL_ENTRY_CONTROLS %08x\n", val));
2909	VMXReadVMCS(VMX_VMCS_CTRL_EXIT_CONTROLS, &val);
2910	Log(("VMX_VMCS_CTRL_EXIT_CONTROLS %08x\n", val));
2911
2912	VMXReadVMCS(VMX_VMCS_HOST_CR0, &val);
2913	Log(("VMX_VMCS_HOST_CR0 %08x\n", val));
2914
2915	VMXReadVMCS(VMX_VMCS_HOST_CR3, &val);
2916	Log(("VMX_VMCS_HOST_CR3 %VHp\n", val));
2917
2918	VMXReadVMCS(VMX_VMCS_HOST_CR4, &val);
2919	Log(("VMX_VMCS_HOST_CR4 %08x\n", val));
2920
2921	VMXReadVMCS(VMX_VMCS_HOST_FIELD_CS, &val);
2922	Log(("VMX_VMCS_HOST_FIELD_CS %08x\n", val));
2923
2924	VMXReadVMCS(VMX_VMCS_GUEST_RFLAGS, &val);
2925	Log(("VMX_VMCS_GUEST_RFLAGS %08x\n", val));
2926
2927	if (val < gdtr.cbGdt)
2928	{
2929	pDesc = &((PX86DESCHC)gdtr.pGdt)[val >> X86_SEL_SHIFT_HC];
2930	HWACCMR0DumpDescriptor(pDesc, val, "CS: ");
2931	}
2932
2933	VMXReadVMCS(VMX_VMCS_HOST_FIELD_DS, &val);
2934	Log(("VMX_VMCS_HOST_FIELD_DS %08x\n", val));
2935	if (val < gdtr.cbGdt)
2936	{
2937	pDesc = &((PX86DESCHC)gdtr.pGdt)[val >> X86_SEL_SHIFT_HC];
2938	HWACCMR0DumpDescriptor(pDesc, val, "DS: ");
2939	}
2940
2941	VMXReadVMCS(VMX_VMCS_HOST_FIELD_ES, &val);
2942	Log(("VMX_VMCS_HOST_FIELD_ES %08x\n", val));
2943	if (val < gdtr.cbGdt)
2944	{
2945	pDesc = &((PX86DESCHC)gdtr.pGdt)[val >> X86_SEL_SHIFT_HC];
2946	HWACCMR0DumpDescriptor(pDesc, val, "ES: ");
2947	}
2948
2949	VMXReadVMCS(VMX_VMCS_HOST_FIELD_FS, &val);
2950	Log(("VMX_VMCS_HOST_FIELD_FS %08x\n", val));
2951	if (val < gdtr.cbGdt)
2952	{
2953	pDesc = &((PX86DESCHC)gdtr.pGdt)[val >> X86_SEL_SHIFT_HC];
2954	HWACCMR0DumpDescriptor(pDesc, val, "FS: ");
2955	}
2956
2957	VMXReadVMCS(VMX_VMCS_HOST_FIELD_GS, &val);
2958	Log(("VMX_VMCS_HOST_FIELD_GS %08x\n", val));
2959	if (val < gdtr.cbGdt)
2960	{
2961	pDesc = &((PX86DESCHC)gdtr.pGdt)[val >> X86_SEL_SHIFT_HC];
2962	HWACCMR0DumpDescriptor(pDesc, val, "GS: ");
2963	}
2964
2965	VMXReadVMCS(VMX_VMCS_HOST_FIELD_SS, &val);
2966	Log(("VMX_VMCS_HOST_FIELD_SS %08x\n", val));
2967	if (val < gdtr.cbGdt)
2968	{
2969	pDesc = &((PX86DESCHC)gdtr.pGdt)[val >> X86_SEL_SHIFT_HC];
2970	HWACCMR0DumpDescriptor(pDesc, val, "SS: ");
2971	}
2972
2973	VMXReadVMCS(VMX_VMCS_HOST_FIELD_TR, &val);
2974	Log(("VMX_VMCS_HOST_FIELD_TR %08x\n", val));
2975	if (val < gdtr.cbGdt)
2976	{
2977	pDesc = &((PX86DESCHC)gdtr.pGdt)[val >> X86_SEL_SHIFT_HC];
2978	HWACCMR0DumpDescriptor(pDesc, val, "TR: ");
2979	}
2980
2981	VMXReadVMCS(VMX_VMCS_HOST_TR_BASE, &val);
2982	Log(("VMX_VMCS_HOST_TR_BASE %VHv\n", val));
2983
2984	VMXReadVMCS(VMX_VMCS_HOST_GDTR_BASE, &val);
2985	Log(("VMX_VMCS_HOST_GDTR_BASE %VHv\n", val));
2986	VMXReadVMCS(VMX_VMCS_HOST_IDTR_BASE, &val);
2987	Log(("VMX_VMCS_HOST_IDTR_BASE %VHv\n", val));
2988
2989	VMXReadVMCS(VMX_VMCS_HOST_SYSENTER_CS, &val);
2990	Log(("VMX_VMCS_HOST_SYSENTER_CS %08x\n", val));
2991
2992	VMXReadVMCS(VMX_VMCS_HOST_SYSENTER_EIP, &val);
2993	Log(("VMX_VMCS_HOST_SYSENTER_EIP %VHv\n", val));
2994
2995	VMXReadVMCS(VMX_VMCS_HOST_SYSENTER_ESP, &val);
2996	Log(("VMX_VMCS_HOST_SYSENTER_ESP %VHv\n", val));
2997
2998	VMXReadVMCS(VMX_VMCS_HOST_RSP, &val);
2999	Log(("VMX_VMCS_HOST_RSP %VHv\n", val));
3000	VMXReadVMCS(VMX_VMCS_HOST_RIP, &val);
3001	Log(("VMX_VMCS_HOST_RIP %VHv\n", val));
3002
3003	#if HC_ARCH_BITS == 64
3004	Log(("MSR_K6_EFER = %VX64\n", ASMRdMsr(MSR_K6_EFER)));
3005	Log(("MSR_K6_STAR = %VX64\n", ASMRdMsr(MSR_K6_STAR)));
3006	Log(("MSR_K8_LSTAR = %VX64\n", ASMRdMsr(MSR_K8_LSTAR)));
3007	Log(("MSR_K8_CSTAR = %VX64\n", ASMRdMsr(MSR_K8_CSTAR)));
3008	Log(("MSR_K8_SF_MASK = %VX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
3009	#endif
3010	}
3011	break;
3012	}
3013
3014	default:
3015	/* impossible */
3016	AssertFailed();
3017	break;
3018	}
3019	}
3020	#endif /* VBOX_STRICT */

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMR0/HWVMXR0.cpp@ 13085

Download in other formats: