HMVMXR0.cpp@ 45344

Last change on this file since 45344 was 45344, checked in by vboxsync, 12 years ago
VMM/VMMR0: HMVMXR0 properties.
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 358.5 KB

Line
1	/* $Id: HMVMXR0.cpp 45344 2013-04-04 17:38:24Z vboxsync $ */
2	/** @file
3	* HM VMX (Intel VT-x) - Host Context Ring-0.
4	*/
5
6	/*
7	* Copyright (C) 2012 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18	/*******************************************************************************
19	* Header Files *
20	*******************************************************************************/
21	#define LOG_GROUP LOG_GROUP_HM
22	#include <iprt/asm-amd64-x86.h>
23	#include <iprt/thread.h>
24	#include <iprt/string.h>
25
26	#include "HMInternal.h"
27	#include <VBox/vmm/vm.h>
28	#include "HWVMXR0.h"
29	#include <VBox/vmm/pdmapi.h>
30	#include <VBox/vmm/dbgf.h>
31	#include <VBox/vmm/iom.h>
32	#include <VBox/vmm/selm.h>
33	#include <VBox/vmm/tm.h>
34	#ifdef VBOX_WITH_REM
35	# include <VBox/vmm/rem.h>
36	#endif
37	#ifdef DEBUG_ramshankar
38	#define VBOX_ALWAYS_TRAP_ALL_EXCEPTIONS
39	#endif
40
41	/*******************************************************************************
42	* Defined Constants And Macros *
43	*******************************************************************************/
44	#if defined(RT_ARCH_AMD64)
45	# define VMX_IS_64BIT_HOST_MODE() (true)
46	#elif defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
47	# define VMX_IS_64BIT_HOST_MODE() (g_fVMXIs64bitHost != 0)
48	#else
49	# define VMX_IS_64BIT_HOST_MODE() (false)
50	#endif
51
52	#define VMX_SEL_UNUSABLE RT_BIT(16)
53
54	/**
55	* Updated-guest-state flags.
56	*/
57	#define VMX_UPDATED_GUEST_FPU RT_BIT(0)
58	#define VMX_UPDATED_GUEST_RIP RT_BIT(1)
59	#define VMX_UPDATED_GUEST_RSP RT_BIT(2)
60	#define VMX_UPDATED_GUEST_RFLAGS RT_BIT(3)
61	#define VMX_UPDATED_GUEST_CR0 RT_BIT(4)
62	#define VMX_UPDATED_GUEST_CR3 RT_BIT(5)
63	#define VMX_UPDATED_GUEST_CR4 RT_BIT(6)
64	#define VMX_UPDATED_GUEST_GDTR RT_BIT(7)
65	#define VMX_UPDATED_GUEST_IDTR RT_BIT(8)
66	#define VMX_UPDATED_GUEST_LDTR RT_BIT(9)
67	#define VMX_UPDATED_GUEST_TR RT_BIT(10)
68	#define VMX_UPDATED_GUEST_SEGMENT_REGS RT_BIT(11)
69	#define VMX_UPDATED_GUEST_DEBUG RT_BIT(12)
70	#define VMX_UPDATED_GUEST_FS_BASE_MSR RT_BIT(13)
71	#define VMX_UPDATED_GUEST_GS_BASE_MSR RT_BIT(14)
72	#define VMX_UPDATED_GUEST_SYSENTER_CS_MSR RT_BIT(15)
73	#define VMX_UPDATED_GUEST_SYSENTER_EIP_MSR RT_BIT(16)
74	#define VMX_UPDATED_GUEST_SYSENTER_ESP_MSR RT_BIT(17)
75	#define VMX_UPDATED_GUEST_INTR_STATE RT_BIT(18)
76	#define VMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS RT_BIT(19)
77	#define VMX_UPDATED_GUEST_ACTIVITY_STATE RT_BIT(20)
78	#define VMX_UPDATED_GUEST_APIC_STATE RT_BIT(21)
79	#define VMX_UPDATED_GUEST_ALL ( VMX_UPDATED_GUEST_FPU \
80	\| VMX_UPDATED_GUEST_RIP \
81	\| VMX_UPDATED_GUEST_RSP \
82	\| VMX_UPDATED_GUEST_RFLAGS \
83	\| VMX_UPDATED_GUEST_CR0 \
84	\| VMX_UPDATED_GUEST_CR3 \
85	\| VMX_UPDATED_GUEST_CR4 \
86	\| VMX_UPDATED_GUEST_GDTR \
87	\| VMX_UPDATED_GUEST_IDTR \
88	\| VMX_UPDATED_GUEST_LDTR \
89	\| VMX_UPDATED_GUEST_TR \
90	\| VMX_UPDATED_GUEST_SEGMENT_REGS \
91	\| VMX_UPDATED_GUEST_DEBUG \
92	\| VMX_UPDATED_GUEST_FS_BASE_MSR \
93	\| VMX_UPDATED_GUEST_GS_BASE_MSR \
94	\| VMX_UPDATED_GUEST_SYSENTER_CS_MSR \
95	\| VMX_UPDATED_GUEST_SYSENTER_EIP_MSR \
96	\| VMX_UPDATED_GUEST_SYSENTER_ESP_MSR \
97	\| VMX_UPDATED_GUEST_INTR_STATE \
98	\| VMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS \
99	\| VMX_UPDATED_GUEST_ACTIVITY_STATE \
100	\| VMX_UPDATED_GUEST_APIC_STATE)
101
102	/**
103	* Flags to skip redundant reads of some common VMCS fields.
104	*/
105	#define VMX_TRANSIENT_IDT_VECTORING_INFO RT_BIT(0)
106	#define VMX_TRANSIENT_IDT_VECTORING_ERROR_CODE RT_BIT(1)
107	#define VMX_TRANSIENT_EXIT_QUALIFICATION RT_BIT(2)
108	#define VMX_TRANSIENT_EXIT_INSTR_LEN RT_BIT(3)
109	#define VMX_TRANSIENT_EXIT_INTERRUPTION_INFO RT_BIT(4)
110	#define VMX_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE RT_BIT(5)
111
112	/*
113	* Exception bitmap mask for real-mode guests (real-on-v86). We need to intercept all exceptions manually (except #PF).
114	* #NM is also handled spearetely, see hmR0VmxLoadGuestControlRegs(). #PF need not be intercepted even in real-mode if
115	* we have Nested Paging support.
116	*/
117	#define VMX_REAL_MODE_XCPT_BITMAP ( RT_BIT(X86_XCPT_DE) \| RT_BIT(X86_XCPT_DB) \| RT_BIT(X86_XCPT_NMI) \
118	\| RT_BIT(X86_XCPT_BP) \| RT_BIT(X86_XCPT_OF) \| RT_BIT(X86_XCPT_BR) \
119	\| RT_BIT(X86_XCPT_UD) /* RT_BIT(X86_XCPT_NM) */ \| RT_BIT(X86_XCPT_DF) \
120	\| RT_BIT(X86_XCPT_CO_SEG_OVERRUN) \| RT_BIT(X86_XCPT_TS) \| RT_BIT(X86_XCPT_NP) \
121	\| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
122	\| RT_BIT(X86_XCPT_MF) \| RT_BIT(X86_XCPT_AC) \| RT_BIT(X86_XCPT_MC) \
123	\| RT_BIT(X86_XCPT_XF))
124
125	/* Maximum VM-instruction error number. */
126	#define VMX_INSTR_ERROR_MAX 28
127
128	/*******************************************************************************
129	* Structures and Typedefs *
130	*******************************************************************************/
131	/**
132	* A state structure for holding miscellaneous information across
133	* VMX non-root operation and restored after the transition.
134	*/
135	typedef struct VMXTRANSIENT
136	{
137	/** The host's rflags/eflags. */
138	RTCCUINTREG uEFlags;
139	/** The guest's LSTAR MSR value used for TPR patching for 32-bit guests. */
140	uint64_t u64LStarMsr;
141	/** The guest's TPR value used for TPR shadowing. */
142	uint8_t u8GuestTpr;
143
144	/** The basic VM-exit reason. */
145	uint16_t uExitReason;
146	/** The VM-exit exit qualification. */
147	RTGCUINTPTR uExitQualification;
148	/** The VM-exit interruption error code. */
149	uint32_t uExitIntrErrorCode;
150
151	/** The VM-exit interruption-information field. */
152	uint32_t uExitIntrInfo;
153	/** Whether the VM-entry failed or not. */
154	bool fVMEntryFailed;
155	/** The VM-exit instruction-length field. */
156	uint32_t cbInstr;
157
158	/** The VM-entry interruption-information field. */
159	uint32_t uEntryIntrInfo;
160	/** The VM-entry exception error code field. */
161	uint32_t uEntryXcptErrorCode;
162	/** The VM-entry instruction length field. */
163	uint32_t cbEntryInstr;
164
165	/** IDT-vectoring information field. */
166	uint32_t uIdtVectoringInfo;
167	/** IDT-vectoring error code. */
168	uint32_t uIdtVectoringErrorCode;
169
170	/** Mask of currently read VMCS fields; VMX_TRANSIENT_. /
171	uint32_t fVmcsFieldsRead;
172	/** Whether TSC-offsetting should be setup before VM-entry. */
173	bool fUpdateTscOffsetting;
174	/** Whether the VM-exit was caused by a page-fault during delivery of a
175	* contributary exception or a page-fault. */
176	bool fVectoringPF;
177	} VMXTRANSIENT, *PVMXTRANSIENT;
178
179	/**
180	* MSR-bitmap read permissions.
181	*/
182	typedef enum VMXMSREXITREAD
183	{
184	/** Reading this MSR causes a VM-exit. */
185	VMXMSREXIT_INTERCEPT_READ = 0xb,
186	/** Reading this MSR does not cause a VM-exit. */
187	VMXMSREXIT_PASSTHRU_READ
188	} VMXMSREXITREAD;
189
190	/**
191	* MSR-bitmap write permissions.
192	*/
193	typedef enum VMXMSREXITWRITE
194	{
195	/** Writing to this MSR causes a VM-exit. */
196	VMXMSREXIT_INTERCEPT_WRITE = 0xd,
197	/** Writing to this MSR does not cause a VM-exit. */
198	VMXMSREXIT_PASSTHRU_WRITE
199	} VMXMSREXITWRITE;
200
201	/*******************************************************************************
202	* Internal Functions *
203	*******************************************************************************/
204	static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_VPID enmFlush, RTGCPTR GCPtr);
205	static int hmR0VmxInjectEventVmcs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntrInfo,
206	uint32_t cbInstr, uint32_t u32ErrCode);
207
208	static DECLCALLBACK(int) hmR0VmxExitXcptNmi(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
209	static DECLCALLBACK(int) hmR0VmxExitExtInt(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
210	static DECLCALLBACK(int) hmR0VmxExitTripleFault(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
211	static DECLCALLBACK(int) hmR0VmxExitInitSignal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
212	static DECLCALLBACK(int) hmR0VmxExitSipi(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
213	static DECLCALLBACK(int) hmR0VmxExitIoSmi(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
214	static DECLCALLBACK(int) hmR0VmxExitSmi(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
215	static DECLCALLBACK(int) hmR0VmxExitIntWindow(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
216	static DECLCALLBACK(int) hmR0VmxExitNmiWindow(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
217	static DECLCALLBACK(int) hmR0VmxExitTaskSwitch(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
218	static DECLCALLBACK(int) hmR0VmxExitCpuid(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
219	static DECLCALLBACK(int) hmR0VmxExitGetsec(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
220	static DECLCALLBACK(int) hmR0VmxExitHlt(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
221	static DECLCALLBACK(int) hmR0VmxExitInvd(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
222	static DECLCALLBACK(int) hmR0VmxExitInvlpg(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
223	static DECLCALLBACK(int) hmR0VmxExitRdpmc(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
224	static DECLCALLBACK(int) hmR0VmxExitRdtsc(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
225	static DECLCALLBACK(int) hmR0VmxExitRsm(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
226	static DECLCALLBACK(int) hmR0VmxExitInjectXcptUD(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
227	static DECLCALLBACK(int) hmR0VmxExitMovCRx(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
228	static DECLCALLBACK(int) hmR0VmxExitMovDRx(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
229	static DECLCALLBACK(int) hmR0VmxExitIoInstr(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
230	static DECLCALLBACK(int) hmR0VmxExitRdmsr(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
231	static DECLCALLBACK(int) hmR0VmxExitWrmsr(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
232	static DECLCALLBACK(int) hmR0VmxExitErrInvalidGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
233	static DECLCALLBACK(int) hmR0VmxExitErrMsrLoad(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
234	static DECLCALLBACK(int) hmR0VmxExitErrUndefined(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
235	static DECLCALLBACK(int) hmR0VmxExitMwait(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
236	static DECLCALLBACK(int) hmR0VmxExitMtf(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
237	static DECLCALLBACK(int) hmR0VmxExitMonitor(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
238	static DECLCALLBACK(int) hmR0VmxExitPause(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
239	static DECLCALLBACK(int) hmR0VmxExitErrMachineCheck(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
240	static DECLCALLBACK(int) hmR0VmxExitTprBelowThreshold(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
241	static DECLCALLBACK(int) hmR0VmxExitApicAccess(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
242	static DECLCALLBACK(int) hmR0VmxExitXdtrAccess(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
243	static DECLCALLBACK(int) hmR0VmxExitXdtrAccess(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
244	static DECLCALLBACK(int) hmR0VmxExitEptViolation(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
245	static DECLCALLBACK(int) hmR0VmxExitEptMisconfig(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
246	static DECLCALLBACK(int) hmR0VmxExitRdtscp(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
247	static DECLCALLBACK(int) hmR0VmxExitPreemptionTimer(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
248	static DECLCALLBACK(int) hmR0VmxExitWbinvd(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
249	static DECLCALLBACK(int) hmR0VmxExitXsetbv(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
250	static DECLCALLBACK(int) hmR0VmxExitRdrand(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
251	static DECLCALLBACK(int) hmR0VmxExitInvpcid(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
252	static DECLCALLBACK(int) hmR0VmxExitXcptNM(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
253	static DECLCALLBACK(int) hmR0VmxExitXcptPF(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
254	static DECLCALLBACK(int) hmR0VmxExitXcptMF(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
255	static DECLCALLBACK(int) hmR0VmxExitXcptDB(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
256	static DECLCALLBACK(int) hmR0VmxExitXcptBP(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
257	static DECLCALLBACK(int) hmR0VmxExitXcptGP(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
258	static DECLCALLBACK(int) hmR0VmxExitXcptGeneric(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
259
260	/*******************************************************************************
261	* Global Variables *
262	*******************************************************************************/
263	/** @todo Move this to hm_vmx.h. */
264	/**
265	* VM-exit handler.
266	*
267	* @returns VBox status code.
268	* @param pVM Pointer to the VM.
269	* @param pVCpu Pointer to the VMCPU.
270	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
271	* out-of-sync. Make sure to update the required
272	* fields before using them.
273	* @param pVmxTransient Pointer to the VMX-transient structure.
274	*/
275	typedef DECLCALLBACK(int) FNVMEXITHANDLER(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
276	/** Pointer to VM-exit handler. */
277	typedef FNVMEXITHANDLER *PFNVMEXITHANDLER;
278
279	static const PFNVMEXITHANDLER s_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
280	{
281	/* 00 VMX_EXIT_XCPT_NMI */ hmR0VmxExitXcptNmi,
282	/* 01 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
283	/* 02 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
284	/* 03 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitInitSignal,
285	/* 04 VMX_EXIT_SIPI */ hmR0VmxExitSipi,
286	/* 05 VMX_EXIT_IO_SMI */ hmR0VmxExitIoSmi,
287	/* 06 VMX_EXIT_SMI */ hmR0VmxExitSmi,
288	/* 07 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
289	/* 08 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
290	/* 09 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
291	/* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
292	/* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
293	/* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
294	/* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
295	/* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
296	/* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
297	/* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
298	/* 17 VMX_EXIT_RSM */ hmR0VmxExitRsm,
299	/* 18 VMX_EXIT_VMCALL */ hmR0VmxExitInjectXcptUD,
300	/* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitInjectXcptUD,
301	/* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitInjectXcptUD,
302	/* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitInjectXcptUD,
303	/* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitInjectXcptUD,
304	/* 23 VMX_EXIT_VMREAD */ hmR0VmxExitInjectXcptUD,
305	/* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitInjectXcptUD,
306	/* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitInjectXcptUD,
307	/* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitInjectXcptUD,
308	/* 27 VMX_EXIT_VMXON */ hmR0VmxExitInjectXcptUD,
309	/* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
310	/* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
311	/* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
312	/* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
313	/* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
314	/* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
315	/* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrMsrLoad,
316	/* 35 UNDEFINED */ hmR0VmxExitErrUndefined,
317	/* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
318	/* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
319	/* 38 UNDEFINED */ hmR0VmxExitErrUndefined,
320	/* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
321	/* 40 UNDEFINED */ hmR0VmxExitPause,
322	/* 41 VMX_EXIT_PAUSE */ hmR0VmxExitErrMachineCheck,
323	/* 42 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUndefined,
324	/* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
325	/* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
326	/* 45 UNDEFINED */ hmR0VmxExitErrUndefined,
327	/* 46 VMX_EXIT_XDTR_ACCESS */ hmR0VmxExitXdtrAccess,
328	/* 47 VMX_EXIT_TR_ACCESS */ hmR0VmxExitXdtrAccess,
329	/* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
330	/* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
331	/* 50 VMX_EXIT_INVEPT */ hmR0VmxExitInjectXcptUD,
332	/* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
333	/* 52 VMX_EXIT_PREEMPTION_TIMER */ hmR0VmxExitPreemptionTimer,
334	/* 53 VMX_EXIT_INVVPID */ hmR0VmxExitInjectXcptUD,
335	/* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
336	/* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
337	/* 56 UNDEFINED */ hmR0VmxExitErrUndefined,
338	/* 57 VMX_EXIT_RDRAND */ hmR0VmxExitRdrand,
339	/* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
340	/* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitInjectXcptUD
341	};
342
343	static const char* const s_apszVmxInstrErrors[VMX_INSTR_ERROR_MAX + 1] =
344	{
345	/* 0 */ "(Not Used)",
346	/* 1 */ "VMCALL executed in VMX root operation.",
347	/* 2 */ "VMCLEAR with invalid physical address.",
348	/* 3 */ "VMCLEAR with VMXON pointer.",
349	/* 4 */ "VMLAUNCH with non-clear VMCS.",
350	/* 5 */ "VMRESUME with non-launched VMCS.",
351	/* 6 */ "VMRESUME after VMXOFF",
352	/* 7 */ "VM entry with invalid control fields.",
353	/* 8 */ "VM entry with invalid host state fields.",
354	/* 9 */ "VMPTRLD with invalid physical address.",
355	/* 10 */ "VMPTRLD with VMXON pointer.",
356	/* 11 */ "VMPTRLD with incorrect revision identifier.",
357	/* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
358	/* 13 */ "VMWRITE to read-only VMCS component.",
359	/* 14 */ "(Not Used)",
360	/* 15 */ "VMXON executed in VMX root operation.",
361	/* 16 */ "VM entry with invalid executive-VMCS pointer.",
362	/* 17 */ "VM entry with non-launched executing VMCS.",
363	/* 18 */ "VM entry with executive-VMCS pointer not VMXON pointer.",
364	/* 19 */ "VMCALL with non-clear VMCS.",
365	/* 20 */ "VMCALL with invalid VM-exit control fields.",
366	/* 21 */ "(Not Used)",
367	/* 22 */ "VMCALL with incorrect MSEG revision identifier.",
368	/* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
369	/* 24 */ "VMCALL with invalid SMM-monitor features.",
370	/* 25 */ "VM entry with invalid VM-execution control fields in executive VMCS.",
371	/* 26 */ "VM entry with events blocked by MOV SS.",
372	/* 27 */ "(Not Used)",
373	/* 28 */ "Invalid operand to INVEPT/INVVPID."
374	};
375
376
377	/**
378	* Disables longjmps to ring-3.
379	* @param pVCpu Pointer to the VMCPU.
380	*/
381	DECLINLINE(void) hmR0VmxCallRing3Disable(PVMCPU pVCpu)
382	{
383	while (VMMRZCallRing3IsEnabled(pVCpu))
384	VMMRZCallRing3Disable(pVCpu);
385	Assert(VMMR0IsLogFlushDisabled(pVCpu));
386	}
387
388
389	/**
390	* Enables longjmps to ring-3.
391	* @param pVCpu Pointer to the VMCPU.
392	*/
393	DECLINLINE(void) hmR0VmxCallRing3Enable(PVMCPU pVCpu)
394	{
395	while (!VMMRZCallRing3IsEnabled(pVCpu))
396	VMMRZCallRing3Enable(pVCpu);
397	Assert(!VMMR0IsLogFlushDisabled(pVCpu));
398	}
399
400
401	/**
402	* Updates the VM's last error record. If there was a VMX instruction error,
403	* reads the error data from the VMCS and updates VCPU's last error record as
404	* well.
405	*
406	* @param pVM Pointer to the VM.
407	* @param pVCpu Pointer to the VMCPU (can be NULL if @a rc is not
408	* VERR_VMX_GENERIC).
409	* @param rc The error code.
410	*/
411	static void hmR0VmxUpdateErrorRecord(PVM pVM, PVMCPU pVCpu, int rc)
412	{
413	AssertPtr(pVM);
414	if (rc == VERR_VMX_GENERIC)
415	{
416	AssertPtrReturnVoid(pVCpu);
417	VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.lasterror.u32InstrError);
418	}
419	pVM->hm.s.lLastError = rc;
420	}
421
422
423	/**
424	* Reads the VM-entry interruption-information field from the VMCS into the VMX
425	* transient structure.
426	*
427	* @returns VBox status code.
428	* @param pVmxTransient Pointer to the VMX transient structure.
429	*
430	* @remarks No-long-jump zone!!!
431	*/
432	DECLINLINE(int) hmR0VmxReadEntryIntrInfoVmcs(PVMXTRANSIENT pVmxTransient)
433	{
434	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntrInfo);
435	AssertRCReturn(rc, rc);
436	return VINF_SUCCESS;
437	}
438
439
440	/**
441	* Reads the VM-entry exception error code field from the VMCS into
442	* the VMX transient structure.
443	*
444	* @returns VBox status code.
445	* @param pVmxTransient Pointer to the VMX transient structure.
446	*
447	* @remarks No-long-jump zone!!!
448	*/
449	DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
450	{
451	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
452	AssertRCReturn(rc, rc);
453	return VINF_SUCCESS;
454	}
455
456
457	/**
458	* Reads the VM-entry exception error code field from the VMCS into
459	* the VMX transient structure.
460	*
461	* @returns VBox status code.
462	* @param pVmxTransient Pointer to the VMX transient structure.
463	*
464	* @remarks No-long-jump zone!!!
465	*/
466	DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
467	{
468	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
469	AssertRCReturn(rc, rc);
470	return VINF_SUCCESS;
471	}
472
473
474	/**
475	* Reads the VM-exit interruption-information field from the VMCS into the VMX
476	* transient structure.
477	*
478	* @returns VBox status code.
479	* @param pVmxTransient Pointer to the VMX transient structure.
480	*/
481	DECLINLINE(int) hmR0VmxReadExitIntrInfoVmcs(PVMXTRANSIENT pVmxTransient)
482	{
483	if (!(pVmxTransient->fVmcsFieldsRead & VMX_TRANSIENT_EXIT_INTERRUPTION_INFO))
484	{
485	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntrInfo);
486	AssertRCReturn(rc, rc);
487	pVmxTransient->fVmcsFieldsRead \|= VMX_TRANSIENT_EXIT_INTERRUPTION_INFO;
488	}
489	return VINF_SUCCESS;
490	}
491
492
493	/**
494	* Reads the VM-exit interruption error code from the VMCS into the VMX
495	* transient structure.
496	*
497	* @returns VBox status code.
498	* @param pVmxTransient Pointer to the VMX transient structure.
499	*/
500	DECLINLINE(int) hmR0VmxReadExitIntrErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
501	{
502	if (!(pVmxTransient->fVmcsFieldsRead & VMX_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE))
503	{
504	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntrErrorCode);
505	AssertRCReturn(rc, rc);
506	pVmxTransient->fVmcsFieldsRead \|= VMX_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE;
507	}
508	return VINF_SUCCESS;
509	}
510
511
512	/**
513	* Reads the VM-exit instruction length field from the VMCS into the VMX
514	* transient structure.
515	*
516	* @returns VBox status code.
517	* @param pVmxTransient Pointer to the VMX transient structure.
518	*/
519	DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
520	{
521	if (!(pVmxTransient->fVmcsFieldsRead & VMX_TRANSIENT_EXIT_INSTR_LEN))
522	{
523	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
524	AssertRCReturn(rc, rc);
525	pVmxTransient->fVmcsFieldsRead \|= VMX_TRANSIENT_EXIT_INSTR_LEN;
526	}
527	return VINF_SUCCESS;
528	}
529
530
531	/**
532	* Reads the exit qualification from the VMCS into the VMX transient structure.
533	*
534	* @returns VBox status code.
535	* @param pVmxTransient Pointer to the VMX transient structure.
536	*/
537	DECLINLINE(int) hmR0VmxReadExitQualificationVmcs(PVMXTRANSIENT pVmxTransient)
538	{
539	if (!(pVmxTransient->fVmcsFieldsRead & VMX_TRANSIENT_EXIT_QUALIFICATION))
540	{
541	int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQualification);
542	AssertRCReturn(rc, rc);
543	pVmxTransient->fVmcsFieldsRead \|= VMX_TRANSIENT_EXIT_QUALIFICATION;
544	}
545	return VINF_SUCCESS;
546	}
547
548
549	/**
550	* Reads the IDT-vectoring information field from the VMCS into the VMX
551	* transient structure.
552	*
553	* @returns VBox status code.
554	* @param pVmxTransient Pointer to the VMX transient structure.
555	*
556	* @remarks No-long-jump zone!!!
557	*/
558	DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
559	{
560	if (!(pVmxTransient->fVmcsFieldsRead & VMX_TRANSIENT_IDT_VECTORING_INFO))
561	{
562	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_INFO, &pVmxTransient->uIdtVectoringInfo);
563	AssertRCReturn(rc, rc);
564	pVmxTransient->fVmcsFieldsRead \|= VMX_TRANSIENT_IDT_VECTORING_INFO;
565	}
566	return VINF_SUCCESS;
567	}
568
569
570	/**
571	* Reads the IDT-vectoring error code from the VMCS into the VMX
572	* transient structure.
573	*
574	* @returns VBox status code.
575	* @param pVmxTransient Pointer to the VMX transient structure.
576	*/
577	DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
578	{
579	if (!(pVmxTransient->fVmcsFieldsRead & VMX_TRANSIENT_IDT_VECTORING_ERROR_CODE))
580	{
581	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
582	AssertRCReturn(rc, rc);
583	pVmxTransient->fVmcsFieldsRead \|= VMX_TRANSIENT_IDT_VECTORING_ERROR_CODE;
584	}
585	return VINF_SUCCESS;
586	}
587
588
589	/**
590	* Enters VMX root mode operation on the current CPU.
591	*
592	* @returns VBox status code.
593	* @param pVM Pointer to the VM (optional, can be NULL, after
594	* a resume).
595	* @param HCPhysCpuPage Physical address of the VMXON region.
596	* @param pvCpuPage Pointer to the VMXON region.
597	*/
598	DECLINLINE(int) hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
599	{
600	AssertReturn(HCPhysCpuPage != 0 && HCPhysCpuPage != NIL_RTHCPHYS, VERR_INVALID_PARAMETER);
601	AssertReturn(pvCpuPage, VERR_INVALID_PARAMETER);
602
603	if (pVM)
604	{
605	/* Write the VMCS revision dword to the VMXON region. */
606	(uint32_t )pvCpuPage = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.msr.vmx_basic_info);
607	}
608
609	/* Disable interrupts. Interrupts handlers might, in theory, change CR4. */
610	RTCCUINTREG fFlags = ASMIntDisableFlags();
611
612	/* Enable the VMX bit in CR4. */
613	RTCCUINTREG uCr4 = ASMGetCR4();
614	if (!(uCr4 & X86_CR4_VMXE))
615	ASMSetCR4(uCr4 \| X86_CR4_VMXE);
616
617	/* Enter VMXON root mode. */
618	int rc = VMXEnable(HCPhysCpuPage);
619
620	/* Restore interrupts. */
621	ASMSetFlags(fFlags);
622	return rc;
623	}
624
625
626	/**
627	* Exits VMX root mode operation on the current CPU.
628	*
629	* @returns VBox status code.
630	*/
631	static int hmR0VmxLeaveRootMode(void)
632	{
633	/* Disable interrupts. Interrupt handlers might, in theory, change CR4. */
634	RTCCUINTREG fFlags = ASMIntDisableFlags();
635	int rc = VINF_SUCCESS;
636
637	/* If we're for some reason not in VMX root mode, then don't leave it. */
638	if (ASMGetCR4() & X86_CR4_VMXE)
639	{
640	/* Exit root mode using VMXOFF & clear the VMX bit in CR4 */
641	VMXDisable();
642	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
643	}
644	else
645	rc = VERR_VMX_NOT_IN_VMX_ROOT_MODE;
646
647	/* Restore interrupts. */
648	ASMSetFlags(fFlags);
649	return rc;
650	}
651
652
653	/**
654	* Allocates and maps one physically contiguous page. The allocated page is
655	* zero'd out. (Used by various VT-x structures).
656	*
657	* @returns IPRT status code.
658	* @param pMemObj Pointer to the ring-0 memory object.
659	* @param ppVirt Where to store the virtual address of the
660	* allocation.
661	* @param pPhys Where to store the physical address of the
662	* allocation.
663	*/
664	DECLINLINE(int) hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
665	{
666	AssertPtrReturn(pMemObj, VERR_INVALID_PARAMETER);
667	AssertPtrReturn(ppVirt, VERR_INVALID_PARAMETER);
668	AssertPtrReturn(pHCPhys, VERR_INVALID_PARAMETER);
669
670	int rc = RTR0MemObjAllocCont(pMemObj, PAGE_SIZE, false /* fExecutable */);
671	if (RT_FAILURE(rc))
672	return rc;
673	ppVirt = RTR0MemObjAddress(pMemObj);
674	pHCPhys = RTR0MemObjGetPagePhysAddr(pMemObj, 0 /* iPage */);
675	ASMMemZero32(*ppVirt, PAGE_SIZE);
676	return VINF_SUCCESS;
677	}
678
679
680	/**
681	* Frees and unmaps an allocated physical page.
682	*
683	* @param pMemObj Pointer to the ring-0 memory object.
684	* @param ppVirt Where to re-initialize the virtual address of
685	* allocation as 0.
686	* @param pHCPhys Where to re-initialize the physical address of the
687	* allocation as 0.
688	*/
689	DECLINLINE(void) hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
690	{
691	AssertPtr(pMemObj);
692	AssertPtr(ppVirt);
693	AssertPtr(pHCPhys);
694	if (*pMemObj != NIL_RTR0MEMOBJ)
695	{
696	int rc = RTR0MemObjFree(pMemObj, true / fFreeMappings */);
697	AssertRC(rc);
698	*pMemObj = NIL_RTR0MEMOBJ;
699	*ppVirt = 0;
700	*pHCPhys = 0;
701	}
702	}
703
704
705	/**
706	* Worker function to free VT-x related structures.
707	*
708	* @returns IPRT status code.
709	* @param pVM Pointer to the VM.
710	*/
711	static void hmR0VmxStructsFree(PVM pVM)
712	{
713	for (VMCPUID i = 0; i < pVM->cCpus; i++)
714	{
715	PVMCPU pVCpu = &pVM->aCpus[i];
716	AssertPtr(pVCpu);
717
718	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
719	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
720
721	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
722	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap, &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
723
724	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic, &pVCpu->hm.s.vmx.HCPhysVirtApic);
725	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
726	}
727
728	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
729	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
730	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
731	#endif
732	}
733
734
735	/**
736	* Worker function to allocate VT-x related VM structures.
737	*
738	* @returns IPRT status code.
739	* @param pVM Pointer to the VM.
740	*/
741	static int hmR0VmxStructsAlloc(PVM pVM)
742	{
743	/*
744	* Initialize members up-front so we can cleanup properly on allocation failure.
745	*/
746	#define VMXLOCAL_INIT_VM_MEMOBJ(a_Name, a_VirtPrefix) \
747	pVM->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
748	pVM->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
749	pVM->hm.s.vmx.HCPhys##a_Name = 0;
750
751	#define VMXLOCAL_INIT_VMCPU_MEMOBJ(a_Name, a_VirtPrefix) \
752	pVCpu->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
753	pVCpu->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
754	pVCpu->hm.s.vmx.HCPhys##a_Name = 0;
755
756	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
757	VMXLOCAL_INIT_VM_MEMOBJ(Scratch, pv);
758	#endif
759	VMXLOCAL_INIT_VM_MEMOBJ(ApicAccess, pb);
760
761	AssertCompile(sizeof(VMCPUID) == sizeof(pVM->cCpus));
762	for (VMCPUID i = 0; i < pVM->cCpus; i++)
763	{
764	PVMCPU pVCpu = &pVM->aCpus[i];
765	VMXLOCAL_INIT_VMCPU_MEMOBJ(Vmcs, pv);
766	VMXLOCAL_INIT_VMCPU_MEMOBJ(VirtApic, pb);
767	VMXLOCAL_INIT_VMCPU_MEMOBJ(MsrBitmap, pv);
768	VMXLOCAL_INIT_VMCPU_MEMOBJ(GuestMsr, pv);
769	VMXLOCAL_INIT_VMCPU_MEMOBJ(HostMsr, pv);
770	}
771	#undef VMXLOCAL_INIT_VMCPU_MEMOBJ
772	#undef VMXLOCAL_INIT_VM_MEMOBJ
773
774	/*
775	* Allocate all the VT-x structures.
776	*/
777	int rc = VINF_SUCCESS;
778	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
779	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
780	if (RT_FAILURE(rc))
781	goto cleanup;
782	strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
783	(uint64_t )(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xDEADBEEFDEADBEEF);
784	#endif
785
786	/* Allocate the APIC-access page for trapping APIC accesses from the guest. */
787	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
788	{
789	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
790	&pVM->hm.s.vmx.HCPhysApicAccess);
791	if (RT_FAILURE(rc))
792	goto cleanup;
793	}
794
795	/*
796	* Initialize per-VCPU VT-x structures.
797	*/
798	for (VMCPUID i =0; i < pVM->cCpus; i++)
799	{
800	PVMCPU pVCpu = &pVM->aCpus[i];
801	AssertPtr(pVCpu);
802
803	/* Allocate the VM control structure (VMCS). */
804	AssertReturn(MSR_IA32_VMX_BASIC_INFO_VMCS_SIZE(pVM->hm.s.vmx.msr.vmx_basic_info) <= PAGE_SIZE, VERR_INTERNAL_ERROR);
805	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
806	if (RT_FAILURE(rc))
807	goto cleanup;
808
809	/* Allocate the Virtual-APIC page for transparent TPR accesses. */
810	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
811	{
812	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic,
813	&pVCpu->hm.s.vmx.HCPhysVirtApic);
814	if (RT_FAILURE(rc))
815	goto cleanup;
816	}
817
818	/* Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for transparent accesses of specific MSRs. */
819	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
820	{
821	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap,
822	&pVCpu->hm.s.vmx.HCPhysMsrBitmap);
823	if (RT_FAILURE(rc))
824	goto cleanup;
825	memset(pVCpu->hm.s.vmx.pvMsrBitmap, 0xff, PAGE_SIZE);
826	}
827
828	/* Allocate the VM-entry MSR-load and VM-exit MSR-store page for the guest MSRs. */
829	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
830	if (RT_FAILURE(rc))
831	goto cleanup;
832
833	/* Allocate the VM-exit MSR-load page for the host MSRs. */
834	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
835	if (RT_FAILURE(rc))
836	goto cleanup;
837	}
838
839	return VINF_SUCCESS;
840
841	cleanup:
842	hmR0VmxStructsFree(pVM);
843	return rc;
844	}
845
846
847	/**
848	* Does global VT-x initialization (called during module initialization).
849	*
850	* @returns VBox status code.
851	*/
852	VMMR0DECL(int) VMXR0GlobalInit(void)
853	{
854	/* Setup the main VM exit handlers. */
855	/** @todo I'm told lookup table for function pointers is a bad idea in terms
856	* of performance. Make these static for now and not do change anything
857	* at runtime so we can easily switch to a switch-case approach later. */
858	AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(s_apfnVMExitHandlers));
859	#ifdef DEBUG
860	for (unsigned i = 0; i < RT_ELEMENTS(s_apfnVMExitHandlers); i++)
861	Assert(s_apfnVMExitHandlers[i]);
862	#endif
863	return VINF_SUCCESS;
864	}
865
866
867	/**
868	* Does global VT-x termination (called during module termination).
869	*/
870	VMMR0DECL(void) VMXR0GlobalTerm()
871	{
872	/* Nothing to do currently. */
873	}
874
875
876	/**
877	* Sets up and activates VT-x on the current CPU.
878	*
879	* @returns VBox status code.
880	* @param pCpu Pointer to the global CPU info struct.
881	* @param pVM Pointer to the VM (can be NULL after a host resume
882	* operation).
883	* @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
884	* fEnabledByHost is true).
885	* @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
886	* @a fEnabledByHost is true).
887	* @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
888	* enable VT-x/AMD-V on the host.
889	*/
890	VMMR0DECL(int) VMXR0EnableCpu(PHMGLOBLCPUINFO pCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost)
891	{
892	AssertReturn(pCpu, VERR_INVALID_PARAMETER);
893	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
894
895	if (!fEnabledByHost)
896	{
897	int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
898	if (RT_FAILURE(rc))
899	return rc;
900	}
901
902	/*
903	* Flush all VPIDs (in case we or any other hypervisor have been using VPIDs) so that
904	* we can avoid an explicit flush while using new VPIDs. We would still need to flush
905	* each time while reusing a VPID after hitting the MaxASID limit once.
906	*/
907	if ( pVM
908	&& pVM->hm.s.vmx.fVpid
909	&& (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS))
910	{
911	hmR0VmxFlushVpid(pVM, NULL /* pvCpu /, VMX_FLUSH_VPID_ALL_CONTEXTS, 0 / GCPtr */);
912	pCpu->fFlushAsidBeforeUse = false;
913	}
914	else
915	pCpu->fFlushAsidBeforeUse = true;
916
917	/* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
918	++pCpu->cTlbFlushes;
919
920	return VINF_SUCCESS;
921	}
922
923
924	/**
925	* Deactivates VT-x on the current CPU.
926	*
927	* @returns VBox status code.
928	* @param pCpu Pointer to the global CPU info struct.
929	* @param pvCpuPage Pointer to the VMXON region.
930	* @param HCPhysCpuPage Physical address of the VMXON region.
931	*/
932	VMMR0DECL(int) VMXR0DisableCpu(PHMGLOBLCPUINFO pCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
933	{
934	NOREF(pCpu);
935	NOREF(pvCpuPage);
936	NOREF(HCPhysCpuPage);
937
938	hmR0VmxLeaveRootMode();
939	return VINF_SUCCESS;
940	}
941
942
943	/**
944	* Sets the permission bits for the specified MSR in the MSR bitmap.
945	*
946	* @param pVCpu Pointer to the VMCPU.
947	* @param ulMSR The MSR value.
948	* @param enmRead Whether reading this MSR causes a VM-exit.
949	* @param enmWrite Whether writing this MSR causes a VM-exit.
950	*/
951	static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, unsigned ulMsr, VMXMSREXITREAD enmRead, VMXMSREXITWRITE enmWrite)
952	{
953	unsigned ulBit;
954	uint8_t pbMsrBitmap = (uint8_t )pVCpu->hm.s.vmx.pvMsrBitmap;
955
956	/*
957	* Layout:
958	* 0x000 - 0x3ff - Low MSR read bits
959	* 0x400 - 0x7ff - High MSR read bits
960	* 0x800 - 0xbff - Low MSR write bits
961	* 0xc00 - 0xfff - High MSR write bits
962	*/
963	if (ulMsr <= 0x00001FFF)
964	{
965	/* Pentium-compatible MSRs */
966	ulBit = ulMsr;
967	}
968	else if ( ulMsr >= 0xC0000000
969	&& ulMsr <= 0xC0001FFF)
970	{
971	/* AMD Sixth Generation x86 Processor MSRs */
972	ulBit = (ulMsr - 0xC0000000);
973	pbMsrBitmap += 0x400;
974	}
975	else
976	{
977	AssertMsgFailed(("Invalid MSR %lx\n", ulMsr));
978	return;
979	}
980
981	Assert(ulBit <= 0x1fff);
982	if (enmRead == VMXMSREXIT_INTERCEPT_READ)
983	ASMBitSet(pbMsrBitmap, ulBit);
984	else
985	ASMBitClear(pbMsrBitmap, ulBit);
986
987	if (enmWrite == VMXMSREXIT_INTERCEPT_WRITE)
988	ASMBitSet(pbMsrBitmap + 0x800, ulBit);
989	else
990	ASMBitClear(pbMsrBitmap + 0x800, ulBit);
991	}
992
993
994	/**
995	* Flushes the TLB using EPT.
996	*
997	* @returns VBox status code.
998	* @param pVM Pointer to the VM.
999	* @param pVCpu Pointer to the VMCPU.
1000	* @param enmFlush Type of flush.
1001	*/
1002	static void hmR0VmxFlushEpt(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_EPT enmFlush)
1003	{
1004	AssertPtr(pVM);
1005	Assert(pVM->hm.s.fNestedPaging);
1006
1007	LogFlowFunc(("pVM=%p pVCpu=%p enmFlush=%d\n", pVM, pVCpu, enmFlush));
1008
1009	uint64_t descriptor[2];
1010	descriptor[0] = pVCpu->hm.s.vmx.GCPhysEPTP;
1011	descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
1012
1013	int rc = VMXR0InvEPT(enmFlush, &descriptor[0]);
1014	AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %#x %RGv failed with %Rrc\n", enmFlush, pVCpu->hm.s.vmx.GCPhysEPTP, rc));
1015	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
1016	}
1017
1018
1019	/**
1020	* Flushes the TLB using VPID.
1021	*
1022	* @returns VBox status code.
1023	* @param pVM Pointer to the VM.
1024	* @param pVCpu Pointer to the VMCPU (can be NULL depending on @a
1025	* enmFlush).
1026	* @param enmFlush Type of flush.
1027	* @param GCPtr Virtual address of the page to flush (can be 0 depending
1028	* on @a enmFlush).
1029	*/
1030	static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_VPID enmFlush, RTGCPTR GCPtr)
1031	{
1032	AssertPtr(pVM);
1033	Assert(pVM->hm.s.vmx.fVpid);
1034
1035	uint64_t descriptor[2];
1036	if (enmFlush == VMX_FLUSH_VPID_ALL_CONTEXTS)
1037	{
1038	descriptor[0] = 0;
1039	descriptor[1] = 0;
1040	}
1041	else
1042	{
1043	AssertPtr(pVCpu);
1044	AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1045	AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1046	descriptor[0] = pVCpu->hm.s.uCurrentAsid;
1047	descriptor[1] = GCPtr;
1048	}
1049
1050	int rc = VMXR0InvVPID(enmFlush, &descriptor[0]); NOREF(rc);
1051	AssertMsg(rc == VINF_SUCCESS,
1052	("VMXR0InvVPID %#x %u %RGv failed with %d\n", enmFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
1053	if ( RT_SUCCESS(rc)
1054	&& pVCpu)
1055	{
1056	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
1057	}
1058	}
1059
1060
1061	/**
1062	* Invalidates a guest page by guest virtual address. Only relevant for
1063	* EPT/VPID, otherwise there is nothing really to invalidate.
1064	*
1065	* @returns VBox status code.
1066	* @param pVM Pointer to the VM.
1067	* @param pVCpu Pointer to the VMCPU.
1068	* @param GCVirt Guest virtual address of the page to invalidate.
1069	*/
1070	VMMR0DECL(int) VMXR0InvalidatePage(PVM pVM, PVMCPU pVCpu, RTGCPTR GCVirt)
1071	{
1072	AssertPtr(pVM);
1073	AssertPtr(pVCpu);
1074	LogFlowFunc(("pVM=%p pVCpu=%p GCVirt=%RGv\n", pVM, pVCpu, GCVirt));
1075
1076	bool fFlushPending = VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1077	if (!fFlushPending)
1078	{
1079	/*
1080	* We must invalidate the guest TLB entry in either case, we cannot ignore it even for the EPT case
1081	* See @bugref{6043} and @bugref{6177}.
1082	*
1083	* Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*() as this
1084	* function maybe called in a loop with individual addresses.
1085	*/
1086	if (pVM->hm.s.vmx.fVpid)
1087	{
1088	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1089	{
1090	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, GCVirt);
1091	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
1092	}
1093	else
1094	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1095	}
1096	else if (pVM->hm.s.fNestedPaging)
1097	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1098	}
1099
1100	return VINF_SUCCESS;
1101	}
1102
1103
1104	/**
1105	* Invalidates a guest page by physical address. Only relevant for EPT/VPID,
1106	* otherwise there is nothing really to invalidate.
1107	*
1108	* @returns VBox status code.
1109	* @param pVM Pointer to the VM.
1110	* @param pVCpu Pointer to the VMCPU.
1111	* @param GCPhys Guest physical address of the page to invalidate.
1112	*/
1113	VMMR0DECL(int) VMXR0InvalidatePhysPage(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys)
1114	{
1115	LogFlowFunc(("%RGp\n", GCPhys));
1116
1117	/*
1118	* We cannot flush a page by guest-physical address. invvpid takes only a linear address while invept only flushes
1119	* by EPT not individual addresses. We update the force flag here and flush before the next VM-entry in hmR0VmxFlushTLB*().
1120	* This function might be called in a loop.
1121	*/
1122	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1123	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgPhys);
1124	return VINF_SUCCESS;
1125	}
1126
1127
1128	/**
1129	* Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
1130	* case where neither EPT nor VPID is supported by the CPU.
1131	*
1132	* @param pVM Pointer to the VM.
1133	* @param pVCpu Pointer to the VMCPU.
1134	*
1135	* @remarks Called with interrupts disabled.
1136	*/
1137	static DECLCALLBACK(void) hmR0VmxFlushTaggedTlbNone(PVM pVM, PVMCPU pVCpu)
1138	{
1139	NOREF(pVM);
1140	AssertPtr(pVCpu);
1141	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
1142	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1143
1144	PHMGLOBLCPUINFO pCpu = HMR0GetCurrentCpu();
1145	AssertPtr(pCpu);
1146
1147	pVCpu->hm.s.TlbShootdown.cPages = 0;
1148	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1149	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1150	pVCpu->hm.s.fForceTLBFlush = false;
1151	return;
1152	}
1153
1154
1155	/**
1156	* Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
1157	*
1158	* @param pVM Pointer to the VM.
1159	* @param pVCpu Pointer to the VMCPU.
1160	* @remarks All references to "ASID" in this function pertains to "VPID" in
1161	* Intel's nomenclature. The reason is, to avoid confusion in compare
1162	* statements since the host-CPU copies are named "ASID".
1163	*
1164	* @remarks Called with interrupts disabled.
1165	* @todo Statistics.
1166	*/
1167	static DECLCALLBACK(void) hmR0VmxFlushTaggedTlbBoth(PVM pVM, PVMCPU pVCpu)
1168	{
1169	AssertPtr(pVM);
1170	AssertPtr(pVCpu);
1171	AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
1172	("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
1173	"fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
1174
1175	PHMGLOBLCPUINFO pCpu = HMR0GetCurrentCpu();
1176	AssertPtr(pCpu);
1177
1178	/*
1179	* Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1180	* This can happen both for start & resume due to long jumps back to ring-3.
1181	* If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1182	* or the host Cpu is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1183	*/
1184	bool fNewASID = false;
1185	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1186	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1187	{
1188	pVCpu->hm.s.fForceTLBFlush = true;
1189	fNewASID = true;
1190	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1191	}
1192
1193	/*
1194	* Check for explicit TLB shootdowns.
1195	*/
1196	if (VMCPU_FF_TESTANDCLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1197	{
1198	pVCpu->hm.s.fForceTLBFlush = true;
1199	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1200	}
1201
1202	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1203	if (pVCpu->hm.s.fForceTLBFlush)
1204	{
1205	if (fNewASID)
1206	{
1207	++pCpu->uCurrentAsid;
1208	if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1209	{
1210	pCpu->uCurrentAsid = 1; /* start at 1; host uses 0 */
1211	pCpu->cTlbFlushes++;
1212	pCpu->fFlushAsidBeforeUse = true;
1213	}
1214
1215	pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1216	if (pCpu->fFlushAsidBeforeUse)
1217	hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
1218	}
1219	else
1220	{
1221	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
1222	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_SINGLE_CONTEXT, 0 /* GCPtr */);
1223	else
1224	hmR0VmxFlushEpt(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1225	}
1226
1227	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1228	pVCpu->hm.s.fForceTLBFlush = false;
1229	}
1230	else
1231	{
1232	AssertMsg(pVCpu->hm.s.uCurrentAsid && pCpu->uCurrentAsid,
1233	("hm->uCurrentAsid=%lu hm->cTlbFlushes=%lu cpu->uCurrentAsid=%lu cpu->cTlbFlushes=%lu\n",
1234	pVCpu->hm.s.uCurrentAsid, pVCpu->hm.s.cTlbFlushes,
1235	pCpu->uCurrentAsid, pCpu->cTlbFlushes));
1236
1237	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
1238	* not be executed. See hmQueueInvlPage() where it is commented
1239	* out. Support individual entry flushing someday. */
1240	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1241	{
1242	STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
1243
1244	/*
1245	* Flush individual guest entries using VPID from the TLB or as little as possible with EPT
1246	* as supported by the CPU.
1247	*/
1248	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1249	{
1250	for (unsigned i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
1251	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
1252	}
1253	else
1254	hmR0VmxFlushEpt(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1255	}
1256	else
1257	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
1258	}
1259	pVCpu->hm.s.TlbShootdown.cPages = 0;
1260	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1261
1262	AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
1263	("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
1264	AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
1265	("cpu%d uCurrentAsid = %u\n", pCpu->idCpu, pCpu->uCurrentAsid));
1266	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
1267	("cpu%d VM uCurrentAsid = %u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
1268
1269	/* Update VMCS with the VPID. */
1270	int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
1271	AssertRC(rc);
1272	}
1273
1274
1275	/**
1276	* Flushes the tagged-TLB entries for EPT CPUs as necessary.
1277	*
1278	* @returns VBox status code.
1279	* @param pVM Pointer to the VM.
1280	* @param pVCpu Pointer to the VMCPU.
1281	*
1282	* @remarks Called with interrupts disabled.
1283	* @todo Statistics.
1284	*/
1285	static DECLCALLBACK(void) hmR0VmxFlushTaggedTlbEpt(PVM pVM, PVMCPU pVCpu)
1286	{
1287	AssertPtr(pVM);
1288	AssertPtr(pVCpu);
1289	AssertMsg(pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with NestedPaging disabled."));
1290	AssertMsg(!pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID enabled."));
1291
1292	PHMGLOBLCPUINFO pCpu = HMR0GetCurrentCpu();
1293	AssertPtr(pCpu);
1294
1295	/*
1296	* Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1297	* This can happen both for start & resume due to long jumps back to ring-3.
1298	* A change in the TLB flush count implies the host CPU is online after a suspend/resume.
1299	*/
1300	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1301	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1302	{
1303	pVCpu->hm.s.fForceTLBFlush = true;
1304	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1305	}
1306
1307	/* Check for explicit TLB shootdown flushes. */
1308	if (VMCPU_FF_TESTANDCLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1309	{
1310	pVCpu->hm.s.fForceTLBFlush = true;
1311	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1312	}
1313
1314	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1315	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1316
1317	if (pVCpu->hm.s.fForceTLBFlush)
1318	{
1319	hmR0VmxFlushEpt(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1320	pVCpu->hm.s.fForceTLBFlush = false;
1321	}
1322	else
1323	{
1324	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
1325	* not be executed. See hmQueueInvlPage() where it is commented
1326	* out. Support individual entry flushing someday. */
1327	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1328	{
1329	/* We cannot flush individual entries without VPID support. Flush using EPT. */
1330	STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
1331	hmR0VmxFlushEpt(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1332	}
1333	else
1334	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
1335	}
1336
1337	pVCpu->hm.s.TlbShootdown.cPages= 0;
1338	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1339	}
1340
1341
1342	/**
1343	* Flushes the tagged-TLB entries for VPID CPUs as necessary.
1344	*
1345	* @returns VBox status code.
1346	* @param pVM Pointer to the VM.
1347	* @param pVCpu Pointer to the VMCPU.
1348	*
1349	* @remarks Called with interrupts disabled.
1350	* @todo Statistics.
1351	*/
1352	static DECLCALLBACK(void) hmR0VmxFlushTaggedTlbVpid(PVM pVM, PVMCPU pVCpu)
1353	{
1354	AssertPtr(pVM);
1355	AssertPtr(pVCpu);
1356	AssertMsg(pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked with VPID disabled."));
1357	AssertMsg(!pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging enabled"));
1358
1359	PHMGLOBLCPUINFO pCpu = HMR0GetCurrentCpu();
1360
1361	/*
1362	* Force a TLB flush for the first world switch if the current CPU differs from the one we ran on last.
1363	* This can happen both for start & resume due to long jumps back to ring-3.
1364	* If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1365	* or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1366	*/
1367	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1368	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1369	{
1370	pVCpu->hm.s.fForceTLBFlush = true;
1371	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1372	}
1373
1374	/* Check for explicit TLB shootdown flushes. */
1375	if (VMCPU_FF_TESTANDCLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1376	{
1377	/*
1378	* If we ever support VPID flush combinations other than ALL or SINGLE-context (see hmR0VmxSetupTaggedTlb())
1379	* we would need to explicitly flush in this case (add an fExplicitFlush = true here and change the
1380	* pCpu->fFlushAsidBeforeUse check below to include fExplicitFlush's too) - an obscure corner case.
1381	*/
1382	pVCpu->hm.s.fForceTLBFlush = true;
1383	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1384	}
1385
1386	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1387	if (pVCpu->hm.s.fForceTLBFlush)
1388	{
1389	++pCpu->uCurrentAsid;
1390	if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1391	{
1392	pCpu->uCurrentAsid = 1; /* start at 1; host uses 0 */
1393	pCpu->fFlushAsidBeforeUse = true;
1394	pCpu->cTlbFlushes++;
1395	}
1396
1397	pVCpu->hm.s.fForceTLBFlush = false;
1398	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1399	pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1400	if (pCpu->fFlushAsidBeforeUse)
1401	hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
1402	}
1403	else
1404	{
1405	AssertMsg(pVCpu->hm.s.uCurrentAsid && pCpu->uCurrentAsid,
1406	("hm->uCurrentAsid=%lu hm->cTlbFlushes=%lu cpu->uCurrentAsid=%lu cpu->cTlbFlushes=%lu\n",
1407	pVCpu->hm.s.uCurrentAsid, pVCpu->hm.s.cTlbFlushes,
1408	pCpu->uCurrentAsid, pCpu->cTlbFlushes));
1409
1410	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
1411	* not be executed. See hmQueueInvlPage() where it is commented
1412	* out. Support individual entry flushing someday. */
1413	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1414	{
1415	/* Flush individual guest entries using VPID or as little as possible with EPT as supported by the CPU. */
1416	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1417	{
1418	for (unsigned i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
1419	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
1420	}
1421	else
1422	hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
1423	}
1424	else
1425	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
1426	}
1427
1428	pVCpu->hm.s.TlbShootdown.cPages = 0;
1429	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1430
1431	AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
1432	("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
1433	AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
1434	("cpu%d uCurrentAsid = %u\n", pCpu->idCpu, pCpu->uCurrentAsid));
1435	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
1436	("cpu%d VM uCurrentAsid = %u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
1437
1438	int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
1439	AssertRC(rc);
1440	}
1441
1442
1443	/**
1444	* Sets up the appropriate tagged TLB-flush level and handler for flushing guest
1445	* TLB entries from the host TLB before VM-entry.
1446	*
1447	* @returns VBox status code.
1448	* @param pVM Pointer to the VM.
1449	*/
1450	static int hmR0VmxSetupTaggedTlb(PVM pVM)
1451	{
1452	/*
1453	* Determine optimal flush type for nested paging.
1454	* We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup unrestricted
1455	* guest execution (see hmR3InitFinalizeR0()).
1456	*/
1457	if (pVM->hm.s.fNestedPaging)
1458	{
1459	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
1460	{
1461	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
1462	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_SINGLE_CONTEXT;
1463	else if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
1464	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_ALL_CONTEXTS;
1465	else
1466	{
1467	/* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
1468	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
1469	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1470	}
1471
1472	/* Make sure the write-back cacheable memory type for EPT is supported. */
1473	if (!(pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB))
1474	{
1475	LogRel(("hmR0VmxSetupTaggedTlb: Unsupported EPTP memory type %#x.\n", pVM->hm.s.vmx.msr.vmx_ept_vpid_caps));
1476	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
1477	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1478	}
1479	}
1480	else
1481	{
1482	/* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
1483	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
1484	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1485	}
1486	}
1487
1488	/*
1489	* Determine optimal flush type for VPID.
1490	*/
1491	if (pVM->hm.s.vmx.fVpid)
1492	{
1493	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
1494	{
1495	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
1496	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_SINGLE_CONTEXT;
1497	else if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
1498	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_ALL_CONTEXTS;
1499	else
1500	{
1501	/* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
1502	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1503	LogRel(("hmR0VmxSetupTaggedTlb: Only INDIV_ADDR supported. Ignoring VPID.\n"));
1504	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
1505	LogRel(("hmR0VmxSetupTaggedTlb: Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
1506	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NOT_SUPPORTED;
1507	pVM->hm.s.vmx.fVpid = false;
1508	}
1509	}
1510	else
1511	{
1512	/* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
1513	Log(("hmR0VmxSetupTaggedTlb: VPID supported without INVEPT support. Ignoring VPID.\n"));
1514	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NOT_SUPPORTED;
1515	pVM->hm.s.vmx.fVpid = false;
1516	}
1517	}
1518
1519	/*
1520	* Setup the handler for flushing tagged-TLBs.
1521	*/
1522	if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
1523	pVM->hm.s.vmx.pfnFlushTaggedTlb = hmR0VmxFlushTaggedTlbBoth;
1524	else if (pVM->hm.s.fNestedPaging)
1525	pVM->hm.s.vmx.pfnFlushTaggedTlb = hmR0VmxFlushTaggedTlbEpt;
1526	else if (pVM->hm.s.vmx.fVpid)
1527	pVM->hm.s.vmx.pfnFlushTaggedTlb = hmR0VmxFlushTaggedTlbVpid;
1528	else
1529	pVM->hm.s.vmx.pfnFlushTaggedTlb = hmR0VmxFlushTaggedTlbNone;
1530	return VINF_SUCCESS;
1531	}
1532
1533
1534	/**
1535	* Sets up pin-based VM-execution controls in the VMCS.
1536	*
1537	* @returns VBox status code.
1538	* @param pVM Pointer to the VM.
1539	* @param pVCpu Pointer to the VMCPU.
1540	*/
1541	static int hmR0VmxSetupPinCtls(PVM pVM, PVMCPU pVCpu)
1542	{
1543	AssertPtr(pVM);
1544	AssertPtr(pVCpu);
1545
1546	uint32_t val = pVM->hm.s.vmx.msr.vmx_pin_ctls.n.disallowed0; /* Bits set here must always be set. */
1547	uint32_t zap = pVM->hm.s.vmx.msr.vmx_pin_ctls.n.allowed1; /* Bits cleared here must always be cleared. */
1548
1549	val \|= VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_EXT_INT_EXIT /* External interrupts causes a VM-exits. */
1550	\| VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_NMI_EXIT; /* Non-maskable interrupts causes a VM-exit. */
1551	Assert(!(val & VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_VIRTUAL_NMI));
1552
1553	/* Enable the VMX preemption timer. */
1554	if (pVM->hm.s.vmx.fUsePreemptTimer)
1555	val \|= VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_PREEMPT_TIMER;
1556
1557	if ((val & zap) != val)
1558	{
1559	LogRel(("hmR0VmxSetupPinCtls: invalid pin-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
1560	pVM->hm.s.vmx.msr.vmx_pin_ctls.n.disallowed0, val, zap));
1561	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1562	}
1563
1564	val &= zap;
1565	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC_CONTROLS, val);
1566	AssertRCReturn(rc, rc);
1567
1568	/* Update VCPU with the currently set pin-based VM-execution controls. */
1569	pVCpu->hm.s.vmx.u32PinCtls = val;
1570	return rc;
1571	}
1572
1573
1574	/**
1575	* Sets up processor-based VM-execution controls in the VMCS.
1576	*
1577	* @returns VBox status code.
1578	* @param pVM Pointer to the VM.
1579	* @param pVMCPU Pointer to the VMCPU.
1580	*/
1581	static int hmR0VmxSetupProcCtls(PVM pVM, PVMCPU pVCpu)
1582	{
1583	AssertPtr(pVM);
1584	AssertPtr(pVCpu);
1585
1586	int rc = VERR_INTERNAL_ERROR_5;
1587	uint32_t val = pVM->hm.s.vmx.msr.vmx_proc_ctls.n.disallowed0; /* Bits set here must be set in the VMCS. */
1588	uint32_t zap = pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
1589
1590	val \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_HLT_EXIT /* HLT causes a VM-exit. */
1591	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
1592	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
1593	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
1594	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDPMC_EXIT /* RDPMC causes a VM-exit. */
1595	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_EXIT /* MONITOR causes a VM-exit. */
1596	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
1597
1598	/* We toggle VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
1599	if ( !(pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT)
1600	\|\| (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.disallowed0 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT))
1601	{
1602	LogRel(("hmR0VmxSetupProcCtls: unsupported VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT combo!"));
1603	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1604	}
1605
1606	/* Without nested paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
1607	if (!pVM->hm.s.fNestedPaging)
1608	{
1609	Assert(!pVM->hm.s.vmx.fUnrestrictedGuest); /* Paranoia. */
1610	val \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INVLPG_EXIT
1611	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
1612	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT;
1613	}
1614
1615	/* Use TPR shadowing if supported by the CPU. */
1616	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
1617	{
1618	Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
1619	Assert(!(pVCpu->hm.s.vmx.HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
1620	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, 0);
1621	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL, pVCpu->hm.s.vmx.HCPhysVirtApic);
1622
1623	val \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
1624	/* CR8 writes causes a VM-exit based on TPR threshold. */
1625	Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR8_STORE_EXIT));
1626	Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR8_LOAD_EXIT));
1627	}
1628	else
1629	{
1630	val \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR8_STORE_EXIT /* CR8 reads causes a VM-exit. */
1631	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR8_LOAD_EXIT; /* CR8 writes causes a VM-exit. */
1632	}
1633
1634	/* Use MSR-bitmaps if supported by the CPU. */
1635	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
1636	{
1637	val \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS;
1638
1639	Assert(pVCpu->hm.s.vmx.HCPhysMsrBitmap);
1640	Assert(!(pVCpu->hm.s.vmx.HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
1641	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, pVCpu->hm.s.vmx.HCPhysMsrBitmap);
1642	AssertRCReturn(rc, rc);
1643
1644	/*
1645	* The guest can access the following MSRs (read, write) without causing VM-exits; they are loaded/stored
1646	* automatically (either as part of the MSR-load/store areas or dedicated fields in the VMCS).
1647	*/
1648	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_CS, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1649	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_ESP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1650	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_EIP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1651	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1652	hmR0VmxSetMsrPermission(pVCpu, MSR_K6_STAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1653	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_SF_MASK, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1654	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1655	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1656	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_FS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1657	}
1658
1659	/* Use the secondary processor-based VM-execution controls if supported by the CPU. */
1660	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
1661	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
1662
1663	if ((val & zap) != val)
1664	{
1665	LogRel(("hmR0VmxSetupProcCtls: invalid processor-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
1666	pVM->hm.s.vmx.msr.vmx_proc_ctls.n.disallowed0, val, zap));
1667	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1668	}
1669
1670	val &= zap;
1671	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, val);
1672	AssertRCReturn(rc, rc);
1673
1674	/* Update VCPU with the currently set processor-based VM-execution controls. */
1675	pVCpu->hm.s.vmx.u32ProcCtls = val;
1676
1677	/*
1678	* Secondary processor-based VM-execution controls.
1679	*/
1680	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
1681	{
1682	val = pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.disallowed0; /* Bits set here must be set in the VMCS. */
1683	zap = pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
1684
1685	val \|= VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT; /* WBINVD causes a VM-exit. */
1686
1687	if (pVM->hm.s.fNestedPaging)
1688	val \|= VMX_VMCS_CTRL_PROC_EXEC2_EPT; /* Enable EPT. */
1689	else
1690	{
1691	/*
1692	* Without Nested Paging, INVPCID should cause a VM-exit. Enabling this bit causes the CPU to refer to
1693	* VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INVLPG_EXIT when INVPCID is executed by the guest.
1694	* See Intel spec. 25.4 "Changes to instruction behaviour in VMX non-root operation".
1695	*/
1696	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_INVPCID)
1697	val \|= VMX_VMCS_CTRL_PROC_EXEC2_INVPCID;
1698	}
1699
1700	if (pVM->hm.s.vmx.fVpid)
1701	val \|= VMX_VMCS_CTRL_PROC_EXEC2_VPID; /* Enable VPID. */
1702
1703	if (pVM->hm.s.vmx.fUnrestrictedGuest)
1704	val \|= VMX_VMCS_CTRL_PROC_EXEC2_UNRESTRICTED_GUEST; /* Enable Unrestricted Execution. */
1705
1706	/* Enable Virtual-APIC page accesses if supported by the CPU. This is essentially where the TPR shadow resides. */
1707	/** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
1708	* done dynamically. */
1709	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
1710	{
1711	Assert(pVM->hm.s.vmx.HCPhysApicAccess);
1712	Assert(!(pVM->hm.s.vmx.HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
1713	val \|= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC; /* Virtualize APIC accesses. */
1714	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, pVM->hm.s.vmx.HCPhysApicAccess);
1715	AssertRCReturn(rc, rc);
1716	}
1717
1718	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
1719	{
1720	val \|= VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP; /* Enable RDTSCP support. */
1721	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
1722	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_TSC_AUX, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1723	}
1724
1725	if ((val & zap) != val)
1726	{
1727	LogRel(("hmR0VmxSetupProcCtls: invalid secondary processor-based VM-execution controls combo! "
1728	"cpu=%#RX64 val=%#RX64 zap=%#RX64\n", pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.disallowed0, val, zap));
1729	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1730	}
1731
1732	val &= zap;
1733	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS2, val);
1734	AssertRCReturn(rc, rc);
1735
1736	/* Update VCPU with the currently set secondary processor-based VM-execution controls. */
1737	pVCpu->hm.s.vmx.u32ProcCtls2 = val;
1738	}
1739
1740	return VINF_SUCCESS;
1741	}
1742
1743
1744	/**
1745	* Sets up miscellaneous (everything other than Pin & Processor-based
1746	* VM-execution) control fields in the VMCS.
1747	*
1748	* @returns VBox status code.
1749	* @param pVM Pointer to the VM.
1750	* @param pVCpu Pointer to the VMCPU.
1751	*/
1752	static int hmR0VmxSetupMiscCtls(PVM pVM, PVMCPU pVCpu)
1753	{
1754	AssertPtr(pVM);
1755	AssertPtr(pVCpu);
1756
1757	int rc = VERR_GENERAL_FAILURE;
1758
1759	/* All CR3 accesses cause VM-exits. Later we optimize CR3 accesses (see hmR0VmxLoadGuestControlRegs())*/
1760	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, 0);
1761
1762	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, 0);
1763
1764	/*
1765	* Set MASK & MATCH to 0. VMX checks if GuestPFErrCode & MASK == MATCH. If equal (in our case it always is)
1766	* and if the X86_XCPT_PF bit in the exception bitmap is set it causes a VM-exit, if clear doesn't cause an exit.
1767	* We thus use the exception bitmap to control it rather than use both.
1768	*/
1769	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, 0);
1770	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, 0);
1771
1772	/** @todo Explore possibility of using IO-bitmaps. */
1773	/* All IO & IOIO instructions cause VM-exits. */
1774	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_A_FULL, 0);
1775	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_B_FULL, 0);
1776
1777	/* Setup MSR autoloading/autostoring. */
1778	Assert(pVCpu->hm.s.vmx.HCPhysGuestMsr);
1779	Assert(!(pVCpu->hm.s.vmx.HCPhysGuestMsr & 0xf)); /* Lower 4 bits MBZ. */
1780	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
1781	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
1782	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, 0);
1783	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, 0);
1784
1785	Assert(pVCpu->hm.s.vmx.HCPhysHostMsr);
1786	Assert(!(pVCpu->hm.s.vmx.HCPhysHostMsr & 0xf)); /* Lower 4 bits MBZ. */
1787	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysHostMsr);
1788	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, 0);
1789
1790	/* Set VMCS link pointer. Reserved for future use, must be -1. Intel spec. 24.4 "Guest-State Area". */
1791	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, 0xffffffffffffffffULL);
1792	AssertRCReturn(rc, rc);
1793	return rc;
1794	}
1795
1796
1797	/**
1798	* Sets up the initial exception bitmap in the VMCS based on static conditions
1799	* (i.e. conditions that cannot ever change at runtime).
1800	*
1801	* @returns VBox status code.
1802	* @param pVM Pointer to the VM.
1803	* @param pVCpu Pointer to the VMCPU.
1804	*/
1805	static int hmR0VmxInitXcptBitmap(PVM pVM, PVMCPU pVCpu)
1806	{
1807	AssertPtr(pVM);
1808	AssertPtr(pVCpu);
1809
1810	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
1811
1812	uint32_t u32XcptBitmap = 0;
1813
1814	/* Without nested paging, #PF must cause a VM-exit so we can sync our shadow page tables. */
1815	if (!pVM->hm.s.fNestedPaging)
1816	u32XcptBitmap \|= RT_BIT(X86_XCPT_PF);
1817
1818	pVCpu->hm.s.vmx.u32XcptBitmap = u32XcptBitmap;
1819	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
1820	AssertRCReturn(rc, rc);
1821	return rc;
1822	}
1823
1824
1825	/**
1826	* Sets up the initial guest-state mask. The guest-state mask is consulted
1827	* before reading guest-state fields from the VMCS as VMREADs can be expensive
1828	* for the nested virtualization case (as it would cause a VM-exit).
1829	*
1830	* @param pVCpu Pointer to the VMCPU.
1831	*/
1832	static int hmR0VmxInitUpdatedGuestStateMask(PVMCPU pVCpu)
1833	{
1834	/* Initially the guest-state is up-to-date as there is nothing in the VMCS. */
1835	pVCpu->hm.s.vmx.fUpdatedGuestState = VMX_UPDATED_GUEST_ALL;
1836	return VINF_SUCCESS;
1837	}
1838
1839
1840	/**
1841	* Does per-VM VT-x initialization.
1842	*
1843	* @returns VBox status code.
1844	* @param pVM Pointer to the VM.
1845	*/
1846	VMMR0DECL(int) VMXR0InitVM(PVM pVM)
1847	{
1848	LogFlowFunc(("pVM=%p\n", pVM));
1849
1850	int rc = hmR0VmxStructsAlloc(pVM);
1851	if (RT_FAILURE(rc))
1852	{
1853	LogRel(("VMXR0InitVM: hmR0VmxStructsAlloc failed! rc=%Rrc\n", rc));
1854	return rc;
1855	}
1856
1857	for (VMCPUID i = 0; i < pVM->cCpus; i++)
1858	{
1859	PVMCPU pVCpu = &pVM->aCpus[i];
1860
1861	/* Current guest paging mode. */
1862	pVCpu->hm.s.vmx.enmLastSeenGuestMode = PGMMODE_REAL;
1863	}
1864
1865	return VINF_SUCCESS;
1866	}
1867
1868
1869	/**
1870	* Does per-VM VT-x termination.
1871	*
1872	* @returns VBox status code.
1873	* @param pVM Pointer to the VM.
1874	*/
1875	VMMR0DECL(int) VMXR0TermVM(PVM pVM)
1876	{
1877	LogFlowFunc(("pVM=%p\n", pVM));
1878
1879	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
1880	if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
1881	ASMMemZero32(pVM->hm.s.vmx.pvScratch, PAGE_SIZE);
1882	#endif
1883	hmR0VmxStructsFree(pVM);
1884	return VINF_SUCCESS;
1885	}
1886
1887
1888	/**
1889	* Sets up the VM for execution under VT-x.
1890	* This function is only called once per-VM during initalization.
1891	*
1892	* @returns VBox status code.
1893	* @param pVM Pointer to the VM.
1894	*/
1895	VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
1896	{
1897	AssertPtrReturn(pVM, VERR_INVALID_PARAMETER);
1898	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1899
1900	LogFlowFunc(("pVM=%p\n", pVM));
1901
1902	/*
1903	* Without UnrestrictedGuest, pRealModeTSS and pNonPagingModeEPTPageTable must always be allocated.
1904	* We no longer support the highly unlikely case of UnrestrictedGuest without pRealModeTSS. See hmR3InitFinalizeR0().
1905	*/
1906	/* -XXX- change hmR3InitFinalizeR0() to fail if pRealModeTSS alloc fails. */
1907	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
1908	&& ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
1909	\|\| !pVM->hm.s.vmx.pRealModeTSS))
1910	{
1911	LogRel(("VMXR0SetupVM: invalid real-on-v86 state.\n"));
1912	return VERR_INTERNAL_ERROR;
1913	}
1914
1915	/* Initialize these always, see hmR3InitFinalizeR0().*/
1916	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NONE;
1917	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NONE;
1918
1919	/* Setup the tagged-TLB flush handlers. */
1920	int rc = hmR0VmxSetupTaggedTlb(pVM);
1921	if (RT_FAILURE(rc))
1922	{
1923	LogRel(("VMXR0SetupVM: hmR0VmxSetupTaggedTlb failed! rc=%Rrc\n", rc));
1924	return rc;
1925	}
1926
1927	for (VMCPUID i = 0; i < pVM->cCpus; i++)
1928	{
1929	PVMCPU pVCpu = &pVM->aCpus[i];
1930	AssertPtr(pVCpu);
1931	AssertPtr(pVCpu->hm.s.vmx.pvVmcs);
1932
1933	/* Set revision dword at the beginning of the VMCS structure. */
1934	(uint32_t )pVCpu->hm.s.vmx.pvVmcs = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.msr.vmx_basic_info);
1935
1936	/* Initialize our VMCS region in memory, set the VMCS launch state to "clear". */
1937	rc = VMXClearVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
1938	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVMCS failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
1939	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
1940
1941	/* Load this VMCS as the current VMCS. */
1942	rc = VMXActivateVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
1943	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXActivateVMCS failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
1944	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
1945
1946	/* Setup the pin-based VM-execution controls. */
1947	rc = hmR0VmxSetupPinCtls(pVM, pVCpu);
1948	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupPinCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
1949	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
1950
1951	/* Setup the processor-based VM-execution controls. */
1952	rc = hmR0VmxSetupProcCtls(pVM, pVCpu);
1953	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupProcCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
1954	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
1955
1956	/* Setup the rest (miscellaneous) VM-execution controls. */
1957	rc = hmR0VmxSetupMiscCtls(pVM, pVCpu);
1958	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupMiscCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
1959	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
1960
1961	/* Setup the initial exception bitmap. */
1962	rc = hmR0VmxInitXcptBitmap(pVM, pVCpu);
1963	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitXcptBitmap failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
1964	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
1965
1966	/* Setup the initial guest-state mask. */
1967	rc = hmR0VmxInitUpdatedGuestStateMask(pVCpu);
1968	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitUpdatedGuestStateMask failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
1969	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
1970
1971	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
1972	/* Setup the VMCS read cache as we queue up certain VMWRITEs that can only be done in 64-bit mode for 64-bit guests. */
1973	rc = hmR0VmxInitVmcsReadCache(pVM, pVCpu);
1974	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitVmcsReadCache failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
1975	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
1976	#endif
1977
1978	/* Re-sync the CPU's internal data into our VMCS memory region & reset the launch state to "clear". */
1979	rc = VMXClearVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
1980	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVMCS(2) failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
1981	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
1982
1983	/* Update the last error record for this VCPU. */
1984	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc);
1985	}
1986
1987	return VINF_SUCCESS;
1988	}
1989
1990
1991	/**
1992	* Saves the host control registers (CR0, CR3, CR4) into the host-state area in
1993	* the VMCS.
1994	*
1995	* @returns VBox status code.
1996	* @param pVM Pointer to the VM.
1997	* @param pVCpu Pointer to the VMCPU.
1998	*/
1999	DECLINLINE(int) hmR0VmxSaveHostControlRegs(PVM pVM, PVMCPU pVCpu)
2000	{
2001	RTCCUINTREG uReg = ASMGetCR0();
2002	int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
2003	AssertRCReturn(rc, rc);
2004
2005	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2006	/* For the darwin 32-bit hybrid kernel, we need the 64-bit CR3 as it uses 64-bit paging. */
2007	if (VMX_IS_64BIT_HOST_MODE())
2008	{
2009	uint64_t uReg = hmR0Get64bitCR3();
2010	rc = VMXWriteVmcs64(VMX_VMCS_HOST_CR3, uReg);
2011	}
2012	else
2013	#endif
2014	{
2015	uReg = ASMGetCR3();
2016	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
2017	}
2018	AssertRCReturn(rc, rc);
2019
2020	uReg = ASMGetCR4();
2021	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
2022	AssertRCReturn(rc, rc);
2023	return rc;
2024	}
2025
2026
2027	/**
2028	* Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
2029	* the host-state area in the VMCS.
2030	*
2031	* @returns VBox status code.
2032	* @param pVM Pointer to the VM.
2033	* @param pVCpu Pointer to the VMCPU.
2034	*/
2035	DECLINLINE(int) hmR0VmxSaveHostSegmentRegs(PVM pVM, PVMCPU pVCpu)
2036	{
2037	int rc = VERR_INTERNAL_ERROR_5;
2038	RTSEL uSelCS = 0;
2039	RTSEL uSelSS = 0;
2040	RTSEL uSelDS = 0;
2041	RTSEL uSelES = 0;
2042	RTSEL uSelFS = 0;
2043	RTSEL uSelGS = 0;
2044	RTSEL uSelTR = 0;
2045
2046	/*
2047	* Host Selector registers.
2048	*/
2049	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2050	if (VMX_IS_64BIT_HOST_MODE())
2051	{
2052	uSelCS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelCS;
2053	uSelSS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelSS;
2054	}
2055	else
2056	{
2057	/* Seems darwin uses the LDT (TI flag is set) in the CS & SS selectors which VT-x doesn't like. */
2058	uSelCS = (RTSEL)(uintptr_t)&SUPR0AbsKernelCS;
2059	uSelSS = (RTSEL)(uintptr_t)&SUPR0AbsKernelSS;
2060	}
2061	#else
2062	uSelCS = ASMGetCS();
2063	uSelSS = ASMGetSS();
2064	#endif
2065
2066	/* Note: VT-x is picky about the RPL of the selectors here; we'll restore them manually. */
2067	/** @todo Verify if we have any platform that actually run with DS or ES with
2068	* RPL != 0 in kernel space. */
2069	uSelDS = 0;
2070	uSelES = 0;
2071	uSelFS = 0;
2072	uSelGS = 0;
2073	uSelTR = ASMGetTR();
2074
2075	/* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
2076	Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
2077	Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
2078	Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
2079	Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
2080	Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
2081	Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
2082	Assert(uSelCS != 0);
2083	Assert(uSelTR != 0);
2084
2085	/* Assertion is right but we would not have updated u32ExitCtls yet. */
2086	#if 0
2087	if (!(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_ADDR_SPACE_SIZE))
2088	Assert(uSelSS != 0);
2089	#endif
2090
2091	/* Write these host selector fields into the host-state area in the VMCS. */
2092	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_CS, uSelCS);
2093	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_SS, uSelSS);
2094	/* Avoid the VMWRITEs as we set the following segments to 0 and the VMCS fields are already 0 (since g_HvmR0 is static) */
2095	#if 0
2096	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_DS, uSelDS);
2097	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_ES, uSelES);
2098	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_FS, uSelFS);
2099	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_GS, uSelGS);
2100	#endif
2101	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_TR, uSelTR);
2102	AssertRCReturn(rc, rc);
2103
2104	/*
2105	* Host GDTR and IDTR.
2106	*/
2107	/** @todo Despite VT-x -not- restoring the limits on GDTR and IDTR it should
2108	* be safe to -not- save and restore GDTR and IDTR in the assembly
2109	* code and just do it here and don't care if the limits are zapped on
2110	* VM-exit. */
2111	RTGDTR Gdtr;
2112	RT_ZERO(Gdtr);
2113	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2114	if (VMX_IS_64BIT_HOST_MODE())
2115	{
2116	X86XDTR64 Gtr64;
2117	X86XDTR64 Idtr64;
2118	hmR0Get64bitGdtrAndIdtr(&Gdtr64, &Idtr64);
2119	rc = VMXWriteVmcs64(VMX_VMCS_HOST_GDTR_BASE, Gdtr64.uAddr);
2120	rc \|= VMXWriteVmcs64(VMX_VMCS_HOST_IDTR_BASE, Idtr64.uAddr);
2121	Gdtr.cbGdt = Gdtr64.cb;
2122	Gdtr.pGdt = (uintptr_t)Gdtr64.uAddr;
2123	}
2124	else
2125	#endif
2126	{
2127	RTIDTR Idtr;
2128	ASMGetGDTR(&Gdtr);
2129	ASMGetIDTR(&Idtr);
2130	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt);
2131	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt);
2132	}
2133	AssertRCReturn(rc, rc);
2134
2135	/*
2136	* Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI and RPL bits
2137	* is effectively what the CPU does for "scaling by 8". TI is always 0 and RPL should be too in most cases.
2138	*/
2139	if ((uSelTR & X86_SEL_MASK) > Gdtr.cbGdt)
2140	{
2141	AssertMsgFailed(("hmR0VmxSaveHostSegmentRegs: TR selector exceeds limit.TR=%RTsel Gdtr.cbGdt=%#x\n", uSelTR, Gdtr.cbGdt));
2142	return VERR_VMX_INVALID_HOST_STATE;
2143	}
2144
2145	PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
2146	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2147	if (VMX_IS_64BIT_HOST_MODE())
2148	{
2149	/* We need the 64-bit TR base for hybrid darwin. */
2150	uint64_t u64TRBase = X86DESC64_BASE((PX86DESC64)pDesc);
2151	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, u64TRBase);
2152	}
2153	else
2154	#endif
2155	{
2156	uintptr_t uTRBase;
2157	#if HC_ARCH_BITS == 64
2158	uTRBase = X86DESC64_BASE(pDesc);
2159	#else
2160	uTRBase = X86DESC_BASE(pDesc);
2161	#endif
2162	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
2163	}
2164	AssertRCReturn(rc, rc);
2165
2166	/*
2167	* Host FS base and GS base.
2168	* For 32-bit hosts the base is handled by the assembly code where we push/pop FS and GS which .
2169	* would take care of the bases. In 64-bit, the MSRs come into play.
2170	*/
2171	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2172	if (VMX_IS_64BIT_HOST_MODE())
2173	{
2174	uint64_t u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
2175	uint64_t u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
2176	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_FS_BASE, u64FSBase);
2177	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_GS_BASE, u64GSBase);
2178	AssertRCReturn(rc, rc);
2179	}
2180	#endif
2181	return rc;
2182	}
2183
2184
2185	/**
2186	* Saves certain host MSRs in the VM-Exit MSR-load area and some in the
2187	* host-state area of the VMCS. Theses MSRs will be automatically restored on
2188	* the host after every successful VM exit.
2189	*
2190	* @returns VBox status code.
2191	* @param pVM Pointer to the VM.
2192	* @param pVCpu Pointer to the VMCPU.
2193	*/
2194	DECLINLINE(int) hmR0VmxSaveHostMsrs(PVM pVM, PVMCPU pVCpu)
2195	{
2196	AssertPtr(pVCpu);
2197	AssertPtr(pVCpu->hm.s.vmx.pvHostMsr);
2198
2199	PVMXMSR pHostMsr = (PVMXMSR)pVCpu->hm.s.vmx.pvHostMsr;
2200	unsigned idxHostMsr = 0;
2201	uint32_t u32HostExtFeatures = pVM->hm.s.cpuid.u32AMDFeatureEDX;
2202
2203	if (u32HostExtFeatures & (X86_CPUID_EXT_FEATURE_EDX_NX \| X86_CPUID_EXT_FEATURE_EDX_LONG_MODE))
2204	{
2205	pHostMsr->u32IndexMSR = MSR_K6_EFER;
2206	pHostMsr->u32Reserved = 0;
2207	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2208	if (CPUMIsGuestInLongMode(pVCpu))
2209	{
2210	/* Must match the EFER value in our 64 bits switcher. */
2211	pHostMsr->u64Value = ASMRdMsr(MSR_K6_EFER) \| MSR_K6_EFER_LME \| MSR_K6_EFER_SCE \| MSR_K6_EFER_NXE;
2212	}
2213	else
2214	#endif
2215	pHostMsr->u64Value = ASMRdMsr(MSR_K6_EFER);
2216	pHostMsr++; idxHostMsr++;
2217	}
2218
2219	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2220	if (VMX_IS_64BIT_HOST_MODE())
2221	{
2222	pHostMsr->u32IndexMSR = MSR_K6_STAR;
2223	pHostMsr->u32Reserved = 0;
2224	pHostMsr->u64Value = ASMRdMsr(MSR_K6_STAR); /* legacy syscall eip, cs & ss */
2225	pHostMsr++; idxHostMsr++;
2226	pHostMsr->u32IndexMSR = MSR_K8_LSTAR;
2227	pHostMsr->u32Reserved = 0;
2228	pHostMsr->u64Value = ASMRdMsr(MSR_K8_LSTAR); /* 64 bits mode syscall rip */
2229	pHostMsr++; idxHostMsr++;
2230	pHostMsr->u32IndexMSR = MSR_K8_SF_MASK;
2231	pHostMsr->u32Reserved = 0;
2232	pHostMsr->u64Value = ASMRdMsr(MSR_K8_SF_MASK); /* syscall flag mask */
2233	pHostMsr++; idxHostMsr++;
2234	/* The KERNEL_GS_BASE MSR doesn't work reliably with auto load/store. See @bugref{6208} */
2235	#if 0
2236	pMsr->u32IndexMSR = MSR_K8_KERNEL_GS_BASE;
2237	pMsr->u32Reserved = 0;
2238	pMsr->u64Value = ASMRdMsr(MSR_K8_KERNEL_GS_BASE); /* swapgs exchange value */
2239	pHostMsr++; idxHostMsr++;
2240	#endif
2241	}
2242	#endif
2243
2244	/* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
2245	if (idxHostMsr > MSR_IA32_VMX_MISC_MAX_MSR(pVM->hm.s.vmx.msr.vmx_misc))
2246	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2247
2248	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, idxHostMsr);
2249	AssertRCReturn(rc, rc);
2250
2251	/*
2252	* Host Sysenter MSRs.
2253	*/
2254	rc = VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
2255	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2256	if (VMX_IS_64BIT_HOST_MODE())
2257	{
2258	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
2259	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
2260	}
2261	else
2262	{
2263	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
2264	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
2265	}
2266	#elif HC_ARCH_BITS == 32
2267	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
2268	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
2269	#else
2270	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
2271	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
2272	#endif
2273	AssertRCReturn(rc, rc);
2274
2275	/** @todo IA32_PERF_GLOBALCTRL, IA32_PAT, IA32_EFER, also see
2276	* hmR0VmxSetupExitCtls() !! */
2277	return rc;
2278	}
2279
2280
2281	/**
2282	* Sets up VM-entry controls in the VMCS. These controls can affect things done
2283	* on VM-exit; e.g. "load debug controls", see Intel spec. 24.8.1 "VM-entry
2284	* controls".
2285	*
2286	* @returns VBox status code.
2287	* @param pVM Pointer to the VM.
2288	* @param pVCpu Pointer to the VMCPU.
2289	* @param pCtx Pointer to the guest-CPU context.
2290	*
2291	* @remarks No-long-jump zone!!!
2292	*/
2293	DECLINLINE(int) hmR0VmxLoadGuestEntryCtls(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
2294	{
2295	int rc = VINF_SUCCESS;
2296	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_ENTRY_CTLS)
2297	{
2298	uint32_t val = pVM->hm.s.vmx.msr.vmx_entry.n.disallowed0; /* Bits set here must be set in the VMCS. */
2299	uint32_t zap = pVM->hm.s.vmx.msr.vmx_entry.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2300
2301	/* Load debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x capable CPUs only supports the 1-setting of this bit. */
2302	val \|= VMX_VMCS_CTRL_ENTRY_CONTROLS_LOAD_DEBUG;
2303
2304	/* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry. */
2305	if (CPUMIsGuestInLongModeEx(pCtx))
2306	val \|= VMX_VMCS_CTRL_ENTRY_CONTROLS_IA32E_MODE_GUEST;
2307	else
2308	Assert(!(val & VMX_VMCS_CTRL_ENTRY_CONTROLS_IA32E_MODE_GUEST));
2309
2310	/*
2311	* The following should not be set (since we're not in SMM mode):
2312	* - VMX_VMCS_CTRL_ENTRY_CONTROLS_ENTRY_SMM
2313	* - VMX_VMCS_CTRL_ENTRY_CONTROLS_DEACTIVATE_DUALMON
2314	*/
2315
2316	/** @todo VMX_VMCS_CTRL_ENTRY_CONTROLS_LOAD_GUEST_PERF_MSR,
2317	* VMX_VMCS_CTRL_ENTRY_CONTROLS_LOAD_GUEST_PAT_MSR,
2318	* VMX_VMCS_CTRL_ENTRY_CONTROLS_LOAD_GUEST_EFER_MSR */
2319
2320	if ((val & zap) != val)
2321	{
2322	LogRel(("hmR0VmxLoadGuestEntryCtls: invalid VM-entry controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
2323	pVM->hm.s.vmx.msr.vmx_entry.n.disallowed0, val, zap));
2324	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2325	}
2326
2327	val &= zap;
2328	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_CONTROLS, val);
2329	AssertRCReturn(rc, rc);
2330
2331	/* Update VCPU with the currently set VM-exit controls. */
2332	pVCpu->hm.s.vmx.u32EntryCtls = val;
2333	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_ENTRY_CTLS;
2334	}
2335	return rc;
2336	}
2337
2338
2339	/**
2340	* Sets up the VM-exit controls in the VMCS.
2341	*
2342	* @returns VBox status code.
2343	* @param pVM Pointer to the VM.
2344	* @param pVCpu Pointer to the VMCPU.
2345	*/
2346	DECLINLINE(int) hmR0VmxLoadGuestExitCtls(PVM pVM, PVMCPU pVCpu)
2347	{
2348	int rc = VINF_SUCCESS;
2349	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_EXIT_CTLS)
2350	{
2351	uint32_t val = pVM->hm.s.vmx.msr.vmx_exit.n.disallowed0; /* Bits set here must be set in the VMCS. */
2352	uint32_t zap = pVM->hm.s.vmx.msr.vmx_exit.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2353
2354	/* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only supported the 1-setting of this bit. */
2355	val \|= VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_DEBUG;
2356
2357	/* Set the host long mode active (EFER.LMA) bit (which Intel calls "Host address-space size") if necessary. */
2358	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2359	if (VMX_IS_64BIT_HOST_MODE())
2360	val \|= VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_ADDR_SPACE_SIZE;
2361	else
2362	Assert(!(val & VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_ADDR_SPACE_SIZE));
2363	#elif HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
2364	if (CPUMIsGuestInLongModeEx(pCtx))
2365	val \|= VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_ADDR_SPACE_SIZE; /* The switcher goes to long mode. */
2366	else
2367	Assert(!(val & VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_ADDR_SPACE_SIZE));
2368	#endif
2369
2370	/* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
2371	Assert(!(val & VMX_VMCS_CTRL_EXIT_CONTROLS_ACK_EXT_INT));
2372
2373	/** @todo VMX_VMCS_CTRL_EXIT_CONTROLS_LOAD_PERF_MSR,
2374	* VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_GUEST_PAT_MSR,
2375	* VMX_VMCS_CTRL_EXIT_CONTROLS_LOAD_HOST_PAT_MSR,
2376	* VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_GUEST_EFER_MSR,
2377	* VMX_VMCS_CTRL_EXIT_CONTROLS_LOAD_HOST_EFER_MSR,
2378	* VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_VMX_PREEMPT_TIMER. */
2379
2380	if ((val & zap) != val)
2381	{
2382	LogRel(("hmR0VmxSetupProcCtls: invalid VM-exit controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
2383	pVM->hm.s.vmx.msr.vmx_exit.n.disallowed0, val, zap));
2384	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2385	}
2386
2387	val &= zap;
2388	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_CONTROLS, val);
2389
2390	/* Update VCPU with the currently set VM-exit controls. */
2391	pVCpu->hm.s.vmx.u32ExitCtls = val;
2392	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_EXIT_CTLS;
2393	}
2394	return rc;
2395	}
2396
2397
2398	/**
2399	* Loads the guest APIC and related state.
2400	*
2401	* @returns VBox status code.
2402	* @param pVM Pointer to the VM.
2403	* @param pVCpu Pointer to the VMCPU.
2404	* @param pCtx Pointer to the guest-CPU context.
2405	*/
2406	DECLINLINE(int) hmR0VmxLoadGuestApicState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
2407	{
2408	int rc = VINF_SUCCESS;
2409	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_GUEST_APIC_STATE)
2410	{
2411	/* Setup TPR shadowing. Also setup TPR patching for 32-bit guests. */
2412	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
2413	{
2414	Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
2415
2416	bool fPendingIntr = false;
2417	uint8_t u8GuestTpr = 0;
2418	rc = PDMApicGetTPR(pVCpu, &u8GuestTpr, &fPendingIntr);
2419	AssertRCReturn(rc, rc);
2420
2421	/*
2422	* If there are external interrupts pending but masked by the TPR value, apply the threshold so that if the guest
2423	* lowers the TPR, it would cause a VM-exit and we can deliver the interrupt.
2424	* If there are no external interrupts pending, set threshold to 0 to not cause a VM-exit. We will eventually deliver
2425	* the interrupt when we VM-exit for other reasons.
2426	*/
2427	pVCpu->hm.s.vmx.pbVirtApic[0x80] = u8GuestTpr; /* Offset 0x80 is TPR in the APIC MMIO range. */
2428	/* Bits 3-0 of the TPR threshold field correspond to bits 7-4 of the TPR (which is the Task-Priority Class). */
2429	uint32_t u32TprThreshold = fPendingIntr ? (u8GuestTpr >> 4) : 0;
2430	Assert(!(u32TprThreshold & 0xfffffff0)); /* Bits 31:4 MBZ. */
2431
2432	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
2433	AssertRCReturn(rc, rc);
2434
2435	/* 32-bit guests uses LSTAR MSR for patching guest code which touches the TPR. */
2436	if (pVM->hm.s.fTPRPatchingActive)
2437	{
2438	Assert(!CPUMIsGuestInLongModeEx(pCtx)); /* EFER always up-to-date. */
2439	pCtx->msrLSTAR = u8GuestTpr;
2440	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
2441	{
2442	/* If there are interrupts pending, intercept CR8 writes, otherwise don't intercept CR8 reads or writes. */
2443	if (fPendingIntr)
2444	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_INTERCEPT_WRITE);
2445	else
2446	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2447	}
2448	}
2449	}
2450
2451	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_GUEST_APIC_STATE;
2452	}
2453	return rc;
2454	}
2455
2456
2457	/**
2458	* Loads the guest's RIP into the guest-state area in the VMCS.
2459	*
2460	* @returns VBox status code.
2461	* @param pVM Pointer to the VM.
2462	* @param pVCpu Pointer to the VMCPU.
2463	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2464	* out-of-sync. Make sure to update the required fields
2465	* before using them.
2466	*
2467	* @remarks No-long-jump zone!!!
2468	*/
2469	DECLINLINE(int) hmR0VmxLoadGuestRip(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2470	{
2471	int rc = VINF_SUCCESS;
2472	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_RIP)
2473	{
2474	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pMixedCtx->rip);
2475	AssertRCReturn(rc, rc);
2476	Log(("VMX_VMCS_GUEST_RIP=%#RX64\n", pMixedCtx->rip));
2477	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_RIP;
2478	}
2479	return rc;
2480	}
2481
2482
2483	/**
2484	* Loads the guest's RSP into the guest-state area in the VMCS.
2485	*
2486	* @returns VBox status code.
2487	* @param pVM Pointer to the VM.
2488	* @param pVCpu Pointer to the VMCPU.
2489	* @param pCtx Pointer to the guest-CPU context.
2490	*
2491	* @remarks No-long-jump zone!!!
2492	*/
2493	DECLINLINE(int) hmR0VmxLoadGuestRsp(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2494	{
2495	int rc = VINF_SUCCESS;
2496	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_RSP)
2497	{
2498	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pMixedCtx->rsp);
2499	AssertRCReturn(rc, rc);
2500	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_RSP;
2501	}
2502	return rc;
2503	}
2504
2505
2506	/**
2507	* Loads the guest's RFLAGS into the guest-state area in the VMCS.
2508	*
2509	* @returns VBox status code.
2510	* @param pVM Pointer to the VM.
2511	* @param pVCpu Pointer to the VMCPU.
2512	* @param pCtx Pointer to the guest-CPU context.
2513	*
2514	* @remarks No-long-jump zone!!!
2515	*/
2516	DECLINLINE(int) hmR0VmxLoadGuestRflags(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
2517	{
2518	int rc = VINF_SUCCESS;
2519	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_RFLAGS)
2520	{
2521	/* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
2522	Let us assert it as such and use native-width VMWRITE. */
2523	X86RFLAGS uRFlags = pCtx->rflags;
2524	Assert(uRFlags.u64 >> 32 == 0);
2525	uRFlags.u64 &= VMX_EFLAGS_RESERVED_0; /* Bits 22-31, 15, 5 & 3 MBZ. */
2526	uRFlags.u64 \|= VMX_EFLAGS_RESERVED_1; /* Bit 1 MB1. */
2527
2528	/*
2529	* If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so we can restore them on VM exit.
2530	* Modify the real-mode guest's eflags so that VT-x can run the real-mode guest code under Virtual 8086 mode.
2531	*/
2532	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
2533	{
2534	Assert(pVM->hm.s.vmx.pRealModeTSS);
2535	Assert(PDMVmmDevHeapIsEnabled(pVM));
2536	pVCpu->hm.s.vmx.RealMode.eflags.u32 = uRFlags.u64; /* Save the original eflags of the real-mode guest. */
2537	uRFlags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
2538	uRFlags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
2539	}
2540
2541	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RFLAGS, uRFlags.u64);
2542	AssertRCReturn(rc, rc);
2543
2544	Log(("VMX_VMCS_GUEST_RFLAGS=%#RX64\n", uRFlags.u64));
2545	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_RFLAGS;
2546	}
2547	return rc;
2548	}
2549
2550
2551	/**
2552	* Loads the guest's general purpose registers (GPRs) - RIP, RSP and RFLAGS
2553	* into the guest-state area in the VMCS. The remaining GPRs are handled in the
2554	* assembly code.
2555	*
2556	* @returns VBox status code.
2557	* @param pVM Pointer to the VM.
2558	* @param pVCpu Pointer to the VMCPU.
2559	* @param pCtx Pointer to the guest-CPU context.
2560	*
2561	* @remarks No-long-jump zone!!!
2562	*/
2563	DECLINLINE(int) hmR0VmxLoadGuestGprs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
2564	{
2565	LogFlowFunc(("pVM=%p pVCpu=%p pCtx=%p\n", pVM, pVCpu, pCtx));
2566	int rc = hmR0VmxLoadGuestRip(pVM, pVCpu, pCtx);
2567	rc \|= hmR0VmxLoadGuestRsp(pVM, pVCpu, pCtx);
2568	rc \|= hmR0VmxLoadGuestRflags(pVM, pVCpu, pCtx);
2569	AssertRCReturn(rc, rc);
2570	return rc;
2571	}
2572
2573
2574	/**
2575	* Loads the guest control registers (CR0, CR3, CR4) into the guest-state area
2576	* in the VMCS.
2577	*
2578	* @returns VBox status code.
2579	* @param pVM Pointer to the VM.
2580	* @param pVCpu Pointer to the VMCPU.
2581	* @param pCtx Pointer to the guest-CPU context.
2582	*
2583	* @remarks No-long-jump zone!!!
2584	*/
2585	DECLINLINE(int) hmR0VmxLoadGuestControlRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
2586	{
2587	int rc = VINF_SUCCESS;
2588
2589	/*
2590	* Guest CR0.
2591	* Guest FPU.
2592	*/
2593	if ( (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR0)
2594	\|\| (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_FPU))
2595	{
2596	uint64_t u64GuestCR0 = pCtx->cr0;
2597
2598	/* The guest's view (read access) of its CR0 is unblemished. */
2599	rc = VMXWriteVmcsGstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, u64GuestCR0);
2600	AssertRCReturn(rc, rc);
2601	Log2(("VMX_VMCS_CTRL_CR0_READ_SHADOW=%#RX64\n", u64GuestCR0));
2602
2603	/* Setup VT-x's view of the guest CR0. */
2604	/* Minimize VM-exits due to CR3 changes when we have NestedPaging. */
2605	if (pVM->hm.s.fNestedPaging)
2606	{
2607	if (CPUMIsGuestPagingEnabledEx(pCtx))
2608	{
2609	/* The guest has paging enabled, let it access CR3 without causing a VM exit if supported. */
2610	pVCpu->hm.s.vmx.u32ProcCtls &= ~( VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
2611	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT);
2612	}
2613	else
2614	{
2615	/* The guest doesn't have paging enabled, make CR3 access to cause VM exits to update our shadow. */
2616	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
2617	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT;
2618	}
2619
2620	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls);
2621	AssertRCReturn(rc, rc);
2622	}
2623	else
2624	u64GuestCR0 \|= X86_CR0_WP; /* Guest CPL 0 writes to its read-only pages should cause a VM-exit. */
2625
2626	/*
2627	* Guest FPU bits.
2628	* Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be set on the first
2629	* CPUs to support VT-x (prob. means all the way up to Nehalem) and no mention of with regards to UX in VM-entry checks.
2630	*/
2631	u64GuestCR0 \|= X86_CR0_NE;
2632	bool fInterceptNM = false;
2633	if (CPUMIsGuestFPUStateActive(pVCpu))
2634	{
2635	fInterceptNM = false; /* Guest FPU active, no need to VM-exit on #NM. */
2636	/* The guest should still get #NM exceptions when it expects it to, so we should not clear TS & MP bits here.
2637	We're only concerned about -us- not intercepting #NMs when the guest-FPU is active. Not the guest itself! */
2638	}
2639	else
2640	{
2641	fInterceptNM = true; /* Guest FPU inactive, VM-exit on #NM for lazy FPU loading. */
2642	u64GuestCR0 \|= X86_CR0_TS /* Guest can task switch quickly and do lazy FPU syncing. */
2643	\| X86_CR0_MP; /* FWAIT/WAIT should not ignore CR0.TS and should generate #NM. */
2644	}
2645
2646	/* Catch floating point exceptions if we need to report them to the guest in a different way. */
2647	bool fInterceptMF = false;
2648	if (!(pCtx->cr0 & X86_CR0_NE))
2649	fInterceptMF = true;
2650
2651	/* Finally, intercept all exceptions as we cannot directly inject them in real-mode, see hmR0VmxInjectEventVmcs(). */
2652	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
2653	{
2654	Assert(PDMVmmDevHeapIsEnabled(pVM));
2655	Assert(pVM->hm.s.vmx.pRealModeTSS);
2656	pVCpu->hm.s.vmx.u32XcptBitmap \|= VMX_REAL_MODE_XCPT_BITMAP;
2657	fInterceptNM = true;
2658	fInterceptMF = true;
2659	}
2660	else
2661	pVCpu->hm.s.vmx.u32XcptBitmap &= ~VMX_REAL_MODE_XCPT_BITMAP;
2662
2663	if (fInterceptNM)
2664	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_NM);
2665	else
2666	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_NM);
2667
2668	if (fInterceptMF)
2669	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_MF);
2670	else
2671	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_MF);
2672
2673	/* Additional intercepts for debugging, define these yourself explicitly. */
2674	#ifdef VBOX_ALWAYS_TRAP_ALL_EXCEPTIONS
2675	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_BP)
2676	\| RT_BIT(X86_XCPT_DB)
2677	\| RT_BIT(X86_XCPT_DE)
2678	\| RT_BIT(X86_XCPT_NM)
2679	\| RT_BIT(X86_XCPT_UD)
2680	\| RT_BIT(X86_XCPT_NP)
2681	\| RT_BIT(X86_XCPT_SS)
2682	\| RT_BIT(X86_XCPT_GP)
2683	\| RT_BIT(X86_XCPT_PF)
2684	\| RT_BIT(X86_XCPT_MF);
2685	#elif defined(VBOX_ALWAYS_TRAP_PF)
2686	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_PF)
2687	#endif
2688
2689	Assert(pVM->hm.s.fNestedPaging \|\| (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_PF)));
2690
2691	/* Set/clear the CR0 specific bits along with their exceptions (PE, PG, CD, NW). */
2692	uint64_t uSetCR0 = (pVM->hm.s.vmx.msr.vmx_cr0_fixed0 & pVM->hm.s.vmx.msr.vmx_cr0_fixed1);
2693	uint64_t uZapCR0 = (pVM->hm.s.vmx.msr.vmx_cr0_fixed0 \| pVM->hm.s.vmx.msr.vmx_cr0_fixed1);
2694	if (pVM->hm.s.vmx.fUnrestrictedGuest) /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG). */
2695	uSetCR0 &= ~(X86_CR0_PE \| X86_CR0_PG);
2696	else
2697	Assert((uSetCR0 & (X86_CR0_PE \| X86_CR0_PG)) == (X86_CR0_PE \| X86_CR0_PG));
2698
2699	u64GuestCR0 \|= uSetCR0;
2700	u64GuestCR0 &= uZapCR0;
2701	u64GuestCR0 &= ~(X86_CR0_CD \| X86_CR0_NW); /* Always enable caching. */
2702
2703	/* Write VT-x's view of the guest CR0 into the VMCS and update the exception bitmap. */
2704	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR0, u64GuestCR0);
2705	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
2706	Log2(("VMX_VMCS_GUEST_CR0=%#RX32\n", (uint32_t)u64GuestCR0));
2707
2708	/*
2709	* CR0 is shared between host and guest along with a CR0 read shadow. Therefore, certain bits must not be changed
2710	* by the guest because VT-x ignores saving/restoring them (namely CD, ET, NW) and for certain other bits
2711	* we want to be notified immediately of guest CR0 changes (e.g. PG to update our shadow page tables).
2712	*/
2713	uint64_t u64CR0Mask = 0;
2714	u64CR0Mask = X86_CR0_PE
2715	\| X86_CR0_WP /** @todo do we need to monitor WP with nested paging? */
2716	\| X86_CR0_PG /** @todo do we need to monitor PG with nested paging? */
2717	\| X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
2718	\| X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
2719	\| X86_CR0_NW; /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
2720
2721	/* We don't need to intercept changes to CR0.PE with unrestricted guests. */
2722	if (pVM->hm.s.vmx.fUnrestrictedGuest)
2723	u64CR0Mask &= ~X86_CR0_PE;
2724
2725	/* If the guest FPU state is active, don't need to VM-exit on writes to FPU related bits in CR0. */
2726	if (fInterceptNM)
2727	u64CR0Mask \|= (X86_CR0_TS \| X86_CR0_MP);
2728	else
2729	u64CR0Mask &= ~(X86_CR0_TS \| X86_CR0_MP);
2730
2731	/* Write the CR0 mask into the VMCS and update the VCPU's copy of the current CR0 mask. */
2732	pVCpu->hm.s.vmx.cr0_mask = u64CR0Mask;
2733	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, u64CR0Mask);
2734	AssertRCReturn(rc, rc);
2735
2736	pVCpu->hm.s.fContextUseFlags &= ~(HM_CHANGED_GUEST_CR0 \| HM_CHANGED_GUEST_FPU);
2737	}
2738
2739	/*
2740	* Guest CR2.
2741	* It's always loaded late in the assembler code. Nothing to do here.
2742	*/
2743	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_CR2;
2744
2745	/*
2746	* Guest CR3.
2747	*/
2748	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR3)
2749	{
2750	uint64_t u64GuestCR3 = 0;
2751	if (pVM->hm.s.fNestedPaging)
2752	{
2753	pVCpu->hm.s.vmx.GCPhysEPTP = PGMGetHyperCR3(pVCpu);
2754
2755	/* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
2756	Assert(pVCpu->hm.s.vmx.GCPhysEPTP);
2757	Assert(!(pVCpu->hm.s.vmx.GCPhysEPTP & 0xfff0000000000000ULL));
2758	Assert(!(pVCpu->hm.s.vmx.GCPhysEPTP & 0xfff));
2759
2760	/* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
2761	pVCpu->hm.s.vmx.GCPhysEPTP \|= VMX_EPT_MEMTYPE_WB
2762	\| (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
2763
2764	/* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
2765	AssertMsg( ((pVCpu->hm.s.vmx.GCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
2766	&& ((pVCpu->hm.s.vmx.GCPhysEPTP >> 6) & 0x3f) == 0, /* Bits 6:11 MBZ. */
2767	("EPTP %#RX64\n", pVCpu->hm.s.vmx.GCPhysEPTP));
2768
2769	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVCpu->hm.s.vmx.GCPhysEPTP);
2770	AssertRCReturn(rc, rc);
2771	Log(("VMX_VMCS64_CTRL_EPTP_FULL=%#RX64\n", pVCpu->hm.s.vmx.GCPhysEPTP));
2772
2773	if ( CPUMIsGuestPagingEnabledEx(pCtx)
2774	\|\| pVM->hm.s.vmx.fUnrestrictedGuest)
2775	{
2776	/* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
2777	if (CPUMIsGuestInPAEModeEx(pCtx))
2778	{
2779	rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
2780	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u);
2781	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u);
2782	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u);
2783	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u);
2784	AssertRCReturn(rc, rc);
2785	}
2786
2787	/* The guest's view of its CR3 is unblemished with Nested Paging when the guest is using paging or we
2788	have Unrestricted Execution to handle the guest when it's not using paging. */
2789	u64GuestCR3 = pCtx->cr3;
2790	}
2791	else
2792	{
2793	/*
2794	* The guest is not using paging, but the CPU (VT-x) has to. While the guest thinks it accesses physical memory
2795	* directly, we use our identity-mapped page table to map guest-linear to guest-physical addresses.
2796	* EPT takes care of translating it to host-physical addresses.
2797	*/
2798	RTGCPHYS GCPhys;
2799	Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
2800	Assert(PDMVmmDevHeapIsEnabled(pVM));
2801
2802	/* We obtain it here every time as the guest could have relocated this PCI region. */
2803	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
2804	AssertRCReturn(rc, rc);
2805
2806	u64GuestCR3 = GCPhys;
2807	}
2808	}
2809	else
2810	{
2811	/* Non-nested paging case, just use the hypervisor's CR3. */
2812	u64GuestCR3 = PGMGetHyperCR3(pVCpu);
2813	}
2814
2815	Log2(("VMX_VMCS_GUEST_CR3=%#RX64\n", u64GuestCR3));
2816	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, u64GuestCR3);
2817	AssertRCReturn(rc, rc);
2818
2819	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_CR3;
2820	}
2821
2822	/*
2823	* Guest CR4.
2824	*/
2825	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR4)
2826	{
2827	uint64_t u64GuestCR4 = pCtx->cr4;
2828
2829	/* The guest's view of its CR4 is unblemished. */
2830	rc = VMXWriteVmcsGstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, u64GuestCR4);
2831	AssertRCReturn(rc, rc);
2832	Log2(("VMX_VMCS_CTRL_CR4_READ_SHADOW=%#RGv\n", u64GuestCR4));
2833
2834	/* Setup VT-x's view of the guest CR4. */
2835	/*
2836	* If we're emulating real-mode using virtual-8086 mode, we want to redirect software interrupts to the 8086 program
2837	* interrupt handler. Clear the VME bit (the interrupt redirection bitmap is already all 0, see hmR3InitFinalizeR0())
2838	* See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
2839	*/
2840	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
2841	{
2842	Assert(pVM->hm.s.vmx.pRealModeTSS);
2843	Assert(PDMVmmDevHeapIsEnabled(pVM));
2844	u64GuestCR4 &= ~X86_CR4_VME;
2845	}
2846
2847	if (pVM->hm.s.fNestedPaging)
2848	{
2849	if ( !CPUMIsGuestPagingEnabledEx(pCtx)
2850	&& !pVM->hm.s.vmx.fUnrestrictedGuest)
2851	{
2852	/* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
2853	u64GuestCR4 \|= X86_CR4_PSE;
2854	/* Our identity mapping is a 32 bits page directory. */
2855	u64GuestCR4 &= ~X86_CR4_PAE;
2856	}
2857	/* else use guest CR4.*/
2858	}
2859	else
2860	{
2861	/*
2862	* The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
2863	* paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
2864	*/
2865	switch (pVCpu->hm.s.enmShadowMode)
2866	{
2867	case PGMMODE_REAL: /* Real-mode. */
2868	case PGMMODE_PROTECTED: /* Protected mode without paging. */
2869	case PGMMODE_32_BIT: /* 32-bit paging. */
2870	{
2871	u64GuestCR4 &= ~X86_CR4_PAE;
2872	break;
2873	}
2874
2875	case PGMMODE_PAE: /* PAE paging. */
2876	case PGMMODE_PAE_NX: /* PAE paging with NX. */
2877	{
2878	u64GuestCR4 \|= X86_CR4_PAE;
2879	break;
2880	}
2881
2882	case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
2883	case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
2884	#ifdef VBOX_ENABLE_64_BITS_GUESTS
2885	break;
2886	#endif
2887	default:
2888	AssertFailed();
2889	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
2890	}
2891	}
2892
2893	/* We need to set and clear the CR4 specific bits here (mainly the X86_CR4_VMXE bit). */
2894	uint64_t uSetCR4 = (pVM->hm.s.vmx.msr.vmx_cr4_fixed0 & pVM->hm.s.vmx.msr.vmx_cr4_fixed1);
2895	uint64_t uZapCR4 = (pVM->hm.s.vmx.msr.vmx_cr4_fixed0 \| pVM->hm.s.vmx.msr.vmx_cr4_fixed1);
2896	u64GuestCR4 \|= uSetCR4;
2897	u64GuestCR4 &= uZapCR4;
2898
2899	/* Write VT-x's view of the guest CR4 into the VMCS. */
2900	Log2(("VMX_VMCS_GUEST_CR4=%#RGv (Set=%#RX32 Zap=%#RX32)\n", u64GuestCR4, uSetCR4, uZapCR4));
2901	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR4, u64GuestCR4);
2902
2903	/* Setup CR4 mask. CR4 flags owned by the host, if the guest attempts to change them, that would cause a VM exit. */
2904	uint64_t u64CR4Mask = 0;
2905	u64CR4Mask = X86_CR4_VME
2906	\| X86_CR4_PAE /** @todo should we intercept this bit with Nested Paging? */
2907	\| X86_CR4_PGE /** @todo why should we care if guest changes PGE bit or not with Nested Paging? */
2908	\| X86_CR4_PSE /** @todo do we care about page-size extensions in the Nested Paging case? */
2909	\| X86_CR4_VMXE;
2910	pVCpu->hm.s.vmx.cr4_mask = u64CR4Mask;
2911	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, u64CR4Mask);
2912	AssertRCReturn(rc, rc);
2913
2914	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_CR4;
2915	}
2916	return rc;
2917	}
2918
2919
2920	/**
2921	* Loads the guest debug registers into the guest-state area in the VMCS.
2922	* This also sets up whether #DB and MOV DRx accesses cause VM exits.
2923	*
2924	* @returns VBox status code.
2925	* @param pVM Pointer to the VM.
2926	* @param pVCpu Pointer to the VMCPU.
2927	* @param pCtx Pointer to the guest-CPU context.
2928	*
2929	* @remarks No-long-jump zone!!!
2930	*/
2931	DECLINLINE(int) hmR0VmxLoadGuestDebugRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
2932	{
2933	if (!(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_DEBUG))
2934	return VINF_SUCCESS;
2935
2936	/* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
2937	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_CONTROLS_LOAD_DEBUG)
2938	{
2939	Assert((pCtx->dr[7] & 0xffffffff00000000ULL) == 0); /* upper 32 bits are reserved (MBZ). */
2940	/* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
2941	Assert((pCtx->dr[7] & 0xd800) == 0); /* bits 15, 14, 12, 11 are reserved (MBZ). */
2942	Assert((pCtx->dr[7] & 0x400) == 0x400); /* bit 10 is reserved (MB1). */
2943	}
2944
2945	int rc = VERR_INTERNAL_ERROR_5;
2946	bool fInterceptDB = false;
2947	bool fInterceptMovDRx = false;
2948	if (DBGFIsStepping(pVCpu))
2949	{
2950	/* If the CPU supports the monitor trap flag, use it for single stepping in DBGF. */
2951	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_TRAP_FLAG)
2952	{
2953	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_TRAP_FLAG;
2954	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls);
2955	AssertRCReturn(rc, rc);
2956	Assert(fInterceptDB == false);
2957	}
2958	else if (pCtx->eflags.Bits.u1TF) /* If the guest is using its TF bit, we cannot single step in DBGF. */
2959	{
2960	Assert(fInterceptDB == false);
2961	/** @todo can we somehow signal DBGF that it cannot single-step instead of
2962	* just continuing? */
2963	}
2964	else
2965	fInterceptDB = true;
2966	}
2967	else
2968	Assert(fInterceptDB == false); /* If we are not single stepping in DBGF, there is no need to intercept #DB. */
2969
2970	/*
2971	* If the guest is using its DRx registers and the host DRx does not yet contain the guest DRx values,
2972	* load the guest DRx registers into the host and don't cause VM-exits on guest's MOV DRx accesses.
2973	* The same for the hypervisor DRx registers, priority is for the guest here.
2974	*/
2975	if ( (pCtx->dr[7] & (X86_DR7_ENABLED_MASK \| X86_DR7_GD))
2976	&& !CPUMIsGuestDebugStateActive(pVCpu))
2977	{
2978	/* Save the host and load the guest debug registers. This will make the guest debug state active. */
2979	rc = CPUMR0LoadGuestDebugState(pVM, pVCpu, pCtx, true /* include DR6 */);
2980	AssertRCReturn(rc, rc);
2981	Assert(CPUMIsGuestDebugStateActive(pVCpu));
2982	Assert(fInterceptMovDRx == false);
2983	}
2984	else if ( CPUMGetHyperDR7(pVCpu) & (X86_DR7_ENABLED_MASK \| X86_DR7_GD)
2985	&& !CPUMIsHyperDebugStateActive(pVCpu))
2986	{
2987	/* Save the host and load the hypervisor debug registers. This will make the hyper debug state active. */
2988	rc = CPUMR0LoadHyperDebugState(pVM, pVCpu, pCtx, true /* include DR6 */);
2989	AssertRCReturn(rc, rc);
2990	Assert(CPUMIsHyperDebugStateActive(pVCpu));
2991	fInterceptMovDRx = true;
2992	}
2993	else
2994	Assert(fInterceptMovDRx == false); /* No need to intercept MOV DRx if DBGF is not active nor the guest is debugging. */
2995
2996	/* Update the exception bitmap regarding intercepting #DB generated by the guest. */
2997	if (fInterceptDB)
2998	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_DB);
2999	else if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3000	{
3001	#ifndef VBOX_ALWAYS_TRAP_ALL_EXCEPTIONS
3002	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_DB);
3003	#endif
3004	}
3005
3006	/* Update the processor-based VM-execution controls regarding intercepting MOV DRx instructions. */
3007	if (fInterceptMovDRx)
3008	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT;
3009	else
3010	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT;
3011
3012	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
3013	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls);
3014
3015	/* The guest's view of its DR7 is unblemished. */
3016	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_DR7, pCtx->dr[7]);
3017
3018	/* Setup other debug controls */
3019	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, 0); /** @todo think about this. */
3020	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0); /** @todo Intel spec. 26.6.3 think about this */
3021	AssertRCReturn(rc, rc);
3022
3023	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_DEBUG;
3024	return rc;
3025	}
3026
3027
3028	#ifdef DEBUG
3029	/**
3030	* Debug function to validate segment registers.
3031	*/
3032	static void hmR0VmxValidateSegmentRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
3033	{
3034	/* Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers". */
3035	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
3036	&& ( !CPUMIsGuestInRealModeEx(pCtx)
3037	&& !CPUMIsGuestInV86ModeEx(pCtx)))
3038	{
3039	/** @todo DPL checks for CS, SS. */
3040	/* Protected mode checks */
3041	/* CS */
3042	Assert(pCtx->cs.Attr.n.u1Present);
3043	Assert(!(pCtx->cs.Attr.u & 0xf00));
3044	Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
3045	Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
3046	\|\| !(pCtx->cs.Attr.n.u1Granularity));
3047	Assert( !(pCtx->cs.u32Limit & 0xfff00000)
3048	\|\| (pCtx->cs.Attr.n.u1Granularity));
3049	Assert(pCtx->cs.Attr.u && pCtx->cs.Attr.u != VMX_SEL_UNUSABLE); /* CS cannot be loaded with NULL in protected mode. */
3050	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
3051	Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
3052	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
3053	Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
3054	else
3055	AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
3056	/* SS */
3057	if (pCtx->ss.Attr.u && pCtx->ss.Attr.u != VMX_SEL_UNUSABLE)
3058	{
3059	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
3060	Assert(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7);
3061	Assert(pCtx->ss.Attr.n.u1Present);
3062	Assert(!(pCtx->ss.Attr.u & 0xf00));
3063	Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
3064	Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
3065	\|\| !(pCtx->ss.Attr.n.u1Granularity));
3066	Assert( !(pCtx->ss.u32Limit & 0xfff00000)
3067	\|\| (pCtx->ss.Attr.n.u1Granularity));
3068	}
3069	Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
3070	/* CR0 might not be up-to-date here always, hence disabled. */
3071	#if 0
3072	if (!pCtx->cr0 & X86_CR0_PE)
3073	Assert(!pCtx->ss.Attr.n.u2Dpl);
3074	#endif
3075	/* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
3076	if (pCtx->ds.Attr.u && pCtx->ds.Attr.u != VMX_SEL_UNUSABLE)
3077	{
3078	Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3079	Assert(pCtx->ds.Attr.n.u1Present);
3080	Assert(pCtx->ds.Attr.n.u4Type > 11 \|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
3081	Assert(!(pCtx->ds.Attr.u & 0xf00));
3082	Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
3083	Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
3084	\|\| !(pCtx->ds.Attr.n.u1Granularity));
3085	Assert( !(pCtx->ds.u32Limit & 0xfff00000)
3086	\|\| (pCtx->ds.Attr.n.u1Granularity));
3087	Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3088	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
3089	}
3090	if (pCtx->es.Attr.u && pCtx->es.Attr.u != VMX_SEL_UNUSABLE)
3091	{
3092	Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3093	Assert(pCtx->es.Attr.n.u1Present);
3094	Assert(pCtx->es.Attr.n.u4Type > 11 \|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
3095	Assert(!(pCtx->es.Attr.u & 0xf00));
3096	Assert(!(pCtx->es.Attr.u & 0xfffe0000));
3097	Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
3098	\|\| !(pCtx->es.Attr.n.u1Granularity));
3099	Assert( !(pCtx->es.u32Limit & 0xfff00000)
3100	\|\| (pCtx->es.Attr.n.u1Granularity));
3101	Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3102	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
3103	}
3104	if (pCtx->fs.Attr.u && pCtx->fs.Attr.u != VMX_SEL_UNUSABLE)
3105	{
3106	Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3107	Assert(pCtx->fs.Attr.n.u1Present);
3108	Assert(pCtx->fs.Attr.n.u4Type > 11 \|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
3109	Assert(!(pCtx->fs.Attr.u & 0xf00));
3110	Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
3111	Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
3112	\|\| !(pCtx->fs.Attr.n.u1Granularity));
3113	Assert( !(pCtx->fs.u32Limit & 0xfff00000)
3114	\|\| (pCtx->fs.Attr.n.u1Granularity));
3115	Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3116	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
3117	}
3118	if (pCtx->gs.Attr.u && pCtx->gs.Attr.u != VMX_SEL_UNUSABLE)
3119	{
3120	Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3121	Assert(pCtx->gs.Attr.n.u1Present);
3122	Assert(pCtx->gs.Attr.n.u4Type > 11 \|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
3123	Assert(!(pCtx->gs.Attr.u & 0xf00));
3124	Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
3125	Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
3126	\|\| !(pCtx->gs.Attr.n.u1Granularity));
3127	Assert( !(pCtx->gs.u32Limit & 0xfff00000)
3128	\|\| (pCtx->gs.Attr.n.u1Granularity));
3129	Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3130	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
3131	}
3132	/* 64-bit capable CPUs. */
3133	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3134	Assert(!(pCtx->cs.u64Base >> 32));
3135	Assert(!pCtx->ss.Attr.u \|\| !(pCtx->ss.u64Base >> 32));
3136	Assert(!pCtx->ds.Attr.u \|\| !(pCtx->ds.u64Base >> 32));
3137	Assert(!pCtx->es.Attr.u \|\| !(pCtx->es.u64Base >> 32));
3138	# endif
3139	}
3140	else if ( CPUMIsGuestInV86ModeEx(pCtx)
3141	\|\| ( CPUMIsGuestInRealModeEx(pCtx)
3142	&& !pVM->hm.s.vmx.fUnrestrictedGuest))
3143	{
3144	/* Real and v86 mode checks. */
3145	/* hmR0VmxWriteSegmentReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
3146	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
3147	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3148	{
3149	u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3; u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
3150	}
3151	else
3152	{
3153	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
3154	u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
3155	}
3156
3157	/* CS */
3158	AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
3159	Assert(pCtx->cs.u32Limit == 0xffff);
3160	Assert(u32CSAttr == 0xf3);
3161	/* SS */
3162	Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
3163	Assert(pCtx->ss.u32Limit == 0xffff);
3164	Assert(u32SSAttr == 0xf3);
3165	/* DS */
3166	Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
3167	Assert(pCtx->ds.u32Limit == 0xffff);
3168	Assert(u32DSAttr == 0xf3);
3169	/* ES */
3170	Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
3171	Assert(pCtx->es.u32Limit == 0xffff);
3172	Assert(u32ESAttr == 0xf3);
3173	/* FS */
3174	Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
3175	Assert(pCtx->fs.u32Limit == 0xffff);
3176	Assert(u32FSAttr == 0xf3);
3177	/* GS */
3178	Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
3179	Assert(pCtx->gs.u32Limit == 0xffff);
3180	Assert(u32GSAttr == 0xf3);
3181	/* 64-bit capable CPUs. */
3182	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3183	Assert(!(pCtx->cs.u64Base >> 32));
3184	Assert(!u32SSAttr \|\| !(pCtx->ss.u64Base >> 32));
3185	Assert(!u32DSAttr \|\| !(pCtx->ds.u64Base >> 32));
3186	Assert(!u32ESAttr \|\| !(pCtx->es.u64Base >> 32));
3187	# endif
3188	}
3189	}
3190	#endif /* DEBUG */
3191
3192
3193	/**
3194	* Writes a guest segment register into the guest-state area in the VMCS.
3195	*
3196	* @returns VBox status code.
3197	* @param pVM Pointer to the VM.
3198	* @param pVCpu Pointer to the VMCPU.
3199	* @param idxSel Index of the selector in the VMCS.
3200	* @param idxLimit Index of the segment limit in the VMCS.
3201	* @param idxBase Index of the segment base in the VMCS.
3202	* @param idxAccess Index of the access rights of the segment in the VMCS.
3203	* @param pSelReg Pointer to the segment selector.
3204	* @param pCtx Pointer to the guest-CPU context.
3205	*
3206	* @remarks No-long-jump zone!!!
3207	*/
3208	DECLINLINE(int) hmR0VmxWriteSegmentReg(PVM pVM, PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase,
3209	uint32_t idxAccess, PCPUMSELREG pSelReg, PCPUMCTX pCtx)
3210	{
3211	int rc;
3212	rc = VMXWriteVmcs32(idxSel, pSelReg->Sel); /* 16-bit guest selector field. */
3213	rc \|= VMXWriteVmcs32(idxLimit, pSelReg->u32Limit); /* 32-bit guest segment limit field. */
3214	rc \|= VMXWriteVmcsGstN(idxBase, pSelReg->u64Base); /* Natural width guest segment base field.*/
3215	AssertRCReturn(rc, rc);
3216
3217	uint32_t u32Access = pSelReg->Attr.u;
3218	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3219	{
3220	/* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
3221	u32Access = 0xf3;
3222	Assert(pVM->hm.s.vmx.pRealModeTSS);
3223	Assert(PDMVmmDevHeapIsEnabled(pVM));
3224	}
3225	else
3226	{
3227	/*
3228	* The way to differentiate between whether this is really a null selector or was just a selector loaded with 0 in
3229	* real-mode is using the segment attributes. A selector loaded in real-mode with the value 0 is valid and usable in
3230	* protected-mode and we should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures NULL selectors
3231	* loaded in protected-mode have their attribute as 0.
3232	*/
3233	if (!u32Access)
3234	u32Access = VMX_SEL_UNUSABLE;
3235	}
3236
3237	/* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
3238	AssertMsg((u32Access == VMX_SEL_UNUSABLE) \|\| (u32Access & X86_SEL_TYPE_ACCESSED),
3239	("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
3240
3241	rc = VMXWriteVmcs32(idxAccess, u32Access); /* 32-bit guest segment access-rights field. */
3242	AssertRCReturn(rc, rc);
3243	return rc;
3244	}
3245
3246
3247	/**
3248	* Loads the guest segment registers, GDTR, IDTR, LDTR, (TR, FS and GS bases)
3249	* into the guest-state area in the VMCS.
3250	*
3251	* @returns VBox status code.
3252	* @param pVM Pointer to the VM.
3253	* @param pVCPU Pointer to the VMCPU.
3254	* @param pCtx Pointer to the guest-CPU context.
3255	*
3256	* @remarks No-long-jump zone!!!
3257	* @remarks Requires RFLAGS (for debug assertions).
3258	*/
3259	DECLINLINE(int) hmR0VmxLoadGuestSegmentRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
3260	{
3261	int rc = VERR_INTERNAL_ERROR_5;
3262
3263	/*
3264	* Guest Segment registers: CS, SS, DS, ES, FS, GS.
3265	*/
3266	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SEGMENT_REGS)
3267	{
3268	/* Save the segment attributes for real-on-v86 mode hack, so we can restore them on VM-exit. */
3269	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3270	{
3271	pVCpu->hm.s.vmx.RealMode.uAttrCS.u = pCtx->cs.Attr.u;
3272	pVCpu->hm.s.vmx.RealMode.uAttrSS.u = pCtx->ss.Attr.u;
3273	pVCpu->hm.s.vmx.RealMode.uAttrDS.u = pCtx->ds.Attr.u;
3274	pVCpu->hm.s.vmx.RealMode.uAttrES.u = pCtx->es.Attr.u;
3275	pVCpu->hm.s.vmx.RealMode.uAttrFS.u = pCtx->fs.Attr.u;
3276	pVCpu->hm.s.vmx.RealMode.uAttrGS.u = pCtx->gs.Attr.u;
3277	}
3278
3279	#ifdef VBOX_WITH_REM
3280	if (!pVM->hm.s.vmx.fUnrestrictedGuest)
3281	{
3282	Assert(pVM->hm.s.vmx.pRealModeTSS);
3283	PGMMODE enmGuestMode = PGMGetGuestMode(pVCpu);
3284	if (pVCpu->hm.s.vmx.enmLastSeenGuestMode != enmGuestMode)
3285	{
3286	AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
3287	if ( pVCpu->hm.s.vmx.enmLastSeenGuestMode == PGMMODE_REAL
3288	&& enmGuestMode >= PGMMODE_PROTECTED)
3289	{
3290	/* Signal that recompiler must flush its code-cache as the guest -may- rewrite code it will later execute
3291	in real-mode (e.g. OpenBSD 4.0) */
3292	REMFlushTBs(pVM);
3293	Log2(("Switch to protected mode detected!\n"));
3294	}
3295	pVCpu->hm.s.vmx.enmLastSeenGuestMode = enmGuestMode;
3296	}
3297	}
3298	#endif
3299	rc = hmR0VmxWriteSegmentReg(pVM, pVCpu, VMX_VMCS16_GUEST_FIELD_CS, VMX_VMCS32_GUEST_CS_LIMIT, VMX_VMCS_GUEST_CS_BASE,
3300	VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS, &pCtx->cs, pCtx);
3301	rc \|= hmR0VmxWriteSegmentReg(pVM, pVCpu, VMX_VMCS16_GUEST_FIELD_SS, VMX_VMCS32_GUEST_SS_LIMIT, VMX_VMCS_GUEST_SS_BASE,
3302	VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS, &pCtx->ss, pCtx);
3303	rc \|= hmR0VmxWriteSegmentReg(pVM, pVCpu, VMX_VMCS16_GUEST_FIELD_DS, VMX_VMCS32_GUEST_DS_LIMIT, VMX_VMCS_GUEST_DS_BASE,
3304	VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS, &pCtx->ds, pCtx);
3305	rc \|= hmR0VmxWriteSegmentReg(pVM, pVCpu, VMX_VMCS16_GUEST_FIELD_ES, VMX_VMCS32_GUEST_ES_LIMIT, VMX_VMCS_GUEST_ES_BASE,
3306	VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS, &pCtx->es, pCtx);
3307	rc \|= hmR0VmxWriteSegmentReg(pVM, pVCpu, VMX_VMCS16_GUEST_FIELD_FS, VMX_VMCS32_GUEST_FS_LIMIT, VMX_VMCS_GUEST_FS_BASE,
3308	VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS, &pCtx->fs, pCtx);
3309	rc \|= hmR0VmxWriteSegmentReg(pVM, pVCpu, VMX_VMCS16_GUEST_FIELD_GS, VMX_VMCS32_GUEST_GS_LIMIT, VMX_VMCS_GUEST_GS_BASE,
3310	VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS, &pCtx->gs, pCtx);
3311	AssertRCReturn(rc, rc);
3312
3313	#ifdef DEBUG
3314	hmR0VmxValidateSegmentRegs(pVM, pVCpu, pCtx);
3315	#endif
3316	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SEGMENT_REGS;
3317	}
3318
3319	/*
3320	* Guest TR.
3321	*/
3322	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_TR)
3323	{
3324	/*
3325	* Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is achieved
3326	* using the interrupt redirection bitmap (all bits cleared to let the guest handle INT-n's) in the TSS.
3327	* See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
3328	*/
3329	uint16_t u16Sel = 0;
3330	uint32_t u32Limit = 0;
3331	uint64_t u64Base = 0;
3332	uint32_t u32AccessRights = 0;
3333
3334	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3335	{
3336	u16Sel = pCtx->tr.Sel;
3337	u32Limit = pCtx->tr.u32Limit;
3338	u64Base = pCtx->tr.u64Base;
3339	u32AccessRights = pCtx->tr.Attr.u;
3340	}
3341	else
3342	{
3343	Assert(pVM->hm.s.vmx.pRealModeTSS);
3344	Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMR3CanExecuteGuest() -XXX- what about inner loop changes? */
3345
3346	/* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
3347	RTGCPHYS GCPhys;
3348	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
3349	AssertRCReturn(rc, rc);
3350
3351	X86DESCATTR DescAttr;
3352	DescAttr.u = 0;
3353	DescAttr.n.u1Present = 1;
3354	DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
3355
3356	u16Sel = 0;
3357	u32Limit = HM_VTX_TSS_SIZE;
3358	u64Base = GCPhys; /* in real-mode phys = virt. */
3359	u32AccessRights = DescAttr.u;
3360	}
3361
3362	/* Validate. */
3363	Assert(!(u16Sel & RT_BIT(2)));
3364	AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
3365	\|\| (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
3366	AssertMsg(!(u32AccessRights & VMX_SEL_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
3367	Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
3368	Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
3369	Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
3370	Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
3371	Assert( (u32Limit & 0xfff) == 0xfff
3372	\|\| !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
3373	Assert( !(pCtx->tr.u32Limit & 0xfff00000)
3374	\|\| (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
3375
3376	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_TR, u16Sel);
3377	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit);
3378	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base);
3379	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights);
3380	AssertRCReturn(rc, rc);
3381
3382	Log2(("VMX_VMCS_GUEST_TR_BASE=%#RX64\n", u64Base));
3383	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_TR;
3384	}
3385
3386	/*
3387	* Guest GDTR.
3388	*/
3389	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_GDTR)
3390	{
3391	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pCtx->gdtr.cbGdt);
3392	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pCtx->gdtr.pGdt);
3393	AssertRCReturn(rc, rc);
3394
3395	Assert(!(pCtx->gdtr.cbGdt & 0xffff0000ULL)); /* Bits 31:16 MBZ. */
3396	Log2(("VMX_VMCS_GUEST_GDTR_BASE=%#RX64\n", pCtx->gdtr.pGdt));
3397	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_GDTR;
3398	}
3399
3400	/*
3401	* Guest LDTR.
3402	*/
3403	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_LDTR)
3404	{
3405	/* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
3406	uint32_t u32Access = 0;
3407	if (!(pCtx->ldtr.Attr.u & VMX_SEL_UNUSABLE))
3408	u32Access = VMX_SEL_UNUSABLE;
3409	else
3410	u32Access = pCtx->ldtr.Attr.u;
3411
3412	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_LDTR, pCtx->ldtr.Sel);
3413	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pCtx->ldtr.u32Limit);
3414	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pCtx->ldtr.u64Base);
3415	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access);
3416	AssertRCReturn(rc, rc);
3417
3418	/* Validate. */
3419	if (!(u32Access & VMX_SEL_UNUSABLE))
3420	{
3421	Assert(!(pCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
3422	Assert(pCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
3423	Assert(!pCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
3424	Assert(pCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
3425	Assert(!pCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
3426	Assert(!(pCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
3427	Assert( (pCtx->ldtr.u32Limit & 0xfff) == 0xfff
3428	\|\| !pCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
3429	Assert( !(pCtx->ldtr.u32Limit & 0xfff00000)
3430	\|\| pCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
3431	}
3432
3433	Log2(("VMX_VMCS_GUEST_LDTR_BASE=%#RX64\n", pCtx->ldtr.u64Base));
3434	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_LDTR;
3435	}
3436
3437	/*
3438	* Guest IDTR.
3439	*/
3440	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_IDTR)
3441	{
3442	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pCtx->idtr.cbIdt);
3443	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pCtx->idtr.pIdt);
3444	AssertRCReturn(rc, rc);
3445
3446	Assert(!(pCtx->idtr.cbIdt & 0xffff0000ULL)); /* Bits 31:16 MBZ. */
3447	Log2(("VMX_VMCS_GUEST_IDTR_BASE=%#RX64\n", pCtx->idtr.pIdt));
3448	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_IDTR;
3449	}
3450
3451	/*
3452	* Guest FS & GS base MSRs.
3453	* We already initialized the FS & GS base as part of the guest segment registers, but the guest's FS/GS base
3454	* MSRs might have changed (e.g. due to WRMSR) and we need to update the bases if that happened. These MSRs
3455	* are only available in 64-bit mode.
3456	*/
3457	/** @todo Avoid duplication of this code in assembly (see MYPUSHSEGS) - it
3458	* should not be necessary to do it in assembly again. */
3459	if (CPUMIsGuestInLongModeEx(pCtx))
3460	{
3461	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_FS_BASE_MSR)
3462	{
3463	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_FS_BASE, pCtx->fs.u64Base);
3464	AssertRCReturn(rc, rc);
3465	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_FS_BASE_MSR;
3466	}
3467
3468	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_GS_BASE_MSR)
3469	{
3470	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_GS_BASE, pCtx->gs.u64Base);
3471	AssertRCReturn(rc, rc);
3472	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_GS_BASE_MSR;
3473	}
3474	}
3475	else
3476	pVCpu->hm.s.fContextUseFlags &= ~(HM_CHANGED_GUEST_FS_BASE_MSR \| HM_CHANGED_GUEST_GS_BASE_MSR);
3477
3478	return VINF_SUCCESS;
3479	}
3480
3481
3482	/**
3483	* Loads certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
3484	* areas. These MSRs will automatically be loaded to the host CPU on every
3485	* successful VM entry and stored from the host CPU on every successful VM exit.
3486	* Also loads the sysenter MSRs into the guest-state area in the VMCS.
3487	*
3488	* @returns VBox status code.
3489	* @param pVM Pointer to the VM.
3490	* @param pVCpu Pointer to the VMCPU.
3491	* @param pCtx Pointer to the guest-CPU context.
3492	*
3493	* @remarks No-long-jump zone!!!
3494	*/
3495	DECLINLINE(int) hmR0VmxLoadGuestMsrs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
3496	{
3497	AssertPtr(pVCpu);
3498	AssertPtr(pVCpu->hm.s.vmx.pvGuestMsr);
3499
3500	/*
3501	* MSRs covered by Auto-load/store: EFER, LSTAR, STAR, SF_MASK, TSC_AUX (RDTSCP).
3502	*/
3503	int rc = VINF_SUCCESS;
3504	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_GUEST_AUTO_MSRS)
3505	{
3506	PVMXMSR pGuestMsr = (PVMXMSR)pVCpu->hm.s.vmx.pvGuestMsr;
3507	unsigned cGuestMsrs = 0;
3508
3509	/* See Intel spec. 4.1.4 "Enumeration of Paging Features by CPUID". */
3510	const bool fSupportsNX = CPUMGetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_NX);
3511	const bool fSupportsLongMode = CPUMGetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_LONG_MODE);
3512	if (fSupportsNX \|\| fSupportsLongMode)
3513	{
3514	/** @todo support save IA32_EFER, i.e.
3515	* VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_GUEST_EFER_MSR, in which case the
3516	* guest EFER need not be part of the VM-entry MSR-load area. */
3517	pGuestMsr->u32IndexMSR = MSR_K6_EFER;
3518	pGuestMsr->u32Reserved = 0;
3519	pGuestMsr->u64Value = pCtx->msrEFER;
3520	/* VT-x will complain if only MSR_K6_EFER_LME is set. See Intel spec. 26.4 "Loading MSRs" for details. */
3521	if (!CPUMIsGuestInLongModeEx(pCtx))
3522	pGuestMsr->u64Value &= ~(MSR_K6_EFER_LMA \| MSR_K6_EFER_LME);
3523	pGuestMsr++; cGuestMsrs++;
3524	if (fSupportsLongMode)
3525	{
3526	pGuestMsr->u32IndexMSR = MSR_K8_LSTAR;
3527	pGuestMsr->u32Reserved = 0;
3528	pGuestMsr->u64Value = pCtx->msrLSTAR; /* 64 bits mode syscall rip */
3529	pGuestMsr++; cGuestMsrs++;
3530	pGuestMsr->u32IndexMSR = MSR_K6_STAR;
3531	pGuestMsr->u32Reserved = 0;
3532	pGuestMsr->u64Value = pCtx->msrSTAR; /* legacy syscall eip, cs & ss */
3533	pGuestMsr++; cGuestMsrs++;
3534	pGuestMsr->u32IndexMSR = MSR_K8_SF_MASK;
3535	pGuestMsr->u32Reserved = 0;
3536	pGuestMsr->u64Value = pCtx->msrSFMASK; /* syscall flag mask */
3537	pGuestMsr++; cGuestMsrs++;
3538	/* The KERNEL_GS_BASE MSR doesn't work reliably with auto load/store. See @bugref{6208} */
3539	#if 0
3540	pGuestMsr->u32IndexMSR = MSR_K8_KERNEL_GS_BASE;
3541	pGuestMsr->u32Reserved = 0;
3542	pGuestMsr->u64Value = pCtx->msrKERNELGSBASE; /* swapgs exchange value */
3543	pGuestMsr++; cGuestMsrs++;
3544	#endif
3545	}
3546	}
3547
3548	/*
3549	* RDTSCP requires the TSC_AUX MSR. Host and guest share the physical MSR. So we have to
3550	* load the guest's copy if the guest can execute RDTSCP without causing VM-exits.
3551	*/
3552	if ( CPUMGetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_RDTSCP)
3553	&& (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP))
3554	{
3555	pGuestMsr->u32IndexMSR = MSR_K8_TSC_AUX;
3556	pGuestMsr->u32Reserved = 0;
3557	rc = CPUMQueryGuestMsr(pVCpu, MSR_K8_TSC_AUX, &pGuestMsr->u64Value);
3558	AssertRCReturn(rc, rc);
3559	pGuestMsr++; cGuestMsrs++;
3560	}
3561
3562	/* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
3563	if (cGuestMsrs > MSR_IA32_VMX_MISC_MAX_MSR(pVM->hm.s.vmx.msr.vmx_misc))
3564	{
3565	LogRel(("CPU autoload/store MSR count in VMCS exceeded cGuestMsrs=%u.\n", cGuestMsrs));
3566	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3567	}
3568
3569	/* Update the VCPU's copy of the guest MSR count. */
3570	pVCpu->hm.s.vmx.cGuestMsrs = cGuestMsrs;
3571	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cGuestMsrs);
3572	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cGuestMsrs);
3573	AssertRCReturn(rc, rc);
3574
3575	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_GUEST_AUTO_MSRS;
3576	}
3577
3578	/*
3579	* Guest Sysenter MSRs.
3580	* These flags are only set when MSR-bitmaps are not supported by the CPU and we cause
3581	* VM exits on WRMSRs for these MSRs.
3582	*/
3583	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SYSENTER_CS_MSR)
3584	{
3585	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pCtx->SysEnter.cs);
3586	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SYSENTER_CS_MSR;
3587	}
3588	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SYSENTER_EIP_MSR)
3589	{
3590	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pCtx->SysEnter.eip);
3591	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SYSENTER_EIP_MSR;
3592	}
3593	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SYSENTER_ESP_MSR)
3594	{
3595	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pCtx->SysEnter.esp);
3596	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SYSENTER_ESP_MSR;
3597	}
3598	AssertRCReturn(rc, rc);
3599
3600	return rc;
3601	}
3602
3603
3604	/**
3605	* Loads the guest activity state into the guest-state area in the VMCS.
3606	*
3607	* @returns VBox status code.
3608	* @param pVM Pointer to the VM.
3609	* @param pVCpu Pointer to the VMCPU.
3610	* @param pCtx Pointer to the guest-CPU context.
3611	*
3612	* @remarks No-long-jump zone!!!
3613	*/
3614	DECLINLINE(int) hmR0VmxLoadGuestActivityState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
3615	{
3616	/** @todo See if we can make use of other states, e.g.
3617	* VMX_VMCS_GUEST_ACTIVITY_SHUTDOWN or HLT. */
3618	int rc = VINF_SUCCESS;
3619	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_GUEST_ACTIVITY_STATE)
3620	{
3621	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, VMX_VMCS_GUEST_ACTIVITY_ACTIVE);
3622	AssertRCReturn(rc, rc);
3623	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_GUEST_ACTIVITY_STATE;
3624	}
3625	return rc;
3626	}
3627
3628
3629	/**
3630	* Loads the guest-interruptibility state (or "interrupt shadow" as AMD calls
3631	* it) into the guest-state area in the VMCS.
3632	*
3633	* @returns VBox status code.
3634	* @param pVM Pointer to the VM.
3635	* @param pVCpu Pointer to the VMCPU.
3636	* @param pCtx Pointer to the guest-CPU context.
3637	*
3638	* @remarks No-long-jump zone!!!
3639	* @remarks Requires RIP, RFLAGS (debug assert).
3640	*/
3641	DECLINLINE(int) hmR0VmxLoadGuestIntrState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
3642	{
3643	if (!(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_INTR_STATE))
3644	return VINF_SUCCESS;
3645
3646	/*
3647	* Instructions like STI and MOV SS inhibit interrupts till the next instruction completes. Check if we should
3648	* inhibit interrupts or clear any existing interrupt-inhibition.
3649	*/
3650	uint32_t uIntrState = 0;
3651	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3652	{
3653	if (pCtx->rip != EMGetInhibitInterruptsPC(pVCpu))
3654	{
3655	/*
3656	* We can clear the inhibit force flag as even if we go back to the recompiler without executing guest code in VT-x
3657	* the flag's condition to be cleared is met and thus the cleared state is correct. Additionally, this means
3658	* we need not re-read the VMCS field on the VM-exit path and clear/set this flag on every VM-exit. Finally,
3659	* hmR0VmxInjectPendingInterrupt() relies on us clearing this flag here.
3660	*/
3661	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
3662	uIntrState = 0; /* Clear interrupt inhibition. */
3663	}
3664	else if (pCtx->eflags.u32 & X86_EFL_IF)
3665	{
3666	/*
3667	* We don't have enough information to distinguish a block-by-STI vs. block-by-MOV SS. Intel seems to think there
3668	* is a slight difference regarding MOV SS additionally blocking some debug exceptions.
3669	* See Intel spec. 24.2.2 "Guest Non-Register State" table "Format of Interruptibility State".
3670	*/
3671	uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
3672	}
3673	}
3674	else
3675	uIntrState = 0; /* No interrupt inhibition. */
3676
3677	Assert((uIntrState & 0x3) != 0x3); /* Block-by-STI and MOV SS cannot be simultaneously set. */
3678	Assert((pCtx->eflags.u32 & X86_EFL_IF) \|\| uIntrState == 0); /* If EFLAGS.IF is not set, no interrupt inhibition. */
3679	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, uIntrState);
3680	AssertRCReturn(rc ,rc);
3681	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_INTR_STATE;
3682	return rc;
3683	}
3684
3685
3686	/**
3687	* Sets up the appropriate function to run guest code.
3688	*
3689	* @returns VBox status code.
3690	* @param pVM Pointer to the VM.
3691	* @param pVCpu Pointer to the VMCPU.
3692	* @param pCtx Pointer to the guest-CPU context.
3693	*
3694	* @remarks No-long-jump zone!!!
3695	* @todo change this to return void.
3696	*/
3697	DECLINLINE(int) hmR0VmxSetupVMRunHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
3698	{
3699	if (CPUMIsGuestInLongModeEx(pCtx))
3700	{
3701	#ifndef VBOX_ENABLE_64_BITS_GUESTS
3702	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
3703	#endif
3704	Assert(pVM->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
3705	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3706	/* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
3707	pVCpu->hm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
3708	#else
3709	/* 64-bit host or hybrid host. */
3710	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM64;
3711	#endif
3712	}
3713	else
3714	{
3715	/* Guest is not in long mode, use the 32-bit handler. */
3716	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
3717	}
3718	Assert(pVCpu->hm.s.vmx.pfnStartVM);
3719	return VINF_SUCCESS;
3720	}
3721
3722
3723	/**
3724	* Wrapper for running the guest code in VT-x.
3725	*
3726	* @returns VBox strict status code.
3727	* @param pVM Pointer to the VM.
3728	* @param pVCpu Pointer to the VMCPU.
3729	* @param pCtx Pointer to the guest-CPU context.
3730	*
3731	* @remarks No-long-jump zone!!!
3732	*/
3733	DECLINLINE(int) hmR0VmxRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
3734	{
3735	/*
3736	* 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses floating-point operations
3737	* using SSE instructions. Some XMM registers (XMM6-XMM15) are callee-saved and thus the need for this XMM wrapper.
3738	* Refer MSDN docs. "Configuring Programs for 64-bit / x64 Software Conventions / Register Usage" for details.
3739	*/
3740	#ifdef VBOX_WITH_KERNEL_USING_XMM
3741	return hmR0VMXStartVMWrapXMM(pVCpu->hm.s.fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hm.s.vmx.pfnStartVM);
3742	#else
3743	return pVCpu->hm.s.vmx.pfnStartVM(pVCpu->hm.s.fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu);
3744	#endif
3745	}
3746
3747
3748	/**
3749	* Report world-switch error and dump some useful debug info.
3750	*
3751	* @param pVM Pointer to the VM.
3752	* @param pVCpu Pointer to the VMCPU.
3753	* @param rcVMRun The return code from VMLAUNCH/VMRESUME.
3754	* @param pCtx Pointer to the guest-CPU context.
3755	* @param pVmxTransient Pointer to the VMX transient structure (only
3756	* exitReason updated).
3757	*/
3758	static void hmR0VmxReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, int rcVMRun, PCPUMCTX pCtx, PVMXTRANSIENT pVmxTransient)
3759	{
3760	Assert(pVM);
3761	Assert(pVCpu);
3762	Assert(pCtx);
3763	Assert(pVmxTransient);
3764	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
3765
3766	Log(("VM-entry failure: %Rrc\n", rcVMRun));
3767	switch (rcVMRun)
3768	{
3769	case VERR_VMX_INVALID_VMXON_PTR:
3770	AssertFailed();
3771	break;
3772	case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
3773	case VERR_VMX_UNABLE_TO_START_VM:
3774	case VERR_VMX_UNABLE_TO_RESUME_VM:
3775	{
3776	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.lasterror.u32ExitReason);
3777	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.lasterror.u32InstrError);
3778	rc \|= hmR0VmxReadExitQualificationVmcs(pVmxTransient);
3779	AssertRC(rc);
3780
3781	#ifdef VBOX_STRICT
3782	Log(("uExitReason %#x (VmxTransient %#x)\n", pVCpu->hm.s.vmx.lasterror.u32ExitReason,
3783	pVmxTransient->uExitReason));
3784	Log(("Exit Qualification %#x\n", pVmxTransient->uExitQualification));
3785	Log(("InstrError %#x\n", pVCpu->hm.s.vmx.lasterror.u32InstrError));
3786	if (pVCpu->hm.s.vmx.lasterror.u32InstrError <= VMX_INSTR_ERROR_MAX)
3787	Log(("InstrError Desc. \"%s\"\n", s_apszVmxInstrErrors[pVCpu->hm.s.vmx.lasterror.u32InstrError]));
3788	else
3789	Log(("InstrError Desc. Range exceeded %u\n", VMX_INSTR_ERROR_MAX));
3790
3791	/* VMX control bits. */
3792	uint32_t u32Val;
3793	uint64_t u64Val;
3794	RTHCUINTREG uHCReg;
3795	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC_CONTROLS, &u32Val); AssertRC(rc);
3796	Log(("VMX_VMCS32_CTRL_PIN_EXEC_CONTROLS %#RX32\n", u32Val));
3797	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, &u32Val); AssertRC(rc);
3798	Log(("VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS %#RX32\n", u32Val));
3799	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS2, &u32Val); AssertRC(rc);
3800	Log(("VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS2 %#RX32\n", u32Val));
3801	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_CONTROLS, &u32Val); AssertRC(rc);
3802	Log(("VMX_VMCS32_CTRL_ENTRY_CONTROLS %#RX32\n", u32Val));
3803	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_CONTROLS, &u32Val); AssertRC(rc);
3804	Log(("VMX_VMCS32_CTRL_EXIT_CONTROLS %#RX32\n", u32Val));
3805	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
3806	Log(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
3807	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
3808	Log(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
3809	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
3810	Log(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
3811	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
3812	Log(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
3813	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
3814	Log(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
3815	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
3816	Log(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
3817	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
3818	Log(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
3819	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
3820	Log(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
3821	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
3822	Log(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
3823	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
3824	Log(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
3825	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
3826	Log(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
3827	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
3828	Log(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
3829	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
3830	Log(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
3831	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
3832	Log(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
3833	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
3834	Log(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
3835	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
3836	Log(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
3837
3838	/* Guest bits. */
3839	RTGCUINTREG uGCReg;
3840	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &uGCReg); AssertRC(rc);
3841	Log(("Old Guest Rip %#RGv New %#RGv\n", (RTGCPTR)pCtx->rip, (RTGCPTR)uGCReg));
3842	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &uGCReg); AssertRC(rc);
3843	Log(("Old Guest Rsp %#RGv New %#RGv\n", (RTGCPTR)pCtx->rsp, (RTGCPTR)uGCReg));
3844	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RFLAGS, &uGCReg); AssertRC(rc);
3845	Log(("Old Guest Rflags %#RGr New %#RGr\n", (RTGCPTR)pCtx->rflags.u64, (RTGCPTR)uGCReg));
3846	rc = VMXReadVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, &u32Val); AssertRC(rc);
3847	Log(("VMX_VMCS16_GUEST_FIELD_VPID %u\n", u32Val));
3848
3849	/* Host bits. */
3850	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
3851	Log(("Host CR0 %#RHr\n", uHCReg));
3852	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
3853	Log(("Host CR3 %#RHr\n", uHCReg));
3854	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
3855	Log(("Host CR4 %#RHr\n", uHCReg));
3856
3857	RTGDTR HostGdtr;
3858	PCX86DESCHC pDesc;
3859	ASMGetGDTR(&HostGdtr);
3860	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_CS, &u32Val);
3861	Log(("Host CS %#08x\n", u32Val));
3862	if (u32Val < HostGdtr.cbGdt)
3863	{
3864	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
3865	HMR0DumpDescriptor(pDesc, u32Val, "CS: ");
3866	}
3867
3868	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_DS, &u32Val); AssertRC(rc);
3869	Log(("Host DS %#08x\n", u32Val));
3870	if (u32Val < HostGdtr.cbGdt)
3871	{
3872	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
3873	HMR0DumpDescriptor(pDesc, u32Val, "DS: ");
3874	}
3875
3876	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_ES, &u32Val); AssertRC(rc);
3877	Log(("Host ES %#08x\n", u32Val));
3878	if (u32Val < HostGdtr.cbGdt)
3879	{
3880	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
3881	HMR0DumpDescriptor(pDesc, u32Val, "ES: ");
3882	}
3883
3884	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_FS, &u32Val); AssertRC(rc);
3885	Log(("Host FS %#08x\n", u32Val));
3886	if (u32Val < HostGdtr.cbGdt)
3887	{
3888	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
3889	HMR0DumpDescriptor(pDesc, u32Val, "FS: ");
3890	}
3891
3892	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_GS, &u32Val); AssertRC(rc);
3893	Log(("Host GS %#08x\n", u32Val));
3894	if (u32Val < HostGdtr.cbGdt)
3895	{
3896	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
3897	HMR0DumpDescriptor(pDesc, u32Val, "GS: ");
3898	}
3899
3900	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_SS, &u32Val); AssertRC(rc);
3901	Log(("Host SS %#08x\n", u32Val));
3902	if (u32Val < HostGdtr.cbGdt)
3903	{
3904	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
3905	HMR0DumpDescriptor(pDesc, u32Val, "SS: ");
3906	}
3907
3908	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_TR, &u32Val); AssertRC(rc);
3909	Log(("Host TR %#08x\n", u32Val));
3910	if (u32Val < HostGdtr.cbGdt)
3911	{
3912	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
3913	HMR0DumpDescriptor(pDesc, u32Val, "TR: ");
3914	}
3915
3916	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
3917	Log(("Host TR Base %#RHv\n", uHCReg));
3918	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
3919	Log(("Host GDTR Base %#RHv\n", uHCReg));
3920	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
3921	Log(("Host IDTR Base %#RHv\n", uHCReg));
3922	rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
3923	Log(("Host SYSENTER CS %#08x\n", u32Val));
3924	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
3925	Log(("Host SYSENTER EIP %#RHv\n", uHCReg));
3926	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
3927	Log(("Host SYSENTER ESP %#RHv\n", uHCReg));
3928	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
3929	Log(("Host RSP %#RHv\n", uHCReg));
3930	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
3931	Log(("Host RIP %#RHv\n", uHCReg));
3932	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3933	if (VMX_IS_64BIT_HOST_MODE())
3934	{
3935	Log(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
3936	Log(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
3937	Log(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
3938	Log(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
3939	Log(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
3940	Log(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
3941	}
3942	# endif
3943	#endif /* VBOX_STRICT */
3944	break;
3945	}
3946
3947	default:
3948	/* Impossible */
3949	AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
3950	break;
3951	}
3952	NOREF(pVM);
3953	}
3954
3955
3956	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3957	#ifndef VMX_USE_CACHED_VMCS_ACCESSES
3958	# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
3959	#endif
3960
3961	/**
3962	* Executes the specified handler in 64-bit mode.
3963	*
3964	* @returns VBox status code.
3965	* @param pVM Pointer to the VM.
3966	* @param pVCpu Pointer to the VMCPU.
3967	* @param pCtx Pointer to the guest CPU context.
3968	* @param pfnHandler Pointer to the RC handler function.
3969	* @param cbParam Number of parameters.
3970	* @param paParam Array of 32-bit parameters.
3971	*/
3972	VMMR0DECL(int) VMXR0Execute64BitsHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, RTRCPTR pfnHandler, uint32_t cbParam,
3973	uint32_t *paParam)
3974	{
3975	int rc, rc2;
3976	PHMGLOBLCPUINFO pCpu;
3977	RTHCPHYS HCPhysCpuPage;
3978	RTHCUINTREG uOldEFlags;
3979
3980	AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
3981	Assert(pfnHandler);
3982	Assert(pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Write.aField));
3983	Assert(pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Read.aField));
3984
3985	#ifdef VBOX_STRICT
3986	for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries; i++)
3987	Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VMCSCache.Write.aField[i]));
3988
3989	for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries; i++)
3990	Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VMCSCache.Read.aField[i]));
3991	#endif
3992
3993	/* Disable interrupts. */
3994	uOldEFlags = ASMIntDisableFlags();
3995
3996	#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
3997	RTCPUID idHostCpu = RTMpCpuId();
3998	CPUMR0SetLApic(pVM, idHostCpu);
3999	#endif
4000
4001	pCpu = HMR0GetCurrentCpu();
4002	HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
4003
4004	/* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
4005	VMXClearVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
4006
4007	/* Leave VMX Root Mode. */
4008	VMXDisable();
4009
4010	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
4011
4012	CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
4013	CPUMSetHyperEIP(pVCpu, pfnHandler);
4014	for (int i = (int)cbParam - 1; i >= 0; i--)
4015	CPUMPushHyper(pVCpu, paParam[i]);
4016
4017	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
4018
4019	/* Call the switcher. */
4020	rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_OFFSETOF(VM, aCpus[pVCpu->idCpu].cpum) - RT_OFFSETOF(VM, cpum));
4021	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
4022
4023	/** @todo replace with hmR0VmxEnterRootMode() and LeaveRootMode(). */
4024	/* Make sure the VMX instructions don't cause #UD faults. */
4025	ASMSetCR4(ASMGetCR4() \| X86_CR4_VMXE);
4026
4027	/* Re-enter VMX Root Mode */
4028	rc2 = VMXEnable(HCPhysCpuPage);
4029	if (RT_FAILURE(rc2))
4030	{
4031	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
4032	ASMSetFlags(uOldEFlags);
4033	return VERR_VMX_VMXON_FAILED;
4034	}
4035
4036	rc2 = VMXActivateVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
4037	AssertRC(rc2);
4038	Assert(!(ASMGetFlags() & X86_EFL_IF));
4039	ASMSetFlags(uOldEFlags);
4040	return rc;
4041	}
4042
4043
4044	/**
4045	* Prepares for and executes VMLAUNCH (64 bits guests) for 32-bit hosts
4046	* supporting 64-bit guests.
4047	*
4048	* @returns VBox status code.
4049	* @param fResume Whether to VMLAUNCH or VMRESUME.
4050	* @param pCtx Pointer to the guest-CPU context.
4051	* @param pCache Pointer to the VMCS cache.
4052	* @param pVM Pointer to the VM.
4053	* @param pVCpu Pointer to the VMCPU.
4054	*/
4055	DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
4056	{
4057	uint32_t aParam[6];
4058	PHMGLOBLCPUINFO pCpu = NULL;
4059	RTHCPHYS HCPhysCpuPage = 0;
4060	int rc = VERR_INTERNAL_ERROR_5;
4061
4062	pCpu = HMR0GetCurrentCpu();
4063	HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
4064
4065	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4066	pCache->uPos = 1;
4067	pCache->interPD = PGMGetInterPaeCR3(pVM);
4068	pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
4069	#endif
4070
4071	#ifdef DEBUG
4072	pCache->TestIn.HCPhysCpuPage= 0;
4073	pCache->TestIn.HCPhysVmcs = 0;
4074	pCache->TestIn.pCache = 0;
4075	pCache->TestOut.HCPhysVmcs = 0;
4076	pCache->TestOut.pCache = 0;
4077	pCache->TestOut.pCtx = 0;
4078	pCache->TestOut.eflags = 0;
4079	#endif
4080
4081	aParam[0] = (uint32_t)(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
4082	aParam[1] = (uint32_t)(HCPhysCpuPage >> 32); /* Param 1: VMXON physical address - Hi. */
4083	aParam[2] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
4084	aParam[3] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs >> 32); /* Param 2: VMCS physical address - Hi. */
4085	aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache);
4086	aParam[5] = 0;
4087
4088	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4089	pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
4090	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
4091	#endif
4092	rc = VMXR0Execute64BitsHandler(pVM, pVCpu, pCtx, pVM->hm.s.pfnVMXGCStartVM64, 6, &aParam[0]);
4093
4094	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4095	Assert((uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
4096	Assert(pCtx->dr[4] == 10);
4097	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
4098	#endif
4099
4100	#ifdef DEBUG
4101	AssertMsg(pCache->TestIn.HCPhysCpuPage== HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
4102	AssertMsg(pCache->TestIn.HCPhysVmcs == pVCpu->hm.s.vmx.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
4103	pVCpu->hm.s.vmx.HCPhysVmcs));
4104	AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
4105	pCache->TestOut.HCPhysVmcs));
4106	AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
4107	pCache->TestOut.pCache));
4108	AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache),
4109	("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache)));
4110	AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
4111	pCache->TestOut.pCtx));
4112	Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
4113	#endif
4114	return rc;
4115	}
4116
4117
4118	/**
4119	* Initialize the VMCS-Read cache. The VMCS cache is used for 32-bit hosts
4120	* running 64-bit guests (except 32-bit Darwin which runs with 64-bit paging in
4121	* 32-bit mode) for 64-bit fields that cannot be accessed in 32-bit mode. Some
4122	* 64-bit fields -can- be accessed (those that have a 32-bit FULL & HIGH part).
4123	*
4124	* @returns VBox status code.
4125	* @param pVM Pointer to the VM.
4126	* @param pVCpu Pointer to the VMCPU.
4127	*/
4128	static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu)
4129	{
4130	#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
4131	{ \
4132	Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
4133	pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
4134	pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
4135	}
4136
4137	#define VMXLOCAL_INIT_VMCS_SELREG(REG, pCache) \
4138	{ \
4139	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS16_GUEST_FIELD_##REG); \
4140	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS32_GUEST_##REG##_LIMIT); \
4141	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_##REG##_BASE); \
4142	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS32_GUEST_##REG##_ACCESS_RIGHTS); \
4143	}
4144
4145	AssertPtr(pVM);
4146	AssertPtr(pVCpu);
4147	PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
4148
4149	/* 16-bit guest-state fields (16-bit selectors and their corresponding 32-bit limit & 32-bit access-rights fields). */
4150	VMXLOCAL_INIT_VMCS_SELREG(ES, pCache);
4151	VMXLOCAL_INIT_VMCS_SELREG(CS, pCache);
4152	VMXLOCAL_INIT_VMCS_SELREG(SS, pCache);
4153	VMXLOCAL_INIT_VMCS_SELREG(DS, pCache);
4154	VMXLOCAL_INIT_VMCS_SELREG(FS, pCache);
4155	VMXLOCAL_INIT_VMCS_SELREG(GS, pCache);
4156	VMXLOCAL_INIT_VMCS_SELREG(LDTR, pCache);
4157	VMXLOCAL_INIT_VMCS_SELREG(TR, pCache);
4158
4159	/* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for these 64-bit fields (using "FULL" and "HIGH" fields). */
4160	#if 0
4161	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
4162	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
4163	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
4164	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
4165	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
4166	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
4167	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
4168	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
4169	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
4170	#endif
4171
4172	/* 32-bit guest-state fields. */
4173	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS32_GUEST_GDTR_LIMIT);
4174	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS32_GUEST_IDTR_LIMIT);
4175	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE);
4176	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS32_GUEST_SYSENTER_CS);
4177	/* Unused 32-bit guest-state fields. */
4178	#if 0
4179	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS32_GUEST_ACTIVITY_STATE);
4180	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS32_GUEST_SMBASE);
4181	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS32_GUEST_PREEMPTION_TIMER_VALUE);
4182	#endif
4183
4184	/* Natural width guest-state fields. */
4185	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
4186	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
4187	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
4188	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
4189	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
4190	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
4191	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
4192	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
4193	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
4194	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
4195	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
4196	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
4197	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DR7);
4198	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
4199	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
4200	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RFLAGS);
4201	/* Unused natural width guest-state fields. */
4202	#if 0
4203	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DEBUG_EXCEPTIONS);
4204	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in Nested Paging case */
4205	#endif
4206	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
4207	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
4208
4209	if (pVM->hm.s.fNestedPaging)
4210	{
4211	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
4212	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL);
4213	pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
4214	}
4215	else
4216	pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
4217
4218	#undef VMXLOCAL_INIT_VMCS_SELREG
4219	#undef VMXLOCAL_INIT_READ_CACHE_FIELD
4220	return VINF_SUCCESS;
4221	}
4222
4223
4224	/**
4225	* Writes a field into the VMCS. This can either directly invoke a VMWRITE or
4226	* queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
4227	* darwin, running 64-bit guests).
4228	*
4229	* @returns VBox status code.
4230	* @param pVCpu Pointer to the VMCPU.
4231	* @param idxField The VMCS field encoding.
4232	* @param u64Val 16, 32 or 64 bits value.
4233	*/
4234	VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
4235	{
4236	int rc;
4237	switch (idxField)
4238	{
4239	/*
4240	* These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
4241	*/
4242	/* 64-bit Control fields. */
4243	case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
4244	case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
4245	case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
4246	case VMX_VMCS64_CTRL_VMEXIT_MSR_STORE_FULL:
4247	case VMX_VMCS64_CTRL_VMEXIT_MSR_LOAD_FULL:
4248	case VMX_VMCS64_CTRL_VMENTRY_MSR_LOAD_FULL:
4249	case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
4250	case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
4251	case VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL:
4252	case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
4253	case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
4254	case VMX_VMCS64_CTRL_EPTP_FULL:
4255	case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
4256	/* 64-bit Read-only data fields. */
4257	case VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL:
4258	/* 64-bit Guest-state fields. */
4259	case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
4260	case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
4261	case VMX_VMCS64_GUEST_PAT_FULL:
4262	case VMX_VMCS64_GUEST_EFER_FULL:
4263	case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
4264	case VMX_VMCS64_GUEST_PDPTE0_FULL:
4265	case VMX_VMCS64_GUEST_PDPTE1_FULL:
4266	case VMX_VMCS64_GUEST_PDPTE2_FULL:
4267	case VMX_VMCS64_GUEST_PDPTE3_FULL:
4268	/* 64-bit Host-state fields. */
4269	case VMX_VMCS64_HOST_FIELD_PAT_FULL:
4270	case VMX_VMCS64_HOST_FIELD_EFER_FULL:
4271	case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
4272	{
4273	rc = VMXWriteVmcs32(idxField, u64Val);
4274	rc \|= VMXWriteVmcs32(idxField + 1, (uint32_t)(u64Val >> 32ULL)); /* hmpf, do we really need the "ULL" suffix? */
4275	break;
4276	}
4277
4278	/*
4279	* These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
4280	* values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
4281	*/
4282	/* Natural-width Guest-state fields. */
4283	case VMX_VMCS_GUEST_CR0:
4284	case VMX_VMCS_GUEST_CR3:
4285	case VMX_VMCS_GUEST_CR4:
4286	case VMX_VMCS_GUEST_ES_BASE:
4287	case VMX_VMCS_GUEST_CS_BASE:
4288	case VMX_VMCS_GUEST_SS_BASE:
4289	case VMX_VMCS_GUEST_DS_BASE:
4290	case VMX_VMCS_GUEST_FS_BASE:
4291	case VMX_VMCS_GUEST_GS_BASE:
4292	case VMX_VMCS_GUEST_LDTR_BASE:
4293	case VMX_VMCS_GUEST_TR_BASE:
4294	case VMX_VMCS_GUEST_GDTR_BASE:
4295	case VMX_VMCS_GUEST_IDTR_BASE:
4296	case VMX_VMCS_GUEST_DR7:
4297	case VMX_VMCS_GUEST_RSP:
4298	case VMX_VMCS_GUEST_RIP:
4299	case VMX_VMCS_GUEST_RFLAGS:
4300	case VMX_VMCS_GUEST_DEBUG_EXCEPTIONS:
4301	case VMX_VMCS_GUEST_SYSENTER_ESP:
4302	case VMX_VMCS_GUEST_SYSENTER_EIP:
4303	{
4304	if ((u64Val >> 32ULL) == 0)
4305	{
4306	/* If this field is 64-bit, VT-x will zero out the top bits. */
4307	rc = VMXWriteVmcs32(idxField, (uint32_t)u64Val);
4308	}
4309	else
4310	{
4311	/* Assert that only the 32->64 switcher case should ever come here. */
4312	Assert(pVM->hm.s.fAllow64BitGuests);
4313	rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
4314	}
4315	break;
4316	}
4317
4318	default:
4319	{
4320	AssertMsgFailed(("VMXWriteVmcs64Ex: invalid field %#x (pVCpu=%p u64Val=%RX64)\n", (unsigned)idxField, pVCpu, u64Val));
4321	rc = VERR_INVALID_PARAMETER;
4322	break;
4323	}
4324	}
4325	AssertRCReturn(rc, rc);
4326	return rc;
4327	}
4328
4329
4330	/**
4331	* Queue up a VMWRITE by using the VMCS write cache. This is only used on 32-bit
4332	* hosts (except darwin) for 64-bit guests.
4333	*
4334	* @param pVCpu Pointer to the VMCPU.
4335	* @param idxField The VMCS field encoding.
4336	* @param u64Val 16, 32 or 64 bits value.
4337	*/
4338	VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
4339	{
4340	AssertPtr(pVCpu);
4341	PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
4342
4343	AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1,
4344	("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
4345
4346	/* Make sure there are no duplicates. */
4347	for (unsigned i = 0; i < pCache->Write.cValidEntries; i++)
4348	{
4349	if (pCache->Write.aField[i] == idxField)
4350	{
4351	pCache->Write.aFieldVal[i] = u64Val;
4352	return VINF_SUCCESS;
4353	}
4354	}
4355
4356	pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
4357	pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
4358	pCache->Write.cValidEntries++;
4359	return VINF_SUCCESS;
4360	}
4361
4362
4363	/**
4364	* Loads the VMCS write-cache into the CPU (by executing VMWRITEs).
4365	*
4366	* @param pCache Pointer to the VMCS cache.
4367	*/
4368	VMMR0DECL(void) VMXWriteCachedVmcsLoad(PVMCSCACHE pCache)
4369	{
4370	AssertPtr(pCache);
4371	for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
4372	{
4373	int rc = VMXWriteVmcs64(pCache->Write.aField[i], pCache->Write.aFieldVal[i]);
4374	AssertRC(rc, rc);
4375	}
4376	pCache->Write.cValidEntries = 0;
4377	}
4378
4379
4380	/**
4381	* Stores the VMCS read-cache from the CPU (by executing VMREADs).
4382	*
4383	* @param pCache Pointer to the VMCS cache.
4384	* @remarks No-long-jump zone!!!
4385	*/
4386	VMMR0DECL(void) VMXReadCachedVmcsStore(PVMCSCACHE pCache)
4387	{
4388	AssertPtr(pCache);
4389	for (uint32_t i = 0; i < pCache->Read.cValidEntries; i++)
4390	{
4391	int rc = VMXReadVmcs64(pCache->Read.aField[i], &pCache->Read.aFieldVal[i]);
4392	AssertRC(rc);
4393	}
4394	}
4395	#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL) */
4396
4397
4398	/**
4399	* Sets up the usage of TSC-offsetting and updates the VMCS. If offsetting is
4400	* not possible, cause VM-exits on RDTSC(P)s. Also sets up the VMX preemption
4401	* timer.
4402	*
4403	* @returns VBox status code.
4404	* @param pVM Pointer to the VM.
4405	* @param pVCpu Pointer to the VMCPU.
4406	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4407	* out-of-sync. Make sure to update the required fields
4408	* before using them.
4409	* @remarks No-long-jump zone!!!
4410	*/
4411	static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4412	{
4413	int rc = VERR_INTERNAL_ERROR_5;
4414	bool fOffsettedTsc = false;
4415	if (pVM->hm.s.vmx.fUsePreemptTimer)
4416	{
4417	uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVCpu, &fOffsettedTsc, &pVCpu->hm.s.vmx.u64TSCOffset);
4418
4419	/* Make sure the returned values have sane upper and lower boundaries. */
4420	uint64_t u64CpuHz = SUPGetCpuHzFromGIP(g_pSUPGlobalInfoPage);
4421	cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
4422	cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
4423	cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
4424
4425	uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
4426	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_PREEMPTION_TIMER_VALUE, cPreemptionTickCount); AssertRC(rc);
4427	}
4428	else
4429	fOffsettedTsc = TMCpuTickCanUseRealTSC(pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset);
4430
4431	if (fOffsettedTsc)
4432	{
4433	uint64_t u64CurTSC = ASMReadTSC();
4434	if (u64CurTSC + pVCpu->hm.s.vmx.u64TSCOffset >= TMCpuTickGetLastSeen(pVCpu))
4435	{
4436	/* Note: VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT takes precedence over TSC_OFFSET, applies to RDTSCP too. */
4437	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, pVCpu->hm.s.vmx.u64TSCOffset); AssertRC(rc);
4438
4439	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT;
4440	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
4441	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
4442	}
4443	else
4444	{
4445	/* VM-exit on RDTSC(P) as we would otherwise pass decreasing TSC values to the guest. */
4446	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT;
4447	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
4448	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscInterceptOverFlow);
4449	}
4450	}
4451	else
4452	{
4453	/* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
4454	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT;
4455	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
4456	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
4457	}
4458	}
4459
4460
4461	/**
4462	* Determines if an exception is a benign exception. Benign exceptions
4463	* are ones which cannot cause double-faults.
4464	*
4465	* @returns true if the exception is benign, false otherwise.
4466	* @param uVector The exception vector.
4467	*/
4468	DECLINLINE(bool) hmR0VmxIsBenignXcpt(const uint8_t uVector)
4469	{
4470	switch (uVector)
4471	{
4472	case X86_XCPT_DB:
4473	case X86_XCPT_NMI:
4474	case X86_XCPT_BP:
4475	case X86_XCPT_OF:
4476	case X86_XCPT_BR:
4477	case X86_XCPT_UD:
4478	case X86_XCPT_NM:
4479	case X86_XCPT_CO_SEG_OVERRUN:
4480	case X86_XCPT_MF:
4481	case X86_XCPT_AC:
4482	case X86_XCPT_MC:
4483	case X86_XCPT_XF:
4484	return true;
4485	default:
4486	return false;
4487	}
4488	}
4489
4490
4491	/**
4492	* Determines if an exception is a contributory exception. Contributory
4493	* exceptions are ones which can cause double-faults.
4494	*
4495	* @returns true if the exception is contributory, false otherwise.
4496	* @param uVector The exception vector.
4497	*/
4498	DECLINLINE(bool) hmR0VmxIsContributoryXcpt(const uint8_t uVector)
4499	{
4500	switch (uVector)
4501	{
4502	case X86_XCPT_GP:
4503	case X86_XCPT_SS:
4504	case X86_XCPT_NP:
4505	case X86_XCPT_TS:
4506	case X86_XCPT_DE:
4507	return true;
4508	default:
4509	return false;
4510	}
4511	return false;
4512	}
4513
4514
4515	/**
4516	* Determines if we are intercepting any contributory exceptions.
4517	*
4518	* @returns true if we are intercepting any contributory exception, false
4519	* otherwise.
4520	* @param pVCpu Pointer to the VMCPU.
4521	*/
4522	DECLINLINE(bool) hmR0VmxInterceptingContributoryXcpts(PVMCPU pVCpu)
4523	{
4524	if ( (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_GP))
4525	\|\| (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_SS))
4526	\|\| (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_NP))
4527	\|\| (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_TS))
4528	\|\| (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_DE)))
4529	{
4530	return true;
4531	}
4532	return false;
4533	}
4534
4535
4536	/**
4537	* Saves, if necessary, any event that occurred during event delivery as a
4538	* pending VMX event to handle before the next VM-entry or to be translated as a
4539	* TRPM event in the case of exiting to ring-3.
4540	*
4541	* @returns VBox status code (informational error codes included).
4542	* @retval VINF_SUCCESS if we should continue handling VM-exits.
4543	* @retval VINF_VMX_DOUBLE_FAULT if a #DF condition was detected and we ought to
4544	* continue execution of the guest which will delivery the #DF.
4545	* @retval VINF_EM_RESET if we detected a triple-fault condition.
4546	*
4547	* @param pVM Pointer to the VM.
4548	* @param pVCpu Pointer to the VMCPU.
4549	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4550	* out-of-sync. Make sure to update the required fields
4551	* before using them.
4552	* @param pVmxTransient Pointer to the VMX transient structure.
4553	*
4554	* @remarks No-long-jump zone!!!
4555	* @remarks Called unconditionally after every VM-exit.
4556	*
4557	*/
4558	static int hmR0VmxSavePendingEventDueToEventDelivery(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
4559	{
4560
4561	Assert(pVCpu);
4562	Assert(pVmxTransient);
4563
4564	if (RT_UNLIKELY(pVmxTransient->fVMEntryFailed)) /* Don't bother with pending events if the VM-entry itself failed. */
4565	return VINF_SUCCESS;
4566
4567	int rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
4568	AssertRCReturn(rc, rc);
4569	if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
4570	{
4571	rc = hmR0VmxReadExitIntrInfoVmcs(pVmxTransient);
4572	AssertRCReturn(rc, rc);
4573
4574	uint8_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
4575	uint8_t uExitVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntrInfo);
4576	uint8_t uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
4577
4578	typedef enum
4579	{
4580	VMXREFLECTXCPT_XCPT, /* Reflect Idt-vectoring exception. */
4581	VMXREFLECTXCPT_DF, /* Reflect a double-fault to the guest. */
4582	VMXREFLECTXCPT_TF, /* Reflect a triple fault state to the VMM. */
4583	VMXREFLECTXCPT_NONE /* Nothing to reflect. */
4584	} VMXREFLECTXCPT;
4585
4586	/* See Intel spec. 30.7.1.1 "Reflecting Exceptions to Guest Software". */
4587	VMXREFLECTXCPT enmReflect = VMXREFLECTXCPT_NONE;
4588	if (uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT)
4589	{
4590	if ( hmR0VmxIsBenignXcpt(uIdtVector)
4591	\|\| hmR0VmxIsBenignXcpt(uExitVector)
4592	\|\| ( hmR0VmxIsContributoryXcpt(uIdtVector)
4593	&& uExitVector == X86_XCPT_PF))
4594	{
4595	enmReflect = VMXREFLECTXCPT_XCPT;
4596	}
4597	else if ( hmR0VmxIsContributoryXcpt(uIdtVector)
4598	&& hmR0VmxIsContributoryXcpt(uExitVector))
4599	{
4600	enmReflect = VMXREFLECTXCPT_DF;
4601	}
4602	else if ( (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_PF))
4603	&& uIdtVector == X86_XCPT_PF
4604	&& uExitVector == X86_XCPT_PF)
4605	{
4606	pVmxTransient->fVectoringPF = true;
4607	}
4608	else if ( hmR0VmxInterceptingContributoryXcpts(pVCpu)
4609	&& uIdtVector == X86_XCPT_PF
4610	&& hmR0VmxIsContributoryXcpt(uExitVector))
4611	{
4612	enmReflect = VMXREFLECTXCPT_DF;
4613	}
4614	else if (uIdtVector == X86_XCPT_DF)
4615	enmReflect = VMXREFLECTXCPT_TF;
4616	}
4617	else if ( uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
4618	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
4619	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
4620	{
4621	/*
4622	* Ignore software interrupts (INT n), software exceptions (#BP, #OF) and privileged software exception
4623	* (whatever they are) as they reoccur when restarting the instruction.
4624	*/
4625	enmReflect = VMXREFLECTXCPT_XCPT;
4626	}
4627
4628	Assert(pVmxTransient->fVectoringPF == false \|\| enmReflect == VMXREFLECTXCPT_NONE);
4629	switch (enmReflect)
4630	{
4631	case VMXREFLECTXCPT_XCPT:
4632	{
4633	Assert(!pVCpu->hm.s.Event.fPending);
4634	pVCpu->hm.s.Event.fPending = true;
4635	pVCpu->hm.s.Event.u64IntrInfo = VMX_ENTRY_INTR_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo);
4636	if (VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntrInfo))
4637	{
4638	rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
4639	AssertRCReturn(rc, rc);
4640	pVCpu->hm.s.Event.u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
4641	}
4642	else
4643	pVCpu->hm.s.Event.u32ErrCode = 0;
4644	Log(("Pending event %#RX64 Err=%#RX32\n", pVCpu->hm.s.Event.u64IntrInfo, pVCpu->hm.s.Event.u32ErrCode));
4645	break;
4646	}
4647
4648	case VMXREFLECTXCPT_DF:
4649	{
4650	uint32_t u32IntrInfo;
4651	u32IntrInfo = X86_XCPT_DF \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
4652	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
4653	u32IntrInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
4654
4655	Assert(!pVCpu->hm.s.Event.fPending);
4656	pVCpu->hm.s.Event.fPending = true;
4657	pVCpu->hm.s.Event.u64IntrInfo = u32IntrInfo;
4658	pVCpu->hm.s.Event.u32ErrCode = 0;
4659	rc = VINF_VMX_DOUBLE_FAULT;
4660	Log(("Pending #DF %#RX64 uIdt=%#x uExit=%#x\n", pVCpu->hm.s.Event.u64IntrInfo, uIdtVector, uExitVector));
4661	break;
4662	}
4663
4664	case VMXREFLECTXCPT_TF:
4665	{
4666	Log(("Pending triple-fault uIdt=%#x uExit=%#x\n", uIdtVector, uExitVector));
4667	rc = VINF_EM_RESET;
4668	break;
4669	}
4670
4671	default: /* shut up gcc. */
4672	break;
4673	}
4674	}
4675	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RESET \|\| rc == VINF_VMX_DOUBLE_FAULT);
4676	return rc;
4677	}
4678
4679
4680	/**
4681	* Saves the guest's CR0 register from the VMCS into the guest-CPU context.
4682	*
4683	* @returns VBox status code.
4684	* @param pVM Pointer to the VM.
4685	* @param pVCpu Pointer to the VMCPU.
4686	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4687	* out-of-sync. Make sure to update the required fields
4688	* before using them.
4689	*
4690	* @remarks No-long-jump zone!!!
4691	*/
4692	DECLINLINE(int) hmR0VmxSaveGuestCR0(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4693	{
4694	int rc = VINF_SUCCESS;
4695	if ( !(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_CR0)
4696	\|\| !(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_FPU))
4697	{
4698	RTGCUINTREG uVal = 0;
4699	RTCCUINTREG uShadow = 0;
4700	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR0, &uVal);
4701	rc \|= VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uShadow);
4702	AssertRCReturn(rc, rc);
4703	uVal = (uShadow & pVCpu->hm.s.vmx.cr0_mask) \| (uVal & ~pVCpu->hm.s.vmx.cr0_mask);
4704	CPUMSetGuestCR0(pVCpu, uVal);
4705	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_CR0 \| VMX_UPDATED_GUEST_FPU;
4706	}
4707	return rc;
4708	}
4709
4710
4711	/**
4712	* Saves the guest's CR4 register from the VMCS into the guest-CPU context.
4713	*
4714	* @returns VBox status code.
4715	* @param pVM Pointer to the VM.
4716	* @param pVCpu Pointer to the VMCPU.
4717	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4718	* out-of-sync. Make sure to update the required fields
4719	* before using them.
4720	*
4721	* @remarks No-long-jump zone!!!
4722	*/
4723	DECLINLINE(int) hmR0VmxSaveGuestCR4(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4724	{
4725	int rc = VINF_SUCCESS;
4726	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_CR4))
4727	{
4728	RTGCUINTREG uVal = 0;
4729	RTCCUINTREG uShadow = 0;
4730	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR4, &uVal);
4731	rc \|= VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uShadow);
4732	AssertRCReturn(rc, rc);
4733	uVal = (uShadow & pVCpu->hm.s.vmx.cr4_mask) \| (uVal & ~pVCpu->hm.s.vmx.cr4_mask);
4734	CPUMSetGuestCR4(pVCpu, uVal);
4735	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_CR4;
4736	}
4737	return rc;
4738	}
4739
4740
4741	/**
4742	* Saves the guest's RIP register from the VMCS into the guest-CPU context.
4743	*
4744	* @returns VBox status code.
4745	* @param pVM Pointer to the VM.
4746	* @param pVCpu Pointer to the VMCPU.
4747	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4748	* out-of-sync. Make sure to update the required fields
4749	* before using them.
4750	*
4751	* @remarks No-long-jump zone!!!
4752	*/
4753	DECLINLINE(int) hmR0VmxSaveGuestRip(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4754	{
4755	if (pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_RIP)
4756	return VINF_SUCCESS;
4757
4758	RTGCUINTREG uVal = 0;
4759	int rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &uVal);
4760	AssertRCReturn(rc, rc);
4761	pMixedCtx->rip = uVal;
4762	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_RIP;
4763	return rc;
4764	}
4765
4766
4767	/**
4768	* Saves the guest's RSP register from the VMCS into the guest-CPU context.
4769	*
4770	* @returns VBox status code.
4771	* @param pVM Pointer to the VM.
4772	* @param pVCpu Pointer to the VMCPU.
4773	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4774	* out-of-sync. Make sure to update the required fields
4775	* before using them.
4776	*
4777	* @remarks No-long-jump zone!!!
4778	*/
4779	DECLINLINE(int) hmR0VmxSaveGuestRsp(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4780	{
4781	if (pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_RSP)
4782	return VINF_SUCCESS;
4783
4784	RTGCUINTREG uVal = 0;
4785	int rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &uVal);
4786	AssertRCReturn(rc, rc);
4787	pMixedCtx->rsp = uVal;
4788	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_RSP;
4789	return rc;
4790	}
4791
4792
4793	/**
4794	* Saves the guest's RFLAGS from the VMCS into the guest-CPU context.
4795	*
4796	* @returns VBox status code.
4797	* @param pVM Pointer to the VM.
4798	* @param pVCpu Pointer to the VMCPU.
4799	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4800	* out-of-sync. Make sure to update the required fields
4801	* before using them.
4802	*
4803	* @remarks No-long-jump zone!!!
4804	*/
4805	DECLINLINE(int) hmR0VmxSaveGuestRflags(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4806	{
4807	if (pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_RFLAGS)
4808	return VINF_SUCCESS;
4809
4810	RTGCUINTREG uVal = 0;
4811	int rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RFLAGS, &uVal);
4812	AssertRCReturn(rc, rc);
4813	pMixedCtx->rflags.u64 = uVal;
4814
4815	/* Undo our real-on-v86-mode changes to eflags if necessary. */
4816	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4817	{
4818	Assert(pVM->hm.s.vmx.pRealModeTSS);
4819	Log(("Saving real-mode RFLAGS VT-x view=%#RX64\n", pMixedCtx->rflags.u64));
4820	pMixedCtx->eflags.Bits.u1VM = 0;
4821	pMixedCtx->eflags.Bits.u2IOPL = pVCpu->hm.s.vmx.RealMode.eflags.Bits.u2IOPL;
4822	}
4823
4824	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_RFLAGS;
4825	return rc;
4826	}
4827
4828
4829	/**
4830	* Wrapper for saving the guest's RIP, RSP and RFLAGS from the VMCS into the
4831	* guest-CPU context.
4832	*/
4833	static int hmR0VmxSaveGuestGprs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4834	{
4835	int rc = hmR0VmxSaveGuestRip(pVM, pVCpu, pMixedCtx);
4836	rc \|= hmR0VmxSaveGuestRsp(pVM, pVCpu, pMixedCtx);
4837	rc \|= hmR0VmxSaveGuestRflags(pVM, pVCpu, pMixedCtx);
4838	AssertRCReturn(rc, rc);
4839	return rc;
4840	}
4841
4842
4843	/**
4844	* Saves the guest's interruptibility state.
4845	*
4846	* @returns VBox status code.
4847	* @param pVM Pointer to the VM.
4848	* @param pVCpu Pointer to the VMCPU.
4849	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4850	* out-of-sync. Make sure to update the required fields
4851	* before using them.
4852	*
4853	* @remarks No-long-jump zone!!!
4854	*/
4855	DECLINLINE(int) hmR0VmxSaveGuestIntrState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4856	{
4857	if (pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_INTR_STATE)
4858	return VINF_SUCCESS;
4859
4860	uint32_t uIntrState = 0;
4861	int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
4862	if (uIntrState != 0)
4863	{
4864	Assert( uIntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
4865	\|\| uIntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
4866	rc = hmR0VmxSaveGuestRip(pVM, pVCpu, pMixedCtx);
4867	AssertRCReturn(rc, rc);
4868	EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
4869	Assert(VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
4870	}
4871	else
4872	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
4873
4874	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_INTR_STATE;
4875	return rc;
4876	}
4877
4878
4879	/**
4880	* Saves the guest's activity state.
4881	*
4882	* @returns VBox status code.
4883	* @param pVM Pointer to the VM.
4884	* @param pVCpu Pointer to the VMCPU.
4885	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4886	* out-of-sync. Make sure to update the required fields
4887	* before using them.
4888	*
4889	* @remarks No-long-jump zone!!!
4890	*/
4891	DECLINLINE(int) hmR0VmxSaveGuestActivityState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4892	{
4893	/* Nothing to do for now until we make use of different guest-CPU activity state. Just update the flag. */
4894	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_ACTIVITY_STATE;
4895	return VINF_SUCCESS;
4896	}
4897
4898
4899	/**
4900	* Saves the guest SYSENTER MSRs (SYSENTER_CS, SYSENTER_EIP, SYSENTER_ESP) from
4901	* the current VMCS into the guest-CPU context.
4902	*
4903	* @returns VBox status code.
4904	* @param pVM Pointer to the VM.
4905	* @param pVCpu Pointer to the VMCPU.
4906	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4907	* out-of-sync. Make sure to update the required fields
4908	* before using them.
4909	*
4910	* @remarks No-long-jump zone!!!
4911	*/
4912	DECLINLINE(int) hmR0VmxSaveGuestSysenterMsrs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4913	{
4914	int rc = VINF_SUCCESS;
4915	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_SYSENTER_CS_MSR))
4916	{
4917	uint32_t u32Val = 0;
4918	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val); AssertRCReturn(rc, rc);
4919	pMixedCtx->SysEnter.cs = u32Val;
4920	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_SYSENTER_CS_MSR;
4921	}
4922
4923	RTGCUINTREG uGCVal = 0;
4924	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_SYSENTER_EIP_MSR))
4925	{
4926	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &uGCVal); AssertRCReturn(rc, rc);
4927	pMixedCtx->SysEnter.eip = uGCVal;
4928	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_SYSENTER_EIP_MSR;
4929	}
4930	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_SYSENTER_ESP_MSR))
4931	{
4932	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &uGCVal); AssertRCReturn(rc, rc);
4933	pMixedCtx->SysEnter.esp = uGCVal;
4934	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_SYSENTER_ESP_MSR;
4935	}
4936	return rc;
4937	}
4938
4939
4940	/**
4941	* Saves the guest FS_BASE MSRs from the current VMCS into the guest-CPU
4942	* context.
4943	*
4944	* @returns VBox status code.
4945	* @param pVM Pointer to the VM.
4946	* @param pVCpu Pointer to the VMCPU.
4947	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4948	* out-of-sync. Make sure to update the required fields
4949	* before using them.
4950	*
4951	* @remarks No-long-jump zone!!!
4952	*/
4953	DECLINLINE(int) hmR0VmxSaveGuestFSBaseMsr(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4954	{
4955	RTGCUINTREG uVal = 0;
4956	int rc = VINF_SUCCESS;
4957	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_FS_BASE_MSR))
4958	{
4959	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_FS_BASE, &uVal); AssertRCReturn(rc, rc);
4960	pMixedCtx->fs.u64Base = uVal;
4961	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_FS_BASE_MSR;
4962	}
4963	return rc;
4964	}
4965
4966
4967	/**
4968	* Saves the guest GS_BASE MSRs from the current VMCS into the guest-CPU
4969	* context.
4970	*
4971	* @returns VBox status code.
4972	* @param pVM Pointer to the VM.
4973	* @param pVCpu Pointer to the VMCPU.
4974	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4975	* out-of-sync. Make sure to update the required fields
4976	* before using them.
4977	*
4978	* @remarks No-long-jump zone!!!
4979	*/
4980	DECLINLINE(int) hmR0VmxSaveGuestGSBaseMsr(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4981	{
4982	RTGCUINTREG uVal = 0;
4983	int rc = VINF_SUCCESS;
4984	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_GS_BASE_MSR))
4985	{
4986	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GS_BASE, &uVal); AssertRCReturn(rc, rc);
4987	pMixedCtx->gs.u64Base = uVal;
4988	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_GS_BASE_MSR;
4989	}
4990	return rc;
4991	}
4992
4993
4994	/**
4995	* Saves the auto load/store'd guest MSRs from the current VMCS into the
4996	* guest-CPU context. Currently these are LSTAR, STAR, SFMASK and TSC_AUX.
4997	*
4998	* @returns VBox status code.
4999	* @param pVM Pointer to the VM.
5000	* @param pVCpu Pointer to the VMCPU.
5001	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5002	* out-of-sync. Make sure to update the required fields
5003	* before using them.
5004	*
5005	* @remarks No-long-jump zone!!!
5006	*/
5007	static int hmR0VmxSaveGuestAutoLoadStoreMsrs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5008	{
5009	if (pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS)
5010	return VINF_SUCCESS;
5011
5012	for (unsigned i = 0; i < pVCpu->hm.s.vmx.cGuestMsrs; i++)
5013	{
5014	PVMXMSR pMsr = (PVMXMSR)pVCpu->hm.s.vmx.pvGuestMsr;
5015	pMsr += i;
5016	switch (pMsr->u32IndexMSR)
5017	{
5018	case MSR_K8_LSTAR: pMixedCtx->msrLSTAR = pMsr->u64Value; break;
5019	case MSR_K6_STAR: pMixedCtx->msrSTAR = pMsr->u64Value; break;
5020	case MSR_K8_SF_MASK: pMixedCtx->msrSFMASK = pMsr->u64Value; break;
5021	case MSR_K8_TSC_AUX: CPUMSetGuestMsr(pVCpu, MSR_K8_TSC_AUX, pMsr->u64Value); break;
5022	#if 0
5023	/* The KERNEL_GS_BASE MSR doesn't work reliably with auto load/store. See @bugref{6208} */
5024	case MSR_K8_KERNEL_GS_BASE: pMixedCtx->msrKERNELGSBASE = pMsr->u64Value; break;
5025	#endif
5026	case MSR_K6_EFER: /* EFER can't be changed without causing a VM-exit. */ break;
5027	default:
5028	{
5029	AssertFailed();
5030	return VERR_HM_UNEXPECTED_LD_ST_MSR;
5031	}
5032	}
5033	}
5034	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS;
5035	return VINF_SUCCESS;
5036	}
5037
5038
5039	/**
5040	* Saves the guest control registers from the current VMCS into the guest-CPU
5041	* context.
5042	*
5043	* @returns VBox status code.
5044	* @param pVM Pointer to the VM.
5045	* @param pVCpu Pointer to the VMCPU.
5046	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5047	* out-of-sync. Make sure to update the required fields
5048	* before using them.
5049	*
5050	* @remarks No-long-jump zone!!!
5051	*/
5052	DECLINLINE(int) hmR0VmxSaveGuestControlRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5053	{
5054	RTGCUINTREG uVal = 0;
5055	RTGCUINTREG uShadow = 0;
5056	int rc = VINF_SUCCESS;
5057
5058	/* Guest CR0. Guest FPU. */
5059	rc = hmR0VmxSaveGuestCR0(pVM, pVCpu, pMixedCtx);
5060	AssertRCReturn(rc, rc);
5061
5062	/* Guest CR4. */
5063	rc = hmR0VmxSaveGuestCR4(pVM, pVCpu, pMixedCtx);
5064	AssertRCReturn(rc, rc);
5065
5066	/* Guest CR3. Only changes with Nested Paging. This must be done -after- saving CR0 and CR4 from the guest! */
5067	if ( pVM->hm.s.fNestedPaging
5068	&& CPUMIsGuestPagingEnabledEx(pMixedCtx))
5069	{
5070	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_CR3))
5071	{
5072	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &uVal);
5073	AssertRCReturn(rc, rc);
5074	if (pMixedCtx->cr3 != uVal)
5075	{
5076	CPUMSetGuestCR3(pVCpu, uVal);
5077	/* Set the force flag to inform PGM about it when necessary. It is cleared by PGMUpdateCR3().*/
5078	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
5079	}
5080
5081	/* We require EFER to check PAE mode. */
5082	rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVM, pVCpu, pMixedCtx);
5083	AssertRCReturn(rc, rc);
5084
5085	/* If the guest is in PAE mode, sync back the PDPE's into the guest state. */
5086	if (CPUMIsGuestInPAEModeEx(pMixedCtx)) /* Reads CR0, CR4 and EFER MSR. */
5087	{
5088	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u);
5089	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u);
5090	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u);
5091	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u);
5092	AssertRCReturn(rc, rc);
5093	/* Set the force flag to inform PGM about it when necessary. It is cleared by PGMGstUpdatePaePdpes(). */
5094	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
5095	}
5096
5097	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_CR3;
5098	}
5099	}
5100	else
5101	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_CR3;
5102	return rc;
5103	}
5104
5105
5106	/**
5107	* Reads a guest segment register from the current VMCS into the guest-CPU
5108	* context.
5109	*
5110	* @returns VBox status code.
5111	* @param idxSel Index of the selector in the VMCS.
5112	* @param idxLimit Index of the segment limit in the VMCS.
5113	* @param idxBase Index of the segment base in the VMCS.
5114	* @param idxAccess Index of the access rights of the segment in the VMCS.
5115	* @param pSelReg Pointer to the segment selector.
5116	*
5117	* @remarks No-long-jump zone!!!
5118	*/
5119	DECLINLINE(int) hmR0VmxReadSegmentReg(uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase, uint32_t idxAccess,
5120	PCPUMSELREG pSelReg)
5121	{
5122	uint32_t u32Val = 0;
5123	int rc = VMXReadVmcs32(idxSel, &u32Val); AssertRCReturn(rc, rc);
5124	pSelReg->Sel = (uint16_t)u32Val;
5125	pSelReg->ValidSel = (uint16_t)u32Val;
5126	pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
5127
5128	rc = VMXReadVmcs32(idxLimit, &u32Val); AssertRCReturn(rc, rc);
5129	pSelReg->u32Limit = u32Val;
5130
5131	RTGCUINTREG uGCVal = 0;
5132	rc = VMXReadVmcsGstN(idxBase, &uGCVal); AssertRCReturn(rc, rc);
5133	pSelReg->u64Base = uGCVal;
5134
5135	rc = VMXReadVmcs32(idxAccess, &u32Val); AssertRCReturn(rc, rc);
5136	pSelReg->Attr.u = u32Val;
5137
5138	/*
5139	* If VT-x marks the segment as unusable, the rest of the attributes are undefined.
5140	* See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers.
5141	*/
5142	if (pSelReg->Attr.u & VMX_SEL_UNUSABLE)
5143	{
5144	Assert(idxSel != VMX_VMCS16_GUEST_FIELD_TR);
5145	pSelReg->Attr.u = VMX_SEL_UNUSABLE;
5146	}
5147	return rc;
5148	}
5149
5150
5151	/**
5152	* Saves the guest segment registers from the current VMCS into the guest-CPU
5153	* context.
5154	*
5155	* @returns VBox status code.
5156	* @param pVM Pointer to the VM.
5157	* @param pVCpu Pointer to the VMCPU.
5158	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5159	* out-of-sync. Make sure to update the required fields
5160	* before using them.
5161	*
5162	* @remarks No-long-jump zone!!!
5163	*/
5164	static int hmR0VmxSaveGuestSegmentRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5165	{
5166	int rc = VINF_SUCCESS;
5167
5168	/* Guest segment registers. */
5169	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_SEGMENT_REGS))
5170	{
5171	rc = hmR0VmxSaveGuestCR0(pVM, pVCpu, pMixedCtx);
5172	AssertRCReturn(rc, rc);
5173
5174	rc = hmR0VmxReadSegmentReg(VMX_VMCS16_GUEST_FIELD_CS, VMX_VMCS32_GUEST_CS_LIMIT, VMX_VMCS_GUEST_CS_BASE,
5175	VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS, &pMixedCtx->cs);
5176	rc \|= hmR0VmxReadSegmentReg(VMX_VMCS16_GUEST_FIELD_SS, VMX_VMCS32_GUEST_SS_LIMIT, VMX_VMCS_GUEST_SS_BASE,
5177	VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS, &pMixedCtx->ss);
5178	rc \|= hmR0VmxReadSegmentReg(VMX_VMCS16_GUEST_FIELD_DS, VMX_VMCS32_GUEST_DS_LIMIT, VMX_VMCS_GUEST_DS_BASE,
5179	VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS, &pMixedCtx->ds);
5180	rc \|= hmR0VmxReadSegmentReg(VMX_VMCS16_GUEST_FIELD_ES, VMX_VMCS32_GUEST_ES_LIMIT, VMX_VMCS_GUEST_ES_BASE,
5181	VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS, &pMixedCtx->es);
5182	rc \|= hmR0VmxReadSegmentReg(VMX_VMCS16_GUEST_FIELD_FS, VMX_VMCS32_GUEST_FS_LIMIT, VMX_VMCS_GUEST_FS_BASE,
5183	VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS, &pMixedCtx->fs);
5184	rc \|= hmR0VmxReadSegmentReg(VMX_VMCS16_GUEST_FIELD_GS, VMX_VMCS32_GUEST_GS_LIMIT, VMX_VMCS_GUEST_GS_BASE,
5185	VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS, &pMixedCtx->gs);
5186	AssertRCReturn(rc, rc);
5187
5188	/* Restore segment attributes for real-on-v86 mode hack. */
5189	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
5190	{
5191	pMixedCtx->cs.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrCS.u;
5192	pMixedCtx->ss.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrSS.u;
5193	pMixedCtx->ds.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrDS.u;
5194	pMixedCtx->es.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrES.u;
5195	pMixedCtx->fs.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrFS.u;
5196	pMixedCtx->gs.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrGS.u;
5197	}
5198
5199	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_SEGMENT_REGS;
5200	}
5201
5202	/* Guest LDTR. */
5203	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_LDTR))
5204	{
5205	rc = hmR0VmxReadSegmentReg(VMX_VMCS16_GUEST_FIELD_LDTR, VMX_VMCS32_GUEST_LDTR_LIMIT, VMX_VMCS_GUEST_LDTR_BASE,
5206	VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, &pMixedCtx->ldtr);
5207	AssertRCReturn(rc, rc);
5208	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_LDTR;
5209	}
5210
5211	/* Guest GDTR. */
5212	RTGCUINTREG uGCVal = 0;
5213	uint32_t u32Val = 0;
5214	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_GDTR))
5215	{
5216	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &uGCVal);
5217	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
5218	pMixedCtx->gdtr.pGdt = uGCVal;
5219	pMixedCtx->gdtr.cbGdt = u32Val;
5220	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_GDTR;
5221	}
5222
5223	/* Guest IDTR. */
5224	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_IDTR))
5225	{
5226	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &uGCVal);
5227	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
5228	pMixedCtx->idtr.pIdt = uGCVal;
5229	pMixedCtx->idtr.cbIdt = u32Val;
5230	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_IDTR;
5231	}
5232
5233	/* Guest TR. */
5234	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_TR))
5235	{
5236	rc = hmR0VmxSaveGuestCR0(pVM, pVCpu, pMixedCtx);
5237	AssertRCReturn(rc, rc);
5238
5239	/* For real-mode emulation using virtual-8086 mode we have the fake TSS (pRealModeTSS) in TR, don't sync the fake one. */
5240	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
5241	{
5242	rc = hmR0VmxReadSegmentReg(VMX_VMCS16_GUEST_FIELD_TR, VMX_VMCS32_GUEST_TR_LIMIT, VMX_VMCS_GUEST_TR_BASE,
5243	VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, &pMixedCtx->tr);
5244	AssertRCReturn(rc, rc);
5245	}
5246	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_TR;
5247	}
5248	return rc;
5249	}
5250
5251
5252	/**
5253	* Saves the guest debug registers from the current VMCS into the guest-CPU
5254	* context.
5255	*
5256	* @returns VBox status code.
5257	* @param pVM Pointer to the VM.
5258	* @param pVCpu Pointer to the VMCPU.
5259	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5260	* out-of-sync. Make sure to update the required fields
5261	* before using them.
5262	*
5263	* @remarks No-long-jump zone!!!
5264	*/
5265	DECLINLINE(int) hmR0VmxSaveGuestDebugRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5266	{
5267	int rc = VINF_SUCCESS;
5268	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_DEBUG))
5269	{
5270	RTGCUINTREG uVal;
5271	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_DR7, &uVal); AssertRCReturn(rc, rc);
5272	pMixedCtx->dr[7] = uVal;
5273
5274	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_DEBUG;
5275	}
5276	return rc;
5277	}
5278
5279
5280	/**
5281	* Saves the guest APIC state from the currentl VMCS into the guest-CPU context.
5282	*
5283	* @returns VBox status code.
5284	* @param pVM Pointer to the VM.
5285	* @param pVCpu Pointer to the VMCPU.
5286	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5287	* out-of-sync. Make sure to update the required fields
5288	* before using them.
5289	*
5290	* @remarks No-long-jump zone!!!
5291	*/
5292	DECLINLINE(int) hmR0VmxSaveGuestApicState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5293	{
5294	/* Updating TPR is already done in hmR0VmxPostRunGuest(). Just update the flag. */
5295	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_APIC_STATE;
5296	return VINF_SUCCESS;
5297	}
5298
5299
5300	/**
5301	* Saves the entire guest state from the currently active VMCS into the
5302	* guest-CPU context. This essentially VMREADs all guest-data.
5303	*
5304	* @returns VBox status code.
5305	* @param pVM Pointer to the VM.
5306	* @param pVCpu Pointer to the VMCPU.
5307	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5308	* out-of-sync. Make sure to update the required fields
5309	* before using them.
5310	*/
5311	static int hmR0VmxSaveGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5312	{
5313	Assert(pVM);
5314	Assert(pVCpu);
5315	Assert(pMixedCtx);
5316
5317	if (pVCpu->hm.s.vmx.fUpdatedGuestState == VMX_UPDATED_GUEST_ALL)
5318	return VINF_SUCCESS;
5319
5320	VMMRZCallRing3Disable(pVCpu); /* We're not using hmR0VmxCallRing3Disable() as we want to respect nesting here. */
5321
5322	int rc = hmR0VmxSaveGuestGprs(pVM, pVCpu, pMixedCtx);
5323	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestGprs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
5324
5325	rc = hmR0VmxSaveGuestControlRegs(pVM, pVCpu, pMixedCtx);
5326	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestControlRegs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
5327
5328	rc = hmR0VmxSaveGuestSegmentRegs(pVM, pVCpu, pMixedCtx);
5329	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSegmentRegs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
5330
5331	rc = hmR0VmxSaveGuestDebugRegs(pVM, pVCpu, pMixedCtx);
5332	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestDebugRegs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
5333
5334	rc = hmR0VmxSaveGuestSysenterMsrs(pVM, pVCpu, pMixedCtx);
5335	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSysenterMsrs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
5336
5337	rc = hmR0VmxSaveGuestFSBaseMsr(pVM, pVCpu, pMixedCtx);
5338	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestFSBaseMsr failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
5339
5340	rc = hmR0VmxSaveGuestGSBaseMsr(pVM, pVCpu, pMixedCtx);
5341	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestGSBaseMsr failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
5342
5343	rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVM, pVCpu, pMixedCtx);
5344	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestAutoLoadStoreMsrs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
5345
5346	rc = hmR0VmxSaveGuestIntrState(pVM, pVCpu, pMixedCtx);
5347	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestIntrState failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
5348
5349	rc = hmR0VmxSaveGuestActivityState(pVM, pVCpu, pMixedCtx);
5350	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestActivityState failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
5351
5352	rc = hmR0VmxSaveGuestApicState(pVM, pVCpu, pMixedCtx);
5353	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestDebugRegs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
5354
5355	AssertMsg(pVCpu->hm.s.vmx.fUpdatedGuestState == VMX_UPDATED_GUEST_ALL,
5356	("Missed guest state bits while saving state; residue %RX32\n", pVCpu->hm.s.vmx.fUpdatedGuestState));
5357
5358	VMMRZCallRing3Enable(pVCpu);
5359	return rc;
5360	}
5361
5362
5363	/**
5364	* Check per-VM and per-VCPU force flag actions that require us to go back to
5365	* ring-3 for one reason or another.
5366	*
5367	* @returns VBox status code (information status code included).
5368	* @retval VINF_SUCCESS if we don't have any actions that require going back to
5369	* ring-3.
5370	* @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
5371	* @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
5372	* interrupts)
5373	* @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
5374	* all EMTs to be in ring-3.
5375	* @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
5376	* @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
5377	* to the EM loop.
5378	*
5379	* @param pVM Pointer to the VM.
5380	* @param pVCpu Pointer to the VMCPU.
5381	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5382	* out-of-sync. Make sure to update the required fields
5383	* before using them.
5384	*/
5385	static int hmR0VmxCheckForceFlags(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5386	{
5387	Assert(VMMRZCallRing3IsEnabled(pVCpu));
5388
5389	int rc = VERR_INTERNAL_ERROR_5;
5390	if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK \| VM_FF_REQUEST \| VM_FF_PGM_POOL_FLUSH_PENDING \| VM_FF_PDM_DMA)
5391	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK \| VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL
5392	\| VMCPU_FF_REQUEST \| VMCPU_FF_HM_UPDATE_CR3 \| VMCPU_FF_HM_UPDATE_PAE_PDPES))
5393	{
5394	/* We need the control registers now, make sure the guest-CPU context is updated. */
5395	rc = hmR0VmxSaveGuestControlRegs(pVM, pVCpu, pMixedCtx);
5396	AssertRCReturn(rc, rc);
5397
5398	/* Pending HM CR3 sync. */
5399	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
5400	{
5401	rc = PGMUpdateCR3(pVCpu, pMixedCtx->cr3);
5402	Assert(rc == VINF_SUCCESS \|\| rc == VINF_PGM_SYNC_CR3);
5403	}
5404	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
5405	{
5406	rc = PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
5407	AssertRCReturn(rc, rc);
5408	}
5409
5410	/* Pending PGM C3 sync. */
5411	if (VMCPU_FF_IS_PENDING(pVCpu,VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
5412	{
5413	rc = PGMSyncCR3(pVCpu, pMixedCtx->cr0, pMixedCtx->cr3, pMixedCtx->cr4, VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
5414	if (rc != VINF_SUCCESS)
5415	{
5416	AssertRC(rc);
5417	Log2(("hmR0VmxCheckForceFlags: PGMSyncCR3 forcing us back to ring-3. rc=%d\n", rc));
5418	return rc;
5419	}
5420	}
5421
5422	/* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
5423	/* -XXX- what was that about single stepping? */
5424	if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK)
5425	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
5426	{
5427	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
5428	rc = RT_UNLIKELY(VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
5429	Log2(("hmR0VmxCheckForceFlags: HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc));
5430	return rc;
5431	}
5432
5433	/* Pending VM request packets, such as hardware interrupts. */
5434	if ( VM_FF_IS_PENDING(pVM, VM_FF_REQUEST)
5435	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_REQUEST))
5436	{
5437	Log2(("hmR0VmxCheckForceFlags: Pending VM request forcing us back to ring-3\n"));
5438	return VINF_EM_PENDING_REQUEST;
5439	}
5440
5441	/* Pending PGM pool flushes. */
5442	if (VM_FF_IS_PENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
5443	{
5444	Log2(("hmR0VmxCheckForceFlags: PGM pool flush pending forcing us back to ring-3\n"));
5445	return rc = VINF_PGM_POOL_FLUSH_PENDING;
5446	}
5447
5448	/* Pending DMA requests. */
5449	if (VM_FF_IS_PENDING(pVM, VM_FF_PDM_DMA))
5450	{
5451	Log2(("hmR0VmxCheckForceFlags: Pending DMA request forcing us back to ring-3\n"));
5452	return VINF_EM_RAW_TO_R3;
5453	}
5454	}
5455
5456	/* Paranoia. */
5457	Assert(rc != VERR_EM_INTERPRETER);
5458	return VINF_SUCCESS;
5459	}
5460
5461
5462	/**
5463	* Converts any pending VMX event into a TRPM trap. Typically used when leaving
5464	* VT-x to execute any instruction.
5465	*
5466	* @param pvCpu Pointer to the VMCPU.
5467	*/
5468	static void hmR0VmxUpdateTRPMTrap(PVMCPU pVCpu)
5469	{
5470	if (pVCpu->hm.s.Event.fPending)
5471	{
5472	uint32_t uVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVCpu->hm.s.Event.u64IntrInfo);
5473	uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVCpu->hm.s.Event.u64IntrInfo);
5474	bool fErrorCodeValid = VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntrInfo);
5475	uint32_t uErrorCode = pVCpu->hm.s.Event.u32ErrCode;
5476
5477	/* If a trap was already pending, we did something wrong! */
5478	Assert(TRPMQueryTrap(pVCpu, NULL, NULL) == VERR_TRPM_NO_ACTIVE_TRAP);
5479
5480	/* A page-fault exception during a page-fault would become a double-fault. */
5481	AssertMsg(uVectorType != VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT \|\| uVector != X86_XCPT_PF,
5482	("%#RX64 uVectorType=%#x uVector=%#x\n", pVCpu->hm.s.Event.u64IntrInfo, uVectorType, uVector));
5483
5484	TRPMEVENT enmTrapType;
5485	switch (uVectorType)
5486	{
5487	case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
5488	case VMX_IDT_VECTORING_INFO_TYPE_NMI:
5489	enmTrapType = TRPM_HARDWARE_INT;
5490	break;
5491	case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
5492	enmTrapType = TRPM_SOFTWARE_INT;
5493	break;
5494	case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
5495	case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: /* #BP and #OF */
5496	case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
5497	enmTrapType = TRPM_TRAP;
5498	break;
5499	default:
5500	AssertMsgFailed(("Invalid trap type %#x\n", uVectorType));
5501	enmTrapType = TRPM_32BIT_HACK;
5502	break;
5503	}
5504	int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
5505	AssertRC(rc);
5506	if (fErrorCodeValid)
5507	TRPMSetErrorCode(pVCpu, uErrorCode);
5508
5509	/* Clear the VT-x state bits now that TRPM has the information. */
5510	pVCpu->hm.s.Event.fPending = false;
5511	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, 0);
5512	AssertRC(rc);
5513	}
5514	}
5515
5516
5517	/**
5518	* Does the necessary state syncing before doing a longjmp to ring-3.
5519	*
5520	* @param pVM Pointer to the VM.
5521	* @param pVCpu Pointer to the VMCPU.
5522	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5523	* out-of-sync. Make sure to update the required fields
5524	* before using them.
5525	* @param rcExit The reason for exiting to ring-3. Can be
5526	* VINF_VMM_UNKNOWN_RING3_CALL.
5527	*
5528	* @remarks No-long-jmp zone!!!
5529	*/
5530	static void hmR0VmxLongJmpToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, int rcExit)
5531	{
5532	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
5533	Log(("hmR0VmxLongJmpToRing3: rcExit=%d\n", rcExit));
5534
5535	/* We're going back to ring-3, clear the flag that we need to go back to ring-3. */
5536	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
5537
5538	int rc = hmR0VmxSaveGuestState(pVM, pVCpu, pMixedCtx);
5539	AssertRC(rc);
5540
5541	/* Restore debug registers if necessary and resync on next R0 re-entry. */
5542	if (CPUMIsGuestDebugStateActive(pVCpu))
5543	{
5544	CPUMR0SaveGuestDebugState(pVM, pVCpu, pMixedCtx, true /* save DR6 */);
5545	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_DEBUG;
5546	}
5547	else if (CPUMIsHyperDebugStateActive(pVCpu))
5548	{
5549	CPUMR0LoadHostDebugState(pVM, pVCpu);
5550	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_DEBUG;
5551	}
5552
5553	/* Signal changes to the recompiler. */
5554	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
5555	\| CPUM_CHANGED_LDTR
5556	\| CPUM_CHANGED_GDTR
5557	\| CPUM_CHANGED_IDTR
5558	\| CPUM_CHANGED_TR
5559	\| CPUM_CHANGED_HIDDEN_SEL_REGS);
5560
5561	/* On our way back from ring-3 the following needs to be done. */
5562	/** @todo This can change with preemption hooks. */
5563	if (rcExit == VINF_EM_RAW_INTERRUPT)
5564	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_HOST_CONTEXT;
5565	else
5566	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_ALL_GUEST;
5567
5568	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchToR3);
5569	}
5570
5571
5572	/**
5573	* An action requires us to go back to ring-3. This function does the necessary
5574	* steps before we can safely return to ring-3. This is not the same as longjmps
5575	* to ring-3, this is voluntary.
5576	*
5577	* @param pVM Pointer to the VM.
5578	* @param pVCpu Pointer to the VMCPU.
5579	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5580	* out-of-sync. Make sure to update the required fields
5581	* before using them.
5582	* @param rcExit The reason for exiting to ring-3. Can be
5583	* VINF_VMM_UNKNOWN_RING3_CALL.
5584	*/
5585	static void hmR0VmxExitToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, int rcExit)
5586	{
5587	Assert(pVM);
5588	Assert(pVCpu);
5589	Assert(pMixedCtx);
5590	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
5591
5592	/* We want to see what the guest-state was before VM-entry, don't resync here, as we will never continue guest execution.*/
5593	if (rcExit == VERR_VMX_INVALID_GUEST_STATE)
5594	return;
5595
5596	/* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
5597	VMMRZCallRing3Disable(pVCpu);
5598	Log(("hmR0VmxExitToRing3: rcExit=%d\n", rcExit));
5599
5600	/* We need to do this only while truly exiting the "inner loop" back to ring-3 and -not- for any longjmp to ring3. */
5601	hmR0VmxUpdateTRPMTrap(pVCpu);
5602
5603	/* Sync. the rest of the state before going back to ring-3. */
5604	hmR0VmxLongJmpToRing3(pVM, pVCpu, pMixedCtx, rcExit);
5605
5606	VMMRZCallRing3Enable(pVCpu);
5607	}
5608
5609
5610	/**
5611	* VMMRZCallRing3 callback wrapper which saves the guest state before we
5612	* longjump to ring-3 and possibly get preempted.
5613	*
5614	* @param pVCpu Pointer to the VMCPU.
5615	* @param enmOperation The operation causing the ring-3 longjump.
5616	* @param pvUser The user argument (pointer to the possibly
5617	* out-of-date guest-CPU context).
5618	*
5619	* @remarks Must never be called with @a enmOperation ==
5620	* VMMCALLRING3_VM_R0_ASSERTION.
5621	*/
5622	DECLCALLBACK(void) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
5623	{
5624	/* VMMRZCallRing3() already makes sure we never get called as a result of an longjmp due to an assertion, */
5625	Assert(pVCpu);
5626	Assert(pvUser);
5627	Assert(VMMRZCallRing3IsEnabled(pVCpu));
5628	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
5629
5630	VMMRZCallRing3Disable(pVCpu);
5631	hmR0VmxLongJmpToRing3(pVCpu->CTX_SUFF(pVM), pVCpu, (PCPUMCTX)pvUser, VINF_VMM_UNKNOWN_RING3_CALL);
5632	VMMRZCallRing3Enable(pVCpu);
5633	}
5634
5635
5636	/**
5637	* Injects any pending TRPM trap into the VM by updating the VMCS.
5638	*
5639	* @returns VBox status code (informational status code included).
5640	* @param pVM Pointer to the VM.
5641	* @param pVCpu Pointer to the VMCPU.
5642	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5643	* out-of-sync. Make sure to update the required fields
5644	* before using them.
5645	*/
5646	static int hmR0VmxInjectTRPMTrap(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5647	{
5648	if (!TRPMHasTrap(pVCpu))
5649	return VINF_SUCCESS;
5650
5651	uint8_t u8Vector = 0;
5652	TRPMEVENT enmTrpmEvent = TRPM_SOFTWARE_INT;
5653	RTGCUINT uErrCode = 0;
5654
5655	int rc = TRPMQueryTrapAll(pVCpu, &u8Vector, &enmTrpmEvent, &uErrCode, NULL /* puCr2 */);
5656	AssertRCReturn(rc, rc);
5657	Assert(enmTrpmEvent != TRPM_SOFTWARE_INT);
5658
5659	rc = TRPMResetTrap(pVCpu);
5660	AssertRCReturn(rc, rc);
5661
5662	/* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntrInfo. */
5663	uint32_t u32IntrInfo = u8Vector \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
5664	if (enmTrpmEvent == TRPM_TRAP)
5665	{
5666	switch (u8Vector)
5667	{
5668	case X86_XCPT_BP:
5669	case X86_XCPT_OF:
5670	{
5671	/* These exceptions must be delivered as software exceptions. They have no error codes associated with them. */
5672	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5673	break;
5674	}
5675
5676	case X86_XCPT_DF:
5677	case X86_XCPT_TS:
5678	case X86_XCPT_NP:
5679	case X86_XCPT_SS:
5680	case X86_XCPT_GP:
5681	case X86_XCPT_PF:
5682	case X86_XCPT_AC:
5683	/* These exceptions must be delivered as hardware exceptions. They have error codes associated with
5684	them which VT-x/VMM pushes to the guest stack. */
5685	u32IntrInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
5686	/* no break! */
5687	default:
5688	{
5689	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5690	break;
5691	}
5692	}
5693	}
5694	else if (enmTrpmEvent == TRPM_HARDWARE_INT)
5695	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5696	else
5697	AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
5698
5699	STAM_COUNTER_INC(&pVCpu->hm.s.StatIntInject);
5700	return hmR0VmxInjectEventVmcs(pVM, pVCpu, pMixedCtx, u32IntrInfo, 0 /* cbInstr */, uErrCode);
5701	}
5702
5703
5704	/**
5705	* Checks if there are any pending guest interrupts to be delivered and injects
5706	* them into the VM by updating the VMCS.
5707	*
5708	* @returns VBox status code (informational status codes included).
5709	* @param pVM Pointer to the VM.
5710	* @param pVCpu Pointer to the VMCPU.
5711	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5712	* out-of-sync. Make sure to update the required fields
5713	* before using them.
5714	*
5715	* @remarks This must be called only after hmR0VmxSaveGuestIntrState().
5716	*/
5717	static int hmR0VmxInjectPendingInterrupt(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5718	{
5719	/* First inject any pending HM interrupts. */
5720	if (pVCpu->hm.s.Event.fPending)
5721	{
5722	int rc = hmR0VmxInjectEventVmcs(pVM, pVCpu, pMixedCtx, pVCpu->hm.s.Event.u64IntrInfo, 0 /* cbInstr */,
5723	pVCpu->hm.s.Event.u32ErrCode);
5724	AssertRCReturn(rc, rc);
5725	pVCpu->hm.s.Event.fPending = false;
5726	return rc;
5727	}
5728
5729	/** @todo SMI. SMIs take priority over NMIs. */
5730
5731	/* NMI. NMIs take priority over regular interrupts . */
5732	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI))
5733	{
5734	/* Construct an NMI interrupt and inject it into the VMCS. */
5735	RTGCUINTPTR uIntrInfo;
5736	uIntrInfo = X86_XCPT_NMI;
5737	uIntrInfo \|= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
5738	uIntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5739	int rc = hmR0VmxInjectEventVmcs(pVM, pVCpu, pMixedCtx, uIntrInfo, 0 /* cbInstr /, 0 / u32ErrCode */);
5740	AssertRCReturn(rc, rc);
5741	return rc;
5742	}
5743
5744	/* We need the guests's RFLAGS for sure from this point on, make sure it is updated. */
5745	int rc = hmR0VmxSaveGuestRflags(pVM, pVCpu, pMixedCtx);
5746	AssertRCReturn(rc, rc);
5747
5748	/* If there isn't any active trap, check if we have pending interrupts and convert them to TRPM traps and deliver them. */
5749	if (!TRPMHasTrap(pVCpu))
5750	{
5751	/* Check if there are guest external interrupts (PIC/APIC) pending. */
5752	if (VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC)))
5753	{
5754	/*
5755	* When external interrupts are pending and the guest has disabled interrupts, cause a VM-exit using "interrupt-window
5756	* exiting" so we can deliver the interrupt when the guest is ready to receive them. Otherwise, if the guest
5757	* can receive interrupts now, convert the PDM interrupt into a TRPM event and inject it.
5758	*/
5759	if (!(pMixedCtx->eflags.u32 & X86_EFL_IF)) /** @todo we can use interrupt-window exiting for block-by-STI. */
5760	{
5761	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INT_WINDOW_EXIT))
5762	{
5763	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INT_WINDOW_EXIT;
5764	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls);
5765	AssertRCReturn(rc, rc);
5766	}
5767	/* else we will deliver interrupts whenever the guest exits next and it's in a state to receive interrupts. */
5768	}
5769	else if (!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
5770	{
5771	uint8_t u8Interrupt = 0;
5772	rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
5773	if (RT_SUCCESS(rc))
5774	{
5775	/* Convert pending interrupt from PIC/APIC into TRPM and handle it below. */
5776	rc = TRPMAssertTrap(pVCpu, u8Interrupt, TRPM_HARDWARE_INT);
5777	AssertRCReturn(rc, rc);
5778	}
5779	else
5780	{
5781	/** @todo Does this actually happen? If not turn it into an assertion. */
5782	Assert(!VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC)));
5783	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
5784	}
5785	}
5786	}
5787	}
5788
5789	/* If interrupts can be delivered, inject it into the VM. */
5790	if ( (pMixedCtx->eflags.u32 & X86_EFL_IF)
5791	&& !VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS)
5792	&& TRPMHasTrap(pVCpu))
5793	{
5794	rc = hmR0VmxInjectTRPMTrap(pVM, pVCpu, pMixedCtx);
5795	AssertRCReturn(rc, rc);
5796	}
5797	return rc;
5798	}
5799
5800	/**
5801	* Injects an invalid-opcode (#UD) exception into the VM.
5802	*
5803	* @returns VBox status code (informational status code included).
5804	* @param pVM Pointer to the VM.
5805	* @param pVCpu Pointer to the VMCPU.
5806	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5807	* out-of-sync. Make sure to update the required fields
5808	* before using them.
5809	*/
5810	DECLINLINE(int) hmR0VmxInjectXcptUD(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5811	{
5812	/* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntrInfo. */
5813	uint32_t u32IntrInfo = X86_XCPT_UD \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
5814	STAM_COUNTER_INC(&pVCpu->hm.s.StatIntInject);
5815	return hmR0VmxInjectEventVmcs(pVM, pVCpu, pMixedCtx, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode */);
5816	}
5817
5818
5819	/**
5820	* Injects a double-fault (#DF) exception into the VM.
5821	*
5822	* @returns VBox status code (informational status code included).
5823	* @param pVM Pointer to the VM.
5824	* @param pVCpu Pointer to the VMCPU.
5825	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5826	* out-of-sync. Make sure to update the required fields
5827	* before using them.
5828	*/
5829	DECLINLINE(int) hmR0VmxInjectXcptDF(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5830	{
5831	/* Inject the double-fault. */
5832	uint32_t u32IntrInfo = X86_XCPT_DF \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
5833	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5834	u32IntrInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
5835	STAM_COUNTER_INC(&pVCpu->hm.s.StatIntInject);
5836	return hmR0VmxInjectEventVmcs(pVM, pVCpu, pMixedCtx, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode */);
5837	}
5838
5839
5840	/**
5841	* Injects a debug (#DB) exception into the VM.
5842	*
5843	* @returns VBox status code (informational status code included).
5844	* @param pVM Pointer to the VM.
5845	* @param pVCpu Pointer to the VMCPU.
5846	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5847	* out-of-sync. Make sure to update the required fields
5848	* before using them.
5849	*/
5850	DECLINLINE(int) hmR0VmxInjectXcptDB(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5851	{
5852	/* Inject the debug-exception. */
5853	uint32_t u32IntrInfo = X86_XCPT_DB \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
5854	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5855	STAM_COUNTER_INC(&pVCpu->hm.s.StatIntInject);
5856	return hmR0VmxInjectEventVmcs(pVM, pVCpu, pMixedCtx, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode */);
5857	}
5858
5859
5860	/**
5861	* Injects a overflow (#OF) exception into the VM.
5862	*
5863	* @returns VBox status code (informational status code included).
5864	* @param pVM Pointer to the VM.
5865	* @param pVCpu Pointer to the VMCPU.
5866	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5867	* out-of-sync. Make sure to update the required fields
5868	* before using them.
5869	* @param cbInstr The value of RIP that is to be pushed on the guest
5870	* stack.
5871	*/
5872	DECLINLINE(int) hmR0VmxInjectXcptOF(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
5873	{
5874	/* Inject the overflow exception. */
5875	uint32_t u32IntrInfo = X86_XCPT_OF \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
5876	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5877	STAM_COUNTER_INC(&pVCpu->hm.s.StatIntInject);
5878	return hmR0VmxInjectEventVmcs(pVM, pVCpu, pMixedCtx, u32IntrInfo, cbInstr, 0 /* u32ErrCode */);
5879	}
5880
5881
5882	/**
5883	* Injects a general-protection (#GP) fault into the VM.
5884	*
5885	* @returns VBox status code (informational status code included).
5886	* @param pVM Pointer to the VM.
5887	* @param pVCpu Pointer to the VMCPU.
5888	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5889	* out-of-sync. Make sure to update the required fields
5890	* before using them.
5891	* @param u32ErrorCode The error code associated with the #GP.
5892	*/
5893	DECLINLINE(int) hmR0VmxInjectXcptGP(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fErrorCodeValid, uint32_t u32ErrorCode)
5894	{
5895	/* Inject the general-protection fault. */
5896	uint32_t u32IntrInfo = X86_XCPT_GP \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
5897	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5898	if (fErrorCodeValid)
5899	u32IntrInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
5900	STAM_COUNTER_INC(&pVCpu->hm.s.StatIntInject);
5901	return hmR0VmxInjectEventVmcs(pVM, pVCpu, pMixedCtx, u32IntrInfo, 0 /* cbInstr */, u32ErrorCode);
5902	}
5903
5904
5905	/**
5906	* Injects a software interrupt (INTn) into the VM.
5907	*
5908	* @returns VBox status code (informational status code included).
5909	* @param pVM Pointer to the VM.
5910	* @param pVCpu Pointer to the VMCPU.
5911	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5912	* out-of-sync. Make sure to update the required fields
5913	* before using them.
5914	* @param uVector The software interrupt vector number.
5915	* @param cbInstr The value of RIP that is to be pushed on the guest
5916	* stack.
5917	*/
5918	DECLINLINE(int) hmR0VmxInjectIntN(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint16_t uVector, uint32_t cbInstr)
5919	{
5920	/* Inject the INTn. */
5921	uint32_t u32IntrInfo = uVector \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
5922	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5923	STAM_COUNTER_INC(&pVCpu->hm.s.StatIntInject);
5924	return hmR0VmxInjectEventVmcs(pVM, pVCpu, pMixedCtx, u32IntrInfo, cbInstr, 0 /* u32ErrCode */);
5925	}
5926
5927
5928	/**
5929	* Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
5930	* stack.
5931	*
5932	* @returns VBox status code (information status code included).
5933	* @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
5934	* @param pVM Pointer to the VM.
5935	* @param pMixedCtx Pointer to the guest-CPU context.
5936	* @param uValue The value to push to the guest stack.
5937	*/
5938	static int hmR0VmxRealModeGuestStackPush(PVM pVM, PCPUMCTX pMixedCtx, uint16_t uValue)
5939	{
5940	/*
5941	* The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
5942	* virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
5943	* See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
5944	*/
5945	if (pMixedCtx->sp == 1)
5946	return VINF_EM_RESET;
5947	pMixedCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
5948	int rc = PGMPhysSimpleWriteGCPhys(pVM, pMixedCtx->ss.u64Base + pMixedCtx->sp, &uValue, sizeof(uint16_t));
5949	AssertRCReturn(rc, rc);
5950	return rc;
5951	}
5952
5953
5954	/**
5955	* Injects an event into the guest upon VM-entry by updating the relevant fields
5956	* in the VM-entry area in the VMCS.
5957	*
5958	* @returns VBox status code (informational error codes included).
5959	* @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
5960	* @retval VINF_EM_RESET if event injection resulted in a triple-fault.
5961	*
5962	* @param pVM Pointer to the VM.
5963	* @param pVCpu Pointer to the VMCPU.
5964	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5965	* out-of-sync. Make sure to update the required fields
5966	* before using them.
5967	* @param u64IntrInfo The VM-entry interruption-information field.
5968	* @param cbInstr The VM-entry instruction length in bytes (for software
5969	* interrupts, exceptions and privileged software
5970	* exceptions).
5971	* @param u32ErrCode The VM-entry exception error code.
5972	*
5973	* @remarks No-long-jump zone!!!
5974	*/
5975	static int hmR0VmxInjectEventVmcs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntrInfo, uint32_t cbInstr,
5976	uint32_t u32ErrCode)
5977	{
5978	/* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
5979	AssertMsg(u64IntrInfo >> 32 == 0, ("%#RX64\n", u64IntrInfo));
5980	uint32_t u32IntrInfo = (uint32_t)u64IntrInfo;
5981
5982	/* We require CR0 to check if the guest is in real-mode. */
5983	int rc = hmR0VmxSaveGuestCR0(pVM, pVCpu, pMixedCtx);
5984	AssertRCReturn(rc, rc);
5985
5986	const uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(u32IntrInfo);
5987	STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
5988
5989	/*
5990	* Hardware interrupts & exceptions cannot be delivered through the software interrupt redirection bitmap to the real
5991	* mode task in virtual-8086 mode. We must jump to the interrupt handler in the (real-mode) guest.
5992	* See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode" for interrupt & exception classes.
5993	* See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
5994	*/
5995	if (CPUMIsGuestInRealModeEx(pMixedCtx))
5996	{
5997	if (!pVM->hm.s.vmx.fUnrestrictedGuest)
5998	{
5999	Assert(PDMVmmDevHeapIsEnabled(pVM));
6000	Assert(pVM->hm.s.vmx.pRealModeTSS);
6001
6002	/* Save the required guest state bits from the VMCS. */
6003	rc = hmR0VmxSaveGuestSegmentRegs(pVM, pVCpu, pMixedCtx);
6004	rc \|= hmR0VmxSaveGuestGprs(pVM, pVCpu, pMixedCtx);
6005	AssertRCReturn(rc, rc);
6006
6007	/* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
6008	const size_t cbIdtEntry = 4;
6009	if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pMixedCtx->idtr.cbIdt)
6010	{
6011	/* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
6012	if (uVector == X86_XCPT_DF)
6013	return VINF_EM_RESET;
6014	else if (uVector == X86_XCPT_GP)
6015	{
6016	/* If we're injecting a #GP with no valid IDT entry, inject a double-fault. */
6017	return hmR0VmxInjectXcptDF(pVM, pVCpu, pMixedCtx);
6018	}
6019
6020	/* If we're injecting an interrupt/exception with no valid IDT entry, inject a general-protection fault. */
6021	/* No error codes for exceptions in real-mode. See Intel spec. 20.1.4 "Interrupt and Exception Handling" */
6022	return hmR0VmxInjectXcptGP(pVM, pVCpu, pMixedCtx, false /* fErrCodeValid /, 0 / u32ErrCode */);
6023	}
6024
6025	/* Software exceptions (#BP and #OF exceptions thrown as a result of INT 3 or INTO) */
6026	uint16_t uGuestIp = pMixedCtx->ip;
6027	if (VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntrInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT)
6028	{
6029	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF);
6030	/* #BP and #OF are both benign traps, we need to resume the next instruction. */
6031	uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
6032	}
6033	else if (VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntrInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT)
6034	uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
6035
6036	/* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
6037	uint16_t offIdtEntry = 0;
6038	RTSEL selIdtEntry = 0;
6039	RTGCPHYS GCPhysIdtEntry = (RTGCPHYS)pMixedCtx->idtr.pIdt + uVector * cbIdtEntry;
6040	rc = PGMPhysSimpleReadGCPhys(pVM, &offIdtEntry, GCPhysIdtEntry, sizeof(offIdtEntry));
6041	rc \|= PGMPhysSimpleReadGCPhys(pVM, &selIdtEntry, GCPhysIdtEntry + 2, sizeof(selIdtEntry));
6042	AssertRCReturn(rc, rc);
6043
6044	/* Construct the stack frame for the interrupt/exception handler. */
6045	rc = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->eflags.u32);
6046	rc \|= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->cs.Sel);
6047	rc \|= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, uGuestIp);
6048	AssertRCReturn(rc, rc);
6049
6050	/* Clear the required eflag bits and jump to the interrupt/exception handler. */
6051	if (rc == VINF_SUCCESS)
6052	{
6053	pMixedCtx->eflags.u32 &= ~(X86_EFL_IF \| X86_EFL_TF \| X86_EFL_RF \| X86_EFL_AC);
6054	pMixedCtx->rip = offIdtEntry;
6055	pMixedCtx->cs.Sel = selIdtEntry;
6056	pMixedCtx->cs.u64Base = selIdtEntry << cbIdtEntry;
6057
6058	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SEGMENT_REGS
6059	\| HM_CHANGED_GUEST_RIP
6060	\| HM_CHANGED_GUEST_RFLAGS
6061	\| HM_CHANGED_GUEST_RSP;
6062	AssertRCReturn(rc, rc);
6063	}
6064	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RESET);
6065	return rc;
6066	}
6067	else
6068	{
6069	/*
6070	* For unrestricted execution enabled CPUs running real-mode guests, we must not set the deliver-error-code bit.
6071	* See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
6072	*/
6073	u32IntrInfo &= ~VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6074	}
6075	}
6076
6077	/* Add the valid bit, maybe the caller was lazy. */
6078	u32IntrInfo \|= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
6079
6080	Assert(!VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK(u32IntrInfo)); /* Bit 12 MBZ. */
6081	Assert(!(u32IntrInfo & 0x7ffff000)); /* Bits 30:12 MBZ. */
6082	Log(("Injecting u32IntrInfo=%#x u32ErrCode=%#x instrlen=%#x\n", u32IntrInfo, u32ErrCode, cbInstr));
6083
6084	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntrInfo);
6085	if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntrInfo))
6086	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
6087	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
6088	AssertRCReturn(rc, rc);
6089
6090	return rc;
6091	}
6092
6093
6094	/**
6095	* Enters the VT-x session.
6096	*
6097	* @returns VBox status code.
6098	* @param pVM Pointer to the VM.
6099	* @param pVCpu Pointer to the VMCPU.
6100	* @param pCpu Pointer to the CPU info struct.
6101	*/
6102	VMMR0DECL(int) VMXR0Enter(PVM pVM, PVMCPU pVCpu, PHMGLOBLCPUINFO pCpu)
6103	{
6104	AssertPtr(pVM);
6105	AssertPtr(pVCpu);
6106	Assert(pVM->hm.s.vmx.fSupported);
6107	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6108	NOREF(pCpu);
6109
6110	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
6111
6112	/* Make sure we're in VMX root mode. */
6113	RTCCUINTREG u32HostCR4 = ASMGetCR4();
6114	if (!(u32HostCR4 & X86_CR4_VMXE))
6115	{
6116	LogRel(("VMXR0Enter: X86_CR4_VMXE bit in CR4 is not set!\n"));
6117	return VERR_VMX_X86_CR4_VMXE_CLEARED;
6118	}
6119
6120	/* Load the active VMCS as the current one. */
6121	int rc = VMXActivateVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
6122	if (RT_FAILURE(rc))
6123	return rc;
6124
6125	/** @todo this will change with preemption hooks where can can VMRESUME as long
6126	* as we're no preempted. */
6127	pVCpu->hm.s.fResumeVM = false;
6128	return VINF_SUCCESS;
6129	}
6130
6131
6132	/**
6133	* Leaves the VT-x session.
6134	*
6135	* @returns VBox status code.
6136	* @param pVM Pointer to the VM.
6137	* @param pVCpu Pointer to the VMCPU.
6138	* @param pCtx Pointer to the guest-CPU context.
6139	*/
6140	VMMR0DECL(int) VMXR0Leave(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
6141	{
6142	AssertPtr(pVCpu);
6143	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6144	NOREF(pVM);
6145	NOREF(pCtx);
6146
6147	/** @todo this will change with preemption hooks where we only VMCLEAR when
6148	* we are actually going to be preempted, not all the time like we
6149	* currently do. */
6150	/*
6151	* Sync the current VMCS (writes back internal data back into the VMCS region in memory)
6152	* and mark the VMCS launch-state as "clear".
6153	*/
6154	int rc = VMXClearVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
6155	return rc;
6156	}
6157
6158
6159	/**
6160	* Saves the host state in the VMCS host-state.
6161	* Sets up the VM-exit MSR-load area.
6162	*
6163	* The CPU state will be loaded from these fields on every successful VM-exit.
6164	*
6165	* @returns VBox status code.
6166	* @param pVM Pointer to the VM.
6167	* @param pVCpu Pointer to the VMCPU.
6168	*
6169	* @remarks No-long-jump zone!!!
6170	*/
6171	VMMR0DECL(int) VMXR0SaveHostState(PVM pVM, PVMCPU pVCpu)
6172	{
6173	AssertPtr(pVM);
6174	AssertPtr(pVCpu);
6175	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6176
6177	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
6178
6179	/* Nothing to do if the host-state-changed flag isn't set. This will later be optimized when preemption hooks are in place. */
6180	if (!(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_HOST_CONTEXT))
6181	return VINF_SUCCESS;
6182
6183	int rc = hmR0VmxSaveHostControlRegs(pVM, pVCpu);
6184	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostControlRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6185
6186	rc = hmR0VmxSaveHostSegmentRegs(pVM, pVCpu);
6187	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostSegmentRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6188
6189	rc = hmR0VmxSaveHostMsrs(pVM, pVCpu);
6190	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostMsrs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6191
6192	/* Reset the host-state-changed flag. */
6193	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_HOST_CONTEXT;
6194	return rc;
6195	}
6196
6197
6198	/**
6199	* Loads the guest state into the VMCS guest-state area. The CPU state will be
6200	* loaded from these fields on every successful VM-entry.
6201	*
6202	* Sets up the VM-entry MSR-load and VM-exit MSR-store areas.
6203	* Sets up the VM-entry controls.
6204	* Sets up the appropriate VMX non-root function to execute guest code based on
6205	* the guest CPU mode.
6206	*
6207	* @returns VBox status code.
6208	* @param pVM Pointer to the VM.
6209	* @param pVCpu Pointer to the VMCPU.
6210	* @param pCtx Pointer to the guest-CPU context.
6211	*
6212	* @remarks No-long-jump zone!!!
6213	*/
6214	VMMR0DECL(int) VMXR0LoadGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
6215	{
6216	AssertPtr(pVM);
6217	AssertPtr(pVCpu);
6218	AssertPtr(pCtx);
6219	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6220
6221	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
6222
6223	/* For longjmp reentrants we need not load the guest state all over again. */
6224	if (!pVCpu->hm.s.fContextUseFlags)
6225	return VINF_SUCCESS;
6226
6227	/* Determine real-on-v86 mode. */
6228	pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = false;
6229	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
6230	&& CPUMIsGuestInRealModeEx(pCtx))
6231	{
6232	pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = true;
6233	}
6234
6235	int rc = hmR0VmxLoadGuestEntryCtls(pVM, pVCpu, pCtx);
6236	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestEntryCtls! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6237
6238	rc = hmR0VmxLoadGuestExitCtls(pVM, pVCpu);
6239	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupExitCtls failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6240
6241	rc = hmR0VmxLoadGuestActivityState(pVM, pVCpu, pCtx);
6242	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestActivityState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6243
6244	rc = hmR0VmxLoadGuestControlRegs(pVM, pVCpu, pCtx);
6245	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestControlRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6246
6247	rc = hmR0VmxLoadGuestSegmentRegs(pVM, pVCpu, pCtx);
6248	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestSegmentRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6249
6250	rc = hmR0VmxLoadGuestDebugRegs(pVM, pVCpu, pCtx);
6251	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestDebugRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6252
6253	rc = hmR0VmxLoadGuestMsrs(pVM, pVCpu, pCtx);
6254	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestMsrs! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6255
6256	rc = hmR0VmxLoadGuestApicState(pVM, pVCpu, pCtx);
6257	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestApicState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6258
6259	rc = hmR0VmxLoadGuestGprs(pVM, pVCpu, pCtx);
6260	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestGprs! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6261
6262	rc = hmR0VmxLoadGuestIntrState(pVM, pVCpu, pCtx);
6263	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestIntrState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6264
6265	rc = hmR0VmxSetupVMRunHandler(pVM, pVCpu, pCtx);
6266	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupVMRunHandler! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6267
6268	AssertMsg(!pVCpu->hm.s.fContextUseFlags,
6269	("Missed updating flags while loading guest state. pVM=%p pVCpu=%p fContextUseFlags=%#RX32\n",
6270	pVM, pVCpu, pVCpu->hm.s.fContextUseFlags));
6271	return rc;
6272	}
6273
6274
6275	/**
6276	* Does the preparations before executing guest code in VT-x.
6277	*
6278	* This may cause longjmps to ring-3 and may even result in rescheduling to the
6279	* recompiler. We must be cautious what we do here regarding committing
6280	* guest-state information into the the VMCS assuming we assuredly execute the
6281	* guest in VT-x. If we fall back to the recompiler after updating VMCS and
6282	* clearing the common-state (TRPM/forceflags), we must undo those changes so
6283	* that the recompiler can (and should) use them when it resumes guest
6284	* execution. Otherwise such operations must be done when we can no longer
6285	* exit to ring-3.
6286	*
6287	* @returns VBox status code (informational status codes included).
6288	* @retval VINF_SUCCESS if we can proceed with running the guest.
6289	* @retval VINF_EM_RESET if a triple-fault occurs while injecting a double-fault
6290	* into the guest.
6291	* @retval VINF_* scheduling changes, we have to go back to ring-3.
6292	*
6293	* @param pVM Pointer to the VM.
6294	* @param pVCpu Pointer to the VMCPU.
6295	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6296	* out-of-sync. Make sure to update the required fields
6297	* before using them.
6298	* @param pVmxTransient Pointer to the VMX transient structure.
6299	*
6300	* @remarks Called with preemption disabled.
6301	*/
6302	DECLINLINE(int) hmR0VmxPreRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6303	{
6304	Assert(VMMRZCallRing3IsEnabled(pVCpu));
6305
6306	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
6307	PGMRZDynMapFlushAutoSet(pVCpu);
6308	#endif
6309
6310	/* Check force flag actions that might require us to go back to ring-3. */
6311	int rc = hmR0VmxCheckForceFlags(pVM, pVCpu, pMixedCtx);
6312	if (rc != VINF_SUCCESS)
6313	return rc;
6314
6315	/* Setup the Virtualized APIC accesses. pMixedCtx->msrApicBase is always up-to-date. It's not part of the VMCS. */
6316	if ( pVCpu->hm.s.vmx.u64MsrApicBase != pMixedCtx->msrApicBase
6317	&& (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC))
6318	{
6319	Assert(pVM->hm.s.vmx.HCPhysApicAccess);
6320	RTGCPHYS GCPhysApicBase;
6321	GCPhysApicBase = pMixedCtx->msrApicBase;
6322	GCPhysApicBase &= PAGE_BASE_GC_MASK;
6323
6324	/* Unalias any existing mapping. */
6325	rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
6326	AssertRCReturn(rc, rc);
6327
6328	/* Map the HC APIC-access page into the GC space, this also updates the shadow page tables if necessary. */
6329	Log2(("Mapped HC APIC-access page into GC: GCPhysApicBase=%#RGv\n", GCPhysApicBase));
6330	rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW \| X86_PTE_P);
6331	AssertRCReturn(rc, rc);
6332
6333	pVCpu->hm.s.vmx.u64MsrApicBase = pMixedCtx->msrApicBase;
6334	}
6335
6336	#ifdef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
6337	/* We disable interrupts so that we don't miss any interrupts that would flag preemption (IPI/timers etc.) */
6338	pVmxTransient->uEFlags = ASMIntDisableFlags();
6339	if (RTThreadPreemptIsPending(NIL_RTTHREAD))
6340	{
6341	ASMSetFlags(pVmxTransient->uEFlags);
6342	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptPending);
6343	/* Don't use VINF_EM_RAW_INTERRUPT_HYPER as we can't assume the host does kernel preemption. Maybe some day? */
6344	return VINF_EM_RAW_INTERRUPT;
6345	}
6346	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
6347	#endif
6348
6349	/*
6350	* This clears force-flags, TRPM traps & pending HM events. We cannot safely restore the state if we exit to ring-3
6351	* (before running guest code) after calling this function (e.g. how do we reverse the effects of calling PDMGetInterrupt()?)
6352	* This is why this is done after all possible exits-to-ring-3 paths in this code.
6353	*/
6354	rc = hmR0VmxInjectPendingInterrupt(pVM, pVCpu, pMixedCtx);
6355	AssertRCReturn(rc, rc);
6356	return rc;
6357	}
6358
6359
6360	/**
6361	* Prepares to run guest code in VT-x and we've committed to doing so. This
6362	* means there is no backing out to ring-3 or anywhere else at this
6363	* point.
6364	*
6365	* @param pVM Pointer to the VM.
6366	* @param pVCpu Pointer to the VMCPU.
6367	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6368	* out-of-sync. Make sure to update the required fields
6369	* before using them.
6370	* @param pVmxTransient Pointer to the VMX transient structure.
6371	*
6372	* @remarks Called with preemption disabled.
6373	* @remarks No-long-jump zone!!!
6374	*/
6375	DECLINLINE(void) hmR0VmxPreRunGuestCommitted(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6376	{
6377	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6378
6379	#ifndef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
6380	/** @todo I don't see the point of this, VMMR0EntryFast() already disables interrupts for the entire period. */
6381	pVmxTransient->uEFlags = ASMIntDisableFlags();
6382	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
6383	#endif
6384
6385	/* Load the required guest state bits (for guest-state changes in the inner execution loop). */
6386	Assert(!(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_HOST_CONTEXT));
6387	int rc = VMXR0LoadGuestState(pVM, pVCpu, pMixedCtx);
6388	AssertRC(rc);
6389	AssertMsg(pVCpu->hm.s.fContextUseFlags == 0, ("fContextUseFlags =%#x\n", pVCpu->hm.s.fContextUseFlags));
6390
6391	/* Cache the TPR-shadow for checking on every VM-exit if it might have changed. */
6392	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
6393	pVmxTransient->u8GuestTpr = pVCpu->hm.s.vmx.pbVirtApic[0x80];
6394
6395	Assert(pVM->hm.s.vmx.pfnFlushTaggedTlb);
6396	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB-shootdowns, set this across the world switch. */
6397	pVM->hm.s.vmx.pfnFlushTaggedTlb(pVM, pVCpu); /* Flush the TLB of guest entries as necessary. */
6398
6399	/* Setup TSC-offsetting or intercept RDTSC(P)s and update the preemption timer. */
6400	if (pVmxTransient->fUpdateTscOffsetting)
6401	{
6402	hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVM, pVCpu, pMixedCtx);
6403	pVmxTransient->fUpdateTscOffsetting = false;
6404	}
6405
6406	/*
6407	* TPR patching (only active for 32-bit guests on 64-bit capable CPUs) when the CPU does not supported virtualizing
6408	* APIC accesses feature (VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC).
6409	*/
6410	if (pVM->hm.s.fTPRPatchingActive)
6411	{
6412	Assert(!CPUMIsGuestInLongMode(pVCpu));
6413
6414	/* Need guest's LSTAR MSR (which is part of the auto load/store MSRs in the VMCS), ensure we have the updated one. */
6415	rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVM, pVCpu, pMixedCtx);
6416	AssertRC(rc);
6417
6418	/* The patch code uses the LSTAR as it's not used by a guest in 32-bit mode implicitly (i.e. SYSCALL is 64-bit only). */
6419	pVmxTransient->u64LStarMsr = ASMRdMsr(MSR_K8_LSTAR);
6420	ASMWrMsr(MSR_K8_LSTAR, pMixedCtx->msrLSTAR); /* pMixedCtx->msrLSTAR contains the guest's TPR,
6421	see hmR0VmxLoadGuestApicState(). */
6422	}
6423
6424	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
6425	TMNotifyStartOfExecution(pVCpu); /* Finally, notify TM to resume its clocks as we're about
6426	to start executing. */
6427	}
6428
6429
6430	/**
6431	* Performs some essential restoration of state after running guest code in
6432	* VT-x.
6433	*
6434	* @param pVM Pointer to the VM.
6435	* @param pVCpu Pointer to the VMCPU.
6436	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6437	* out-of-sync. Make sure to update the required fields
6438	* before using them.
6439	* @param pVmxTransient Pointer to the VMX transient structure.
6440	* @param rcVMRun Return code of VMLAUNCH/VMRESUME.
6441	*
6442	* @remarks Called with interrupts disabled.
6443	* @remarks No-long-jump zone!!! This function will however re-enable longjmps
6444	* unconditionally when it is safe to do so.
6445	*/
6446	DECLINLINE(void) hmR0VmxPostRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, int rcVMRun)
6447	{
6448	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6449	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatInGC, x);
6450
6451	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB-shootdowns. */
6452	ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for TLB-shootdowns. */
6453	pVCpu->hm.s.vmx.fUpdatedGuestState = 0; /* Exits/longjmps to ring-3 requires saving the guest state. */
6454	pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
6455	pVmxTransient->fVectoringPF = false; /* Clear the vectoring page-fault flag. */
6456
6457	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT))
6458	{
6459	/** @todo Find a way to fix hardcoding a guestimate. */
6460	TMCpuTickSetLastSeen(pVCpu, ASMReadTSC()
6461	+ pVCpu->hm.s.vmx.u64TSCOffset - 0x400 /* guestimate of world switch overhead in clock ticks */);
6462	}
6463
6464	TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
6465	Assert(!(ASMGetFlags() & X86_EFL_IF));
6466	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED);
6467
6468	/* Restore the effects of TPR patching if any. */
6469	if (pVM->hm.s.fTPRPatchingActive)
6470	{
6471	int rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVM, pVCpu, pMixedCtx);
6472	AssertRC(rc);
6473	pMixedCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR); /* MSR_K8_LSTAR contains the guest TPR. */
6474	ASMWrMsr(MSR_K8_LSTAR, pVmxTransient->u64LStarMsr);
6475	}
6476
6477	ASMSetFlags(pVmxTransient->uEFlags); /* Enable interrupts. */
6478	pVCpu->hm.s.fResumeVM = true; /* Use VMRESUME instead of VMLAUNCH in the next run. */
6479
6480	/* Save the basic VM-exit reason. Refer Intel spec. 24.9.1 "Basic VM-exit Information". */
6481	uint32_t uExitReason;
6482	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
6483	rc \|= hmR0VmxReadEntryIntrInfoVmcs(pVmxTransient);
6484	AssertRC(rc);
6485	pVmxTransient->uExitReason = (uint16_t)VMX_EXIT_REASON_BASIC(uExitReason);
6486	pVmxTransient->fVMEntryFailed = VMX_ENTRY_INTERRUPTION_INFO_VALID(pVmxTransient->uEntryIntrInfo);
6487
6488	VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pMixedCtx);
6489	hmR0VmxCallRing3Enable(pVCpu); /* It is now safe to do longjmps to ring-3!!! */
6490
6491	/* If the VMLAUNCH/VMRESUME failed, we can bail out early. This does -not- cover VMX_EXIT_ERR_. /
6492	if (RT_UNLIKELY(rcVMRun != VINF_SUCCESS))
6493	{
6494	Log(("VM-entry failure: rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", rcVMRun, pVmxTransient->fVMEntryFailed));
6495	return;
6496	}
6497
6498	/* We need to update our interruptibility-state on every VM-exit and VM-entry. */
6499	rc = hmR0VmxSaveGuestIntrState(pVM, pVCpu, pMixedCtx);
6500	AssertRC(rc);
6501	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_INTR_STATE;
6502
6503	/*
6504	* If the TPR was raised by the guest, it wouldn't cause a VM-exit immediately. Instead we sync the TPR lazily whenever we
6505	* eventually get a VM-exit for any reason. This maybe expensive as PDMApicSetTPR() can longjmp to ring-3; also why we do
6506	* it outside of hmR0VmxSaveGuestState() which must never cause longjmps.
6507	*/
6508	if ( !pVmxTransient->fVMEntryFailed
6509	&& (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
6510	&& pVmxTransient->u8GuestTpr != pVCpu->hm.s.vmx.pbVirtApic[0x80])
6511	{
6512	rc = PDMApicSetTPR(pVCpu, pVCpu->hm.s.vmx.pbVirtApic[0x80]);
6513	AssertRC(rc);
6514	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_VMX_GUEST_APIC_STATE;
6515	}
6516	}
6517
6518
6519	/**
6520	* Runs the guest code using VT-x.
6521	*
6522	* @returns VBox status code.
6523	* @param pVM Pointer to the VM.
6524	* @param pVCpu Pointer to the VMCPU.
6525	* @param pCtx Pointer to the guest-CPU context.
6526	*
6527	* @remarks Called with preemption disabled.
6528	*/
6529	VMMR0DECL(int) VMXR0RunGuestCode(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
6530	{
6531	Assert(VMMRZCallRing3IsEnabled(pVCpu));
6532	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6533	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
6534
6535	VMXTRANSIENT VmxTransient;
6536	VmxTransient.fUpdateTscOffsetting = true;
6537	int rc = VERR_INTERNAL_ERROR_5;
6538	unsigned cLoops = 0;
6539
6540	for (;; cLoops++)
6541	{
6542	Assert(!HMR0SuspendPending());
6543	AssertMsg(pVCpu->hm.s.idEnteredCpu == RTMpCpuId(),
6544	("Illegal migration! Entered on CPU %u Current %u cLoops=%u\n", (unsigned)pVCpu->hm.s.idEnteredCpu,
6545	(unsigned)RTMpCpuId(), cLoops));
6546
6547	/* Preparatory work for running guest code, this may return to ring-3 for some last minute updates. */
6548	rc = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient);
6549	if (rc != VINF_SUCCESS)
6550	break;
6551
6552	/*
6553	* No longjmps to ring-3 from this point on!!!
6554	* Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
6555	* This also disables flushing of the R0-logger instance (if any).
6556	*/
6557	hmR0VmxCallRing3Disable(pVCpu);
6558	VMMRZCallRing3RemoveNotification(pVCpu);
6559	hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
6560
6561	rc = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
6562	/* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
6563
6564	/*
6565	* Restore any residual host-state and save any bits shared between host and guest into the guest-CPU state.
6566	* This will also re-enable longjmps to ring-3 when it has reached a safe point!!!
6567	*/
6568	hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rc);
6569	if (RT_UNLIKELY(rc != VINF_SUCCESS)) /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
6570	{
6571	hmR0VmxReportWorldSwitchError(pVM, pVCpu, rc, pCtx, &VmxTransient);
6572	return rc;
6573	}
6574
6575	/* Handle any exception that caused a VM-exit while delivering an event to the guest. */
6576	rc = hmR0VmxSavePendingEventDueToEventDelivery(pVM, pVCpu, pCtx, &VmxTransient);
6577	if (RT_LIKELY(rc == VINF_SUCCESS))
6578	{
6579	/* Handle VM-exits. */
6580	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("Invalid VM-exit %#x\n", VmxTransient.uExitReason));
6581	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
6582	rc = (*s_apfnVMExitHandlers[VmxTransient.uExitReason])(pVM, pVCpu, pCtx, &VmxTransient);
6583	if (rc != VINF_SUCCESS)
6584	break;
6585	else if (cLoops > pVM->hm.s.cMaxResumeLoops)
6586	{
6587	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMaxResume);
6588	rc = VINF_EM_RAW_INTERRUPT;
6589	break;
6590	}
6591	}
6592	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
6593	break;
6594	/* else continue guest execution for (VINF_VMX_DOUBLE_FAULT) */
6595	}
6596
6597	if (rc == VERR_EM_INTERPRETER)
6598	rc = VINF_EM_RAW_EMULATE_INSTR;
6599	hmR0VmxExitToRing3(pVM, pVCpu, pCtx, rc);
6600	return rc;
6601	}
6602
6603	/* Validates input parameters for VM-exit handler functions. Later change this to be debug builds only. */
6604	#ifdef DEBUG
6605	/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
6606	# define VMX_ASSERT_PREEMPT_CPUID_VAR() \
6607	RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
6608	# define VMX_ASSERT_PREEMPT_CPUID() \
6609	do \
6610	{ \
6611	RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
6612	AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
6613	} while (0)
6614
6615	# define VMX_VALIDATE_EXIT_HANDLER_PARAMS() \
6616	do { \
6617	AssertPtr(pVM); \
6618	AssertPtr(pVCpu); \
6619	AssertPtr(pMixedCtx); \
6620	AssertPtr(pVmxTransient); \
6621	Assert(pVmxTransient->fVMEntryFailed == false); \
6622	Assert(ASMIntAreEnabled() == true); \
6623	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD)); \
6624	VMX_ASSERT_PREEMPT_CPUID_VAR(); \
6625	LogFunc(("\n")); \
6626	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD)); \
6627	if (VMMR0IsLogFlushDisabled(pVCpu)) \
6628	VMX_ASSERT_PREEMPT_CPUID(); \
6629	} while (0)
6630	# define VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() \
6631	do { \
6632	LogFunc(("\n")); \
6633	} while(0)
6634	#else /* Release builds */
6635	# define VMX_VALIDATE_EXIT_HANDLER_PARAMS() do { } while(0)
6636	# define VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() do { } while(0)
6637	#endif
6638
6639
6640	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
6641	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
6642	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
6643	/**
6644	* VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
6645	*/
6646	static DECLCALLBACK(int) hmR0VmxExitExtInt(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6647	{
6648	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
6649	STAM_COUNTER_INC(&pVCpu->hm.s.StatPendingHostIrq);
6650	return VINF_EM_RAW_INTERRUPT;
6651	}
6652
6653
6654	/**
6655	* VM-exit handler for exceptions and NMIs (VMX_EXIT_XCPT_NMI).
6656	*/
6657	static DECLCALLBACK(int) hmR0VmxExitXcptNmi(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6658	{
6659	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
6660	int rc = hmR0VmxReadExitIntrInfoVmcs(pVmxTransient);
6661	rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
6662	rc \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVmxTransient);
6663	AssertRCReturn(rc, rc);
6664
6665	uint8_t u8IntrType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntrInfo);
6666	Assert( !(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_CONTROLS_ACK_EXT_INT)
6667	&& u8IntrType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
6668
6669	if (u8IntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
6670	return VINF_EM_RAW_INTERRUPT;
6671
6672	uint32_t uExitIntrInfo = pVmxTransient->uExitIntrInfo;
6673	Assert(VMX_EXIT_INTERRUPTION_INFO_VALID(uExitIntrInfo));
6674
6675	uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(uExitIntrInfo);
6676	switch (VMX_EXIT_INTERRUPTION_INFO_TYPE(uExitIntrInfo))
6677	{
6678	case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
6679	Assert(uVector == X86_XCPT_DB \|\| uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF);
6680	/* no break */
6681	case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT: /* Hardware exception. */
6682	{
6683	switch (uVector)
6684	{
6685	case X86_XCPT_PF: rc = hmR0VmxExitXcptPF(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6686	case X86_XCPT_GP: rc = hmR0VmxExitXcptGP(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6687	case X86_XCPT_NM: rc = hmR0VmxExitXcptNM(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6688	case X86_XCPT_MF: rc = hmR0VmxExitXcptMF(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6689	case X86_XCPT_DB: rc = hmR0VmxExitXcptDB(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6690	case X86_XCPT_BP: rc = hmR0VmxExitXcptBP(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6691	#ifdef VBOX_ALWAYS_TRAP_ALL_EXCEPTIONS
6692	case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF);
6693	rc = hmR0VmxExitXcptGeneric(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6694	case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
6695	rc = hmR0VmxExitXcptGeneric(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6696	case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
6697	rc = hmR0VmxExitXcptGeneric(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6698	case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
6699	rc = hmR0VmxExitXcptGeneric(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6700	case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
6701	rc = hmR0VmxExitXcptGeneric(pVM, pVCpu, pMixedCtx, pVmxTransient); break;
6702	#endif
6703	default:
6704	{
6705	rc = hmR0VmxSaveGuestCR0(pVM, pVCpu, pMixedCtx);
6706	AssertRCReturn(rc, rc);
6707
6708	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
6709	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6710	{
6711	Assert(pVM->hm.s.vmx.pRealModeTSS);
6712	Assert(PDMVmmDevHeapIsEnabled(pVM));
6713	rc = hmR0VmxInjectEventVmcs(pVM, pVCpu, pMixedCtx,
6714	VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(uExitIntrInfo),
6715	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode);
6716	AssertRCReturn(rc, rc);
6717	}
6718	else
6719	{
6720	AssertMsgFailed(("Unexpected VM-exit caused by exception %#x\n", uVector));
6721	rc = VERR_VMX_UNEXPECTED_EXCEPTION;
6722	}
6723	break;
6724	}
6725	}
6726	break;
6727	}
6728
6729	case VMX_EXIT_INTERRUPTION_INFO_TYPE_DB_XCPT:
6730	default:
6731	{
6732	rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_CODE;
6733	AssertMsgFailed(("Unexpected interruption code %#x\n", VMX_EXIT_INTERRUPTION_INFO_TYPE(uExitIntrInfo)));
6734	break;
6735	}
6736	}
6737	return rc;
6738	}
6739
6740
6741	/**
6742	* VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
6743	*/
6744	static DECLCALLBACK(int) hmR0VmxExitIntWindow(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6745	{
6746	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
6747	int rc = VERR_INTERNAL_ERROR_5;
6748	#ifdef DEBUG
6749	rc = hmR0VmxSaveGuestRflags(pVM, pVCpu, pMixedCtx);
6750	AssertRC(rc);
6751	Assert(pMixedCtx->eflags.u32 & X86_EFL_IF);
6752	#endif
6753
6754	/* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
6755	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INT_WINDOW_EXIT;
6756	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls);
6757	AssertRCReturn(rc, rc);
6758
6759	/* Deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectPendingInterrupt() and resume guest execution. */
6760	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
6761	return VINF_SUCCESS;
6762	}
6763
6764
6765	/**
6766	* VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
6767	*/
6768	static DECLCALLBACK(int) hmR0VmxExitNmiWindow(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6769	{
6770	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
6771	AssertMsgFailed(("Unexpected NMI-window exit.\n"));
6772	return VERR_VMX_UNEXPECTED_EXIT_CODE;
6773	}
6774
6775
6776	/**
6777	* VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
6778	*/
6779	static DECLCALLBACK(int) hmR0VmxExitWbinvd(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6780	{
6781	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
6782	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
6783	rc \|= hmR0VmxSaveGuestRip(pVM, pVCpu, pMixedCtx);
6784	AssertRCReturn(rc, rc);
6785
6786	pMixedCtx->rip += pVmxTransient->cbInstr;
6787	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
6788
6789	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWbinvd);
6790	return VINF_SUCCESS;
6791	}
6792
6793
6794	/**
6795	* VM-exit handler for WBINVD (VMX_EXIT_INVD). Unconditional VM-exit.
6796	*/
6797	static DECLCALLBACK(int) hmR0VmxExitInvd(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6798	{
6799	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
6800	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
6801	rc \|= hmR0VmxSaveGuestRip(pVM, pVCpu, pMixedCtx);
6802	AssertRCReturn(rc, rc);
6803
6804	pMixedCtx->rip += pVmxTransient->cbInstr;
6805	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
6806
6807	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvd);
6808	return VINF_SUCCESS;
6809	}
6810
6811
6812	/**
6813	* VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
6814	*/
6815	static DECLCALLBACK(int) hmR0VmxExitCpuid(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6816	{
6817	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
6818	int rc = EMInterpretCpuId(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
6819	if (RT_LIKELY(rc == VINF_SUCCESS))
6820	{
6821	rc = hmR0VmxSaveGuestRip(pVM, pVCpu, pMixedCtx);
6822	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
6823	AssertRCReturn(rc, rc);
6824	Assert(pVmxTransient->cbInstr == 2);
6825
6826	Log(("hmR0VmxExitCpuid: RIP=%#RX64\n", pMixedCtx->rip));
6827	pMixedCtx->rip += pVmxTransient->cbInstr;
6828	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
6829	}
6830	else
6831	{
6832	AssertMsgFailed(("hmR0VmxExitCpuid: EMInterpretCpuId failed with %Rrc\n", rc));
6833	rc = VERR_EM_INTERPRETER;
6834	}
6835	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCpuid);
6836	return rc;
6837	}
6838
6839
6840	/**
6841	* VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
6842	*/
6843	static DECLCALLBACK(int) hmR0VmxExitGetsec(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6844	{
6845	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
6846	int rc = hmR0VmxSaveGuestCR4(pVM, pVCpu, pMixedCtx);
6847	AssertRCReturn(rc, rc);
6848
6849	if (pMixedCtx->cr4 & X86_CR4_SMXE)
6850	return VINF_EM_RAW_EMULATE_INSTR;
6851
6852	AssertMsgFailed(("hmR0VmxExitGetsec: unexpected VM-exit when CR4.SMXE is 0.\n"));
6853	return VERR_VMX_UNEXPECTED_EXIT_CODE;
6854	}
6855
6856
6857	/**
6858	* VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
6859	*/
6860	static DECLCALLBACK(int) hmR0VmxExitRdtsc(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6861	{
6862	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
6863	int rc = hmR0VmxSaveGuestCR4(pVM, pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
6864	AssertRCReturn(rc, rc);
6865
6866	rc = EMInterpretRdtsc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
6867	if (RT_LIKELY(rc == VINF_SUCCESS))
6868	{
6869	rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
6870	rc \|= hmR0VmxSaveGuestRip(pVM, pVCpu, pMixedCtx);
6871	AssertRCReturn(rc, rc);
6872	Assert(pVmxTransient->cbInstr == 2);
6873
6874	pMixedCtx->rip += pVmxTransient->cbInstr;
6875	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
6876
6877	/* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
6878	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TSC_OFFSETTING)
6879	pVmxTransient->fUpdateTscOffsetting = true;
6880	}
6881	else
6882	{
6883	AssertMsgFailed(("hmR0VmxExitRdtsc: EMInterpretRdtsc failed with %Rrc\n", rc));
6884	rc = VERR_EM_INTERPRETER;
6885	}
6886	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
6887	return rc;
6888	}
6889
6890
6891	/**
6892	* VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
6893	*/
6894	static DECLCALLBACK(int) hmR0VmxExitRdtscp(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6895	{
6896	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
6897	int rc = hmR0VmxSaveGuestCR4(pVM, pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
6898	rc \|= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVM, pVCpu, pMixedCtx); /* For MSR_K8_TSC_AUX */
6899	AssertRCReturn(rc, rc);
6900
6901	rc = EMInterpretRdtscp(pVM, pVCpu, pMixedCtx);
6902	if (RT_LIKELY(rc == VINF_SUCCESS))
6903	{
6904	rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
6905	rc \|= hmR0VmxSaveGuestRip(pVM, pVCpu, pMixedCtx);
6906	AssertRCReturn(rc, rc);
6907	Assert(pVmxTransient->cbInstr == 3);
6908
6909	pMixedCtx->rip += pVmxTransient->cbInstr;
6910	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
6911
6912	/* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
6913	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TSC_OFFSETTING)
6914	pVmxTransient->fUpdateTscOffsetting = true;
6915	}
6916	else
6917	{
6918	AssertMsgFailed(("hmR0VmxExitRdtscp: EMInterpretRdtscp failed with %Rrc\n", rc));
6919	rc = VERR_EM_INTERPRETER;
6920	}
6921	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
6922	return rc;
6923	}
6924
6925
6926	/**
6927	* VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
6928	*/
6929	static DECLCALLBACK(int) hmR0VmxExitRdpmc(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6930	{
6931	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
6932	int rc = hmR0VmxSaveGuestCR4(pVM, pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
6933	rc \|= hmR0VmxSaveGuestCR0(pVM, pVCpu, pMixedCtx); /** @todo review if CR0 is really required by EM. */
6934	AssertRCReturn(rc, rc);
6935
6936	rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
6937	if (RT_LIKELY(rc == VINF_SUCCESS))
6938	{
6939	rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
6940	rc \|= hmR0VmxSaveGuestRip(pVM, pVCpu, pMixedCtx);
6941	AssertRCReturn(rc, rc);
6942	Assert(pVmxTransient->cbInstr == 2);
6943
6944	pMixedCtx->rip += pVmxTransient->cbInstr;
6945	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
6946	}
6947	else
6948	{
6949	AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
6950	rc = VERR_EM_INTERPRETER;
6951	}
6952	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdpmc);
6953	return rc;
6954	}
6955
6956
6957	/**
6958	* VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
6959	*/
6960	static DECLCALLBACK(int) hmR0VmxExitInvlpg(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6961	{
6962	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
6963	int rc = hmR0VmxReadExitQualificationVmcs(pVmxTransient);
6964	rc \|= hmR0VmxSaveGuestControlRegs(pVM, pVCpu, pMixedCtx);
6965	AssertRCReturn(rc, rc);
6966
6967	VBOXSTRICTRC rc2 = EMInterpretInvlpg(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), pVmxTransient->uExitQualification);
6968	rc = VBOXSTRICTRC_VAL(rc2);
6969	if (RT_LIKELY(rc == VINF_SUCCESS))
6970	{
6971	rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
6972	rc \|= hmR0VmxSaveGuestRip(pVM, pVCpu, pMixedCtx);
6973	AssertRCReturn(rc, rc);
6974
6975	pMixedCtx->rip += pVmxTransient->cbInstr;
6976	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
6977	}
6978	else
6979	{
6980	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitInvlpg: EMInterpretInvlpg %RGv failed with %Rrc\n",
6981	pVmxTransient->uExitQualification, rc));
6982	rc = VERR_EM_INTERPRETER;
6983	}
6984	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvlpg);
6985	return rc;
6986	}
6987
6988
6989	/**
6990	* VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
6991	*/
6992	static DECLCALLBACK(int) hmR0VmxExitMonitor(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6993	{
6994	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
6995	int rc = hmR0VmxSaveGuestCR0(pVM, pVCpu, pMixedCtx);
6996	rc \|= hmR0VmxSaveGuestRflags(pVM, pVCpu, pMixedCtx);
6997	rc \|= hmR0VmxSaveGuestSegmentRegs(pVM, pVCpu, pMixedCtx);
6998	AssertRCReturn(rc, rc);
6999
7000	rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7001	if (RT_LIKELY(rc == VINF_SUCCESS))
7002	{
7003	rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
7004	rc \|= hmR0VmxSaveGuestRip(pVM, pVCpu, pMixedCtx);
7005	AssertRCReturn(rc, rc);
7006
7007	pMixedCtx->rip += pVmxTransient->cbInstr;
7008	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
7009	}
7010	else
7011	{
7012	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMonitor: EMInterpretMonitor failed with %Rrc\n", rc));
7013	rc = VERR_EM_INTERPRETER;
7014	}
7015	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMonitor);
7016	return rc;
7017	}
7018
7019
7020	/**
7021	* VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
7022	*/
7023	static DECLCALLBACK(int) hmR0VmxExitMwait(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7024	{
7025	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7026	int rc = hmR0VmxSaveGuestCR0(pVM, pVCpu, pMixedCtx);
7027	rc \|= hmR0VmxSaveGuestRflags(pVM, pVCpu, pMixedCtx);
7028	rc \|= hmR0VmxSaveGuestSegmentRegs(pVM, pVCpu, pMixedCtx);
7029	AssertRCReturn(rc, rc);
7030
7031	VBOXSTRICTRC rc2 = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7032	rc = VBOXSTRICTRC_VAL(rc2);
7033	if (RT_LIKELY( rc == VINF_SUCCESS
7034	\|\| rc == VINF_EM_HALT))
7035	{
7036	int rc3 = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
7037	rc3 \|= hmR0VmxSaveGuestRip(pVM, pVCpu, pMixedCtx);
7038	AssertRCReturn(rc3, rc3);
7039
7040	pMixedCtx->rip += pVmxTransient->cbInstr;
7041	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
7042
7043	if ( rc == VINF_EM_HALT
7044	&& EMShouldContinueAfterHalt(pVCpu, pMixedCtx))
7045	{
7046	rc = VINF_SUCCESS;
7047	}
7048	}
7049	else
7050	{
7051	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMwait: EMInterpretMWait failed with %Rrc\n", rc));
7052	rc = VERR_EM_INTERPRETER;
7053	}
7054	AssertMsg(rc == VINF_SUCCESS \|\| rc == VINF_EM_HALT \|\| rc == VERR_EM_INTERPRETER,
7055	("hmR0VmxExitMwait: failed, invalid error code %Rrc\n", rc));
7056	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMwait);
7057	return rc;
7058	}
7059
7060
7061	/**
7062	* VM-exit handler for RSM (VMX_EXIT_RSM). Unconditional VM-exit.
7063	*/
7064	static DECLCALLBACK(int) hmR0VmxExitRsm(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7065	{
7066	/*
7067	* Execution of RSM outside of SMM mode causes #UD regardless of VMX root or VMX non-root mode. In theory, we should never
7068	* get this VM-exit. This can happen only if dual-monitor treatment of SMI and VMX is enabled, which can (only?) be done by
7069	* executing VMCALL in VMX root operation. If we get here something funny is going on.
7070	* See Intel spec. "33.15.5 Enabling the Dual-Monitor Treatment".
7071	*/
7072	AssertMsgFailed(("Unexpected RSM VM-exit. pVM=%p pVCpu=%p pMixedCtx=%p\n", pVM, pVCpu, pMixedCtx));
7073	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7074	}
7075
7076
7077	/**
7078	* VM-exit handler for SMI (VMX_EXIT_SMI). Unconditional VM-exit.
7079	*/
7080	static DECLCALLBACK(int) hmR0VmxExitSmi(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7081	{
7082	/*
7083	* This can only happen if we support dual-monitor treatment of SMI, which can be activated by executing VMCALL in VMX
7084	* root operation. If we get there there is something funny going on.
7085	* See Intel spec. "33.15.6 Activating the Dual-Monitor Treatment" and Intel spec. 25.3 "Other Causes of VM-Exits"
7086	*/
7087	AssertMsgFailed(("Unexpected SMI VM-exit. pVM=%p pVCpu=%p pMixedCtx=%p\n", pVM, pVCpu, pMixedCtx));
7088	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7089	}
7090
7091
7092	/**
7093	* VM-exit handler for IO SMI (VMX_EXIT_IO_SMI). Unconditional VM-exit.
7094	*/
7095	static DECLCALLBACK(int) hmR0VmxExitIoSmi(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7096	{
7097	/* Same treatment as VMX_EXIT_SMI. See comment in hmR0VmxExitSmi(). */
7098	AssertMsgFailed(("Unexpected IO SMI VM-exit. pVM=%p pVCpu=%p pMixedCtx=%p\n", pVM, pVCpu, pMixedCtx));
7099	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7100	}
7101
7102
7103	/**
7104	* VM-exit handler for SIPI (VMX_EXIT_SIPI). Conditional VM-exit.
7105	*/
7106	static DECLCALLBACK(int) hmR0VmxExitSipi(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7107	{
7108	/*
7109	* SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest activity state is used. We currently
7110	* don't make use of it (see hmR0VmxLoadGuestActivityState()) as our guests don't have direct access to the host LAPIC.
7111	* See Intel spec. 25.3 "Other Causes of VM-exits".
7112	*/
7113	AssertMsgFailed(("Unexpected SIPI VM-exit. pVM=%p pVCpu=%p pMixedCtx=%p\n", pVM, pVCpu, pMixedCtx));
7114	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7115	}
7116
7117
7118	/**
7119	* VM-exit handler for INIT signal (VMX_EXIT_INIT_SIGNAL). Unconditional
7120	* VM-exit.
7121	*/
7122	static DECLCALLBACK(int) hmR0VmxExitInitSignal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7123	{
7124	/*
7125	* INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM. See Intel spec. "33.14.1 Default Treatment of
7126	* SMI Delivery" and "29.3 VMX Instructions" for "VMXON". It is -NOT- blocked in VMX non-root operation so we can potentially
7127	* still get these exits. See Intel spec. "23.8 Restrictions on VMX operation".
7128	*/
7129	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7130	return VINF_SUCCESS; /** @todo r=ramshankar: correct?. */
7131	}
7132
7133
7134	/**
7135	* VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
7136	* VM-exit.
7137	*/
7138	static DECLCALLBACK(int) hmR0VmxExitTripleFault(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7139	{
7140	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7141	return VINF_EM_RESET;
7142	}
7143
7144
7145	/**
7146	* VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
7147	*/
7148	static DECLCALLBACK(int) hmR0VmxExitHlt(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7149	{
7150	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7151	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_HLT_EXIT);
7152	int rc = hmR0VmxSaveGuestRip(pVM, pVCpu, pMixedCtx);
7153	rc \|= hmR0VmxSaveGuestRflags(pVM, pVCpu, pMixedCtx);
7154	AssertRCReturn(rc, rc);
7155
7156	pMixedCtx->rip++;
7157	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
7158	if (EMShouldContinueAfterHalt(pVCpu, pMixedCtx)) /* Requires eflags. */
7159	rc = VINF_SUCCESS;
7160	else
7161	rc = VINF_EM_HALT;
7162
7163	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
7164	return rc;
7165	}
7166
7167
7168	/**
7169	* VM-exit handler for instructions that result in a #UD exception delivered to the guest.
7170	*/
7171	static DECLCALLBACK(int) hmR0VmxExitInjectXcptUD(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7172	{
7173	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7174	int rc = hmR0VmxInjectXcptUD(pVM, pVCpu, pMixedCtx);
7175	AssertRCReturn(rc, rc);
7176	return rc;
7177	}
7178
7179
7180	/**
7181	* VM-exit handler for expiry of the VMX preemption timer.
7182	*/
7183	static DECLCALLBACK(int) hmR0VmxExitPreemptionTimer(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7184	{
7185	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7186	/* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
7187	bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
7188	int rc = fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
7189	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
7190	return rc;
7191	}
7192
7193
7194	/**
7195	* VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
7196	*/
7197	static DECLCALLBACK(int) hmR0VmxExitXsetbv(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7198	{
7199	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7200	/* We expose XSETBV to the guest, fallback to the recompiler for emulation. */
7201	/** @todo check if XSETBV is supported by the recompiler. */
7202	return VERR_EM_INTERPRETER;
7203	}
7204
7205
7206	/**
7207	* VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
7208	*/
7209	static DECLCALLBACK(int) hmR0VmxExitInvpcid(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7210	{
7211	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7212	/* The guest should not invalidate the host CPU's TLBs, fallback to recompiler. */
7213	/** @todo implement EMInterpretInvpcid() */
7214	return VERR_EM_INTERPRETER;
7215	}
7216
7217
7218	/**
7219	* VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE).
7220	* Error VM-exit.
7221	*/
7222	static DECLCALLBACK(int) hmR0VmxExitErrInvalidGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7223	{
7224	uint32_t uIntrState;
7225	RTHCUINTREG uHCReg;
7226	uint64_t u64Val;
7227
7228	int rc = hmR0VmxReadEntryIntrInfoVmcs(pVmxTransient);
7229	rc \|= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
7230	rc \|= hmR0VmxReadEntryInstrLenVmcs(pVmxTransient);
7231	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
7232	rc \|= hmR0VmxSaveGuestState(pVM, pVCpu, pMixedCtx);
7233	AssertRCReturn(rc, rc);
7234
7235	Log(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntrInfo));
7236	Log(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
7237	Log(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
7238	Log(("VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE %#RX32\n", uIntrState));
7239
7240	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR0, &u64Val); AssertRC(rc);
7241	Log(("VMX_VMCS_GUEST_CR0 %#RX64\n", u64Val));
7242	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
7243	Log(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
7244	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
7245	Log(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
7246	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
7247	Log(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
7248	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
7249	Log(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
7250	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
7251	Log(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
7252
7253	HMDumpRegs(pVM, pVCpu, pMixedCtx);
7254
7255	return VERR_VMX_INVALID_GUEST_STATE;
7256	}
7257
7258
7259	/**
7260	* VM-exit handler for VM-entry failure due to an MSR-load
7261	* (VMX_EXIT_ERR_MSR_LOAD). Error VM-exit.
7262	*/
7263	static DECLCALLBACK(int) hmR0VmxExitErrMsrLoad(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7264	{
7265	AssertMsgFailed(("Unexpected MSR-load exit. pVM=%p pVCpu=%p pMixedCtx=%p\n", pVM, pVCpu, pMixedCtx));
7266	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7267	}
7268
7269
7270	/**
7271	* VM-exit handler for VM-entry failure due to a machine-check event
7272	* (VMX_EXIT_ERR_MACHINE_CHECK). Error VM-exit.
7273	*/
7274	static DECLCALLBACK(int) hmR0VmxExitErrMachineCheck(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7275	{
7276	AssertMsgFailed(("Unexpected machine-check event exit. pVM=%p pVCpu=%p pMixedCtx=%p\n", pVM, pVCpu, pMixedCtx));
7277	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7278	}
7279
7280
7281	/**
7282	* VM-exit handler for all undefined reasons. Should never ever happen.. in
7283	* theory.
7284	*/
7285	static DECLCALLBACK(int) hmR0VmxExitErrUndefined(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7286	{
7287	AssertMsgFailed(("Huh!? Undefined VM-exit reason %d. pVM=%p pVCpu=%p pMixedCtx=%p\n", pVmxTransient->uExitReason,
7288	pVM, pVCpu, pMixedCtx));
7289	return VERR_VMX_UNDEFINED_EXIT_CODE;
7290	}
7291
7292
7293	/**
7294	* VM-exit handler for XDTR (LGDT, SGDT, LIDT, SIDT) accesses
7295	* (VMX_EXIT_XDTR_ACCESS) and LDT and TR access (LLDT, LTR, SLDT, STR).
7296	* Conditional VM-exit.
7297	*/
7298	static DECLCALLBACK(int) hmR0VmxExitXdtrAccess(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7299	{
7300	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7301	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT. */
7302	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitXdtrAccess);
7303	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT)
7304	return VERR_EM_INTERPRETER;
7305	AssertMsgFailed(("Unexpected XDTR access. pVM=%p pVCpu=%p pMixedCtx=%p\n", pVM, pVCpu, pMixedCtx));
7306	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7307	}
7308
7309
7310	/**
7311	* VM-exit handler for RDRAND (VMX_EXIT_RDRAND). Conditional VM-exit.
7312	*/
7313	static DECLCALLBACK(int) hmR0VmxExitRdrand(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7314	{
7315	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7316	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT. */
7317	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdrand);
7318	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT)
7319	return VERR_EM_INTERPRETER;
7320	AssertMsgFailed(("Unexpected RDRAND exit. pVM=%p pVCpu=%p pMixedCtx=%p\n", pVM, pVCpu, pMixedCtx));
7321	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7322	}
7323
7324
7325	/**
7326	* VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
7327	*/
7328	static DECLCALLBACK(int) hmR0VmxExitRdmsr(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7329	{
7330	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7331	/* EMInterpretRdmsr() requires CR0, Eflags and SS segment register. */
7332	int rc = hmR0VmxSaveGuestCR0(pVM, pVCpu, pMixedCtx);
7333	rc \|= hmR0VmxSaveGuestRflags(pVM, pVCpu, pMixedCtx);
7334	rc \|= hmR0VmxSaveGuestSegmentRegs(pVM, pVCpu, pMixedCtx);
7335	AssertRCReturn(rc, rc);
7336
7337	rc = EMInterpretRdmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7338	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER,
7339	("hmR0VmxExitRdmsr: failed, invalid error code %Rrc\n", rc));
7340	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
7341
7342	/* Update RIP and continue guest execution. */
7343	if (RT_LIKELY(rc == VINF_SUCCESS))
7344	{
7345	rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
7346	rc \|= hmR0VmxSaveGuestRip(pVM, pVCpu, pMixedCtx);
7347	AssertRCReturn(rc, rc);
7348
7349	Assert(pVmxTransient->cbInstr == 2);
7350	pMixedCtx->rip += pVmxTransient->cbInstr;
7351	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
7352	}
7353	return rc;
7354	}
7355
7356
7357	/**
7358	* VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
7359	*/
7360	static DECLCALLBACK(int) hmR0VmxExitWrmsr(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7361	{
7362	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7363	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
7364	AssertRCReturn(rc, rc);
7365	Assert(pVmxTransient->cbInstr == 2);
7366
7367	/* If TPR patching is active, LSTAR holds the guest TPR, writes to it must be propagated to the APIC. */
7368	if ( pVM->hm.s.fTPRPatchingActive
7369	&& pMixedCtx->ecx == MSR_K8_LSTAR)
7370	{
7371	Assert(!CPUMIsGuestInLongModeEx(pMixedCtx)); /* Requires EFER but it's always up-to-date. */
7372	if ((pMixedCtx->eax & 0xff) != pVmxTransient->u8GuestTpr)
7373	{
7374	rc = PDMApicSetTPR(pVCpu, pMixedCtx->eax & 0xff);
7375	AssertRC(rc);
7376	}
7377
7378	rc = hmR0VmxSaveGuestRip(pVM, pVCpu, pMixedCtx);
7379	AssertRCReturn(rc, rc);
7380	pMixedCtx->rip += pVmxTransient->cbInstr;
7381	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
7382	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
7383	return VINF_SUCCESS;
7384	}
7385
7386	/* Update MSRs that are part of the VMCS when MSR-bitmaps are not supported. */
7387	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS))
7388	{
7389	switch (pMixedCtx->ecx)
7390	{
7391	case MSR_IA32_SYSENTER_CS: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SYSENTER_CS_MSR; break;
7392	case MSR_IA32_SYSENTER_EIP: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SYSENTER_EIP_MSR; break;
7393	case MSR_IA32_SYSENTER_ESP: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SYSENTER_ESP_MSR; break;
7394	case MSR_K8_FS_BASE: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_FS_BASE_MSR; break;
7395	case MSR_K8_GS_BASE: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_GS_BASE_MSR; break;
7396	}
7397	}
7398	#ifdef DEBUG
7399	else
7400	{
7401	/* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
7402	switch (pMixedCtx->ecx)
7403	{
7404	case MSR_IA32_SYSENTER_CS:
7405	case MSR_IA32_SYSENTER_EIP:
7406	case MSR_IA32_SYSENTER_ESP:
7407	case MSR_K8_FS_BASE:
7408	case MSR_K8_GS_BASE:
7409	AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%RX32\n", pMixedCtx->ecx));
7410	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7411	case MSR_K8_LSTAR:
7412	case MSR_K6_STAR:
7413	case MSR_K8_SF_MASK:
7414	case MSR_K8_TSC_AUX:
7415	AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%RX32\n", pMixedCtx->ecx));
7416	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7417	}
7418	}
7419	#endif
7420
7421	/* EMInterpretWrmsr() requires CR0, EFLAGS and SS segment register. */
7422	rc = hmR0VmxSaveGuestCR0(pVM, pVCpu, pMixedCtx);
7423	rc \|= hmR0VmxSaveGuestRflags(pVM, pVCpu, pMixedCtx);
7424	rc \|= hmR0VmxSaveGuestSegmentRegs(pVM, pVCpu, pMixedCtx);
7425	AssertRCReturn(rc, rc);
7426
7427	rc = EMInterpretWrmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7428	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER, ("hmR0VmxExitWrmsr: failed, invalid error code %Rrc\n", rc));
7429	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
7430
7431	/* Update guest-state and continue execution. */
7432	if (RT_LIKELY(rc == VINF_SUCCESS))
7433	{
7434	rc = hmR0VmxSaveGuestRip(pVM, pVCpu, pMixedCtx);
7435	AssertRCReturn(rc, rc);
7436
7437	pMixedCtx->rip += pVmxTransient->cbInstr;
7438	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
7439
7440	/* If this is an X2APIC WRMSR access, update the APIC state as well. */
7441	if ( pMixedCtx->ecx >= MSR_IA32_X2APIC_START
7442	&& pMixedCtx->ecx <= MSR_IA32_X2APIC_END)
7443	{
7444	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_VMX_GUEST_APIC_STATE;
7445	}
7446	}
7447	return rc;
7448	}
7449
7450
7451	/**
7452	* VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
7453	*/
7454	static DECLCALLBACK(int) hmR0VmxExitPause(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7455	{
7456	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7457	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_PAUSE_EXIT. */
7458	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPause);
7459	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_PAUSE_EXIT)
7460	return VERR_EM_INTERPRETER;
7461	AssertMsgFailed(("Unexpected PAUSE exit. pVM=%p pVCpu=%p pMixedCtx=%p\n", pVM, pVCpu, pMixedCtx));
7462	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7463	}
7464
7465
7466	/**
7467	* VM-exit handler for when the TPR value is lowered below the specified
7468	* threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
7469	*/
7470	static DECLCALLBACK(int) hmR0VmxExitTprBelowThreshold(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7471	{
7472	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7473	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW);
7474
7475	/*
7476	* The TPR has already been updated, see hmR0VMXPostRunGuest(). RIP is also updated as part of the VM-exit by VT-x. Update
7477	* the threshold in the VMCS, deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectPendingInterrupt() and
7478	* resume guest execution.
7479	*/
7480	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_VMX_GUEST_APIC_STATE;
7481	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
7482	return VINF_SUCCESS;
7483	}
7484
7485
7486	/**
7487	* VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
7488	* VM-exit.
7489	*
7490	* @retval VINF_SUCCESS when guest execution can continue.
7491	* @retval VINF_PGM_CHANGE_MODE when shadow paging mode changed, back to ring-3.
7492	* @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
7493	* @retval VERR_EM_INTERPRETER when something unexpected happened, fallback to
7494	* recompiler.
7495	*/
7496	static DECLCALLBACK(int) hmR0VmxExitMovCRx(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7497	{
7498	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7499	int rc = hmR0VmxReadExitQualificationVmcs(pVmxTransient);
7500	AssertRCReturn(rc, rc);
7501
7502	const RTGCUINTPTR uExitQualification = pVmxTransient->uExitQualification;
7503	const uint8_t uAccessType = VMX_EXIT_QUALIFICATION_CRX_ACCESS(uExitQualification);
7504	switch (uAccessType)
7505	{
7506	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_WRITE: /* MOV to CRx */
7507	{
7508	/* EMInterpretCRxWrite() references a lot of guest state (EFER, RFLAGS, Segment Registers, etc.) Sync entire state */
7509	rc = hmR0VmxSaveGuestState(pVM, pVCpu, pMixedCtx);
7510	AssertRCReturn(rc, rc);
7511
7512	rc = EMInterpretCRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
7513	VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification),
7514	VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification));
7515	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_PGM_SYNC_CR3);
7516
7517	switch (VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification))
7518	{
7519	case 0: /* CR0 */
7520	Log(("CR0 write rc=%d\n", rc));
7521	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
7522	break;
7523	case 2: /* CR2 */
7524	Log(("CR2 write rc=%d\n", rc));
7525	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR2;
7526	break;
7527	case 3: /* CR3 */
7528	Assert(!pVM->hm.s.fNestedPaging \|\| !CPUMIsGuestPagingEnabledEx(pMixedCtx));
7529	Log(("CR3 write rc=%d\n", rc));
7530	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR3;
7531	break;
7532	case 4: /* CR4 */
7533	Log(("CR4 write rc=%d\n", rc));
7534	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR4;
7535	break;
7536	case 8: /* CR8 */
7537	Assert(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW));
7538	/* CR8 contains the APIC TPR. Was updated by EMInterpretCRxWrite(). */
7539	/* We don't need to update HM_CHANGED_VMX_GUEST_APIC_STATE here as this -cannot- happen with TPR shadowing. */
7540	break;
7541	default:
7542	AssertMsgFailed(("Invalid CRx register %#x\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)));
7543	break;
7544	}
7545
7546	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxWrite[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
7547	break;
7548	}
7549
7550	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ: /* MOV from CRx */
7551	{
7552	/* EMInterpretCRxRead() requires EFER MSR, CS. */
7553	rc \|= hmR0VmxSaveGuestSegmentRegs(pVM, pVCpu, pMixedCtx);
7554	AssertRCReturn(rc, rc);
7555	Assert( !pVM->hm.s.fNestedPaging
7556	\|\| !CPUMIsGuestPagingEnabledEx(pMixedCtx)
7557	\|\| VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 3);
7558
7559	/* CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
7560	Assert( VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 8
7561	\|\| !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW));
7562
7563	rc = EMInterpretCRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
7564	VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification),
7565	VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification));
7566	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER);
7567	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxRead[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
7568	Log(("CRX CR%d Read access rc=%d\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification), rc));
7569	break;
7570	}
7571
7572	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_CLTS: /* CLTS (Clear Task-Switch Flag in CR0) */
7573	{
7574	rc = hmR0VmxSaveGuestCR0(pVM, pVCpu, pMixedCtx);
7575	rc \|= EMInterpretCLTS(pVM, pVCpu);
7576	AssertRCReturn(rc, rc);
7577	if (RT_LIKELY(rc == VINF_SUCCESS))
7578	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
7579	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
7580	Log(("CRX CLTS write rc=%d\n", rc));
7581	break;
7582	}
7583
7584	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
7585	{
7586	rc = hmR0VmxSaveGuestCR0(pVM, pVCpu, pMixedCtx);
7587	rc \|= EMInterpretLMSW(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(uExitQualification));
7588	AssertRCReturn(rc, rc);
7589	if (RT_LIKELY(rc == VINF_SUCCESS))
7590	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
7591	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
7592	Log(("CRX LMSW write rc=%d\n", rc));
7593	break;
7594	}
7595
7596	default:
7597	{
7598	AssertMsgFailed(("Invalid access-type in Mov CRx exit qualification %#x\n", uAccessType));
7599	return VERR_VMX_UNEXPECTED_EXCEPTION;
7600	}
7601	}
7602
7603	/* Validate possible error codes. */
7604	Assert(rc == VINF_SUCCESS \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_SYNC_CR3);
7605	if (RT_SUCCESS(rc))
7606	{
7607	int rc2 = hmR0VmxSaveGuestRip(pVM, pVCpu, pMixedCtx);
7608	rc2 \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
7609	AssertRCReturn(rc2, rc2);
7610	pMixedCtx->rip += pVmxTransient->cbInstr;
7611	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
7612	}
7613
7614	return rc;
7615	}
7616
7617
7618	/**
7619	* VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
7620	* VM-exit.
7621	*/
7622	static DECLCALLBACK(int) hmR0VmxExitIoInstr(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7623	{
7624	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7625
7626	int rc = hmR0VmxReadExitQualificationVmcs(pVmxTransient);
7627	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
7628	rc \|= hmR0VmxSaveGuestRip(pVM, pVCpu, pMixedCtx);
7629	rc \|= hmR0VmxSaveGuestRflags(pVM, pVCpu, pMixedCtx); /* Eflag checks in EMInterpretDisasCurrent(). */
7630	rc \|= hmR0VmxSaveGuestControlRegs(pVM, pVCpu, pMixedCtx); /* CR0 checks & PGM* in EMInterpretDisasCurrent(). */
7631	rc \|= hmR0VmxSaveGuestSegmentRegs(pVM, pVCpu, pMixedCtx); /* SELM checks in EMInterpretDisasCurrent(). */
7632	/* EFER also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
7633	AssertRCReturn(rc, rc);
7634
7635	/* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
7636	uint32_t uIOPort = VMX_EXIT_QUALIFICATION_IO_PORT(pVmxTransient->uExitQualification);
7637	uint32_t uIOWidth = VMX_EXIT_QUALIFICATION_IO_WIDTH(pVmxTransient->uExitQualification);
7638	bool fIOWrite = (VMX_EXIT_QUALIFICATION_IO_DIRECTION(pVmxTransient->uExitQualification)
7639	== VMX_EXIT_QUALIFICATION_IO_DIRECTION_OUT);
7640	bool fIOString = (VMX_EXIT_QUALIFICATION_IO_STRING(pVmxTransient->uExitQualification) == 1);
7641	Assert(uIOWidth == 0 \|\| uIOWidth == 1 \|\| uIOWidth == 3);
7642
7643	/* I/O operation lookup arrays. */
7644	static const uint32_t s_aIOSize[4] = { 1, 2, 0, 4 }; /* Size of the I/O Accesses. */
7645	static const uint32_t s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving the result (in AL/AX/EAX). */
7646
7647	const uint32_t cbSize = s_aIOSize[uIOWidth];
7648	const uint32_t cbInstr = pVmxTransient->cbInstr;
7649	if (fIOString)
7650	{
7651	/* INS/OUTS - I/O String instruction. */
7652	PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
7653	/** @todo for now manually disassemble later optimize by getting the fields from
7654	* the VMCS. */
7655	/** @todo VMX_VMCS_RO_EXIT_GUEST_LINEAR_ADDR contains the flat pointer
7656	* operand of the instruction. VMX_VMCS32_RO_EXIT_INSTR_INFO contains
7657	* segment prefix info. */
7658	rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, NULL);
7659	if (RT_SUCCESS(rc))
7660	{
7661	if (fIOWrite)
7662	{
7663	VBOXSTRICTRC rc2 = IOMInterpretOUTSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
7664	(DISCPUMODE)pDis->uAddrMode, cbSize);
7665	rc = VBOXSTRICTRC_VAL(rc2);
7666	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringWrite);
7667	}
7668	else
7669	{
7670	VBOXSTRICTRC rc2 = IOMInterpretINSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
7671	(DISCPUMODE)pDis->uAddrMode, cbSize);
7672	rc = VBOXSTRICTRC_VAL(rc2);
7673	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringRead);
7674	}
7675	}
7676	else
7677	{
7678	AssertMsg(rc == VERR_EM_INTERPRETER, ("rc=%Rrc RIP %#RX64\n", rc, pMixedCtx->rip));
7679	rc = VINF_EM_RAW_EMULATE_INSTR;
7680	}
7681	}
7682	else
7683	{
7684	/* IN/OUT - I/O instruction. */
7685	const uint32_t uAndVal = s_aIOOpAnd[uIOWidth];
7686	Assert(!VMX_EXIT_QUALIFICATION_IO_REP(pVmxTransient->uExitQualification));
7687	if (fIOWrite)
7688	{
7689	VBOXSTRICTRC rc2 = IOMIOPortWrite(pVM, pVCpu, uIOPort, pMixedCtx->eax & uAndVal, cbSize);
7690	rc = VBOXSTRICTRC_VAL(rc2);
7691	if (rc == VINF_IOM_R3_IOPORT_WRITE)
7692	HMR0SavePendingIOPortWrite(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbSize);
7693	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
7694	}
7695	else
7696	{
7697	uint32_t u32Result = 0;
7698	VBOXSTRICTRC rc2 = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbSize);
7699	rc = VBOXSTRICTRC_VAL(rc2);
7700	if (IOM_SUCCESS(rc))
7701	{
7702	/* Save result of I/O IN instr. in AL/AX/EAX. */
7703	pMixedCtx->eax = (pMixedCtx->eax & ~uAndVal) \| (u32Result & uAndVal);
7704	}
7705	else if (rc == VINF_IOM_R3_IOPORT_READ)
7706	HMR0SavePendingIOPortRead(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbSize);
7707	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
7708	}
7709	}
7710
7711	if (IOM_SUCCESS(rc))
7712	{
7713	pMixedCtx->rip += cbInstr;
7714	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
7715	if (RT_LIKELY(rc == VINF_SUCCESS))
7716	{
7717	rc = hmR0VmxSaveGuestDebugRegs(pVM, pVCpu, pMixedCtx); /* For DR7. */
7718	AssertRCReturn(rc, rc);
7719
7720	/* If any IO breakpoints are armed, then we should check if a debug trap needs to be generated. */
7721	if (pMixedCtx->dr[7] & X86_DR7_ENABLED_MASK)
7722	{
7723	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
7724	for (unsigned i = 0; i < 4; i++)
7725	{
7726	unsigned uBPLen = s_aIOSize[X86_DR7_GET_LEN(pMixedCtx->dr[7], i)];
7727	if ( ( uIOPort >= pMixedCtx->dr[i]
7728	&& uIOPort < pMixedCtx->dr[i] + uBPLen)
7729	&& (pMixedCtx->dr[7] & (X86_DR7_L(i) \| X86_DR7_G(i)))
7730	&& (pMixedCtx->dr[7] & X86_DR7_RW(i, X86_DR7_RW_IO)) == X86_DR7_RW(i, X86_DR7_RW_IO))
7731	{
7732	Assert(CPUMIsGuestDebugStateActive(pVCpu));
7733	uint64_t uDR6 = ASMGetDR6();
7734
7735	/* Clear all breakpoint status flags and set the one we just hit. */
7736	uDR6 &= ~(X86_DR6_B0 \| X86_DR6_B1 \| X86_DR6_B2 \| X86_DR6_B3);
7737	uDR6 \|= (uint64_t)RT_BIT(i);
7738
7739	/*
7740	* Note: AMD64 Architecture Programmer's Manual 13.1:
7741	* Bits 15:13 of the DR6 register is never cleared by the processor and must
7742	* be cleared by software after the contents have been read.
7743	*/
7744	ASMSetDR6(uDR6);
7745
7746	/* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
7747	pMixedCtx->dr[7] &= ~X86_DR7_GD;
7748
7749	/* Paranoia. */
7750	pMixedCtx->dr[7] &= 0xffffffff; /* Upper 32 bits reserved. */
7751	pMixedCtx->dr[7] &= ~(RT_BIT(11) \| RT_BIT(12) \| RT_BIT(14) \| RT_BIT(15)); /* MBZ. */
7752	pMixedCtx->dr[7] \|= 0x400; /* MB1. */
7753
7754	/* Resync DR7 */
7755	/** @todo probably cheaper to just reload DR7, nothing else needs changing. */
7756	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_DEBUG;
7757
7758	/* Inject #DB and get on with guest execution. */
7759	rc = hmR0VmxInjectXcptDB(pVM, pVCpu, pMixedCtx);
7760	AssertRCReturn(rc, rc);
7761	break;
7762	}
7763	}
7764	}
7765	}
7766	}
7767
7768	#ifdef DEBUG
7769	if (rc == VINF_IOM_R3_IOPORT_READ)
7770	Assert(!fIOWrite);
7771	else if (rc == VINF_IOM_R3_IOPORT_WRITE)
7772	Assert(fIOWrite);
7773	else
7774	{
7775	AssertMsg( RT_FAILURE(rc)
7776	\|\| rc == VINF_SUCCESS
7777	\|\| rc == VINF_EM_RAW_EMULATE_INSTR
7778	\|\| rc == VINF_EM_RAW_GUEST_TRAP
7779	\|\| rc == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", rc));
7780	}
7781	#endif
7782
7783	return rc;
7784	}
7785
7786
7787	/**
7788	* VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
7789	* VM-exit.
7790	*/
7791	static DECLCALLBACK(int) hmR0VmxExitTaskSwitch(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7792	{
7793	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7794
7795	/** @todo Emulate task switch someday, currently just going back to ring-3 for
7796	* emulation. */
7797
7798	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
7799	return VERR_EM_INTERPRETER;
7800	}
7801
7802
7803	/**
7804	* VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
7805	*/
7806	static DECLCALLBACK(int) hmR0VmxExitMtf(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7807	{
7808	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7809	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_TRAP_FLAG);
7810	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_TRAP_FLAG;
7811	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls);
7812	AssertRCReturn(rc, rc);
7813	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMtf);
7814	return VINF_EM_DBG_STOP;
7815	}
7816
7817
7818	/**
7819	* VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
7820	*/
7821	static DECLCALLBACK(int) hmR0VmxExitApicAccess(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7822	{
7823	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7824	int rc = hmR0VmxReadExitQualificationVmcs(pVmxTransient);
7825
7826	/** @todo Investigate if IOMMMIOPhysHandler() requires a lot of state, for now
7827	* just sync the whole thing. */
7828	rc \|= hmR0VmxSaveGuestState(pVM, pVCpu, pMixedCtx);
7829	AssertRCReturn(rc, rc);
7830
7831	/* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
7832	unsigned uAccessType = VMX_EXIT_QUALIFICATION_APIC_ACCESS_TYPE(pVmxTransient->uExitQualification);
7833	switch (uAccessType)
7834	{
7835	case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
7836	case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
7837	{
7838	if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
7839	&& VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification) == 0x80)
7840	{
7841	AssertMsgFailed(("hmR0VmxExitApicAccess: can't touch TPR offset while using TPR shadowing.\n"));
7842	}
7843
7844	RTGCPHYS GCPhys = pMixedCtx->msrApicBase; /* Always up-to-date, msrApicBase is not part of the VMCS. */
7845	GCPhys &= PAGE_BASE_GC_MASK;
7846	GCPhys += VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification);
7847	Log(("ApicAccess %RGp %#x\n", GCPhys, VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification)));
7848	VBOXSTRICTRC rc2 = IOMMMIOPhysHandler(pVM, pVCpu, (uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ) ? 0 : X86_TRAP_PF_RW,
7849	CPUMCTX2CORE(pMixedCtx), GCPhys);
7850	rc = VBOXSTRICTRC_VAL(rc2);
7851	if ( rc == VINF_SUCCESS
7852	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
7853	\|\| rc == VERR_PAGE_NOT_PRESENT)
7854	{
7855	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
7856	\| HM_CHANGED_VMX_GUEST_APIC_STATE;
7857	rc = VINF_SUCCESS;
7858	}
7859	break;
7860	}
7861
7862	default:
7863	rc = VINF_EM_RAW_EMULATE_INSTR;
7864	break;
7865	}
7866
7867	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
7868	return rc;
7869	}
7870
7871
7872	/**
7873	* VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
7874	* VM-exit.
7875	*/
7876	static DECLCALLBACK(int) hmR0VmxExitMovDRx(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7877	{
7878	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7879
7880	/* We should -not- get this VM-exit if the guest is debugging. */
7881	if (CPUMIsGuestDebugStateActive(pVCpu))
7882	{
7883	AssertMsgFailed(("Unexpected MOV DRx exit. pVM=%p pVCpu=%p pMixedCtx=%p\n", pVM, pVCpu, pMixedCtx));
7884	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7885	}
7886
7887	int rc = VERR_INTERNAL_ERROR_5;
7888	if ( !DBGFIsStepping(pVCpu)
7889	&& !CPUMIsHyperDebugStateActive(pVCpu))
7890	{
7891	Assert(!CPUMIsGuestDebugStateActive(pVCpu));
7892
7893	/* Don't intercept MOV DRx. */
7894	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT;
7895	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls);
7896	AssertRCReturn(rc, rc);
7897
7898	/* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
7899	rc = CPUMR0LoadGuestDebugState(pVM, pVCpu, pMixedCtx, true /* include DR6 */);
7900	AssertRCReturn(rc, rc);
7901	Assert(CPUMIsGuestDebugStateActive(pVCpu));
7902
7903	#ifdef VBOX_WITH_STATISTICS
7904	rc = hmR0VmxReadExitQualificationVmcs(pVmxTransient);
7905	AssertRCReturn(rc, rc);
7906	if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
7907	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
7908	else
7909	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
7910	#endif
7911	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
7912	return VINF_SUCCESS;
7913	}
7914
7915	/** @todo clear VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT after the first
7916	* time and restore DRx registers afterwards */
7917	/*
7918	* EMInterpretDRx[Write\|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER, CS. EFER is always up-to-date, see
7919	* hmR0VmxSaveGuestAutoLoadStoreMsrs(). Update only the segment registers from the CPU.
7920	*/
7921	rc = hmR0VmxReadExitQualificationVmcs(pVmxTransient);
7922	rc \|= hmR0VmxSaveGuestSegmentRegs(pVM, pVCpu, pMixedCtx);
7923	AssertRCReturn(rc, rc);
7924
7925	if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
7926	{
7927	rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
7928	VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification),
7929	VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification));
7930	if (RT_SUCCESS(rc))
7931	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_DEBUG;
7932	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
7933	}
7934	else
7935	{
7936	rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
7937	VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification),
7938	VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification));
7939	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
7940	}
7941
7942	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER);
7943	if (RT_SUCCESS(rc))
7944	{
7945	int rc2 = hmR0VmxSaveGuestRip(pVM, pVCpu, pMixedCtx);
7946	rc2 \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
7947	AssertRCReturn(rc2, rc2);
7948	pMixedCtx->rip += pVmxTransient->cbInstr;
7949	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
7950	}
7951	return rc;
7952	}
7953
7954
7955	/**
7956	* VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
7957	* Conditional VM-exit.
7958	*/
7959	static DECLCALLBACK(int) hmR0VmxExitEptMisconfig(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7960	{
7961	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7962	Assert(pVM->hm.s.fNestedPaging);
7963
7964	RTGCPHYS GCPhys = 0;
7965	int rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
7966	AssertRCReturn(rc, rc);
7967
7968	rc = hmR0VmxSaveGuestState(pVM, pVCpu, pMixedCtx); /** @todo Can we do better? */
7969	AssertRCReturn(rc, rc);
7970
7971	Log(("EPT misconfig at %#RX64 RIP=%#RX64\n", GCPhys, pMixedCtx->rip));
7972
7973	/*
7974	* If we succeed, resume guest execution.
7975	* If we fail in interpreting the instruction because we couldn't get the guest physical address
7976	* of the page containing the instruction via the guest's page tables (we would invalidate the guest page
7977	* in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
7978	* weird case. See @bugref{6043}.
7979	*/
7980	VBOXSTRICTRC rc2 = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pMixedCtx), GCPhys, UINT32_MAX);
7981	rc = VBOXSTRICTRC_VAL(rc2);
7982	Log(("EPT misconfig rc=%d\n", rc));
7983	if ( rc == VINF_SUCCESS
7984	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
7985	\|\| rc == VERR_PAGE_NOT_PRESENT)
7986	{
7987	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
7988	\| HM_CHANGED_VMX_GUEST_APIC_STATE;
7989	return VINF_SUCCESS;
7990	}
7991	return rc;
7992	}
7993
7994
7995	/**
7996	* VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
7997	* VM-exit.
7998	*/
7999	static DECLCALLBACK(int) hmR0VmxExitEptViolation(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8000	{
8001	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
8002	Assert(pVM->hm.s.fNestedPaging);
8003
8004	RTGCPHYS GCPhys = 0;
8005	int rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
8006	rc \|= hmR0VmxReadExitQualificationVmcs(pVmxTransient);
8007	rc \|= hmR0VmxSaveGuestState(pVM, pVCpu, pMixedCtx); /** @todo can we do better? */
8008	AssertRCReturn(rc, rc);
8009
8010	/* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
8011	AssertMsg(((pVmxTransient->uExitQualification >> 7) & 3) != 2, ("%#RGv", pVmxTransient->uExitQualification));
8012
8013	RTGCUINT uErrorCode = 0;
8014	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_INSTR_FETCH)
8015	uErrorCode \|= X86_TRAP_PF_ID;
8016	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_DATA_WRITE)
8017	uErrorCode \|= X86_TRAP_PF_RW;
8018	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_ENTRY_PRESENT)
8019	uErrorCode \|= X86_TRAP_PF_P;
8020
8021	TRPMAssertTrap(pVCpu, X86_XCPT_PF, TRPM_TRAP);
8022	TRPMSetErrorCode(pVCpu, uErrorCode);
8023	TRPMSetFaultAddress(pVCpu, GCPhys);
8024
8025	Log(("EPT violation %#x at %#RGv ErrorCode %#x CS:EIP=%04x:%#RX64\n", (uint32_t)pVmxTransient->uExitQualification, GCPhys,
8026	uErrorCode, pMixedCtx->cs.Sel, pMixedCtx->rip));
8027
8028	/* Handle the pagefault trap for the nested shadow table. */
8029	rc = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pMixedCtx), GCPhys);
8030	TRPMResetTrap(pVCpu);
8031
8032	/* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
8033	if ( rc == VINF_SUCCESS
8034	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
8035	\|\| rc == VERR_PAGE_NOT_PRESENT)
8036	{
8037	/* Successfully synced our shadow page tables or emulation MMIO instruction. */
8038	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
8039	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
8040	\| HM_CHANGED_VMX_GUEST_APIC_STATE;
8041	return VINF_SUCCESS;
8042	}
8043
8044	Log(("EPT return to ring-3 rc=%d\n"));
8045
8046	/* We need to go back to ring-3 to emulate the instruction as we could not handle it correctly, tell TRPM. */
8047	/** @todo Shouldn't we update TRPM here? */
8048	return rc;
8049	}
8050
8051
8052	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
8053	/* -=-=-=-=-=-=-=-=-=- VM-exit Exception Handlers -=-=-=-=-=-=-=-=-=-=- */
8054	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
8055	/**
8056	* VM-exit exception handler for #MF (Math Fault: floating point exception).
8057	*/
8058	static DECLCALLBACK(int) hmR0VmxExitXcptMF(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8059	{
8060	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8061
8062	int rc = hmR0VmxSaveGuestCR0(pVM, pVCpu, pMixedCtx);
8063	AssertRCReturn(rc, rc);
8064	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
8065
8066	if (!(pMixedCtx->cr0 & X86_CR0_NE))
8067	{
8068	/* Old-style FPU error reporting needs some extra work. */
8069	/** @todo don't fall back to the recompiler, but do it manually. */
8070	return VERR_EM_INTERPRETER;
8071	}
8072	rc = hmR0VmxInjectEventVmcs(pVM, pVCpu, pMixedCtx,
8073	VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8074	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode);
8075	AssertRCReturn(rc, rc);
8076	return rc;
8077	}
8078
8079
8080	/**
8081	* VM-exit exception handler for #BP (Breakpoint exception).
8082	*/
8083	static DECLCALLBACK(int) hmR0VmxExitXcptBP(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8084	{
8085	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8086
8087	/** @todo Try optimize this by not saving the entire guest state unless
8088	* really needed. */
8089	int rc = hmR0VmxSaveGuestState(pVM, pVCpu, pMixedCtx);
8090	AssertRCReturn(rc, rc);
8091	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
8092
8093	rc = DBGFRZTrap03Handler(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
8094	if (rc == VINF_EM_RAW_GUEST_TRAP)
8095	{
8096	rc = hmR0VmxReadExitIntrInfoVmcs(pVmxTransient);
8097	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
8098	rc \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVmxTransient);
8099	AssertRCReturn(rc, rc);
8100
8101	rc = hmR0VmxInjectEventVmcs(pVM, pVCpu, pMixedCtx,
8102	VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8103	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode);
8104	AssertRCReturn(rc, rc);
8105	}
8106
8107	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RESET);
8108	return rc;
8109	}
8110
8111
8112	/**
8113	* VM-exit exception handler for #DB (Debug exception).
8114	*/
8115	static DECLCALLBACK(int) hmR0VmxExitXcptDB(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8116	{
8117	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8118
8119	int rc = hmR0VmxReadExitQualificationVmcs(pVmxTransient);
8120	rc \|= hmR0VmxSaveGuestSegmentRegs(pVM, pVCpu, pMixedCtx);
8121	rc \|= hmR0VmxSaveGuestRflags(pVM, pVCpu, pMixedCtx);
8122	AssertRCReturn(rc, rc);
8123
8124	/* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
8125	uint64_t uDR6 = X86_DR6_INIT_VAL;
8126	uDR6 \|= (pVmxTransient->uExitQualification
8127	& (X86_DR6_B0 \| X86_DR6_B1 \| X86_DR6_B2 \| X86_DR6_B3 \| X86_DR6_BD \| X86_DR6_BS));
8128	rc = DBGFRZTrap01Handler(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uDR6);
8129	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
8130	if (rc == VINF_EM_RAW_GUEST_TRAP)
8131	{
8132	/** @todo revisit this. */
8133	/* DR6, DR7.GD and IA32_DEBUGCTL.LBR are not updated yet. See Intel spec. 27.1 "Architectural State before a VM-Exit". */
8134	pMixedCtx->dr[6] = uDR6;
8135
8136	if (CPUMIsGuestDebugStateActive(pVCpu))
8137	ASMSetDR6(pMixedCtx->dr[6]);
8138
8139	/* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
8140	pMixedCtx->dr[7] &= ~X86_DR7_GD;
8141
8142	/* Paranoia. */
8143	pMixedCtx->dr[7] &= 0xffffffff; /* upper 32 bits reserved */
8144	pMixedCtx->dr[7] &= ~(RT_BIT(11) \| RT_BIT(12) \| RT_BIT(14) \| RT_BIT(15)); /* must be zero */
8145	pMixedCtx->dr[7] \|= 0x400; /* must be one */
8146
8147	/* Resync DR7. */
8148	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_DR7, pMixedCtx->dr[7]);
8149
8150	rc \|= hmR0VmxReadExitIntrInfoVmcs(pVmxTransient);
8151	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
8152	rc \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVmxTransient);
8153	rc \|= hmR0VmxInjectEventVmcs(pVM, pVCpu, pMixedCtx,
8154	VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8155	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode);
8156	AssertRCReturn(rc,rc);
8157	return rc;
8158	}
8159	/* Return to ring 3 to deal with the debug exit code. */
8160	return rc;
8161	}
8162
8163
8164	/**
8165	* VM-exit exception handler for #NM (Device-not-available exception: floating
8166	* point exception).
8167	*/
8168	static DECLCALLBACK(int) hmR0VmxExitXcptNM(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8169	{
8170	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8171
8172	#ifndef VBOX_ALWAYS_TRAP_ALL_EXCEPTIONS
8173	Assert(!CPUMIsGuestFPUStateActive(pVCpu));
8174	#endif
8175
8176	/* We require CR0 and EFER. EFER is always up-to-date. */
8177	int rc = hmR0VmxSaveGuestControlRegs(pVM, pVCpu, pMixedCtx);
8178	AssertRCReturn(rc, rc);
8179
8180	/* Lazy FPU loading; Load the guest-FPU state transparently and continue execution of the guest. */
8181	rc = CPUMR0LoadGuestFPU(pVM, pVCpu, pMixedCtx);
8182	if (rc == VINF_SUCCESS)
8183	{
8184	Assert(CPUMIsGuestFPUStateActive(pVCpu));
8185
8186	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_FPU;
8187	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowNM);
8188	return VINF_SUCCESS;
8189	}
8190
8191	/* Forward #NM to the guest. */
8192	Assert(rc == VINF_EM_RAW_GUEST_TRAP);
8193	rc = hmR0VmxReadExitIntrInfoVmcs(pVmxTransient);
8194	AssertRCReturn(rc, rc);
8195	rc = hmR0VmxInjectEventVmcs(pVM, pVCpu, pMixedCtx,
8196	VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8197	pVmxTransient->cbInstr, 0 /* error code */);
8198	AssertRCReturn(rc, rc);
8199	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
8200	return rc;
8201	}
8202
8203
8204	/**
8205	* VM-exit exception handler for #GP (General-protection exception).
8206	*
8207	* @remarks Requires pVmxTransient->uExitIntrInfo to be up-to-date.
8208	*/
8209	static DECLCALLBACK(int) hmR0VmxExitXcptGP(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8210	{
8211	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8212
8213	int rc = VERR_INTERNAL_ERROR_5;
8214
8215	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
8216	{
8217	#ifdef VBOX_ALWAYS_TRAP_ALL_EXCEPTIONS
8218	/* If the guest is not in real-mode or we have unrestricted execution support, reflect #GP to the guest. */
8219	rc = hmR0VmxReadExitIntrInfoVmcs(pVmxTransient);
8220	rc \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVmxTransient);
8221	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
8222	rc \|= hmR0VmxInjectEventVmcs(pVM, pVCpu, pMixedCtx,
8223	VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8224	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode);
8225	AssertRCReturn(rc, rc);
8226	return rc;
8227	#else
8228	/* We don't intercept #GP. */
8229	AssertMsgFailed(("Unexpected VM-exit caused by #GP exception\n"));
8230	return VERR_VMX_UNEXPECTED_EXCEPTION;
8231	#endif
8232	}
8233
8234	Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
8235	Assert(!pVM->hm.s.vmx.fUnrestrictedGuest);
8236
8237	/* EMInterpretDisasCurrent() requires a lot of the state, save the entire state. */
8238	rc = hmR0VmxSaveGuestState(pVM, pVCpu, pMixedCtx);
8239	AssertRCReturn(rc, rc);
8240
8241	PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
8242	unsigned int cbOp = 0;
8243	rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbOp);
8244	if (RT_SUCCESS(rc))
8245	{
8246	rc = VINF_SUCCESS;
8247	Assert(cbOp == pDis->cbInstr);
8248	Log2(("#GP Disas OpCode=%u CS:EIP %04x:%#RX64\n", pDis->pCurInstr->uOpcode, pMixedCtx->cs.Sel, pMixedCtx->rip));
8249	switch (pDis->pCurInstr->uOpcode)
8250	{
8251	case OP_CLI:
8252	pMixedCtx->eflags.Bits.u1IF = 0;
8253	pMixedCtx->rip += pDis->cbInstr;
8254	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS;
8255	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCli);
8256	break;
8257
8258	case OP_STI:
8259	pMixedCtx->eflags.Bits.u1IF = 1;
8260	pMixedCtx->rip += pDis->cbInstr;
8261	EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
8262	Assert(VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
8263	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_INTR_STATE \| HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS;
8264	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitSti);
8265	break;
8266
8267	case OP_HLT:
8268	rc = VINF_EM_HALT;
8269	pMixedCtx->rip += pDis->cbInstr;
8270	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
8271	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
8272	break;
8273
8274	case OP_POPF:
8275	{
8276	Log(("POPF CS:RIP %04x:%#RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
8277	size_t cbParm = 0;
8278	uint32_t uMask = 0;
8279	if (pDis->fPrefix & DISPREFIX_OPSIZE)
8280	{
8281	cbParm = 4;
8282	uMask = 0xffffffff;
8283	}
8284	else
8285	{
8286	cbParm = 2;
8287	uMask = 0xffff;
8288	}
8289
8290	/* Get the stack pointer & pop the contents of the stack onto EFlags. */
8291	RTGCPTR GCPtrStack = 0;
8292	X86EFLAGS uEflags;
8293	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
8294	&GCPtrStack);
8295	if (RT_SUCCESS(rc))
8296	{
8297	Assert(sizeof(uEflags.u32) >= cbParm);
8298	uEflags.u32 = 0;
8299	rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &uEflags.u32, cbParm);
8300	}
8301	if (RT_FAILURE(rc))
8302	{
8303	rc = VERR_EM_INTERPRETER;
8304	break;
8305	}
8306	Log(("POPF %x -> %RGv mask=%x RIP=%#RX64\n", uEflags.u, pMixedCtx->rsp, uMask, pMixedCtx->rip));
8307	pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~(X86_EFL_POPF_BITS & uMask))
8308	\| (uEflags.u32 & X86_EFL_POPF_BITS & uMask);
8309	/* The RF bit is always cleared by POPF; see Intel Instruction reference for POPF. */
8310	pMixedCtx->eflags.Bits.u1RF = 0;
8311	pMixedCtx->esp += cbParm;
8312	pMixedCtx->esp &= uMask;
8313	pMixedCtx->rip += pDis->cbInstr;
8314	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS;
8315	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPopf);
8316	break;
8317	}
8318
8319	case OP_PUSHF:
8320	{
8321	size_t cbParm = 0;
8322	uint32_t uMask = 0;
8323	if (pDis->fPrefix & DISPREFIX_OPSIZE)
8324	{
8325	cbParm = 4;
8326	uMask = 0xffffffff;
8327	}
8328	else
8329	{
8330	cbParm = 2;
8331	uMask = 0xffff;
8332	}
8333
8334	/* Get the stack pointer & push the contents of eflags onto the stack. */
8335	RTGCPTR GCPtrStack = 0;
8336	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), (pMixedCtx->esp - cbParm) & uMask,
8337	SELMTOFLAT_FLAGS_CPL0, &GCPtrStack);
8338	if (RT_FAILURE(rc))
8339	{
8340	rc = VERR_EM_INTERPRETER;
8341	break;
8342	}
8343	X86EFLAGS uEflags;
8344	uEflags = pMixedCtx->eflags;
8345	/* The RF & VM bits are cleared on image stored on stack; see Intel Instruction reference for PUSHF. */
8346	uEflags.Bits.u1RF = 0;
8347	uEflags.Bits.u1VM = 0;
8348
8349	rc = PGMPhysWrite(pVM, (RTGCPHYS)GCPtrStack, &uEflags.u, cbParm);
8350	if (RT_FAILURE(rc))
8351	{
8352	rc = VERR_EM_INTERPRETER;
8353	break;
8354	}
8355	Log(("PUSHF %x -> %RGv\n", uEflags.u, GCPtrStack));
8356	pMixedCtx->esp -= cbParm;
8357	pMixedCtx->esp &= uMask;
8358	pMixedCtx->rip += pDis->cbInstr;
8359	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP;
8360	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPushf);
8361	break;
8362	}
8363
8364	case OP_IRET:
8365	{
8366	/** @todo Handle 32-bit operand sizes and check stack limits. See Intel
8367	* instruction reference. */
8368	RTGCPTR GCPtrStack = 0;
8369	uint32_t uMask = 0xffff;
8370	uint16_t aIretFrame[3];
8371	if (pDis->fPrefix & (DISPREFIX_OPSIZE \| DISPREFIX_ADDRSIZE))
8372	{
8373	rc = VERR_EM_INTERPRETER;
8374	break;
8375	}
8376	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
8377	&GCPtrStack);
8378	if (RT_SUCCESS(rc))
8379	rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &aIretFrame[0], sizeof(aIretFrame));
8380	if (RT_FAILURE(rc))
8381	{
8382	rc = VERR_EM_INTERPRETER;
8383	break;
8384	}
8385	pMixedCtx->eip = 0;
8386	pMixedCtx->ip = aIretFrame[0];
8387	pMixedCtx->cs.Sel = aIretFrame[1];
8388	pMixedCtx->cs.ValidSel = aIretFrame[1];
8389	pMixedCtx->cs.u64Base = (uint64_t)pMixedCtx->cs.Sel << 4;
8390	pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~(X86_EFL_POPF_BITS & uMask))
8391	\| (aIretFrame[2] & X86_EFL_POPF_BITS & uMask);
8392	pMixedCtx->sp += sizeof(aIretFrame);
8393	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_SEGMENT_REGS \| HM_CHANGED_GUEST_RSP
8394	\| HM_CHANGED_GUEST_RFLAGS;
8395	Log(("IRET %#RX32 to %04x:%x\n", GCPtrStack, pMixedCtx->cs.Sel, pMixedCtx->ip));
8396	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIret);
8397	break;
8398	}
8399
8400	case OP_INT:
8401	{
8402	uint16_t uVector = pDis->Param1.uValue & 0xff;
8403	rc = hmR0VmxInjectIntN(pVM, pVCpu, pMixedCtx, uVector, pDis->cbInstr);
8404	AssertRCReturn(rc, rc);
8405	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
8406	break;
8407	}
8408
8409	case OP_INTO:
8410	{
8411	if (pMixedCtx->eflags.Bits.u1OF)
8412	{
8413	rc = hmR0VmxInjectXcptOF(pVM, pVCpu, pMixedCtx, pDis->cbInstr);
8414	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
8415	}
8416	break;
8417	}
8418
8419	default:
8420	{
8421	VBOXSTRICTRC rc2 = EMInterpretInstructionDisasState(pVCpu, pDis, CPUMCTX2CORE(pMixedCtx), 0 /* pvFault */,
8422	EMCODETYPE_SUPERVISOR);
8423	rc = VBOXSTRICTRC_VAL(rc2);
8424	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_ALL_GUEST;
8425	Log2(("#GP rc=%Rrc\n", rc));
8426	break;
8427	}
8428	}
8429	}
8430	else
8431	rc = VERR_EM_INTERPRETER;
8432
8433	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_EM_HALT
8434	\|\| rc == VINF_EM_RESET /* injection caused triple fault */,
8435	("#GP Unexpected rc=%Rrc\n", rc));
8436	return rc;
8437	}
8438
8439
8440	/**
8441	* VM-exit exception handler wrapper for generic exceptions. Simply re-injects
8442	* the exception reported in the VMX transient structure back into the VM.
8443	*
8444	* @remarks Requires uExitIntrInfo, uExitIntrErrorCode, cbInstr fields in the
8445	* VMX transient structure to be up-to-date.
8446	*/
8447	static DECLCALLBACK(int) hmR0VmxExitXcptGeneric(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8448	{
8449	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8450
8451	/* Re-inject the exception into the guest. This cannot be a double-fault condition which are handled in
8452	hmR0VmxSavePendingEventDueToEventDelivery(). */
8453	int rc = hmR0VmxInjectEventVmcs(pVM, pVCpu, pMixedCtx,
8454	VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8455	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode);
8456	AssertRCReturn(rc, rc);
8457	return rc;
8458	}
8459
8460
8461	/**
8462	* VM-exit exception handler for #PF (Page-fault exception).
8463	*/
8464	static DECLCALLBACK(int) hmR0VmxExitXcptPF(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8465	{
8466	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8467
8468	int rc = hmR0VmxReadExitQualificationVmcs(pVmxTransient);
8469	rc \|= hmR0VmxReadExitIntrInfoVmcs(pVmxTransient);
8470	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
8471	rc \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVmxTransient);
8472	AssertRCReturn(rc, rc);
8473
8474	#if defined(VBOX_ALWAYS_TRAP_ALL_EXCEPTIONS) \|\| defined(VBOX_ALWAYS_TRAP_PF)
8475	if (pVM->hm.s.fNestedPaging)
8476	{
8477	if (RT_LIKELY(!pVmxTransient->fVectoringPF))
8478	{
8479	pMixedCtx->cr2 = pVmxTransient->uExitQualification;
8480	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR2;
8481	rc = hmR0VmxInjectEventVmcs(pVM, pVCpu, pMixedCtx,
8482	VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8483	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode);
8484	AssertRCReturn(rc, rc);
8485	}
8486	else
8487	{
8488	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
8489	Assert(!pVCpu->hm.s.Event.fPending);
8490	rc = hmR0VmxInjectXcptDF(pVM, pVCpu, pMixedCtx);
8491	}
8492	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
8493	return rc;
8494	}
8495	#else
8496	Assert(!pVM->hm.s.fNestedPaging);
8497	#endif
8498
8499	#ifdef VBOX_HM_WITH_GUEST_PATCHING
8500	rc = hmR0VmxSaveGuestControlRegs(pVM, pVCpu, pMixedCtx);
8501	rc \|= hmR0VmxSaveGuestSegmentRegs(pVM, pVCpu, pMixedCtx);
8502	rc \|= hmR0VmxSaveGuestRflags(pVM, pVCpu, pMixedCtx);
8503	AssertRCReturn(rc, rc);
8504	/* Shortcut for APIC TPR access, only for 32-bit guests. */
8505	if ( pVM->hm.s.fTRPPatchingAllowed
8506	&& pVM->hm.s.pGuestPatchMem
8507	&& (pVmxTransient->uExitQualification & 0xfff) == 0x80 /* TPR offset */
8508	&& !(pVmxTransient->uExitIntrErrorCode & X86_TRAP_PF_P) /* Page not present */
8509	&& CPUMGetGuestCPL(pVCpu) == 0 /* Requires CR0, EFLAGS, segments. */
8510	&& !CPUMIsGuestInLongModeEx(pMixedCtx) /* Requires EFER. */
8511	&& pVM->hm.s.cPatches < RT_ELEMENTS(pVM->hm.s.aPatches))
8512	{
8513	RTGCPHYS GCPhys;
8514	RTGCPHYS GCPhysApicBase = (pMixedCtx->msrApicBase & PAGE_BASE_GC_MASK);
8515	rc = PGMGstGetPage(pVCpu, (RTGCPTR)pVmxTransient->uExitQualification, NULL /* pfFlags */, &GCPhys);
8516	if ( rc == VINF_SUCCESS
8517	&& GCPhys == GCPhysApicBase)
8518	{
8519	rc = hmR0VmxSaveGuestRip(pVM, pVCpu, pMixedCtx);
8520	AssertRCReturn(rc, rc);
8521
8522	/* Only attempt to patch the instruction once. */
8523	PHMTPRPATCH pPatch = (PHMTPRPATCH)RTAvloU32Get(&pVM->hm.s.PatchTree, (AVLOU32KEY)pMixedCtx->eip);
8524	if (!pPatch)
8525	return VINF_EM_HM_PATCH_TPR_INSTR;
8526	}
8527	}
8528	#endif
8529
8530	TRPMAssertTrap(pVCpu, X86_XCPT_PF, TRPM_TRAP);
8531	TRPMSetFaultAddress(pVCpu, pVmxTransient->uExitQualification);
8532	TRPMSetErrorCode(pVCpu, pVmxTransient->uExitIntrErrorCode);
8533
8534	rc = hmR0VmxSaveGuestState(pVM, pVCpu, pMixedCtx);
8535	AssertRCReturn(rc, rc);
8536
8537	/* Forward it to the trap handler first. */
8538	rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntrErrorCode, CPUMCTX2CORE(pMixedCtx),
8539	(RTGCPTR)pVmxTransient->uExitQualification);
8540
8541	Log(("#PF: cr2=%RGv cs:rip=%04x:%RGv errorcode %#RX32 rc=%d\n", pVmxTransient->uExitQualification, pMixedCtx->cs.Sel,
8542	pMixedCtx->rip, pVmxTransient->uExitIntrErrorCode, rc));
8543
8544	if (rc == VINF_SUCCESS)
8545	{
8546	/* Successfully synced shadow pages tables or emulated an MMIO instruction. */
8547	/** @todo this isn't quite right, what if guest does lgdt with some MMIO
8548	* memory? We don't update the whole state here... */
8549	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
8550	\| HM_CHANGED_VMX_GUEST_APIC_STATE;
8551
8552	TRPMResetTrap(pVCpu);
8553	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
8554	return rc;
8555	}
8556	else if (rc == VINF_EM_RAW_GUEST_TRAP)
8557	{
8558	if (RT_LIKELY(!pVmxTransient->fVectoringPF))
8559	{
8560	/* It's a guest page fault and needs to be reflected to the guest. */
8561	uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
8562	TRPMResetTrap(pVCpu);
8563	pMixedCtx->cr2 = pVmxTransient->uExitQualification;
8564	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR2;
8565	rc = hmR0VmxInjectEventVmcs(pVM, pVCpu, pMixedCtx,
8566	VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8567	pVmxTransient->cbInstr, uGstErrorCode);
8568	AssertRCReturn(rc, rc);
8569	}
8570	else
8571	{
8572	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
8573	Assert(!pVCpu->hm.s.Event.fPending);
8574	TRPMResetTrap(pVCpu);
8575	rc = hmR0VmxInjectXcptDF(pVM, pVCpu, pMixedCtx);
8576	}
8577	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
8578	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RESET);
8579	return rc;
8580	}
8581
8582	TRPMResetTrap(pVCpu);
8583	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
8584	return rc;
8585	}
8586

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 45344

Download in other formats: