HMVMXR0.cpp@ 45543

Last change on this file since 45543 was 45543, checked in by vboxsync, 12 years ago
VMM/VMMR0: HM bits.
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 373.2 KB

Line
1	/* $Id: HMVMXR0.cpp 45543 2013-04-14 16:19:03Z vboxsync $ */
2	/** @file
3	* HM VMX (Intel VT-x) - Host Context Ring-0.
4	*/
5
6	/*
7	* Copyright (C) 2012-2013 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18	/*******************************************************************************
19	* Header Files *
20	*******************************************************************************/
21	#define LOG_GROUP LOG_GROUP_HM
22	#include <iprt/asm-amd64-x86.h>
23	#include <iprt/thread.h>
24	#include <iprt/string.h>
25
26	#include "HMInternal.h"
27	#include <VBox/vmm/vm.h>
28	#include "HWVMXR0.h"
29	#include <VBox/vmm/pdmapi.h>
30	#include <VBox/vmm/dbgf.h>
31	#include <VBox/vmm/iom.h>
32	#include <VBox/vmm/selm.h>
33	#include <VBox/vmm/tm.h>
34	#ifdef VBOX_WITH_REM
35	# include <VBox/vmm/rem.h>
36	#endif
37	#ifdef DEBUG_ramshankar
38	#define VBOX_ALWAYS_TRAP_ALL_EXCEPTIONS
39	#endif
40
41	/*******************************************************************************
42	* Defined Constants And Macros *
43	*******************************************************************************/
44	#if defined(RT_ARCH_AMD64)
45	# define VMX_IS_64BIT_HOST_MODE() (true)
46	#elif defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
47	# define VMX_IS_64BIT_HOST_MODE() (g_fVMXIs64bitHost != 0)
48	#else
49	# define VMX_IS_64BIT_HOST_MODE() (false)
50	#endif
51
52	#define VMX_SEL_UNUSABLE RT_BIT(16)
53	#define VMX_FLUSH_TAGGED_TLB_EPT_VPID 0
54	#define VMX_FLUSH_TAGGED_TLB_EPT 1
55	#define VMX_FLUSH_TAGGED_TLB_VPID 2
56	#define VMX_FLUSH_TAGGED_TLB_NONE 3
57
58	/**
59	* Updated-guest-state flags.
60	*/
61	#define VMX_UPDATED_GUEST_FPU RT_BIT(0)
62	#define VMX_UPDATED_GUEST_RIP RT_BIT(1)
63	#define VMX_UPDATED_GUEST_RSP RT_BIT(2)
64	#define VMX_UPDATED_GUEST_RFLAGS RT_BIT(3)
65	#define VMX_UPDATED_GUEST_CR0 RT_BIT(4)
66	#define VMX_UPDATED_GUEST_CR3 RT_BIT(5)
67	#define VMX_UPDATED_GUEST_CR4 RT_BIT(6)
68	#define VMX_UPDATED_GUEST_GDTR RT_BIT(7)
69	#define VMX_UPDATED_GUEST_IDTR RT_BIT(8)
70	#define VMX_UPDATED_GUEST_LDTR RT_BIT(9)
71	#define VMX_UPDATED_GUEST_TR RT_BIT(10)
72	#define VMX_UPDATED_GUEST_SEGMENT_REGS RT_BIT(11)
73	#define VMX_UPDATED_GUEST_DEBUG RT_BIT(12)
74	#define VMX_UPDATED_GUEST_FS_BASE_MSR RT_BIT(13)
75	#define VMX_UPDATED_GUEST_GS_BASE_MSR RT_BIT(14)
76	#define VMX_UPDATED_GUEST_SYSENTER_CS_MSR RT_BIT(15)
77	#define VMX_UPDATED_GUEST_SYSENTER_EIP_MSR RT_BIT(16)
78	#define VMX_UPDATED_GUEST_SYSENTER_ESP_MSR RT_BIT(17)
79	#define VMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS RT_BIT(18)
80	#define VMX_UPDATED_GUEST_ACTIVITY_STATE RT_BIT(19)
81	#define VMX_UPDATED_GUEST_APIC_STATE RT_BIT(20)
82	#define VMX_UPDATED_GUEST_ALL ( VMX_UPDATED_GUEST_FPU \
83	\| VMX_UPDATED_GUEST_RIP \
84	\| VMX_UPDATED_GUEST_RSP \
85	\| VMX_UPDATED_GUEST_RFLAGS \
86	\| VMX_UPDATED_GUEST_CR0 \
87	\| VMX_UPDATED_GUEST_CR3 \
88	\| VMX_UPDATED_GUEST_CR4 \
89	\| VMX_UPDATED_GUEST_GDTR \
90	\| VMX_UPDATED_GUEST_IDTR \
91	\| VMX_UPDATED_GUEST_LDTR \
92	\| VMX_UPDATED_GUEST_TR \
93	\| VMX_UPDATED_GUEST_SEGMENT_REGS \
94	\| VMX_UPDATED_GUEST_DEBUG \
95	\| VMX_UPDATED_GUEST_FS_BASE_MSR \
96	\| VMX_UPDATED_GUEST_GS_BASE_MSR \
97	\| VMX_UPDATED_GUEST_SYSENTER_CS_MSR \
98	\| VMX_UPDATED_GUEST_SYSENTER_EIP_MSR \
99	\| VMX_UPDATED_GUEST_SYSENTER_ESP_MSR \
100	\| VMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS \
101	\| VMX_UPDATED_GUEST_ACTIVITY_STATE \
102	\| VMX_UPDATED_GUEST_APIC_STATE)
103
104	/**
105	* Flags to skip redundant reads of some common VMCS fields.
106	*/
107	#define VMX_TRANSIENT_IDT_VECTORING_INFO RT_BIT(0)
108	#define VMX_TRANSIENT_IDT_VECTORING_ERROR_CODE RT_BIT(1)
109	#define VMX_TRANSIENT_EXIT_QUALIFICATION RT_BIT(2)
110	#define VMX_TRANSIENT_EXIT_INSTR_LEN RT_BIT(3)
111	#define VMX_TRANSIENT_EXIT_INTERRUPTION_INFO RT_BIT(4)
112	#define VMX_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE RT_BIT(5)
113
114	/**
115	* Exception bitmap mask for real-mode guests (real-on-v86). We need to intercept all exceptions manually (except #PF).
116	* #NM is also handled spearetely, see hmR0VmxLoadGuestControlRegs(). #PF need not be intercepted even in real-mode if
117	* we have Nested Paging support.
118	*/
119	#define VMX_REAL_MODE_XCPT_BITMAP ( RT_BIT(X86_XCPT_DE) \| RT_BIT(X86_XCPT_DB) \| RT_BIT(X86_XCPT_NMI) \
120	\| RT_BIT(X86_XCPT_BP) \| RT_BIT(X86_XCPT_OF) \| RT_BIT(X86_XCPT_BR) \
121	\| RT_BIT(X86_XCPT_UD) /* RT_BIT(X86_XCPT_NM) */ \| RT_BIT(X86_XCPT_DF) \
122	\| RT_BIT(X86_XCPT_CO_SEG_OVERRUN) \| RT_BIT(X86_XCPT_TS) \| RT_BIT(X86_XCPT_NP) \
123	\| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
124	\| RT_BIT(X86_XCPT_MF) \| RT_BIT(X86_XCPT_AC) \| RT_BIT(X86_XCPT_MC) \
125	\| RT_BIT(X86_XCPT_XF))
126
127	/**
128	* Exception bitmap mask for all contributory exceptions.
129	*/
130	#define VMX_CONTRIBUTORY_XCPT_BITMAP ( RT_BIT(X86_XCPT_GP) \| RT_BIT(X86_XCPT_NP) \| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_TS) \
131	\| RT_BIT(X86_XCPT_DE))
132
133	/** Maximum VM-instruction error number. */
134	#define VMX_INSTR_ERROR_MAX 28
135
136	/*******************************************************************************
137	* Structures and Typedefs *
138	*******************************************************************************/
139	/**
140	* A state structure for holding miscellaneous information across
141	* VMX non-root operation and restored after the transition.
142	*/
143	typedef struct VMXTRANSIENT
144	{
145	/** The host's rflags/eflags. */
146	RTCCUINTREG uEFlags;
147	/** The guest's LSTAR MSR value used for TPR patching for 32-bit guests. */
148	uint64_t u64LStarMsr;
149	/** The guest's TPR value used for TPR shadowing. */
150	uint8_t u8GuestTpr;
151
152	/** The basic VM-exit reason. */
153	uint16_t uExitReason;
154	/** The VM-exit exit qualification. */
155	RTGCUINTPTR uExitQualification;
156	/** The VM-exit interruption error code. */
157	uint32_t uExitIntrErrorCode;
158
159	/** The VM-exit interruption-information field. */
160	uint32_t uExitIntrInfo;
161	/** Whether the VM-entry failed or not. */
162	bool fVMEntryFailed;
163	/** The VM-exit instruction-length field. */
164	uint32_t cbInstr;
165
166	/** The VM-entry interruption-information field. */
167	uint32_t uEntryIntrInfo;
168	/** The VM-entry exception error code field. */
169	uint32_t uEntryXcptErrorCode;
170	/** The VM-entry instruction length field. */
171	uint32_t cbEntryInstr;
172
173	/** IDT-vectoring information field. */
174	uint32_t uIdtVectoringInfo;
175	/** IDT-vectoring error code. */
176	uint32_t uIdtVectoringErrorCode;
177
178	/** Mask of currently read VMCS fields; VMX_TRANSIENT_. /
179	uint32_t fVmcsFieldsRead;
180	/** Whether TSC-offsetting should be setup before VM-entry. */
181	bool fUpdateTscOffsettingAndPreemptTimer;
182	/** Whether the VM-exit was caused by a page-fault during delivery of a
183	* contributary exception or a page-fault. */
184	bool fVectoringPF;
185	} VMXTRANSIENT, *PVMXTRANSIENT;
186
187	/**
188	* MSR-bitmap read permissions.
189	*/
190	typedef enum VMXMSREXITREAD
191	{
192	/** Reading this MSR causes a VM-exit. */
193	VMXMSREXIT_INTERCEPT_READ = 0xb,
194	/** Reading this MSR does not cause a VM-exit. */
195	VMXMSREXIT_PASSTHRU_READ
196	} VMXMSREXITREAD;
197
198	/**
199	* MSR-bitmap write permissions.
200	*/
201	typedef enum VMXMSREXITWRITE
202	{
203	/** Writing to this MSR causes a VM-exit. */
204	VMXMSREXIT_INTERCEPT_WRITE = 0xd,
205	/** Writing to this MSR does not cause a VM-exit. */
206	VMXMSREXIT_PASSTHRU_WRITE
207	} VMXMSREXITWRITE;
208
209	/*******************************************************************************
210	* Internal Functions *
211	*******************************************************************************/
212	static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_VPID enmFlush, RTGCPTR GCPtr);
213	static int hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntrInfo,
214	uint32_t cbInstr, uint32_t u32ErrCode, uint32_t *puIntrState);
215	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
216	static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu);
217	#endif
218	#if 1
219	DECLINLINE(int) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason);
220	#endif
221
222	static DECLCALLBACK(int) hmR0VmxExitXcptNmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
223	static DECLCALLBACK(int) hmR0VmxExitExtInt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
224	static DECLCALLBACK(int) hmR0VmxExitTripleFault(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
225	static DECLCALLBACK(int) hmR0VmxExitInitSignal(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
226	static DECLCALLBACK(int) hmR0VmxExitSipi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
227	static DECLCALLBACK(int) hmR0VmxExitIoSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
228	static DECLCALLBACK(int) hmR0VmxExitSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
229	static DECLCALLBACK(int) hmR0VmxExitIntWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
230	static DECLCALLBACK(int) hmR0VmxExitNmiWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
231	static DECLCALLBACK(int) hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
232	static DECLCALLBACK(int) hmR0VmxExitCpuid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
233	static DECLCALLBACK(int) hmR0VmxExitGetsec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
234	static DECLCALLBACK(int) hmR0VmxExitHlt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
235	static DECLCALLBACK(int) hmR0VmxExitInvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
236	static DECLCALLBACK(int) hmR0VmxExitInvlpg(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
237	static DECLCALLBACK(int) hmR0VmxExitRdpmc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
238	static DECLCALLBACK(int) hmR0VmxExitRdtsc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
239	static DECLCALLBACK(int) hmR0VmxExitRsm(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
240	static DECLCALLBACK(int) hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
241	static DECLCALLBACK(int) hmR0VmxExitMovCRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
242	static DECLCALLBACK(int) hmR0VmxExitMovDRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
243	static DECLCALLBACK(int) hmR0VmxExitIoInstr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
244	static DECLCALLBACK(int) hmR0VmxExitRdmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
245	static DECLCALLBACK(int) hmR0VmxExitWrmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
246	static DECLCALLBACK(int) hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
247	static DECLCALLBACK(int) hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
248	static DECLCALLBACK(int) hmR0VmxExitErrUndefined(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
249	static DECLCALLBACK(int) hmR0VmxExitMwait(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
250	static DECLCALLBACK(int) hmR0VmxExitMtf(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
251	static DECLCALLBACK(int) hmR0VmxExitMonitor(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
252	static DECLCALLBACK(int) hmR0VmxExitPause(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
253	static DECLCALLBACK(int) hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
254	static DECLCALLBACK(int) hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
255	static DECLCALLBACK(int) hmR0VmxExitApicAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
256	static DECLCALLBACK(int) hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
257	static DECLCALLBACK(int) hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
258	static DECLCALLBACK(int) hmR0VmxExitEptViolation(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
259	static DECLCALLBACK(int) hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
260	static DECLCALLBACK(int) hmR0VmxExitRdtscp(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
261	static DECLCALLBACK(int) hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
262	static DECLCALLBACK(int) hmR0VmxExitWbinvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
263	static DECLCALLBACK(int) hmR0VmxExitXsetbv(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
264	static DECLCALLBACK(int) hmR0VmxExitRdrand(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
265	static DECLCALLBACK(int) hmR0VmxExitInvpcid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
266	static DECLCALLBACK(int) hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
267	static DECLCALLBACK(int) hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
268	static DECLCALLBACK(int) hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
269	static DECLCALLBACK(int) hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
270	static DECLCALLBACK(int) hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
271	static DECLCALLBACK(int) hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
272	static DECLCALLBACK(int) hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
273
274	/*******************************************************************************
275	* Global Variables *
276	*******************************************************************************/
277	/**
278	* VM-exit handler.
279	*
280	* @returns VBox status code.
281	* @param pVCpu Pointer to the VMCPU.
282	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
283	* out-of-sync. Make sure to update the required
284	* fields before using them.
285	* @param pVmxTransient Pointer to the VMX-transient structure.
286	*/
287	typedef DECLCALLBACK(int) FNVMEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
288	/** Pointer to VM-exit handler. */
289	typedef FNVMEXITHANDLER *const PFNVMEXITHANDLER;
290
291	/**
292	* VMX_EXIT dispatch table.
293	*/
294	static const PFNVMEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
295	{
296	/* 00 VMX_EXIT_XCPT_NMI */ hmR0VmxExitXcptNmi,
297	/* 01 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
298	/* 02 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
299	/* 03 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitInitSignal,
300	/* 04 VMX_EXIT_SIPI */ hmR0VmxExitSipi,
301	/* 05 VMX_EXIT_IO_SMI */ hmR0VmxExitIoSmi,
302	/* 06 VMX_EXIT_SMI */ hmR0VmxExitSmi,
303	/* 07 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
304	/* 08 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
305	/* 09 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
306	/* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
307	/* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
308	/* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
309	/* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
310	/* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
311	/* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
312	/* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
313	/* 17 VMX_EXIT_RSM */ hmR0VmxExitRsm,
314	/* 18 VMX_EXIT_VMCALL */ hmR0VmxExitSetPendingXcptUD,
315	/* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
316	/* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
317	/* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
318	/* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
319	/* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
320	/* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
321	/* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
322	/* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
323	/* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
324	/* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
325	/* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
326	/* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
327	/* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
328	/* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
329	/* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
330	/* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrMsrLoad,
331	/* 35 UNDEFINED */ hmR0VmxExitErrUndefined,
332	/* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
333	/* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
334	/* 38 UNDEFINED */ hmR0VmxExitErrUndefined,
335	/* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
336	/* 40 UNDEFINED */ hmR0VmxExitPause,
337	/* 41 VMX_EXIT_PAUSE */ hmR0VmxExitErrMachineCheck,
338	/* 42 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUndefined,
339	/* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
340	/* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
341	/* 45 UNDEFINED */ hmR0VmxExitErrUndefined,
342	/* 46 VMX_EXIT_XDTR_ACCESS */ hmR0VmxExitXdtrAccess,
343	/* 47 VMX_EXIT_TR_ACCESS */ hmR0VmxExitXdtrAccess,
344	/* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
345	/* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
346	/* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
347	/* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
348	/* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
349	/* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
350	/* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
351	/* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
352	/* 56 UNDEFINED */ hmR0VmxExitErrUndefined,
353	/* 57 VMX_EXIT_RDRAND */ hmR0VmxExitRdrand,
354	/* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
355	/* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitSetPendingXcptUD
356	};
357
358	#ifdef VBOX_STRICT
359	static const char* const g_apszVmxInstrErrors[VMX_INSTR_ERROR_MAX + 1] =
360	{
361	/* 0 */ "(Not Used)",
362	/* 1 */ "VMCALL executed in VMX root operation.",
363	/* 2 */ "VMCLEAR with invalid physical address.",
364	/* 3 */ "VMCLEAR with VMXON pointer.",
365	/* 4 */ "VMLAUNCH with non-clear VMCS.",
366	/* 5 */ "VMRESUME with non-launched VMCS.",
367	/* 6 */ "VMRESUME after VMXOFF",
368	/* 7 */ "VM entry with invalid control fields.",
369	/* 8 */ "VM entry with invalid host state fields.",
370	/* 9 */ "VMPTRLD with invalid physical address.",
371	/* 10 */ "VMPTRLD with VMXON pointer.",
372	/* 11 */ "VMPTRLD with incorrect revision identifier.",
373	/* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
374	/* 13 */ "VMWRITE to read-only VMCS component.",
375	/* 14 */ "(Not Used)",
376	/* 15 */ "VMXON executed in VMX root operation.",
377	/* 16 */ "VM entry with invalid executive-VMCS pointer.",
378	/* 17 */ "VM entry with non-launched executing VMCS.",
379	/* 18 */ "VM entry with executive-VMCS pointer not VMXON pointer.",
380	/* 19 */ "VMCALL with non-clear VMCS.",
381	/* 20 */ "VMCALL with invalid VM-exit control fields.",
382	/* 21 */ "(Not Used)",
383	/* 22 */ "VMCALL with incorrect MSEG revision identifier.",
384	/* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
385	/* 24 */ "VMCALL with invalid SMM-monitor features.",
386	/* 25 */ "VM entry with invalid VM-execution control fields in executive VMCS.",
387	/* 26 */ "VM entry with events blocked by MOV SS.",
388	/* 27 */ "(Not Used)",
389	/* 28 */ "Invalid operand to INVEPT/INVVPID."
390	};
391	#endif
392
393	/**
394	* Updates the VM's last error record. If there was a VMX instruction error,
395	* reads the error data from the VMCS and updates VCPU's last error record as
396	* well.
397	*
398	* @param pVM Pointer to the VM.
399	* @param pVCpu Pointer to the VMCPU (can be NULL if @a rc is not
400	* VERR_VMX_UNABLE_TO_START_VM or
401	* VERR_VMX_INVALID_VMCS_FIELD).
402	* @param rc The error code.
403	*/
404	static void hmR0VmxUpdateErrorRecord(PVM pVM, PVMCPU pVCpu, int rc)
405	{
406	AssertPtr(pVM);
407	if ( rc == VERR_VMX_INVALID_VMCS_FIELD
408	\|\| rc == VERR_VMX_UNABLE_TO_START_VM)
409	{
410	AssertPtrReturnVoid(pVCpu);
411	VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.lasterror.u32InstrError);
412	}
413	pVM->hm.s.lLastError = rc;
414	}
415
416
417	/**
418	* Reads the VM-entry interruption-information field from the VMCS into the VMX
419	* transient structure.
420	*
421	* @returns VBox status code.
422	* @param pVmxTransient Pointer to the VMX transient structure.
423	*
424	* @remarks No-long-jump zone!!!
425	*/
426	DECLINLINE(int) hmR0VmxReadEntryIntrInfoVmcs(PVMXTRANSIENT pVmxTransient)
427	{
428	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntrInfo);
429	AssertRCReturn(rc, rc);
430	return VINF_SUCCESS;
431	}
432
433
434	/**
435	* Reads the VM-entry exception error code field from the VMCS into
436	* the VMX transient structure.
437	*
438	* @returns VBox status code.
439	* @param pVmxTransient Pointer to the VMX transient structure.
440	*
441	* @remarks No-long-jump zone!!!
442	*/
443	DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
444	{
445	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
446	AssertRCReturn(rc, rc);
447	return VINF_SUCCESS;
448	}
449
450
451	/**
452	* Reads the VM-entry exception error code field from the VMCS into
453	* the VMX transient structure.
454	*
455	* @returns VBox status code.
456	* @param pVCpu Pointer to the VMCPU.
457	* @param pVmxTransient Pointer to the VMX transient structure.
458	*
459	* @remarks No-long-jump zone!!!
460	*/
461	DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
462	{
463	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
464	AssertRCReturn(rc, rc);
465	return VINF_SUCCESS;
466	}
467
468
469	/**
470	* Reads the VM-exit interruption-information field from the VMCS into the VMX
471	* transient structure.
472	*
473	* @returns VBox status code.
474	* @param pVCpu Pointer to the VMCPU.
475	* @param pVmxTransient Pointer to the VMX transient structure.
476	*/
477	DECLINLINE(int) hmR0VmxReadExitIntrInfoVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
478	{
479	if (!(pVmxTransient->fVmcsFieldsRead & VMX_TRANSIENT_EXIT_INTERRUPTION_INFO))
480	{
481	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntrInfo);
482	AssertRCReturn(rc, rc);
483	pVmxTransient->fVmcsFieldsRead \|= VMX_TRANSIENT_EXIT_INTERRUPTION_INFO;
484	}
485	return VINF_SUCCESS;
486	}
487
488
489	/**
490	* Reads the VM-exit interruption error code from the VMCS into the VMX
491	* transient structure.
492	*
493	* @returns VBox status code.
494	* @param pVCpu Pointer to the VMCPU.
495	* @param pVmxTransient Pointer to the VMX transient structure.
496	*/
497	DECLINLINE(int) hmR0VmxReadExitIntrErrorCodeVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
498	{
499	if (!(pVmxTransient->fVmcsFieldsRead & VMX_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE))
500	{
501	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntrErrorCode);
502	AssertRCReturn(rc, rc);
503	pVmxTransient->fVmcsFieldsRead \|= VMX_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE;
504	}
505	return VINF_SUCCESS;
506	}
507
508
509	/**
510	* Reads the VM-exit instruction length field from the VMCS into the VMX
511	* transient structure.
512	*
513	* @returns VBox status code.
514	* @param pVCpu Pointer to the VMCPU.
515	* @param pVmxTransient Pointer to the VMX transient structure.
516	*/
517	DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
518	{
519	if (!(pVmxTransient->fVmcsFieldsRead & VMX_TRANSIENT_EXIT_INSTR_LEN))
520	{
521	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
522	AssertRCReturn(rc, rc);
523	pVmxTransient->fVmcsFieldsRead \|= VMX_TRANSIENT_EXIT_INSTR_LEN;
524	}
525	return VINF_SUCCESS;
526	}
527
528
529	/**
530	* Reads the exit qualification from the VMCS into the VMX transient structure.
531	*
532	* @returns VBox status code.
533	* @param pVCpu Pointer to the VMCPU.
534	* @param pVmxTransient Pointer to the VMX transient structure.
535	*/
536	DECLINLINE(int) hmR0VmxReadExitQualificationVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
537	{
538	if (!(pVmxTransient->fVmcsFieldsRead & VMX_TRANSIENT_EXIT_QUALIFICATION))
539	{
540	int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQualification);
541	AssertRCReturn(rc, rc);
542	pVmxTransient->fVmcsFieldsRead \|= VMX_TRANSIENT_EXIT_QUALIFICATION;
543	}
544	return VINF_SUCCESS;
545	}
546
547
548	/**
549	* Reads the IDT-vectoring information field from the VMCS into the VMX
550	* transient structure.
551	*
552	* @returns VBox status code.
553	* @param pVmxTransient Pointer to the VMX transient structure.
554	*
555	* @remarks No-long-jump zone!!!
556	*/
557	DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
558	{
559	if (!(pVmxTransient->fVmcsFieldsRead & VMX_TRANSIENT_IDT_VECTORING_INFO))
560	{
561	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_INFO, &pVmxTransient->uIdtVectoringInfo);
562	AssertRCReturn(rc, rc);
563	pVmxTransient->fVmcsFieldsRead \|= VMX_TRANSIENT_IDT_VECTORING_INFO;
564	}
565	return VINF_SUCCESS;
566	}
567
568
569	/**
570	* Reads the IDT-vectoring error code from the VMCS into the VMX
571	* transient structure.
572	*
573	* @returns VBox status code.
574	* @param pVmxTransient Pointer to the VMX transient structure.
575	*/
576	DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
577	{
578	if (!(pVmxTransient->fVmcsFieldsRead & VMX_TRANSIENT_IDT_VECTORING_ERROR_CODE))
579	{
580	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
581	AssertRCReturn(rc, rc);
582	pVmxTransient->fVmcsFieldsRead \|= VMX_TRANSIENT_IDT_VECTORING_ERROR_CODE;
583	}
584	return VINF_SUCCESS;
585	}
586
587
588	/**
589	* Enters VMX root mode operation on the current CPU.
590	*
591	* @returns VBox status code.
592	* @param pVM Pointer to the VM (optional, can be NULL, after
593	* a resume).
594	* @param HCPhysCpuPage Physical address of the VMXON region.
595	* @param pvCpuPage Pointer to the VMXON region.
596	*/
597	DECLINLINE(int) hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
598	{
599	AssertReturn(HCPhysCpuPage != 0 && HCPhysCpuPage != NIL_RTHCPHYS, VERR_INVALID_PARAMETER);
600	AssertReturn(pvCpuPage, VERR_INVALID_PARAMETER);
601
602	if (pVM)
603	{
604	/* Write the VMCS revision dword to the VMXON region. */
605	(uint32_t )pvCpuPage = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.msr.vmx_basic_info);
606	}
607
608	/* Disable interrupts. Interrupts handlers might, in theory, change CR4. */
609	RTCCUINTREG fFlags = ASMIntDisableFlags();
610
611	/* Enable the VMX bit in CR4 if necessary. */
612	RTCCUINTREG uCr4 = ASMGetCR4();
613	if (!(uCr4 & X86_CR4_VMXE))
614	ASMSetCR4(uCr4 \| X86_CR4_VMXE);
615
616	/* Enter VMX root mode. */
617	int rc = VMXEnable(HCPhysCpuPage); /** @todo This would #GP(0) if we are already in VMX root mode... try skip it? */
618	if (RT_FAILURE(rc))
619	ASMSetCR4(uCr4);
620
621	/* Restore interrupts. */
622	ASMSetFlags(fFlags);
623	return rc;
624	}
625
626
627	/**
628	* Exits VMX root mode operation on the current CPU.
629	*
630	* @returns VBox status code.
631	*/
632	static int hmR0VmxLeaveRootMode(void)
633	{
634	/* Disable interrupts. Interrupt handlers might, in theory, change CR4. */
635	RTCCUINTREG fFlags = ASMIntDisableFlags();
636	int rc = VINF_SUCCESS;
637
638	/* If we're for some reason not in VMX root mode, then don't leave it. */
639	if (ASMGetCR4() & X86_CR4_VMXE)
640	{
641	/* Exit VMX root mode and clear the VMX bit in CR4 */
642	VMXDisable();
643	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
644	}
645	else
646	rc = VERR_VMX_NOT_IN_VMX_ROOT_MODE;
647
648	/* Restore interrupts. */
649	ASMSetFlags(fFlags);
650	return rc;
651	}
652
653
654	/**
655	* Allocates and maps one physically contiguous page. The allocated page is
656	* zero'd out. (Used by various VT-x structures).
657	*
658	* @returns IPRT status code.
659	* @param pMemObj Pointer to the ring-0 memory object.
660	* @param ppVirt Where to store the virtual address of the
661	* allocation.
662	* @param pPhys Where to store the physical address of the
663	* allocation.
664	*/
665	DECLINLINE(int) hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
666	{
667	AssertPtrReturn(pMemObj, VERR_INVALID_PARAMETER);
668	AssertPtrReturn(ppVirt, VERR_INVALID_PARAMETER);
669	AssertPtrReturn(pHCPhys, VERR_INVALID_PARAMETER);
670
671	int rc = RTR0MemObjAllocCont(pMemObj, PAGE_SIZE, false /* fExecutable */);
672	if (RT_FAILURE(rc))
673	return rc;
674	ppVirt = RTR0MemObjAddress(pMemObj);
675	pHCPhys = RTR0MemObjGetPagePhysAddr(pMemObj, 0 /* iPage */);
676	ASMMemZero32(*ppVirt, PAGE_SIZE);
677	return VINF_SUCCESS;
678	}
679
680
681	/**
682	* Frees and unmaps an allocated physical page.
683	*
684	* @param pMemObj Pointer to the ring-0 memory object.
685	* @param ppVirt Where to re-initialize the virtual address of
686	* allocation as 0.
687	* @param pHCPhys Where to re-initialize the physical address of the
688	* allocation as 0.
689	*/
690	DECLINLINE(void) hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
691	{
692	AssertPtr(pMemObj);
693	AssertPtr(ppVirt);
694	AssertPtr(pHCPhys);
695	if (*pMemObj != NIL_RTR0MEMOBJ)
696	{
697	int rc = RTR0MemObjFree(pMemObj, true / fFreeMappings */);
698	AssertRC(rc);
699	*pMemObj = NIL_RTR0MEMOBJ;
700	*ppVirt = 0;
701	*pHCPhys = 0;
702	}
703	}
704
705
706	/**
707	* Worker function to free VT-x related structures.
708	*
709	* @returns IPRT status code.
710	* @param pVM Pointer to the VM.
711	*/
712	static void hmR0VmxStructsFree(PVM pVM)
713	{
714	for (VMCPUID i = 0; i < pVM->cCpus; i++)
715	{
716	PVMCPU pVCpu = &pVM->aCpus[i];
717	AssertPtr(pVCpu);
718
719	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
720	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
721
722	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
723	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap, &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
724
725	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic, &pVCpu->hm.s.vmx.HCPhysVirtApic);
726	hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
727	}
728
729	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
730	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
731	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
732	#endif
733	}
734
735
736	/**
737	* Worker function to allocate VT-x related VM structures.
738	*
739	* @returns IPRT status code.
740	* @param pVM Pointer to the VM.
741	*/
742	static int hmR0VmxStructsAlloc(PVM pVM)
743	{
744	/*
745	* Initialize members up-front so we can cleanup properly on allocation failure.
746	*/
747	#define VMXLOCAL_INIT_VM_MEMOBJ(a_Name, a_VirtPrefix) \
748	pVM->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
749	pVM->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
750	pVM->hm.s.vmx.HCPhys##a_Name = 0;
751
752	#define VMXLOCAL_INIT_VMCPU_MEMOBJ(a_Name, a_VirtPrefix) \
753	pVCpu->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
754	pVCpu->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
755	pVCpu->hm.s.vmx.HCPhys##a_Name = 0;
756
757	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
758	VMXLOCAL_INIT_VM_MEMOBJ(Scratch, pv);
759	#endif
760	VMXLOCAL_INIT_VM_MEMOBJ(ApicAccess, pb);
761
762	AssertCompile(sizeof(VMCPUID) == sizeof(pVM->cCpus));
763	for (VMCPUID i = 0; i < pVM->cCpus; i++)
764	{
765	PVMCPU pVCpu = &pVM->aCpus[i];
766	VMXLOCAL_INIT_VMCPU_MEMOBJ(Vmcs, pv);
767	VMXLOCAL_INIT_VMCPU_MEMOBJ(VirtApic, pb);
768	VMXLOCAL_INIT_VMCPU_MEMOBJ(MsrBitmap, pv);
769	VMXLOCAL_INIT_VMCPU_MEMOBJ(GuestMsr, pv);
770	VMXLOCAL_INIT_VMCPU_MEMOBJ(HostMsr, pv);
771	}
772	#undef VMXLOCAL_INIT_VMCPU_MEMOBJ
773	#undef VMXLOCAL_INIT_VM_MEMOBJ
774
775	/*
776	* Allocate all the VT-x structures.
777	*/
778	int rc = VINF_SUCCESS;
779	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
780	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
781	if (RT_FAILURE(rc))
782	goto cleanup;
783	strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
784	(uint64_t )(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xDEADBEEFDEADBEEF);
785	#endif
786
787	/* Allocate the APIC-access page for trapping APIC accesses from the guest. */
788	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
789	{
790	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
791	&pVM->hm.s.vmx.HCPhysApicAccess);
792	if (RT_FAILURE(rc))
793	goto cleanup;
794	}
795
796	/*
797	* Initialize per-VCPU VT-x structures.
798	*/
799	for (VMCPUID i =0; i < pVM->cCpus; i++)
800	{
801	PVMCPU pVCpu = &pVM->aCpus[i];
802	AssertPtr(pVCpu);
803
804	/* Allocate the VM control structure (VMCS). */
805	AssertReturn(MSR_IA32_VMX_BASIC_INFO_VMCS_SIZE(pVM->hm.s.vmx.msr.vmx_basic_info) <= PAGE_SIZE, VERR_INTERNAL_ERROR);
806	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
807	if (RT_FAILURE(rc))
808	goto cleanup;
809
810	/* Allocate the Virtual-APIC page for transparent TPR accesses. */
811	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
812	{
813	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic,
814	&pVCpu->hm.s.vmx.HCPhysVirtApic);
815	if (RT_FAILURE(rc))
816	goto cleanup;
817	}
818
819	/* Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for transparent accesses of specific MSRs. */
820	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
821	{
822	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap,
823	&pVCpu->hm.s.vmx.HCPhysMsrBitmap);
824	if (RT_FAILURE(rc))
825	goto cleanup;
826	memset(pVCpu->hm.s.vmx.pvMsrBitmap, 0xff, PAGE_SIZE);
827	}
828
829	/* Allocate the VM-entry MSR-load and VM-exit MSR-store page for the guest MSRs. */
830	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
831	if (RT_FAILURE(rc))
832	goto cleanup;
833
834	/* Allocate the VM-exit MSR-load page for the host MSRs. */
835	rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
836	if (RT_FAILURE(rc))
837	goto cleanup;
838	}
839
840	return VINF_SUCCESS;
841
842	cleanup:
843	hmR0VmxStructsFree(pVM);
844	return rc;
845	}
846
847
848	/**
849	* Does global VT-x initialization (called during module initialization).
850	*
851	* @returns VBox status code.
852	*/
853	VMMR0DECL(int) VMXR0GlobalInit(void)
854	{
855	/* Setup the main VM exit handlers. */
856	AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
857	#ifdef VBOX_STRICT
858	for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
859	Assert(g_apfnVMExitHandlers[i]);
860	#endif
861	return VINF_SUCCESS;
862	}
863
864
865	/**
866	* Does global VT-x termination (called during module termination).
867	*/
868	VMMR0DECL(void) VMXR0GlobalTerm()
869	{
870	/* Nothing to do currently. */
871	}
872
873
874	/**
875	* Sets up and activates VT-x on the current CPU.
876	*
877	* @returns VBox status code.
878	* @param pCpu Pointer to the global CPU info struct.
879	* @param pVM Pointer to the VM (can be NULL after a host resume
880	* operation).
881	* @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
882	* fEnabledByHost is true).
883	* @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
884	* @a fEnabledByHost is true).
885	* @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
886	* enable VT-x/AMD-V on the host.
887	*/
888	VMMR0DECL(int) VMXR0EnableCpu(PHMGLOBLCPUINFO pCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost)
889	{
890	AssertReturn(pCpu, VERR_INVALID_PARAMETER);
891	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
892
893	if (!fEnabledByHost)
894	{
895	int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
896	if (RT_FAILURE(rc))
897	return rc;
898	}
899
900	/*
901	* Flush all VPIDs (in case we or any other hypervisor have been using VPIDs) so that
902	* we can avoid an explicit flush while using new VPIDs. We would still need to flush
903	* each time while reusing a VPID after hitting the MaxASID limit once.
904	*/
905	if ( pVM
906	&& pVM->hm.s.vmx.fVpid
907	&& (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS))
908	{
909	hmR0VmxFlushVpid(pVM, NULL /* pvCpu /, VMX_FLUSH_VPID_ALL_CONTEXTS, 0 / GCPtr */);
910	pCpu->fFlushAsidBeforeUse = false;
911	}
912	else
913	pCpu->fFlushAsidBeforeUse = true;
914
915	/* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
916	++pCpu->cTlbFlushes;
917
918	return VINF_SUCCESS;
919	}
920
921
922	/**
923	* Deactivates VT-x on the current CPU.
924	*
925	* @returns VBox status code.
926	* @param pCpu Pointer to the global CPU info struct.
927	* @param pvCpuPage Pointer to the VMXON region.
928	* @param HCPhysCpuPage Physical address of the VMXON region.
929	*/
930	VMMR0DECL(int) VMXR0DisableCpu(PHMGLOBLCPUINFO pCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
931	{
932	NOREF(pCpu);
933	NOREF(pvCpuPage);
934	NOREF(HCPhysCpuPage);
935
936	hmR0VmxLeaveRootMode();
937	return VINF_SUCCESS;
938	}
939
940
941	/**
942	* Sets the permission bits for the specified MSR in the MSR bitmap.
943	*
944	* @param pVCpu Pointer to the VMCPU.
945	* @param uMSR The MSR value.
946	* @param enmRead Whether reading this MSR causes a VM-exit.
947	* @param enmWrite Whether writing this MSR causes a VM-exit.
948	*/
949	static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, VMXMSREXITREAD enmRead, VMXMSREXITWRITE enmWrite)
950	{
951	int32_t iBit;
952	uint8_t pbMsrBitmap = (uint8_t )pVCpu->hm.s.vmx.pvMsrBitmap;
953
954	/*
955	* Layout:
956	* 0x000 - 0x3ff - Low MSR read bits
957	* 0x400 - 0x7ff - High MSR read bits
958	* 0x800 - 0xbff - Low MSR write bits
959	* 0xc00 - 0xfff - High MSR write bits
960	*/
961	if (uMsr <= 0x00001FFF)
962	{
963	/* Pentium-compatible MSRs */
964	iBit = uMsr;
965	}
966	else if ( uMsr >= 0xC0000000
967	&& uMsr <= 0xC0001FFF)
968	{
969	/* AMD Sixth Generation x86 Processor MSRs */
970	iBit = (uMsr - 0xC0000000);
971	pbMsrBitmap += 0x400;
972	}
973	else
974	{
975	AssertMsgFailed(("hmR0VmxSetMsrPermission: Invalid MSR %#RX32\n", uMsr));
976	return;
977	}
978
979	Assert(iBit <= 0x1fff);
980	if (enmRead == VMXMSREXIT_INTERCEPT_READ)
981	ASMBitSet(pbMsrBitmap, iBit);
982	else
983	ASMBitClear(pbMsrBitmap, iBit);
984
985	if (enmWrite == VMXMSREXIT_INTERCEPT_WRITE)
986	ASMBitSet(pbMsrBitmap + 0x800, iBit);
987	else
988	ASMBitClear(pbMsrBitmap + 0x800, iBit);
989	}
990
991
992	/**
993	* Flushes the TLB using EPT.
994	*
995	* @returns VBox status code.
996	* @param pVM Pointer to the VM.
997	* @param pVCpu Pointer to the VMCPU.
998	* @param enmFlush Type of flush.
999	*/
1000	static void hmR0VmxFlushEpt(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_EPT enmFlush)
1001	{
1002	AssertPtr(pVM);
1003	Assert(pVM->hm.s.fNestedPaging);
1004
1005	LogFlowFunc(("pVM=%p pVCpu=%p enmFlush=%d\n", pVM, pVCpu, enmFlush));
1006
1007	uint64_t descriptor[2];
1008	descriptor[0] = pVCpu->hm.s.vmx.HCPhysEPTP;
1009	descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
1010
1011	int rc = VMXR0InvEPT(enmFlush, &descriptor[0]);
1012	AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %#x %RGv failed with %Rrc\n", enmFlush, pVCpu->hm.s.vmx.HCPhysEPTP, rc));
1013	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
1014	}
1015
1016
1017	/**
1018	* Flushes the TLB using VPID.
1019	*
1020	* @returns VBox status code.
1021	* @param pVM Pointer to the VM.
1022	* @param pVCpu Pointer to the VMCPU (can be NULL depending on @a
1023	* enmFlush).
1024	* @param enmFlush Type of flush.
1025	* @param GCPtr Virtual address of the page to flush (can be 0 depending
1026	* on @a enmFlush).
1027	*/
1028	static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_VPID enmFlush, RTGCPTR GCPtr)
1029	{
1030	AssertPtr(pVM);
1031	Assert(pVM->hm.s.vmx.fVpid);
1032
1033	uint64_t descriptor[2];
1034	if (enmFlush == VMX_FLUSH_VPID_ALL_CONTEXTS)
1035	{
1036	descriptor[0] = 0;
1037	descriptor[1] = 0;
1038	}
1039	else
1040	{
1041	AssertPtr(pVCpu);
1042	AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1043	AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1044	descriptor[0] = pVCpu->hm.s.uCurrentAsid;
1045	descriptor[1] = GCPtr;
1046	}
1047
1048	int rc = VMXR0InvVPID(enmFlush, &descriptor[0]); NOREF(rc);
1049	AssertMsg(rc == VINF_SUCCESS,
1050	("VMXR0InvVPID %#x %u %RGv failed with %d\n", enmFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
1051	if ( RT_SUCCESS(rc)
1052	&& pVCpu)
1053	{
1054	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
1055	}
1056	}
1057
1058
1059	/**
1060	* Invalidates a guest page by guest virtual address. Only relevant for
1061	* EPT/VPID, otherwise there is nothing really to invalidate.
1062	*
1063	* @returns VBox status code.
1064	* @param pVM Pointer to the VM.
1065	* @param pVCpu Pointer to the VMCPU.
1066	* @param GCVirt Guest virtual address of the page to invalidate.
1067	*/
1068	VMMR0DECL(int) VMXR0InvalidatePage(PVM pVM, PVMCPU pVCpu, RTGCPTR GCVirt)
1069	{
1070	AssertPtr(pVM);
1071	AssertPtr(pVCpu);
1072	LogFlowFunc(("pVM=%p pVCpu=%p GCVirt=%RGv\n", pVM, pVCpu, GCVirt));
1073
1074	bool fFlushPending = VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1075	if (!fFlushPending)
1076	{
1077	/*
1078	* We must invalidate the guest TLB entry in either case, we cannot ignore it even for the EPT case
1079	* See @bugref{6043} and @bugref{6177}.
1080	*
1081	* Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*() as this
1082	* function maybe called in a loop with individual addresses.
1083	*/
1084	if (pVM->hm.s.vmx.fVpid)
1085	{
1086	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1087	{
1088	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, GCVirt);
1089	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
1090	}
1091	else
1092	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1093	}
1094	else if (pVM->hm.s.fNestedPaging)
1095	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1096	}
1097
1098	return VINF_SUCCESS;
1099	}
1100
1101
1102	/**
1103	* Invalidates a guest page by physical address. Only relevant for EPT/VPID,
1104	* otherwise there is nothing really to invalidate.
1105	*
1106	* @returns VBox status code.
1107	* @param pVM Pointer to the VM.
1108	* @param pVCpu Pointer to the VMCPU.
1109	* @param GCPhys Guest physical address of the page to invalidate.
1110	*/
1111	VMMR0DECL(int) VMXR0InvalidatePhysPage(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys)
1112	{
1113	LogFlowFunc(("%RGp\n", GCPhys));
1114
1115	/*
1116	* We cannot flush a page by guest-physical address. invvpid takes only a linear address while invept only flushes
1117	* by EPT not individual addresses. We update the force flag here and flush before the next VM-entry in hmR0VmxFlushTLB*().
1118	* This function might be called in a loop.
1119	*/
1120	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1121	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgPhys);
1122	return VINF_SUCCESS;
1123	}
1124
1125
1126	/**
1127	* Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
1128	* case where neither EPT nor VPID is supported by the CPU.
1129	*
1130	* @param pVM Pointer to the VM.
1131	* @param pVCpu Pointer to the VMCPU.
1132	*
1133	* @remarks Called with interrupts disabled.
1134	*/
1135	static DECLCALLBACK(void) hmR0VmxFlushTaggedTlbNone(PVM pVM, PVMCPU pVCpu)
1136	{
1137	NOREF(pVM);
1138	AssertPtr(pVCpu);
1139	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
1140	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1141
1142	PHMGLOBLCPUINFO pCpu = HMR0GetCurrentCpu();
1143	AssertPtr(pCpu);
1144
1145	pVCpu->hm.s.TlbShootdown.cPages = 0;
1146	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1147	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1148	pVCpu->hm.s.fForceTLBFlush = false;
1149	return;
1150	}
1151
1152
1153	/**
1154	* Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
1155	*
1156	* @param pVM Pointer to the VM.
1157	* @param pVCpu Pointer to the VMCPU.
1158	* @remarks All references to "ASID" in this function pertains to "VPID" in
1159	* Intel's nomenclature. The reason is, to avoid confusion in compare
1160	* statements since the host-CPU copies are named "ASID".
1161	*
1162	* @remarks Called with interrupts disabled.
1163	*/
1164	static DECLCALLBACK(void) hmR0VmxFlushTaggedTlbBoth(PVM pVM, PVMCPU pVCpu)
1165	{
1166	AssertPtr(pVM);
1167	AssertPtr(pVCpu);
1168	AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
1169	("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
1170	"fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
1171
1172	PHMGLOBLCPUINFO pCpu = HMR0GetCurrentCpu();
1173	AssertPtr(pCpu);
1174
1175	/*
1176	* Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1177	* This can happen both for start & resume due to long jumps back to ring-3.
1178	* If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1179	* or the host Cpu is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1180	*/
1181	bool fNewASID = false;
1182	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1183	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1184	{
1185	pVCpu->hm.s.fForceTLBFlush = true;
1186	fNewASID = true;
1187	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1188	}
1189
1190	/*
1191	* Check for explicit TLB shootdowns.
1192	*/
1193	if (VMCPU_FF_TESTANDCLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1194	{
1195	pVCpu->hm.s.fForceTLBFlush = true;
1196	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1197	}
1198
1199	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1200	if (pVCpu->hm.s.fForceTLBFlush)
1201	{
1202	if (fNewASID)
1203	{
1204	++pCpu->uCurrentAsid;
1205	if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1206	{
1207	pCpu->uCurrentAsid = 1; /* start at 1; host uses 0 */
1208	pCpu->cTlbFlushes++;
1209	pCpu->fFlushAsidBeforeUse = true;
1210	}
1211
1212	pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1213	if (pCpu->fFlushAsidBeforeUse)
1214	hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
1215	}
1216	else
1217	{
1218	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
1219	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_SINGLE_CONTEXT, 0 /* GCPtr */);
1220	else
1221	hmR0VmxFlushEpt(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1222	}
1223
1224	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1225	pVCpu->hm.s.fForceTLBFlush = false;
1226	}
1227	else
1228	{
1229	AssertMsg(pVCpu->hm.s.uCurrentAsid && pCpu->uCurrentAsid,
1230	("hm->uCurrentAsid=%lu hm->cTlbFlushes=%lu cpu->uCurrentAsid=%lu cpu->cTlbFlushes=%lu\n",
1231	pVCpu->hm.s.uCurrentAsid, pVCpu->hm.s.cTlbFlushes,
1232	pCpu->uCurrentAsid, pCpu->cTlbFlushes));
1233
1234	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
1235	* not be executed. See hmQueueInvlPage() where it is commented
1236	* out. Support individual entry flushing someday. */
1237	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1238	{
1239	STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
1240
1241	/*
1242	* Flush individual guest entries using VPID from the TLB or as little as possible with EPT
1243	* as supported by the CPU.
1244	*/
1245	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1246	{
1247	for (uint32_t i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
1248	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
1249	}
1250	else
1251	hmR0VmxFlushEpt(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1252	}
1253	else
1254	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
1255	}
1256	pVCpu->hm.s.TlbShootdown.cPages = 0;
1257	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1258
1259	AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
1260	("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
1261	AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
1262	("cpu%d uCurrentAsid = %u\n", pCpu->idCpu, pCpu->uCurrentAsid));
1263	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
1264	("cpu%d VM uCurrentAsid = %u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
1265
1266	/* Update VMCS with the VPID. */
1267	int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
1268	AssertRC(rc);
1269	}
1270
1271
1272	/**
1273	* Flushes the tagged-TLB entries for EPT CPUs as necessary.
1274	*
1275	* @returns VBox status code.
1276	* @param pVM Pointer to the VM.
1277	* @param pVCpu Pointer to the VMCPU.
1278	*
1279	* @remarks Called with interrupts disabled.
1280	*/
1281	static DECLCALLBACK(void) hmR0VmxFlushTaggedTlbEpt(PVM pVM, PVMCPU pVCpu)
1282	{
1283	AssertPtr(pVM);
1284	AssertPtr(pVCpu);
1285	AssertMsg(pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with NestedPaging disabled."));
1286	AssertMsg(!pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID enabled."));
1287
1288	PHMGLOBLCPUINFO pCpu = HMR0GetCurrentCpu();
1289	AssertPtr(pCpu);
1290
1291	/*
1292	* Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1293	* This can happen both for start & resume due to long jumps back to ring-3.
1294	* A change in the TLB flush count implies the host CPU is online after a suspend/resume.
1295	*/
1296	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1297	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1298	{
1299	pVCpu->hm.s.fForceTLBFlush = true;
1300	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1301	}
1302
1303	/* Check for explicit TLB shootdown flushes. */
1304	if (VMCPU_FF_TESTANDCLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1305	{
1306	pVCpu->hm.s.fForceTLBFlush = true;
1307	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1308	}
1309
1310	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1311	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1312
1313	if (pVCpu->hm.s.fForceTLBFlush)
1314	{
1315	hmR0VmxFlushEpt(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1316	pVCpu->hm.s.fForceTLBFlush = false;
1317	}
1318	else
1319	{
1320	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
1321	* not be executed. See hmQueueInvlPage() where it is commented
1322	* out. Support individual entry flushing someday. */
1323	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1324	{
1325	/* We cannot flush individual entries without VPID support. Flush using EPT. */
1326	STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
1327	hmR0VmxFlushEpt(pVM, pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1328	}
1329	else
1330	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
1331	}
1332
1333	pVCpu->hm.s.TlbShootdown.cPages = 0;
1334	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1335	}
1336
1337
1338	/**
1339	* Flushes the tagged-TLB entries for VPID CPUs as necessary.
1340	*
1341	* @returns VBox status code.
1342	* @param pVM Pointer to the VM.
1343	* @param pVCpu Pointer to the VMCPU.
1344	*
1345	* @remarks Called with interrupts disabled.
1346	*/
1347	static DECLCALLBACK(void) hmR0VmxFlushTaggedTlbVpid(PVM pVM, PVMCPU pVCpu)
1348	{
1349	AssertPtr(pVM);
1350	AssertPtr(pVCpu);
1351	AssertMsg(pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked with VPID disabled."));
1352	AssertMsg(!pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging enabled"));
1353
1354	PHMGLOBLCPUINFO pCpu = HMR0GetCurrentCpu();
1355
1356	/*
1357	* Force a TLB flush for the first world switch if the current CPU differs from the one we ran on last.
1358	* This can happen both for start & resume due to long jumps back to ring-3.
1359	* If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1360	* or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1361	*/
1362	if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1363	\|\| pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1364	{
1365	pVCpu->hm.s.fForceTLBFlush = true;
1366	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1367	}
1368
1369	/* Check for explicit TLB shootdown flushes. */
1370	if (VMCPU_FF_TESTANDCLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1371	{
1372	/*
1373	* If we ever support VPID flush combinations other than ALL or SINGLE-context (see hmR0VmxSetupTaggedTlb())
1374	* we would need to explicitly flush in this case (add an fExplicitFlush = true here and change the
1375	* pCpu->fFlushAsidBeforeUse check below to include fExplicitFlush's too) - an obscure corner case.
1376	*/
1377	pVCpu->hm.s.fForceTLBFlush = true;
1378	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1379	}
1380
1381	pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1382	if (pVCpu->hm.s.fForceTLBFlush)
1383	{
1384	++pCpu->uCurrentAsid;
1385	if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1386	{
1387	pCpu->uCurrentAsid = 1; /* start at 1; host uses 0 */
1388	pCpu->fFlushAsidBeforeUse = true;
1389	pCpu->cTlbFlushes++;
1390	}
1391
1392	pVCpu->hm.s.fForceTLBFlush = false;
1393	pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1394	pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1395	if (pCpu->fFlushAsidBeforeUse)
1396	hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
1397	}
1398	else
1399	{
1400	AssertMsg(pVCpu->hm.s.uCurrentAsid && pCpu->uCurrentAsid,
1401	("hm->uCurrentAsid=%lu hm->cTlbFlushes=%lu cpu->uCurrentAsid=%lu cpu->cTlbFlushes=%lu\n",
1402	pVCpu->hm.s.uCurrentAsid, pVCpu->hm.s.cTlbFlushes,
1403	pCpu->uCurrentAsid, pCpu->cTlbFlushes));
1404
1405	/** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere so this path should
1406	* not be executed. See hmQueueInvlPage() where it is commented
1407	* out. Support individual entry flushing someday. */
1408	if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1409	{
1410	/* Flush individual guest entries using VPID or as little as possible with EPT as supported by the CPU. */
1411	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1412	{
1413	for (uint32_t i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
1414	hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
1415	}
1416	else
1417	hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
1418	}
1419	else
1420	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
1421	}
1422
1423	pVCpu->hm.s.TlbShootdown.cPages = 0;
1424	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1425
1426	AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
1427	("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
1428	AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
1429	("cpu%d uCurrentAsid = %u\n", pCpu->idCpu, pCpu->uCurrentAsid));
1430	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
1431	("cpu%d VM uCurrentAsid = %u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
1432
1433	int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
1434	AssertRC(rc);
1435	}
1436
1437
1438	/**
1439	* Flushes the guest TLB entry based on CPU capabilities.
1440	*
1441	* @param pVCpu Pointer to the VMCPU.
1442	*/
1443	DECLINLINE(void) hmR0VmxFlushTaggedTlb(PVMCPU pVCpu)
1444	{
1445	PVM pVM = pVCpu->CTX_SUFF(pVM);
1446	switch (pVM->hm.s.vmx.uFlushTaggedTlb)
1447	{
1448	case VMX_FLUSH_TAGGED_TLB_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pVM, pVCpu); break;
1449	case VMX_FLUSH_TAGGED_TLB_EPT: hmR0VmxFlushTaggedTlbEpt(pVM, pVCpu); break;
1450	case VMX_FLUSH_TAGGED_TLB_VPID: hmR0VmxFlushTaggedTlbVpid(pVM, pVCpu); break;
1451	case VMX_FLUSH_TAGGED_TLB_NONE: hmR0VmxFlushTaggedTlbNone(pVM, pVCpu); break;
1452	default:
1453	AssertMsgFailed(("Invalid flush-tag function identifier\n"));
1454	break;
1455	}
1456	}
1457
1458
1459	/**
1460	* Sets up the appropriate tagged TLB-flush level and handler for flushing guest
1461	* TLB entries from the host TLB before VM-entry.
1462	*
1463	* @returns VBox status code.
1464	* @param pVM Pointer to the VM.
1465	*/
1466	static int hmR0VmxSetupTaggedTlb(PVM pVM)
1467	{
1468	/*
1469	* Determine optimal flush type for nested paging.
1470	* We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup unrestricted
1471	* guest execution (see hmR3InitFinalizeR0()).
1472	*/
1473	if (pVM->hm.s.fNestedPaging)
1474	{
1475	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
1476	{
1477	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
1478	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_SINGLE_CONTEXT;
1479	else if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
1480	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_ALL_CONTEXTS;
1481	else
1482	{
1483	/* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
1484	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
1485	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1486	}
1487
1488	/* Make sure the write-back cacheable memory type for EPT is supported. */
1489	if (!(pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB))
1490	{
1491	LogRel(("hmR0VmxSetupTaggedTlb: Unsupported EPTP memory type %#x.\n", pVM->hm.s.vmx.msr.vmx_ept_vpid_caps));
1492	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
1493	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1494	}
1495	}
1496	else
1497	{
1498	/* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
1499	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
1500	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1501	}
1502	}
1503
1504	/*
1505	* Determine optimal flush type for VPID.
1506	*/
1507	if (pVM->hm.s.vmx.fVpid)
1508	{
1509	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
1510	{
1511	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
1512	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_SINGLE_CONTEXT;
1513	else if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
1514	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_ALL_CONTEXTS;
1515	else
1516	{
1517	/* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
1518	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1519	LogRel(("hmR0VmxSetupTaggedTlb: Only INDIV_ADDR supported. Ignoring VPID.\n"));
1520	if (pVM->hm.s.vmx.msr.vmx_ept_vpid_caps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
1521	LogRel(("hmR0VmxSetupTaggedTlb: Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
1522	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NOT_SUPPORTED;
1523	pVM->hm.s.vmx.fVpid = false;
1524	}
1525	}
1526	else
1527	{
1528	/* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
1529	Log(("hmR0VmxSetupTaggedTlb: VPID supported without INVEPT support. Ignoring VPID.\n"));
1530	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NOT_SUPPORTED;
1531	pVM->hm.s.vmx.fVpid = false;
1532	}
1533	}
1534
1535	/*
1536	* Setup the handler for flushing tagged-TLBs.
1537	*/
1538	if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
1539	pVM->hm.s.vmx.uFlushTaggedTlb = VMX_FLUSH_TAGGED_TLB_EPT_VPID;
1540	else if (pVM->hm.s.fNestedPaging)
1541	pVM->hm.s.vmx.uFlushTaggedTlb = VMX_FLUSH_TAGGED_TLB_EPT;
1542	else if (pVM->hm.s.vmx.fVpid)
1543	pVM->hm.s.vmx.uFlushTaggedTlb = VMX_FLUSH_TAGGED_TLB_VPID;
1544	else
1545	pVM->hm.s.vmx.uFlushTaggedTlb = VMX_FLUSH_TAGGED_TLB_NONE;
1546	return VINF_SUCCESS;
1547	}
1548
1549
1550	/**
1551	* Sets up pin-based VM-execution controls in the VMCS.
1552	*
1553	* @returns VBox status code.
1554	* @param pVM Pointer to the VM.
1555	* @param pVCpu Pointer to the VMCPU.
1556	*/
1557	static int hmR0VmxSetupPinCtls(PVM pVM, PVMCPU pVCpu)
1558	{
1559	AssertPtr(pVM);
1560	AssertPtr(pVCpu);
1561
1562	uint32_t val = pVM->hm.s.vmx.msr.vmx_pin_ctls.n.disallowed0; /* Bits set here must always be set. */
1563	uint32_t zap = pVM->hm.s.vmx.msr.vmx_pin_ctls.n.allowed1; /* Bits cleared here must always be cleared. */
1564
1565	val \|= VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_EXT_INT_EXIT /* External interrupts causes a VM-exits. */
1566	\| VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_NMI_EXIT; /* Non-maskable interrupts causes a VM-exit. */
1567	Assert(!(val & VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_VIRTUAL_NMI));
1568
1569	/* Enable the VMX preemption timer. */
1570	if (pVM->hm.s.vmx.fUsePreemptTimer)
1571	{
1572	Assert(pVM->hm.s.vmx.msr.vmx_pin_ctls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_PREEMPT_TIMER);
1573	val \|= VMX_VMCS_CTRL_PIN_EXEC_CONTROLS_PREEMPT_TIMER;
1574	}
1575
1576	if ((val & zap) != val)
1577	{
1578	LogRel(("hmR0VmxSetupPinCtls: invalid pin-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
1579	pVM->hm.s.vmx.msr.vmx_pin_ctls.n.disallowed0, val, zap));
1580	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1581	}
1582
1583	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC_CONTROLS, val);
1584	AssertRCReturn(rc, rc);
1585
1586	/* Update VCPU with the currently set pin-based VM-execution controls. */
1587	pVCpu->hm.s.vmx.u32PinCtls = val;
1588	return rc;
1589	}
1590
1591
1592	/**
1593	* Sets up processor-based VM-execution controls in the VMCS.
1594	*
1595	* @returns VBox status code.
1596	* @param pVM Pointer to the VM.
1597	* @param pVMCPU Pointer to the VMCPU.
1598	*/
1599	static int hmR0VmxSetupProcCtls(PVM pVM, PVMCPU pVCpu)
1600	{
1601	AssertPtr(pVM);
1602	AssertPtr(pVCpu);
1603
1604	int rc = VERR_INTERNAL_ERROR_5;
1605	uint32_t val = pVM->hm.s.vmx.msr.vmx_proc_ctls.n.disallowed0; /* Bits set here must be set in the VMCS. */
1606	uint32_t zap = pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
1607
1608	val \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_HLT_EXIT /* HLT causes a VM-exit. */
1609	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
1610	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
1611	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
1612	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDPMC_EXIT /* RDPMC causes a VM-exit. */
1613	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_EXIT /* MONITOR causes a VM-exit. */
1614	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
1615
1616	/* We toggle VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
1617	if ( !(pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT)
1618	\|\| (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.disallowed0 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT))
1619	{
1620	LogRel(("hmR0VmxSetupProcCtls: unsupported VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT combo!"));
1621	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1622	}
1623
1624	/* Without nested paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
1625	if (!pVM->hm.s.fNestedPaging)
1626	{
1627	Assert(!pVM->hm.s.vmx.fUnrestrictedGuest); /* Paranoia. */
1628	val \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INVLPG_EXIT
1629	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
1630	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT;
1631	}
1632
1633	/* Use TPR shadowing if supported by the CPU. */
1634	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
1635	{
1636	Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
1637	Assert(!(pVCpu->hm.s.vmx.HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
1638	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, 0);
1639	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL, pVCpu->hm.s.vmx.HCPhysVirtApic);
1640	AssertRCReturn(rc, rc);
1641
1642	val \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
1643	/* CR8 writes causes a VM-exit based on TPR threshold. */
1644	Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR8_STORE_EXIT));
1645	Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR8_LOAD_EXIT));
1646	}
1647	else
1648	{
1649	val \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR8_STORE_EXIT /* CR8 reads causes a VM-exit. */
1650	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR8_LOAD_EXIT; /* CR8 writes causes a VM-exit. */
1651	}
1652
1653	/* Use MSR-bitmaps if supported by the CPU. */
1654	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
1655	{
1656	val \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS;
1657
1658	Assert(pVCpu->hm.s.vmx.HCPhysMsrBitmap);
1659	Assert(!(pVCpu->hm.s.vmx.HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
1660	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, pVCpu->hm.s.vmx.HCPhysMsrBitmap);
1661	AssertRCReturn(rc, rc);
1662
1663	/*
1664	* The guest can access the following MSRs (read, write) without causing VM-exits; they are loaded/stored
1665	* automatically (either as part of the MSR-load/store areas or dedicated fields in the VMCS).
1666	*/
1667	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_CS, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1668	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_ESP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1669	hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_EIP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1670	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1671	hmR0VmxSetMsrPermission(pVCpu, MSR_K6_STAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1672	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_SF_MASK, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1673	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1674	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1675	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_FS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1676	}
1677
1678	/* Use the secondary processor-based VM-execution controls if supported by the CPU. */
1679	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
1680	val \|= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
1681
1682	if ((val & zap) != val)
1683	{
1684	LogRel(("hmR0VmxSetupProcCtls: invalid processor-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
1685	pVM->hm.s.vmx.msr.vmx_proc_ctls.n.disallowed0, val, zap));
1686	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1687	}
1688
1689	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, val);
1690	AssertRCReturn(rc, rc);
1691
1692	/* Update VCPU with the currently set processor-based VM-execution controls. */
1693	pVCpu->hm.s.vmx.u32ProcCtls = val;
1694
1695	/*
1696	* Secondary processor-based VM-execution controls.
1697	*/
1698	if (RT_LIKELY(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL))
1699	{
1700	val = pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.disallowed0; /* Bits set here must be set in the VMCS. */
1701	zap = pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
1702
1703	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT)
1704	val \|= VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT; /* WBINVD causes a VM-exit. */
1705
1706	if (pVM->hm.s.fNestedPaging)
1707	val \|= VMX_VMCS_CTRL_PROC_EXEC2_EPT; /* Enable EPT. */
1708	else
1709	{
1710	/*
1711	* Without Nested Paging, INVPCID should cause a VM-exit. Enabling this bit causes the CPU to refer to
1712	* VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INVLPG_EXIT when INVPCID is executed by the guest.
1713	* See Intel spec. 25.4 "Changes to instruction behaviour in VMX non-root operation".
1714	*/
1715	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_INVPCID)
1716	val \|= VMX_VMCS_CTRL_PROC_EXEC2_INVPCID;
1717	}
1718
1719	if (pVM->hm.s.vmx.fVpid)
1720	val \|= VMX_VMCS_CTRL_PROC_EXEC2_VPID; /* Enable VPID. */
1721
1722	if (pVM->hm.s.vmx.fUnrestrictedGuest)
1723	val \|= VMX_VMCS_CTRL_PROC_EXEC2_UNRESTRICTED_GUEST; /* Enable Unrestricted Execution. */
1724
1725	/* Enable Virtual-APIC page accesses if supported by the CPU. This is essentially where the TPR shadow resides. */
1726	/** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
1727	* done dynamically. */
1728	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
1729	{
1730	Assert(pVM->hm.s.vmx.HCPhysApicAccess);
1731	Assert(!(pVM->hm.s.vmx.HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
1732	val \|= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC; /* Virtualize APIC accesses. */
1733	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, pVM->hm.s.vmx.HCPhysApicAccess);
1734	AssertRCReturn(rc, rc);
1735	}
1736
1737	if (pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
1738	{
1739	val \|= VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP; /* Enable RDTSCP support. */
1740	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
1741	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_TSC_AUX, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1742	}
1743
1744	if ((val & zap) != val)
1745	{
1746	LogRel(("hmR0VmxSetupProcCtls: invalid secondary processor-based VM-execution controls combo! "
1747	"cpu=%#RX64 val=%#RX64 zap=%#RX64\n", pVM->hm.s.vmx.msr.vmx_proc_ctls2.n.disallowed0, val, zap));
1748	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1749	}
1750
1751	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS2, val);
1752	AssertRCReturn(rc, rc);
1753
1754	/* Update VCPU with the currently set secondary processor-based VM-execution controls. */
1755	pVCpu->hm.s.vmx.u32ProcCtls2 = val;
1756	}
1757
1758	return VINF_SUCCESS;
1759	}
1760
1761
1762	/**
1763	* Sets up miscellaneous (everything other than Pin & Processor-based
1764	* VM-execution) control fields in the VMCS.
1765	*
1766	* @returns VBox status code.
1767	* @param pVM Pointer to the VM.
1768	* @param pVCpu Pointer to the VMCPU.
1769	*/
1770	static int hmR0VmxSetupMiscCtls(PVM pVM, PVMCPU pVCpu)
1771	{
1772	AssertPtr(pVM);
1773	AssertPtr(pVCpu);
1774
1775	int rc = VERR_GENERAL_FAILURE;
1776
1777	/* All CR3 accesses cause VM-exits. Later we optimize CR3 accesses (see hmR0VmxLoadGuestControlRegs())*/
1778	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, 0);
1779
1780	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, 0);
1781
1782	/*
1783	* Set MASK & MATCH to 0. VMX checks if GuestPFErrCode & MASK == MATCH. If equal (in our case it always is)
1784	* and if the X86_XCPT_PF bit in the exception bitmap is set it causes a VM-exit, if clear doesn't cause an exit.
1785	* We thus use the exception bitmap to control it rather than use both.
1786	*/
1787	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, 0);
1788	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, 0);
1789
1790	/** @todo Explore possibility of using IO-bitmaps. */
1791	/* All IO & IOIO instructions cause VM-exits. */
1792	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_A_FULL, 0);
1793	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_B_FULL, 0);
1794
1795	/* Setup MSR autoloading/autostoring. */
1796	Assert(pVCpu->hm.s.vmx.HCPhysGuestMsr);
1797	Assert(!(pVCpu->hm.s.vmx.HCPhysGuestMsr & 0xf)); /* Lower 4 bits MBZ. */
1798	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
1799	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
1800	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, 0);
1801	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, 0);
1802
1803	Assert(pVCpu->hm.s.vmx.HCPhysHostMsr);
1804	Assert(!(pVCpu->hm.s.vmx.HCPhysHostMsr & 0xf)); /* Lower 4 bits MBZ. */
1805	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysHostMsr);
1806	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, 0);
1807
1808	/* Set VMCS link pointer. Reserved for future use, must be -1. Intel spec. 24.4 "Guest-State Area". */
1809	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, 0xffffffffffffffffULL);
1810
1811	/* Setup debug controls */
1812	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, 0); /** @todo think about this. */
1813	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0); /** @todo Intel spec. 26.6.3 think about this */
1814	AssertRCReturn(rc, rc);
1815	return rc;
1816	}
1817
1818
1819	/**
1820	* Sets up the initial exception bitmap in the VMCS based on static conditions
1821	* (i.e. conditions that cannot ever change at runtime).
1822	*
1823	* @returns VBox status code.
1824	* @param pVM Pointer to the VM.
1825	* @param pVCpu Pointer to the VMCPU.
1826	*/
1827	static int hmR0VmxInitXcptBitmap(PVM pVM, PVMCPU pVCpu)
1828	{
1829	AssertPtr(pVM);
1830	AssertPtr(pVCpu);
1831
1832	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
1833
1834	uint32_t u32XcptBitmap = 0;
1835
1836	/* Without nested paging, #PF must cause a VM-exit so we can sync our shadow page tables. */
1837	if (!pVM->hm.s.fNestedPaging)
1838	u32XcptBitmap \|= RT_BIT(X86_XCPT_PF);
1839
1840	pVCpu->hm.s.vmx.u32XcptBitmap = u32XcptBitmap;
1841	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
1842	AssertRCReturn(rc, rc);
1843	return rc;
1844	}
1845
1846
1847	/**
1848	* Sets up the initial guest-state mask. The guest-state mask is consulted
1849	* before reading guest-state fields from the VMCS as VMREADs can be expensive
1850	* for the nested virtualization case (as it would cause a VM-exit).
1851	*
1852	* @param pVCpu Pointer to the VMCPU.
1853	*/
1854	static int hmR0VmxInitUpdatedGuestStateMask(PVMCPU pVCpu)
1855	{
1856	/* Initially the guest-state is up-to-date as there is nothing in the VMCS. */
1857	pVCpu->hm.s.vmx.fUpdatedGuestState = VMX_UPDATED_GUEST_ALL;
1858	return VINF_SUCCESS;
1859	}
1860
1861
1862	/**
1863	* Does per-VM VT-x initialization.
1864	*
1865	* @returns VBox status code.
1866	* @param pVM Pointer to the VM.
1867	*/
1868	VMMR0DECL(int) VMXR0InitVM(PVM pVM)
1869	{
1870	LogFlowFunc(("pVM=%p\n", pVM));
1871
1872	int rc = hmR0VmxStructsAlloc(pVM);
1873	if (RT_FAILURE(rc))
1874	{
1875	LogRel(("VMXR0InitVM: hmR0VmxStructsAlloc failed! rc=%Rrc\n", rc));
1876	return rc;
1877	}
1878
1879	for (VMCPUID i = 0; i < pVM->cCpus; i++)
1880	{
1881	PVMCPU pVCpu = &pVM->aCpus[i];
1882
1883	/* Current guest paging mode. */
1884	pVCpu->hm.s.vmx.enmLastSeenGuestMode = PGMMODE_REAL;
1885	}
1886
1887	return VINF_SUCCESS;
1888	}
1889
1890
1891	/**
1892	* Does per-VM VT-x termination.
1893	*
1894	* @returns VBox status code.
1895	* @param pVM Pointer to the VM.
1896	*/
1897	VMMR0DECL(int) VMXR0TermVM(PVM pVM)
1898	{
1899	LogFlowFunc(("pVM=%p\n", pVM));
1900
1901	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
1902	if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
1903	ASMMemZero32(pVM->hm.s.vmx.pvScratch, PAGE_SIZE);
1904	#endif
1905	hmR0VmxStructsFree(pVM);
1906	return VINF_SUCCESS;
1907	}
1908
1909
1910	/**
1911	* Sets up the VM for execution under VT-x.
1912	* This function is only called once per-VM during initalization.
1913	*
1914	* @returns VBox status code.
1915	* @param pVM Pointer to the VM.
1916	*/
1917	VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
1918	{
1919	AssertPtrReturn(pVM, VERR_INVALID_PARAMETER);
1920	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1921
1922	LogFlowFunc(("pVM=%p\n", pVM));
1923
1924	/*
1925	* Without UnrestrictedGuest, pRealModeTSS and pNonPagingModeEPTPageTable must always be allocated.
1926	* We no longer support the highly unlikely case of UnrestrictedGuest without pRealModeTSS. See hmR3InitFinalizeR0().
1927	*/
1928	/* -XXX- change hmR3InitFinalizeR0() to fail if pRealModeTSS alloc fails. */
1929	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
1930	&& ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
1931	\|\| !pVM->hm.s.vmx.pRealModeTSS))
1932	{
1933	LogRel(("VMXR0SetupVM: invalid real-on-v86 state.\n"));
1934	return VERR_INTERNAL_ERROR;
1935	}
1936
1937	/* Initialize these always, see hmR3InitFinalizeR0().*/
1938	pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NONE;
1939	pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NONE;
1940
1941	/* Setup the tagged-TLB flush handlers. */
1942	int rc = hmR0VmxSetupTaggedTlb(pVM);
1943	if (RT_FAILURE(rc))
1944	{
1945	LogRel(("VMXR0SetupVM: hmR0VmxSetupTaggedTlb failed! rc=%Rrc\n", rc));
1946	return rc;
1947	}
1948
1949	for (VMCPUID i = 0; i < pVM->cCpus; i++)
1950	{
1951	PVMCPU pVCpu = &pVM->aCpus[i];
1952	AssertPtr(pVCpu);
1953	AssertPtr(pVCpu->hm.s.vmx.pvVmcs);
1954
1955	/* Set revision dword at the beginning of the VMCS structure. */
1956	(uint32_t )pVCpu->hm.s.vmx.pvVmcs = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.msr.vmx_basic_info);
1957
1958	/* Initialize our VMCS region in memory, set the VMCS launch state to "clear". */
1959	rc = VMXClearVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
1960	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVMCS failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
1961	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
1962
1963	/* Load this VMCS as the current VMCS. */
1964	rc = VMXActivateVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
1965	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXActivateVMCS failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
1966	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
1967
1968	/* Setup the pin-based VM-execution controls. */
1969	rc = hmR0VmxSetupPinCtls(pVM, pVCpu);
1970	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupPinCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
1971	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
1972
1973	/* Setup the processor-based VM-execution controls. */
1974	rc = hmR0VmxSetupProcCtls(pVM, pVCpu);
1975	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupProcCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
1976	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
1977
1978	/* Setup the rest (miscellaneous) VM-execution controls. */
1979	rc = hmR0VmxSetupMiscCtls(pVM, pVCpu);
1980	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupMiscCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
1981	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
1982
1983	/* Setup the initial exception bitmap. */
1984	rc = hmR0VmxInitXcptBitmap(pVM, pVCpu);
1985	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitXcptBitmap failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
1986	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
1987
1988	/* Setup the initial guest-state mask. */
1989	rc = hmR0VmxInitUpdatedGuestStateMask(pVCpu);
1990	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitUpdatedGuestStateMask failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
1991	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
1992
1993	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
1994	/* Setup the VMCS read cache as we queue up certain VMWRITEs that can only be done in 64-bit mode for 64-bit guests. */
1995	rc = hmR0VmxInitVmcsReadCache(pVM, pVCpu);
1996	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitVmcsReadCache failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
1997	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
1998	#endif
1999
2000	/* Re-sync the CPU's internal data into our VMCS memory region & reset the launch state to "clear". */
2001	rc = VMXClearVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
2002	AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVMCS(2) failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2003	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2004
2005	/* Update the last error record for this VCPU. */
2006	hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc);
2007	}
2008
2009	return VINF_SUCCESS;
2010	}
2011
2012
2013	/**
2014	* Saves the host control registers (CR0, CR3, CR4) into the host-state area in
2015	* the VMCS.
2016	*
2017	* @returns VBox status code.
2018	* @param pVM Pointer to the VM.
2019	* @param pVCpu Pointer to the VMCPU.
2020	*/
2021	DECLINLINE(int) hmR0VmxSaveHostControlRegs(PVM pVM, PVMCPU pVCpu)
2022	{
2023	RTCCUINTREG uReg = ASMGetCR0();
2024	int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
2025
2026	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2027	/* For the darwin 32-bit hybrid kernel, we need the 64-bit CR3 as it uses 64-bit paging. */
2028	if (VMX_IS_64BIT_HOST_MODE())
2029	{
2030	uint64_t uReg = hmR0Get64bitCR3();
2031	rc \|= VMXWriteVmcs64(VMX_VMCS_HOST_CR3, uReg);
2032	}
2033	else
2034	#endif
2035	{
2036	uReg = ASMGetCR3();
2037	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
2038	}
2039
2040	uReg = ASMGetCR4();
2041	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
2042	AssertRCReturn(rc, rc);
2043	return rc;
2044	}
2045
2046
2047	/**
2048	* Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
2049	* the host-state area in the VMCS.
2050	*
2051	* @returns VBox status code.
2052	* @param pVM Pointer to the VM.
2053	* @param pVCpu Pointer to the VMCPU.
2054	*/
2055	DECLINLINE(int) hmR0VmxSaveHostSegmentRegs(PVM pVM, PVMCPU pVCpu)
2056	{
2057	int rc = VERR_INTERNAL_ERROR_5;
2058	RTSEL uSelCS = 0;
2059	RTSEL uSelSS = 0;
2060	RTSEL uSelDS = 0;
2061	RTSEL uSelES = 0;
2062	RTSEL uSelFS = 0;
2063	RTSEL uSelGS = 0;
2064	RTSEL uSelTR = 0;
2065
2066	/*
2067	* Host Selector registers.
2068	*/
2069	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2070	if (VMX_IS_64BIT_HOST_MODE())
2071	{
2072	uSelCS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelCS;
2073	uSelSS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelSS;
2074	}
2075	else
2076	{
2077	/* Seems darwin uses the LDT (TI flag is set) in the CS & SS selectors which VT-x doesn't like. */
2078	uSelCS = (RTSEL)(uintptr_t)&SUPR0AbsKernelCS;
2079	uSelSS = (RTSEL)(uintptr_t)&SUPR0AbsKernelSS;
2080	}
2081	#else
2082	uSelCS = ASMGetCS();
2083	uSelSS = ASMGetSS();
2084	#endif
2085
2086	/* Note: VT-x is picky about the RPL of the selectors here; we'll restore them manually. */
2087	/** @todo Verify if we have any platform that actually run with DS or ES with
2088	* RPL != 0 in kernel space. */
2089	uSelDS = 0;
2090	uSelES = 0;
2091	uSelFS = 0;
2092	uSelGS = 0;
2093	uSelTR = ASMGetTR();
2094
2095	/* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
2096	Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
2097	Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
2098	Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
2099	Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
2100	Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
2101	Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
2102	Assert(uSelCS != 0);
2103	Assert(uSelTR != 0);
2104
2105	/* Assertion is right but we would not have updated u32ExitCtls yet. */
2106	#if 0
2107	if (!(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_ADDR_SPACE_SIZE))
2108	Assert(uSelSS != 0);
2109	#endif
2110
2111	/* Write these host selector fields into the host-state area in the VMCS. */
2112	rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_CS, uSelCS);
2113	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_SS, uSelSS);
2114	/* Avoid the VMWRITEs as we set the following segments to 0 and the VMCS fields are already 0 (since g_HvmR0 is static) */
2115	#if 0
2116	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_DS, uSelDS);
2117	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_ES, uSelES);
2118	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_FS, uSelFS);
2119	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_GS, uSelGS);
2120	#endif
2121	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_TR, uSelTR);
2122	AssertRCReturn(rc, rc);
2123
2124	/*
2125	* Host GDTR and IDTR.
2126	*/
2127	/** @todo Despite VT-x -not- restoring the limits on GDTR and IDTR it should
2128	* be safe to -not- save and restore GDTR and IDTR in the assembly
2129	* code and just do it here and don't care if the limits are zapped on
2130	* VM-exit. */
2131	RTGDTR Gdtr;
2132	RT_ZERO(Gdtr);
2133	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2134	if (VMX_IS_64BIT_HOST_MODE())
2135	{
2136	X86XDTR64 Gtr64;
2137	X86XDTR64 Idtr64;
2138	hmR0Get64bitGdtrAndIdtr(&Gdtr64, &Idtr64);
2139	rc = VMXWriteVmcs64(VMX_VMCS_HOST_GDTR_BASE, Gdtr64.uAddr);
2140	rc \|= VMXWriteVmcs64(VMX_VMCS_HOST_IDTR_BASE, Idtr64.uAddr);
2141	Gdtr.cbGdt = Gdtr64.cb;
2142	Gdtr.pGdt = (uintptr_t)Gdtr64.uAddr;
2143	}
2144	else
2145	#endif
2146	{
2147	RTIDTR Idtr;
2148	ASMGetGDTR(&Gdtr);
2149	ASMGetIDTR(&Idtr);
2150	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt);
2151	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt);
2152	}
2153	AssertRCReturn(rc, rc);
2154
2155	/*
2156	* Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI and RPL bits
2157	* is effectively what the CPU does for "scaling by 8". TI is always 0 and RPL should be too in most cases.
2158	*/
2159	if ((uSelTR & X86_SEL_MASK) > Gdtr.cbGdt)
2160	{
2161	AssertMsgFailed(("hmR0VmxSaveHostSegmentRegs: TR selector exceeds limit.TR=%RTsel Gdtr.cbGdt=%#x\n", uSelTR, Gdtr.cbGdt));
2162	return VERR_VMX_INVALID_HOST_STATE;
2163	}
2164
2165	PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
2166	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2167	if (VMX_IS_64BIT_HOST_MODE())
2168	{
2169	/* We need the 64-bit TR base for hybrid darwin. */
2170	uint64_t u64TRBase = X86DESC64_BASE((PX86DESC64)pDesc);
2171	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, u64TRBase);
2172	}
2173	else
2174	#endif
2175	{
2176	uintptr_t uTRBase;
2177	#if HC_ARCH_BITS == 64
2178	uTRBase = X86DESC64_BASE(pDesc);
2179	#else
2180	uTRBase = X86DESC_BASE(pDesc);
2181	#endif
2182	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
2183	}
2184	AssertRCReturn(rc, rc);
2185
2186	/*
2187	* Host FS base and GS base.
2188	* For 32-bit hosts the base is handled by the assembly code where we push/pop FS and GS which .
2189	* would take care of the bases. In 64-bit, the MSRs come into play.
2190	*/
2191	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2192	if (VMX_IS_64BIT_HOST_MODE())
2193	{
2194	uint64_t u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
2195	uint64_t u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
2196	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_FS_BASE, u64FSBase);
2197	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_GS_BASE, u64GSBase);
2198	AssertRCReturn(rc, rc);
2199	}
2200	#endif
2201	return rc;
2202	}
2203
2204
2205	/**
2206	* Saves certain host MSRs in the VM-Exit MSR-load area and some in the
2207	* host-state area of the VMCS. Theses MSRs will be automatically restored on
2208	* the host after every successful VM exit.
2209	*
2210	* @returns VBox status code.
2211	* @param pVM Pointer to the VM.
2212	* @param pVCpu Pointer to the VMCPU.
2213	*/
2214	DECLINLINE(int) hmR0VmxSaveHostMsrs(PVM pVM, PVMCPU pVCpu)
2215	{
2216	AssertPtr(pVCpu);
2217	AssertPtr(pVCpu->hm.s.vmx.pvHostMsr);
2218
2219	PVMXMSR pHostMsr = (PVMXMSR)pVCpu->hm.s.vmx.pvHostMsr;
2220	uint32_t cHostMsrs = 0;
2221	uint32_t u32HostExtFeatures = pVM->hm.s.cpuid.u32AMDFeatureEDX;
2222
2223	if (u32HostExtFeatures & (X86_CPUID_EXT_FEATURE_EDX_NX \| X86_CPUID_EXT_FEATURE_EDX_LONG_MODE))
2224	{
2225	pHostMsr->u32IndexMSR = MSR_K6_EFER;
2226	pHostMsr->u32Reserved = 0;
2227	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2228	if (CPUMIsGuestInLongMode(pVCpu))
2229	{
2230	/* Must match the EFER value in our 64 bits switcher. */
2231	pHostMsr->u64Value = ASMRdMsr(MSR_K6_EFER) \| MSR_K6_EFER_LME \| MSR_K6_EFER_SCE \| MSR_K6_EFER_NXE;
2232	}
2233	else
2234	#endif
2235	pHostMsr->u64Value = ASMRdMsr(MSR_K6_EFER);
2236	pHostMsr++; cHostMsrs++;
2237	}
2238
2239	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2240	if (VMX_IS_64BIT_HOST_MODE())
2241	{
2242	pHostMsr->u32IndexMSR = MSR_K6_STAR;
2243	pHostMsr->u32Reserved = 0;
2244	pHostMsr->u64Value = ASMRdMsr(MSR_K6_STAR); /* legacy syscall eip, cs & ss */
2245	pHostMsr++; cHostMsrs++;
2246	pHostMsr->u32IndexMSR = MSR_K8_LSTAR;
2247	pHostMsr->u32Reserved = 0;
2248	pHostMsr->u64Value = ASMRdMsr(MSR_K8_LSTAR); /* 64 bits mode syscall rip */
2249	pHostMsr++; cHostMsrs++;
2250	pHostMsr->u32IndexMSR = MSR_K8_SF_MASK;
2251	pHostMsr->u32Reserved = 0;
2252	pHostMsr->u64Value = ASMRdMsr(MSR_K8_SF_MASK); /* syscall flag mask */
2253	pHostMsr++; cHostMsrs++;
2254	/* The KERNEL_GS_BASE MSR doesn't work reliably with auto load/store. See @bugref{6208} */
2255	#if 0
2256	pMsr->u32IndexMSR = MSR_K8_KERNEL_GS_BASE;
2257	pMsr->u32Reserved = 0;
2258	pMsr->u64Value = ASMRdMsr(MSR_K8_KERNEL_GS_BASE); /* swapgs exchange value */
2259	pHostMsr++; cHostMsrs++;
2260	#endif
2261	}
2262	#endif
2263
2264	/* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
2265	if (RT_UNLIKELY(cHostMsrs > MSR_IA32_VMX_MISC_MAX_MSR(pVM->hm.s.vmx.msr.vmx_misc)))
2266	{
2267	LogRel(("cHostMsrs=%u Cpu=%u\n", cHostMsrs, (unsigned)MSR_IA32_VMX_MISC_MAX_MSR(pVM->hm.s.vmx.msr.vmx_misc)));
2268	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2269	}
2270
2271	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cHostMsrs);
2272
2273	/*
2274	* Host Sysenter MSRs.
2275	*/
2276	rc \|= VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
2277	#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2278	if (VMX_IS_64BIT_HOST_MODE())
2279	{
2280	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
2281	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
2282	}
2283	else
2284	{
2285	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
2286	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
2287	}
2288	#elif HC_ARCH_BITS == 32
2289	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
2290	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
2291	#else
2292	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
2293	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
2294	#endif
2295	AssertRCReturn(rc, rc);
2296
2297	/** @todo IA32_PERF_GLOBALCTRL, IA32_PAT, IA32_EFER, also see
2298	* hmR0VmxSetupExitCtls() !! */
2299	return rc;
2300	}
2301
2302
2303	/**
2304	* Sets up VM-entry controls in the VMCS. These controls can affect things done
2305	* on VM-exit; e.g. "load debug controls", see Intel spec. 24.8.1 "VM-entry
2306	* controls".
2307	*
2308	* @returns VBox status code.
2309	* @param pVM Pointer to the VM.
2310	* @param pVCpu Pointer to the VMCPU.
2311	* @param pCtx Pointer to the guest-CPU context.
2312	*
2313	* @remarks No-long-jump zone!!!
2314	*/
2315	DECLINLINE(int) hmR0VmxLoadGuestEntryCtls(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
2316	{
2317	int rc = VINF_SUCCESS;
2318	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_ENTRY_CTLS)
2319	{
2320	uint32_t val = pVM->hm.s.vmx.msr.vmx_entry.n.disallowed0; /* Bits set here must be set in the VMCS. */
2321	uint32_t zap = pVM->hm.s.vmx.msr.vmx_entry.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2322
2323	/* Load debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x capable CPUs only supports the 1-setting of this bit. */
2324	val \|= VMX_VMCS_CTRL_ENTRY_CONTROLS_LOAD_DEBUG;
2325
2326	/* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry. */
2327	if (CPUMIsGuestInLongModeEx(pCtx))
2328	val \|= VMX_VMCS_CTRL_ENTRY_CONTROLS_IA32E_MODE_GUEST;
2329	else
2330	Assert(!(val & VMX_VMCS_CTRL_ENTRY_CONTROLS_IA32E_MODE_GUEST));
2331
2332	/*
2333	* The following should not be set (since we're not in SMM mode):
2334	* - VMX_VMCS_CTRL_ENTRY_CONTROLS_ENTRY_SMM
2335	* - VMX_VMCS_CTRL_ENTRY_CONTROLS_DEACTIVATE_DUALMON
2336	*/
2337
2338	/** @todo VMX_VMCS_CTRL_ENTRY_CONTROLS_LOAD_GUEST_PERF_MSR,
2339	* VMX_VMCS_CTRL_ENTRY_CONTROLS_LOAD_GUEST_PAT_MSR,
2340	* VMX_VMCS_CTRL_ENTRY_CONTROLS_LOAD_GUEST_EFER_MSR */
2341
2342	if ((val & zap) != val)
2343	{
2344	LogRel(("hmR0VmxLoadGuestEntryCtls: invalid VM-entry controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
2345	pVM->hm.s.vmx.msr.vmx_entry.n.disallowed0, val, zap));
2346	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2347	}
2348
2349	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_CONTROLS, val);
2350	AssertRCReturn(rc, rc);
2351
2352	/* Update VCPU with the currently set VM-exit controls. */
2353	pVCpu->hm.s.vmx.u32EntryCtls = val;
2354	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_ENTRY_CTLS;
2355	}
2356	return rc;
2357	}
2358
2359
2360	/**
2361	* Sets up the VM-exit controls in the VMCS.
2362	*
2363	* @returns VBox status code.
2364	* @param pVM Pointer to the VM.
2365	* @param pVCpu Pointer to the VMCPU.
2366	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2367	* out-of-sync. Make sure to update the required fields
2368	* before using them.
2369	*
2370	* @remarks requires EFER.
2371	*/
2372	DECLINLINE(int) hmR0VmxLoadGuestExitCtls(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2373	{
2374	int rc = VINF_SUCCESS;
2375	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_EXIT_CTLS)
2376	{
2377	uint32_t val = pVM->hm.s.vmx.msr.vmx_exit.n.disallowed0; /* Bits set here must be set in the VMCS. */
2378	uint32_t zap = pVM->hm.s.vmx.msr.vmx_exit.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2379
2380	/* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only supported the 1-setting of this bit. */
2381	val \|= VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_DEBUG;
2382
2383	/* Set the host long mode active (EFER.LMA) bit (which Intel calls "Host address-space size") if necessary. */
2384	#if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2385	if (VMX_IS_64BIT_HOST_MODE())
2386	val \|= VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_ADDR_SPACE_SIZE;
2387	else
2388	Assert(!(val & VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_ADDR_SPACE_SIZE));
2389	#elif HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
2390	if (CPUMIsGuestInLongModeEx(pMixedCtx))
2391	val \|= VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_ADDR_SPACE_SIZE; /* The switcher goes to long mode. */
2392	else
2393	Assert(!(val & VMX_VMCS_CTRL_EXIT_CONTROLS_HOST_ADDR_SPACE_SIZE));
2394	#endif
2395
2396	/* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
2397	Assert(!(val & VMX_VMCS_CTRL_EXIT_CONTROLS_ACK_EXT_INT));
2398
2399	/** @todo VMX_VMCS_CTRL_EXIT_CONTROLS_LOAD_PERF_MSR,
2400	* VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_GUEST_PAT_MSR,
2401	* VMX_VMCS_CTRL_EXIT_CONTROLS_LOAD_HOST_PAT_MSR,
2402	* VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_GUEST_EFER_MSR,
2403	* VMX_VMCS_CTRL_EXIT_CONTROLS_LOAD_HOST_EFER_MSR. */
2404
2405	if (pVM->hm.s.vmx.msr.vmx_exit.n.allowed1 & VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_VMX_PREEMPT_TIMER)
2406	val \|= VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_VMX_PREEMPT_TIMER;
2407
2408	if ((val & zap) != val)
2409	{
2410	LogRel(("hmR0VmxSetupProcCtls: invalid VM-exit controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
2411	pVM->hm.s.vmx.msr.vmx_exit.n.disallowed0, val, zap));
2412	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2413	}
2414
2415	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_CONTROLS, val);
2416	AssertRCReturn(rc, rc);
2417
2418	/* Update VCPU with the currently set VM-exit controls. */
2419	pVCpu->hm.s.vmx.u32ExitCtls = val;
2420	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_EXIT_CTLS;
2421	}
2422	return rc;
2423	}
2424
2425
2426	/**
2427	* Loads the guest APIC and related state.
2428	*
2429	* @returns VBox status code.
2430	* @param pVM Pointer to the VM.
2431	* @param pVCpu Pointer to the VMCPU.
2432	* @param pCtx Pointer to the guest-CPU context.
2433	*/
2434	DECLINLINE(int) hmR0VmxLoadGuestApicState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
2435	{
2436	int rc = VINF_SUCCESS;
2437	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_GUEST_APIC_STATE)
2438	{
2439	/* Setup TPR shadowing. Also setup TPR patching for 32-bit guests. */
2440	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
2441	{
2442	Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
2443
2444	bool fPendingIntr = false;
2445	uint8_t u8GuestTpr = 0;
2446	rc = PDMApicGetTPR(pVCpu, &u8GuestTpr, &fPendingIntr);
2447	AssertRCReturn(rc, rc);
2448
2449	/*
2450	* If there are external interrupts pending but masked by the TPR value, apply the threshold so that if the guest
2451	* lowers the TPR, it would cause a VM-exit and we can deliver the interrupt.
2452	* If there are no external interrupts pending, set threshold to 0 to not cause a VM-exit. We will eventually deliver
2453	* the interrupt when we VM-exit for other reasons.
2454	*/
2455	pVCpu->hm.s.vmx.pbVirtApic[0x80] = u8GuestTpr; /* Offset 0x80 is TPR in the APIC MMIO range. */
2456	/* Bits 3-0 of the TPR threshold field correspond to bits 7-4 of the TPR (which is the Task-Priority Class). */
2457	uint32_t u32TprThreshold = fPendingIntr ? (u8GuestTpr >> 4) : 0;
2458	Assert(!(u32TprThreshold & 0xfffffff0)); /* Bits 31:4 MBZ. */
2459
2460	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
2461	AssertRCReturn(rc, rc);
2462
2463	/* 32-bit guests uses LSTAR MSR for patching guest code which touches the TPR. */
2464	if (pVM->hm.s.fTPRPatchingActive)
2465	{
2466	Assert(!CPUMIsGuestInLongModeEx(pCtx)); /* EFER always up-to-date. */
2467	pCtx->msrLSTAR = u8GuestTpr;
2468	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS)
2469	{
2470	/* If there are interrupts pending, intercept CR8 writes, otherwise don't intercept CR8 reads or writes. */
2471	if (fPendingIntr)
2472	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_INTERCEPT_WRITE);
2473	else
2474	hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2475	}
2476	}
2477	}
2478
2479	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_GUEST_APIC_STATE;
2480	}
2481	return rc;
2482	}
2483
2484
2485	/**
2486	* Gets the guest's interruptibility-state ("interrupt shadow" as AMD calls it).
2487	*
2488	* @returns
2489	* @param pVCpu Pointer to the VMCPU.
2490	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2491	* out-of-sync. Make sure to update the required fields
2492	* before using them.
2493	*
2494	* @remarks No-long-jump zone!!!
2495	* @remarks Has side-effects with VMCPU_FF_INHIBIT_INTERRUPTS force-flag.
2496	*/
2497	DECLINLINE(uint32_t) hmR0VmxGetGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2498	{
2499	/*
2500	* Instructions like STI and MOV SS inhibit interrupts till the next instruction completes. Check if we should
2501	* inhibit interrupts or clear any existing interrupt-inhibition.
2502	*/
2503	uint32_t uIntrState = 0;
2504	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
2505	{
2506	/* If inhibition is active, RIP & RFLAGS should've been accessed (i.e. read previously from the VMCS or from ring-3). */
2507	AssertMsg((pVCpu->hm.s.vmx.fUpdatedGuestState & (VMX_UPDATED_GUEST_RIP \| VMX_UPDATED_GUEST_RFLAGS))
2508	== (VMX_UPDATED_GUEST_RIP \| VMX_UPDATED_GUEST_RFLAGS), ("%#x\n", pVCpu->hm.s.vmx.fUpdatedGuestState));
2509	if (pMixedCtx->rip != EMGetInhibitInterruptsPC(pVCpu))
2510	{
2511	/*
2512	* We can clear the inhibit force flag as even if we go back to the recompiler without executing guest code in
2513	* VT-x the flag's condition to be cleared is met and thus the cleared state is correct.
2514	*/
2515	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
2516	}
2517	else if (pMixedCtx->eflags.Bits.u1IF)
2518	uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
2519	else
2520	uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS;
2521	}
2522	return uIntrState;
2523	}
2524
2525
2526	/**
2527	* Loads the guest's interruptibility-state into the guest-state area in the
2528	* VMCS.
2529	*
2530	* @returns VBox status code.
2531	* @param pVCpu Pointer to the VMCPU.
2532	* @param uIntrState The interruptibility-state to set.
2533	*/
2534	DECLINLINE(int) hmR0VmxLoadGuestIntrState(PVMCPU pVCpu, uint32_t uIntrState)
2535	{
2536	AssertMsg(!(uIntrState & 0xfffffff0), ("%#x\n", uIntrState)); /* Bits 31:4 MBZ. */
2537	Assert((uIntrState & 0x3) != 0x3); /* Block-by-STI and MOV SS cannot be simultaneously set. */
2538	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, uIntrState);
2539	AssertRCReturn(rc, rc);
2540	return rc;
2541	}
2542
2543
2544	/**
2545	* Loads the guest's RIP into the guest-state area in the VMCS.
2546	*
2547	* @returns VBox status code.
2548	* @param pVM Pointer to the VM.
2549	* @param pVCpu Pointer to the VMCPU.
2550	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
2551	* out-of-sync. Make sure to update the required fields
2552	* before using them.
2553	*
2554	* @remarks No-long-jump zone!!!
2555	*/
2556	DECLINLINE(int) hmR0VmxLoadGuestRip(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2557	{
2558	int rc = VINF_SUCCESS;
2559	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_RIP)
2560	{
2561	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pMixedCtx->rip);
2562	AssertRCReturn(rc, rc);
2563	Log(("VMX_VMCS_GUEST_RIP=%#RX64\n", pMixedCtx->rip));
2564	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_RIP;
2565	}
2566	return rc;
2567	}
2568
2569
2570	/**
2571	* Loads the guest's RSP into the guest-state area in the VMCS.
2572	*
2573	* @returns VBox status code.
2574	* @param pVM Pointer to the VM.
2575	* @param pVCpu Pointer to the VMCPU.
2576	* @param pCtx Pointer to the guest-CPU context.
2577	*
2578	* @remarks No-long-jump zone!!!
2579	*/
2580	DECLINLINE(int) hmR0VmxLoadGuestRsp(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
2581	{
2582	int rc = VINF_SUCCESS;
2583	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_RSP)
2584	{
2585	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pMixedCtx->rsp);
2586	AssertRCReturn(rc, rc);
2587	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_RSP;
2588	}
2589	return rc;
2590	}
2591
2592
2593	/**
2594	* Loads the guest's RFLAGS into the guest-state area in the VMCS.
2595	*
2596	* @returns VBox status code.
2597	* @param pVM Pointer to the VM.
2598	* @param pVCpu Pointer to the VMCPU.
2599	* @param pCtx Pointer to the guest-CPU context.
2600	*
2601	* @remarks No-long-jump zone!!!
2602	*/
2603	DECLINLINE(int) hmR0VmxLoadGuestRflags(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
2604	{
2605	int rc = VINF_SUCCESS;
2606	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_RFLAGS)
2607	{
2608	/* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
2609	Let us assert it as such and use native-width VMWRITE. */
2610	X86RFLAGS uRFlags = pCtx->rflags;
2611	Assert(uRFlags.u64 >> 32 == 0);
2612	uRFlags.u64 &= VMX_EFLAGS_RESERVED_0; /* Bits 22-31, 15, 5 & 3 MBZ. */
2613	uRFlags.u64 \|= VMX_EFLAGS_RESERVED_1; /* Bit 1 MB1. */
2614
2615	/*
2616	* If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so we can restore them on VM exit.
2617	* Modify the real-mode guest's eflags so that VT-x can run the real-mode guest code under Virtual 8086 mode.
2618	*/
2619	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
2620	{
2621	Assert(pVM->hm.s.vmx.pRealModeTSS);
2622	Assert(PDMVmmDevHeapIsEnabled(pVM));
2623	pVCpu->hm.s.vmx.RealMode.eflags.u32 = uRFlags.u64; /* Save the original eflags of the real-mode guest. */
2624	uRFlags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
2625	uRFlags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
2626	}
2627
2628	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RFLAGS, uRFlags.u64);
2629	AssertRCReturn(rc, rc);
2630
2631	Log(("VMX_VMCS_GUEST_RFLAGS=%#RX64\n", uRFlags.u64));
2632	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_RFLAGS;
2633	}
2634	return rc;
2635	}
2636
2637
2638	/**
2639	* Loads the guest's general purpose registers (GPRs) - RIP, RSP and RFLAGS
2640	* into the guest-state area in the VMCS. The remaining GPRs are handled in the
2641	* assembly code.
2642	*
2643	* @returns VBox status code.
2644	* @param pVM Pointer to the VM.
2645	* @param pVCpu Pointer to the VMCPU.
2646	* @param pCtx Pointer to the guest-CPU context.
2647	*
2648	* @remarks No-long-jump zone!!!
2649	*/
2650	DECLINLINE(int) hmR0VmxLoadGuestGprs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
2651	{
2652	LogFlowFunc(("pVM=%p pVCpu=%p pCtx=%p\n", pVM, pVCpu, pCtx));
2653	int rc = hmR0VmxLoadGuestRip(pVM, pVCpu, pCtx);
2654	rc \|= hmR0VmxLoadGuestRsp(pVM, pVCpu, pCtx);
2655	rc \|= hmR0VmxLoadGuestRflags(pVM, pVCpu, pCtx);
2656	return rc;
2657	}
2658
2659
2660	/**
2661	* Loads the guest control registers (CR0, CR3, CR4) into the guest-state area
2662	* in the VMCS.
2663	*
2664	* @returns VBox status code.
2665	* @param pVM Pointer to the VM.
2666	* @param pVCpu Pointer to the VMCPU.
2667	* @param pCtx Pointer to the guest-CPU context.
2668	*
2669	* @remarks No-long-jump zone!!!
2670	*/
2671	DECLINLINE(int) hmR0VmxLoadGuestControlRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
2672	{
2673	int rc = VINF_SUCCESS;
2674
2675	/*
2676	* Guest CR0.
2677	* Guest FPU.
2678	*/
2679	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR0)
2680	{
2681	uint64_t u64GuestCR0 = pCtx->cr0;
2682
2683	/* The guest's view (read access) of its CR0 is unblemished. */
2684	rc = VMXWriteVmcsGstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, u64GuestCR0);
2685	AssertRCReturn(rc, rc);
2686	Log2(("VMX_VMCS_CTRL_CR0_READ_SHADOW=%#RX64\n", u64GuestCR0));
2687
2688	/* Setup VT-x's view of the guest CR0. */
2689	/* Minimize VM-exits due to CR3 changes when we have NestedPaging. */
2690	if (pVM->hm.s.fNestedPaging)
2691	{
2692	if (CPUMIsGuestPagingEnabledEx(pCtx))
2693	{
2694	/* The guest has paging enabled, let it access CR3 without causing a VM exit if supported. */
2695	pVCpu->hm.s.vmx.u32ProcCtls &= ~( VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
2696	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT);
2697	}
2698	else
2699	{
2700	/* The guest doesn't have paging enabled, make CR3 access to cause VM exits to update our shadow. */
2701	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_LOAD_EXIT
2702	\| VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_CR3_STORE_EXIT;
2703	}
2704
2705	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls);
2706	AssertRCReturn(rc, rc);
2707	}
2708	else
2709	u64GuestCR0 \|= X86_CR0_WP; /* Guest CPL 0 writes to its read-only pages should cause a VM-exit. */
2710
2711	/*
2712	* Guest FPU bits.
2713	* Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be set on the first
2714	* CPUs to support VT-x (prob. means all the way up to Nehalem) and no mention of with regards to UX in VM-entry checks.
2715	*/
2716	u64GuestCR0 \|= X86_CR0_NE;
2717	bool fInterceptNM = false;
2718	if (CPUMIsGuestFPUStateActive(pVCpu))
2719	{
2720	fInterceptNM = false; /* Guest FPU active, no need to VM-exit on #NM. */
2721	/* The guest should still get #NM exceptions when it expects it to, so we should not clear TS & MP bits here.
2722	We're only concerned about -us- not intercepting #NMs when the guest-FPU is active. Not the guest itself! */
2723	}
2724	else
2725	{
2726	fInterceptNM = true; /* Guest FPU inactive, VM-exit on #NM for lazy FPU loading. */
2727	u64GuestCR0 \|= X86_CR0_TS /* Guest can task switch quickly and do lazy FPU syncing. */
2728	\| X86_CR0_MP; /* FWAIT/WAIT should not ignore CR0.TS and should generate #NM. */
2729	}
2730
2731	/* Catch floating point exceptions if we need to report them to the guest in a different way. */
2732	bool fInterceptMF = false;
2733	if (!(pCtx->cr0 & X86_CR0_NE))
2734	fInterceptMF = true;
2735
2736	/* Finally, intercept all exceptions as we cannot directly inject them in real-mode, see hmR0VmxInjectEventVmcs(). */
2737	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
2738	{
2739	Assert(PDMVmmDevHeapIsEnabled(pVM));
2740	Assert(pVM->hm.s.vmx.pRealModeTSS);
2741	pVCpu->hm.s.vmx.u32XcptBitmap \|= VMX_REAL_MODE_XCPT_BITMAP;
2742	fInterceptNM = true;
2743	fInterceptMF = true;
2744	}
2745	else
2746	pVCpu->hm.s.vmx.u32XcptBitmap &= ~VMX_REAL_MODE_XCPT_BITMAP;
2747
2748	if (fInterceptNM)
2749	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_NM);
2750	else
2751	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_NM);
2752
2753	if (fInterceptMF)
2754	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_MF);
2755	else
2756	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_MF);
2757
2758	/* Additional intercepts for debugging, define these yourself explicitly. */
2759	#ifdef VBOX_ALWAYS_TRAP_ALL_EXCEPTIONS
2760	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_BP)
2761	\| RT_BIT(X86_XCPT_DB)
2762	\| RT_BIT(X86_XCPT_DE)
2763	\| RT_BIT(X86_XCPT_NM)
2764	\| RT_BIT(X86_XCPT_UD)
2765	\| RT_BIT(X86_XCPT_NP)
2766	\| RT_BIT(X86_XCPT_SS)
2767	\| RT_BIT(X86_XCPT_GP)
2768	\| RT_BIT(X86_XCPT_PF)
2769	\| RT_BIT(X86_XCPT_MF);
2770	#elif defined(VBOX_ALWAYS_TRAP_PF)
2771	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_PF)
2772	#endif
2773
2774	Assert(pVM->hm.s.fNestedPaging \|\| (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_PF)));
2775
2776	/* Set/clear the CR0 specific bits along with their exceptions (PE, PG, CD, NW). */
2777	uint64_t uSetCR0 = (pVM->hm.s.vmx.msr.vmx_cr0_fixed0 & pVM->hm.s.vmx.msr.vmx_cr0_fixed1);
2778	uint64_t uZapCR0 = (pVM->hm.s.vmx.msr.vmx_cr0_fixed0 \| pVM->hm.s.vmx.msr.vmx_cr0_fixed1);
2779	if (pVM->hm.s.vmx.fUnrestrictedGuest) /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG). */
2780	uSetCR0 &= ~(X86_CR0_PE \| X86_CR0_PG);
2781	else
2782	Assert((uSetCR0 & (X86_CR0_PE \| X86_CR0_PG)) == (X86_CR0_PE \| X86_CR0_PG));
2783
2784	u64GuestCR0 \|= uSetCR0;
2785	u64GuestCR0 &= uZapCR0;
2786	u64GuestCR0 &= ~(X86_CR0_CD \| X86_CR0_NW); /* Always enable caching. */
2787
2788	/* Write VT-x's view of the guest CR0 into the VMCS and update the exception bitmap. */
2789	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR0, u64GuestCR0);
2790	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
2791	Log2(("VMX_VMCS_GUEST_CR0=%#RX32\n", (uint32_t)u64GuestCR0));
2792
2793	/*
2794	* CR0 is shared between host and guest along with a CR0 read shadow. Therefore, certain bits must not be changed
2795	* by the guest because VT-x ignores saving/restoring them (namely CD, ET, NW) and for certain other bits
2796	* we want to be notified immediately of guest CR0 changes (e.g. PG to update our shadow page tables).
2797	*/
2798	uint64_t u64CR0Mask = 0;
2799	u64CR0Mask = X86_CR0_PE
2800	\| X86_CR0_WP
2801	\| X86_CR0_PG
2802	\| X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
2803	\| X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
2804	\| X86_CR0_NW; /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
2805
2806	/* We don't need to intercept changes to CR0.PE with unrestricted guests. */
2807	if (pVM->hm.s.vmx.fUnrestrictedGuest)
2808	u64CR0Mask &= ~X86_CR0_PE;
2809
2810	/* If the guest FPU state is active, don't need to VM-exit on writes to FPU related bits in CR0. */
2811	if (fInterceptNM)
2812	u64CR0Mask \|= (X86_CR0_TS \| X86_CR0_MP);
2813	else
2814	u64CR0Mask &= ~(X86_CR0_TS \| X86_CR0_MP);
2815
2816	/* Write the CR0 mask into the VMCS and update the VCPU's copy of the current CR0 mask. */
2817	pVCpu->hm.s.vmx.cr0_mask = u64CR0Mask;
2818	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, u64CR0Mask);
2819	AssertRCReturn(rc, rc);
2820
2821	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_CR0;
2822	}
2823
2824	/*
2825	* Guest CR2.
2826	* It's always loaded in the assembler code. Nothing to do here.
2827	*/
2828
2829	/*
2830	* Guest CR3.
2831	*/
2832	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR3)
2833	{
2834	uint64_t u64GuestCR3 = 0;
2835	if (pVM->hm.s.fNestedPaging)
2836	{
2837	pVCpu->hm.s.vmx.HCPhysEPTP = PGMGetHyperCR3(pVCpu);
2838
2839	/* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
2840	Assert(pVCpu->hm.s.vmx.HCPhysEPTP);
2841	Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & 0xfff0000000000000ULL));
2842	Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & 0xfff));
2843
2844	/* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
2845	pVCpu->hm.s.vmx.HCPhysEPTP \|= VMX_EPT_MEMTYPE_WB
2846	\| (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
2847
2848	/* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
2849	AssertMsg( ((pVCpu->hm.s.vmx.HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
2850	&& ((pVCpu->hm.s.vmx.HCPhysEPTP >> 6) & 0x3f) == 0, /* Bits 6:11 MBZ. */
2851	("EPTP %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
2852
2853	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVCpu->hm.s.vmx.HCPhysEPTP);
2854	AssertRCReturn(rc, rc);
2855	Log(("VMX_VMCS64_CTRL_EPTP_FULL=%#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
2856
2857	if ( pVM->hm.s.vmx.fUnrestrictedGuest
2858	\|\| CPUMIsGuestPagingEnabledEx(pCtx))
2859	{
2860	/* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
2861	if (CPUMIsGuestInPAEModeEx(pCtx))
2862	{
2863	rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
2864	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u);
2865	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u);
2866	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u);
2867	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u);
2868	AssertRCReturn(rc, rc);
2869	}
2870
2871	/* The guest's view of its CR3 is unblemished with Nested Paging when the guest is using paging or we
2872	have Unrestricted Execution to handle the guest when it's not using paging. */
2873	u64GuestCR3 = pCtx->cr3;
2874	}
2875	else
2876	{
2877	/*
2878	* The guest is not using paging, but the CPU (VT-x) has to. While the guest thinks it accesses physical memory
2879	* directly, we use our identity-mapped page table to map guest-linear to guest-physical addresses.
2880	* EPT takes care of translating it to host-physical addresses.
2881	*/
2882	RTGCPHYS GCPhys;
2883	Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
2884	Assert(PDMVmmDevHeapIsEnabled(pVM));
2885
2886	/* We obtain it here every time as the guest could have relocated this PCI region. */
2887	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
2888	AssertRCReturn(rc, rc);
2889
2890	u64GuestCR3 = GCPhys;
2891	}
2892	}
2893	else
2894	{
2895	/* Non-nested paging case, just use the hypervisor's CR3. */
2896	u64GuestCR3 = PGMGetHyperCR3(pVCpu);
2897	}
2898
2899	Log2(("VMX_VMCS_GUEST_CR3=%#RX64\n", u64GuestCR3));
2900	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, u64GuestCR3);
2901	AssertRCReturn(rc, rc);
2902
2903	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_CR3;
2904	}
2905
2906	/*
2907	* Guest CR4.
2908	*/
2909	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_CR4)
2910	{
2911	uint64_t u64GuestCR4 = pCtx->cr4;
2912
2913	/* The guest's view of its CR4 is unblemished. */
2914	rc = VMXWriteVmcsGstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, u64GuestCR4);
2915	AssertRCReturn(rc, rc);
2916	Log2(("VMX_VMCS_CTRL_CR4_READ_SHADOW=%#RGv\n", u64GuestCR4));
2917
2918	/* Setup VT-x's view of the guest CR4. */
2919	/*
2920	* If we're emulating real-mode using virtual-8086 mode, we want to redirect software interrupts to the 8086 program
2921	* interrupt handler. Clear the VME bit (the interrupt redirection bitmap is already all 0, see hmR3InitFinalizeR0())
2922	* See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
2923	*/
2924	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
2925	{
2926	Assert(pVM->hm.s.vmx.pRealModeTSS);
2927	Assert(PDMVmmDevHeapIsEnabled(pVM));
2928	u64GuestCR4 &= ~X86_CR4_VME;
2929	}
2930
2931	if (pVM->hm.s.fNestedPaging)
2932	{
2933	if ( !CPUMIsGuestPagingEnabledEx(pCtx)
2934	&& !pVM->hm.s.vmx.fUnrestrictedGuest)
2935	{
2936	/* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
2937	u64GuestCR4 \|= X86_CR4_PSE;
2938	/* Our identity mapping is a 32 bits page directory. */
2939	u64GuestCR4 &= ~X86_CR4_PAE;
2940	}
2941	/* else use guest CR4.*/
2942	}
2943	else
2944	{
2945	/*
2946	* The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
2947	* paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
2948	*/
2949	switch (pVCpu->hm.s.enmShadowMode)
2950	{
2951	case PGMMODE_REAL: /* Real-mode. */
2952	case PGMMODE_PROTECTED: /* Protected mode without paging. */
2953	case PGMMODE_32_BIT: /* 32-bit paging. */
2954	{
2955	u64GuestCR4 &= ~X86_CR4_PAE;
2956	break;
2957	}
2958
2959	case PGMMODE_PAE: /* PAE paging. */
2960	case PGMMODE_PAE_NX: /* PAE paging with NX. */
2961	{
2962	u64GuestCR4 \|= X86_CR4_PAE;
2963	break;
2964	}
2965
2966	case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
2967	case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
2968	#ifdef VBOX_ENABLE_64_BITS_GUESTS
2969	break;
2970	#endif
2971	default:
2972	AssertFailed();
2973	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
2974	}
2975	}
2976
2977	/* We need to set and clear the CR4 specific bits here (mainly the X86_CR4_VMXE bit). */
2978	uint64_t uSetCR4 = (pVM->hm.s.vmx.msr.vmx_cr4_fixed0 & pVM->hm.s.vmx.msr.vmx_cr4_fixed1);
2979	uint64_t uZapCR4 = (pVM->hm.s.vmx.msr.vmx_cr4_fixed0 \| pVM->hm.s.vmx.msr.vmx_cr4_fixed1);
2980	u64GuestCR4 \|= uSetCR4;
2981	u64GuestCR4 &= uZapCR4;
2982
2983	/* Write VT-x's view of the guest CR4 into the VMCS. */
2984	Log2(("VMX_VMCS_GUEST_CR4=%#RGv (Set=%#RX32 Zap=%#RX32)\n", u64GuestCR4, uSetCR4, uZapCR4));
2985	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR4, u64GuestCR4);
2986
2987	/* Setup CR4 mask. CR4 flags owned by the host, if the guest attempts to change them, that would cause a VM exit. */
2988	uint64_t u64CR4Mask = 0;
2989	u64CR4Mask = X86_CR4_VME
2990	\| X86_CR4_PAE
2991	\| X86_CR4_PGE
2992	\| X86_CR4_PSE
2993	\| X86_CR4_VMXE;
2994	pVCpu->hm.s.vmx.cr4_mask = u64CR4Mask;
2995	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, u64CR4Mask);
2996	AssertRCReturn(rc, rc);
2997
2998	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_CR4;
2999	}
3000	return rc;
3001	}
3002
3003
3004	/**
3005	* Loads the guest debug registers into the guest-state area in the VMCS.
3006	* This also sets up whether #DB and MOV DRx accesses cause VM exits.
3007	*
3008	* @returns VBox status code.
3009	* @param pVM Pointer to the VM.
3010	* @param pVCpu Pointer to the VMCPU.
3011	* @param pCtx Pointer to the guest-CPU context.
3012	*
3013	* @remarks No-long-jump zone!!!
3014	*/
3015	DECLINLINE(int) hmR0VmxLoadGuestDebugRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
3016	{
3017	if (!(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_DEBUG))
3018	return VINF_SUCCESS;
3019
3020	#ifdef VBOX_STRICT
3021	/* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
3022	if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_CONTROLS_LOAD_DEBUG)
3023	{
3024	Assert((pCtx->dr[7] & 0xffffffff00000000ULL) == 0); /* upper 32 bits are reserved (MBZ). */
3025	/* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
3026	Assert((pCtx->dr[7] & 0xd800) == 0); /* bits 15, 14, 12, 11 are reserved (MBZ). */
3027	Assert((pCtx->dr[7] & 0x400) == 0x400); /* bit 10 is reserved (MB1). */
3028	}
3029	#endif
3030
3031	int rc = VERR_INTERNAL_ERROR_5;
3032	bool fInterceptDB = false;
3033	bool fInterceptMovDRx = false;
3034	if (DBGFIsStepping(pVCpu))
3035	{
3036	/* If the CPU supports the monitor trap flag, use it for single stepping in DBGF. */
3037	if (pVM->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_TRAP_FLAG)
3038	{
3039	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_TRAP_FLAG;
3040	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls);
3041	AssertRCReturn(rc, rc);
3042	Assert(fInterceptDB == false);
3043	}
3044	else if (pCtx->eflags.Bits.u1TF) /* If the guest is using its TF bit, we cannot single step in DBGF. */
3045	{
3046	Assert(fInterceptDB == false);
3047	/** @todo can we somehow signal DBGF that it cannot single-step instead of
3048	* just continuing? */
3049	}
3050	else
3051	fInterceptDB = true;
3052	}
3053	else
3054	Assert(fInterceptDB == false); /* If we are not single stepping in DBGF, there is no need to intercept #DB. */
3055
3056	/*
3057	* If the guest is using its DRx registers and the host DRx does not yet contain the guest DRx values,
3058	* load the guest DRx registers into the host and don't cause VM-exits on guest's MOV DRx accesses.
3059	* The same for the hypervisor DRx registers, priority is for the guest here.
3060	*/
3061	if ( (pCtx->dr[7] & (X86_DR7_ENABLED_MASK \| X86_DR7_GD))
3062	&& !CPUMIsGuestDebugStateActive(pVCpu))
3063	{
3064	/* Save the host and load the guest debug registers. This will make the guest debug state active. */
3065	rc = CPUMR0LoadGuestDebugState(pVM, pVCpu, pCtx, true /* include DR6 */);
3066	AssertRC(rc);
3067	Assert(CPUMIsGuestDebugStateActive(pVCpu));
3068	Assert(fInterceptMovDRx == false);
3069	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
3070	}
3071	else if ( CPUMGetHyperDR7(pVCpu) & (X86_DR7_ENABLED_MASK \| X86_DR7_GD)
3072	&& !CPUMIsHyperDebugStateActive(pVCpu))
3073	{
3074	/* Save the host and load the hypervisor debug registers. This will make the hyper debug state active. */
3075	rc = CPUMR0LoadHyperDebugState(pVM, pVCpu, pCtx, true /* include DR6 */);
3076	AssertRC(rc);
3077	Assert(CPUMIsHyperDebugStateActive(pVCpu));
3078	fInterceptMovDRx = true;
3079	}
3080	else
3081	Assert(fInterceptMovDRx == false); /* No need to intercept MOV DRx if DBGF is not active nor the guest is debugging. */
3082
3083	/* Update the exception bitmap regarding intercepting #DB generated by the guest. */
3084	if (fInterceptDB)
3085	pVCpu->hm.s.vmx.u32XcptBitmap \|= RT_BIT(X86_XCPT_DB);
3086	else if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3087	{
3088	#ifndef VBOX_ALWAYS_TRAP_ALL_EXCEPTIONS
3089	pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_DB);
3090	#endif
3091	}
3092
3093	/* Update the processor-based VM-execution controls regarding intercepting MOV DRx instructions. */
3094	if (fInterceptMovDRx)
3095	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT;
3096	else
3097	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT;
3098
3099	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
3100	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls);
3101
3102	/* The guest's view of its DR7 is unblemished. */
3103	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_DR7, pCtx->dr[7]);
3104
3105	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_DEBUG;
3106	return rc;
3107	}
3108
3109
3110	#ifdef VBOX_STRICT
3111	/**
3112	* Strict function to validate segment registers.
3113	*/
3114	static void hmR0VmxValidateSegmentRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
3115	{
3116	/* Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers". */
3117	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
3118	&& ( !CPUMIsGuestInRealModeEx(pCtx)
3119	&& !CPUMIsGuestInV86ModeEx(pCtx)))
3120	{
3121	/* Protected mode checks */
3122	/* CS */
3123	Assert(pCtx->cs.Attr.n.u1Present);
3124	Assert(!(pCtx->cs.Attr.u & 0xf00));
3125	Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
3126	Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
3127	\|\| !(pCtx->cs.Attr.n.u1Granularity));
3128	Assert( !(pCtx->cs.u32Limit & 0xfff00000)
3129	\|\| (pCtx->cs.Attr.n.u1Granularity));
3130	Assert(pCtx->cs.Attr.u && pCtx->cs.Attr.u != VMX_SEL_UNUSABLE); /* CS cannot be loaded with NULL in protected mode. */
3131	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
3132	Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
3133	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
3134	Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
3135	else
3136	AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
3137	/* SS */
3138	if (pCtx->ss.Attr.u && pCtx->ss.Attr.u != VMX_SEL_UNUSABLE)
3139	{
3140	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
3141	Assert(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7);
3142	Assert(pCtx->ss.Attr.n.u1Present);
3143	Assert(!(pCtx->ss.Attr.u & 0xf00));
3144	Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
3145	Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
3146	\|\| !(pCtx->ss.Attr.n.u1Granularity));
3147	Assert( !(pCtx->ss.u32Limit & 0xfff00000)
3148	\|\| (pCtx->ss.Attr.n.u1Granularity));
3149	}
3150	Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
3151	/* CR0 might not be up-to-date here always, hence disabled. */
3152	#if 0
3153	if (!pCtx->cr0 & X86_CR0_PE)
3154	Assert(!pCtx->ss.Attr.n.u2Dpl);
3155	#endif
3156	/* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
3157	if (pCtx->ds.Attr.u && pCtx->ds.Attr.u != VMX_SEL_UNUSABLE)
3158	{
3159	Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3160	Assert(pCtx->ds.Attr.n.u1Present);
3161	Assert(pCtx->ds.Attr.n.u4Type > 11 \|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
3162	Assert(!(pCtx->ds.Attr.u & 0xf00));
3163	Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
3164	Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
3165	\|\| !(pCtx->ds.Attr.n.u1Granularity));
3166	Assert( !(pCtx->ds.u32Limit & 0xfff00000)
3167	\|\| (pCtx->ds.Attr.n.u1Granularity));
3168	Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3169	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
3170	}
3171	if (pCtx->es.Attr.u && pCtx->es.Attr.u != VMX_SEL_UNUSABLE)
3172	{
3173	Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3174	Assert(pCtx->es.Attr.n.u1Present);
3175	Assert(pCtx->es.Attr.n.u4Type > 11 \|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
3176	Assert(!(pCtx->es.Attr.u & 0xf00));
3177	Assert(!(pCtx->es.Attr.u & 0xfffe0000));
3178	Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
3179	\|\| !(pCtx->es.Attr.n.u1Granularity));
3180	Assert( !(pCtx->es.u32Limit & 0xfff00000)
3181	\|\| (pCtx->es.Attr.n.u1Granularity));
3182	Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3183	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
3184	}
3185	if (pCtx->fs.Attr.u && pCtx->fs.Attr.u != VMX_SEL_UNUSABLE)
3186	{
3187	Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3188	Assert(pCtx->fs.Attr.n.u1Present);
3189	Assert(pCtx->fs.Attr.n.u4Type > 11 \|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
3190	Assert(!(pCtx->fs.Attr.u & 0xf00));
3191	Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
3192	Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
3193	\|\| !(pCtx->fs.Attr.n.u1Granularity));
3194	Assert( !(pCtx->fs.u32Limit & 0xfff00000)
3195	\|\| (pCtx->fs.Attr.n.u1Granularity));
3196	Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3197	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
3198	}
3199	if (pCtx->gs.Attr.u && pCtx->gs.Attr.u != VMX_SEL_UNUSABLE)
3200	{
3201	Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
3202	Assert(pCtx->gs.Attr.n.u1Present);
3203	Assert(pCtx->gs.Attr.n.u4Type > 11 \|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
3204	Assert(!(pCtx->gs.Attr.u & 0xf00));
3205	Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
3206	Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
3207	\|\| !(pCtx->gs.Attr.n.u1Granularity));
3208	Assert( !(pCtx->gs.u32Limit & 0xfff00000)
3209	\|\| (pCtx->gs.Attr.n.u1Granularity));
3210	Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
3211	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
3212	}
3213	/* 64-bit capable CPUs. */
3214	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3215	Assert(!(pCtx->cs.u64Base >> 32));
3216	Assert(!pCtx->ss.Attr.u \|\| !(pCtx->ss.u64Base >> 32));
3217	Assert(!pCtx->ds.Attr.u \|\| !(pCtx->ds.u64Base >> 32));
3218	Assert(!pCtx->es.Attr.u \|\| !(pCtx->es.u64Base >> 32));
3219	# endif
3220	}
3221	else if ( CPUMIsGuestInV86ModeEx(pCtx)
3222	\|\| ( CPUMIsGuestInRealModeEx(pCtx)
3223	&& !pVM->hm.s.vmx.fUnrestrictedGuest))
3224	{
3225	/* Real and v86 mode checks. */
3226	/* hmR0VmxWriteSegmentReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
3227	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
3228	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3229	{
3230	u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3; u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
3231	}
3232	else
3233	{
3234	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
3235	u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
3236	}
3237
3238	/* CS */
3239	AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
3240	Assert(pCtx->cs.u32Limit == 0xffff);
3241	Assert(u32CSAttr == 0xf3);
3242	/* SS */
3243	Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
3244	Assert(pCtx->ss.u32Limit == 0xffff);
3245	Assert(u32SSAttr == 0xf3);
3246	/* DS */
3247	Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
3248	Assert(pCtx->ds.u32Limit == 0xffff);
3249	Assert(u32DSAttr == 0xf3);
3250	/* ES */
3251	Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
3252	Assert(pCtx->es.u32Limit == 0xffff);
3253	Assert(u32ESAttr == 0xf3);
3254	/* FS */
3255	Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
3256	Assert(pCtx->fs.u32Limit == 0xffff);
3257	Assert(u32FSAttr == 0xf3);
3258	/* GS */
3259	Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
3260	Assert(pCtx->gs.u32Limit == 0xffff);
3261	Assert(u32GSAttr == 0xf3);
3262	/* 64-bit capable CPUs. */
3263	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3264	Assert(!(pCtx->cs.u64Base >> 32));
3265	Assert(!u32SSAttr \|\| !(pCtx->ss.u64Base >> 32));
3266	Assert(!u32DSAttr \|\| !(pCtx->ds.u64Base >> 32));
3267	Assert(!u32ESAttr \|\| !(pCtx->es.u64Base >> 32));
3268	# endif
3269	}
3270	}
3271	#endif /* VBOX_STRICT */
3272
3273
3274	/**
3275	* Writes a guest segment register into the guest-state area in the VMCS.
3276	*
3277	* @returns VBox status code.
3278	* @param pVM Pointer to the VM.
3279	* @param pVCpu Pointer to the VMCPU.
3280	* @param idxSel Index of the selector in the VMCS.
3281	* @param idxLimit Index of the segment limit in the VMCS.
3282	* @param idxBase Index of the segment base in the VMCS.
3283	* @param idxAccess Index of the access rights of the segment in the VMCS.
3284	* @param pSelReg Pointer to the segment selector.
3285	* @param pCtx Pointer to the guest-CPU context.
3286	*
3287	* @remarks No-long-jump zone!!!
3288	*/
3289	DECLINLINE(int) hmR0VmxWriteSegmentReg(PVM pVM, PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase,
3290	uint32_t idxAccess, PCPUMSELREG pSelReg, PCPUMCTX pCtx)
3291	{
3292	int rc;
3293	rc = VMXWriteVmcs32(idxSel, pSelReg->Sel); /* 16-bit guest selector field. */
3294	rc \|= VMXWriteVmcs32(idxLimit, pSelReg->u32Limit); /* 32-bit guest segment limit field. */
3295	rc \|= VMXWriteVmcsGstN(idxBase, pSelReg->u64Base); /* Natural width guest segment base field.*/
3296	AssertRCReturn(rc, rc);
3297
3298	uint32_t u32Access = pSelReg->Attr.u;
3299	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3300	{
3301	/* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
3302	u32Access = 0xf3;
3303	Assert(pVM->hm.s.vmx.pRealModeTSS);
3304	Assert(PDMVmmDevHeapIsEnabled(pVM));
3305	}
3306	else
3307	{
3308	/*
3309	* The way to differentiate between whether this is really a null selector or was just a selector loaded with 0 in
3310	* real-mode is using the segment attributes. A selector loaded in real-mode with the value 0 is valid and usable in
3311	* protected-mode and we should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures NULL selectors
3312	* loaded in protected-mode have their attribute as 0.
3313	*/
3314	if (!u32Access)
3315	u32Access = VMX_SEL_UNUSABLE;
3316	}
3317
3318	/* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
3319	AssertMsg((u32Access == VMX_SEL_UNUSABLE) \|\| (u32Access & X86_SEL_TYPE_ACCESSED),
3320	("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
3321
3322	rc = VMXWriteVmcs32(idxAccess, u32Access); /* 32-bit guest segment access-rights field. */
3323	AssertRCReturn(rc, rc);
3324	return rc;
3325	}
3326
3327
3328	/**
3329	* Loads the guest segment registers, GDTR, IDTR, LDTR, (TR, FS and GS bases)
3330	* into the guest-state area in the VMCS.
3331	*
3332	* @returns VBox status code.
3333	* @param pVM Pointer to the VM.
3334	* @param pVCPU Pointer to the VMCPU.
3335	* @param pCtx Pointer to the guest-CPU context.
3336	*
3337	* @remarks No-long-jump zone!!!
3338	*/
3339	DECLINLINE(int) hmR0VmxLoadGuestSegmentRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
3340	{
3341	int rc = VERR_INTERNAL_ERROR_5;
3342
3343	/*
3344	* Guest Segment registers: CS, SS, DS, ES, FS, GS.
3345	*/
3346	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SEGMENT_REGS)
3347	{
3348	/* Save the segment attributes for real-on-v86 mode hack, so we can restore them on VM-exit. */
3349	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3350	{
3351	pVCpu->hm.s.vmx.RealMode.uAttrCS.u = pCtx->cs.Attr.u;
3352	pVCpu->hm.s.vmx.RealMode.uAttrSS.u = pCtx->ss.Attr.u;
3353	pVCpu->hm.s.vmx.RealMode.uAttrDS.u = pCtx->ds.Attr.u;
3354	pVCpu->hm.s.vmx.RealMode.uAttrES.u = pCtx->es.Attr.u;
3355	pVCpu->hm.s.vmx.RealMode.uAttrFS.u = pCtx->fs.Attr.u;
3356	pVCpu->hm.s.vmx.RealMode.uAttrGS.u = pCtx->gs.Attr.u;
3357	}
3358
3359	#ifdef VBOX_WITH_REM
3360	if (!pVM->hm.s.vmx.fUnrestrictedGuest)
3361	{
3362	Assert(pVM->hm.s.vmx.pRealModeTSS);
3363	PGMMODE enmGuestMode = PGMGetGuestMode(pVCpu);
3364	if (pVCpu->hm.s.vmx.enmLastSeenGuestMode != enmGuestMode)
3365	{
3366	AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
3367	if ( pVCpu->hm.s.vmx.enmLastSeenGuestMode == PGMMODE_REAL
3368	&& enmGuestMode >= PGMMODE_PROTECTED)
3369	{
3370	/* Signal that recompiler must flush its code-cache as the guest -may- rewrite code it will later execute
3371	in real-mode (e.g. OpenBSD 4.0) */
3372	REMFlushTBs(pVM);
3373	Log2(("Switch to protected mode detected!\n"));
3374	}
3375	pVCpu->hm.s.vmx.enmLastSeenGuestMode = enmGuestMode;
3376	}
3377	}
3378	#endif
3379	rc = hmR0VmxWriteSegmentReg(pVM, pVCpu, VMX_VMCS16_GUEST_FIELD_CS, VMX_VMCS32_GUEST_CS_LIMIT, VMX_VMCS_GUEST_CS_BASE,
3380	VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS, &pCtx->cs, pCtx);
3381	rc \|= hmR0VmxWriteSegmentReg(pVM, pVCpu, VMX_VMCS16_GUEST_FIELD_SS, VMX_VMCS32_GUEST_SS_LIMIT, VMX_VMCS_GUEST_SS_BASE,
3382	VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS, &pCtx->ss, pCtx);
3383	rc \|= hmR0VmxWriteSegmentReg(pVM, pVCpu, VMX_VMCS16_GUEST_FIELD_DS, VMX_VMCS32_GUEST_DS_LIMIT, VMX_VMCS_GUEST_DS_BASE,
3384	VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS, &pCtx->ds, pCtx);
3385	rc \|= hmR0VmxWriteSegmentReg(pVM, pVCpu, VMX_VMCS16_GUEST_FIELD_ES, VMX_VMCS32_GUEST_ES_LIMIT, VMX_VMCS_GUEST_ES_BASE,
3386	VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS, &pCtx->es, pCtx);
3387	rc \|= hmR0VmxWriteSegmentReg(pVM, pVCpu, VMX_VMCS16_GUEST_FIELD_FS, VMX_VMCS32_GUEST_FS_LIMIT, VMX_VMCS_GUEST_FS_BASE,
3388	VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS, &pCtx->fs, pCtx);
3389	rc \|= hmR0VmxWriteSegmentReg(pVM, pVCpu, VMX_VMCS16_GUEST_FIELD_GS, VMX_VMCS32_GUEST_GS_LIMIT, VMX_VMCS_GUEST_GS_BASE,
3390	VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS, &pCtx->gs, pCtx);
3391	AssertRCReturn(rc, rc);
3392
3393	#ifdef VBOX_STRICT
3394	hmR0VmxValidateSegmentRegs(pVM, pVCpu, pCtx);
3395	#endif
3396	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SEGMENT_REGS;
3397	}
3398
3399	/*
3400	* Guest TR.
3401	*/
3402	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_TR)
3403	{
3404	/*
3405	* Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is achieved
3406	* using the interrupt redirection bitmap (all bits cleared to let the guest handle INT-n's) in the TSS.
3407	* See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
3408	*/
3409	uint16_t u16Sel = 0;
3410	uint32_t u32Limit = 0;
3411	uint64_t u64Base = 0;
3412	uint32_t u32AccessRights = 0;
3413
3414	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3415	{
3416	u16Sel = pCtx->tr.Sel;
3417	u32Limit = pCtx->tr.u32Limit;
3418	u64Base = pCtx->tr.u64Base;
3419	u32AccessRights = pCtx->tr.Attr.u;
3420	}
3421	else
3422	{
3423	Assert(pVM->hm.s.vmx.pRealModeTSS);
3424	Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMR3CanExecuteGuest() -XXX- what about inner loop changes? */
3425
3426	/* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
3427	RTGCPHYS GCPhys;
3428	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
3429	AssertRCReturn(rc, rc);
3430
3431	X86DESCATTR DescAttr;
3432	DescAttr.u = 0;
3433	DescAttr.n.u1Present = 1;
3434	DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
3435
3436	u16Sel = 0;
3437	u32Limit = HM_VTX_TSS_SIZE;
3438	u64Base = GCPhys; /* in real-mode phys = virt. */
3439	u32AccessRights = DescAttr.u;
3440	}
3441
3442	/* Validate. */
3443	Assert(!(u16Sel & RT_BIT(2)));
3444	AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
3445	\|\| (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
3446	AssertMsg(!(u32AccessRights & VMX_SEL_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
3447	Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
3448	Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
3449	Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
3450	Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
3451	Assert( (u32Limit & 0xfff) == 0xfff
3452	\|\| !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
3453	Assert( !(pCtx->tr.u32Limit & 0xfff00000)
3454	\|\| (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
3455
3456	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_TR, u16Sel);
3457	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit);
3458	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base);
3459	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights);
3460	AssertRCReturn(rc, rc);
3461
3462	Log2(("VMX_VMCS_GUEST_TR_BASE=%#RX64\n", u64Base));
3463	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_TR;
3464	}
3465
3466	/*
3467	* Guest GDTR.
3468	*/
3469	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_GDTR)
3470	{
3471	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pCtx->gdtr.cbGdt);
3472	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pCtx->gdtr.pGdt);
3473	AssertRCReturn(rc, rc);
3474
3475	Assert(!(pCtx->gdtr.cbGdt & 0xffff0000ULL)); /* Bits 31:16 MBZ. */
3476	Log2(("VMX_VMCS_GUEST_GDTR_BASE=%#RX64\n", pCtx->gdtr.pGdt));
3477	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_GDTR;
3478	}
3479
3480	/*
3481	* Guest LDTR.
3482	*/
3483	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_LDTR)
3484	{
3485	/* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
3486	uint32_t u32Access = 0;
3487	if (!pCtx->ldtr.Attr.u)
3488	u32Access = VMX_SEL_UNUSABLE;
3489	else
3490	u32Access = pCtx->ldtr.Attr.u;
3491
3492	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_LDTR, pCtx->ldtr.Sel);
3493	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pCtx->ldtr.u32Limit);
3494	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pCtx->ldtr.u64Base);
3495	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access);
3496	AssertRCReturn(rc, rc);
3497
3498	/* Validate. */
3499	if (!(u32Access & VMX_SEL_UNUSABLE))
3500	{
3501	Assert(!(pCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
3502	Assert(pCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
3503	Assert(!pCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
3504	Assert(pCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
3505	Assert(!pCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
3506	Assert(!(pCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
3507	Assert( (pCtx->ldtr.u32Limit & 0xfff) == 0xfff
3508	\|\| !pCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
3509	Assert( !(pCtx->ldtr.u32Limit & 0xfff00000)
3510	\|\| pCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
3511	}
3512
3513	Log2(("VMX_VMCS_GUEST_LDTR_BASE=%#RX64\n", pCtx->ldtr.u64Base));
3514	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_LDTR;
3515	}
3516
3517	/*
3518	* Guest IDTR.
3519	*/
3520	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_IDTR)
3521	{
3522	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pCtx->idtr.cbIdt);
3523	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pCtx->idtr.pIdt);
3524	AssertRCReturn(rc, rc);
3525
3526	Assert(!(pCtx->idtr.cbIdt & 0xffff0000ULL)); /* Bits 31:16 MBZ. */
3527	Log2(("VMX_VMCS_GUEST_IDTR_BASE=%#RX64\n", pCtx->idtr.pIdt));
3528	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_IDTR;
3529	}
3530
3531	/*
3532	* Guest FS & GS base MSRs.
3533	* We already initialized the FS & GS base as part of the guest segment registers, but the guest's FS/GS base
3534	* MSRs might have changed (e.g. due to WRMSR) and we need to update the bases if that happened. These MSRs
3535	* are only available in 64-bit mode.
3536	*/
3537	/** @todo Avoid duplication of this code in assembly (see MYPUSHSEGS) - it
3538	* should not be necessary to do it in assembly again. */
3539	if (CPUMIsGuestInLongModeEx(pCtx))
3540	{
3541	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_FS_BASE_MSR)
3542	{
3543	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_FS_BASE, pCtx->fs.u64Base);
3544	AssertRCReturn(rc, rc);
3545	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_FS_BASE_MSR;
3546	}
3547
3548	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_GS_BASE_MSR)
3549	{
3550	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_GS_BASE, pCtx->gs.u64Base);
3551	AssertRCReturn(rc, rc);
3552	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_GS_BASE_MSR;
3553	}
3554	}
3555	else
3556	pVCpu->hm.s.fContextUseFlags &= ~(HM_CHANGED_GUEST_FS_BASE_MSR \| HM_CHANGED_GUEST_GS_BASE_MSR);
3557
3558	return VINF_SUCCESS;
3559	}
3560
3561
3562	/**
3563	* Loads certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
3564	* areas. These MSRs will automatically be loaded to the host CPU on every
3565	* successful VM entry and stored from the host CPU on every successful VM exit.
3566	* Also loads the sysenter MSRs into the guest-state area in the VMCS.
3567	*
3568	* @returns VBox status code.
3569	* @param pVM Pointer to the VM.
3570	* @param pVCpu Pointer to the VMCPU.
3571	* @param pCtx Pointer to the guest-CPU context.
3572	*
3573	* @remarks No-long-jump zone!!!
3574	*/
3575	DECLINLINE(int) hmR0VmxLoadGuestMsrs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
3576	{
3577	AssertPtr(pVCpu);
3578	AssertPtr(pVCpu->hm.s.vmx.pvGuestMsr);
3579
3580	/*
3581	* MSRs covered by Auto-load/store: EFER, LSTAR, STAR, SF_MASK, TSC_AUX (RDTSCP).
3582	*/
3583	int rc = VINF_SUCCESS;
3584	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_GUEST_AUTO_MSRS)
3585	{
3586	PVMXMSR pGuestMsr = (PVMXMSR)pVCpu->hm.s.vmx.pvGuestMsr;
3587	uint32_t cGuestMsrs = 0;
3588
3589	/* See Intel spec. 4.1.4 "Enumeration of Paging Features by CPUID". */
3590	const bool fSupportsNX = CPUMGetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_NX);
3591	const bool fSupportsLongMode = CPUMGetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_LONG_MODE);
3592	if (fSupportsNX \|\| fSupportsLongMode)
3593	{
3594	/** @todo support save IA32_EFER, i.e.
3595	* VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_GUEST_EFER_MSR, in which case the
3596	* guest EFER need not be part of the VM-entry MSR-load area. */
3597	pGuestMsr->u32IndexMSR = MSR_K6_EFER;
3598	pGuestMsr->u32Reserved = 0;
3599	pGuestMsr->u64Value = pCtx->msrEFER;
3600	/* VT-x will complain if only MSR_K6_EFER_LME is set. See Intel spec. 26.4 "Loading MSRs" for details. */
3601	if (!CPUMIsGuestInLongModeEx(pCtx))
3602	pGuestMsr->u64Value &= ~(MSR_K6_EFER_LMA \| MSR_K6_EFER_LME);
3603	pGuestMsr++; cGuestMsrs++;
3604	if (fSupportsLongMode)
3605	{
3606	pGuestMsr->u32IndexMSR = MSR_K8_LSTAR;
3607	pGuestMsr->u32Reserved = 0;
3608	pGuestMsr->u64Value = pCtx->msrLSTAR; /* 64 bits mode syscall rip */
3609	pGuestMsr++; cGuestMsrs++;
3610	pGuestMsr->u32IndexMSR = MSR_K6_STAR;
3611	pGuestMsr->u32Reserved = 0;
3612	pGuestMsr->u64Value = pCtx->msrSTAR; /* legacy syscall eip, cs & ss */
3613	pGuestMsr++; cGuestMsrs++;
3614	pGuestMsr->u32IndexMSR = MSR_K8_SF_MASK;
3615	pGuestMsr->u32Reserved = 0;
3616	pGuestMsr->u64Value = pCtx->msrSFMASK; /* syscall flag mask */
3617	pGuestMsr++; cGuestMsrs++;
3618	/* The KERNEL_GS_BASE MSR doesn't work reliably with auto load/store. See @bugref{6208} */
3619	#if 0
3620	pGuestMsr->u32IndexMSR = MSR_K8_KERNEL_GS_BASE;
3621	pGuestMsr->u32Reserved = 0;
3622	pGuestMsr->u64Value = pCtx->msrKERNELGSBASE; /* swapgs exchange value */
3623	pGuestMsr++; cGuestMsrs++;
3624	#endif
3625	}
3626	}
3627
3628	/*
3629	* RDTSCP requires the TSC_AUX MSR. Host and guest share the physical MSR. So we have to
3630	* load the guest's copy if the guest can execute RDTSCP without causing VM-exits.
3631	*/
3632	if ( CPUMGetGuestCpuIdFeature(pVM, CPUMCPUIDFEATURE_RDTSCP)
3633	&& (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP))
3634	{
3635	pGuestMsr->u32IndexMSR = MSR_K8_TSC_AUX;
3636	pGuestMsr->u32Reserved = 0;
3637	rc = CPUMQueryGuestMsr(pVCpu, MSR_K8_TSC_AUX, &pGuestMsr->u64Value);
3638	AssertRCReturn(rc, rc);
3639	pGuestMsr++; cGuestMsrs++;
3640	}
3641
3642	/* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
3643	if (cGuestMsrs > MSR_IA32_VMX_MISC_MAX_MSR(pVM->hm.s.vmx.msr.vmx_misc))
3644	{
3645	LogRel(("CPU autoload/store MSR count in VMCS exceeded cGuestMsrs=%u.\n", cGuestMsrs));
3646	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3647	}
3648
3649	/* Update the VCPU's copy of the guest MSR count. */
3650	pVCpu->hm.s.vmx.cGuestMsrs = cGuestMsrs;
3651	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cGuestMsrs);
3652	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cGuestMsrs);
3653	AssertRCReturn(rc, rc);
3654
3655	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_GUEST_AUTO_MSRS;
3656	}
3657
3658	/*
3659	* Guest Sysenter MSRs.
3660	* These flags are only set when MSR-bitmaps are not supported by the CPU and we cause
3661	* VM exits on WRMSRs for these MSRs.
3662	*/
3663	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SYSENTER_CS_MSR)
3664	{
3665	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pCtx->SysEnter.cs);
3666	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SYSENTER_CS_MSR;
3667	}
3668	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SYSENTER_EIP_MSR)
3669	{
3670	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pCtx->SysEnter.eip);
3671	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SYSENTER_EIP_MSR;
3672	}
3673	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_GUEST_SYSENTER_ESP_MSR)
3674	{
3675	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pCtx->SysEnter.esp);
3676	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_GUEST_SYSENTER_ESP_MSR;
3677	}
3678	AssertRCReturn(rc, rc);
3679
3680	return rc;
3681	}
3682
3683
3684	/**
3685	* Loads the guest activity state into the guest-state area in the VMCS.
3686	*
3687	* @returns VBox status code.
3688	* @param pVM Pointer to the VM.
3689	* @param pVCpu Pointer to the VMCPU.
3690	* @param pCtx Pointer to the guest-CPU context.
3691	*
3692	* @remarks No-long-jump zone!!!
3693	*/
3694	DECLINLINE(int) hmR0VmxLoadGuestActivityState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
3695	{
3696	/** @todo See if we can make use of other states, e.g.
3697	* VMX_VMCS_GUEST_ACTIVITY_SHUTDOWN or HLT. */
3698	int rc = VINF_SUCCESS;
3699	if (pVCpu->hm.s.fContextUseFlags & HM_CHANGED_VMX_GUEST_ACTIVITY_STATE)
3700	{
3701	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, VMX_VMCS_GUEST_ACTIVITY_ACTIVE);
3702	AssertRCReturn(rc, rc);
3703	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_VMX_GUEST_ACTIVITY_STATE;
3704	}
3705	return rc;
3706	}
3707
3708
3709	/**
3710	* Sets up the appropriate function to run guest code.
3711	*
3712	* @returns VBox status code.
3713	* @param pVM Pointer to the VM.
3714	* @param pVCpu Pointer to the VMCPU.
3715	* @param pCtx Pointer to the guest-CPU context.
3716	*
3717	* @remarks No-long-jump zone!!!
3718	*/
3719	DECLINLINE(int) hmR0VmxSetupVMRunHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
3720	{
3721	if (CPUMIsGuestInLongModeEx(pCtx))
3722	{
3723	#ifndef VBOX_ENABLE_64_BITS_GUESTS
3724	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
3725	#endif
3726	Assert(pVM->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
3727	#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3728	/* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
3729	pVCpu->hm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
3730	#else
3731	/* 64-bit host or hybrid host. */
3732	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM64;
3733	#endif
3734	}
3735	else
3736	{
3737	/* Guest is not in long mode, use the 32-bit handler. */
3738	pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
3739	}
3740	Assert(pVCpu->hm.s.vmx.pfnStartVM);
3741	return VINF_SUCCESS;
3742	}
3743
3744
3745	/**
3746	* Wrapper for running the guest code in VT-x.
3747	*
3748	* @returns VBox strict status code.
3749	* @param pVM Pointer to the VM.
3750	* @param pVCpu Pointer to the VMCPU.
3751	* @param pCtx Pointer to the guest-CPU context.
3752	*
3753	* @remarks No-long-jump zone!!!
3754	*/
3755	DECLINLINE(int) hmR0VmxRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
3756	{
3757	/*
3758	* 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses floating-point operations
3759	* using SSE instructions. Some XMM registers (XMM6-XMM15) are callee-saved and thus the need for this XMM wrapper.
3760	* Refer MSDN docs. "Configuring Programs for 64-bit / x64 Software Conventions / Register Usage" for details.
3761	*/
3762	#ifdef VBOX_WITH_KERNEL_USING_XMM
3763	return hmR0VMXStartVMWrapXMM(pVCpu->hm.s.fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hm.s.vmx.pfnStartVM);
3764	#else
3765	return pVCpu->hm.s.vmx.pfnStartVM(pVCpu->hm.s.fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu);
3766	#endif
3767	}
3768
3769
3770	/**
3771	* Report world-switch error and dump some useful debug info.
3772	*
3773	* @param pVM Pointer to the VM.
3774	* @param pVCpu Pointer to the VMCPU.
3775	* @param rcVMRun The return code from VMLAUNCH/VMRESUME.
3776	* @param pCtx Pointer to the guest-CPU context.
3777	* @param pVmxTransient Pointer to the VMX transient structure (only
3778	* exitReason updated).
3779	*/
3780	static void hmR0VmxReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, int rcVMRun, PCPUMCTX pCtx, PVMXTRANSIENT pVmxTransient)
3781	{
3782	Assert(pVM);
3783	Assert(pVCpu);
3784	Assert(pCtx);
3785	Assert(pVmxTransient);
3786	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
3787
3788	Log(("VM-entry failure: %Rrc\n", rcVMRun));
3789	switch (rcVMRun)
3790	{
3791	case VERR_VMX_INVALID_VMXON_PTR:
3792	AssertFailed();
3793	break;
3794	case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
3795	case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
3796	{
3797	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.lasterror.u32ExitReason);
3798	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.lasterror.u32InstrError);
3799	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
3800	AssertRC(rc);
3801
3802	#ifdef VBOX_STRICT
3803	Log(("uExitReason %#x (VmxTransient %#x)\n", pVCpu->hm.s.vmx.lasterror.u32ExitReason,
3804	pVmxTransient->uExitReason));
3805	Log(("Exit Qualification %#x\n", pVmxTransient->uExitQualification));
3806	Log(("InstrError %#x\n", pVCpu->hm.s.vmx.lasterror.u32InstrError));
3807	if (pVCpu->hm.s.vmx.lasterror.u32InstrError <= VMX_INSTR_ERROR_MAX)
3808	Log(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.lasterror.u32InstrError]));
3809	else
3810	Log(("InstrError Desc. Range exceeded %u\n", VMX_INSTR_ERROR_MAX));
3811
3812	/* VMX control bits. */
3813	uint32_t u32Val;
3814	uint64_t u64Val;
3815	RTHCUINTREG uHCReg;
3816	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC_CONTROLS, &u32Val); AssertRC(rc);
3817	Log(("VMX_VMCS32_CTRL_PIN_EXEC_CONTROLS %#RX32\n", u32Val));
3818	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, &u32Val); AssertRC(rc);
3819	Log(("VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS %#RX32\n", u32Val));
3820	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS2, &u32Val); AssertRC(rc);
3821	Log(("VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS2 %#RX32\n", u32Val));
3822	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_CONTROLS, &u32Val); AssertRC(rc);
3823	Log(("VMX_VMCS32_CTRL_ENTRY_CONTROLS %#RX32\n", u32Val));
3824	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_CONTROLS, &u32Val); AssertRC(rc);
3825	Log(("VMX_VMCS32_CTRL_EXIT_CONTROLS %#RX32\n", u32Val));
3826	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
3827	Log(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
3828	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
3829	Log(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
3830	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
3831	Log(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
3832	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
3833	Log(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
3834	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
3835	Log(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
3836	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
3837	Log(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
3838	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
3839	Log(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
3840	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
3841	Log(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
3842	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
3843	Log(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
3844	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
3845	Log(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
3846	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
3847	Log(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
3848	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
3849	Log(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
3850	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
3851	Log(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
3852	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
3853	Log(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
3854	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
3855	Log(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
3856	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
3857	Log(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
3858
3859	/* Guest bits. */
3860	RTGCUINTREG uGCReg;
3861	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &uGCReg); AssertRC(rc);
3862	Log(("Old Guest Rip %#RGv New %#RGv\n", (RTGCPTR)pCtx->rip, (RTGCPTR)uGCReg));
3863	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &uGCReg); AssertRC(rc);
3864	Log(("Old Guest Rsp %#RGv New %#RGv\n", (RTGCPTR)pCtx->rsp, (RTGCPTR)uGCReg));
3865	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RFLAGS, &uGCReg); AssertRC(rc);
3866	Log(("Old Guest Rflags %#RGr New %#RGr\n", (RTGCPTR)pCtx->rflags.u64, (RTGCPTR)uGCReg));
3867	rc = VMXReadVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, &u32Val); AssertRC(rc);
3868	Log(("VMX_VMCS16_GUEST_FIELD_VPID %u\n", u32Val));
3869
3870	/* Host bits. */
3871	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
3872	Log(("Host CR0 %#RHr\n", uHCReg));
3873	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
3874	Log(("Host CR3 %#RHr\n", uHCReg));
3875	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
3876	Log(("Host CR4 %#RHr\n", uHCReg));
3877
3878	RTGDTR HostGdtr;
3879	PCX86DESCHC pDesc;
3880	ASMGetGDTR(&HostGdtr);
3881	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_CS, &u32Val);
3882	Log(("Host CS %#08x\n", u32Val));
3883	if (u32Val < HostGdtr.cbGdt)
3884	{
3885	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
3886	HMR0DumpDescriptor(pDesc, u32Val, "CS: ");
3887	}
3888
3889	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_DS, &u32Val); AssertRC(rc);
3890	Log(("Host DS %#08x\n", u32Val));
3891	if (u32Val < HostGdtr.cbGdt)
3892	{
3893	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
3894	HMR0DumpDescriptor(pDesc, u32Val, "DS: ");
3895	}
3896
3897	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_ES, &u32Val); AssertRC(rc);
3898	Log(("Host ES %#08x\n", u32Val));
3899	if (u32Val < HostGdtr.cbGdt)
3900	{
3901	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
3902	HMR0DumpDescriptor(pDesc, u32Val, "ES: ");
3903	}
3904
3905	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_FS, &u32Val); AssertRC(rc);
3906	Log(("Host FS %#08x\n", u32Val));
3907	if (u32Val < HostGdtr.cbGdt)
3908	{
3909	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
3910	HMR0DumpDescriptor(pDesc, u32Val, "FS: ");
3911	}
3912
3913	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_GS, &u32Val); AssertRC(rc);
3914	Log(("Host GS %#08x\n", u32Val));
3915	if (u32Val < HostGdtr.cbGdt)
3916	{
3917	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
3918	HMR0DumpDescriptor(pDesc, u32Val, "GS: ");
3919	}
3920
3921	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_SS, &u32Val); AssertRC(rc);
3922	Log(("Host SS %#08x\n", u32Val));
3923	if (u32Val < HostGdtr.cbGdt)
3924	{
3925	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
3926	HMR0DumpDescriptor(pDesc, u32Val, "SS: ");
3927	}
3928
3929	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_TR, &u32Val); AssertRC(rc);
3930	Log(("Host TR %#08x\n", u32Val));
3931	if (u32Val < HostGdtr.cbGdt)
3932	{
3933	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
3934	HMR0DumpDescriptor(pDesc, u32Val, "TR: ");
3935	}
3936
3937	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
3938	Log(("Host TR Base %#RHv\n", uHCReg));
3939	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
3940	Log(("Host GDTR Base %#RHv\n", uHCReg));
3941	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
3942	Log(("Host IDTR Base %#RHv\n", uHCReg));
3943	rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
3944	Log(("Host SYSENTER CS %#08x\n", u32Val));
3945	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
3946	Log(("Host SYSENTER EIP %#RHv\n", uHCReg));
3947	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
3948	Log(("Host SYSENTER ESP %#RHv\n", uHCReg));
3949	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
3950	Log(("Host RSP %#RHv\n", uHCReg));
3951	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
3952	Log(("Host RIP %#RHv\n", uHCReg));
3953	# if HC_ARCH_BITS == 64 \|\| defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3954	if (VMX_IS_64BIT_HOST_MODE())
3955	{
3956	Log(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
3957	Log(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
3958	Log(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
3959	Log(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
3960	Log(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
3961	Log(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
3962	}
3963	# endif
3964	#endif /* VBOX_STRICT */
3965	break;
3966	}
3967
3968	default:
3969	/* Impossible */
3970	AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
3971	break;
3972	}
3973	NOREF(pVM);
3974	}
3975
3976
3977	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3978	#ifndef VMX_USE_CACHED_VMCS_ACCESSES
3979	# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
3980	#endif
3981
3982	#ifdef VBOX_STRICT
3983	static bool hmR0VmxIsValidReadField(uint32_t idxField)
3984	{
3985	switch (idxField)
3986	{
3987	case VMX_VMCS_GUEST_RIP:
3988	case VMX_VMCS_GUEST_RSP:
3989	case VMX_VMCS_GUEST_RFLAGS:
3990	case VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE:
3991	case VMX_VMCS_CTRL_CR0_READ_SHADOW:
3992	case VMX_VMCS_GUEST_CR0:
3993	case VMX_VMCS_CTRL_CR4_READ_SHADOW:
3994	case VMX_VMCS_GUEST_CR4:
3995	case VMX_VMCS_GUEST_DR7:
3996	case VMX_VMCS32_GUEST_SYSENTER_CS:
3997	case VMX_VMCS_GUEST_SYSENTER_EIP:
3998	case VMX_VMCS_GUEST_SYSENTER_ESP:
3999	case VMX_VMCS32_GUEST_GDTR_LIMIT:
4000	case VMX_VMCS_GUEST_GDTR_BASE:
4001	case VMX_VMCS32_GUEST_IDTR_LIMIT:
4002	case VMX_VMCS_GUEST_IDTR_BASE:
4003	case VMX_VMCS16_GUEST_FIELD_CS:
4004	case VMX_VMCS32_GUEST_CS_LIMIT:
4005	case VMX_VMCS_GUEST_CS_BASE:
4006	case VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS:
4007	case VMX_VMCS16_GUEST_FIELD_DS:
4008	case VMX_VMCS32_GUEST_DS_LIMIT:
4009	case VMX_VMCS_GUEST_DS_BASE:
4010	case VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS:
4011	case VMX_VMCS16_GUEST_FIELD_ES:
4012	case VMX_VMCS32_GUEST_ES_LIMIT:
4013	case VMX_VMCS_GUEST_ES_BASE:
4014	case VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS:
4015	case VMX_VMCS16_GUEST_FIELD_FS:
4016	case VMX_VMCS32_GUEST_FS_LIMIT:
4017	case VMX_VMCS_GUEST_FS_BASE:
4018	case VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS:
4019	case VMX_VMCS16_GUEST_FIELD_GS:
4020	case VMX_VMCS32_GUEST_GS_LIMIT:
4021	case VMX_VMCS_GUEST_GS_BASE:
4022	case VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS:
4023	case VMX_VMCS16_GUEST_FIELD_SS:
4024	case VMX_VMCS32_GUEST_SS_LIMIT:
4025	case VMX_VMCS_GUEST_SS_BASE:
4026	case VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS:
4027	case VMX_VMCS16_GUEST_FIELD_LDTR:
4028	case VMX_VMCS32_GUEST_LDTR_LIMIT:
4029	case VMX_VMCS_GUEST_LDTR_BASE:
4030	case VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS:
4031	case VMX_VMCS16_GUEST_FIELD_TR:
4032	case VMX_VMCS32_GUEST_TR_LIMIT:
4033	case VMX_VMCS_GUEST_TR_BASE:
4034	case VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS:
4035	case VMX_VMCS32_RO_EXIT_REASON:
4036	case VMX_VMCS32_RO_VM_INSTR_ERROR:
4037	case VMX_VMCS32_RO_EXIT_INSTR_LENGTH:
4038	case VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE:
4039	case VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO:
4040	case VMX_VMCS32_RO_EXIT_INSTR_INFO:
4041	case VMX_VMCS_RO_EXIT_QUALIFICATION:
4042	case VMX_VMCS32_RO_IDT_INFO:
4043	case VMX_VMCS32_RO_IDT_ERROR_CODE:
4044	case VMX_VMCS_GUEST_CR3:
4045	case VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL:
4046	return true;
4047	}
4048	return false;
4049	}
4050
4051	static bool hmR0VmxIsValidWriteField(uint32_t idxField)
4052	{
4053	switch (idxField)
4054	{
4055	case VMX_VMCS_GUEST_LDTR_BASE:
4056	case VMX_VMCS_GUEST_TR_BASE:
4057	case VMX_VMCS_GUEST_GDTR_BASE:
4058	case VMX_VMCS_GUEST_IDTR_BASE:
4059	case VMX_VMCS_GUEST_SYSENTER_EIP:
4060	case VMX_VMCS_GUEST_SYSENTER_ESP:
4061	case VMX_VMCS_GUEST_CR0:
4062	case VMX_VMCS_GUEST_CR4:
4063	case VMX_VMCS_GUEST_CR3:
4064	case VMX_VMCS_GUEST_DR7:
4065	case VMX_VMCS_GUEST_RIP:
4066	case VMX_VMCS_GUEST_RSP:
4067	case VMX_VMCS_GUEST_CS_BASE:
4068	case VMX_VMCS_GUEST_DS_BASE:
4069	case VMX_VMCS_GUEST_ES_BASE:
4070	case VMX_VMCS_GUEST_FS_BASE:
4071	case VMX_VMCS_GUEST_GS_BASE:
4072	case VMX_VMCS_GUEST_SS_BASE:
4073	return true;
4074	}
4075	return false;
4076	}
4077	#endif /* VBOX_STRICT */
4078
4079	/**
4080	* Executes the specified handler in 64-bit mode.
4081	*
4082	* @returns VBox status code.
4083	* @param pVM Pointer to the VM.
4084	* @param pVCpu Pointer to the VMCPU.
4085	* @param pCtx Pointer to the guest CPU context.
4086	* @param pfnHandler Pointer to the RC handler function.
4087	* @param cbParam Number of parameters.
4088	* @param paParam Array of 32-bit parameters.
4089	*/
4090	VMMR0DECL(int) VMXR0Execute64BitsHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, RTRCPTR pfnHandler, uint32_t cbParam,
4091	uint32_t *paParam)
4092	{
4093	int rc, rc2;
4094	PHMGLOBLCPUINFO pCpu;
4095	RTHCPHYS HCPhysCpuPage;
4096	RTHCUINTREG uOldEFlags;
4097
4098	AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
4099	Assert(pfnHandler);
4100	Assert(pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Write.aField));
4101	Assert(pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Read.aField));
4102
4103	#ifdef VBOX_STRICT
4104	for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries; i++)
4105	Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VMCSCache.Write.aField[i]));
4106
4107	for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries; i++)
4108	Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VMCSCache.Read.aField[i]));
4109	#endif
4110
4111	/* Disable interrupts. */
4112	uOldEFlags = ASMIntDisableFlags();
4113
4114	#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
4115	RTCPUID idHostCpu = RTMpCpuId();
4116	CPUMR0SetLApic(pVM, idHostCpu);
4117	#endif
4118
4119	pCpu = HMR0GetCurrentCpu();
4120	HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
4121
4122	/* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
4123	VMXClearVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
4124
4125	/* Leave VMX Root Mode. */
4126	VMXDisable();
4127
4128	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
4129
4130	CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
4131	CPUMSetHyperEIP(pVCpu, pfnHandler);
4132	for (int i = (int)cbParam - 1; i >= 0; i--)
4133	CPUMPushHyper(pVCpu, paParam[i]);
4134
4135	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
4136
4137	/* Call the switcher. */
4138	rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_OFFSETOF(VM, aCpus[pVCpu->idCpu].cpum) - RT_OFFSETOF(VM, cpum));
4139	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
4140
4141	/** @todo replace with hmR0VmxEnterRootMode() and LeaveRootMode(). */
4142	/* Make sure the VMX instructions don't cause #UD faults. */
4143	ASMSetCR4(ASMGetCR4() \| X86_CR4_VMXE);
4144
4145	/* Re-enter VMX Root Mode */
4146	rc2 = VMXEnable(HCPhysCpuPage);
4147	if (RT_FAILURE(rc2))
4148	{
4149	ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
4150	ASMSetFlags(uOldEFlags);
4151	return rc2;
4152	}
4153
4154	rc2 = VMXActivateVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
4155	AssertRC(rc2);
4156	Assert(!(ASMGetFlags() & X86_EFL_IF));
4157	ASMSetFlags(uOldEFlags);
4158	return rc;
4159	}
4160
4161
4162	/**
4163	* Prepares for and executes VMLAUNCH (64 bits guests) for 32-bit hosts
4164	* supporting 64-bit guests.
4165	*
4166	* @returns VBox status code.
4167	* @param fResume Whether to VMLAUNCH or VMRESUME.
4168	* @param pCtx Pointer to the guest-CPU context.
4169	* @param pCache Pointer to the VMCS cache.
4170	* @param pVM Pointer to the VM.
4171	* @param pVCpu Pointer to the VMCPU.
4172	*/
4173	DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
4174	{
4175	uint32_t aParam[6];
4176	PHMGLOBLCPUINFO pCpu = NULL;
4177	RTHCPHYS HCPhysCpuPage = 0;
4178	int rc = VERR_INTERNAL_ERROR_5;
4179
4180	pCpu = HMR0GetCurrentCpu();
4181	HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
4182
4183	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4184	pCache->uPos = 1;
4185	pCache->interPD = PGMGetInterPaeCR3(pVM);
4186	pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
4187	#endif
4188
4189	#ifdef VBOX_STRICT
4190	pCache->TestIn.HCPhysCpuPage = 0;
4191	pCache->TestIn.HCPhysVmcs = 0;
4192	pCache->TestIn.pCache = 0;
4193	pCache->TestOut.HCPhysVmcs = 0;
4194	pCache->TestOut.pCache = 0;
4195	pCache->TestOut.pCtx = 0;
4196	pCache->TestOut.eflags = 0;
4197	#endif
4198
4199	aParam[0] = (uint32_t)(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
4200	aParam[1] = (uint32_t)(HCPhysCpuPage >> 32); /* Param 1: VMXON physical address - Hi. */
4201	aParam[2] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
4202	aParam[3] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs >> 32); /* Param 2: VMCS physical address - Hi. */
4203	aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache);
4204	aParam[5] = 0;
4205
4206	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4207	pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
4208	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
4209	#endif
4210	rc = VMXR0Execute64BitsHandler(pVM, pVCpu, pCtx, pVM->hm.s.pfnVMXGCStartVM64, 6, &aParam[0]);
4211
4212	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
4213	Assert((uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
4214	Assert(pCtx->dr[4] == 10);
4215	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
4216	#endif
4217
4218	#ifdef VBOX_STRICT
4219	AssertMsg(pCache->TestIn.HCPhysCpuPage == HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
4220	AssertMsg(pCache->TestIn.HCPhysVmcs == pVCpu->hm.s.vmx.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
4221	pVCpu->hm.s.vmx.HCPhysVmcs));
4222	AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
4223	pCache->TestOut.HCPhysVmcs));
4224	AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
4225	pCache->TestOut.pCache));
4226	AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache),
4227	("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache)));
4228	AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
4229	pCache->TestOut.pCtx));
4230	Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
4231	#endif
4232	return rc;
4233	}
4234
4235
4236	/**
4237	* Initialize the VMCS-Read cache. The VMCS cache is used for 32-bit hosts
4238	* running 64-bit guests (except 32-bit Darwin which runs with 64-bit paging in
4239	* 32-bit mode) for 64-bit fields that cannot be accessed in 32-bit mode. Some
4240	* 64-bit fields -can- be accessed (those that have a 32-bit FULL & HIGH part).
4241	*
4242	* @returns VBox status code.
4243	* @param pVM Pointer to the VM.
4244	* @param pVCpu Pointer to the VMCPU.
4245	*/
4246	static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu)
4247	{
4248	#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
4249	{ \
4250	Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
4251	pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
4252	pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
4253	}
4254
4255	#define VMXLOCAL_INIT_VMCS_SELREG(REG, pCache) \
4256	{ \
4257	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS16_GUEST_FIELD_##REG); \
4258	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS32_GUEST_##REG##_LIMIT); \
4259	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_##REG##_BASE); \
4260	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS32_GUEST_##REG##_ACCESS_RIGHTS); \
4261	}
4262
4263	AssertPtr(pVM);
4264	AssertPtr(pVCpu);
4265	PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
4266
4267	/* 16-bit guest-state fields (16-bit selectors and their corresponding 32-bit limit & 32-bit access-rights fields). */
4268	VMXLOCAL_INIT_VMCS_SELREG(ES, pCache);
4269	VMXLOCAL_INIT_VMCS_SELREG(CS, pCache);
4270	VMXLOCAL_INIT_VMCS_SELREG(SS, pCache);
4271	VMXLOCAL_INIT_VMCS_SELREG(DS, pCache);
4272	VMXLOCAL_INIT_VMCS_SELREG(FS, pCache);
4273	VMXLOCAL_INIT_VMCS_SELREG(GS, pCache);
4274	VMXLOCAL_INIT_VMCS_SELREG(LDTR, pCache);
4275	VMXLOCAL_INIT_VMCS_SELREG(TR, pCache);
4276
4277	/* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for these 64-bit fields (using "FULL" and "HIGH" fields). */
4278	#if 0
4279	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
4280	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
4281	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
4282	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
4283	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
4284	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
4285	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
4286	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
4287	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
4288	#endif
4289
4290	/* 32-bit guest-state fields. */
4291	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS32_GUEST_GDTR_LIMIT);
4292	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS32_GUEST_IDTR_LIMIT);
4293	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE);
4294	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS32_GUEST_SYSENTER_CS);
4295	/* Unused 32-bit guest-state fields. */
4296	#if 0
4297	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS32_GUEST_ACTIVITY_STATE);
4298	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS32_GUEST_SMBASE);
4299	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS32_GUEST_PREEMPT_TIMER_VALUE);
4300	#endif
4301
4302	/* Natural width guest-state fields. */
4303	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
4304	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
4305	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
4306	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
4307	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
4308	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
4309	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
4310	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
4311	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
4312	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
4313	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
4314	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
4315	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DR7);
4316	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
4317	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
4318	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RFLAGS);
4319	/* Unused natural width guest-state fields. */
4320	#if 0
4321	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DEBUG_EXCEPTIONS);
4322	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in Nested Paging case */
4323	#endif
4324	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
4325	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
4326
4327	if (pVM->hm.s.fNestedPaging)
4328	{
4329	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
4330	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL);
4331	pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
4332	}
4333	else
4334	pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
4335
4336	#undef VMXLOCAL_INIT_VMCS_SELREG
4337	#undef VMXLOCAL_INIT_READ_CACHE_FIELD
4338	return VINF_SUCCESS;
4339	}
4340
4341
4342	/**
4343	* Writes a field into the VMCS. This can either directly invoke a VMWRITE or
4344	* queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
4345	* darwin, running 64-bit guests).
4346	*
4347	* @returns VBox status code.
4348	* @param pVCpu Pointer to the VMCPU.
4349	* @param idxField The VMCS field encoding.
4350	* @param u64Val 16, 32 or 64 bits value.
4351	*/
4352	VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
4353	{
4354	int rc;
4355	switch (idxField)
4356	{
4357	/*
4358	* These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
4359	*/
4360	/* 64-bit Control fields. */
4361	case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
4362	case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
4363	case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
4364	case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
4365	case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
4366	case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
4367	case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
4368	case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
4369	case VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL:
4370	case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
4371	case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
4372	case VMX_VMCS64_CTRL_EPTP_FULL:
4373	case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
4374	/* 64-bit Read-only data fields. */
4375	case VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL:
4376	/* 64-bit Guest-state fields. */
4377	case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
4378	case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
4379	case VMX_VMCS64_GUEST_PAT_FULL:
4380	case VMX_VMCS64_GUEST_EFER_FULL:
4381	case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
4382	case VMX_VMCS64_GUEST_PDPTE0_FULL:
4383	case VMX_VMCS64_GUEST_PDPTE1_FULL:
4384	case VMX_VMCS64_GUEST_PDPTE2_FULL:
4385	case VMX_VMCS64_GUEST_PDPTE3_FULL:
4386	/* 64-bit Host-state fields. */
4387	case VMX_VMCS64_HOST_FIELD_PAT_FULL:
4388	case VMX_VMCS64_HOST_FIELD_EFER_FULL:
4389	case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
4390	{
4391	rc = VMXWriteVmcs32(idxField, u64Val);
4392	rc \|= VMXWriteVmcs32(idxField + 1, (uint32_t)(u64Val >> 32ULL)); /* hmpf, do we really need the "ULL" suffix? */
4393	break;
4394	}
4395
4396	/*
4397	* These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
4398	* values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
4399	*/
4400	/* Natural-width Guest-state fields. */
4401	case VMX_VMCS_GUEST_CR0:
4402	case VMX_VMCS_GUEST_CR3:
4403	case VMX_VMCS_GUEST_CR4:
4404	case VMX_VMCS_GUEST_ES_BASE:
4405	case VMX_VMCS_GUEST_CS_BASE:
4406	case VMX_VMCS_GUEST_SS_BASE:
4407	case VMX_VMCS_GUEST_DS_BASE:
4408	case VMX_VMCS_GUEST_FS_BASE:
4409	case VMX_VMCS_GUEST_GS_BASE:
4410	case VMX_VMCS_GUEST_LDTR_BASE:
4411	case VMX_VMCS_GUEST_TR_BASE:
4412	case VMX_VMCS_GUEST_GDTR_BASE:
4413	case VMX_VMCS_GUEST_IDTR_BASE:
4414	case VMX_VMCS_GUEST_DR7:
4415	case VMX_VMCS_GUEST_RSP:
4416	case VMX_VMCS_GUEST_RIP:
4417	case VMX_VMCS_GUEST_RFLAGS:
4418	case VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS:
4419	case VMX_VMCS_GUEST_SYSENTER_ESP:
4420	case VMX_VMCS_GUEST_SYSENTER_EIP:
4421	{
4422	if ((u64Val >> 32ULL) == 0)
4423	{
4424	/* If this field is 64-bit, VT-x will zero out the top bits. */
4425	rc = VMXWriteVmcs32(idxField, (uint32_t)u64Val);
4426	}
4427	else
4428	{
4429	/* Assert that only the 32->64 switcher case should ever come here. */
4430	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests);
4431	rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
4432	}
4433	break;
4434	}
4435
4436	default:
4437	{
4438	AssertMsgFailed(("VMXWriteVmcs64Ex: invalid field %#x (pVCpu=%p u64Val=%RX64)\n", (unsigned)idxField, pVCpu, u64Val));
4439	rc = VERR_INVALID_PARAMETER;
4440	break;
4441	}
4442	}
4443	AssertRCReturn(rc, rc);
4444	return rc;
4445	}
4446
4447
4448	/**
4449	* Queue up a VMWRITE by using the VMCS write cache. This is only used on 32-bit
4450	* hosts (except darwin) for 64-bit guests.
4451	*
4452	* @param pVCpu Pointer to the VMCPU.
4453	* @param idxField The VMCS field encoding.
4454	* @param u64Val 16, 32 or 64 bits value.
4455	*/
4456	VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
4457	{
4458	AssertPtr(pVCpu);
4459	PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
4460
4461	AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1,
4462	("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
4463
4464	/* Make sure there are no duplicates. */
4465	for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
4466	{
4467	if (pCache->Write.aField[i] == idxField)
4468	{
4469	pCache->Write.aFieldVal[i] = u64Val;
4470	return VINF_SUCCESS;
4471	}
4472	}
4473
4474	pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
4475	pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
4476	pCache->Write.cValidEntries++;
4477	return VINF_SUCCESS;
4478	}
4479
4480
4481	/**
4482	* Loads the VMCS write-cache into the CPU (by executing VMWRITEs).
4483	*
4484	* @param pVCpu Pointer to the VMCPU.
4485	* @param pCache Pointer to the VMCS cache.
4486	*
4487	* @remarks No-long-jump zone!!!
4488	*/
4489	VMMR0DECL(void) VMXWriteCachedVmcsLoad(PVMCPU pVCpu, PVMCSCACHE pCache)
4490	{
4491	AssertPtr(pCache);
4492	for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
4493	{
4494	int rc = VMXWriteVmcs64(pCache->Write.aField[i], pCache->Write.aFieldVal[i]);
4495	AssertRC(rc);
4496	}
4497	pCache->Write.cValidEntries = 0;
4498	}
4499
4500
4501	/**
4502	* Stores the VMCS read-cache from the CPU (by executing VMREADs).
4503	*
4504	* @param pVCpu Pointer to the VMCPU.
4505	* @param pCache Pointer to the VMCS cache.
4506	*
4507	* @remarks No-long-jump zone!!!
4508	*/
4509	VMMR0DECL(void) VMXReadCachedVmcsStore(PVMCPU pVCpu, PVMCSCACHE pCache)
4510	{
4511	AssertPtr(pCache);
4512	for (uint32_t i = 0; i < pCache->Read.cValidEntries; i++)
4513	{
4514	int rc = VMXReadVmcs64(pCache->Read.aField[i], &pCache->Read.aFieldVal[i]);
4515	AssertRC(rc);
4516	}
4517	}
4518	#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL) */
4519
4520
4521	/**
4522	* Sets up the usage of TSC-offsetting and updates the VMCS. If offsetting is
4523	* not possible, cause VM-exits on RDTSC(P)s. Also sets up the VMX preemption
4524	* timer.
4525	*
4526	* @returns VBox status code.
4527	* @param pVM Pointer to the VM.
4528	* @param pVCpu Pointer to the VMCPU.
4529	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4530	* out-of-sync. Make sure to update the required fields
4531	* before using them.
4532	* @remarks No-long-jump zone!!!
4533	*/
4534	static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4535	{
4536	int rc = VERR_INTERNAL_ERROR_5;
4537	bool fOffsettedTsc = false;
4538	if (pVM->hm.s.vmx.fUsePreemptTimer)
4539	{
4540	uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVCpu, &fOffsettedTsc, &pVCpu->hm.s.vmx.u64TSCOffset);
4541
4542	/* Make sure the returned values have sane upper and lower boundaries. */
4543	uint64_t u64CpuHz = SUPGetCpuHzFromGIP(g_pSUPGlobalInfoPage);
4544	cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
4545	cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
4546	cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
4547
4548	uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
4549	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_PREEMPT_TIMER_VALUE, cPreemptionTickCount); AssertRC(rc);
4550	}
4551	else
4552	fOffsettedTsc = TMCpuTickCanUseRealTSC(pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset);
4553
4554	if (fOffsettedTsc)
4555	{
4556	uint64_t u64CurTSC = ASMReadTSC();
4557	if (u64CurTSC + pVCpu->hm.s.vmx.u64TSCOffset >= TMCpuTickGetLastSeen(pVCpu))
4558	{
4559	/* Note: VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT takes precedence over TSC_OFFSET, applies to RDTSCP too. */
4560	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, pVCpu->hm.s.vmx.u64TSCOffset); AssertRC(rc);
4561
4562	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT;
4563	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
4564	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
4565	}
4566	else
4567	{
4568	/* VM-exit on RDTSC(P) as we would otherwise pass decreasing TSC values to the guest. */
4569	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT;
4570	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
4571	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscInterceptOverFlow);
4572	}
4573	}
4574	else
4575	{
4576	/* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
4577	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT;
4578	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
4579	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
4580	}
4581	}
4582
4583
4584	/**
4585	* Determines if an exception is a contributory exception. Contributory
4586	* exceptions are ones which can cause double-faults.
4587	*
4588	* @returns true if the exception is contributory, false otherwise.
4589	* @param uVector The exception vector.
4590	*/
4591	DECLINLINE(bool) hmR0VmxIsContributoryXcpt(const uint32_t uVector)
4592	{
4593	switch (uVector)
4594	{
4595	case X86_XCPT_GP:
4596	case X86_XCPT_SS:
4597	case X86_XCPT_NP:
4598	case X86_XCPT_TS:
4599	case X86_XCPT_DE:
4600	return true;
4601	default:
4602	break;
4603	}
4604	return false;
4605	}
4606
4607
4608	/**
4609	* Determines if an exception is a benign exception. Benign exceptions
4610	* are ones which cannot cause double-faults.
4611	*
4612	* @returns true if the exception is benign, false otherwise.
4613	* @param uVector The exception vector.
4614	*/
4615	DECLINLINE(bool) hmR0VmxIsBenignXcpt(const uint32_t uVector)
4616	{
4617	if ( uVector == X86_XCPT_PF
4618	\|\| uVector == X86_XCPT_DF)
4619	{
4620	return false;
4621	}
4622
4623	bool fIsBenignXcpt = !hmR0VmxIsContributoryXcpt(uVector);
4624	#ifdef VBOX_STRICT
4625	switch (uVector)
4626	{
4627	case X86_XCPT_DB:
4628	case X86_XCPT_NMI:
4629	case X86_XCPT_BP:
4630	case X86_XCPT_OF:
4631	case X86_XCPT_BR:
4632	case X86_XCPT_UD:
4633	case X86_XCPT_NM:
4634	case X86_XCPT_CO_SEG_OVERRUN:
4635	case X86_XCPT_MF:
4636	case X86_XCPT_AC:
4637	case X86_XCPT_MC:
4638	case X86_XCPT_XF:
4639	AssertMsg(fIsBenignXcpt, ("%#x\n", uVector));
4640	break;
4641	default:
4642	AssertMsg(!fIsBenignXcpt, ("%#x\n", uVector));
4643	break;
4644	}
4645	#endif
4646	return fIsBenignXcpt;
4647	}
4648
4649
4650	/**
4651	* Determines if we are intercepting any contributory exceptions or page-faults.
4652	*
4653	* @returns true if we are intercepting them, false otherwise.
4654	* @param pVCpu Pointer to the VMCPU.
4655	*/
4656	DECLINLINE(bool) hmR0VmxInterceptingContributoryXcptsOrPF(PVMCPU pVCpu)
4657	{
4658	if (pVCpu->hm.s.vmx.u32XcptBitmap & (VMX_CONTRIBUTORY_XCPT_BITMAP \| RT_BIT(X86_XCPT_PF)))
4659	return true;
4660	return false;
4661	}
4662
4663
4664	/**
4665	* Sets an event as a pending event to be injected into the guest.
4666	*
4667	* @param pVCpu Pointer to the VMCPU.
4668	* @param u32IntrInfo The VM-entry interruption-information field.
4669	* @param cbInstr The VM-entry instruction length in bytes (for software
4670	* interrupts, exceptions and privileged software
4671	* exceptions).
4672	* @param u32ErrCode The VM-entry exception error code.
4673	* @param GCPtrFaultAddress The fault-address (CR2) in case it's a
4674	* page-fault.
4675	*/
4676	DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPU pVCpu, uint32_t u32IntrInfo, uint32_t cbInstr, uint32_t u32ErrCode,
4677	RTGCUINTPTR GCPtrFaultAddress)
4678	{
4679	Assert(!pVCpu->hm.s.Event.fPending);
4680	pVCpu->hm.s.Event.fPending = true;
4681	pVCpu->hm.s.Event.u64IntrInfo = u32IntrInfo;
4682	pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
4683	pVCpu->hm.s.Event.cbInstr = cbInstr;
4684	pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
4685	}
4686
4687
4688	/**
4689	* Sets a double-fault (#DF) exception as pending-for-injection into the VM.
4690	*
4691	* @param pVCpu Pointer to the VMCPU.
4692	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4693	* out-of-sync. Make sure to update the required fields
4694	* before using them.
4695	*/
4696	DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4697	{
4698	/* Inject the double-fault. */
4699	uint32_t u32IntrInfo = X86_XCPT_DF \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
4700	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
4701	u32IntrInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
4702	STAM_COUNTER_INC(&pVCpu->hm.s.StatIntInject);
4703	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
4704	}
4705
4706
4707	/**
4708	* Handle a condition that occurred while delivering an event through the guest
4709	* IDT.
4710	*
4711	* @returns VBox status code (informational error codes included).
4712	* @retval VINF_SUCCESS if we should continue handling the VM-exit.
4713	* @retval VINF_VMX_DOUBLE_FAULT if a #DF condition was detected and we ought to
4714	* continue execution of the guest which will delivery the #DF.
4715	* @retval VINF_EM_RESET if we detected a triple-fault condition.
4716	*
4717	* @param pVCpu Pointer to the VMCPU.
4718	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
4719	* out-of-sync. Make sure to update the required fields
4720	* before using them.
4721	* @param pVmxTransient Pointer to the VMX transient structure.
4722	*
4723	* @remarks No-long-jump zone!!!
4724	*/
4725	static int hmR0VmxCheckExitDueToEventDelivery(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
4726	{
4727	int rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
4728	AssertRCReturn(rc, rc);
4729	if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
4730	{
4731	rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
4732	AssertRCReturn(rc, rc);
4733
4734	uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
4735	uint32_t uExitVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntrInfo);
4736	uint32_t uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
4737
4738	typedef enum
4739	{
4740	VMXREFLECTXCPT_XCPT, /* Reflect Idt-vectoring exception. */
4741	VMXREFLECTXCPT_DF, /* Reflect a double-fault to the guest. */
4742	VMXREFLECTXCPT_TF, /* Reflect a triple fault state to the VMM. */
4743	VMXREFLECTXCPT_NONE /* Nothing to reflect. */
4744	} VMXREFLECTXCPT;
4745
4746	/* See Intel spec. 30.7.1.1 "Reflecting Exceptions to Guest Software". */
4747	VMXREFLECTXCPT enmReflect = VMXREFLECTXCPT_NONE;
4748	if (uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT)
4749	{
4750	if ( hmR0VmxIsBenignXcpt(uIdtVector)
4751	\|\| hmR0VmxIsBenignXcpt(uExitVector)
4752	\|\| ( hmR0VmxIsContributoryXcpt(uIdtVector)
4753	&& uExitVector == X86_XCPT_PF))
4754	{
4755	enmReflect = VMXREFLECTXCPT_XCPT;
4756	}
4757	else if ( (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_PF))
4758	&& uIdtVector == X86_XCPT_PF
4759	&& uExitVector == X86_XCPT_PF)
4760	{
4761	pVmxTransient->fVectoringPF = true;
4762	}
4763	else if ( hmR0VmxIsContributoryXcpt(uIdtVector)
4764	&& hmR0VmxIsContributoryXcpt(uExitVector))
4765	{
4766	enmReflect = VMXREFLECTXCPT_DF;
4767	}
4768	else if ( hmR0VmxInterceptingContributoryXcptsOrPF(pVCpu)
4769	&& uIdtVector == X86_XCPT_PF
4770	&& hmR0VmxIsContributoryXcpt(uExitVector))
4771	{
4772	enmReflect = VMXREFLECTXCPT_DF;
4773	}
4774	else if (uIdtVector == X86_XCPT_DF)
4775	enmReflect = VMXREFLECTXCPT_TF;
4776	}
4777	else if ( uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
4778	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
4779	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
4780	{
4781	/*
4782	* Ignore software interrupts (INT n), software exceptions (#BP, #OF) and privileged software exception
4783	* (whatever they are) as they reoccur when restarting the instruction.
4784	*/
4785	enmReflect = VMXREFLECTXCPT_XCPT;
4786	}
4787
4788	Assert(pVmxTransient->fVectoringPF == false \|\| enmReflect == VMXREFLECTXCPT_NONE);
4789	switch (enmReflect)
4790	{
4791	case VMXREFLECTXCPT_XCPT:
4792	{
4793	uint32_t u32ErrCode = 0;
4794	if (VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntrInfo))
4795	{
4796	rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
4797	AssertRCReturn(rc, rc);
4798	u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
4799	}
4800	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INTR_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
4801	0 /* cbInstr /, u32ErrCode, 0 / GCPtrFaultAddress */);
4802	rc = VINF_SUCCESS;
4803	Log(("Pending event %#RX64 Err=%#RX32\n", pVCpu->hm.s.Event.u64IntrInfo, pVCpu->hm.s.Event.u32ErrCode));
4804	break;
4805	}
4806
4807	case VMXREFLECTXCPT_DF:
4808	{
4809	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
4810	rc = VINF_VMX_DOUBLE_FAULT;
4811	Log(("Pending #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->hm.s.Event.u64IntrInfo, uIdtVector,
4812	uExitVector));
4813	break;
4814	}
4815
4816	case VMXREFLECTXCPT_TF:
4817	{
4818	Log(("Pending triple-fault uIdt=%#x uExit=%#x\n", uIdtVector, uExitVector));
4819	rc = VINF_EM_RESET;
4820	break;
4821	}
4822
4823	default: /* shut up gcc. */
4824	break;
4825	}
4826	}
4827	Assert(rc == VINF_SUCCESS \|\| rc == VINF_VMX_DOUBLE_FAULT \|\| rc == VINF_EM_RESET);
4828	return rc;
4829	}
4830
4831
4832	/**
4833	* Saves the guest's CR0 register from the VMCS into the guest-CPU context.
4834	*
4835	* @returns VBox status code.
4836	* @param pVCpu Pointer to the VMCPU.
4837	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4838	* out-of-sync. Make sure to update the required fields
4839	* before using them.
4840	*
4841	* @remarks No-long-jump zone!!!
4842	*/
4843	DECLINLINE(int) hmR0VmxSaveGuestCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4844	{
4845	int rc = VINF_SUCCESS;
4846	if ( !(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_CR0)
4847	\|\| !(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_FPU))
4848	{
4849	RTGCUINTREG uVal = 0;
4850	RTCCUINTREG uShadow = 0;
4851	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR0, &uVal);
4852	rc \|= VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uShadow);
4853	AssertRCReturn(rc, rc);
4854	uVal = (uShadow & pVCpu->hm.s.vmx.cr0_mask) \| (uVal & ~pVCpu->hm.s.vmx.cr0_mask);
4855	CPUMSetGuestCR0(pVCpu, uVal);
4856	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_CR0 \| VMX_UPDATED_GUEST_FPU;
4857	}
4858	return rc;
4859	}
4860
4861
4862	/**
4863	* Saves the guest's CR4 register from the VMCS into the guest-CPU context.
4864	*
4865	* @returns VBox status code.
4866	* @param pVCpu Pointer to the VMCPU.
4867	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4868	* out-of-sync. Make sure to update the required fields
4869	* before using them.
4870	*
4871	* @remarks No-long-jump zone!!!
4872	*/
4873	DECLINLINE(int) hmR0VmxSaveGuestCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4874	{
4875	int rc = VINF_SUCCESS;
4876	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_CR4))
4877	{
4878	RTGCUINTREG uVal = 0;
4879	RTCCUINTREG uShadow = 0;
4880	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR4, &uVal);
4881	rc \|= VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uShadow);
4882	AssertRCReturn(rc, rc);
4883	uVal = (uShadow & pVCpu->hm.s.vmx.cr4_mask) \| (uVal & ~pVCpu->hm.s.vmx.cr4_mask);
4884	CPUMSetGuestCR4(pVCpu, uVal);
4885	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_CR4;
4886	}
4887	return rc;
4888	}
4889
4890
4891	/**
4892	* Saves the guest's RIP register from the VMCS into the guest-CPU context.
4893	*
4894	* @returns VBox status code.
4895	* @param pVCpu Pointer to the VMCPU.
4896	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4897	* out-of-sync. Make sure to update the required fields
4898	* before using them.
4899	*
4900	* @remarks No-long-jump zone!!!
4901	*/
4902	DECLINLINE(int) hmR0VmxSaveGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4903	{
4904	if (pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_RIP)
4905	return VINF_SUCCESS;
4906
4907	RTGCUINTREG uVal = 0;
4908	int rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &uVal);
4909	AssertRCReturn(rc, rc);
4910	pMixedCtx->rip = uVal;
4911	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_RIP;
4912	return rc;
4913	}
4914
4915
4916	/**
4917	* Saves the guest's RSP register from the VMCS into the guest-CPU context.
4918	*
4919	* @returns VBox status code.
4920	* @param pVCpu Pointer to the VMCPU.
4921	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4922	* out-of-sync. Make sure to update the required fields
4923	* before using them.
4924	*
4925	* @remarks No-long-jump zone!!!
4926	*/
4927	DECLINLINE(int) hmR0VmxSaveGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4928	{
4929	if (pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_RSP)
4930	return VINF_SUCCESS;
4931
4932	RTGCUINTREG uVal = 0;
4933	int rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &uVal);
4934	AssertRCReturn(rc, rc);
4935	pMixedCtx->rsp = uVal;
4936	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_RSP;
4937	return rc;
4938	}
4939
4940
4941	/**
4942	* Saves the guest's RFLAGS from the VMCS into the guest-CPU context.
4943	*
4944	* @returns VBox status code.
4945	* @param pVCpu Pointer to the VMCPU.
4946	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4947	* out-of-sync. Make sure to update the required fields
4948	* before using them.
4949	*
4950	* @remarks No-long-jump zone!!!
4951	*/
4952	DECLINLINE(int) hmR0VmxSaveGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4953	{
4954	if (pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_RFLAGS)
4955	return VINF_SUCCESS;
4956
4957	RTGCUINTREG uVal = 0;
4958	int rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RFLAGS, &uVal);
4959	AssertRCReturn(rc, rc);
4960	pMixedCtx->rflags.u64 = uVal;
4961
4962	/* Undo our real-on-v86-mode changes to eflags if necessary. */
4963	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4964	{
4965	PVM pVM = pVCpu->CTX_SUFF(pVM);
4966	Assert(pVM->hm.s.vmx.pRealModeTSS);
4967	Log(("Saving real-mode RFLAGS VT-x view=%#RX64\n", pMixedCtx->rflags.u64));
4968	pMixedCtx->eflags.Bits.u1VM = 0;
4969	pMixedCtx->eflags.Bits.u2IOPL = pVCpu->hm.s.vmx.RealMode.eflags.Bits.u2IOPL;
4970	}
4971
4972	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_RFLAGS;
4973	return rc;
4974	}
4975
4976
4977	/**
4978	* Wrapper for saving the guest's RIP, RSP and RFLAGS from the VMCS into the
4979	* guest-CPU context.
4980	*/
4981	static int hmR0VmxSaveGuestGprs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4982	{
4983	int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
4984	rc \|= hmR0VmxSaveGuestRsp(pVCpu, pMixedCtx);
4985	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
4986	return rc;
4987	}
4988
4989
4990	/**
4991	* Saves the guest's interruptibility-state ("interrupt shadow" as AMD calls it)
4992	* from the guest-state area in the VMCS.
4993	*
4994	* @param pVCpu Pointer to the VMCPU.
4995	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
4996	* out-of-sync. Make sure to update the required fields
4997	* before using them.
4998	*
4999	* @remarks No-long-jump zone!!!
5000	*/
5001	DECLINLINE(void) hmR0VmxSaveGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5002	{
5003	uint32_t uIntrState = 0;
5004	int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
5005	AssertRC(rc);
5006
5007	if (!uIntrState)
5008	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
5009	else
5010	{
5011	Assert( uIntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
5012	\|\| uIntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
5013	rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
5014	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* for hmR0VmxGetGuestIntrState(). */
5015	AssertRC(rc);
5016	EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
5017	Assert(VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
5018	}
5019	}
5020
5021
5022	/**
5023	* Saves the guest's activity state.
5024	*
5025	* @returns VBox status code.
5026	* @param pVCpu Pointer to the VMCPU.
5027	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5028	* out-of-sync. Make sure to update the required fields
5029	* before using them.
5030	*
5031	* @remarks No-long-jump zone!!!
5032	*/
5033	DECLINLINE(int) hmR0VmxSaveGuestActivityState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5034	{
5035	/* Nothing to do for now until we make use of different guest-CPU activity state. Just update the flag. */
5036	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_ACTIVITY_STATE;
5037	return VINF_SUCCESS;
5038	}
5039
5040
5041	/**
5042	* Saves the guest SYSENTER MSRs (SYSENTER_CS, SYSENTER_EIP, SYSENTER_ESP) from
5043	* the current VMCS into the guest-CPU context.
5044	*
5045	* @returns VBox status code.
5046	* @param pVCpu Pointer to the VMCPU.
5047	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5048	* out-of-sync. Make sure to update the required fields
5049	* before using them.
5050	*
5051	* @remarks No-long-jump zone!!!
5052	*/
5053	DECLINLINE(int) hmR0VmxSaveGuestSysenterMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5054	{
5055	int rc = VINF_SUCCESS;
5056	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_SYSENTER_CS_MSR))
5057	{
5058	uint32_t u32Val = 0;
5059	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val); AssertRCReturn(rc, rc);
5060	pMixedCtx->SysEnter.cs = u32Val;
5061	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_SYSENTER_CS_MSR;
5062	}
5063
5064	RTGCUINTREG uGCVal = 0;
5065	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_SYSENTER_EIP_MSR))
5066	{
5067	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &uGCVal); AssertRCReturn(rc, rc);
5068	pMixedCtx->SysEnter.eip = uGCVal;
5069	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_SYSENTER_EIP_MSR;
5070	}
5071	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_SYSENTER_ESP_MSR))
5072	{
5073	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &uGCVal); AssertRCReturn(rc, rc);
5074	pMixedCtx->SysEnter.esp = uGCVal;
5075	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_SYSENTER_ESP_MSR;
5076	}
5077	return rc;
5078	}
5079
5080
5081	/**
5082	* Saves the guest FS_BASE MSRs from the current VMCS into the guest-CPU
5083	* context.
5084	*
5085	* @returns VBox status code.
5086	* @param pVCpu Pointer to the VMCPU.
5087	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5088	* out-of-sync. Make sure to update the required fields
5089	* before using them.
5090	*
5091	* @remarks No-long-jump zone!!!
5092	*/
5093	DECLINLINE(int) hmR0VmxSaveGuestFSBaseMsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5094	{
5095	RTGCUINTREG uVal = 0;
5096	int rc = VINF_SUCCESS;
5097	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_FS_BASE_MSR))
5098	{
5099	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_FS_BASE, &uVal); AssertRCReturn(rc, rc);
5100	pMixedCtx->fs.u64Base = uVal;
5101	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_FS_BASE_MSR;
5102	}
5103	return rc;
5104	}
5105
5106
5107	/**
5108	* Saves the guest GS_BASE MSRs from the current VMCS into the guest-CPU
5109	* context.
5110	*
5111	* @returns VBox status code.
5112	* @param pVCpu Pointer to the VMCPU.
5113	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5114	* out-of-sync. Make sure to update the required fields
5115	* before using them.
5116	*
5117	* @remarks No-long-jump zone!!!
5118	*/
5119	DECLINLINE(int) hmR0VmxSaveGuestGSBaseMsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5120	{
5121	RTGCUINTREG uVal = 0;
5122	int rc = VINF_SUCCESS;
5123	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_GS_BASE_MSR))
5124	{
5125	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GS_BASE, &uVal); AssertRCReturn(rc, rc);
5126	pMixedCtx->gs.u64Base = uVal;
5127	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_GS_BASE_MSR;
5128	}
5129	return rc;
5130	}
5131
5132
5133	/**
5134	* Saves the auto load/store'd guest MSRs from the current VMCS into the
5135	* guest-CPU context. Currently these are LSTAR, STAR, SFMASK and TSC_AUX.
5136	*
5137	* @returns VBox status code.
5138	* @param pVCpu Pointer to the VMCPU.
5139	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5140	* out-of-sync. Make sure to update the required fields
5141	* before using them.
5142	*
5143	* @remarks No-long-jump zone!!!
5144	*/
5145	static int hmR0VmxSaveGuestAutoLoadStoreMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5146	{
5147	if (pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS)
5148	return VINF_SUCCESS;
5149
5150	for (uint32_t i = 0; i < pVCpu->hm.s.vmx.cGuestMsrs; i++)
5151	{
5152	PVMXMSR pMsr = (PVMXMSR)pVCpu->hm.s.vmx.pvGuestMsr;
5153	pMsr += i;
5154	switch (pMsr->u32IndexMSR)
5155	{
5156	case MSR_K8_LSTAR: pMixedCtx->msrLSTAR = pMsr->u64Value; break;
5157	case MSR_K6_STAR: pMixedCtx->msrSTAR = pMsr->u64Value; break;
5158	case MSR_K8_SF_MASK: pMixedCtx->msrSFMASK = pMsr->u64Value; break;
5159	case MSR_K8_TSC_AUX: CPUMSetGuestMsr(pVCpu, MSR_K8_TSC_AUX, pMsr->u64Value); break;
5160	#if 0
5161	/* The KERNEL_GS_BASE MSR doesn't work reliably with auto load/store. See @bugref{6208} */
5162	case MSR_K8_KERNEL_GS_BASE: pMixedCtx->msrKERNELGSBASE = pMsr->u64Value; break;
5163	#endif
5164	case MSR_K6_EFER: /* EFER can't be changed without causing a VM-exit. */ break;
5165	default:
5166	{
5167	AssertFailed();
5168	return VERR_HM_UNEXPECTED_LD_ST_MSR;
5169	}
5170	}
5171	}
5172	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS;
5173	return VINF_SUCCESS;
5174	}
5175
5176
5177	/**
5178	* Saves the guest control registers from the current VMCS into the guest-CPU
5179	* context.
5180	*
5181	* @returns VBox status code.
5182	* @param pVCpu Pointer to the VMCPU.
5183	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5184	* out-of-sync. Make sure to update the required fields
5185	* before using them.
5186	*
5187	* @remarks No-long-jump zone!!!
5188	*/
5189	DECLINLINE(int) hmR0VmxSaveGuestControlRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5190	{
5191	RTGCUINTREG uVal = 0;
5192	RTGCUINTREG uShadow = 0;
5193	int rc = VINF_SUCCESS;
5194
5195	/* Guest CR0. Guest FPU. */
5196	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
5197
5198	/* Guest CR4. */
5199	rc \|= hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
5200	AssertRCReturn(rc, rc);
5201
5202	/* Guest CR3. Only changes with Nested Paging. This must be done -after- saving CR0 and CR4 from the guest! */
5203	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_CR3))
5204	{
5205	PVM pVM = pVCpu->CTX_SUFF(pVM);
5206	if ( pVM->hm.s.fNestedPaging
5207	&& CPUMIsGuestPagingEnabledEx(pMixedCtx))
5208	{
5209	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &uVal);
5210	if (pMixedCtx->cr3 != uVal)
5211	{
5212	CPUMSetGuestCR3(pVCpu, uVal);
5213	/* Set the force flag to inform PGM about it when necessary. It is cleared by PGMUpdateCR3().*/
5214	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
5215	}
5216
5217	/* We require EFER to check PAE mode. */
5218	rc \|= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
5219
5220	/* If the guest is in PAE mode, sync back the PDPE's into the guest state. */
5221	if (CPUMIsGuestInPAEModeEx(pMixedCtx)) /* Reads CR0, CR4 and EFER MSR. */
5222	{
5223	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u);
5224	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u);
5225	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u);
5226	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u);
5227	/* Set the force flag to inform PGM about it when necessary. It is cleared by PGMGstUpdatePaePdpes(). */
5228	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
5229	}
5230	AssertRCReturn(rc, rc);
5231	}
5232	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_CR3;
5233	}
5234	return rc;
5235	}
5236
5237
5238	/**
5239	* Reads a guest segment register from the current VMCS into the guest-CPU
5240	* context.
5241	*
5242	* @returns VBox status code.
5243	* @param pVCpu Pointer to the VMCPU.
5244	* @param idxSel Index of the selector in the VMCS.
5245	* @param idxLimit Index of the segment limit in the VMCS.
5246	* @param idxBase Index of the segment base in the VMCS.
5247	* @param idxAccess Index of the access rights of the segment in the VMCS.
5248	* @param pSelReg Pointer to the segment selector.
5249	*
5250	* @remarks No-long-jump zone!!!
5251	* @remarks Never call this function directly. Use the VMXLOCAL_READ_SEG() macro
5252	* as that takes care of whether to read from the VMCS cache or not.
5253	*/
5254	DECLINLINE(int) hmR0VmxReadSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase, uint32_t idxAccess,
5255	PCPUMSELREG pSelReg)
5256	{
5257	uint32_t u32Val = 0;
5258	int rc = VMXReadVmcs32(idxSel, &u32Val);
5259	pSelReg->Sel = (uint16_t)u32Val;
5260	pSelReg->ValidSel = (uint16_t)u32Val;
5261	pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
5262
5263	rc \|= VMXReadVmcs32(idxLimit, &u32Val);
5264	pSelReg->u32Limit = u32Val;
5265
5266	RTGCUINTREG uGCVal = 0;
5267	rc \|= VMXReadVmcsGstNByIdxVal(idxBase, &uGCVal);
5268	pSelReg->u64Base = uGCVal;
5269
5270	rc \|= VMXReadVmcs32(idxAccess, &u32Val);
5271	pSelReg->Attr.u = u32Val;
5272	AssertRCReturn(rc, rc);
5273
5274	/*
5275	* If VT-x marks the segment as unusable, the rest of the attributes are undefined.
5276	* See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers.
5277	*/
5278	if (pSelReg->Attr.u & VMX_SEL_UNUSABLE)
5279	{
5280	Assert(idxSel != VMX_VMCS16_GUEST_FIELD_TR);
5281	pSelReg->Attr.u = VMX_SEL_UNUSABLE;
5282	}
5283	return rc;
5284	}
5285
5286
5287	/**
5288	* Saves the guest segment registers from the current VMCS into the guest-CPU
5289	* context.
5290	*
5291	* @returns VBox status code.
5292	* @param pVCpu Pointer to the VMCPU.
5293	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5294	* out-of-sync. Make sure to update the required fields
5295	* before using them.
5296	*
5297	* @remarks No-long-jump zone!!!
5298	*/
5299	static int hmR0VmxSaveGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5300	{
5301	#ifdef VMX_USE_CACHED_VMCS_ACCESSES
5302	#define VMXLOCAL_READ_SEG(Sel, CtxSel) \
5303	hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
5304	VMX_VMCS_GUEST_##Sel##_BASE_CACHE_IDX, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
5305	#else
5306	#define VMXLOCAL_READ_SEG(Sel, CtxSel) \
5307	hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
5308	VMX_VMCS_GUEST_##Sel##_BASE, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
5309	#endif
5310
5311	int rc = VINF_SUCCESS;
5312
5313	/* Guest segment registers. */
5314	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_SEGMENT_REGS))
5315	{
5316	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
5317	rc \|= VMXLOCAL_READ_SEG(CS, cs);
5318	rc \|= VMXLOCAL_READ_SEG(SS, ss);
5319	rc \|= VMXLOCAL_READ_SEG(DS, ds);
5320	rc \|= VMXLOCAL_READ_SEG(ES, es);
5321	rc \|= VMXLOCAL_READ_SEG(FS, fs);
5322	rc \|= VMXLOCAL_READ_SEG(GS, gs);
5323	AssertRCReturn(rc, rc);
5324
5325	/* Restore segment attributes for real-on-v86 mode hack. */
5326	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
5327	{
5328	pMixedCtx->cs.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrCS.u;
5329	pMixedCtx->ss.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrSS.u;
5330	pMixedCtx->ds.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrDS.u;
5331	pMixedCtx->es.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrES.u;
5332	pMixedCtx->fs.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrFS.u;
5333	pMixedCtx->gs.Attr.u = pVCpu->hm.s.vmx.RealMode.uAttrGS.u;
5334	}
5335	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_SEGMENT_REGS;
5336	}
5337
5338	/* Guest LDTR. */
5339	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_LDTR))
5340	{
5341	rc = VMXLOCAL_READ_SEG(LDTR, ldtr);
5342	AssertRCReturn(rc, rc);
5343	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_LDTR;
5344	}
5345
5346	/* Guest GDTR. */
5347	RTGCUINTREG uGCVal = 0;
5348	uint32_t u32Val = 0;
5349	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_GDTR))
5350	{
5351	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &uGCVal);
5352	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
5353	pMixedCtx->gdtr.pGdt = uGCVal;
5354	pMixedCtx->gdtr.cbGdt = u32Val;
5355	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_GDTR;
5356	}
5357
5358	/* Guest IDTR. */
5359	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_IDTR))
5360	{
5361	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &uGCVal);
5362	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
5363	pMixedCtx->idtr.pIdt = uGCVal;
5364	pMixedCtx->idtr.cbIdt = u32Val;
5365	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_IDTR;
5366	}
5367
5368	/* Guest TR. */
5369	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_TR))
5370	{
5371	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
5372
5373	/* For real-mode emulation using virtual-8086 mode we have the fake TSS (pRealModeTSS) in TR, don't save the fake one. */
5374	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
5375	rc \|= VMXLOCAL_READ_SEG(TR, tr);
5376	AssertRCReturn(rc, rc);
5377	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_TR;
5378	}
5379	return rc;
5380	}
5381
5382
5383	/**
5384	* Saves the guest debug registers from the current VMCS into the guest-CPU
5385	* context.
5386	*
5387	* @returns VBox status code.
5388	* @param pVCpu Pointer to the VMCPU.
5389	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5390	* out-of-sync. Make sure to update the required fields
5391	* before using them.
5392	*
5393	* @remarks No-long-jump zone!!!
5394	*/
5395	DECLINLINE(int) hmR0VmxSaveGuestDebugRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5396	{
5397	int rc = VINF_SUCCESS;
5398	if (!(pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_DEBUG))
5399	{
5400	RTGCUINTREG uVal;
5401	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_DR7, &uVal); AssertRCReturn(rc, rc);
5402	pMixedCtx->dr[7] = uVal;
5403
5404	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_DEBUG;
5405	}
5406	return rc;
5407	}
5408
5409
5410	/**
5411	* Saves the guest APIC state from the currentl VMCS into the guest-CPU context.
5412	*
5413	* @returns VBox status code.
5414	* @param pVCpu Pointer to the VMCPU.
5415	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5416	* out-of-sync. Make sure to update the required fields
5417	* before using them.
5418	*
5419	* @remarks No-long-jump zone!!!
5420	*/
5421	DECLINLINE(int) hmR0VmxSaveGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5422	{
5423	/* Updating TPR is already done in hmR0VmxPostRunGuest(). Just update the flag. */
5424	pVCpu->hm.s.vmx.fUpdatedGuestState \|= VMX_UPDATED_GUEST_APIC_STATE;
5425	return VINF_SUCCESS;
5426	}
5427
5428
5429	/**
5430	* Saves the entire guest state from the currently active VMCS into the
5431	* guest-CPU context. This essentially VMREADs all guest-data.
5432	*
5433	* @returns VBox status code.
5434	* @param pVCpu Pointer to the VMCPU.
5435	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5436	* out-of-sync. Make sure to update the required fields
5437	* before using them.
5438	*/
5439	static int hmR0VmxSaveGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5440	{
5441	Assert(pVCpu);
5442	Assert(pMixedCtx);
5443
5444	if (pVCpu->hm.s.vmx.fUpdatedGuestState == VMX_UPDATED_GUEST_ALL)
5445	return VINF_SUCCESS;
5446
5447	VMMRZCallRing3Disable(pVCpu);
5448
5449	int rc = hmR0VmxSaveGuestGprs(pVCpu, pMixedCtx);
5450	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestGprs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5451
5452	rc = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
5453	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestControlRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5454
5455	rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
5456	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSegmentRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5457
5458	rc = hmR0VmxSaveGuestDebugRegs(pVCpu, pMixedCtx);
5459	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestDebugRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5460
5461	rc = hmR0VmxSaveGuestSysenterMsrs(pVCpu, pMixedCtx);
5462	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSysenterMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5463
5464	rc = hmR0VmxSaveGuestFSBaseMsr(pVCpu, pMixedCtx);
5465	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestFSBaseMsr failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5466
5467	rc = hmR0VmxSaveGuestGSBaseMsr(pVCpu, pMixedCtx);
5468	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestGSBaseMsr failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5469
5470	rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
5471	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestAutoLoadStoreMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5472
5473	rc = hmR0VmxSaveGuestActivityState(pVCpu, pMixedCtx);
5474	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestActivityState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5475
5476	rc = hmR0VmxSaveGuestApicState(pVCpu, pMixedCtx);
5477	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestDebugRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
5478
5479	AssertMsg(pVCpu->hm.s.vmx.fUpdatedGuestState == VMX_UPDATED_GUEST_ALL,
5480	("Missed guest state bits while saving state; residue %RX32\n", pVCpu->hm.s.vmx.fUpdatedGuestState));
5481
5482	VMMRZCallRing3Enable(pVCpu);
5483	return rc;
5484	}
5485
5486
5487	/**
5488	* Check per-VM and per-VCPU force flag actions that require us to go back to
5489	* ring-3 for one reason or another.
5490	*
5491	* @returns VBox status code (information status code included).
5492	* @retval VINF_SUCCESS if we don't have any actions that require going back to
5493	* ring-3.
5494	* @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
5495	* @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
5496	* interrupts)
5497	* @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
5498	* all EMTs to be in ring-3.
5499	* @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
5500	* @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
5501	* to the EM loop.
5502	*
5503	* @param pVM Pointer to the VM.
5504	* @param pVCpu Pointer to the VMCPU.
5505	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5506	* out-of-sync. Make sure to update the required fields
5507	* before using them.
5508	*/
5509	static int hmR0VmxCheckForceFlags(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5510	{
5511	Assert(VMMRZCallRing3IsEnabled(pVCpu));
5512
5513	int rc = VERR_INTERNAL_ERROR_5;
5514	if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK \| VM_FF_REQUEST \| VM_FF_PGM_POOL_FLUSH_PENDING \| VM_FF_PDM_DMA)
5515	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK \| VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL
5516	\| VMCPU_FF_REQUEST \| VMCPU_FF_HM_UPDATE_CR3 \| VMCPU_FF_HM_UPDATE_PAE_PDPES))
5517	{
5518	/* We need the control registers now, make sure the guest-CPU context is updated. */
5519	rc = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
5520	AssertRCReturn(rc, rc);
5521
5522	/* Pending HM CR3 sync. */
5523	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
5524	{
5525	rc = PGMUpdateCR3(pVCpu, pMixedCtx->cr3);
5526	Assert(rc == VINF_SUCCESS \|\| rc == VINF_PGM_SYNC_CR3);
5527	}
5528	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
5529	{
5530	rc = PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
5531	AssertRC(rc);
5532	}
5533
5534	/* Pending PGM C3 sync. */
5535	if (VMCPU_FF_IS_PENDING(pVCpu,VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
5536	{
5537	rc = PGMSyncCR3(pVCpu, pMixedCtx->cr0, pMixedCtx->cr3, pMixedCtx->cr4, VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
5538	if (rc != VINF_SUCCESS)
5539	{
5540	AssertRC(rc);
5541	Log2(("hmR0VmxCheckForceFlags: PGMSyncCR3 forcing us back to ring-3. rc=%d\n", rc));
5542	return rc;
5543	}
5544	}
5545
5546	/* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
5547	/* -XXX- what was that about single stepping? */
5548	if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK)
5549	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
5550	{
5551	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
5552	rc = RT_UNLIKELY(VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
5553	Log2(("hmR0VmxCheckForceFlags: HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc));
5554	return rc;
5555	}
5556
5557	/* Pending VM request packets, such as hardware interrupts. */
5558	if ( VM_FF_IS_PENDING(pVM, VM_FF_REQUEST)
5559	\|\| VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_REQUEST))
5560	{
5561	Log2(("hmR0VmxCheckForceFlags: Pending VM request forcing us back to ring-3\n"));
5562	return VINF_EM_PENDING_REQUEST;
5563	}
5564
5565	/* Pending PGM pool flushes. */
5566	if (VM_FF_IS_PENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
5567	{
5568	Log2(("hmR0VmxCheckForceFlags: PGM pool flush pending forcing us back to ring-3\n"));
5569	return rc = VINF_PGM_POOL_FLUSH_PENDING;
5570	}
5571
5572	/* Pending DMA requests. */
5573	if (VM_FF_IS_PENDING(pVM, VM_FF_PDM_DMA))
5574	{
5575	Log2(("hmR0VmxCheckForceFlags: Pending DMA request forcing us back to ring-3\n"));
5576	return VINF_EM_RAW_TO_R3;
5577	}
5578	}
5579
5580	/* Paranoia. */
5581	Assert(rc != VERR_EM_INTERPRETER);
5582	return VINF_SUCCESS;
5583	}
5584
5585
5586	/**
5587	* Converts any TRPM trap into a pending VMX event. This is typically used when
5588	* entering from ring-3 (not longjmp returns).
5589	*
5590	* @param pVCpu Pointer to the VMCPU.
5591	* @param pCtx Pointer to the guest-CPU context.
5592	*/
5593	static void hmR0VmxUpdatePendingEvent(PVMCPU pVCpu, PCPUMCTX pCtx)
5594	{
5595	if (!TRPMHasTrap(pVCpu))
5596	{
5597	Assert(!pVCpu->hm.s.Event.fPending);
5598	return;
5599	}
5600
5601	uint8_t uVector;
5602	TRPMEVENT enmTrpmEvent;
5603	RTGCUINT uErrCode;
5604	RTGCUINTPTR GCPtrFaultAddress;
5605	uint8_t cbInstr;
5606
5607	int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
5608	AssertRC(rc);
5609
5610	/* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntrInfo. */
5611	uint32_t u32IntrInfo = uVector \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
5612	uint32_t u32ErrCode = uErrCode;
5613	if (enmTrpmEvent == TRPM_TRAP)
5614	{
5615	switch (uVector)
5616	{
5617	case X86_XCPT_BP:
5618	case X86_XCPT_OF:
5619	{
5620	/* These exceptions must be delivered as software exceptions. They have no error codes associated with them. */
5621	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5622	break;
5623	}
5624
5625	case X86_XCPT_PF:
5626	pCtx->cr2 = GCPtrFaultAddress;
5627	/* no break */
5628	case X86_XCPT_DF:
5629	case X86_XCPT_TS:
5630	case X86_XCPT_NP:
5631	case X86_XCPT_SS:
5632	case X86_XCPT_GP:
5633	case X86_XCPT_AC:
5634	/* These exceptions must be delivered as hardware exceptions. They have error codes associated with
5635	them which VT-x/VMM pushes to the guest stack. */
5636	u32IntrInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
5637	/* no break! */
5638	default:
5639	{
5640	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5641	break;
5642	}
5643	}
5644	}
5645	else if (enmTrpmEvent == TRPM_HARDWARE_INT)
5646	{
5647	if (uVector != X86_XCPT_NMI)
5648	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5649	else
5650	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5651	}
5652	else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
5653	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5654	else
5655	AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
5656
5657	rc = TRPMResetTrap(pVCpu);
5658	AssertRC(rc);
5659	Log(("Converting TRPM trap: u32IntrInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u u32ErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
5660	u32IntrInfo, enmTrpmEvent, u32ErrCode, GCPtrFaultAddress));
5661	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, cbInstr, u32ErrCode, GCPtrFaultAddress);
5662	}
5663
5664
5665	/**
5666	* Converts any pending VMX event into a TRPM trap. Typically used when leaving
5667	* VT-x to execute any instruction.
5668	*
5669	* @param pvCpu Pointer to the VMCPU.
5670	*/
5671	static void hmR0VmxUpdateTRPM(PVMCPU pVCpu)
5672	{
5673	if (pVCpu->hm.s.Event.fPending)
5674	{
5675	uint32_t uVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVCpu->hm.s.Event.u64IntrInfo);
5676	uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVCpu->hm.s.Event.u64IntrInfo);
5677	bool fErrorCodeValid = !!VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntrInfo);
5678	uint32_t uErrorCode = pVCpu->hm.s.Event.u32ErrCode;
5679
5680	/* If a trap was already pending, we did something wrong! */
5681	Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo /, NULL / pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
5682
5683	TRPMEVENT enmTrapType;
5684	switch (uVectorType)
5685	{
5686	case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
5687	case VMX_IDT_VECTORING_INFO_TYPE_NMI:
5688	enmTrapType = TRPM_HARDWARE_INT;
5689	break;
5690	case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
5691	enmTrapType = TRPM_SOFTWARE_INT;
5692	break;
5693	case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
5694	case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: /* #BP and #OF */
5695	case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
5696	enmTrapType = TRPM_TRAP;
5697	break;
5698	default:
5699	AssertMsgFailed(("Invalid trap type %#x\n", uVectorType));
5700	enmTrapType = TRPM_32BIT_HACK;
5701	break;
5702	}
5703
5704	Log(("Converting pending HM event to TRPM trap uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
5705	int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
5706	AssertRC(rc);
5707
5708	if (fErrorCodeValid)
5709	TRPMSetErrorCode(pVCpu, uErrorCode);
5710	/* A page-fault exception during a page-fault would become a double-fault. */
5711	if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
5712	&& uVector == X86_XCPT_PF)
5713	{
5714	TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
5715	}
5716	else if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
5717	\|\| uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT)
5718	{
5719	AssertMsg( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
5720	\|\| (uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF),
5721	("Invalid vector: uVector=%#x uVectorType=%#x\n", uVector, uVectorType));
5722	TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
5723	}
5724	pVCpu->hm.s.Event.fPending = false;
5725	}
5726	}
5727
5728
5729	/**
5730	* Does the necessary state syncing before doing a longjmp to ring-3.
5731	*
5732	* @param pVM Pointer to the VM.
5733	* @param pVCpu Pointer to the VMCPU.
5734	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5735	* out-of-sync. Make sure to update the required fields
5736	* before using them.
5737	* @param rcExit The reason for exiting to ring-3. Can be
5738	* VINF_VMM_UNKNOWN_RING3_CALL.
5739	*
5740	* @remarks No-long-jmp zone!!!
5741	*/
5742	static void hmR0VmxLongJmpToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, int rcExit)
5743	{
5744	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
5745
5746	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
5747	Assert(pVCpu->hm.s.vmx.fUpdatedGuestState == VMX_UPDATED_GUEST_ALL);
5748	AssertRC(rc);
5749
5750	/* Restore FPU state if necessary and resync on next R0 reentry .*/
5751	if (CPUMIsGuestFPUStateActive(pVCpu))
5752	{
5753	CPUMR0SaveGuestFPU(pVM, pVCpu, pMixedCtx);
5754	Assert(!CPUMIsGuestFPUStateActive(pVCpu));
5755	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
5756	}
5757
5758	/* Restore debug registers if necessary and resync on next R0 reentry. */
5759	if (CPUMIsGuestDebugStateActive(pVCpu))
5760	{
5761	CPUMR0SaveGuestDebugState(pVM, pVCpu, pMixedCtx, true /* save DR6 */);
5762	Assert(!CPUMIsGuestDebugStateActive(pVCpu));
5763	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_DEBUG;
5764	}
5765	else if (CPUMIsHyperDebugStateActive(pVCpu))
5766	{
5767	CPUMR0LoadHostDebugState(pVM, pVCpu);
5768	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT);
5769	}
5770
5771	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
5772	}
5773
5774
5775	/**
5776	* An action requires us to go back to ring-3. This function does the necessary
5777	* steps before we can safely return to ring-3. This is not the same as longjmps
5778	* to ring-3, this is voluntary.
5779	*
5780	* @param pVM Pointer to the VM.
5781	* @param pVCpu Pointer to the VMCPU.
5782	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5783	* out-of-sync. Make sure to update the required fields
5784	* before using them.
5785	* @param rcExit The reason for exiting to ring-3. Can be
5786	* VINF_VMM_UNKNOWN_RING3_CALL.
5787	*/
5788	static void hmR0VmxExitToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, int rcExit)
5789	{
5790	Assert(pVM);
5791	Assert(pVCpu);
5792	Assert(pMixedCtx);
5793	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
5794
5795	if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_GUEST_STATE))
5796	{
5797	/* We want to see what the guest-state was before VM-entry, don't resync here, as we won't continue guest execution. */
5798	return;
5799	}
5800	else if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
5801	{
5802	VMXGetActivateVMCS(&pVCpu->hm.s.vmx.lasterror.u64VMCSPhys);
5803	pVCpu->hm.s.vmx.lasterror.u32VMCSRevision = (uint32_t )pVCpu->hm.s.vmx.pvVmcs;
5804	pVCpu->hm.s.vmx.lasterror.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
5805	pVCpu->hm.s.vmx.lasterror.idCurrentCpu = RTMpCpuId();
5806	return;
5807	}
5808
5809	/* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
5810	VMMRZCallRing3Disable(pVCpu);
5811	Log(("hmR0VmxExitToRing3: rcExit=%d\n", rcExit));
5812
5813	/* We need to do this only while truly exiting the "inner loop" back to ring-3 and -not- for any longjmp to ring3. */
5814	hmR0VmxUpdateTRPM(pVCpu);
5815
5816	/* Sync. the guest state. */
5817	hmR0VmxLongJmpToRing3(pVM, pVCpu, pMixedCtx, rcExit);
5818	STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
5819
5820	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
5821	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
5822	\| CPUM_CHANGED_LDTR
5823	\| CPUM_CHANGED_GDTR
5824	\| CPUM_CHANGED_IDTR
5825	\| CPUM_CHANGED_TR
5826	\| CPUM_CHANGED_HIDDEN_SEL_REGS);
5827
5828	/* On our way back from ring-3 the following needs to be done. */
5829	/** @todo This can change with preemption hooks. */
5830	if (rcExit == VINF_EM_RAW_INTERRUPT)
5831	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_HOST_CONTEXT;
5832	else
5833	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_ALL_GUEST;
5834
5835	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
5836	VMMRZCallRing3Enable(pVCpu);
5837	}
5838
5839
5840	/**
5841	* VMMRZCallRing3 callback wrapper which saves the guest state before we
5842	* longjump to ring-3 and possibly get preempted.
5843	*
5844	* @param pVCpu Pointer to the VMCPU.
5845	* @param enmOperation The operation causing the ring-3 longjump.
5846	* @param pvUser The user argument (pointer to the possibly
5847	* out-of-date guest-CPU context).
5848	*
5849	* @remarks Must never be called with @a enmOperation ==
5850	* VMMCALLRING3_VM_R0_ASSERTION.
5851	*/
5852	DECLCALLBACK(void) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
5853	{
5854	/* VMMRZCallRing3() already makes sure we never get called as a result of an longjmp due to an assertion, */
5855	Assert(pVCpu);
5856	Assert(pvUser);
5857	Assert(VMMRZCallRing3IsEnabled(pVCpu));
5858	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
5859
5860	VMMRZCallRing3Disable(pVCpu);
5861	Log(("hmR0VmxLongJmpToRing3\n"));
5862	hmR0VmxLongJmpToRing3(pVCpu->CTX_SUFF(pVM), pVCpu, (PCPUMCTX)pvUser, VINF_VMM_UNKNOWN_RING3_CALL);
5863	VMMRZCallRing3Enable(pVCpu);
5864	}
5865
5866
5867	/**
5868	* Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
5869	* cause a VM-exit as soon as the guest is in a state to receive interrupts.
5870	*
5871	* @returns VBox status code.
5872	* @param pVCpu Pointer to the VMCPU.
5873	*/
5874	DECLINLINE(void) hmR0VmxSetIntWindowExitVmcs(PVMCPU pVCpu)
5875	{
5876	if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.msr.vmx_proc_ctls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INT_WINDOW_EXIT))
5877	{
5878	/* Enable interrupt-window exiting if it's not in effect already. */
5879	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INT_WINDOW_EXIT))
5880	{
5881	pVCpu->hm.s.vmx.u32ProcCtls \|= VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INT_WINDOW_EXIT;
5882	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls);
5883	AssertRC(rc);
5884	}
5885	} /* else we will deliver interrupts whenever the guest exits next and is in a state to receive events. */
5886	}
5887
5888
5889	/**
5890	* Injects any pending events into the guest if the guest is in a state to
5891	* receive them.
5892	*
5893	* @returns VBox status code (informational status codes included).
5894	* @param pVCpu Pointer to the VMCPU.
5895	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
5896	* out-of-sync. Make sure to update the required fields
5897	* before using them.
5898	*/
5899	static int hmR0VmxInjectPendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5900	{
5901	/* Get the current interruptibility-state of the guest and then figure out what can be injected. */
5902	uint32_t uIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
5903	const bool fBlockMovSS = (uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
5904	const bool fBlockSti = (uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
5905
5906	Assert(!fBlockSti \|\| (pVCpu->hm.s.vmx.fUpdatedGuestState & VMX_UPDATED_GUEST_RFLAGS));
5907	Assert( !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI) /* We don't support block-by-NMI and SMI yet.*/
5908	&& !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI));
5909	Assert(!fBlockSti \|\| pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
5910
5911	int rc = VINF_SUCCESS;
5912	if (pVCpu->hm.s.Event.fPending) /* First, inject any pending HM events. */
5913	{
5914	uint32_t uIntrType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hm.s.Event.u64IntrInfo);
5915	bool fInject = true;
5916	if (uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
5917	{
5918	rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
5919	AssertRCReturn(rc, rc);
5920	const bool fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
5921	if ( fBlockInt
5922	\|\| fBlockSti
5923	\|\| fBlockMovSS)
5924	{
5925	fInject = false;
5926	}
5927	}
5928	else if ( uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
5929	&& ( fBlockMovSS
5930	\|\| fBlockSti))
5931	{
5932	/* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
5933	fInject = false;
5934	}
5935
5936	if (fInject)
5937	{
5938	Log(("Injecting pending event\n"));
5939	rc = hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, pVCpu->hm.s.Event.u64IntrInfo, pVCpu->hm.s.Event.cbInstr,
5940	pVCpu->hm.s.Event.u32ErrCode, &uIntrState);
5941	AssertRCReturn(rc, rc);
5942	pVCpu->hm.s.Event.fPending = false;
5943	}
5944	else
5945	hmR0VmxSetIntWindowExitVmcs(pVCpu);
5946	} /** @todo SMI. SMIs take priority over NMIs. */
5947	else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_NMI)) /* NMI. NMIs take priority over regular interrupts . */
5948	{
5949	/* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
5950	if ( !fBlockMovSS
5951	&& !fBlockSti)
5952	{
5953	Log(("Injecting NMI\n"));
5954	RTGCUINTPTR uIntrInfo;
5955	uIntrInfo = X86_XCPT_NMI;
5956	uIntrInfo \|= (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
5957	uIntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5958	rc = hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, uIntrInfo, 0 /* cbInstr /, 0 / u32ErrCode */, &uIntrState);
5959	AssertRCReturn(rc, rc);
5960	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
5961	}
5962	else
5963	hmR0VmxSetIntWindowExitVmcs(pVCpu);
5964	}
5965	else if (VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC)))
5966	{
5967	/* Check if there are guest external interrupts (PIC/APIC) pending and inject them if the guest can receive them. */
5968	rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
5969	AssertRCReturn(rc, rc);
5970	const bool fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
5971	if ( !fBlockInt
5972	&& !fBlockSti
5973	&& !fBlockMovSS)
5974	{
5975	uint8_t u8Interrupt = 0;
5976	rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
5977	if (RT_SUCCESS(rc))
5978	{
5979	Log(("Injecting interrupt u8Interrupt=%#x\n", u8Interrupt));
5980	uint32_t u32IntrInfo = u8Interrupt \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
5981	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5982	rc = hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode */, &uIntrState);
5983	}
5984	else
5985	{
5986	/** @todo Does this actually happen? If not turn it into an assertion. */
5987	Assert(!VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC)));
5988	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
5989	rc = VINF_SUCCESS;
5990	}
5991	}
5992	else
5993	hmR0VmxSetIntWindowExitVmcs(pVCpu);
5994	}
5995
5996	/*
5997	* There's no need to clear the VM entry-interruption information field here if we're not injecting anything.
5998	* VT-x clears the valid bit on every VM-exit. See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
5999	*/
6000	int rc2 = hmR0VmxLoadGuestIntrState(pVCpu, uIntrState);
6001	AssertRC(rc2);
6002	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RESET);
6003	return rc;
6004	}
6005
6006
6007	/**
6008	* Sets an invalid-opcode (#UD) exception as pending-for-injection into the VM.
6009	*
6010	* @param pVCpu Pointer to the VMCPU.
6011	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6012	* out-of-sync. Make sure to update the required fields
6013	* before using them.
6014	*/
6015	DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6016	{
6017	uint32_t u32IntrInfo = X86_XCPT_UD \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
6018	STAM_COUNTER_INC(&pVCpu->hm.s.StatIntInject);
6019	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
6020	}
6021
6022
6023	/**
6024	* Injects a double-fault (#DF) exception into the VM.
6025	*
6026	* @returns VBox status code (informational status code included).
6027	* @param pVCpu Pointer to the VMCPU.
6028	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6029	* out-of-sync. Make sure to update the required fields
6030	* before using them.
6031	*/
6032	DECLINLINE(int) hmR0VmxInjectXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t *puIntrState)
6033	{
6034	/* Inject the double-fault. */
6035	uint32_t u32IntrInfo = X86_XCPT_DF \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
6036	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6037	u32IntrInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6038	STAM_COUNTER_INC(&pVCpu->hm.s.StatIntInject);
6039	return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode */, puIntrState);
6040	}
6041
6042
6043	/**
6044	* Sets a debug (#DB) exception as pending-for-injection into the VM.
6045	*
6046	* @param pVCpu Pointer to the VMCPU.
6047	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6048	* out-of-sync. Make sure to update the required fields
6049	* before using them.
6050	*/
6051	DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6052	{
6053	/* Inject the debug-exception. */
6054	uint32_t u32IntrInfo = X86_XCPT_DB \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
6055	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6056	STAM_COUNTER_INC(&pVCpu->hm.s.StatIntInject);
6057	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
6058	}
6059
6060
6061	/**
6062	* Sets an overflow (#OF) exception as pending-for-injection into the VM.
6063	*
6064	* @param pVCpu Pointer to the VMCPU.
6065	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6066	* out-of-sync. Make sure to update the required fields
6067	* before using them.
6068	* @param cbInstr The value of RIP that is to be pushed on the guest
6069	* stack.
6070	*/
6071	DECLINLINE(void) hmR0VmxSetPendingXcptOF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
6072	{
6073	/* Inject the overflow exception. */
6074	uint32_t u32IntrInfo = X86_XCPT_OF \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
6075	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6076	STAM_COUNTER_INC(&pVCpu->hm.s.StatIntInject);
6077	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, cbInstr, 0 /* u32ErrCode /, 0 / GCPtrFaultAddress */);
6078	}
6079
6080
6081	/**
6082	* Injects a general-protection (#GP) fault into the VM.
6083	*
6084	* @returns VBox status code (informational status code included).
6085	* @param pVCpu Pointer to the VMCPU.
6086	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6087	* out-of-sync. Make sure to update the required fields
6088	* before using them.
6089	* @param u32ErrorCode The error code associated with the #GP.
6090	*/
6091	DECLINLINE(int) hmR0VmxInjectXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fErrorCodeValid, uint32_t u32ErrorCode,
6092	uint32_t *puIntrState)
6093	{
6094	/* Inject the general-protection fault. */
6095	uint32_t u32IntrInfo = X86_XCPT_GP \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
6096	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6097	if (fErrorCodeValid)
6098	u32IntrInfo \|= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6099	STAM_COUNTER_INC(&pVCpu->hm.s.StatIntInject);
6100	return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntrInfo, 0 /* cbInstr */, u32ErrorCode, puIntrState);
6101	}
6102
6103
6104	/**
6105	* Sets a software interrupt (INTn) as pending-for-injection into the VM.
6106	*
6107	* @param pVCpu Pointer to the VMCPU.
6108	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6109	* out-of-sync. Make sure to update the required fields
6110	* before using them.
6111	* @param uVector The software interrupt vector number.
6112	* @param cbInstr The value of RIP that is to be pushed on the guest
6113	* stack.
6114	*/
6115	DECLINLINE(void) hmR0VmxSetPendingIntN(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint16_t uVector, uint32_t cbInstr)
6116	{
6117	/* Inject the INTn. */
6118	uint32_t u32IntrInfo = uVector \| (1 << VMX_EXIT_INTERRUPTION_INFO_VALID_SHIFT);
6119	u32IntrInfo \|= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6120	STAM_COUNTER_INC(&pVCpu->hm.s.StatIntInject);
6121	hmR0VmxSetPendingEvent(pVCpu, u32IntrInfo, cbInstr, 0 /* u32ErrCode /, 0 / GCPtrFaultAddress */);
6122	}
6123
6124
6125	/**
6126	* Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
6127	* stack.
6128	*
6129	* @returns VBox status code (information status code included).
6130	* @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
6131	* @param pVM Pointer to the VM.
6132	* @param pMixedCtx Pointer to the guest-CPU context.
6133	* @param uValue The value to push to the guest stack.
6134	*/
6135	DECLINLINE(int) hmR0VmxRealModeGuestStackPush(PVM pVM, PCPUMCTX pMixedCtx, uint16_t uValue)
6136	{
6137	/*
6138	* The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
6139	* virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
6140	* See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
6141	*/
6142	if (pMixedCtx->sp == 1)
6143	return VINF_EM_RESET;
6144	pMixedCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
6145	int rc = PGMPhysSimpleWriteGCPhys(pVM, pMixedCtx->ss.u64Base + pMixedCtx->sp, &uValue, sizeof(uint16_t));
6146	AssertRCReturn(rc, rc);
6147	return rc;
6148	}
6149
6150
6151	/**
6152	* Injects an event into the guest upon VM-entry by updating the relevant fields
6153	* in the VM-entry area in the VMCS.
6154	*
6155	* @returns VBox status code (informational error codes included).
6156	* @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
6157	* @retval VINF_EM_RESET if event injection resulted in a triple-fault.
6158	*
6159	* @param pVCpu Pointer to the VMCPU.
6160	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6161	* out-of-sync. Make sure to update the required fields
6162	* before using them.
6163	* @param u64IntrInfo The VM-entry interruption-information field.
6164	* @param cbInstr The VM-entry instruction length in bytes (for software
6165	* interrupts, exceptions and privileged software
6166	* exceptions).
6167	* @param u32ErrCode The VM-entry exception error code.
6168	* @param puIntrState Pointer to the current guest interruptibility-state.
6169	* This interruptibility-state will be updated if
6170	* necessary. This cannot not be NULL.
6171	*
6172	* @remarks No-long-jump zone!!!
6173	* @remarks Requires CR0!
6174	*/
6175	static int hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntrInfo, uint32_t cbInstr,
6176	uint32_t u32ErrCode, uint32_t *puIntrState)
6177	{
6178	/* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
6179	AssertMsg(u64IntrInfo >> 32 == 0, ("%#RX64\n", u64IntrInfo));
6180	Assert(puIntrState);
6181	uint32_t u32IntrInfo = (uint32_t)u64IntrInfo;
6182
6183	const uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(u32IntrInfo);
6184	const uint32_t uIntrType = VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntrInfo);
6185
6186	/* Cannot inject an NMI when block-by-MOV SS is in effect. */
6187	Assert( uIntrType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
6188	\|\| !((*puIntrState) & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS));
6189
6190	/* We require CR0 to check if the guest is in real-mode. */
6191	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6192	AssertRCReturn(rc, rc);
6193
6194	STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
6195
6196	/*
6197	* Hardware interrupts & exceptions cannot be delivered through the software interrupt redirection bitmap to the real
6198	* mode task in virtual-8086 mode. We must jump to the interrupt handler in the (real-mode) guest.
6199	* See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode" for interrupt & exception classes.
6200	* See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
6201	*/
6202	if (CPUMIsGuestInRealModeEx(pMixedCtx))
6203	{
6204	PVM pVM = pVCpu->CTX_SUFF(pVM);
6205	if (!pVM->hm.s.vmx.fUnrestrictedGuest)
6206	{
6207	Assert(PDMVmmDevHeapIsEnabled(pVM));
6208	Assert(pVM->hm.s.vmx.pRealModeTSS);
6209
6210	/* Save the required guest state bits from the VMCS. */
6211	rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
6212	rc \|= hmR0VmxSaveGuestGprs(pVCpu, pMixedCtx);
6213	AssertRCReturn(rc, rc);
6214
6215	/* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
6216	const size_t cbIdtEntry = 4;
6217	if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pMixedCtx->idtr.cbIdt)
6218	{
6219	/* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
6220	if (uVector == X86_XCPT_DF)
6221	return VINF_EM_RESET;
6222	else if (uVector == X86_XCPT_GP)
6223	{
6224	/* If we're injecting a #GP with no valid IDT entry, inject a double-fault. */
6225	return hmR0VmxInjectXcptDF(pVCpu, pMixedCtx, puIntrState);
6226	}
6227
6228	/* If we're injecting an interrupt/exception with no valid IDT entry, inject a general-protection fault. */
6229	/* No error codes for exceptions in real-mode. See Intel spec. 20.1.4 "Interrupt and Exception Handling" */
6230	return hmR0VmxInjectXcptGP(pVCpu, pMixedCtx, false /* fErrCodeValid /, 0 / u32ErrCode */, puIntrState);
6231	}
6232
6233	/* Software exceptions (#BP and #OF exceptions thrown as a result of INT 3 or INTO) */
6234	uint16_t uGuestIp = pMixedCtx->ip;
6235	if (VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntrInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT)
6236	{
6237	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF);
6238	/* #BP and #OF are both benign traps, we need to resume the next instruction. */
6239	uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
6240	}
6241	else if (VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntrInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT)
6242	uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
6243
6244	/* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
6245	uint16_t offIdtEntry = 0;
6246	RTSEL selIdtEntry = 0;
6247	RTGCPHYS GCPhysIdtEntry = (RTGCPHYS)pMixedCtx->idtr.pIdt + uVector * cbIdtEntry;
6248	rc = PGMPhysSimpleReadGCPhys(pVM, &offIdtEntry, GCPhysIdtEntry, sizeof(offIdtEntry));
6249	rc \|= PGMPhysSimpleReadGCPhys(pVM, &selIdtEntry, GCPhysIdtEntry + 2, sizeof(selIdtEntry));
6250	AssertRCReturn(rc, rc);
6251
6252	/* Construct the stack frame for the interrupt/exception handler. */
6253	rc = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->eflags.u32);
6254	rc \|= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->cs.Sel);
6255	rc \|= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, uGuestIp);
6256	AssertRCReturn(rc, rc);
6257
6258	/* Clear the required eflag bits and jump to the interrupt/exception handler. */
6259	if (rc == VINF_SUCCESS)
6260	{
6261	pMixedCtx->eflags.u32 &= ~(X86_EFL_IF \| X86_EFL_TF \| X86_EFL_RF \| X86_EFL_AC);
6262	pMixedCtx->rip = offIdtEntry;
6263	pMixedCtx->cs.Sel = selIdtEntry;
6264	pMixedCtx->cs.u64Base = selIdtEntry << cbIdtEntry;
6265	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SEGMENT_REGS
6266	\| HM_CHANGED_GUEST_RIP
6267	\| HM_CHANGED_GUEST_RFLAGS
6268	\| HM_CHANGED_GUEST_RSP;
6269
6270	/* We're clearing interrupts, which means no block-by-STI interrupt-inhibition. */
6271	if (*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
6272	{
6273	Assert( uIntrType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
6274	&& uIntrType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
6275	Log(("Clearing inhibition due to STI.\n"));
6276	*puIntrState &= ~VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
6277	}
6278	Log(("Injecting u32IntrInfo=%#x u32ErrCode=%#x instrlen=%#x\n", u32IntrInfo, u32ErrCode, cbInstr));
6279	}
6280	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RESET);
6281	return rc;
6282	}
6283	else
6284	{
6285	/*
6286	* For unrestricted execution enabled CPUs running real-mode guests, we must not set the deliver-error-code bit.
6287	* See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
6288	*/
6289	u32IntrInfo &= ~VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6290	}
6291	}
6292
6293	/* Validate. */
6294	Assert(VMX_EXIT_INTERRUPTION_INFO_VALID(u32IntrInfo)); /* Bit 31 (Valid bit) must be set by caller. */
6295	Assert(!VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK(u32IntrInfo)); /* Bit 12 MBZ. */
6296	Assert(!(u32IntrInfo & 0x7ffff000)); /* Bits 30:12 MBZ. */
6297
6298	/* Inject. */
6299	Log(("Injecting u32IntrInfo=%#x u32ErrCode=%#x instrlen=%#x\n", u32IntrInfo, u32ErrCode, cbInstr));
6300	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntrInfo);
6301	if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntrInfo))
6302	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
6303	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
6304	AssertRCReturn(rc, rc);
6305	return rc;
6306	}
6307
6308
6309	/**
6310	* Enters the VT-x session.
6311	*
6312	* @returns VBox status code.
6313	* @param pVM Pointer to the VM.
6314	* @param pVCpu Pointer to the VMCPU.
6315	* @param pCpu Pointer to the CPU info struct.
6316	*/
6317	VMMR0DECL(int) VMXR0Enter(PVM pVM, PVMCPU pVCpu, PHMGLOBLCPUINFO pCpu)
6318	{
6319	AssertPtr(pVM);
6320	AssertPtr(pVCpu);
6321	Assert(pVM->hm.s.vmx.fSupported);
6322	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6323	NOREF(pCpu);
6324
6325	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
6326
6327	/* Make sure we're in VMX root mode. */
6328	RTCCUINTREG u32HostCR4 = ASMGetCR4();
6329	if (!(u32HostCR4 & X86_CR4_VMXE))
6330	{
6331	LogRel(("VMXR0Enter: X86_CR4_VMXE bit in CR4 is not set!\n"));
6332	return VERR_VMX_X86_CR4_VMXE_CLEARED;
6333	}
6334
6335	/* Load the active VMCS as the current one. */
6336	int rc = VMXActivateVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
6337	if (RT_FAILURE(rc))
6338	return rc;
6339
6340	/** @todo this will change with preemption hooks where can can VMRESUME as long
6341	* as we're no preempted. */
6342	pVCpu->hm.s.fResumeVM = false;
6343	return VINF_SUCCESS;
6344	}
6345
6346
6347	/**
6348	* Leaves the VT-x session.
6349	*
6350	* @returns VBox status code.
6351	* @param pVM Pointer to the VM.
6352	* @param pVCpu Pointer to the VMCPU.
6353	* @param pCtx Pointer to the guest-CPU context.
6354	*/
6355	VMMR0DECL(int) VMXR0Leave(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
6356	{
6357	AssertPtr(pVCpu);
6358	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6359	NOREF(pVM);
6360	NOREF(pCtx);
6361
6362	/** @todo this will change with preemption hooks where we only VMCLEAR when
6363	* we are actually going to be preempted, not all the time like we
6364	* currently do. */
6365	/*
6366	* Sync the current VMCS (writes back internal data back into the VMCS region in memory)
6367	* and mark the VMCS launch-state as "clear".
6368	*/
6369	int rc = VMXClearVMCS(pVCpu->hm.s.vmx.HCPhysVmcs);
6370	return rc;
6371	}
6372
6373
6374	/**
6375	* Saves the host state in the VMCS host-state.
6376	* Sets up the VM-exit MSR-load area.
6377	*
6378	* The CPU state will be loaded from these fields on every successful VM-exit.
6379	*
6380	* @returns VBox status code.
6381	* @param pVM Pointer to the VM.
6382	* @param pVCpu Pointer to the VMCPU.
6383	*
6384	* @remarks No-long-jump zone!!!
6385	*/
6386	VMMR0DECL(int) VMXR0SaveHostState(PVM pVM, PVMCPU pVCpu)
6387	{
6388	AssertPtr(pVM);
6389	AssertPtr(pVCpu);
6390	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6391
6392	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
6393
6394	/* Nothing to do if the host-state-changed flag isn't set. This will later be optimized when preemption hooks are in place. */
6395	if (!(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_HOST_CONTEXT))
6396	return VINF_SUCCESS;
6397
6398	int rc = hmR0VmxSaveHostControlRegs(pVM, pVCpu);
6399	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostControlRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6400
6401	rc = hmR0VmxSaveHostSegmentRegs(pVM, pVCpu);
6402	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostSegmentRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6403
6404	rc = hmR0VmxSaveHostMsrs(pVM, pVCpu);
6405	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostMsrs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6406
6407	/* Reset the host-state-changed flag. */
6408	pVCpu->hm.s.fContextUseFlags &= ~HM_CHANGED_HOST_CONTEXT;
6409	return rc;
6410	}
6411
6412
6413	/**
6414	* Loads the guest state into the VMCS guest-state area. The CPU state will be
6415	* loaded from these fields on every successful VM-entry.
6416	*
6417	* Sets up the VM-entry MSR-load and VM-exit MSR-store areas.
6418	* Sets up the VM-entry controls.
6419	* Sets up the appropriate VMX non-root function to execute guest code based on
6420	* the guest CPU mode.
6421	*
6422	* @returns VBox status code.
6423	* @param pVM Pointer to the VM.
6424	* @param pVCpu Pointer to the VMCPU.
6425	* @param pCtx Pointer to the guest-CPU context.
6426	*
6427	* @remarks No-long-jump zone!!!
6428	*/
6429	VMMR0DECL(int) VMXR0LoadGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
6430	{
6431	AssertPtr(pVM);
6432	AssertPtr(pVCpu);
6433	AssertPtr(pCtx);
6434	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6435
6436	LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
6437
6438	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestState, x);
6439
6440	/* Determine real-on-v86 mode. */
6441	pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = false;
6442	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
6443	&& CPUMIsGuestInRealModeEx(pCtx))
6444	{
6445	pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = true;
6446	}
6447
6448	int rc = hmR0VmxLoadGuestEntryCtls(pVM, pVCpu, pCtx);
6449	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestEntryCtls! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6450
6451	rc = hmR0VmxLoadGuestExitCtls(pVM, pVCpu, pCtx);
6452	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupExitCtls failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6453
6454	rc = hmR0VmxLoadGuestActivityState(pVM, pVCpu, pCtx);
6455	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestActivityState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6456
6457	rc = hmR0VmxLoadGuestControlRegs(pVM, pVCpu, pCtx);
6458	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestControlRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6459
6460	rc = hmR0VmxLoadGuestSegmentRegs(pVM, pVCpu, pCtx);
6461	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestSegmentRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6462
6463	rc = hmR0VmxLoadGuestDebugRegs(pVM, pVCpu, pCtx);
6464	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestDebugRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6465
6466	rc = hmR0VmxLoadGuestMsrs(pVM, pVCpu, pCtx);
6467	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestMsrs! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6468
6469	rc = hmR0VmxLoadGuestApicState(pVM, pVCpu, pCtx);
6470	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestApicState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6471
6472	rc = hmR0VmxLoadGuestGprs(pVM, pVCpu, pCtx);
6473	AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestGprs! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6474
6475	rc = hmR0VmxSetupVMRunHandler(pVM, pVCpu, pCtx);
6476	AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupVMRunHandler! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
6477
6478	AssertMsg(!pVCpu->hm.s.fContextUseFlags,
6479	("Missed updating flags while loading guest state. pVM=%p pVCpu=%p fContextUseFlags=%#RX32\n",
6480	pVM, pVCpu, pVCpu->hm.s.fContextUseFlags));
6481
6482	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestState, x);
6483	return rc;
6484	}
6485
6486
6487	/**
6488	* Does the preparations before executing guest code in VT-x.
6489	*
6490	* This may cause longjmps to ring-3 and may even result in rescheduling to the
6491	* recompiler. We must be cautious what we do here regarding committing
6492	* guest-state information into the the VMCS assuming we assuredly execute the
6493	* guest in VT-x. If we fall back to the recompiler after updating VMCS and
6494	* clearing the common-state (TRPM/forceflags), we must undo those changes so
6495	* that the recompiler can (and should) use them when it resumes guest
6496	* execution. Otherwise such operations must be done when we can no longer
6497	* exit to ring-3.
6498	*
6499	* @returns VBox status code (informational status codes included).
6500	* @retval VINF_SUCCESS if we can proceed with running the guest.
6501	* @retval VINF_EM_RESET if a triple-fault occurs while injecting a double-fault
6502	* into the guest.
6503	* @retval VINF_* scheduling changes, we have to go back to ring-3.
6504	*
6505	* @param pVM Pointer to the VM.
6506	* @param pVCpu Pointer to the VMCPU.
6507	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6508	* out-of-sync. Make sure to update the required fields
6509	* before using them.
6510	* @param pVmxTransient Pointer to the VMX transient structure.
6511	*
6512	* @remarks Called with preemption disabled.
6513	*/
6514	DECLINLINE(int) hmR0VmxPreRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6515	{
6516	Assert(VMMRZCallRing3IsEnabled(pVCpu));
6517
6518	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
6519	PGMRZDynMapFlushAutoSet(pVCpu);
6520	#endif
6521
6522	/* Check force flag actions that might require us to go back to ring-3. */
6523	int rc = hmR0VmxCheckForceFlags(pVM, pVCpu, pMixedCtx);
6524	if (rc != VINF_SUCCESS)
6525	return rc;
6526
6527	/* Setup the Virtualized APIC accesses. pMixedCtx->msrApicBase is always up-to-date. It's not part of the VMCS. */
6528	if ( pVCpu->hm.s.vmx.u64MsrApicBase != pMixedCtx->msrApicBase
6529	&& (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC))
6530	{
6531	Assert(pVM->hm.s.vmx.HCPhysApicAccess);
6532	RTGCPHYS GCPhysApicBase;
6533	GCPhysApicBase = pMixedCtx->msrApicBase;
6534	GCPhysApicBase &= PAGE_BASE_GC_MASK;
6535
6536	/* Unalias any existing mapping. */
6537	rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
6538	AssertRCReturn(rc, rc);
6539
6540	/* Map the HC APIC-access page into the GC space, this also updates the shadow page tables if necessary. */
6541	Log2(("Mapped HC APIC-access page into GC: GCPhysApicBase=%#RGv\n", GCPhysApicBase));
6542	rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW \| X86_PTE_P);
6543	AssertRCReturn(rc, rc);
6544
6545	pVCpu->hm.s.vmx.u64MsrApicBase = pMixedCtx->msrApicBase;
6546	}
6547
6548	#ifdef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
6549	/* We disable interrupts so that we don't miss any interrupts that would flag preemption (IPI/timers etc.) */
6550	pVmxTransient->uEFlags = ASMIntDisableFlags();
6551	if (RTThreadPreemptIsPending(NIL_RTTHREAD))
6552	{
6553	ASMSetFlags(pVmxTransient->uEFlags);
6554	STAM_COUNTER_INC(&pVCpu->hm.s.StatPendingHostIrq);
6555	/* Don't use VINF_EM_RAW_INTERRUPT_HYPER as we can't assume the host does kernel preemption. Maybe some day? */
6556	return VINF_EM_RAW_INTERRUPT;
6557	}
6558	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
6559	#endif
6560
6561	/*
6562	* This clears force-flags, TRPM traps & pending HM events. We cannot safely restore the state if we exit to ring-3
6563	* (before running guest code) after calling this function (e.g. how do we reverse the effects of calling PDMGetInterrupt()?)
6564	* This is why this is done after all possible exits-to-ring-3 paths in this code.
6565	*/
6566	rc = hmR0VmxInjectPendingEvent(pVCpu, pMixedCtx);
6567	AssertRCReturn(rc, rc);
6568	return rc;
6569	}
6570
6571
6572	/**
6573	* Prepares to run guest code in VT-x and we've committed to doing so. This
6574	* means there is no backing out to ring-3 or anywhere else at this
6575	* point.
6576	*
6577	* @param pVM Pointer to the VM.
6578	* @param pVCpu Pointer to the VMCPU.
6579	* @param pMixedCtx Pointer to the guest-CPU context. The data may be
6580	* out-of-sync. Make sure to update the required fields
6581	* before using them.
6582	* @param pVmxTransient Pointer to the VMX transient structure.
6583	*
6584	* @remarks Called with preemption disabled.
6585	* @remarks No-long-jump zone!!!
6586	*/
6587	DECLINLINE(void) hmR0VmxPreRunGuestCommitted(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6588	{
6589	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6590	Assert(VMMR0IsLogFlushDisabled(pVCpu));
6591
6592	#ifndef VBOX_WITH_VMMR0_DISABLE_PREEMPTION
6593	/** @todo I don't see the point of this, VMMR0EntryFast() already disables interrupts for the entire period. */
6594	pVmxTransient->uEFlags = ASMIntDisableFlags();
6595	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
6596	#endif
6597
6598	/* Load the required guest state bits (for guest-state changes in the inner execution loop). */
6599	Assert(!(pVCpu->hm.s.fContextUseFlags & HM_CHANGED_HOST_CONTEXT));
6600	Log4(("LoadFlags=%#RX32\n", pVCpu->hm.s.fContextUseFlags));
6601	int rc = VINF_SUCCESS;
6602	if (pVCpu->hm.s.fContextUseFlags == HM_CHANGED_GUEST_RIP)
6603	{
6604	rc = hmR0VmxLoadGuestRip(pVM, pVCpu, pMixedCtx);
6605	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadMinimal);
6606	}
6607	else if (pVCpu->hm.s.fContextUseFlags)
6608	{
6609	rc = VMXR0LoadGuestState(pVM, pVCpu, pMixedCtx);
6610	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadFull);
6611	}
6612	AssertRC(rc);
6613	AssertMsg(!pVCpu->hm.s.fContextUseFlags, ("fContextUseFlags =%#x\n", pVCpu->hm.s.fContextUseFlags));
6614
6615	/* Cache the TPR-shadow for checking on every VM-exit if it might have changed. */
6616	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
6617	pVmxTransient->u8GuestTpr = pVCpu->hm.s.vmx.pbVirtApic[0x80];
6618
6619	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB-shootdowns, set this across the world switch. */
6620	hmR0VmxFlushTaggedTlb(pVCpu); /* Invalidate the appropriate guest entries from the TLB. */
6621
6622	/* Setup TSC-offsetting or intercept RDTSC(P)s and update the preemption timer. */
6623	if (pVmxTransient->fUpdateTscOffsettingAndPreemptTimer)
6624	{
6625	hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVM, pVCpu, pMixedCtx);
6626	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = false;
6627	}
6628
6629	/*
6630	* TPR patching (only active for 32-bit guests on 64-bit capable CPUs) when the CPU does not supported virtualizing
6631	* APIC accesses feature (VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC).
6632	*/
6633	if (pVM->hm.s.fTPRPatchingActive)
6634	{
6635	Assert(!CPUMIsGuestInLongMode(pVCpu));
6636
6637	/* Need guest's LSTAR MSR (which is part of the auto load/store MSRs in the VMCS), ensure we have the updated one. */
6638	rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
6639	AssertRC(rc);
6640
6641	/* The patch code uses the LSTAR as it's not used by a guest in 32-bit mode implicitly (i.e. SYSCALL is 64-bit only). */
6642	pVmxTransient->u64LStarMsr = ASMRdMsr(MSR_K8_LSTAR);
6643	ASMWrMsr(MSR_K8_LSTAR, pMixedCtx->msrLSTAR); /* pMixedCtx->msrLSTAR contains the guest's TPR,
6644	see hmR0VmxLoadGuestApicState(). */
6645	}
6646
6647	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
6648	TMNotifyStartOfExecution(pVCpu); /* Finally, notify TM to resume its clocks as we're about
6649	to start executing. */
6650	}
6651
6652
6653	/**
6654	* Performs some essential restoration of state after running guest code in
6655	* VT-x.
6656	*
6657	* @param pVM Pointer to the VM.
6658	* @param pVCpu Pointer to the VMCPU.
6659	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6660	* out-of-sync. Make sure to update the required fields
6661	* before using them.
6662	* @param pVmxTransient Pointer to the VMX transient structure.
6663	* @param rcVMRun Return code of VMLAUNCH/VMRESUME.
6664	*
6665	* @remarks Called with interrupts disabled.
6666	* @remarks No-long-jump zone!!! This function will however re-enable longjmps
6667	* unconditionally when it is safe to do so.
6668	*/
6669	DECLINLINE(void) hmR0VmxPostRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, int rcVMRun)
6670	{
6671	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6672	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatExit1, x);
6673
6674	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB-shootdowns. */
6675	ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for TLB-shootdowns. */
6676	pVCpu->hm.s.vmx.fUpdatedGuestState = 0; /* Exits/longjmps to ring-3 requires saving the guest state. */
6677	pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
6678	pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
6679
6680	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_RDTSC_EXIT))
6681	{
6682	/** @todo Find a way to fix hardcoding a guestimate. */
6683	TMCpuTickSetLastSeen(pVCpu, ASMReadTSC()
6684	+ pVCpu->hm.s.vmx.u64TSCOffset - 0x400 /* guestimate of world switch overhead in clock ticks */);
6685	}
6686
6687	TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
6688	Assert(!(ASMGetFlags() & X86_EFL_IF));
6689	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED);
6690
6691	/* Restore the effects of TPR patching if any. */
6692	if (pVM->hm.s.fTPRPatchingActive)
6693	{
6694	int rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
6695	AssertRC(rc);
6696	pMixedCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR); /* MSR_K8_LSTAR contains the guest TPR. */
6697	ASMWrMsr(MSR_K8_LSTAR, pVmxTransient->u64LStarMsr);
6698	}
6699
6700	ASMSetFlags(pVmxTransient->uEFlags); /* Enable interrupts. */
6701	pVCpu->hm.s.fResumeVM = true; /* Use VMRESUME instead of VMLAUNCH in the next run. */
6702
6703	/* Save the basic VM-exit reason. Refer Intel spec. 24.9.1 "Basic VM-exit Information". */
6704	uint32_t uExitReason;
6705	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
6706	rc \|= hmR0VmxReadEntryIntrInfoVmcs(pVmxTransient);
6707	AssertRC(rc);
6708	pVmxTransient->uExitReason = (uint16_t)VMX_EXIT_REASON_BASIC(uExitReason);
6709	pVmxTransient->fVMEntryFailed = !!VMX_ENTRY_INTERRUPTION_INFO_VALID(pVmxTransient->uEntryIntrInfo);
6710
6711	VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pMixedCtx);
6712	VMMRZCallRing3Enable(pVCpu); /* It is now safe to do longjmps to ring-3!!! */
6713
6714	/* If the VMLAUNCH/VMRESUME failed, we can bail out early. This does -not- cover VMX_EXIT_ERR_. /
6715	if (RT_UNLIKELY(rcVMRun != VINF_SUCCESS))
6716	{
6717	Log(("VM-entry failure: rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", rcVMRun, pVmxTransient->fVMEntryFailed));
6718	return;
6719	}
6720
6721	if (RT_LIKELY(!pVmxTransient->fVMEntryFailed))
6722	{
6723	/* Update the guest interruptibility-state from the VMCS. */
6724	hmR0VmxSaveGuestIntrState(pVCpu, pMixedCtx);
6725
6726	/*
6727	* If the TPR was raised by the guest, it wouldn't cause a VM-exit immediately. Instead we sync the TPR lazily whenever
6728	* we eventually get a VM-exit for any reason. This maybe expensive as PDMApicSetTPR() can longjmp to ring-3; also why
6729	* we do it outside of hmR0VmxSaveGuestState() which must never cause longjmps.
6730	*/
6731	if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
6732	&& pVmxTransient->u8GuestTpr != pVCpu->hm.s.vmx.pbVirtApic[0x80])
6733	{
6734	rc = PDMApicSetTPR(pVCpu, pVCpu->hm.s.vmx.pbVirtApic[0x80]);
6735	AssertRC(rc);
6736	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_VMX_GUEST_APIC_STATE;
6737	}
6738	}
6739	}
6740
6741
6742	/**
6743	* Runs the guest code using VT-x.
6744	*
6745	* @returns VBox status code.
6746	* @param pVM Pointer to the VM.
6747	* @param pVCpu Pointer to the VMCPU.
6748	* @param pCtx Pointer to the guest-CPU context.
6749	*
6750	* @remarks Called with preemption disabled.
6751	*/
6752	VMMR0DECL(int) VMXR0RunGuestCode(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
6753	{
6754	Assert(VMMRZCallRing3IsEnabled(pVCpu));
6755	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6756
6757	VMXTRANSIENT VmxTransient;
6758	VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
6759	int rc = VERR_INTERNAL_ERROR_5;
6760	uint32_t cLoops = 0;
6761	hmR0VmxUpdatePendingEvent(pVCpu, pCtx);
6762
6763	for (;; cLoops++)
6764	{
6765	Assert(!HMR0SuspendPending());
6766	AssertMsg(pVCpu->hm.s.idEnteredCpu == RTMpCpuId(),
6767	("Illegal migration! Entered on CPU %u Current %u cLoops=%u\n", (unsigned)pVCpu->hm.s.idEnteredCpu,
6768	(unsigned)RTMpCpuId(), cLoops));
6769
6770	/* Preparatory work for running guest code, this may return to ring-3 for some last minute updates. */
6771	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
6772	rc = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient);
6773	if (rc != VINF_SUCCESS)
6774	break;
6775
6776	/*
6777	* No longjmps to ring-3 from this point on!!!
6778	* Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
6779	* This also disables flushing of the R0-logger instance (if any).
6780	*/
6781	VMMRZCallRing3Disable(pVCpu);
6782	VMMRZCallRing3RemoveNotification(pVCpu);
6783	hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
6784
6785	rc = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
6786	/* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
6787
6788	/*
6789	* Restore any residual host-state and save any bits shared between host and guest into the guest-CPU state.
6790	* This will also re-enable longjmps to ring-3 when it has reached a safe point!!!
6791	*/
6792	hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rc);
6793	if (RT_UNLIKELY(rc != VINF_SUCCESS)) /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
6794	{
6795	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
6796	hmR0VmxReportWorldSwitchError(pVM, pVCpu, rc, pCtx, &VmxTransient);
6797	return rc;
6798	}
6799
6800	/* Handle the VM-exit. */
6801	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
6802	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
6803	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
6804	rc = (*g_apfnVMExitHandlers[VmxTransient.uExitReason])(pVCpu, pCtx, &VmxTransient);
6805	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
6806	if (rc != VINF_SUCCESS)
6807	break;
6808	else if (cLoops > pVM->hm.s.cMaxResumeLoops)
6809	{
6810	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMaxResume);
6811	rc = VINF_EM_RAW_INTERRUPT;
6812	break;
6813	}
6814	}
6815
6816	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
6817	if (rc == VERR_EM_INTERPRETER)
6818	rc = VINF_EM_RAW_EMULATE_INSTR;
6819	hmR0VmxExitToRing3(pVM, pVCpu, pCtx, rc);
6820	return rc;
6821	}
6822
6823	#if 1
6824	DECLINLINE(int) hmR0VmxHandleExit(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason)
6825	{
6826	int rc;
6827	switch (rcReason)
6828	{
6829	case VMX_EXIT_EPT_MISCONFIG: rc = hmR0VmxExitEptMisconfig(pVCpu, pMixedCtx, pVmxTransient); break;
6830	case VMX_EXIT_EPT_VIOLATION: rc = hmR0VmxExitEptViolation(pVCpu, pMixedCtx, pVmxTransient); break;
6831	case VMX_EXIT_IO_INSTR: rc = hmR0VmxExitIoInstr(pVCpu, pMixedCtx, pVmxTransient); break;
6832	case VMX_EXIT_CPUID: rc = hmR0VmxExitCpuid(pVCpu, pMixedCtx, pVmxTransient); break;
6833	case VMX_EXIT_RDTSC: rc = hmR0VmxExitRdtsc(pVCpu, pMixedCtx, pVmxTransient); break;
6834	case VMX_EXIT_RDTSCP: rc = hmR0VmxExitRdtscp(pVCpu, pMixedCtx, pVmxTransient); break;
6835	case VMX_EXIT_APIC_ACCESS: rc = hmR0VmxExitApicAccess(pVCpu, pMixedCtx, pVmxTransient); break;
6836	case VMX_EXIT_XCPT_NMI: rc = hmR0VmxExitXcptNmi(pVCpu, pMixedCtx, pVmxTransient); break;
6837	case VMX_EXIT_MOV_CRX: rc = hmR0VmxExitMovCRx(pVCpu, pMixedCtx, pVmxTransient); break;
6838	case VMX_EXIT_EXT_INT: rc = hmR0VmxExitExtInt(pVCpu, pMixedCtx, pVmxTransient); break;
6839	case VMX_EXIT_INT_WINDOW: rc = hmR0VmxExitIntWindow(pVCpu, pMixedCtx, pVmxTransient); break;
6840	case VMX_EXIT_MWAIT: rc = hmR0VmxExitMwait(pVCpu, pMixedCtx, pVmxTransient); break;
6841	case VMX_EXIT_MONITOR: rc = hmR0VmxExitMonitor(pVCpu, pMixedCtx, pVmxTransient); break;
6842	case VMX_EXIT_TASK_SWITCH: rc = hmR0VmxExitTaskSwitch(pVCpu, pMixedCtx, pVmxTransient); break;
6843	case VMX_EXIT_PREEMPT_TIMER: rc = hmR0VmxExitPreemptTimer(pVCpu, pMixedCtx, pVmxTransient); break;
6844	case VMX_EXIT_RDMSR: rc = hmR0VmxExitRdmsr(pVCpu, pMixedCtx, pVmxTransient); break;
6845	case VMX_EXIT_WRMSR: rc = hmR0VmxExitWrmsr(pVCpu, pMixedCtx, pVmxTransient); break;
6846	case VMX_EXIT_MOV_DRX: rc = hmR0VmxExitMovDRx(pVCpu, pMixedCtx, pVmxTransient); break;
6847	case VMX_EXIT_TPR_BELOW_THRESHOLD: rc = hmR0VmxExitTprBelowThreshold(pVCpu, pMixedCtx, pVmxTransient); break;
6848	case VMX_EXIT_HLT: rc = hmR0VmxExitHlt(pVCpu, pMixedCtx, pVmxTransient); break;
6849	case VMX_EXIT_INVD: rc = hmR0VmxExitInvd(pVCpu, pMixedCtx, pVmxTransient); break;
6850	case VMX_EXIT_INVLPG: rc = hmR0VmxExitInvlpg(pVCpu, pMixedCtx, pVmxTransient); break;
6851	case VMX_EXIT_RSM: rc = hmR0VmxExitRsm(pVCpu, pMixedCtx, pVmxTransient); break;
6852	case VMX_EXIT_MTF: rc = hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient); break;
6853	case VMX_EXIT_PAUSE: rc = hmR0VmxExitPause(pVCpu, pMixedCtx, pVmxTransient); break;
6854	case VMX_EXIT_XDTR_ACCESS: rc = hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient); break;
6855	case VMX_EXIT_TR_ACCESS: rc = hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient); break;
6856	case VMX_EXIT_WBINVD: rc = hmR0VmxExitWbinvd(pVCpu, pMixedCtx, pVmxTransient); break;
6857	case VMX_EXIT_XSETBV: rc = hmR0VmxExitXsetbv(pVCpu, pMixedCtx, pVmxTransient); break;
6858	case VMX_EXIT_RDRAND: rc = hmR0VmxExitRdrand(pVCpu, pMixedCtx, pVmxTransient); break;
6859	case VMX_EXIT_INVPCID: rc = hmR0VmxExitInvpcid(pVCpu, pMixedCtx, pVmxTransient); break;
6860	case VMX_EXIT_GETSEC: rc = hmR0VmxExitGetsec(pVCpu, pMixedCtx, pVmxTransient); break;
6861	case VMX_EXIT_RDPMC: rc = hmR0VmxExitRdpmc(pVCpu, pMixedCtx, pVmxTransient); break;
6862
6863	case VMX_EXIT_TRIPLE_FAULT: rc = hmR0VmxExitTripleFault(pVCpu, pMixedCtx, pVmxTransient); break;
6864	case VMX_EXIT_NMI_WINDOW: rc = hmR0VmxExitNmiWindow(pVCpu, pMixedCtx, pVmxTransient); break;
6865	case VMX_EXIT_INIT_SIGNAL: rc = hmR0VmxExitInitSignal(pVCpu, pMixedCtx, pVmxTransient); break;
6866	case VMX_EXIT_SIPI: rc = hmR0VmxExitSipi(pVCpu, pMixedCtx, pVmxTransient); break;
6867	case VMX_EXIT_IO_SMI: rc = hmR0VmxExitIoSmi(pVCpu, pMixedCtx, pVmxTransient); break;
6868	case VMX_EXIT_SMI: rc = hmR0VmxExitSmi(pVCpu, pMixedCtx, pVmxTransient); break;
6869	case VMX_EXIT_ERR_MSR_LOAD: rc = hmR0VmxExitErrMsrLoad(pVCpu, pMixedCtx, pVmxTransient); break;
6870	case VMX_EXIT_ERR_INVALID_GUEST_STATE: rc = hmR0VmxExitErrInvalidGuestState(pVCpu, pMixedCtx, pVmxTransient); break;
6871	case VMX_EXIT_ERR_MACHINE_CHECK: rc = hmR0VmxExitErrMachineCheck(pVCpu, pMixedCtx, pVmxTransient); break;
6872
6873	case VMX_EXIT_VMCALL:
6874	case VMX_EXIT_VMCLEAR:
6875	case VMX_EXIT_VMLAUNCH:
6876	case VMX_EXIT_VMPTRLD:
6877	case VMX_EXIT_VMPTRST:
6878	case VMX_EXIT_VMREAD:
6879	case VMX_EXIT_VMRESUME:
6880	case VMX_EXIT_VMWRITE:
6881	case VMX_EXIT_VMXOFF:
6882	case VMX_EXIT_VMXON:
6883	case VMX_EXIT_INVEPT:
6884	case VMX_EXIT_INVVPID:
6885	case VMX_EXIT_VMFUNC:
6886	rc = hmR0VmxExitSetPendingXcptUD(pVCpu, pMixedCtx, pVmxTransient);
6887	break;
6888	default:
6889	rc = hmR0VmxExitErrUndefined(pVCpu, pMixedCtx, pVmxTransient);
6890	break;
6891	}
6892	return rc;
6893	}
6894	#endif
6895
6896	#ifdef DEBUG
6897	/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
6898	# define VMX_ASSERT_PREEMPT_CPUID_VAR() \
6899	RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
6900	# define VMX_ASSERT_PREEMPT_CPUID() \
6901	do \
6902	{ \
6903	RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
6904	AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
6905	} while (0)
6906
6907	# define VMX_VALIDATE_EXIT_HANDLER_PARAMS() \
6908	do { \
6909	AssertPtr(pVCpu); \
6910	AssertPtr(pMixedCtx); \
6911	AssertPtr(pVmxTransient); \
6912	Assert(pVmxTransient->fVMEntryFailed == false); \
6913	Assert(ASMIntAreEnabled()); \
6914	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD)); \
6915	VMX_ASSERT_PREEMPT_CPUID_VAR(); \
6916	LogFunc(("\n")); \
6917	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD)); \
6918	if (VMMR0IsLogFlushDisabled(pVCpu)) \
6919	VMX_ASSERT_PREEMPT_CPUID(); \
6920	} while (0)
6921	# define VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() \
6922	do { \
6923	LogFunc(("\n")); \
6924	} while(0)
6925	#else /* Release builds */
6926	# define VMX_VALIDATE_EXIT_HANDLER_PARAMS() do { } while(0)
6927	# define VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() do { } while(0)
6928	#endif
6929
6930
6931	/**
6932	* Advances the guest RIP after reading it from the VMCS.
6933	*
6934	* @returns VBox status code.
6935	* @param pVCpu Pointer to the VMCPU.
6936	* @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6937	* out-of-sync. Make sure to update the required fields
6938	* before using them.
6939	* @param pVmxTransient Pointer to the VMX transient structure.
6940	*
6941	* @remarks No-long-jump zone!!!
6942	*/
6943	DECLINLINE(int) hmR0VmxAdvanceGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6944	{
6945	int rc = hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
6946	rc \|= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
6947	AssertRCReturn(rc, rc);
6948
6949	pMixedCtx->rip += pVmxTransient->cbInstr;
6950	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
6951	return rc;
6952	}
6953
6954
6955	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
6956	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
6957	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
6958	/**
6959	* VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
6960	*/
6961	static DECLCALLBACK(int) hmR0VmxExitExtInt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6962	{
6963	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
6964	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
6965	return VINF_SUCCESS;
6966	}
6967
6968
6969	/**
6970	* VM-exit handler for exceptions and NMIs (VMX_EXIT_XCPT_NMI).
6971	*/
6972	static DECLCALLBACK(int) hmR0VmxExitXcptNmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
6973	{
6974	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
6975	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
6976
6977	int rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
6978	AssertRCReturn(rc, rc);
6979
6980	uint32_t uIntrType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntrInfo);
6981	Assert( !(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_CONTROLS_ACK_EXT_INT)
6982	&& uIntrType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
6983
6984	if (uIntrType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
6985	{
6986	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
6987	return VINF_EM_RAW_INTERRUPT;
6988	}
6989
6990	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
6991	rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
6992	if (RT_UNLIKELY(rc == VINF_VMX_DOUBLE_FAULT))
6993	{
6994	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
6995	return VINF_SUCCESS;
6996	}
6997	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
6998	{
6999	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
7000	return rc;
7001	}
7002
7003	uint32_t uExitIntrInfo = pVmxTransient->uExitIntrInfo;
7004	uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(uExitIntrInfo);
7005	switch (uIntrType)
7006	{
7007	case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
7008	Assert(uVector == X86_XCPT_DB \|\| uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF);
7009	/* no break */
7010	case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT: /* Hardware exception. */
7011	{
7012	switch (uVector)
7013	{
7014	case X86_XCPT_PF: rc = hmR0VmxExitXcptPF(pVCpu, pMixedCtx, pVmxTransient); break;
7015	case X86_XCPT_GP: rc = hmR0VmxExitXcptGP(pVCpu, pMixedCtx, pVmxTransient); break;
7016	case X86_XCPT_NM: rc = hmR0VmxExitXcptNM(pVCpu, pMixedCtx, pVmxTransient); break;
7017	case X86_XCPT_MF: rc = hmR0VmxExitXcptMF(pVCpu, pMixedCtx, pVmxTransient); break;
7018	case X86_XCPT_DB: rc = hmR0VmxExitXcptDB(pVCpu, pMixedCtx, pVmxTransient); break;
7019	case X86_XCPT_BP: rc = hmR0VmxExitXcptBP(pVCpu, pMixedCtx, pVmxTransient); break;
7020	#ifdef VBOX_ALWAYS_TRAP_ALL_EXCEPTIONS
7021	case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF);
7022	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
7023	case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
7024	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
7025	case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
7026	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
7027	case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
7028	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
7029	case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
7030	rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
7031	#endif
7032	default:
7033	{
7034	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7035	AssertRCReturn(rc, rc);
7036
7037	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
7038	if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
7039	{
7040	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
7041	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
7042	rc = hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
7043	rc \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
7044	AssertRCReturn(rc, rc);
7045	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(uExitIntrInfo),
7046	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode,
7047	0 /* GCPtrFaultAddress */);
7048	AssertRCReturn(rc, rc);
7049	}
7050	else
7051	{
7052	AssertMsgFailed(("Unexpected VM-exit caused by exception %#x\n", uVector));
7053	rc = VERR_VMX_UNEXPECTED_EXCEPTION;
7054	}
7055	break;
7056	}
7057	}
7058	break;
7059	}
7060
7061	case VMX_EXIT_INTERRUPTION_INFO_TYPE_DB_XCPT:
7062	default:
7063	{
7064	rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_CODE;
7065	AssertMsgFailed(("Unexpected interruption code %#x\n", VMX_EXIT_INTERRUPTION_INFO_TYPE(uExitIntrInfo)));
7066	break;
7067	}
7068	}
7069	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
7070	return rc;
7071	}
7072
7073
7074	/**
7075	* VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
7076	*/
7077	static DECLCALLBACK(int) hmR0VmxExitIntWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7078	{
7079	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7080
7081	/* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
7082	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INT_WINDOW_EXIT);
7083	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_INT_WINDOW_EXIT;
7084	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls);
7085	AssertRCReturn(rc, rc);
7086
7087	/* Deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectEvent() and resume guest execution. */
7088	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
7089	return VINF_SUCCESS;
7090	}
7091
7092
7093	/**
7094	* VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
7095	*/
7096	static DECLCALLBACK(int) hmR0VmxExitNmiWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7097	{
7098	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7099	AssertMsgFailed(("Unexpected NMI-window exit.\n"));
7100	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7101	}
7102
7103
7104	/**
7105	* VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
7106	*/
7107	static DECLCALLBACK(int) hmR0VmxExitWbinvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7108	{
7109	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7110	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWbinvd);
7111	return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7112	}
7113
7114
7115	/**
7116	* VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
7117	*/
7118	static DECLCALLBACK(int) hmR0VmxExitInvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7119	{
7120	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7121	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvd);
7122	return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7123	}
7124
7125
7126	/**
7127	* VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
7128	*/
7129	static DECLCALLBACK(int) hmR0VmxExitCpuid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7130	{
7131	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7132	PVM pVM = pVCpu->CTX_SUFF(pVM);
7133	int rc = EMInterpretCpuId(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7134	if (RT_LIKELY(rc == VINF_SUCCESS))
7135	{
7136	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7137	Assert(pVmxTransient->cbInstr == 2);
7138	}
7139	else
7140	{
7141	AssertMsgFailed(("hmR0VmxExitCpuid: EMInterpretCpuId failed with %Rrc\n", rc));
7142	rc = VERR_EM_INTERPRETER;
7143	}
7144	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCpuid);
7145	return rc;
7146	}
7147
7148
7149	/**
7150	* VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
7151	*/
7152	static DECLCALLBACK(int) hmR0VmxExitGetsec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7153	{
7154	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7155	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
7156	AssertRCReturn(rc, rc);
7157
7158	if (pMixedCtx->cr4 & X86_CR4_SMXE)
7159	return VINF_EM_RAW_EMULATE_INSTR;
7160
7161	AssertMsgFailed(("hmR0VmxExitGetsec: unexpected VM-exit when CR4.SMXE is 0.\n"));
7162	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7163	}
7164
7165
7166	/**
7167	* VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
7168	*/
7169	static DECLCALLBACK(int) hmR0VmxExitRdtsc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7170	{
7171	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7172	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
7173	AssertRCReturn(rc, rc);
7174
7175	PVM pVM = pVCpu->CTX_SUFF(pVM);
7176	rc = EMInterpretRdtsc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7177	if (RT_LIKELY(rc == VINF_SUCCESS))
7178	{
7179	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7180	Assert(pVmxTransient->cbInstr == 2);
7181	/* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
7182	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TSC_OFFSETTING)
7183	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
7184	}
7185	else
7186	{
7187	AssertMsgFailed(("hmR0VmxExitRdtsc: EMInterpretRdtsc failed with %Rrc\n", rc));
7188	rc = VERR_EM_INTERPRETER;
7189	}
7190	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
7191	return rc;
7192	}
7193
7194
7195	/**
7196	* VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
7197	*/
7198	static DECLCALLBACK(int) hmR0VmxExitRdtscp(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7199	{
7200	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7201	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
7202	rc \|= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx); /* For MSR_K8_TSC_AUX */
7203	AssertRCReturn(rc, rc);
7204
7205	PVM pVM = pVCpu->CTX_SUFF(pVM);
7206	rc = EMInterpretRdtscp(pVM, pVCpu, pMixedCtx);
7207	if (RT_LIKELY(rc == VINF_SUCCESS))
7208	{
7209	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7210	Assert(pVmxTransient->cbInstr == 3);
7211	/* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
7212	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TSC_OFFSETTING)
7213	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
7214	}
7215	else
7216	{
7217	AssertMsgFailed(("hmR0VmxExitRdtscp: EMInterpretRdtscp failed with %Rrc\n", rc));
7218	rc = VERR_EM_INTERPRETER;
7219	}
7220	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
7221	return rc;
7222	}
7223
7224
7225	/**
7226	* VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
7227	*/
7228	static DECLCALLBACK(int) hmR0VmxExitRdpmc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7229	{
7230	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7231	int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
7232	rc \|= hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx); /** @todo review if CR0 is really required by EM. */
7233	AssertRCReturn(rc, rc);
7234
7235	PVM pVM = pVCpu->CTX_SUFF(pVM);
7236	rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7237	if (RT_LIKELY(rc == VINF_SUCCESS))
7238	{
7239	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7240	Assert(pVmxTransient->cbInstr == 2);
7241	}
7242	else
7243	{
7244	AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
7245	rc = VERR_EM_INTERPRETER;
7246	}
7247	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdpmc);
7248	return rc;
7249	}
7250
7251
7252	/**
7253	* VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
7254	*/
7255	static DECLCALLBACK(int) hmR0VmxExitInvlpg(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7256	{
7257	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7258	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
7259	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
7260	AssertRCReturn(rc, rc);
7261
7262	PVM pVM = pVCpu->CTX_SUFF(pVM);
7263	VBOXSTRICTRC rc2 = EMInterpretInvlpg(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), pVmxTransient->uExitQualification);
7264	rc = VBOXSTRICTRC_VAL(rc2);
7265	if (RT_LIKELY(rc == VINF_SUCCESS))
7266	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7267	else
7268	{
7269	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitInvlpg: EMInterpretInvlpg %RGv failed with %Rrc\n",
7270	pVmxTransient->uExitQualification, rc));
7271	rc = VERR_EM_INTERPRETER;
7272	}
7273	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvlpg);
7274	return rc;
7275	}
7276
7277
7278	/**
7279	* VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
7280	*/
7281	static DECLCALLBACK(int) hmR0VmxExitMonitor(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7282	{
7283	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7284	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7285	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7286	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7287	AssertRCReturn(rc, rc);
7288
7289	PVM pVM = pVCpu->CTX_SUFF(pVM);
7290	rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7291	if (RT_LIKELY(rc == VINF_SUCCESS))
7292	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7293	else
7294	{
7295	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMonitor: EMInterpretMonitor failed with %Rrc\n", rc));
7296	rc = VERR_EM_INTERPRETER;
7297	}
7298	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMonitor);
7299	return rc;
7300	}
7301
7302
7303	/**
7304	* VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
7305	*/
7306	static DECLCALLBACK(int) hmR0VmxExitMwait(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7307	{
7308	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7309	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7310	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7311	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7312	AssertRCReturn(rc, rc);
7313
7314	PVM pVM = pVCpu->CTX_SUFF(pVM);
7315	VBOXSTRICTRC rc2 = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7316	rc = VBOXSTRICTRC_VAL(rc2);
7317	if (RT_LIKELY( rc == VINF_SUCCESS
7318	\|\| rc == VINF_EM_HALT))
7319	{
7320	int rc3 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7321	AssertRCReturn(rc3, rc3);
7322
7323	if ( rc == VINF_EM_HALT
7324	&& EMShouldContinueAfterHalt(pVCpu, pMixedCtx))
7325	{
7326	rc = VINF_SUCCESS;
7327	}
7328	}
7329	else
7330	{
7331	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMwait: EMInterpretMWait failed with %Rrc\n", rc));
7332	rc = VERR_EM_INTERPRETER;
7333	}
7334	AssertMsg(rc == VINF_SUCCESS \|\| rc == VINF_EM_HALT \|\| rc == VERR_EM_INTERPRETER,
7335	("hmR0VmxExitMwait: failed, invalid error code %Rrc\n", rc));
7336	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMwait);
7337	return rc;
7338	}
7339
7340
7341	/**
7342	* VM-exit handler for RSM (VMX_EXIT_RSM). Unconditional VM-exit.
7343	*/
7344	static DECLCALLBACK(int) hmR0VmxExitRsm(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7345	{
7346	/*
7347	* Execution of RSM outside of SMM mode causes #UD regardless of VMX root or VMX non-root mode. In theory, we should never
7348	* get this VM-exit. This can happen only if dual-monitor treatment of SMI and VMX is enabled, which can (only?) be done by
7349	* executing VMCALL in VMX root operation. If we get here something funny is going on.
7350	* See Intel spec. "33.15.5 Enabling the Dual-Monitor Treatment".
7351	*/
7352	AssertMsgFailed(("Unexpected RSM VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7353	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7354	}
7355
7356
7357	/**
7358	* VM-exit handler for SMI (VMX_EXIT_SMI). Unconditional VM-exit.
7359	*/
7360	static DECLCALLBACK(int) hmR0VmxExitSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7361	{
7362	/*
7363	* This can only happen if we support dual-monitor treatment of SMI, which can be activated by executing VMCALL in VMX
7364	* root operation. If we get there there is something funny going on.
7365	* See Intel spec. "33.15.6 Activating the Dual-Monitor Treatment" and Intel spec. 25.3 "Other Causes of VM-Exits"
7366	*/
7367	AssertMsgFailed(("Unexpected SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7368	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7369	}
7370
7371
7372	/**
7373	* VM-exit handler for IO SMI (VMX_EXIT_IO_SMI). Unconditional VM-exit.
7374	*/
7375	static DECLCALLBACK(int) hmR0VmxExitIoSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7376	{
7377	/* Same treatment as VMX_EXIT_SMI. See comment in hmR0VmxExitSmi(). */
7378	AssertMsgFailed(("Unexpected IO SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7379	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7380	}
7381
7382
7383	/**
7384	* VM-exit handler for SIPI (VMX_EXIT_SIPI). Conditional VM-exit.
7385	*/
7386	static DECLCALLBACK(int) hmR0VmxExitSipi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7387	{
7388	/*
7389	* SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest activity state is used. We currently
7390	* don't make use of it (see hmR0VmxLoadGuestActivityState()) as our guests don't have direct access to the host LAPIC.
7391	* See Intel spec. 25.3 "Other Causes of VM-exits".
7392	*/
7393	AssertMsgFailed(("Unexpected SIPI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7394	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7395	}
7396
7397
7398	/**
7399	* VM-exit handler for INIT signal (VMX_EXIT_INIT_SIGNAL). Unconditional
7400	* VM-exit.
7401	*/
7402	static DECLCALLBACK(int) hmR0VmxExitInitSignal(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7403	{
7404	/*
7405	* INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM. See Intel spec. "33.14.1 Default Treatment of
7406	* SMI Delivery" and "29.3 VMX Instructions" for "VMXON". It is -NOT- blocked in VMX non-root operation so we can potentially
7407	* still get these exits. See Intel spec. "23.8 Restrictions on VMX operation".
7408	*/
7409	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7410	return VINF_SUCCESS; /** @todo r=ramshankar: correct?. */
7411	}
7412
7413
7414	/**
7415	* VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
7416	* VM-exit.
7417	*/
7418	static DECLCALLBACK(int) hmR0VmxExitTripleFault(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7419	{
7420	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7421	return VINF_EM_RESET;
7422	}
7423
7424
7425	/**
7426	* VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
7427	*/
7428	static DECLCALLBACK(int) hmR0VmxExitHlt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7429	{
7430	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7431	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_HLT_EXIT);
7432	int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
7433	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7434	AssertRCReturn(rc, rc);
7435
7436	pMixedCtx->rip++;
7437	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
7438	if (EMShouldContinueAfterHalt(pVCpu, pMixedCtx)) /* Requires eflags. */
7439	rc = VINF_SUCCESS;
7440	else
7441	rc = VINF_EM_HALT;
7442
7443	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
7444	return rc;
7445	}
7446
7447
7448	/**
7449	* VM-exit handler for instructions that result in a #UD exception delivered to the guest.
7450	*/
7451	static DECLCALLBACK(int) hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7452	{
7453	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7454	hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
7455	return VINF_SUCCESS;
7456	}
7457
7458
7459	/**
7460	* VM-exit handler for expiry of the VMX preemption timer.
7461	*/
7462	static DECLCALLBACK(int) hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7463	{
7464	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7465
7466	/* If we're saving the preemption-timer value on every VM-exit & we've reached zero, reset it up on next VM-entry. */
7467	if (pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_CONTROLS_SAVE_VMX_PREEMPT_TIMER)
7468	pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
7469
7470	/* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
7471	PVM pVM = pVCpu->CTX_SUFF(pVM);
7472	bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
7473	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
7474	return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
7475	}
7476
7477
7478	/**
7479	* VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
7480	*/
7481	static DECLCALLBACK(int) hmR0VmxExitXsetbv(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7482	{
7483	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7484	/* We expose XSETBV to the guest, fallback to the recompiler for emulation. */
7485	/** @todo check if XSETBV is supported by the recompiler. */
7486	return VERR_EM_INTERPRETER;
7487	}
7488
7489
7490	/**
7491	* VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
7492	*/
7493	static DECLCALLBACK(int) hmR0VmxExitInvpcid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7494	{
7495	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7496	/* The guest should not invalidate the host CPU's TLBs, fallback to recompiler. */
7497	/** @todo implement EMInterpretInvpcid() */
7498	return VERR_EM_INTERPRETER;
7499	}
7500
7501
7502	/**
7503	* VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE).
7504	* Error VM-exit.
7505	*/
7506	static DECLCALLBACK(int) hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7507	{
7508	uint32_t uIntrState;
7509	RTHCUINTREG uHCReg;
7510	uint64_t u64Val;
7511
7512	int rc = hmR0VmxReadEntryIntrInfoVmcs(pVmxTransient);
7513	rc \|= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
7514	rc \|= hmR0VmxReadEntryInstrLenVmcs(pVCpu, pVmxTransient);
7515	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
7516	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
7517	AssertRCReturn(rc, rc);
7518
7519	Log(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntrInfo));
7520	Log(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
7521	Log(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
7522	Log(("VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE %#RX32\n", uIntrState));
7523
7524	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR0, &u64Val); AssertRC(rc);
7525	Log(("VMX_VMCS_GUEST_CR0 %#RX64\n", u64Val));
7526	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
7527	Log(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
7528	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
7529	Log(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
7530	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
7531	Log(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
7532	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
7533	Log(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
7534	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
7535	Log(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
7536
7537	PVM pVM = pVCpu->CTX_SUFF(pVM);
7538	HMDumpRegs(pVM, pVCpu, pMixedCtx);
7539
7540	return VERR_VMX_INVALID_GUEST_STATE;
7541	}
7542
7543
7544	/**
7545	* VM-exit handler for VM-entry failure due to an MSR-load
7546	* (VMX_EXIT_ERR_MSR_LOAD). Error VM-exit.
7547	*/
7548	static DECLCALLBACK(int) hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7549	{
7550	AssertMsgFailed(("Unexpected MSR-load exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7551	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7552	}
7553
7554
7555	/**
7556	* VM-exit handler for VM-entry failure due to a machine-check event
7557	* (VMX_EXIT_ERR_MACHINE_CHECK). Error VM-exit.
7558	*/
7559	static DECLCALLBACK(int) hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7560	{
7561	AssertMsgFailed(("Unexpected machine-check event exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7562	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7563	}
7564
7565
7566	/**
7567	* VM-exit handler for all undefined reasons. Should never ever happen.. in
7568	* theory.
7569	*/
7570	static DECLCALLBACK(int) hmR0VmxExitErrUndefined(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7571	{
7572	AssertMsgFailed(("Huh!? Undefined VM-exit reason %d. pVCpu=%p pMixedCtx=%p\n", pVmxTransient->uExitReason, pVCpu, pMixedCtx));
7573	return VERR_VMX_UNDEFINED_EXIT_CODE;
7574	}
7575
7576
7577	/**
7578	* VM-exit handler for XDTR (LGDT, SGDT, LIDT, SIDT) accesses
7579	* (VMX_EXIT_XDTR_ACCESS) and LDT and TR access (LLDT, LTR, SLDT, STR).
7580	* Conditional VM-exit.
7581	*/
7582	static DECLCALLBACK(int) hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7583	{
7584	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7585	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT. */
7586	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitXdtrAccess);
7587	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT)
7588	return VERR_EM_INTERPRETER;
7589	AssertMsgFailed(("Unexpected XDTR access. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7590	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7591	}
7592
7593
7594	/**
7595	* VM-exit handler for RDRAND (VMX_EXIT_RDRAND). Conditional VM-exit.
7596	*/
7597	static DECLCALLBACK(int) hmR0VmxExitRdrand(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7598	{
7599	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7600	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT. */
7601	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdrand);
7602	if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT)
7603	return VERR_EM_INTERPRETER;
7604	AssertMsgFailed(("Unexpected RDRAND exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7605	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7606	}
7607
7608
7609	/**
7610	* VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
7611	*/
7612	static DECLCALLBACK(int) hmR0VmxExitRdmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7613	{
7614	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7615	/* EMInterpretRdmsr() requires CR0, Eflags and SS segment register. */
7616	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7617	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7618	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7619	AssertRCReturn(rc, rc);
7620
7621	PVM pVM = pVCpu->CTX_SUFF(pVM);
7622	rc = EMInterpretRdmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7623	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER,
7624	("hmR0VmxExitRdmsr: failed, invalid error code %Rrc\n", rc));
7625	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
7626
7627	/* Update RIP and continue guest execution. */
7628	if (RT_LIKELY(rc == VINF_SUCCESS))
7629	{
7630	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7631	Assert(pVmxTransient->cbInstr == 2);
7632	}
7633	return rc;
7634	}
7635
7636
7637	/**
7638	* VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
7639	*/
7640	static DECLCALLBACK(int) hmR0VmxExitWrmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7641	{
7642	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7643	int rc = VINF_SUCCESS;
7644	PVM pVM = pVCpu->CTX_SUFF(pVM);
7645
7646	/* If TPR patching is active, LSTAR holds the guest TPR, writes to it must be propagated to the APIC. */
7647	if ( pVM->hm.s.fTPRPatchingActive
7648	&& pMixedCtx->ecx == MSR_K8_LSTAR)
7649	{
7650	Assert(!CPUMIsGuestInLongModeEx(pMixedCtx)); /* Requires EFER but it's always up-to-date. */
7651	if ((pMixedCtx->eax & 0xff) != pVmxTransient->u8GuestTpr)
7652	{
7653	rc = PDMApicSetTPR(pVCpu, pMixedCtx->eax & 0xff);
7654	AssertRC(rc);
7655	}
7656
7657	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7658	Assert(pVmxTransient->cbInstr == 2);
7659	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
7660	return VINF_SUCCESS;
7661	}
7662
7663	/* Update MSRs that are part of the VMCS when MSR-bitmaps are not supported. */
7664	if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_MSR_BITMAPS))
7665	{
7666	switch (pMixedCtx->ecx)
7667	{
7668	case MSR_IA32_SYSENTER_CS: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SYSENTER_CS_MSR; break;
7669	case MSR_IA32_SYSENTER_EIP: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SYSENTER_EIP_MSR; break;
7670	case MSR_IA32_SYSENTER_ESP: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_SYSENTER_ESP_MSR; break;
7671	case MSR_K8_FS_BASE: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_FS_BASE_MSR; break;
7672	case MSR_K8_GS_BASE: pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_GS_BASE_MSR; break;
7673	}
7674	}
7675	#ifdef VBOX_STRICT
7676	else
7677	{
7678	/* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
7679	switch (pMixedCtx->ecx)
7680	{
7681	case MSR_IA32_SYSENTER_CS:
7682	case MSR_IA32_SYSENTER_EIP:
7683	case MSR_IA32_SYSENTER_ESP:
7684	case MSR_K8_FS_BASE:
7685	case MSR_K8_GS_BASE:
7686	AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%RX32\n", pMixedCtx->ecx));
7687	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7688	case MSR_K8_LSTAR:
7689	case MSR_K6_STAR:
7690	case MSR_K8_SF_MASK:
7691	case MSR_K8_TSC_AUX:
7692	AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%RX32\n", pMixedCtx->ecx));
7693	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7694	}
7695	}
7696	#endif
7697
7698	/* EMInterpretWrmsr() requires CR0, EFLAGS and SS segment register. */
7699	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7700	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7701	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7702	AssertRCReturn(rc, rc);
7703
7704	rc = EMInterpretWrmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
7705	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER, ("hmR0VmxExitWrmsr: failed, invalid error code %Rrc\n", rc));
7706	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
7707
7708	/* Update guest-state and continue execution. */
7709	if (RT_LIKELY(rc == VINF_SUCCESS))
7710	{
7711	rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7712
7713	/* If this is an X2APIC WRMSR access, update the APIC state as well. */
7714	if ( pMixedCtx->ecx >= MSR_IA32_X2APIC_START
7715	&& pMixedCtx->ecx <= MSR_IA32_X2APIC_END)
7716	{
7717	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_VMX_GUEST_APIC_STATE;
7718	}
7719	}
7720	return rc;
7721	}
7722
7723
7724	/**
7725	* VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
7726	*/
7727	static DECLCALLBACK(int) hmR0VmxExitPause(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7728	{
7729	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7730	/* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_PAUSE_EXIT. */
7731	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPause);
7732	if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_PAUSE_EXIT)
7733	return VERR_EM_INTERPRETER;
7734	AssertMsgFailed(("Unexpected PAUSE exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
7735	return VERR_VMX_UNEXPECTED_EXIT_CODE;
7736	}
7737
7738
7739	/**
7740	* VM-exit handler for when the TPR value is lowered below the specified
7741	* threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
7742	*/
7743	static DECLCALLBACK(int) hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7744	{
7745	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7746	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW);
7747
7748	/*
7749	* The TPR has already been updated, see hmR0VMXPostRunGuest(). RIP is also updated as part of the VM-exit by VT-x. Update
7750	* the threshold in the VMCS, deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectEvent() and
7751	* resume guest execution.
7752	*/
7753	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_VMX_GUEST_APIC_STATE;
7754	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
7755	return VINF_SUCCESS;
7756	}
7757
7758
7759	/**
7760	* VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
7761	* VM-exit.
7762	*
7763	* @retval VINF_SUCCESS when guest execution can continue.
7764	* @retval VINF_PGM_CHANGE_MODE when shadow paging mode changed, back to ring-3.
7765	* @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
7766	* @retval VERR_EM_INTERPRETER when something unexpected happened, fallback to
7767	* recompiler.
7768	*/
7769	static DECLCALLBACK(int) hmR0VmxExitMovCRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7770	{
7771	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7772	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
7773	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
7774	AssertRCReturn(rc, rc);
7775
7776	const RTGCUINTPTR uExitQualification = pVmxTransient->uExitQualification;
7777	const uint32_t uAccessType = VMX_EXIT_QUALIFICATION_CRX_ACCESS(uExitQualification);
7778	PVM pVM = pVCpu->CTX_SUFF(pVM);
7779	switch (uAccessType)
7780	{
7781	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_WRITE: /* MOV to CRx */
7782	{
7783	#if 0
7784	/* EMInterpretCRxWrite() references a lot of guest state (EFER, RFLAGS, Segment Registers, etc.) Sync entire state */
7785	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
7786	#else
7787	rc = hmR0VmxSaveGuestGprs(pVCpu, pMixedCtx);
7788	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
7789	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7790	#endif
7791	AssertRCReturn(rc, rc);
7792
7793	rc = EMInterpretCRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
7794	VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification),
7795	VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification));
7796	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_PGM_SYNC_CR3);
7797
7798	switch (VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification))
7799	{
7800	case 0: /* CR0 */
7801	Log(("CR0 write rc=%d\n", rc));
7802	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
7803	break;
7804	case 2: /* C2 **/
7805	/* Nothing to do here, CR2 it's not part of the VMCS. */
7806	break;
7807	case 3: /* CR3 */
7808	Assert(!pVM->hm.s.fNestedPaging \|\| !CPUMIsGuestPagingEnabledEx(pMixedCtx));
7809	Log(("CR3 write rc=%d\n", rc));
7810	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR3;
7811	break;
7812	case 4: /* CR4 */
7813	Log(("CR4 write rc=%d\n", rc));
7814	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR4;
7815	break;
7816	case 8: /* CR8 */
7817	Assert(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW));
7818	/* CR8 contains the APIC TPR. Was updated by EMInterpretCRxWrite(). */
7819	/* We don't need to update HM_CHANGED_VMX_GUEST_APIC_STATE here as this -cannot- happen with TPR shadowing. */
7820	break;
7821	default:
7822	AssertMsgFailed(("Invalid CRx register %#x\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)));
7823	break;
7824	}
7825
7826	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxWrite[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
7827	break;
7828	}
7829
7830	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ: /* MOV from CRx */
7831	{
7832	/* EMInterpretCRxRead() requires EFER MSR, CS. */
7833	rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7834	AssertRCReturn(rc, rc);
7835	Assert( !pVM->hm.s.fNestedPaging
7836	\|\| !CPUMIsGuestPagingEnabledEx(pMixedCtx)
7837	\|\| VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 3);
7838
7839	/* CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
7840	Assert( VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 8
7841	\|\| !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW));
7842
7843	rc = EMInterpretCRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
7844	VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification),
7845	VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification));
7846	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER);
7847	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxRead[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
7848	Log(("CRX CR%d Read access rc=%d\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification), rc));
7849	break;
7850	}
7851
7852	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_CLTS: /* CLTS (Clear Task-Switch Flag in CR0) */
7853	{
7854	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7855	AssertRCReturn(rc, rc);
7856	rc = EMInterpretCLTS(pVM, pVCpu);
7857	AssertRCReturn(rc, rc);
7858	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
7859	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
7860	Log(("CRX CLTS write rc=%d\n", rc));
7861	break;
7862	}
7863
7864	case VMX_EXIT_QUALIFICATION_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
7865	{
7866	rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7867	AssertRCReturn(rc, rc);
7868	rc = EMInterpretLMSW(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(uExitQualification));
7869	if (RT_LIKELY(rc == VINF_SUCCESS))
7870	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
7871	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
7872	Log(("CRX LMSW write rc=%d\n", rc));
7873	break;
7874	}
7875
7876	default:
7877	{
7878	AssertMsgFailed(("Invalid access-type in Mov CRx exit qualification %#x\n", uAccessType));
7879	rc = VERR_VMX_UNEXPECTED_EXCEPTION;
7880	}
7881	}
7882
7883	/* Validate possible error codes. */
7884	Assert(rc == VINF_SUCCESS \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_SYNC_CR3
7885	\|\| rc == VERR_VMX_UNEXPECTED_EXCEPTION);
7886	if (RT_SUCCESS(rc))
7887	{
7888	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
7889	AssertRCReturn(rc2, rc2);
7890	}
7891
7892	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
7893	return rc;
7894	}
7895
7896
7897	/**
7898	* VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
7899	* VM-exit.
7900	*/
7901	static DECLCALLBACK(int) hmR0VmxExitIoInstr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
7902	{
7903	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
7904	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
7905
7906	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
7907	rc \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
7908	rc \|= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
7909	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* Eflag checks in EMInterpretDisasCurrent(). */
7910	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx); /* CR0 checks & PGM* in EMInterpretDisasCurrent(). */
7911	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx); /* SELM checks in EMInterpretDisasCurrent(). */
7912	/* EFER also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
7913	AssertRCReturn(rc, rc);
7914
7915	/* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
7916	uint32_t uIOPort = VMX_EXIT_QUALIFICATION_IO_PORT(pVmxTransient->uExitQualification);
7917	uint32_t uIOWidth = VMX_EXIT_QUALIFICATION_IO_WIDTH(pVmxTransient->uExitQualification);
7918	bool fIOWrite = (VMX_EXIT_QUALIFICATION_IO_DIRECTION(pVmxTransient->uExitQualification)
7919	== VMX_EXIT_QUALIFICATION_IO_DIRECTION_OUT);
7920	bool fIOString = (VMX_EXIT_QUALIFICATION_IO_STRING(pVmxTransient->uExitQualification) == 1);
7921	Assert(uIOWidth == 0 \|\| uIOWidth == 1 \|\| uIOWidth == 3);
7922
7923	/* I/O operation lookup arrays. */
7924	static const uint32_t s_aIOSize[4] = { 1, 2, 0, 4 }; /* Size of the I/O Accesses. */
7925	static const uint32_t s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving the result (in AL/AX/EAX). */
7926
7927	const uint32_t cbSize = s_aIOSize[uIOWidth];
7928	const uint32_t cbInstr = pVmxTransient->cbInstr;
7929	PVM pVM = pVCpu->CTX_SUFF(pVM);
7930	if (fIOString)
7931	{
7932	/* INS/OUTS - I/O String instruction. */
7933	PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
7934	/** @todo for now manually disassemble later optimize by getting the fields from
7935	* the VMCS. */
7936	/** @todo VMX_VMCS_RO_EXIT_GUEST_LINEAR_ADDR contains the flat pointer
7937	* operand of the instruction. VMX_VMCS32_RO_EXIT_INSTR_INFO contains
7938	* segment prefix info. */
7939	rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, NULL);
7940	if (RT_SUCCESS(rc))
7941	{
7942	if (fIOWrite)
7943	{
7944	VBOXSTRICTRC rc2 = IOMInterpretOUTSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
7945	(DISCPUMODE)pDis->uAddrMode, cbSize);
7946	rc = VBOXSTRICTRC_VAL(rc2);
7947	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringWrite);
7948	}
7949	else
7950	{
7951	VBOXSTRICTRC rc2 = IOMInterpretINSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
7952	(DISCPUMODE)pDis->uAddrMode, cbSize);
7953	rc = VBOXSTRICTRC_VAL(rc2);
7954	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringRead);
7955	}
7956	}
7957	else
7958	{
7959	AssertMsg(rc == VERR_EM_INTERPRETER, ("rc=%Rrc RIP %#RX64\n", rc, pMixedCtx->rip));
7960	rc = VINF_EM_RAW_EMULATE_INSTR;
7961	}
7962	}
7963	else
7964	{
7965	/* IN/OUT - I/O instruction. */
7966	const uint32_t uAndVal = s_aIOOpAnd[uIOWidth];
7967	Assert(!VMX_EXIT_QUALIFICATION_IO_REP(pVmxTransient->uExitQualification));
7968	if (fIOWrite)
7969	{
7970	VBOXSTRICTRC rc2 = IOMIOPortWrite(pVM, pVCpu, uIOPort, pMixedCtx->eax & uAndVal, cbSize);
7971	rc = VBOXSTRICTRC_VAL(rc2);
7972	if (rc == VINF_IOM_R3_IOPORT_WRITE)
7973	HMR0SavePendingIOPortWrite(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbSize);
7974	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
7975	}
7976	else
7977	{
7978	uint32_t u32Result = 0;
7979	VBOXSTRICTRC rc2 = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbSize);
7980	rc = VBOXSTRICTRC_VAL(rc2);
7981	if (IOM_SUCCESS(rc))
7982	{
7983	/* Save result of I/O IN instr. in AL/AX/EAX. */
7984	pMixedCtx->eax = (pMixedCtx->eax & ~uAndVal) \| (u32Result & uAndVal);
7985	}
7986	else if (rc == VINF_IOM_R3_IOPORT_READ)
7987	HMR0SavePendingIOPortRead(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbSize);
7988	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
7989	}
7990	}
7991
7992	if (IOM_SUCCESS(rc))
7993	{
7994	pMixedCtx->rip += cbInstr;
7995	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
7996	if (RT_LIKELY(rc == VINF_SUCCESS))
7997	{
7998	rc = hmR0VmxSaveGuestDebugRegs(pVCpu, pMixedCtx); /* For DR7. */
7999	AssertRCReturn(rc, rc);
8000
8001	/* If any IO breakpoints are armed, then we should check if a debug trap needs to be generated. */
8002	if (pMixedCtx->dr[7] & X86_DR7_ENABLED_MASK)
8003	{
8004	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
8005	for (unsigned i = 0; i < 4; i++)
8006	{
8007	uint32_t uBPLen = s_aIOSize[X86_DR7_GET_LEN(pMixedCtx->dr[7], i)];
8008	if ( ( uIOPort >= pMixedCtx->dr[i]
8009	&& uIOPort < pMixedCtx->dr[i] + uBPLen)
8010	&& (pMixedCtx->dr[7] & (X86_DR7_L(i) \| X86_DR7_G(i)))
8011	&& (pMixedCtx->dr[7] & X86_DR7_RW(i, X86_DR7_RW_IO)) == X86_DR7_RW(i, X86_DR7_RW_IO))
8012	{
8013	Assert(CPUMIsGuestDebugStateActive(pVCpu));
8014	uint64_t uDR6 = ASMGetDR6();
8015
8016	/* Clear all breakpoint status flags and set the one we just hit. */
8017	uDR6 &= ~(X86_DR6_B0 \| X86_DR6_B1 \| X86_DR6_B2 \| X86_DR6_B3);
8018	uDR6 \|= (uint64_t)RT_BIT(i);
8019
8020	/*
8021	* Note: AMD64 Architecture Programmer's Manual 13.1:
8022	* Bits 15:13 of the DR6 register is never cleared by the processor and must
8023	* be cleared by software after the contents have been read.
8024	*/
8025	ASMSetDR6(uDR6);
8026
8027	/* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
8028	pMixedCtx->dr[7] &= ~X86_DR7_GD;
8029
8030	/* Paranoia. */
8031	pMixedCtx->dr[7] &= 0xffffffff; /* Upper 32 bits reserved. */
8032	pMixedCtx->dr[7] &= ~(RT_BIT(11) \| RT_BIT(12) \| RT_BIT(14) \| RT_BIT(15)); /* MBZ. */
8033	pMixedCtx->dr[7] \|= 0x400; /* MB1. */
8034
8035	/* Resync DR7 */
8036	/** @todo probably cheaper to just reload DR7, nothing else needs changing. */
8037	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_DEBUG;
8038
8039	/* Set #DB to be injected into the VM and continue guest execution. */
8040	hmR0VmxSetPendingXcptDB(pVCpu, pMixedCtx);
8041	break;
8042	}
8043	}
8044	}
8045	}
8046	}
8047
8048	#ifdef DEBUG
8049	if (rc == VINF_IOM_R3_IOPORT_READ)
8050	Assert(!fIOWrite);
8051	else if (rc == VINF_IOM_R3_IOPORT_WRITE)
8052	Assert(fIOWrite);
8053	else
8054	{
8055	AssertMsg( RT_FAILURE(rc)
8056	\|\| rc == VINF_SUCCESS
8057	\|\| rc == VINF_EM_RAW_EMULATE_INSTR
8058	\|\| rc == VINF_EM_RAW_GUEST_TRAP
8059	\|\| rc == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", rc));
8060	}
8061	#endif
8062
8063	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
8064	return rc;
8065	}
8066
8067
8068	/**
8069	* VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
8070	* VM-exit.
8071	*/
8072	static DECLCALLBACK(int) hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8073	{
8074	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
8075
8076	/* Check if this task-switch occurred while delivery an event through the guest IDT. */
8077	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8078	AssertRCReturn(rc, rc);
8079	if (VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE_IDT)
8080	{
8081	rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
8082	AssertRCReturn(rc, rc);
8083	if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
8084	{
8085	uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
8086	/* Software interrupts and exceptions will be regenerated when the recompiler restarts the instruction. */
8087	if ( uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
8088	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
8089	&& uIntType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
8090	{
8091	/* Save it as a pending event and it'll be converted to a TRPM event on the way out to ring-3. */
8092	Log(("Pending event on TaskSwitch uIntrType=%#x uVector=%#x\n", uIntType,
8093	VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo)));
8094	Assert(!pVCpu->hm.s.Event.fPending);
8095	pVCpu->hm.s.Event.fPending = true;
8096	pVCpu->hm.s.Event.u64IntrInfo = pVmxTransient->uIdtVectoringInfo;
8097	rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
8098	AssertRCReturn(rc, rc);
8099	if (VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringErrorCode))
8100	pVCpu->hm.s.Event.u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
8101	else
8102	pVCpu->hm.s.Event.u32ErrCode = 0;
8103	}
8104	}
8105	}
8106	/** @todo Emulate task switch someday, currently just going back to ring-3 for
8107	* emulation. */
8108	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
8109	return VERR_EM_INTERPRETER;
8110	}
8111
8112
8113	/**
8114	* VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
8115	*/
8116	static DECLCALLBACK(int) hmR0VmxExitMtf(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8117	{
8118	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
8119	Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_TRAP_FLAG);
8120	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MONITOR_TRAP_FLAG;
8121	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls);
8122	AssertRCReturn(rc, rc);
8123	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMtf);
8124	return VINF_EM_DBG_STOP;
8125	}
8126
8127
8128	/**
8129	* VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
8130	*/
8131	static DECLCALLBACK(int) hmR0VmxExitApicAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8132	{
8133	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
8134	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8135
8136	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
8137	rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
8138	if (RT_UNLIKELY(rc == VINF_VMX_DOUBLE_FAULT))
8139	return VINF_SUCCESS;
8140	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
8141	return rc;
8142
8143	#if 0
8144	/** @todo Investigate if IOMMMIOPhysHandler() requires a lot of state, for now
8145	* just sync the whole thing. */
8146	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8147	#else
8148	/* Aggressive state sync. for now. */
8149	rc = hmR0VmxSaveGuestGprs(pVCpu, pMixedCtx);
8150	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
8151	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8152	#endif
8153	AssertRCReturn(rc, rc);
8154
8155	/* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
8156	uint32_t uAccessType = VMX_EXIT_QUALIFICATION_APIC_ACCESS_TYPE(pVmxTransient->uExitQualification);
8157	switch (uAccessType)
8158	{
8159	case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
8160	case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
8161	{
8162	if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_USE_TPR_SHADOW)
8163	&& VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification) == 0x80)
8164	{
8165	AssertMsgFailed(("hmR0VmxExitApicAccess: can't touch TPR offset while using TPR shadowing.\n"));
8166	}
8167
8168	RTGCPHYS GCPhys = pMixedCtx->msrApicBase; /* Always up-to-date, msrApicBase is not part of the VMCS. */
8169	GCPhys &= PAGE_BASE_GC_MASK;
8170	GCPhys += VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification);
8171	PVM pVM = pVCpu->CTX_SUFF(pVM);
8172	VBOXSTRICTRC rc2 = IOMMMIOPhysHandler(pVM, pVCpu,
8173	(uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ) ? 0 : X86_TRAP_PF_RW,
8174	CPUMCTX2CORE(pMixedCtx), GCPhys);
8175	rc = VBOXSTRICTRC_VAL(rc2);
8176	Log(("ApicAccess %RGp %#x rc=%d\n", GCPhys,
8177	VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification), rc));
8178	if ( rc == VINF_SUCCESS
8179	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
8180	\|\| rc == VERR_PAGE_NOT_PRESENT)
8181	{
8182	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
8183	\| HM_CHANGED_VMX_GUEST_APIC_STATE;
8184	rc = VINF_SUCCESS;
8185	}
8186	break;
8187	}
8188
8189	default:
8190	rc = VINF_EM_RAW_EMULATE_INSTR;
8191	break;
8192	}
8193
8194	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
8195	return rc;
8196	}
8197
8198
8199	/**
8200	* VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
8201	* VM-exit.
8202	*/
8203	static DECLCALLBACK(int) hmR0VmxExitMovDRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8204	{
8205	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
8206
8207	/* We should -not- get this VM-exit if the guest is debugging. */
8208	if (CPUMIsGuestDebugStateActive(pVCpu))
8209	{
8210	AssertMsgFailed(("Unexpected MOV DRx exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
8211	return VERR_VMX_UNEXPECTED_EXIT_CODE;
8212	}
8213
8214	int rc = VERR_INTERNAL_ERROR_5;
8215	if ( !DBGFIsStepping(pVCpu)
8216	&& !CPUMIsHyperDebugStateActive(pVCpu))
8217	{
8218	Assert(!CPUMIsGuestDebugStateActive(pVCpu));
8219
8220	/* Don't intercept MOV DRx. */
8221	pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT;
8222	rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC_CONTROLS, pVCpu->hm.s.vmx.u32ProcCtls);
8223	AssertRCReturn(rc, rc);
8224
8225	/* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
8226	PVM pVM = pVCpu->CTX_SUFF(pVM);
8227	rc = CPUMR0LoadGuestDebugState(pVM, pVCpu, pMixedCtx, true /* include DR6 */);
8228	AssertRC(rc);
8229	Assert(CPUMIsGuestDebugStateActive(pVCpu));
8230
8231	#ifdef VBOX_WITH_STATISTICS
8232	rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8233	AssertRCReturn(rc, rc);
8234	if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
8235	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
8236	else
8237	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
8238	#endif
8239	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
8240	return VINF_SUCCESS;
8241	}
8242
8243	/** @todo clear VMX_VMCS_CTRL_PROC_EXEC_CONTROLS_MOV_DR_EXIT after the first
8244	* time and restore DRx registers afterwards */
8245	/*
8246	* EMInterpretDRx[Write\|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER, CS. EFER is always up-to-date, see
8247	* hmR0VmxSaveGuestAutoLoadStoreMsrs(). Update only the segment registers from the CPU.
8248	*/
8249	rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8250	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8251	AssertRCReturn(rc, rc);
8252
8253	PVM pVM = pVCpu->CTX_SUFF(pVM);
8254	if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
8255	{
8256	rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
8257	VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification),
8258	VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification));
8259	if (RT_SUCCESS(rc))
8260	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_DEBUG;
8261	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
8262	}
8263	else
8264	{
8265	rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
8266	VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification),
8267	VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification));
8268	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
8269	}
8270
8271	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER);
8272	if (RT_SUCCESS(rc))
8273	{
8274	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
8275	AssertRCReturn(rc2, rc2);
8276	}
8277	return rc;
8278	}
8279
8280
8281	/**
8282	* VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
8283	* Conditional VM-exit.
8284	*/
8285	static DECLCALLBACK(int) hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8286	{
8287	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
8288	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
8289
8290	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
8291	int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
8292	if (RT_UNLIKELY(rc == VINF_VMX_DOUBLE_FAULT))
8293	return VINF_SUCCESS;
8294	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
8295	return rc;
8296
8297	RTGCPHYS GCPhys = 0;
8298	rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
8299
8300	#if 0
8301	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
8302	#else
8303	/* Aggressive state sync. for now. */
8304	rc \|= hmR0VmxSaveGuestGprs(pVCpu, pMixedCtx);
8305	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
8306	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8307	#endif
8308	AssertRCReturn(rc, rc);
8309
8310	/*
8311	* If we succeed, resume guest execution.
8312	* If we fail in interpreting the instruction because we couldn't get the guest physical address
8313	* of the page containing the instruction via the guest's page tables (we would invalidate the guest page
8314	* in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
8315	* weird case. See @bugref{6043}.
8316	*/
8317	PVM pVM = pVCpu->CTX_SUFF(pVM);
8318	VBOXSTRICTRC rc2 = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pMixedCtx), GCPhys, UINT32_MAX);
8319	Log(("EPT misconfig at %#RX64 RIP=%#RX64 rc=%d\n", GCPhys, pMixedCtx->rip, rc));
8320	rc = VBOXSTRICTRC_VAL(rc2);
8321	if ( rc == VINF_SUCCESS
8322	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
8323	\|\| rc == VERR_PAGE_NOT_PRESENT)
8324	{
8325	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
8326	\| HM_CHANGED_VMX_GUEST_APIC_STATE;
8327	return VINF_SUCCESS;
8328	}
8329	return rc;
8330	}
8331
8332
8333	/**
8334	* VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
8335	* VM-exit.
8336	*/
8337	static DECLCALLBACK(int) hmR0VmxExitEptViolation(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8338	{
8339	VMX_VALIDATE_EXIT_HANDLER_PARAMS();
8340	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
8341
8342	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
8343	int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
8344	if (RT_UNLIKELY(rc == VINF_VMX_DOUBLE_FAULT))
8345	return VINF_SUCCESS;
8346	else if (RT_UNLIKELY(rc == VINF_EM_RESET))
8347	return rc;
8348
8349	RTGCPHYS GCPhys = 0;
8350	rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
8351	rc \|= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8352	#if 0
8353	rc \|= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
8354	#else
8355	/* Aggressive state sync. for now. */
8356	rc \|= hmR0VmxSaveGuestGprs(pVCpu, pMixedCtx);
8357	rc \|= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
8358	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8359	#endif
8360	AssertRCReturn(rc, rc);
8361
8362	/* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
8363	AssertMsg(((pVmxTransient->uExitQualification >> 7) & 3) != 2, ("%#RGv", pVmxTransient->uExitQualification));
8364
8365	RTGCUINT uErrorCode = 0;
8366	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_INSTR_FETCH)
8367	uErrorCode \|= X86_TRAP_PF_ID;
8368	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_DATA_WRITE)
8369	uErrorCode \|= X86_TRAP_PF_RW;
8370	if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_ENTRY_PRESENT)
8371	uErrorCode \|= X86_TRAP_PF_P;
8372
8373	TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
8374
8375	Log(("EPT violation %#x at %#RGv ErrorCode %#x CS:EIP=%04x:%#RX64\n", (uint32_t)pVmxTransient->uExitQualification, GCPhys,
8376	uErrorCode, pMixedCtx->cs.Sel, pMixedCtx->rip));
8377
8378	/* Handle the pagefault trap for the nested shadow table. */
8379	PVM pVM = pVCpu->CTX_SUFF(pVM);
8380	rc = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pMixedCtx), GCPhys);
8381	TRPMResetTrap(pVCpu);
8382
8383	/* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
8384	if ( rc == VINF_SUCCESS
8385	\|\| rc == VERR_PAGE_TABLE_NOT_PRESENT
8386	\|\| rc == VERR_PAGE_NOT_PRESENT)
8387	{
8388	/* Successfully synced our shadow page tables or emulation MMIO instruction. */
8389	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
8390	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
8391	\| HM_CHANGED_VMX_GUEST_APIC_STATE;
8392	return VINF_SUCCESS;
8393	}
8394
8395	Log(("EPT return to ring-3 rc=%d\n"));
8396	return rc;
8397	}
8398
8399
8400	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
8401	/* -=-=-=-=-=-=-=-=-=- VM-exit Exception Handlers -=-=-=-=-=-=-=-=-=-=- */
8402	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
8403	/**
8404	* VM-exit exception handler for #MF (Math Fault: floating point exception).
8405	*/
8406	static DECLCALLBACK(int) hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8407	{
8408	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8409
8410	int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8411	AssertRCReturn(rc, rc);
8412	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
8413
8414	if (!(pMixedCtx->cr0 & X86_CR0_NE))
8415	{
8416	/* Old-style FPU error reporting needs some extra work. */
8417	/** @todo don't fall back to the recompiler, but do it manually. */
8418	return VERR_EM_INTERPRETER;
8419	}
8420	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8421	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
8422	return rc;
8423	}
8424
8425
8426	/**
8427	* VM-exit exception handler for #BP (Breakpoint exception).
8428	*/
8429	static DECLCALLBACK(int) hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8430	{
8431	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8432
8433	/** @todo Try optimize this by not saving the entire guest state unless
8434	* really needed. */
8435	int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8436	AssertRCReturn(rc, rc);
8437	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
8438
8439	PVM pVM = pVCpu->CTX_SUFF(pVM);
8440	rc = DBGFRZTrap03Handler(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
8441	if (rc == VINF_EM_RAW_GUEST_TRAP)
8442	{
8443	rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
8444	rc \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
8445	rc \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
8446	AssertRCReturn(rc, rc);
8447
8448	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8449	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
8450	}
8451
8452	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RAW_GUEST_TRAP \|\| rc == VINF_EM_DBG_BREAKPOINT);
8453	return rc;
8454	}
8455
8456
8457	/**
8458	* VM-exit exception handler for #DB (Debug exception).
8459	*/
8460	static DECLCALLBACK(int) hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8461	{
8462	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8463
8464	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8465	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8466	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
8467	AssertRCReturn(rc, rc);
8468
8469	/* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
8470	uint64_t uDR6 = X86_DR6_INIT_VAL;
8471	uDR6 \|= (pVmxTransient->uExitQualification
8472	& (X86_DR6_B0 \| X86_DR6_B1 \| X86_DR6_B2 \| X86_DR6_B3 \| X86_DR6_BD \| X86_DR6_BS));
8473	PVM pVM = pVCpu->CTX_SUFF(pVM);
8474	rc = DBGFRZTrap01Handler(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uDR6);
8475	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
8476	if (rc == VINF_EM_RAW_GUEST_TRAP)
8477	{
8478	/* DR6, DR7.GD and IA32_DEBUGCTL.LBR are not updated yet. See Intel spec. 27.1 "Architectural State before a VM-Exit". */
8479	pMixedCtx->dr[6] = uDR6;
8480
8481	if (CPUMIsGuestDebugStateActive(pVCpu))
8482	ASMSetDR6(pMixedCtx->dr[6]);
8483
8484	/* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
8485	pMixedCtx->dr[7] &= ~X86_DR7_GD;
8486
8487	/* Paranoia. */
8488	pMixedCtx->dr[7] &= 0xffffffff; /* upper 32 bits reserved */
8489	pMixedCtx->dr[7] &= ~(RT_BIT(11) \| RT_BIT(12) \| RT_BIT(14) \| RT_BIT(15)); /* must be zero */
8490	pMixedCtx->dr[7] \|= 0x400; /* must be one */
8491
8492	/* Resync DR7. */
8493	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_DR7, pMixedCtx->dr[7]);
8494
8495	rc \|= hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
8496	rc \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
8497	rc \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
8498	AssertRCReturn(rc,rc);
8499	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8500	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
8501	return rc;
8502	}
8503	/* Return to ring 3 to deal with the debug exit code. */
8504	return rc;
8505	}
8506
8507
8508	/**
8509	* VM-exit exception handler for #NM (Device-not-available exception: floating
8510	* point exception).
8511	*/
8512	static DECLCALLBACK(int) hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8513	{
8514	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8515
8516	#ifndef VBOX_ALWAYS_TRAP_ALL_EXCEPTIONS
8517	Assert(!CPUMIsGuestFPUStateActive(pVCpu));
8518	#endif
8519
8520	/* We require CR0 and EFER. EFER is always up-to-date. */
8521	int rc = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
8522	AssertRCReturn(rc, rc);
8523
8524	/* Lazy FPU loading; Load the guest-FPU state transparently and continue execution of the guest. */
8525	PVM pVM = pVCpu->CTX_SUFF(pVM);
8526	rc = CPUMR0LoadGuestFPU(pVM, pVCpu, pMixedCtx);
8527	if (rc == VINF_SUCCESS)
8528	{
8529	Assert(CPUMIsGuestFPUStateActive(pVCpu));
8530	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_CR0;
8531	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowNM);
8532	return VINF_SUCCESS;
8533	}
8534
8535	/* Forward #NM to the guest. */
8536	Assert(rc == VINF_EM_RAW_GUEST_TRAP);
8537	rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
8538	AssertRCReturn(rc, rc);
8539	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8540	pVmxTransient->cbInstr, 0 /* error code /, 0 / GCPtrFaultAddress */);
8541	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
8542	return rc;
8543	}
8544
8545
8546	/**
8547	* VM-exit exception handler for #GP (General-protection exception).
8548	*
8549	* @remarks Requires pVmxTransient->uExitIntrInfo to be up-to-date.
8550	*/
8551	static DECLCALLBACK(int) hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8552	{
8553	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8554
8555	int rc = VERR_INTERNAL_ERROR_5;
8556
8557	if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
8558	{
8559	#ifdef VBOX_ALWAYS_TRAP_ALL_EXCEPTIONS
8560	/* If the guest is not in real-mode or we have unrestricted execution support, reflect #GP to the guest. */
8561	rc = hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
8562	rc \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
8563	rc \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
8564	rc \|= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
8565	AssertRCReturn(rc, rc);
8566	Log(("#GP Gst: RIP %#RX64\n", pMixedCtx->rip));
8567	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8568	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
8569	return rc;
8570	#else
8571	/* We don't intercept #GP. */
8572	AssertMsgFailed(("Unexpected VM-exit caused by #GP exception\n"));
8573	return VERR_VMX_UNEXPECTED_EXCEPTION;
8574	#endif
8575	}
8576
8577	Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
8578	Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
8579
8580	/* EMInterpretDisasCurrent() requires a lot of the state, save the entire state. */
8581	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8582	AssertRCReturn(rc, rc);
8583
8584	PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
8585	uint32_t cbOp = 0;
8586	PVM pVM = pVCpu->CTX_SUFF(pVM);
8587	rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbOp);
8588	if (RT_SUCCESS(rc))
8589	{
8590	rc = VINF_SUCCESS;
8591	Assert(cbOp == pDis->cbInstr);
8592	Log2(("#GP Disas OpCode=%u CS:EIP %04x:%#RX64\n", pDis->pCurInstr->uOpcode, pMixedCtx->cs.Sel, pMixedCtx->rip));
8593	switch (pDis->pCurInstr->uOpcode)
8594	{
8595	case OP_CLI:
8596	pMixedCtx->eflags.Bits.u1IF = 0;
8597	pMixedCtx->rip += pDis->cbInstr;
8598	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS;
8599	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCli);
8600	break;
8601
8602	case OP_STI:
8603	pMixedCtx->eflags.Bits.u1IF = 1;
8604	pMixedCtx->rip += pDis->cbInstr;
8605	EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
8606	Assert(VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
8607	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS;
8608	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitSti);
8609	break;
8610
8611	case OP_HLT:
8612	rc = VINF_EM_HALT;
8613	pMixedCtx->rip += pDis->cbInstr;
8614	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP;
8615	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
8616	break;
8617
8618	case OP_POPF:
8619	{
8620	Log(("POPF CS:RIP %04x:%#RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
8621	uint32_t cbParm = 0;
8622	uint32_t uMask = 0;
8623	if (pDis->fPrefix & DISPREFIX_OPSIZE)
8624	{
8625	cbParm = 4;
8626	uMask = 0xffffffff;
8627	}
8628	else
8629	{
8630	cbParm = 2;
8631	uMask = 0xffff;
8632	}
8633
8634	/* Get the stack pointer & pop the contents of the stack onto EFlags. */
8635	RTGCPTR GCPtrStack = 0;
8636	X86EFLAGS uEflags;
8637	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
8638	&GCPtrStack);
8639	if (RT_SUCCESS(rc))
8640	{
8641	Assert(sizeof(uEflags.u32) >= cbParm);
8642	uEflags.u32 = 0;
8643	rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &uEflags.u32, cbParm);
8644	}
8645	if (RT_FAILURE(rc))
8646	{
8647	rc = VERR_EM_INTERPRETER;
8648	break;
8649	}
8650	Log(("POPF %x -> %RGv mask=%x RIP=%#RX64\n", uEflags.u, pMixedCtx->rsp, uMask, pMixedCtx->rip));
8651	pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~(X86_EFL_POPF_BITS & uMask))
8652	\| (uEflags.u32 & X86_EFL_POPF_BITS & uMask);
8653	/* The RF bit is always cleared by POPF; see Intel Instruction reference for POPF. */
8654	pMixedCtx->eflags.Bits.u1RF = 0;
8655	pMixedCtx->esp += cbParm;
8656	pMixedCtx->esp &= uMask;
8657	pMixedCtx->rip += pDis->cbInstr;
8658	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS;
8659	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPopf);
8660	break;
8661	}
8662
8663	case OP_PUSHF:
8664	{
8665	uint32_t cbParm = 0;
8666	uint32_t uMask = 0;
8667	if (pDis->fPrefix & DISPREFIX_OPSIZE)
8668	{
8669	cbParm = 4;
8670	uMask = 0xffffffff;
8671	}
8672	else
8673	{
8674	cbParm = 2;
8675	uMask = 0xffff;
8676	}
8677
8678	/* Get the stack pointer & push the contents of eflags onto the stack. */
8679	RTGCPTR GCPtrStack = 0;
8680	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), (pMixedCtx->esp - cbParm) & uMask,
8681	SELMTOFLAT_FLAGS_CPL0, &GCPtrStack);
8682	if (RT_FAILURE(rc))
8683	{
8684	rc = VERR_EM_INTERPRETER;
8685	break;
8686	}
8687	X86EFLAGS uEflags;
8688	uEflags = pMixedCtx->eflags;
8689	/* The RF & VM bits are cleared on image stored on stack; see Intel Instruction reference for PUSHF. */
8690	uEflags.Bits.u1RF = 0;
8691	uEflags.Bits.u1VM = 0;
8692
8693	rc = PGMPhysWrite(pVM, (RTGCPHYS)GCPtrStack, &uEflags.u, cbParm);
8694	if (RT_FAILURE(rc))
8695	{
8696	rc = VERR_EM_INTERPRETER;
8697	break;
8698	}
8699	Log(("PUSHF %x -> %RGv\n", uEflags.u, GCPtrStack));
8700	pMixedCtx->esp -= cbParm;
8701	pMixedCtx->esp &= uMask;
8702	pMixedCtx->rip += pDis->cbInstr;
8703	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP;
8704	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPushf);
8705	break;
8706	}
8707
8708	case OP_IRET:
8709	{
8710	/** @todo Handle 32-bit operand sizes and check stack limits. See Intel
8711	* instruction reference. */
8712	RTGCPTR GCPtrStack = 0;
8713	uint32_t uMask = 0xffff;
8714	uint16_t aIretFrame[3];
8715	if (pDis->fPrefix & (DISPREFIX_OPSIZE \| DISPREFIX_ADDRSIZE))
8716	{
8717	rc = VERR_EM_INTERPRETER;
8718	break;
8719	}
8720	rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
8721	&GCPtrStack);
8722	if (RT_SUCCESS(rc))
8723	rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &aIretFrame[0], sizeof(aIretFrame));
8724	if (RT_FAILURE(rc))
8725	{
8726	rc = VERR_EM_INTERPRETER;
8727	break;
8728	}
8729	pMixedCtx->eip = 0;
8730	pMixedCtx->ip = aIretFrame[0];
8731	pMixedCtx->cs.Sel = aIretFrame[1];
8732	pMixedCtx->cs.ValidSel = aIretFrame[1];
8733	pMixedCtx->cs.u64Base = (uint64_t)pMixedCtx->cs.Sel << 4;
8734	pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~(X86_EFL_POPF_BITS & uMask))
8735	\| (aIretFrame[2] & X86_EFL_POPF_BITS & uMask);
8736	pMixedCtx->sp += sizeof(aIretFrame);
8737	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_SEGMENT_REGS \| HM_CHANGED_GUEST_RSP
8738	\| HM_CHANGED_GUEST_RFLAGS;
8739	Log(("IRET %#RX32 to %04x:%x\n", GCPtrStack, pMixedCtx->cs.Sel, pMixedCtx->ip));
8740	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIret);
8741	break;
8742	}
8743
8744	case OP_INT:
8745	{
8746	uint16_t uVector = pDis->Param1.uValue & 0xff;
8747	hmR0VmxSetPendingIntN(pVCpu, pMixedCtx, uVector, pDis->cbInstr);
8748	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
8749	break;
8750	}
8751
8752	case OP_INTO:
8753	{
8754	if (pMixedCtx->eflags.Bits.u1OF)
8755	{
8756	hmR0VmxSetPendingXcptOF(pVCpu, pMixedCtx, pDis->cbInstr);
8757	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
8758	}
8759	break;
8760	}
8761
8762	default:
8763	{
8764	VBOXSTRICTRC rc2 = EMInterpretInstructionDisasState(pVCpu, pDis, CPUMCTX2CORE(pMixedCtx), 0 /* pvFault */,
8765	EMCODETYPE_SUPERVISOR);
8766	rc = VBOXSTRICTRC_VAL(rc2);
8767	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_ALL_GUEST;
8768	Log2(("#GP rc=%Rrc\n", rc));
8769	break;
8770	}
8771	}
8772	}
8773	else
8774	rc = VERR_EM_INTERPRETER;
8775
8776	AssertMsg(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER \|\| rc == VINF_PGM_CHANGE_MODE \|\| rc == VINF_EM_HALT,
8777	("#GP Unexpected rc=%Rrc\n", rc));
8778	return rc;
8779	}
8780
8781
8782	/**
8783	* VM-exit exception handler wrapper for generic exceptions. Simply re-injects
8784	* the exception reported in the VMX transient structure back into the VM.
8785	*
8786	* @remarks Requires uExitIntrInfo, uExitIntrErrorCode, cbInstr fields in the
8787	* VMX transient structure to be up-to-date.
8788	*/
8789	static DECLCALLBACK(int) hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8790	{
8791	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8792
8793	/* Re-inject the exception into the guest. This cannot be a double-fault condition which would have been handled in
8794	hmR0VmxCheckExitDueToEventDelivery(). */
8795	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8796	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, 0 /* GCPtrFaultAddress */);
8797	return VINF_SUCCESS;
8798	}
8799
8800
8801	/**
8802	* VM-exit exception handler for #PF (Page-fault exception).
8803	*/
8804	static DECLCALLBACK(int) hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8805	{
8806	VMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
8807	PVM pVM = pVCpu->CTX_SUFF(pVM);
8808	int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
8809	rc \|= hmR0VmxReadExitIntrInfoVmcs(pVCpu, pVmxTransient);
8810	rc \|= hmR0VmxReadExitInstrLenVmcs(pVCpu, pVmxTransient);
8811	rc \|= hmR0VmxReadExitIntrErrorCodeVmcs(pVCpu, pVmxTransient);
8812	AssertRCReturn(rc, rc);
8813
8814	#if defined(VBOX_ALWAYS_TRAP_ALL_EXCEPTIONS) \|\| defined(VBOX_ALWAYS_TRAP_PF)
8815	if (pVM->hm.s.fNestedPaging)
8816	{
8817	if (RT_LIKELY(!pVmxTransient->fVectoringPF))
8818	{
8819	pMixedCtx->cr2 = pVmxTransient->uExitQualification;
8820	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8821	pVmxTransient->cbInstr, pVmxTransient->uExitIntrErrorCode, pMixedCtx->cr2);
8822	}
8823	else
8824	{
8825	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
8826	Log(("Pending #DF due to vectoring #PF.\n"));
8827	rc = hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
8828	}
8829	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
8830	return rc;
8831	}
8832	#else
8833	Assert(!pVM->hm.s.fNestedPaging);
8834	#endif
8835
8836	#ifdef VBOX_HM_WITH_GUEST_PATCHING
8837	rc = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
8838	rc \|= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8839	rc \|= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
8840	AssertRCReturn(rc, rc);
8841	/* Shortcut for APIC TPR access, only for 32-bit guests. */
8842	if ( pVM->hm.s.fTRPPatchingAllowed
8843	&& pVM->hm.s.pGuestPatchMem
8844	&& (pVmxTransient->uExitQualification & 0xfff) == 0x80 /* TPR offset */
8845	&& !(pVmxTransient->uExitIntrErrorCode & X86_TRAP_PF_P) /* Page not present */
8846	&& CPUMGetGuestCPL(pVCpu) == 0 /* Requires CR0, EFLAGS, segments. */
8847	&& !CPUMIsGuestInLongModeEx(pMixedCtx) /* Requires EFER. */
8848	&& pVM->hm.s.cPatches < RT_ELEMENTS(pVM->hm.s.aPatches))
8849	{
8850	RTGCPHYS GCPhys;
8851	RTGCPHYS GCPhysApicBase = (pMixedCtx->msrApicBase & PAGE_BASE_GC_MASK);
8852	rc = PGMGstGetPage(pVCpu, (RTGCPTR)pVmxTransient->uExitQualification, NULL /* pfFlags */, &GCPhys);
8853	if ( rc == VINF_SUCCESS
8854	&& GCPhys == GCPhysApicBase)
8855	{
8856	rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
8857	AssertRCReturn(rc, rc);
8858
8859	/* Only attempt to patch the instruction once. */
8860	PHMTPRPATCH pPatch = (PHMTPRPATCH)RTAvloU32Get(&pVM->hm.s.PatchTree, (AVLOU32KEY)pMixedCtx->eip);
8861	if (!pPatch)
8862	return VINF_EM_HM_PATCH_TPR_INSTR;
8863	}
8864	}
8865	#endif
8866
8867	rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8868	AssertRCReturn(rc, rc);
8869
8870	TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQualification, (RTGCUINT)pVmxTransient->uExitIntrErrorCode);
8871
8872	/* Forward it to the trap handler first. */
8873	rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntrErrorCode, CPUMCTX2CORE(pMixedCtx),
8874	(RTGCPTR)pVmxTransient->uExitQualification);
8875
8876	Log(("#PF: cr2=%RGv cs:rip=%04x:%RGv errorcode %#RX32 rc=%d\n", pVmxTransient->uExitQualification, pMixedCtx->cs.Sel,
8877	pMixedCtx->rip, pVmxTransient->uExitIntrErrorCode, rc));
8878
8879	if (rc == VINF_SUCCESS)
8880	{
8881	/* Successfully synced shadow pages tables or emulated an MMIO instruction. */
8882	/** @todo this isn't quite right, what if guest does lgdt with some MMIO
8883	* memory? We don't update the whole state here... */
8884	pVCpu->hm.s.fContextUseFlags \|= HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
8885	\| HM_CHANGED_VMX_GUEST_APIC_STATE;
8886	TRPMResetTrap(pVCpu);
8887	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
8888	return rc;
8889	}
8890	else if (rc == VINF_EM_RAW_GUEST_TRAP)
8891	{
8892	if (RT_LIKELY(!pVmxTransient->fVectoringPF))
8893	{
8894	/* It's a guest page fault and needs to be reflected to the guest. */
8895	uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
8896	TRPMResetTrap(pVCpu);
8897	pMixedCtx->cr2 = pVmxTransient->uExitQualification;
8898	hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntrInfo),
8899	pVmxTransient->cbInstr, uGstErrorCode, pMixedCtx->cr2);
8900	}
8901	else
8902	{
8903	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
8904	TRPMResetTrap(pVCpu);
8905	hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
8906	Log(("#PF: Pending #DF due to vectoring #PF\n"));
8907	}
8908
8909	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
8910	return VINF_SUCCESS;
8911	}
8912
8913	TRPMResetTrap(pVCpu);
8914	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
8915	return rc;
8916	}
8917

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 45543

Download in other formats: