HMVMXR0.cpp@ 78229

Last change on this file since 78229 was 78229, checked in by vboxsync, 6 years ago
VMM/HMVMXR0: Nested VMX: bugref:9180 32-bit hosts fixes.
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 642.2 KB

Line
1	/* $Id: HMVMXR0.cpp 78229 2019-04-20 05:20:41Z vboxsync $ */
2	/** @file
3	* HM VMX (Intel VT-x) - Host Context Ring-0.
4	*/
5
6	/*
7	* Copyright (C) 2012-2019 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18
19	/*********************************************************************************************************************************
20	* Header Files *
21	*********************************************************************************************************************************/
22	#define LOG_GROUP LOG_GROUP_HM
23	#define VMCPU_INCL_CPUM_GST_CTX
24	#include <iprt/x86.h>
25	#include <iprt/asm-amd64-x86.h>
26	#include <iprt/thread.h>
27
28	#include <VBox/vmm/pdmapi.h>
29	#include <VBox/vmm/dbgf.h>
30	#include <VBox/vmm/iem.h>
31	#include <VBox/vmm/iom.h>
32	#include <VBox/vmm/selm.h>
33	#include <VBox/vmm/tm.h>
34	#include <VBox/vmm/em.h>
35	#include <VBox/vmm/gim.h>
36	#include <VBox/vmm/apic.h>
37	#ifdef VBOX_WITH_REM
38	# include <VBox/vmm/rem.h>
39	#endif
40	#include "HMInternal.h"
41	#include <VBox/vmm/vm.h>
42	#include <VBox/vmm/hmvmxinline.h>
43	#include "HMVMXR0.h"
44	#include "dtrace/VBoxVMM.h"
45
46	# define HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
47	#ifdef DEBUG_ramshankar
48	# define HMVMX_ALWAYS_SAVE_GUEST_RFLAGS
49	# define HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE
50	# define HMVMX_ALWAYS_CHECK_GUEST_STATE
51	# define HMVMX_ALWAYS_TRAP_ALL_XCPTS
52	# define HMVMX_ALWAYS_TRAP_PF
53	# define HMVMX_ALWAYS_FLUSH_TLB
54	# define HMVMX_ALWAYS_SWAP_EFER
55	#endif
56
57
58	/*********************************************************************************************************************************
59	* Defined Constants And Macros *
60	*********************************************************************************************************************************/
61	/** Use the function table. */
62	#define HMVMX_USE_FUNCTION_TABLE
63
64	/** Determine which tagged-TLB flush handler to use. */
65	#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
66	#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
67	#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
68	#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
69
70	/** @name HMVMX_READ_XXX
71	* Flags to skip redundant reads of some common VMCS fields that are not part of
72	* the guest-CPU or VCPU state but are needed while handling VM-exits.
73	*/
74	#define HMVMX_READ_IDT_VECTORING_INFO RT_BIT_32(0)
75	#define HMVMX_READ_IDT_VECTORING_ERROR_CODE RT_BIT_32(1)
76	#define HMVMX_READ_EXIT_QUALIFICATION RT_BIT_32(2)
77	#define HMVMX_READ_EXIT_INSTR_LEN RT_BIT_32(3)
78	#define HMVMX_READ_EXIT_INTERRUPTION_INFO RT_BIT_32(4)
79	#define HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE RT_BIT_32(5)
80	#define HMVMX_READ_EXIT_INSTR_INFO RT_BIT_32(6)
81	#define HMVMX_READ_GUEST_LINEAR_ADDR RT_BIT_32(7)
82	/** @} */
83
84	/**
85	* Subset of the guest-CPU state that is kept by VMX R0 code while executing the
86	* guest using hardware-assisted VMX.
87	*
88	* This excludes state like GPRs (other than RSP) which are always are
89	* swapped and restored across the world-switch and also registers like EFER,
90	* MSR which cannot be modified by the guest without causing a VM-exit.
91	*/
92	#define HMVMX_CPUMCTX_EXTRN_ALL ( CPUMCTX_EXTRN_RIP \
93	\| CPUMCTX_EXTRN_RFLAGS \
94	\| CPUMCTX_EXTRN_RSP \
95	\| CPUMCTX_EXTRN_SREG_MASK \
96	\| CPUMCTX_EXTRN_TABLE_MASK \
97	\| CPUMCTX_EXTRN_KERNEL_GS_BASE \
98	\| CPUMCTX_EXTRN_SYSCALL_MSRS \
99	\| CPUMCTX_EXTRN_SYSENTER_MSRS \
100	\| CPUMCTX_EXTRN_TSC_AUX \
101	\| CPUMCTX_EXTRN_OTHER_MSRS \
102	\| CPUMCTX_EXTRN_CR0 \
103	\| CPUMCTX_EXTRN_CR3 \
104	\| CPUMCTX_EXTRN_CR4 \
105	\| CPUMCTX_EXTRN_DR7 \
106	\| CPUMCTX_EXTRN_HM_VMX_MASK)
107
108	/**
109	* Exception bitmap mask for real-mode guests (real-on-v86).
110	*
111	* We need to intercept all exceptions manually except:
112	* - \#AC and \#DB are always intercepted to prevent the CPU from deadlocking
113	* due to bugs in Intel CPUs.
114	* - \#PF need not be intercepted even in real-mode if we have nested paging
115	* support.
116	*/
117	#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) /* always: \| RT_BIT(X86_XCPT_DB) */ \| RT_BIT(X86_XCPT_NMI) \
118	\| RT_BIT(X86_XCPT_BP) \| RT_BIT(X86_XCPT_OF) \| RT_BIT(X86_XCPT_BR) \
119	\| RT_BIT(X86_XCPT_UD) \| RT_BIT(X86_XCPT_NM) \| RT_BIT(X86_XCPT_DF) \
120	\| RT_BIT(X86_XCPT_CO_SEG_OVERRUN) \| RT_BIT(X86_XCPT_TS) \| RT_BIT(X86_XCPT_NP) \
121	\| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
122	\| RT_BIT(X86_XCPT_MF) /* always: \| RT_BIT(X86_XCPT_AC) */ \| RT_BIT(X86_XCPT_MC) \
123	\| RT_BIT(X86_XCPT_XF))
124
125	/** Maximum VM-instruction error number. */
126	#define HMVMX_INSTR_ERROR_MAX 28
127
128	/** Profiling macro. */
129	#ifdef HM_PROFILE_EXIT_DISPATCH
130	# define HMVMX_START_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed)
131	# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitDispatch, ed)
132	#else
133	# define HMVMX_START_EXIT_DISPATCH_PROF() do { } while (0)
134	# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
135	#endif
136
137	/** Assert that preemption is disabled or covered by thread-context hooks. */
138	#define HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu) Assert( VMMR0ThreadCtxHookIsEnabled((a_pVCpu)) \
139	\|\| !RTThreadPreemptIsEnabled(NIL_RTTHREAD))
140
141	/** Assert that we haven't migrated CPUs when thread-context hooks are not
142	* used. */
143	#define HMVMX_ASSERT_CPU_SAFE(a_pVCpu) AssertMsg( VMMR0ThreadCtxHookIsEnabled((a_pVCpu)) \
144	\|\| (a_pVCpu)->hm.s.idEnteredCpu == RTMpCpuId(), \
145	("Illegal migration! Entered on CPU %u Current %u\n", \
146	(a_pVCpu)->hm.s.idEnteredCpu, RTMpCpuId()))
147
148	/** Asserts that the given CPUMCTX_EXTRN_XXX bits are present in the guest-CPU
149	* context. */
150	#define HMVMX_CPUMCTX_ASSERT(a_pVCpu, a_fExtrnMbz) AssertMsg(!((a_pVCpu)->cpum.GstCtx.fExtrn & (a_fExtrnMbz)), \
151	("fExtrn=%#RX64 fExtrnMbz=%#RX64\n", \
152	(a_pVCpu)->cpum.GstCtx.fExtrn, (a_fExtrnMbz)))
153
154	/** Helper macro for VM-exit handlers called unexpectedly. */
155	#define HMVMX_UNEXPECTED_EXIT_RET(a_pVCpu, a_pVmxTransient) \
156	do { \
157	(a_pVCpu)->hm.s.u32HMError = (a_pVmxTransient)->uExitReason; \
158	return VERR_VMX_UNEXPECTED_EXIT; \
159	} while (0)
160
161	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
162	/** Macro that does the necessary privilege checks and intercepted VM-exits for
163	* guests that attempted to execute a VMX instruction. */
164	# define HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(a_pVCpu, a_uExitReason) \
165	do \
166	{ \
167	VBOXSTRICTRC rcStrictTmp = hmR0VmxCheckExitDueToVmxInstr((a_pVCpu), (a_uExitReason)); \
168	if (rcStrictTmp == VINF_SUCCESS) \
169	{ /* likely */ } \
170	else if (rcStrictTmp == VINF_HM_PENDING_XCPT) \
171	{ \
172	Assert((a_pVCpu)->hm.s.Event.fPending); \
173	Log4Func(("Privilege checks failed -> %#x\n", VMX_ENTRY_INT_INFO_VECTOR((a_pVCpu)->hm.s.Event.u64IntInfo))); \
174	return VINF_SUCCESS; \
175	} \
176	else \
177	{ \
178	int rcTmp = VBOXSTRICTRC_VAL(rcStrictTmp); \
179	AssertMsgFailedReturn(("Unexpected failure. rc=%Rrc", rcTmp), rcTmp); \
180	} \
181	} while (0)
182
183	/** Macro that decodes a memory operand for an instruction VM-exit. */
184	# define HMVMX_DECODE_MEM_OPERAND(a_pVCpu, a_uExitInstrInfo, a_uExitQual, a_enmMemAccess, a_pGCPtrEffAddr) \
185	do \
186	{ \
187	VBOXSTRICTRC rcStrictTmp = hmR0VmxDecodeMemOperand((a_pVCpu), (a_uExitInstrInfo), (a_uExitQual), (a_enmMemAccess), \
188	(a_pGCPtrEffAddr)); \
189	if (rcStrictTmp == VINF_SUCCESS) \
190	{ /* likely */ } \
191	else if (rcStrictTmp == VINF_HM_PENDING_XCPT) \
192	{ \
193	uint8_t const uXcptTmp = VMX_ENTRY_INT_INFO_VECTOR((a_pVCpu)->hm.s.Event.u64IntInfo); \
194	Log4Func(("Memory operand decoding failed, raising xcpt %#x\n", uXcptTmp)); \
195	NOREF(uXcptTmp); \
196	return VINF_SUCCESS; \
197	} \
198	else \
199	{ \
200	Log4Func(("hmR0VmxDecodeMemOperand failed. rc=%Rrc\n", VBOXSTRICTRC_VAL(rcStrictTmp))); \
201	return rcStrictTmp; \
202	} \
203	} while (0)
204
205	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
206
207
208	/*********************************************************************************************************************************
209	* Structures and Typedefs *
210	*********************************************************************************************************************************/
211	/**
212	* VMX transient state.
213	*
214	* A state structure for holding miscellaneous information across
215	* VMX non-root operation and restored after the transition.
216	*/
217	typedef struct VMXTRANSIENT
218	{
219	/** The host's rflags/eflags. */
220	RTCCUINTREG fEFlags;
221	#if HC_ARCH_BITS == 32
222	uint32_t u32Alignment0;
223	#endif
224	/** The guest's TPR value used for TPR shadowing. */
225	uint8_t u8GuestTpr;
226	/** Alignment. */
227	uint8_t abAlignment0[7];
228
229	/** The basic VM-exit reason. */
230	uint16_t uExitReason;
231	/** Alignment. */
232	uint16_t u16Alignment0;
233	/** The VM-exit interruption error code. */
234	uint32_t uExitIntErrorCode;
235	/** The VM-exit exit code qualification. */
236	uint64_t uExitQual;
237	/** The Guest-linear address. */
238	uint64_t uGuestLinearAddr;
239
240	/** The VM-exit interruption-information field. */
241	uint32_t uExitIntInfo;
242	/** The VM-exit instruction-length field. */
243	uint32_t cbInstr;
244	/** The VM-exit instruction-information field. */
245	VMXEXITINSTRINFO ExitInstrInfo;
246	/** Whether the VM-entry failed or not. */
247	bool fVMEntryFailed;
248	/** Whether we are currently executing a nested-guest. */
249	bool fIsNestedGuest;
250	/** Alignment. */
251	uint8_t abAlignment1[2];
252
253	/** The VM-entry interruption-information field. */
254	uint32_t uEntryIntInfo;
255	/** The VM-entry exception error code field. */
256	uint32_t uEntryXcptErrorCode;
257	/** The VM-entry instruction length field. */
258	uint32_t cbEntryInstr;
259
260	/** IDT-vectoring information field. */
261	uint32_t uIdtVectoringInfo;
262	/** IDT-vectoring error code. */
263	uint32_t uIdtVectoringErrorCode;
264
265	/** Mask of currently read VMCS fields; HMVMX_READ_XXX. */
266	uint32_t fVmcsFieldsRead;
267
268	/** Whether the guest debug state was active at the time of VM-exit. */
269	bool fWasGuestDebugStateActive;
270	/** Whether the hyper debug state was active at the time of VM-exit. */
271	bool fWasHyperDebugStateActive;
272	/** Whether TSC-offsetting and VMX-preemption timer was updated before VM-entry. */
273	bool fUpdatedTscOffsettingAndPreemptTimer;
274	/** Whether the VM-exit was caused by a page-fault during delivery of a
275	* contributory exception or a page-fault. */
276	bool fVectoringDoublePF;
277	/** Whether the VM-exit was caused by a page-fault during delivery of an
278	* external interrupt or NMI. */
279	bool fVectoringPF;
280	bool afAlignment0[3];
281
282	/** The VMCS info. object. */
283	PVMXVMCSINFO pVmcsInfo;
284	} VMXTRANSIENT;
285	AssertCompileMemberAlignment(VMXTRANSIENT, uExitReason, sizeof(uint64_t));
286	AssertCompileMemberAlignment(VMXTRANSIENT, uExitIntInfo, sizeof(uint64_t));
287	AssertCompileMemberAlignment(VMXTRANSIENT, uEntryIntInfo, sizeof(uint64_t));
288	AssertCompileMemberAlignment(VMXTRANSIENT, fWasGuestDebugStateActive, sizeof(uint64_t));
289	AssertCompileMemberAlignment(VMXTRANSIENT, pVmcsInfo, sizeof(uint64_t));
290	AssertCompileMemberSize(VMXTRANSIENT, ExitInstrInfo, sizeof(uint32_t));
291	/** Pointer to VMX transient state. */
292	typedef VMXTRANSIENT *PVMXTRANSIENT;
293
294	/**
295	* Memory operand read or write access.
296	*/
297	typedef enum VMXMEMACCESS
298	{
299	VMXMEMACCESS_READ = 0,
300	VMXMEMACCESS_WRITE = 1
301	} VMXMEMACCESS;
302
303	/**
304	* VMX VM-exit handler.
305	*
306	* @returns Strict VBox status code (i.e. informational status codes too).
307	* @param pVCpu The cross context virtual CPU structure.
308	* @param pVmxTransient The VMX-transient structure.
309	*/
310	#ifndef HMVMX_USE_FUNCTION_TABLE
311	typedef VBOXSTRICTRC FNVMXEXITHANDLER(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
312	#else
313	typedef DECLCALLBACK(VBOXSTRICTRC) FNVMXEXITHANDLER(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
314	/** Pointer to VM-exit handler. */
315	typedef FNVMXEXITHANDLER *PFNVMXEXITHANDLER;
316	#endif
317
318	/**
319	* VMX VM-exit handler, non-strict status code.
320	*
321	* This is generally the same as FNVMXEXITHANDLER, the NSRC bit is just FYI.
322	*
323	* @returns VBox status code, no informational status code returned.
324	* @param pVCpu The cross context virtual CPU structure.
325	* @param pVmxTransient The VMX-transient structure.
326	*
327	* @remarks This is not used on anything returning VERR_EM_INTERPRETER as the
328	* use of that status code will be replaced with VINF_EM_SOMETHING
329	* later when switching over to IEM.
330	*/
331	#ifndef HMVMX_USE_FUNCTION_TABLE
332	typedef int FNVMXEXITHANDLERNSRC(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
333	#else
334	typedef FNVMXEXITHANDLER FNVMXEXITHANDLERNSRC;
335	#endif
336
337
338	/*********************************************************************************************************************************
339	* Internal Functions *
340	*********************************************************************************************************************************/
341	#ifndef HMVMX_USE_FUNCTION_TABLE
342	DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
343	# define HMVMX_EXIT_DECL DECLINLINE(VBOXSTRICTRC)
344	# define HMVMX_EXIT_NSRC_DECL DECLINLINE(int)
345	#else
346	# define HMVMX_EXIT_DECL static DECLCALLBACK(VBOXSTRICTRC)
347	# define HMVMX_EXIT_NSRC_DECL HMVMX_EXIT_DECL
348	#endif
349	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
350	DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExitNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
351	#endif
352
353	static int hmR0VmxImportGuestState(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo, uint64_t fWhat);
354	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
355	static void hmR0VmxInitVmcsReadCache(PVMCPU pVCpu);
356	#endif
357
358	/** @name VM-exit handlers.
359	* @{
360	*/
361	static FNVMXEXITHANDLER hmR0VmxExitXcptOrNmi;
362	static FNVMXEXITHANDLER hmR0VmxExitExtInt;
363	static FNVMXEXITHANDLER hmR0VmxExitTripleFault;
364	static FNVMXEXITHANDLERNSRC hmR0VmxExitInitSignal;
365	static FNVMXEXITHANDLERNSRC hmR0VmxExitSipi;
366	static FNVMXEXITHANDLERNSRC hmR0VmxExitIoSmi;
367	static FNVMXEXITHANDLERNSRC hmR0VmxExitSmi;
368	static FNVMXEXITHANDLERNSRC hmR0VmxExitIntWindow;
369	static FNVMXEXITHANDLERNSRC hmR0VmxExitNmiWindow;
370	static FNVMXEXITHANDLER hmR0VmxExitTaskSwitch;
371	static FNVMXEXITHANDLER hmR0VmxExitCpuid;
372	static FNVMXEXITHANDLER hmR0VmxExitGetsec;
373	static FNVMXEXITHANDLER hmR0VmxExitHlt;
374	static FNVMXEXITHANDLERNSRC hmR0VmxExitInvd;
375	static FNVMXEXITHANDLER hmR0VmxExitInvlpg;
376	static FNVMXEXITHANDLER hmR0VmxExitRdpmc;
377	static FNVMXEXITHANDLER hmR0VmxExitVmcall;
378	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
379	static FNVMXEXITHANDLER hmR0VmxExitVmclear;
380	static FNVMXEXITHANDLER hmR0VmxExitVmlaunch;
381	static FNVMXEXITHANDLER hmR0VmxExitVmptrld;
382	static FNVMXEXITHANDLER hmR0VmxExitVmptrst;
383	static FNVMXEXITHANDLER hmR0VmxExitVmread;
384	static FNVMXEXITHANDLER hmR0VmxExitVmresume;
385	static FNVMXEXITHANDLER hmR0VmxExitVmwrite;
386	static FNVMXEXITHANDLER hmR0VmxExitVmxoff;
387	static FNVMXEXITHANDLER hmR0VmxExitVmxon;
388	#endif
389	static FNVMXEXITHANDLER hmR0VmxExitRdtsc;
390	static FNVMXEXITHANDLERNSRC hmR0VmxExitRsm;
391	static FNVMXEXITHANDLERNSRC hmR0VmxExitSetPendingXcptUD;
392	static FNVMXEXITHANDLER hmR0VmxExitMovCRx;
393	static FNVMXEXITHANDLER hmR0VmxExitMovDRx;
394	static FNVMXEXITHANDLER hmR0VmxExitIoInstr;
395	static FNVMXEXITHANDLER hmR0VmxExitRdmsr;
396	static FNVMXEXITHANDLER hmR0VmxExitWrmsr;
397	static FNVMXEXITHANDLERNSRC hmR0VmxExitErrInvalidGuestState;
398	static FNVMXEXITHANDLERNSRC hmR0VmxExitErrMsrLoad;
399	static FNVMXEXITHANDLERNSRC hmR0VmxExitErrUndefined;
400	static FNVMXEXITHANDLER hmR0VmxExitMwait;
401	static FNVMXEXITHANDLER hmR0VmxExitMtf;
402	static FNVMXEXITHANDLER hmR0VmxExitMonitor;
403	static FNVMXEXITHANDLER hmR0VmxExitPause;
404	static FNVMXEXITHANDLERNSRC hmR0VmxExitErrMachineCheck;
405	static FNVMXEXITHANDLERNSRC hmR0VmxExitTprBelowThreshold;
406	static FNVMXEXITHANDLER hmR0VmxExitApicAccess;
407	static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
408	static FNVMXEXITHANDLER hmR0VmxExitEptViolation;
409	static FNVMXEXITHANDLER hmR0VmxExitEptMisconfig;
410	static FNVMXEXITHANDLER hmR0VmxExitRdtscp;
411	static FNVMXEXITHANDLER hmR0VmxExitPreemptTimer;
412	static FNVMXEXITHANDLERNSRC hmR0VmxExitWbinvd;
413	static FNVMXEXITHANDLER hmR0VmxExitXsetbv;
414	static FNVMXEXITHANDLER hmR0VmxExitRdrand;
415	static FNVMXEXITHANDLER hmR0VmxExitInvpcid;
416	/** @} */
417
418	/** @name Helpers for hardware exceptions VM-exit handlers.
419	* @{
420	*/
421	static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
422	static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
423	static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
424	static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
425	static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
426	static int hmR0VmxExitXcptAC(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
427	static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
428	/** @} */
429
430
431	/*********************************************************************************************************************************
432	* Global Variables *
433	*********************************************************************************************************************************/
434	#ifdef VMX_USE_CACHED_VMCS_ACCESSES
435	static const uint32_t g_aVmcsCacheSegBase[] =
436	{
437	VMX_VMCS_GUEST_ES_BASE_CACHE_IDX,
438	VMX_VMCS_GUEST_CS_BASE_CACHE_IDX,
439	VMX_VMCS_GUEST_SS_BASE_CACHE_IDX,
440	VMX_VMCS_GUEST_DS_BASE_CACHE_IDX,
441	VMX_VMCS_GUEST_FS_BASE_CACHE_IDX,
442	VMX_VMCS_GUEST_GS_BASE_CACHE_IDX
443	};
444	AssertCompile(RT_ELEMENTS(g_aVmcsCacheSegBase) == X86_SREG_COUNT);
445	#endif
446	static const uint32_t g_aVmcsSegBase[] =
447	{
448	VMX_VMCS_GUEST_ES_BASE,
449	VMX_VMCS_GUEST_CS_BASE,
450	VMX_VMCS_GUEST_SS_BASE,
451	VMX_VMCS_GUEST_DS_BASE,
452	VMX_VMCS_GUEST_FS_BASE,
453	VMX_VMCS_GUEST_GS_BASE
454	};
455	static const uint32_t g_aVmcsSegSel[] =
456	{
457	VMX_VMCS16_GUEST_ES_SEL,
458	VMX_VMCS16_GUEST_CS_SEL,
459	VMX_VMCS16_GUEST_SS_SEL,
460	VMX_VMCS16_GUEST_DS_SEL,
461	VMX_VMCS16_GUEST_FS_SEL,
462	VMX_VMCS16_GUEST_GS_SEL
463	};
464	static const uint32_t g_aVmcsSegLimit[] =
465	{
466	VMX_VMCS32_GUEST_ES_LIMIT,
467	VMX_VMCS32_GUEST_CS_LIMIT,
468	VMX_VMCS32_GUEST_SS_LIMIT,
469	VMX_VMCS32_GUEST_DS_LIMIT,
470	VMX_VMCS32_GUEST_FS_LIMIT,
471	VMX_VMCS32_GUEST_GS_LIMIT
472	};
473	static const uint32_t g_aVmcsSegAttr[] =
474	{
475	VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS,
476	VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS,
477	VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS,
478	VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS,
479	VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS,
480	VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS
481	};
482	AssertCompile(RT_ELEMENTS(g_aVmcsSegSel) == X86_SREG_COUNT);
483	AssertCompile(RT_ELEMENTS(g_aVmcsSegLimit) == X86_SREG_COUNT);
484	AssertCompile(RT_ELEMENTS(g_aVmcsSegBase) == X86_SREG_COUNT);
485	AssertCompile(RT_ELEMENTS(g_aVmcsSegAttr) == X86_SREG_COUNT);
486
487	#ifdef HMVMX_USE_FUNCTION_TABLE
488	/**
489	* VMX_EXIT dispatch table.
490	*/
491	static const PFNVMXEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
492	{
493	/* 00 VMX_EXIT_XCPT_OR_NMI */ hmR0VmxExitXcptOrNmi,
494	/* 01 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
495	/* 02 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
496	/* 03 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitInitSignal,
497	/* 04 VMX_EXIT_SIPI */ hmR0VmxExitSipi,
498	/* 05 VMX_EXIT_IO_SMI */ hmR0VmxExitIoSmi,
499	/* 06 VMX_EXIT_SMI */ hmR0VmxExitSmi,
500	/* 07 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
501	/* 08 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
502	/* 09 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
503	/* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
504	/* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
505	/* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
506	/* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
507	/* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
508	/* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
509	/* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
510	/* 17 VMX_EXIT_RSM */ hmR0VmxExitRsm,
511	/* 18 VMX_EXIT_VMCALL */ hmR0VmxExitVmcall,
512	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
513	/* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitVmclear,
514	/* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitVmlaunch,
515	/* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitVmptrld,
516	/* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitVmptrst,
517	/* 23 VMX_EXIT_VMREAD */ hmR0VmxExitVmread,
518	/* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitVmresume,
519	/* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitVmwrite,
520	/* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitVmxoff,
521	/* 27 VMX_EXIT_VMXON */ hmR0VmxExitVmxon,
522	#else
523	/* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
524	/* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
525	/* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
526	/* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
527	/* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
528	/* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
529	/* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
530	/* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
531	/* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
532	#endif
533	/* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
534	/* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
535	/* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
536	/* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
537	/* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
538	/* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
539	/* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrMsrLoad,
540	/* 35 UNDEFINED */ hmR0VmxExitErrUndefined,
541	/* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
542	/* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
543	/* 38 UNDEFINED */ hmR0VmxExitErrUndefined,
544	/* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
545	/* 40 UNDEFINED */ hmR0VmxExitPause,
546	/* 41 VMX_EXIT_PAUSE */ hmR0VmxExitErrMachineCheck,
547	/* 42 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUndefined,
548	/* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
549	/* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
550	/* 45 UNDEFINED */ hmR0VmxExitErrUndefined,
551	/* 46 VMX_EXIT_GDTR_IDTR_ACCESS */ hmR0VmxExitXdtrAccess,
552	/* 47 VMX_EXIT_LDTR_TR_ACCESS */ hmR0VmxExitXdtrAccess,
553	/* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
554	/* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
555	/* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
556	/* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
557	/* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
558	/* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
559	/* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
560	/* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
561	/* 56 VMX_EXIT_APIC_WRITE */ hmR0VmxExitErrUndefined,
562	/* 57 VMX_EXIT_RDRAND */ hmR0VmxExitRdrand,
563	/* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
564	/* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitSetPendingXcptUD,
565	/* 60 VMX_EXIT_ENCLS */ hmR0VmxExitErrUndefined,
566	/* 61 VMX_EXIT_RDSEED / hmR0VmxExitErrUndefined, / only spurious exits, so undefined */
567	/* 62 VMX_EXIT_PML_FULL */ hmR0VmxExitErrUndefined,
568	/* 63 VMX_EXIT_XSAVES */ hmR0VmxExitSetPendingXcptUD,
569	/* 64 VMX_EXIT_XRSTORS */ hmR0VmxExitSetPendingXcptUD,
570	};
571	#endif /* HMVMX_USE_FUNCTION_TABLE */
572
573	#if defined(VBOX_STRICT) && defined(LOG_ENABLED)
574	static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
575	{
576	/* 0 */ "(Not Used)",
577	/* 1 */ "VMCALL executed in VMX root operation.",
578	/* 2 */ "VMCLEAR with invalid physical address.",
579	/* 3 */ "VMCLEAR with VMXON pointer.",
580	/* 4 */ "VMLAUNCH with non-clear VMCS.",
581	/* 5 */ "VMRESUME with non-launched VMCS.",
582	/* 6 */ "VMRESUME after VMXOFF",
583	/* 7 */ "VM-entry with invalid control fields.",
584	/* 8 */ "VM-entry with invalid host state fields.",
585	/* 9 */ "VMPTRLD with invalid physical address.",
586	/* 10 */ "VMPTRLD with VMXON pointer.",
587	/* 11 */ "VMPTRLD with incorrect revision identifier.",
588	/* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
589	/* 13 */ "VMWRITE to read-only VMCS component.",
590	/* 14 */ "(Not Used)",
591	/* 15 */ "VMXON executed in VMX root operation.",
592	/* 16 */ "VM-entry with invalid executive-VMCS pointer.",
593	/* 17 */ "VM-entry with non-launched executing VMCS.",
594	/* 18 */ "VM-entry with executive-VMCS pointer not VMXON pointer.",
595	/* 19 */ "VMCALL with non-clear VMCS.",
596	/* 20 */ "VMCALL with invalid VM-exit control fields.",
597	/* 21 */ "(Not Used)",
598	/* 22 */ "VMCALL with incorrect MSEG revision identifier.",
599	/* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
600	/* 24 */ "VMCALL with invalid SMM-monitor features.",
601	/* 25 */ "VM-entry with invalid VM-execution control fields in executive VMCS.",
602	/* 26 */ "VM-entry with events blocked by MOV SS.",
603	/* 27 */ "(Not Used)",
604	/* 28 */ "Invalid operand to INVEPT/INVVPID."
605	};
606	#endif /* VBOX_STRICT */
607
608
609	/**
610	* Get the CR0 guest/host mask that does not change through the lifetime of a VM.
611	*
612	* Any bit set in this mask is owned by the host/hypervisor and would cause a
613	* VM-exit when modified by the guest.
614	*
615	* @returns The static CR0 guest/host mask.
616	* @param pVCpu The cross context virtual CPU structure.
617	*/
618	DECL_FORCE_INLINE(uint64_t) hmR0VmxGetFixedCr0Mask(PVMCPU pVCpu)
619	{
620	/*
621	* Modifications to CR0 bits that VT-x ignores saving/restoring (CD, ET, NW) and
622	* to CR0 bits that we require for shadow paging (PG) by the guest must cause VM-exits.
623	*/
624	/** @todo Avoid intercepting CR0.PE with unrestricted guest execution. Fix PGM
625	* enmGuestMode to be in-sync with the current mode. See @bugref{6398}
626	* and @bugref{6944}. */
627	PVM pVM = pVCpu->CTX_SUFF(pVM);
628	return ( X86_CR0_PE
629	\| X86_CR0_NE
630	\| (pVM->hm.s.fNestedPaging ? 0 : X86_CR0_WP)
631	\| X86_CR0_PG
632	\| X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
633	\| X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
634	\| X86_CR0_NW); /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
635	}
636
637
638	/**
639	* Gets the CR4 guest/host mask that does not change through the lifetime of a VM.
640	*
641	* Any bit set in this mask is owned by the host/hypervisor and would cause a
642	* VM-exit when modified by the guest.
643	*
644	* @returns The static CR4 guest/host mask.
645	* @param pVCpu The cross context virtual CPU structure.
646	*/
647	DECL_FORCE_INLINE(uint64_t) hmR0VmxGetFixedCr4Mask(PVMCPU pVCpu)
648	{
649	/*
650	* We need to look at the host features here (for e.g. OSXSAVE, PCID) because
651	* these bits are reserved on hardware that does not support them. Since the
652	* CPU cannot refer to our virtual CPUID, we need to intercept CR4 changes to
653	* these bits and handle it depending on whether we expose them to the guest.
654	*/
655	PVM pVM = pVCpu->CTX_SUFF(pVM);
656	bool const fXSaveRstor = pVM->cpum.ro.HostFeatures.fXSaveRstor;
657	bool const fPcid = pVM->cpum.ro.HostFeatures.fPcid;
658	return ( X86_CR4_VMXE
659	\| X86_CR4_VME
660	\| X86_CR4_PAE
661	\| X86_CR4_PGE
662	\| X86_CR4_PSE
663	\| (fXSaveRstor ? X86_CR4_OSXSAVE : 0)
664	\| (fPcid ? X86_CR4_PCIDE : 0));
665	}
666
667
668	/**
669	* Returns whether the the VM-exit MSR-store area differs from the VM-exit MSR-load
670	* area.
671	*
672	* @returns @c true if it's different, @c false otherwise.
673	* @param pVmcsInfo The VMCS info. object.
674	*/
675	DECL_FORCE_INLINE(bool) hmR0VmxIsSeparateExitMsrStoreAreaVmcs(PCVMXVMCSINFO pVmcsInfo)
676	{
677	return RT_BOOL( pVmcsInfo->pvGuestMsrStore != pVmcsInfo->pvGuestMsrLoad
678	&& pVmcsInfo->pvGuestMsrStore);
679	}
680
681
682	/**
683	* Adds one or more exceptions to the exception bitmap and commits it to the current
684	* VMCS.
685	*
686	* @returns VBox status code.
687	* @param pVmxTransient The VMX-transient structure.
688	* @param uXcptMask The exception(s) to add.
689	*/
690	static int hmR0VmxAddXcptInterceptMask(PVMXTRANSIENT pVmxTransient, uint32_t uXcptMask)
691	{
692	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
693	uint32_t uXcptBitmap = pVmcsInfo->u32XcptBitmap;
694	if ((uXcptBitmap & uXcptMask) != uXcptMask)
695	{
696	uXcptBitmap \|= uXcptMask;
697	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
698	AssertRCReturn(rc, rc);
699	pVmcsInfo->u32XcptBitmap = uXcptBitmap;
700	}
701	return VINF_SUCCESS;
702	}
703
704
705	/**
706	* Adds an exception to the exception bitmap and commits it to the current VMCS.
707	*
708	* @returns VBox status code.
709	* @param pVmxTransient The VMX-transient structure.
710	* @param uXcpt The exception to add.
711	*/
712	static int hmR0VmxAddXcptIntercept(PVMXTRANSIENT pVmxTransient, uint8_t uXcpt)
713	{
714	Assert(uXcpt <= X86_XCPT_LAST);
715	return hmR0VmxAddXcptInterceptMask(pVmxTransient, RT_BIT_32(uXcpt));
716	}
717
718
719	/**
720	* Remove one or more exceptions from the exception bitmap and commits it to the
721	* current VMCS.
722	*
723	* This takes care of not removing the exception intercept if a nested-guest
724	* requires the exception to be intercepted.
725	*
726	* @returns VBox status code.
727	* @param pVCpu The cross context virtual CPU structure.
728	* @param pVmxTransient The VMX-transient structure.
729	* @param uXcptMask The exception(s) to remove.
730	*/
731	static int hmR0VmxRemoveXcptInterceptMask(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t uXcptMask)
732	{
733	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
734	uint32_t u32XcptBitmap = pVmcsInfo->u32XcptBitmap;
735	if (u32XcptBitmap & uXcptMask)
736	{
737	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
738	if (!pVmxTransient->fIsNestedGuest)
739	{ /* likely */ }
740	else
741	{
742	PCVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
743	uXcptMask &= ~pVmcsNstGst->u32XcptBitmap;
744	}
745	#endif
746	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
747	uXcptMask &= ~( RT_BIT(X86_XCPT_BP)
748	\| RT_BIT(X86_XCPT_DE)
749	\| RT_BIT(X86_XCPT_NM)
750	\| RT_BIT(X86_XCPT_TS)
751	\| RT_BIT(X86_XCPT_UD)
752	\| RT_BIT(X86_XCPT_NP)
753	\| RT_BIT(X86_XCPT_SS)
754	\| RT_BIT(X86_XCPT_GP)
755	\| RT_BIT(X86_XCPT_PF)
756	\| RT_BIT(X86_XCPT_MF));
757	#elif defined(HMVMX_ALWAYS_TRAP_PF)
758	uXcptMask &= ~RT_BIT(X86_XCPT_PF);
759	#endif
760	if (uXcptMask)
761	{
762	/* Validate we are not removing any essential exception intercepts. */
763	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging \|\| !(uXcptMask & RT_BIT(X86_XCPT_PF))); RT_NOREF(pVCpu);
764	Assert(!(uXcptMask & RT_BIT(X86_XCPT_DB)));
765	Assert(!(uXcptMask & RT_BIT(X86_XCPT_AC)));
766
767	/* Remove it from the exception bitmap. */
768	u32XcptBitmap &= ~uXcptMask;
769
770	/* Commit and update the cache if necessary. */
771	if (pVmcsInfo->u32XcptBitmap != u32XcptBitmap)
772	{
773	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
774	AssertRCReturn(rc, rc);
775	pVmcsInfo->u32XcptBitmap = u32XcptBitmap;
776	}
777	}
778	}
779	return VINF_SUCCESS;
780	}
781
782
783	/**
784	* Remove an exceptions from the exception bitmap and commits it to the current
785	* VMCS.
786	*
787	* @returns VBox status code.
788	* @param pVCpu The cross context virtual CPU structure.
789	* @param pVmxTransient The VMX-transient structure.
790	* @param uXcpt The exception to remove.
791	*/
792	static int hmR0VmxRemoveXcptIntercept(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint8_t uXcpt)
793	{
794	return hmR0VmxRemoveXcptInterceptMask(pVCpu, pVmxTransient, RT_BIT(uXcpt));
795	}
796
797
798	/**
799	* Loads the VMCS specified by the VMCS info. object.
800	*
801	* @returns VBox status code.
802	* @param pVmcsInfo The VMCS info. object.
803	*/
804	static int hmR0VmxLoadVmcs(PVMXVMCSINFO pVmcsInfo)
805	{
806	Assert(pVmcsInfo);
807	Assert(pVmcsInfo->HCPhysVmcs);
808	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
809
810	if (pVmcsInfo->fVmcsState & VMX_V_VMCS_LAUNCH_STATE_CLEAR)
811	{
812	int rc = VMXLoadVmcs(pVmcsInfo->HCPhysVmcs);
813	if (RT_SUCCESS(rc))
814	{
815	pVmcsInfo->fVmcsState \|= VMX_V_VMCS_LAUNCH_STATE_CURRENT;
816	return VINF_SUCCESS;
817	}
818	return rc;
819	}
820	return VERR_VMX_INVALID_VMCS_LAUNCH_STATE;
821	}
822
823
824	/**
825	* Clears the VMCS specified by the VMCS info. object.
826	*
827	* @returns VBox status code.
828	* @param pVmcsInfo The VMCS info. object.
829	*/
830	static int hmR0VmxClearVmcs(PVMXVMCSINFO pVmcsInfo)
831	{
832	Assert(pVmcsInfo);
833	Assert(pVmcsInfo->HCPhysVmcs);
834	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
835
836	int rc = VMXClearVmcs(pVmcsInfo->HCPhysVmcs);
837	if (RT_SUCCESS(rc))
838	pVmcsInfo->fVmcsState = VMX_V_VMCS_LAUNCH_STATE_CLEAR;
839	return rc;
840	}
841
842
843	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
844	/**
845	* Switches the current VMCS to the one specified.
846	*
847	* @returns VBox status code.
848	* @param pVmcsInfoFrom The VMCS info. object we are switching from.
849	* @param pVmcsInfoTo The VMCS info. object we are switching to.
850	*
851	* @remarks Called with interrupts disabled.
852	*/
853	static int hmR0VmxSwitchVmcs(PVMXVMCSINFO pVmcsInfoFrom, PVMXVMCSINFO pVmcsInfoTo)
854	{
855	Assert(pVmcsInfoFrom);
856	Assert(pVmcsInfoTo);
857
858	/*
859	* Clear the VMCS we are switching out if it has not already been cleared.
860	* This will sync any CPU internal data back to the VMCS.
861	*/
862	if (pVmcsInfoFrom->fVmcsState != VMX_V_VMCS_LAUNCH_STATE_CLEAR)
863	{
864	int rc = hmR0VmxClearVmcs(pVmcsInfoFrom);
865	if (RT_SUCCESS(rc))
866	{ /* likely */ }
867	else
868	return rc;
869	}
870
871	/*
872	* Clear the VMCS we are switching to if it has not already been cleared.
873	* This will initialize the VMCS launch state to "clear" required for loading it.
874	*
875	* See Intel spec. 31.6 "Preparation And Launching A Virtual Machine".
876	*/
877	if (pVmcsInfoTo->fVmcsState != VMX_V_VMCS_LAUNCH_STATE_CLEAR)
878	{
879	int rc = hmR0VmxClearVmcs(pVmcsInfoTo);
880	if (RT_SUCCESS(rc))
881	{ /* likely */ }
882	else
883	return rc;
884	}
885
886	/*
887	* Finally, load the VMCS we are switching to.
888	*/
889	return hmR0VmxLoadVmcs(pVmcsInfoTo);
890	}
891	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
892
893
894	/**
895	* Updates the VM's last error record.
896	*
897	* If there was a VMX instruction error, reads the error data from the VMCS and
898	* updates VCPU's last error record as well.
899	*
900	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
901	* Can be NULL if @a rc is not VERR_VMX_UNABLE_TO_START_VM or
902	* VERR_VMX_INVALID_VMCS_FIELD.
903	* @param rc The error code.
904	*/
905	static void hmR0VmxUpdateErrorRecord(PVMCPU pVCpu, int rc)
906	{
907	if ( rc == VERR_VMX_INVALID_VMCS_FIELD
908	\|\| rc == VERR_VMX_UNABLE_TO_START_VM)
909	{
910	AssertPtrReturnVoid(pVCpu);
911	VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
912	}
913	pVCpu->CTX_SUFF(pVM)->hm.s.rcInit = rc;
914	}
915
916
917	#ifdef VBOX_STRICT
918	/**
919	* Reads the VM-entry interruption-information field from the VMCS into the VMX
920	* transient structure.
921	*
922	* @returns VBox status code.
923	* @param pVmxTransient The VMX-transient structure.
924	*
925	* @remarks No-long-jump zone!!!
926	*/
927	DECLINLINE(int) hmR0VmxReadEntryIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
928	{
929	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntInfo);
930	AssertRCReturn(rc, rc);
931	return VINF_SUCCESS;
932	}
933
934
935	/**
936	* Reads the VM-entry exception error code field from the VMCS into
937	* the VMX transient structure.
938	*
939	* @returns VBox status code.
940	* @param pVmxTransient The VMX-transient structure.
941	*
942	* @remarks No-long-jump zone!!!
943	*/
944	DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
945	{
946	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
947	AssertRCReturn(rc, rc);
948	return VINF_SUCCESS;
949	}
950
951
952	/**
953	* Reads the VM-entry exception error code field from the VMCS into
954	* the VMX transient structure.
955	*
956	* @returns VBox status code.
957	* @param pVmxTransient The VMX-transient structure.
958	*
959	* @remarks No-long-jump zone!!!
960	*/
961	DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
962	{
963	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
964	AssertRCReturn(rc, rc);
965	return VINF_SUCCESS;
966	}
967	#endif /* VBOX_STRICT */
968
969
970	/**
971	* Reads the VM-exit interruption-information field from the VMCS into the VMX
972	* transient structure.
973	*
974	* @returns VBox status code.
975	* @param pVmxTransient The VMX-transient structure.
976	*/
977	DECLINLINE(int) hmR0VmxReadExitIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
978	{
979	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INTERRUPTION_INFO))
980	{
981	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntInfo);
982	AssertRCReturn(rc,rc);
983	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_INTERRUPTION_INFO;
984	}
985	return VINF_SUCCESS;
986	}
987
988
989	/**
990	* Reads the VM-exit interruption error code from the VMCS into the VMX
991	* transient structure.
992	*
993	* @returns VBox status code.
994	* @param pVmxTransient The VMX-transient structure.
995	*/
996	DECLINLINE(int) hmR0VmxReadExitIntErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
997	{
998	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE))
999	{
1000	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntErrorCode);
1001	AssertRCReturn(rc, rc);
1002	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE;
1003	}
1004	return VINF_SUCCESS;
1005	}
1006
1007
1008	/**
1009	* Reads the VM-exit instruction length field from the VMCS into the VMX
1010	* transient structure.
1011	*
1012	* @returns VBox status code.
1013	* @param pVmxTransient The VMX-transient structure.
1014	*/
1015	DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
1016	{
1017	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INSTR_LEN))
1018	{
1019	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
1020	AssertRCReturn(rc, rc);
1021	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_INSTR_LEN;
1022	}
1023	return VINF_SUCCESS;
1024	}
1025
1026
1027	/**
1028	* Reads the VM-exit instruction-information field from the VMCS into
1029	* the VMX transient structure.
1030	*
1031	* @returns VBox status code.
1032	* @param pVmxTransient The VMX-transient structure.
1033	*/
1034	DECLINLINE(int) hmR0VmxReadExitInstrInfoVmcs(PVMXTRANSIENT pVmxTransient)
1035	{
1036	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INSTR_INFO))
1037	{
1038	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->ExitInstrInfo.u);
1039	AssertRCReturn(rc, rc);
1040	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_INSTR_INFO;
1041	}
1042	return VINF_SUCCESS;
1043	}
1044
1045
1046	/**
1047	* Reads the VM-exit Qualification from the VMCS into the VMX transient structure.
1048	*
1049	* @returns VBox status code.
1050	* @param pVCpu The cross context virtual CPU structure of the
1051	* calling EMT. (Required for the VMCS cache case.)
1052	* @param pVmxTransient The VMX-transient structure.
1053	*/
1054	DECLINLINE(int) hmR0VmxReadExitQualVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
1055	{
1056	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_QUALIFICATION))
1057	{
1058	int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQual); NOREF(pVCpu);
1059	AssertRCReturn(rc, rc);
1060	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_QUALIFICATION;
1061	}
1062	return VINF_SUCCESS;
1063	}
1064
1065
1066	/**
1067	* Reads the Guest-linear address from the VMCS into the VMX transient structure.
1068	*
1069	* @returns VBox status code.
1070	* @param pVCpu The cross context virtual CPU structure of the
1071	* calling EMT. (Required for the VMCS cache case.)
1072	* @param pVmxTransient The VMX-transient structure.
1073	*/
1074	DECLINLINE(int) hmR0VmxReadGuestLinearAddrVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
1075	{
1076	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_GUEST_LINEAR_ADDR))
1077	{
1078	int rc = VMXReadVmcsGstN(VMX_VMCS_RO_GUEST_LINEAR_ADDR, &pVmxTransient->uGuestLinearAddr); NOREF(pVCpu);
1079	AssertRCReturn(rc, rc);
1080	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_GUEST_LINEAR_ADDR;
1081	}
1082	return VINF_SUCCESS;
1083	}
1084
1085
1086	/**
1087	* Reads the IDT-vectoring information field from the VMCS into the VMX
1088	* transient structure.
1089	*
1090	* @returns VBox status code.
1091	* @param pVmxTransient The VMX-transient structure.
1092	*
1093	* @remarks No-long-jump zone!!!
1094	*/
1095	DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
1096	{
1097	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_IDT_VECTORING_INFO))
1098	{
1099	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_VECTORING_INFO, &pVmxTransient->uIdtVectoringInfo);
1100	AssertRCReturn(rc, rc);
1101	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_IDT_VECTORING_INFO;
1102	}
1103	return VINF_SUCCESS;
1104	}
1105
1106
1107	/**
1108	* Reads the IDT-vectoring error code from the VMCS into the VMX
1109	* transient structure.
1110	*
1111	* @returns VBox status code.
1112	* @param pVmxTransient The VMX-transient structure.
1113	*/
1114	DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
1115	{
1116	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_IDT_VECTORING_ERROR_CODE))
1117	{
1118	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_VECTORING_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
1119	AssertRCReturn(rc, rc);
1120	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_IDT_VECTORING_ERROR_CODE;
1121	}
1122	return VINF_SUCCESS;
1123	}
1124
1125
1126	/**
1127	* Enters VMX root mode operation on the current CPU.
1128	*
1129	* @returns VBox status code.
1130	* @param pVM The cross context VM structure. Can be
1131	* NULL, after a resume.
1132	* @param HCPhysCpuPage Physical address of the VMXON region.
1133	* @param pvCpuPage Pointer to the VMXON region.
1134	*/
1135	static int hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
1136	{
1137	Assert(HCPhysCpuPage && HCPhysCpuPage != NIL_RTHCPHYS);
1138	Assert(RT_ALIGN_T(HCPhysCpuPage, _4K, RTHCPHYS) == HCPhysCpuPage);
1139	Assert(pvCpuPage);
1140	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1141
1142	if (pVM)
1143	{
1144	/* Write the VMCS revision identifier to the VMXON region. */
1145	(uint32_t )pvCpuPage = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_ID);
1146	}
1147
1148	/* Paranoid: Disable interrupts as, in theory, interrupt handlers might mess with CR4. */
1149	RTCCUINTREG fEFlags = ASMIntDisableFlags();
1150
1151	/* Enable the VMX bit in CR4 if necessary. */
1152	RTCCUINTREG uOldCr4 = SUPR0ChangeCR4(X86_CR4_VMXE, RTCCUINTREG_MAX);
1153
1154	/* Enter VMX root mode. */
1155	int rc = VMXEnable(HCPhysCpuPage);
1156	if (RT_FAILURE(rc))
1157	{
1158	if (!(uOldCr4 & X86_CR4_VMXE))
1159	SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
1160
1161	if (pVM)
1162	pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
1163	}
1164
1165	/* Restore interrupts. */
1166	ASMSetFlags(fEFlags);
1167	return rc;
1168	}
1169
1170
1171	/**
1172	* Exits VMX root mode operation on the current CPU.
1173	*
1174	* @returns VBox status code.
1175	*/
1176	static int hmR0VmxLeaveRootMode(void)
1177	{
1178	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1179
1180	/* Paranoid: Disable interrupts as, in theory, interrupts handlers might mess with CR4. */
1181	RTCCUINTREG fEFlags = ASMIntDisableFlags();
1182
1183	/* If we're for some reason not in VMX root mode, then don't leave it. */
1184	RTCCUINTREG uHostCR4 = ASMGetCR4();
1185
1186	int rc;
1187	if (uHostCR4 & X86_CR4_VMXE)
1188	{
1189	/* Exit VMX root mode and clear the VMX bit in CR4. */
1190	VMXDisable();
1191	SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
1192	rc = VINF_SUCCESS;
1193	}
1194	else
1195	rc = VERR_VMX_NOT_IN_VMX_ROOT_MODE;
1196
1197	/* Restore interrupts. */
1198	ASMSetFlags(fEFlags);
1199	return rc;
1200	}
1201
1202
1203	/**
1204	* Allocates and maps a physically contiguous page. The allocated page is
1205	* zero'd out (used by various VT-x structures).
1206	*
1207	* @returns IPRT status code.
1208	* @param pMemObj Pointer to the ring-0 memory object.
1209	* @param ppVirt Where to store the virtual address of the
1210	* allocation.
1211	* @param pHCPhys Where to store the physical address of the
1212	* allocation.
1213	*/
1214	static int hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
1215	{
1216	AssertPtr(pMemObj);
1217	AssertPtr(ppVirt);
1218	AssertPtr(pHCPhys);
1219	int rc = RTR0MemObjAllocCont(pMemObj, X86_PAGE_4K_SIZE, false /* fExecutable */);
1220	if (RT_FAILURE(rc))
1221	return rc;
1222	ppVirt = RTR0MemObjAddress(pMemObj);
1223	pHCPhys = RTR0MemObjGetPagePhysAddr(pMemObj, 0 /* iPage */);
1224	ASMMemZero32(*ppVirt, X86_PAGE_4K_SIZE);
1225	return VINF_SUCCESS;
1226	}
1227
1228
1229	/**
1230	* Frees and unmaps an allocated, physical page.
1231	*
1232	* @param pMemObj Pointer to the ring-0 memory object.
1233	* @param ppVirt Where to re-initialize the virtual address of
1234	* allocation as 0.
1235	* @param pHCPhys Where to re-initialize the physical address of the
1236	* allocation as 0.
1237	*/
1238	static void hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
1239	{
1240	AssertPtr(pMemObj);
1241	AssertPtr(ppVirt);
1242	AssertPtr(pHCPhys);
1243	/* NULL is valid, accepted and ignored by the free function below. */
1244	RTR0MemObjFree(pMemObj, true / fFreeMappings */);
1245	*pMemObj = NIL_RTR0MEMOBJ;
1246	*ppVirt = NULL;
1247	*pHCPhys = NIL_RTHCPHYS;
1248	}
1249
1250
1251	/**
1252	* Initializes a VMCS info. object.
1253	*
1254	* @param pVmcsInfo The VMCS info. object.
1255	*/
1256	static void hmR0VmxInitVmcsInfo(PVMXVMCSINFO pVmcsInfo)
1257	{
1258	RT_ZERO(*pVmcsInfo);
1259
1260	Assert(pVmcsInfo->hMemObjVmcs == NIL_RTR0MEMOBJ);
1261	Assert(pVmcsInfo->hMemObjMsrBitmap == NIL_RTR0MEMOBJ);
1262	Assert(pVmcsInfo->hMemObjGuestMsrLoad == NIL_RTR0MEMOBJ);
1263	Assert(pVmcsInfo->hMemObjGuestMsrStore == NIL_RTR0MEMOBJ);
1264	Assert(pVmcsInfo->hMemObjHostMsrLoad == NIL_RTR0MEMOBJ);
1265	pVmcsInfo->HCPhysVmcs = NIL_RTHCPHYS;
1266	pVmcsInfo->HCPhysMsrBitmap = NIL_RTHCPHYS;
1267	pVmcsInfo->HCPhysGuestMsrLoad = NIL_RTHCPHYS;
1268	pVmcsInfo->HCPhysGuestMsrStore = NIL_RTHCPHYS;
1269	pVmcsInfo->HCPhysHostMsrLoad = NIL_RTHCPHYS;
1270	pVmcsInfo->HCPhysVirtApic = NIL_RTHCPHYS;
1271	pVmcsInfo->HCPhysEPTP = NIL_RTHCPHYS;
1272	pVmcsInfo->u64VmcsLinkPtr = NIL_RTHCPHYS;
1273	}
1274
1275
1276	/**
1277	* Frees the VT-x structures for a VMCS info. object.
1278	*
1279	* @param pVM The cross context VM structure.
1280	* @param pVmcsInfo The VMCS info. object.
1281	*/
1282	static void hmR0VmxFreeVmcsInfo(PVM pVM, PVMXVMCSINFO pVmcsInfo)
1283	{
1284	hmR0VmxPageFree(&pVmcsInfo->hMemObjVmcs, &pVmcsInfo->pvVmcs, &pVmcsInfo->HCPhysVmcs);
1285
1286	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
1287	hmR0VmxPageFree(&pVmcsInfo->hMemObjMsrBitmap, &pVmcsInfo->pvMsrBitmap, &pVmcsInfo->HCPhysMsrBitmap);
1288
1289	hmR0VmxPageFree(&pVmcsInfo->hMemObjHostMsrLoad, &pVmcsInfo->pvHostMsrLoad, &pVmcsInfo->HCPhysHostMsrLoad);
1290	hmR0VmxPageFree(&pVmcsInfo->hMemObjGuestMsrLoad, &pVmcsInfo->pvGuestMsrLoad, &pVmcsInfo->HCPhysGuestMsrLoad);
1291	hmR0VmxPageFree(&pVmcsInfo->hMemObjGuestMsrStore, &pVmcsInfo->pvGuestMsrStore, &pVmcsInfo->HCPhysGuestMsrStore);
1292
1293	hmR0VmxInitVmcsInfo(pVmcsInfo);
1294	}
1295
1296
1297	/**
1298	* Allocates the VT-x structures for a VMCS info. object.
1299	*
1300	* @returns VBox status code.
1301	* @param pVCpu The cross context virtual CPU structure.
1302	* @param pVmcsInfo The VMCS info. object.
1303	* @param fIsNstGstVmcs Whether this is a nested-guest VMCS.
1304	*/
1305	static int hmR0VmxAllocVmcsInfo(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, bool fIsNstGstVmcs)
1306	{
1307	PVM pVM = pVCpu->CTX_SUFF(pVM);
1308
1309	/* Allocate the guest VM control structure (VMCS). */
1310	int rc = hmR0VmxPageAllocZ(&pVmcsInfo->hMemObjVmcs, &pVmcsInfo->pvVmcs, &pVmcsInfo->HCPhysVmcs);
1311	if (RT_SUCCESS(rc))
1312	{
1313	if (!fIsNstGstVmcs)
1314	{
1315	/* Get the allocated virtual-APIC page from the virtual APIC device. */
1316	if ( PDMHasApic(pVCpu->CTX_SUFF(pVM))
1317	&& (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_TPR_SHADOW))
1318	{
1319	rc = APICGetApicPageForCpu(pVCpu, &pVmcsInfo->HCPhysVirtApic, (PRTR0PTR)&pVmcsInfo->pbVirtApic,
1320	NULL /* pR3Ptr /, NULL / pRCPtr */);
1321	}
1322	}
1323	else
1324	{
1325	Assert(pVmcsInfo->HCPhysVirtApic == NIL_RTHCPHYS);
1326	Assert(!pVmcsInfo->pbVirtApic);
1327	}
1328
1329	if (RT_SUCCESS(rc))
1330	{
1331	/*
1332	* Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for
1333	* transparent accesses of specific MSRs.
1334	*
1335	* If the condition for enabling MSR bitmaps changes here, don't forget to
1336	* update HMIsMsrBitmapActive().
1337	*
1338	* We don't share MSR bitmaps between the guest and nested-guest as we then
1339	* don't need to care about carefully restoring the guest MSR bitmap.
1340	* The guest visible nested-guest MSR bitmap needs to remain unchanged.
1341	* Hence, allocate a separate MSR bitmap for the guest and nested-guest.
1342	*/
1343	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
1344	{
1345	rc = hmR0VmxPageAllocZ(&pVmcsInfo->hMemObjMsrBitmap, &pVmcsInfo->pvMsrBitmap, &pVmcsInfo->HCPhysMsrBitmap);
1346	if (RT_SUCCESS(rc))
1347	ASMMemFill32(pVmcsInfo->pvMsrBitmap, X86_PAGE_4K_SIZE, UINT32_C(0xffffffff));
1348	}
1349
1350	if (RT_SUCCESS(rc))
1351	{
1352	/*
1353	* Allocate the VM-entry MSR-load area for the guest MSRs.
1354	*
1355	* Similar to MSR-bitmaps, we do not share the auto MSR-load/store are between
1356	* the guest and nested-guest.
1357	*/
1358	rc = hmR0VmxPageAllocZ(&pVmcsInfo->hMemObjGuestMsrLoad, &pVmcsInfo->pvGuestMsrLoad,
1359	&pVmcsInfo->HCPhysGuestMsrLoad);
1360	if (RT_SUCCESS(rc))
1361	{
1362	/*
1363	* We use the same page for VM-entry MSR-load and VM-exit MSR store areas.
1364	* These contain the guest MSRs to load on VM-entry and store on VM-exit.
1365	*/
1366	Assert(pVmcsInfo->hMemObjGuestMsrStore == NIL_RTR0MEMOBJ);
1367	pVmcsInfo->pvGuestMsrStore = pVmcsInfo->pvGuestMsrLoad;
1368	pVmcsInfo->HCPhysGuestMsrStore = pVmcsInfo->HCPhysGuestMsrLoad;
1369
1370	/* Allocate the VM-exit MSR-load page for the host MSRs. */
1371	rc = hmR0VmxPageAllocZ(&pVmcsInfo->hMemObjHostMsrLoad, &pVmcsInfo->pvHostMsrLoad,
1372	&pVmcsInfo->HCPhysHostMsrLoad);
1373	}
1374	}
1375	}
1376	}
1377
1378	return rc;
1379	}
1380
1381
1382	/**
1383	* Free all VT-x structures for the VM.
1384	*
1385	* @returns IPRT status code.
1386	* @param pVM The cross context VM structure.
1387	*/
1388	static void hmR0VmxStructsFree(PVM pVM)
1389	{
1390	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
1391	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
1392	#endif
1393	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
1394
1395	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
1396	{
1397	PVMCPU pVCpu = &pVM->aCpus[idCpu];
1398	PVMXVMCSINFO pVmcsInfo = &pVCpu->hm.s.vmx.VmcsInfo;
1399	hmR0VmxFreeVmcsInfo(pVM, pVmcsInfo);
1400	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1401	if (pVM->cpum.ro.GuestFeatures.fVmx)
1402	{
1403	pVmcsInfo = &pVCpu->hm.s.vmx.VmcsInfoNstGst;
1404	hmR0VmxFreeVmcsInfo(pVM, pVmcsInfo);
1405	}
1406	#endif
1407	}
1408	}
1409
1410
1411	/**
1412	* Allocate all VT-x structures for the VM.
1413	*
1414	* @returns IPRT status code.
1415	* @param pVM The cross context VM structure.
1416	*/
1417	static int hmR0VmxStructsAlloc(PVM pVM)
1418	{
1419	/*
1420	* Sanity check the VMCS size reported by the CPU as we assume 4KB allocations.
1421	* The VMCS size cannot be more than 4096 bytes.
1422	*
1423	* See Intel spec. Appendix A.1 "Basic VMX Information".
1424	*/
1425	uint32_t const cbVmcs = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_SIZE);
1426	if (cbVmcs <= X86_PAGE_4K_SIZE)
1427	{ /* likely */ }
1428	else
1429	{
1430	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_INVALID_VMCS_SIZE;
1431	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1432	}
1433
1434	/*
1435	* Initialize/check members up-front so we can cleanup en masse on allocation failures.
1436	*/
1437	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
1438	Assert(pVM->hm.s.vmx.hMemObjScratch == NIL_RTR0MEMOBJ);
1439	Assert(pVM->hm.s.vmx.pbScratch == NULL);
1440	pVM->hm.s.vmx.HCPhysScratch = NIL_RTHCPHYS;
1441	#endif
1442
1443	Assert(pVM->hm.s.vmx.hMemObjApicAccess == NIL_RTR0MEMOBJ);
1444	Assert(pVM->hm.s.vmx.pbApicAccess == NULL);
1445	pVM->hm.s.vmx.HCPhysApicAccess = NIL_RTHCPHYS;
1446
1447	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
1448	{
1449	PVMCPU pVCpu = &pVM->aCpus[idCpu];
1450	hmR0VmxInitVmcsInfo(&pVCpu->hm.s.vmx.VmcsInfo);
1451	hmR0VmxInitVmcsInfo(&pVCpu->hm.s.vmx.VmcsInfoNstGst);
1452	}
1453
1454	/*
1455	* Allocate per-VM VT-x structures.
1456	*/
1457	int rc = VINF_SUCCESS;
1458	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
1459	/* Allocate crash-dump magic scratch page. */
1460	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
1461	if (RT_FAILURE(rc))
1462	{
1463	hmR0VmxStructsFree(pVM);
1464	return rc;
1465	}
1466	strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
1467	(uint64_t )(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xdeadbeefdeadbeef);
1468	#endif
1469
1470	/* Allocate the APIC-access page for trapping APIC accesses from the guest. */
1471	if (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
1472	{
1473	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
1474	&pVM->hm.s.vmx.HCPhysApicAccess);
1475	if (RT_FAILURE(rc))
1476	{
1477	hmR0VmxStructsFree(pVM);
1478	return rc;
1479	}
1480	}
1481
1482	/*
1483	* Initialize per-VCPU VT-x structures.
1484	*/
1485	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
1486	{
1487	/* Allocate the guest VMCS structures. */
1488	PVMCPU pVCpu = &pVM->aCpus[idCpu];
1489	rc = hmR0VmxAllocVmcsInfo(pVCpu, &pVCpu->hm.s.vmx.VmcsInfo, false /* fIsNstGstVmcs */);
1490	if (RT_SUCCESS(rc))
1491	{
1492	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1493	/* Allocate the nested-guest VMCS structures, when the VMX feature is exposed to the guest. */
1494	if (pVM->cpum.ro.GuestFeatures.fVmx)
1495	{
1496	rc = hmR0VmxAllocVmcsInfo(pVCpu, &pVCpu->hm.s.vmx.VmcsInfoNstGst, true /* fIsNstGstVmcs */);
1497	if (RT_SUCCESS(rc))
1498	{ /* likely */ }
1499	else
1500	break;
1501	}
1502	#endif
1503	}
1504	else
1505	break;
1506	}
1507
1508	if (RT_FAILURE(rc))
1509	{
1510	hmR0VmxStructsFree(pVM);
1511	return rc;
1512	}
1513
1514	return VINF_SUCCESS;
1515	}
1516
1517
1518	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1519	/**
1520	* Returns whether an MSR at the given MSR-bitmap offset is intercepted or not.
1521	*
1522	* @returns @c true if the MSR is intercepted, @c false otherwise.
1523	* @param pvMsrBitmap The MSR bitmap.
1524	* @param offMsr The MSR byte offset.
1525	* @param iBit The bit offset from the byte offset.
1526	*/
1527	DECLINLINE(bool) hmR0VmxIsMsrBitSet(const void *pvMsrBitmap, uint16_t offMsr, int32_t iBit)
1528	{
1529	uint8_t const * const pbMsrBitmap = (uint8_t const * const)pvMsrBitmap;
1530	Assert(pbMsrBitmap);
1531	Assert(offMsr + (iBit >> 3) <= X86_PAGE_4K_SIZE);
1532	return ASMBitTest(pbMsrBitmap + offMsr, iBit);
1533	}
1534	#endif
1535
1536
1537	/**
1538	* Sets the permission bits for the specified MSR in the given MSR bitmap.
1539	*
1540	* If the passed VMCS is a nested-guest VMCS, this function ensures that the
1541	* read/write intercept is cleared from the MSR bitmap used for hardware-assisted
1542	* VMX execution of the nested-guest, only if nested-guest is also not intercepting
1543	* the read/write access of this MSR.
1544	*
1545	* @param pVCpu The cross context virtual CPU structure.
1546	* @param pVmcsInfo The VMCS info. object.
1547	* @param fIsNstGstVmcs Whether this is a nested-guest VMCS.
1548	* @param idMsr The MSR value.
1549	* @param fMsrpm The MSR permissions (see VMXMSRPM_XXX). This must
1550	* include both a read -and- a write permission!
1551	*
1552	* @sa HMGetVmxMsrPermission.
1553	*/
1554	static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, bool fIsNstGstVmcs, uint32_t idMsr, uint32_t fMsrpm)
1555	{
1556	uint8_t pbMsrBitmap = (uint8_t )pVmcsInfo->pvMsrBitmap;
1557	Assert(pbMsrBitmap);
1558	Assert(VMXMSRPM_IS_FLAG_VALID(fMsrpm));
1559
1560	/*
1561	* MSR-bitmap Layout:
1562	* Byte index MSR range Interpreted as
1563	* 0x000 - 0x3ff 0x00000000 - 0x00001fff Low MSR read bits.
1564	* 0x400 - 0x7ff 0xc0000000 - 0xc0001fff High MSR read bits.
1565	* 0x800 - 0xbff 0x00000000 - 0x00001fff Low MSR write bits.
1566	* 0xc00 - 0xfff 0xc0000000 - 0xc0001fff High MSR write bits.
1567	*
1568	* A bit corresponding to an MSR within the above range causes a VM-exit
1569	* if the bit is 1 on executions of RDMSR/WRMSR. If an MSR falls out of
1570	* the MSR range, it always cause a VM-exit.
1571	*
1572	* See Intel spec. 24.6.9 "MSR-Bitmap Address".
1573	*/
1574	uint16_t const offBitmapRead = 0;
1575	uint16_t const offBitmapWrite = 0x800;
1576	uint16_t offMsr;
1577	int32_t iBit;
1578	if (idMsr <= UINT32_C(0x00001fff))
1579	{
1580	offMsr = 0;
1581	iBit = idMsr;
1582	}
1583	else if (idMsr - UINT32_C(0xc0000000) <= UINT32_C(0x00001fff))
1584	{
1585	offMsr = 0x400;
1586	iBit = idMsr - UINT32_C(0xc0000000);
1587	}
1588	else
1589	AssertMsgFailedReturnVoid(("Invalid MSR %#RX32\n", idMsr));
1590
1591	/*
1592	* Set the MSR read permission.
1593	*/
1594	uint16_t const offMsrRead = offBitmapRead + offMsr;
1595	Assert(offMsrRead + (iBit >> 3) < offBitmapWrite);
1596	if (fMsrpm & VMXMSRPM_ALLOW_RD)
1597	{
1598	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1599	bool const fClear = !fIsNstGstVmcs ? true
1600	: !hmR0VmxIsMsrBitSet(pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvMsrBitmap), offMsrRead, iBit);
1601	#else
1602	RT_NOREF2(pVCpu, fIsNstGstVmcs);
1603	bool const fClear = true;
1604	#endif
1605	if (fClear)
1606	ASMBitClear(pbMsrBitmap + offMsrRead, iBit);
1607	}
1608	else
1609	ASMBitSet(pbMsrBitmap + offMsrRead, iBit);
1610
1611	/*
1612	* Set the MSR write permission.
1613	*/
1614	uint16_t const offMsrWrite = offBitmapWrite + offMsr;
1615	Assert(offMsrWrite + (iBit >> 3) < X86_PAGE_4K_SIZE);
1616	if (fMsrpm & VMXMSRPM_ALLOW_WR)
1617	{
1618	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1619	bool const fClear = !fIsNstGstVmcs ? true
1620	: !hmR0VmxIsMsrBitSet(pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvMsrBitmap), offMsrWrite, iBit);
1621	#else
1622	RT_NOREF2(pVCpu, fIsNstGstVmcs);
1623	bool const fClear = true;
1624	#endif
1625	if (fClear)
1626	ASMBitClear(pbMsrBitmap + offMsrWrite, iBit);
1627	}
1628	else
1629	ASMBitSet(pbMsrBitmap + offMsrWrite, iBit);
1630	}
1631
1632
1633	/**
1634	* Updates the VMCS with the number of effective MSRs in the auto-load/store MSR
1635	* area.
1636	*
1637	* @returns VBox status code.
1638	* @param pVCpu The cross context virtual CPU structure.
1639	* @param pVmcsInfo The VMCS info. object.
1640	* @param cMsrs The number of MSRs.
1641	*/
1642	static int hmR0VmxSetAutoLoadStoreMsrCount(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, uint32_t cMsrs)
1643	{
1644	/* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
1645	uint32_t const cMaxSupportedMsrs = VMX_MISC_MAX_MSRS(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc);
1646	if (RT_UNLIKELY(cMsrs >= cMaxSupportedMsrs))
1647	{
1648	LogRel(("Auto-load/store MSR count exceeded! cMsrs=%u Supported=%u.\n", cMsrs, cMaxSupportedMsrs));
1649	pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_GUEST_MSR_STORAGE;
1650	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1651	}
1652
1653	/* Commit the MSR counts to the VMCS and update the cache. */
1654	int rc = VINF_SUCCESS;
1655	if (pVmcsInfo->cEntryMsrLoad != cMsrs)
1656	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cMsrs);
1657	if (pVmcsInfo->cExitMsrStore != cMsrs)
1658	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cMsrs);
1659	if (pVmcsInfo->cExitMsrLoad != cMsrs)
1660	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cMsrs);
1661	AssertRCReturn(rc, rc);
1662
1663	pVmcsInfo->cEntryMsrLoad = cMsrs;
1664	pVmcsInfo->cExitMsrStore = cMsrs;
1665	pVmcsInfo->cExitMsrLoad = cMsrs;
1666
1667	return VINF_SUCCESS;
1668	}
1669
1670
1671	/**
1672	* Adds a new (or updates the value of an existing) guest/host MSR
1673	* pair to be swapped during the world-switch as part of the
1674	* auto-load/store MSR area in the VMCS.
1675	*
1676	* @returns VBox status code.
1677	* @param pVCpu The cross context virtual CPU structure.
1678	* @param pVmxTransient The VMX-transient structure.
1679	* @param idMsr The MSR.
1680	* @param uGuestMsrValue Value of the guest MSR.
1681	* @param fSetReadWrite Whether to set the guest read/write access of this
1682	* MSR (thus not causing a VM-exit).
1683	* @param fUpdateHostMsr Whether to update the value of the host MSR if
1684	* necessary.
1685	*/
1686	static int hmR0VmxAddAutoLoadStoreMsr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t idMsr, uint64_t uGuestMsrValue,
1687	bool fSetReadWrite, bool fUpdateHostMsr)
1688	{
1689	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
1690	bool const fIsNstGstVmcs = pVmxTransient->fIsNestedGuest;
1691	PVMXAUTOMSR pGuestMsrLoad = (PVMXAUTOMSR)pVmcsInfo->pvGuestMsrLoad;
1692	uint32_t cMsrs = pVmcsInfo->cEntryMsrLoad;
1693	uint32_t i;
1694
1695	/* Paranoia. */
1696	Assert(pGuestMsrLoad);
1697
1698	/* Check if the MSR already exists in the VM-entry MSR-load area. */
1699	for (i = 0; i < cMsrs; i++)
1700	{
1701	if (pGuestMsrLoad->u32Msr == idMsr)
1702	break;
1703	pGuestMsrLoad++;
1704	}
1705
1706	bool fAdded = false;
1707	if (i == cMsrs)
1708	{
1709	/* The MSR does not exist, bump the MSR coun to make room for the new MSR. */
1710	++cMsrs;
1711	int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, pVmcsInfo, cMsrs);
1712	AssertMsgRCReturn(rc, ("Insufficient space to add MSR to VM-entry MSR-load/store area %u\n", idMsr), rc);
1713
1714	/* Set the guest to read/write this MSR without causing VM-exits. */
1715	if ( fSetReadWrite
1716	&& (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS))
1717	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, idMsr, VMXMSRPM_ALLOW_RD \| VMXMSRPM_ALLOW_WR);
1718
1719	fAdded = true;
1720	}
1721
1722	/* Update the MSR value for the newly added or already existing MSR. */
1723	pGuestMsrLoad->u32Msr = idMsr;
1724	pGuestMsrLoad->u64Value = uGuestMsrValue;
1725
1726	/* Create the corresponding slot in the VM-exit MSR-store area if we use a different page. */
1727	if (hmR0VmxIsSeparateExitMsrStoreAreaVmcs(pVmcsInfo))
1728	{
1729	PVMXAUTOMSR pGuestMsrStore = (PVMXAUTOMSR)pVmcsInfo->pvGuestMsrStore;
1730	pGuestMsrStore += i;
1731	pGuestMsrStore->u32Msr = idMsr;
1732	pGuestMsrStore->u64Value = 0;
1733	}
1734
1735	/* Update the corresponding slot in the host MSR area. */
1736	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVmcsInfo->pvHostMsrLoad;
1737	Assert(pHostMsr != pVmcsInfo->pvGuestMsrLoad && pHostMsr != pVmcsInfo->pvGuestMsrStore);
1738	pHostMsr += i;
1739	pHostMsr->u32Msr = idMsr;
1740
1741	/*
1742	* Only if the caller requests to update the host MSR value AND we've newly added the
1743	* MSR to the host MSR area do we actually update the value. Otherwise, it will be
1744	* updated by hmR0VmxUpdateAutoLoadHostMsrs().
1745	*
1746	* We do this for performance reasons since reading MSRs may be quite expensive.
1747	*/
1748	if ( fAdded
1749	&& fUpdateHostMsr)
1750	{
1751	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1752	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1753	pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1754	}
1755	return VINF_SUCCESS;
1756	}
1757
1758
1759	/**
1760	* Removes a guest/host MSR pair to be swapped during the world-switch from the
1761	* auto-load/store MSR area in the VMCS.
1762	*
1763	* @returns VBox status code.
1764	* @param pVCpu The cross context virtual CPU structure.
1765	* @param pVmxTransient The VMX-transient structure.
1766	* @param idMsr The MSR.
1767	*/
1768	static int hmR0VmxRemoveAutoLoadStoreMsr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t idMsr)
1769	{
1770	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
1771	bool const fIsNstGstVmcs = pVmxTransient->fIsNestedGuest;
1772	PVMXAUTOMSR pGuestMsrLoad = (PVMXAUTOMSR)pVmcsInfo->pvGuestMsrLoad;
1773	uint32_t cMsrs = pVmcsInfo->cEntryMsrLoad;
1774
1775	bool const fSeparateExitMsrStorePage = hmR0VmxIsSeparateExitMsrStoreAreaVmcs(pVmcsInfo);
1776	for (uint32_t i = 0; i < cMsrs; i++)
1777	{
1778	/* Find the MSR. */
1779	if (pGuestMsrLoad->u32Msr == idMsr)
1780	{
1781	/* If it's the last MSR, simply reduce the count. */
1782	if (i == cMsrs - 1)
1783	{
1784	--cMsrs;
1785	break;
1786	}
1787
1788	/* Remove it by copying the last MSR in place of it, and reducing the count. */
1789	PVMXAUTOMSR pLastGuestMsrLoad = (PVMXAUTOMSR)pVmcsInfo->pvGuestMsrLoad;
1790	pLastGuestMsrLoad += cMsrs - 1;
1791	pGuestMsrLoad->u32Msr = pLastGuestMsrLoad->u32Msr;
1792	pGuestMsrLoad->u64Value = pLastGuestMsrLoad->u64Value;
1793
1794	/* Remove it from the VM-exit MSR-store area if we are using a different page. */
1795	if (fSeparateExitMsrStorePage)
1796	{
1797	PVMXAUTOMSR pGuestMsrStore = (PVMXAUTOMSR)pVmcsInfo->pvGuestMsrStore;
1798	PVMXAUTOMSR pLastGuestMsrStore = (PVMXAUTOMSR)pVmcsInfo->pvGuestMsrStore;
1799	pGuestMsrStore += i;
1800	pLastGuestMsrStore += cMsrs - 1;
1801	Assert(pGuestMsrStore->u32Msr == idMsr);
1802	pGuestMsrStore->u32Msr = pLastGuestMsrStore->u32Msr;
1803	pGuestMsrStore->u64Value = pLastGuestMsrStore->u64Value;
1804	}
1805
1806	/* Remove it from the VM-exit MSR-load area. */
1807	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVmcsInfo->pvHostMsrLoad;
1808	PVMXAUTOMSR pLastHostMsr = (PVMXAUTOMSR)pVmcsInfo->pvHostMsrLoad;
1809	pHostMsr += i;
1810	pLastHostMsr += cMsrs - 1;
1811	Assert(pHostMsr->u32Msr == idMsr);
1812	pHostMsr->u32Msr = pLastHostMsr->u32Msr;
1813	pHostMsr->u64Value = pLastHostMsr->u64Value;
1814	--cMsrs;
1815	break;
1816	}
1817	pGuestMsrLoad++;
1818	}
1819
1820	/* Update the VMCS if the count changed (meaning the MSR was found). */
1821	if (cMsrs != pVmcsInfo->cEntryMsrLoad)
1822	{
1823	int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, pVmcsInfo, cMsrs);
1824	AssertRCReturn(rc, rc);
1825
1826	/* We're no longer swapping MSRs during the world-switch, intercept guest read/writes to them. */
1827	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
1828	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, idMsr, VMXMSRPM_EXIT_RD \| VMXMSRPM_EXIT_WR);
1829
1830	Log4Func(("Removed MSR %#RX32, cMsrs=%u\n", idMsr, cMsrs));
1831	return VINF_SUCCESS;
1832	}
1833
1834	return VERR_NOT_FOUND;
1835	}
1836
1837
1838	/**
1839	* Checks if the specified guest MSR is part of the VM-entry MSR-load area.
1840	*
1841	* @returns @c true if found, @c false otherwise.
1842	* @param pVmcsInfo The VMCS info. object.
1843	* @param idMsr The MSR to find.
1844	*/
1845	static bool hmR0VmxIsAutoLoadGuestMsr(PCVMXVMCSINFO pVmcsInfo, uint32_t idMsr)
1846	{
1847	PCVMXAUTOMSR pGuestMsrLoad = (PCVMXAUTOMSR)pVmcsInfo->pvGuestMsrLoad;
1848	uint32_t const cMsrs = pVmcsInfo->cEntryMsrLoad;
1849	for (uint32_t i = 0; i < cMsrs; i++)
1850	{
1851	if (pGuestMsrLoad->u32Msr == idMsr)
1852	return true;
1853	pGuestMsrLoad++;
1854	}
1855	return false;
1856	}
1857
1858
1859	/**
1860	* Updates the value of all host MSRs in the VM-exit MSR-load area.
1861	*
1862	* @param pVCpu The cross context virtual CPU structure.
1863	* @param pVmcsInfo The VMCS info. object.
1864	*
1865	* @remarks No-long-jump zone!!!
1866	*/
1867	static void hmR0VmxUpdateAutoLoadHostMsrs(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
1868	{
1869	PVMXAUTOMSR pHostMsrLoad = (PVMXAUTOMSR)pVmcsInfo->pvHostMsrLoad;
1870	uint32_t const cMsrs = pVmcsInfo->cExitMsrLoad;
1871
1872	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1873	Assert(pHostMsrLoad);
1874
1875	for (uint32_t i = 0; i < cMsrs; i++, pHostMsrLoad++)
1876	{
1877	/*
1878	* Performance hack for the host EFER MSR. We use the cached value rather than re-read it.
1879	* Strict builds will catch mismatches in hmR0VmxCheckAutoLoadStoreMsrs(). See @bugref{7368}.
1880	*/
1881	if (pHostMsrLoad->u32Msr == MSR_K6_EFER)
1882	pHostMsrLoad->u64Value = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostMsrEfer;
1883	else
1884	pHostMsrLoad->u64Value = ASMRdMsr(pHostMsrLoad->u32Msr);
1885	}
1886	}
1887
1888
1889	/**
1890	* Saves a set of host MSRs to allow read/write passthru access to the guest and
1891	* perform lazy restoration of the host MSRs while leaving VT-x.
1892	*
1893	* @param pVCpu The cross context virtual CPU structure.
1894	*
1895	* @remarks No-long-jump zone!!!
1896	*/
1897	static void hmR0VmxLazySaveHostMsrs(PVMCPU pVCpu)
1898	{
1899	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1900
1901	/*
1902	* Note: If you're adding MSRs here, make sure to update the MSR-bitmap accesses in hmR0VmxSetupVmcsProcCtls().
1903	*/
1904	if (!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST))
1905	{
1906	Assert(!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)); /* Guest MSRs better not be loaded now. */
1907	#if HC_ARCH_BITS == 64
1908	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1909	{
1910	pVCpu->hm.s.vmx.u64HostMsrLStar = ASMRdMsr(MSR_K8_LSTAR);
1911	pVCpu->hm.s.vmx.u64HostMsrStar = ASMRdMsr(MSR_K6_STAR);
1912	pVCpu->hm.s.vmx.u64HostMsrSfMask = ASMRdMsr(MSR_K8_SF_MASK);
1913	pVCpu->hm.s.vmx.u64HostMsrKernelGsBase = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1914	}
1915	#endif
1916	pVCpu->hm.s.vmx.fLazyMsrs \|= VMX_LAZY_MSRS_SAVED_HOST;
1917	}
1918	}
1919
1920
1921	/**
1922	* Checks whether the MSR belongs to the set of guest MSRs that we restore
1923	* lazily while leaving VT-x.
1924	*
1925	* @returns true if it does, false otherwise.
1926	* @param pVCpu The cross context virtual CPU structure.
1927	* @param idMsr The MSR to check.
1928	*/
1929	static bool hmR0VmxIsLazyGuestMsr(PVMCPU pVCpu, uint32_t idMsr)
1930	{
1931	NOREF(pVCpu);
1932	#if HC_ARCH_BITS == 64
1933	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1934	{
1935	switch (idMsr)
1936	{
1937	case MSR_K8_LSTAR:
1938	case MSR_K6_STAR:
1939	case MSR_K8_SF_MASK:
1940	case MSR_K8_KERNEL_GS_BASE:
1941	return true;
1942	}
1943	}
1944	#else
1945	RT_NOREF(pVCpu, idMsr);
1946	#endif
1947	return false;
1948	}
1949
1950
1951	/**
1952	* Loads a set of guests MSRs to allow read/passthru to the guest.
1953	*
1954	* The name of this function is slightly confusing. This function does NOT
1955	* postpone loading, but loads the MSR right now. "hmR0VmxLazy" is simply a
1956	* common prefix for functions dealing with "lazy restoration" of the shared
1957	* MSRs.
1958	*
1959	* @param pVCpu The cross context virtual CPU structure.
1960	*
1961	* @remarks No-long-jump zone!!!
1962	*/
1963	static void hmR0VmxLazyLoadGuestMsrs(PVMCPU pVCpu)
1964	{
1965	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1966	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1967
1968	Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1969	#if HC_ARCH_BITS == 64
1970	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1971	{
1972	/*
1973	* If the guest MSRs are not loaded -and- if all the guest MSRs are identical
1974	* to the MSRs on the CPU (which are the saved host MSRs, see assertion above) then
1975	* we can skip a few MSR writes.
1976	*
1977	* Otherwise, it implies either 1. they're not loaded, or 2. they're loaded but the
1978	* guest MSR values in the guest-CPU context might be different to what's currently
1979	* loaded in the CPU. In either case, we need to write the new guest MSR values to the
1980	* CPU, see @bugref{8728}.
1981	*/
1982	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
1983	if ( !(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1984	&& pCtx->msrKERNELGSBASE == pVCpu->hm.s.vmx.u64HostMsrKernelGsBase
1985	&& pCtx->msrLSTAR == pVCpu->hm.s.vmx.u64HostMsrLStar
1986	&& pCtx->msrSTAR == pVCpu->hm.s.vmx.u64HostMsrStar
1987	&& pCtx->msrSFMASK == pVCpu->hm.s.vmx.u64HostMsrSfMask)
1988	{
1989	#ifdef VBOX_STRICT
1990	Assert(ASMRdMsr(MSR_K8_KERNEL_GS_BASE) == pCtx->msrKERNELGSBASE);
1991	Assert(ASMRdMsr(MSR_K8_LSTAR) == pCtx->msrLSTAR);
1992	Assert(ASMRdMsr(MSR_K6_STAR) == pCtx->msrSTAR);
1993	Assert(ASMRdMsr(MSR_K8_SF_MASK) == pCtx->msrSFMASK);
1994	#endif
1995	}
1996	else
1997	{
1998	ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pCtx->msrKERNELGSBASE);
1999	ASMWrMsr(MSR_K8_LSTAR, pCtx->msrLSTAR);
2000	ASMWrMsr(MSR_K6_STAR, pCtx->msrSTAR);
2001	ASMWrMsr(MSR_K8_SF_MASK, pCtx->msrSFMASK);
2002	}
2003	}
2004	#endif
2005	pVCpu->hm.s.vmx.fLazyMsrs \|= VMX_LAZY_MSRS_LOADED_GUEST;
2006	}
2007
2008
2009	/**
2010	* Performs lazy restoration of the set of host MSRs if they were previously
2011	* loaded with guest MSR values.
2012	*
2013	* @param pVCpu The cross context virtual CPU structure.
2014	*
2015	* @remarks No-long-jump zone!!!
2016	* @remarks The guest MSRs should have been saved back into the guest-CPU
2017	* context by hmR0VmxImportGuestState()!!!
2018	*/
2019	static void hmR0VmxLazyRestoreHostMsrs(PVMCPU pVCpu)
2020	{
2021	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2022	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
2023
2024	if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
2025	{
2026	Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
2027	#if HC_ARCH_BITS == 64
2028	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
2029	{
2030	ASMWrMsr(MSR_K8_LSTAR, pVCpu->hm.s.vmx.u64HostMsrLStar);
2031	ASMWrMsr(MSR_K6_STAR, pVCpu->hm.s.vmx.u64HostMsrStar);
2032	ASMWrMsr(MSR_K8_SF_MASK, pVCpu->hm.s.vmx.u64HostMsrSfMask);
2033	ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pVCpu->hm.s.vmx.u64HostMsrKernelGsBase);
2034	}
2035	#endif
2036	}
2037	pVCpu->hm.s.vmx.fLazyMsrs &= ~(VMX_LAZY_MSRS_LOADED_GUEST \| VMX_LAZY_MSRS_SAVED_HOST);
2038	}
2039
2040
2041	/**
2042	* Verifies that our cached values of the VMCS fields are all consistent with
2043	* what's actually present in the VMCS.
2044	*
2045	* @returns VBox status code.
2046	* @retval VINF_SUCCESS if all our caches match their respective VMCS fields.
2047	* @retval VERR_VMX_VMCS_FIELD_CACHE_INVALID if a cache field doesn't match the
2048	* VMCS content. HMCPU error-field is
2049	* updated, see VMX_VCI_XXX.
2050	* @param pVCpu The cross context virtual CPU structure.
2051	* @param pVmcsInfo The VMCS info. object.
2052	*/
2053	static int hmR0VmxCheckVmcsCtls(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
2054	{
2055	uint32_t u32Val;
2056	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
2057	AssertRCReturn(rc, rc);
2058	AssertMsgReturnStmt(pVmcsInfo->u32EntryCtls == u32Val,
2059	("Cache=%#RX32 VMCS=%#RX32\n", pVmcsInfo->u32EntryCtls, u32Val),
2060	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_ENTRY,
2061	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2062
2063	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val);
2064	AssertRCReturn(rc, rc);
2065	AssertMsgReturnStmt(pVmcsInfo->u32ExitCtls == u32Val,
2066	("Cache=%#RX32 VMCS=%#RX32\n", pVmcsInfo->u32ExitCtls, u32Val),
2067	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_EXIT,
2068	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2069
2070	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val);
2071	AssertRCReturn(rc, rc);
2072	AssertMsgReturnStmt(pVmcsInfo->u32PinCtls == u32Val,
2073	("Cache=%#RX32 VMCS=%#RX32\n", pVmcsInfo->u32PinCtls, u32Val),
2074	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_PIN_EXEC,
2075	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2076
2077	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val);
2078	AssertRCReturn(rc, rc);
2079	AssertMsgReturnStmt(pVmcsInfo->u32ProcCtls == u32Val,
2080	("Cache=%#RX32 VMCS=%#RX32\n", pVmcsInfo->u32ProcCtls, u32Val),
2081	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_PROC_EXEC,
2082	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2083
2084	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
2085	{
2086	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val);
2087	AssertRCReturn(rc, rc);
2088	AssertMsgReturnStmt(pVmcsInfo->u32ProcCtls2 == u32Val,
2089	("Cache=%#RX32 VMCS=%#RX32\n", pVmcsInfo->u32ProcCtls2, u32Val),
2090	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_PROC_EXEC2,
2091	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2092	}
2093
2094	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val);
2095	AssertRCReturn(rc, rc);
2096	AssertMsgReturnStmt(pVmcsInfo->u32XcptBitmap == u32Val,
2097	("Cache=%#RX32 VMCS=%#RX32\n", pVmcsInfo->u32XcptBitmap, u32Val),
2098	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_XCPT_BITMAP,
2099	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2100
2101	uint64_t u64Val;
2102	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, &u64Val);
2103	AssertRCReturn(rc, rc);
2104	AssertMsgReturnStmt(pVmcsInfo->u64TscOffset == u64Val,
2105	("Cache=%#RX64 VMCS=%#RX64\n", pVmcsInfo->u64TscOffset, u64Val),
2106	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_TSC_OFFSET,
2107	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2108
2109	return VINF_SUCCESS;
2110	}
2111
2112
2113	#ifdef VBOX_STRICT
2114	/**
2115	* Verifies that our cached host EFER MSR value has not changed since we cached it.
2116	*
2117	* @param pVCpu The cross context virtual CPU structure.
2118	* @param pVmcsInfo The VMCS info. object.
2119	*/
2120	static void hmR0VmxCheckHostEferMsr(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
2121	{
2122	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2123
2124	if (pVmcsInfo->u32ExitCtls & VMX_EXIT_CTLS_LOAD_EFER_MSR)
2125	{
2126	uint64_t const uHostEferMsr = ASMRdMsr(MSR_K6_EFER);
2127	uint64_t const uHostEferMsrCache = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostMsrEfer;
2128	uint64_t uVmcsEferMsrVmcs;
2129	int rc = VMXReadVmcs64(VMX_VMCS64_HOST_EFER_FULL, &uVmcsEferMsrVmcs);
2130	AssertRC(rc);
2131
2132	AssertMsgReturnVoid(uHostEferMsr == uVmcsEferMsrVmcs,
2133	("EFER Host/VMCS mismatch! host=%#RX64 vmcs=%#RX64\n", uHostEferMsr, uVmcsEferMsrVmcs));
2134	AssertMsgReturnVoid(uHostEferMsr == uHostEferMsrCache,
2135	("EFER Host/Cache mismatch! host=%#RX64 cache=%#RX64\n", uHostEferMsr, uHostEferMsrCache));
2136	}
2137	}
2138
2139
2140	/**
2141	* Verifies whether the guest/host MSR pairs in the auto-load/store area in the
2142	* VMCS are correct.
2143	*
2144	* @param pVCpu The cross context virtual CPU structure.
2145	* @param pVmcsInfo The VMCS info. object.
2146	*/
2147	static void hmR0VmxCheckAutoLoadStoreMsrs(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
2148	{
2149	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2150
2151	/* Verify MSR counts in the VMCS are what we think it should be. */
2152	uint32_t cMsrs;
2153	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &cMsrs);
2154	AssertRC(rc);
2155	Assert(cMsrs == pVmcsInfo->cEntryMsrLoad);
2156
2157	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &cMsrs);
2158	AssertRC(rc);
2159	Assert(cMsrs == pVmcsInfo->cExitMsrStore);
2160
2161	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &cMsrs);
2162	AssertRC(rc);
2163	Assert(cMsrs == pVmcsInfo->cExitMsrLoad);
2164
2165	/* Verify the MSR counts do not exceed the maximum count supported by the hardware. */
2166	Assert(cMsrs < VMX_MISC_MAX_MSRS(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc));
2167
2168	PCVMXAUTOMSR pGuestMsrLoad = (PCVMXAUTOMSR)pVmcsInfo->pvGuestMsrLoad;
2169	PCVMXAUTOMSR pGuestMsrStore = (PCVMXAUTOMSR)pVmcsInfo->pvGuestMsrStore;
2170	PCVMXAUTOMSR pHostMsrLoad = (PCVMXAUTOMSR)pVmcsInfo->pvHostMsrLoad;
2171	bool const fSeparateExitMsrStorePage = hmR0VmxIsSeparateExitMsrStoreAreaVmcs(pVmcsInfo);
2172	for (uint32_t i = 0; i < cMsrs; i++)
2173	{
2174	/* Verify that the MSRs are paired properly and that the host MSR has the correct value. */
2175	if (fSeparateExitMsrStorePage)
2176	{
2177	AssertMsgReturnVoid(pGuestMsrLoad->u32Msr == pGuestMsrStore->u32Msr,
2178	("GuestMsrLoad=%#RX32 GuestMsrStore=%#RX32 cMsrs=%u\n",
2179	pGuestMsrLoad->u32Msr, pGuestMsrStore->u32Msr, cMsrs));
2180	}
2181
2182	AssertMsgReturnVoid(pHostMsrLoad->u32Msr == pGuestMsrLoad->u32Msr,
2183	("HostMsrLoad=%#RX32 GuestMsrLoad=%#RX32 cMsrs=%u\n",
2184	pHostMsrLoad->u32Msr, pGuestMsrLoad->u32Msr, cMsrs));
2185
2186	uint64_t const u64Msr = ASMRdMsr(pHostMsrLoad->u32Msr);
2187	AssertMsgReturnVoid(pHostMsrLoad->u64Value == u64Msr,
2188	("u32Msr=%#RX32 VMCS Value=%#RX64 ASMRdMsr=%#RX64 cMsrs=%u\n",
2189	pHostMsrLoad->u32Msr, pHostMsrLoad->u64Value, u64Msr, cMsrs));
2190
2191	/* Verify that the accesses are as expected in the MSR bitmap for auto-load/store MSRs. */
2192	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
2193	{
2194	uint32_t fMsrpm = HMGetVmxMsrPermission(pVmcsInfo->pvMsrBitmap, pGuestMsrLoad->u32Msr);
2195	if (pGuestMsrLoad->u32Msr == MSR_K6_EFER)
2196	{
2197	AssertMsgReturnVoid((fMsrpm & VMXMSRPM_EXIT_RD), ("Passthru read for EFER MSR!?\n"));
2198	AssertMsgReturnVoid((fMsrpm & VMXMSRPM_EXIT_WR), ("Passthru write for EFER MSR!?\n"));
2199	}
2200	else
2201	{
2202	AssertMsgReturnVoid((fMsrpm & (VMXMSRPM_ALLOW_RD \| VMXMSRPM_ALLOW_WR))
2203	== (VMXMSRPM_ALLOW_RD \| VMXMSRPM_ALLOW_WR),
2204	("u32Msr=%#RX32 cMsrs=%u No passthru read/write!\n", pGuestMsrLoad->u32Msr, cMsrs));
2205	}
2206	}
2207
2208	/* Move to the next MSR. */
2209	pHostMsrLoad++;
2210	pGuestMsrLoad++;
2211	pGuestMsrStore++;
2212	}
2213	}
2214	#endif /* VBOX_STRICT */
2215
2216
2217	/**
2218	* Flushes the TLB using EPT.
2219	*
2220	* @returns VBox status code.
2221	* @param pVCpu The cross context virtual CPU structure of the calling
2222	* EMT. Can be NULL depending on @a enmTlbFlush.
2223	* @param pVmcsInfo The VMCS info. object. Can be NULL depending on @a
2224	* enmTlbFlush.
2225	* @param enmTlbFlush Type of flush.
2226	*
2227	* @remarks Caller is responsible for making sure this function is called only
2228	* when NestedPaging is supported and providing @a enmTlbFlush that is
2229	* supported by the CPU.
2230	* @remarks Can be called with interrupts disabled.
2231	*/
2232	static void hmR0VmxFlushEpt(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo, VMXTLBFLUSHEPT enmTlbFlush)
2233	{
2234	uint64_t au64Descriptor[2];
2235	if (enmTlbFlush == VMXTLBFLUSHEPT_ALL_CONTEXTS)
2236	au64Descriptor[0] = 0;
2237	else
2238	{
2239	Assert(pVCpu);
2240	Assert(pVmcsInfo);
2241	au64Descriptor[0] = pVmcsInfo->HCPhysEPTP;
2242	}
2243	au64Descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
2244
2245	int rc = VMXR0InvEPT(enmTlbFlush, &au64Descriptor[0]);
2246	AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %#x %#RHp failed. rc=%Rrc\n", enmTlbFlush, au64Descriptor[0], rc));
2247
2248	if ( RT_SUCCESS(rc)
2249	&& pVCpu)
2250	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
2251	}
2252
2253
2254	/**
2255	* Flushes the TLB using VPID.
2256	*
2257	* @returns VBox status code.
2258	* @param pVCpu The cross context virtual CPU structure of the calling
2259	* EMT. Can be NULL depending on @a enmTlbFlush.
2260	* @param enmTlbFlush Type of flush.
2261	* @param GCPtr Virtual address of the page to flush (can be 0 depending
2262	* on @a enmTlbFlush).
2263	*
2264	* @remarks Can be called with interrupts disabled.
2265	*/
2266	static void hmR0VmxFlushVpid(PVMCPU pVCpu, VMXTLBFLUSHVPID enmTlbFlush, RTGCPTR GCPtr)
2267	{
2268	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid);
2269
2270	uint64_t au64Descriptor[2];
2271	if (enmTlbFlush == VMXTLBFLUSHVPID_ALL_CONTEXTS)
2272	{
2273	au64Descriptor[0] = 0;
2274	au64Descriptor[1] = 0;
2275	}
2276	else
2277	{
2278	AssertPtr(pVCpu);
2279	AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
2280	AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
2281	au64Descriptor[0] = pVCpu->hm.s.uCurrentAsid;
2282	au64Descriptor[1] = GCPtr;
2283	}
2284
2285	int rc = VMXR0InvVPID(enmTlbFlush, &au64Descriptor[0]);
2286	AssertMsg(rc == VINF_SUCCESS,
2287	("VMXR0InvVPID %#x %u %RGv failed with %Rrc\n", enmTlbFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
2288
2289	if ( RT_SUCCESS(rc)
2290	&& pVCpu)
2291	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
2292	NOREF(rc);
2293	}
2294
2295
2296	/**
2297	* Invalidates a guest page by guest virtual address. Only relevant for EPT/VPID,
2298	* otherwise there is nothing really to invalidate.
2299	*
2300	* @returns VBox status code.
2301	* @param pVCpu The cross context virtual CPU structure.
2302	* @param GCVirt Guest virtual address of the page to invalidate.
2303	*/
2304	VMMR0DECL(int) VMXR0InvalidatePage(PVMCPU pVCpu, RTGCPTR GCVirt)
2305	{
2306	AssertPtr(pVCpu);
2307	LogFlowFunc(("pVCpu=%p GCVirt=%RGv\n", pVCpu, GCVirt));
2308
2309	if (!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_TLB_FLUSH))
2310	{
2311	/*
2312	* We must invalidate the guest TLB entry in either case, we cannot ignore it even for
2313	* the EPT case. See @bugref{6043} and @bugref{6177}.
2314	*
2315	* Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*()
2316	* as this function maybe called in a loop with individual addresses.
2317	*/
2318	PVM pVM = pVCpu->CTX_SUFF(pVM);
2319	if (pVM->hm.s.vmx.fVpid)
2320	{
2321	bool fVpidFlush = RT_BOOL(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR);
2322
2323	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
2324	/*
2325	* Workaround Erratum BV75, AAJ159 and others that affect several Intel CPUs
2326	* where executing INVVPID outside 64-bit mode does not flush translations of
2327	* 64-bit linear addresses, see @bugref{6208#c72}.
2328	*/
2329	if (RT_HI_U32(GCVirt))
2330	fVpidFlush = false;
2331	#endif
2332
2333	if (fVpidFlush)
2334	{
2335	hmR0VmxFlushVpid(pVCpu, VMXTLBFLUSHVPID_INDIV_ADDR, GCVirt);
2336	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
2337	}
2338	else
2339	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2340	}
2341	else if (pVM->hm.s.fNestedPaging)
2342	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2343	}
2344
2345	return VINF_SUCCESS;
2346	}
2347
2348
2349	/**
2350	* Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
2351	* case where neither EPT nor VPID is supported by the CPU.
2352	*
2353	* @param pHostCpu The HM physical-CPU structure.
2354	* @param pVCpu The cross context virtual CPU structure.
2355	*
2356	* @remarks Called with interrupts disabled.
2357	*/
2358	static void hmR0VmxFlushTaggedTlbNone(PHMPHYSCPU pHostCpu, PVMCPU pVCpu)
2359	{
2360	AssertPtr(pVCpu);
2361	AssertPtr(pHostCpu);
2362
2363	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
2364
2365	Assert(pHostCpu->idCpu != NIL_RTCPUID);
2366	pVCpu->hm.s.idLastCpu = pHostCpu->idCpu;
2367	pVCpu->hm.s.cTlbFlushes = pHostCpu->cTlbFlushes;
2368	pVCpu->hm.s.fForceTLBFlush = false;
2369	return;
2370	}
2371
2372
2373	/**
2374	* Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
2375	*
2376	* @param pHostCpu The HM physical-CPU structure.
2377	* @param pVCpu The cross context virtual CPU structure.
2378	* @param pVmcsInfo The VMCS info. object.
2379	*
2380	* @remarks All references to "ASID" in this function pertains to "VPID" in Intel's
2381	* nomenclature. The reason is, to avoid confusion in compare statements
2382	* since the host-CPU copies are named "ASID".
2383	*
2384	* @remarks Called with interrupts disabled.
2385	*/
2386	static void hmR0VmxFlushTaggedTlbBoth(PHMPHYSCPU pHostCpu, PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
2387	{
2388	#ifdef VBOX_WITH_STATISTICS
2389	bool fTlbFlushed = false;
2390	# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { fTlbFlushed = true; } while (0)
2391	# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { \
2392	if (!fTlbFlushed) \
2393	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch); \
2394	} while (0)
2395	#else
2396	# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { } while (0)
2397	# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { } while (0)
2398	#endif
2399
2400	AssertPtr(pVCpu);
2401	AssertPtr(pHostCpu);
2402	Assert(pHostCpu->idCpu != NIL_RTCPUID);
2403
2404	PVM pVM = pVCpu->CTX_SUFF(pVM);
2405	AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
2406	("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
2407	"fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
2408
2409	/*
2410	* Force a TLB flush for the first world-switch if the current CPU differs from the one we
2411	* ran on last. If the TLB flush count changed, another VM (VCPU rather) has hit the ASID
2412	* limit while flushing the TLB or the host CPU is online after a suspend/resume, so we
2413	* cannot reuse the current ASID anymore.
2414	*/
2415	if ( pVCpu->hm.s.idLastCpu != pHostCpu->idCpu
2416	\|\| pVCpu->hm.s.cTlbFlushes != pHostCpu->cTlbFlushes)
2417	{
2418	++pHostCpu->uCurrentAsid;
2419	if (pHostCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
2420	{
2421	pHostCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0. */
2422	pHostCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
2423	pHostCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
2424	}
2425
2426	pVCpu->hm.s.uCurrentAsid = pHostCpu->uCurrentAsid;
2427	pVCpu->hm.s.idLastCpu = pHostCpu->idCpu;
2428	pVCpu->hm.s.cTlbFlushes = pHostCpu->cTlbFlushes;
2429
2430	/*
2431	* Flush by EPT when we get rescheduled to a new host CPU to ensure EPT-only tagged mappings are also
2432	* invalidated. We don't need to flush-by-VPID here as flushing by EPT covers it. See @bugref{6568}.
2433	*/
2434	hmR0VmxFlushEpt(pVCpu, pVmcsInfo, pVM->hm.s.vmx.enmTlbFlushEpt);
2435	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2436	HMVMX_SET_TAGGED_TLB_FLUSHED();
2437	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
2438	}
2439	else if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH)) /* Check for explicit TLB flushes. */
2440	{
2441	/*
2442	* Changes to the EPT paging structure by VMM requires flushing-by-EPT as the CPU
2443	* creates guest-physical (ie. only EPT-tagged) mappings while traversing the EPT
2444	* tables when EPT is in use. Flushing-by-VPID will only flush linear (only
2445	* VPID-tagged) and combined (EPT+VPID tagged) mappings but not guest-physical
2446	* mappings, see @bugref{6568}.
2447	*
2448	* See Intel spec. 28.3.2 "Creating and Using Cached Translation Information".
2449	*/
2450	hmR0VmxFlushEpt(pVCpu, pVmcsInfo, pVM->hm.s.vmx.enmTlbFlushEpt);
2451	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2452	HMVMX_SET_TAGGED_TLB_FLUSHED();
2453	}
2454
2455	pVCpu->hm.s.fForceTLBFlush = false;
2456	HMVMX_UPDATE_FLUSH_SKIPPED_STAT();
2457
2458	Assert(pVCpu->hm.s.idLastCpu == pHostCpu->idCpu);
2459	Assert(pVCpu->hm.s.cTlbFlushes == pHostCpu->cTlbFlushes);
2460	AssertMsg(pVCpu->hm.s.cTlbFlushes == pHostCpu->cTlbFlushes,
2461	("Flush count mismatch for cpu %d (%u vs %u)\n", pHostCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pHostCpu->cTlbFlushes));
2462	AssertMsg(pHostCpu->uCurrentAsid >= 1 && pHostCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2463	("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pHostCpu->idCpu,
2464	pHostCpu->uCurrentAsid, pHostCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2465	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2466	("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pHostCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2467
2468	/* Update VMCS with the VPID. */
2469	int rc = VMXWriteVmcs32(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
2470	AssertRC(rc);
2471
2472	#undef HMVMX_SET_TAGGED_TLB_FLUSHED
2473	}
2474
2475
2476	/**
2477	* Flushes the tagged-TLB entries for EPT CPUs as necessary.
2478	*
2479	* @param pHostCpu The HM physical-CPU structure.
2480	* @param pVCpu The cross context virtual CPU structure.
2481	* @param pVmcsInfo The VMCS info. object.
2482	*
2483	* @remarks Called with interrupts disabled.
2484	*/
2485	static void hmR0VmxFlushTaggedTlbEpt(PHMPHYSCPU pHostCpu, PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
2486	{
2487	AssertPtr(pVCpu);
2488	AssertPtr(pHostCpu);
2489	Assert(pHostCpu->idCpu != NIL_RTCPUID);
2490	AssertMsg(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked without NestedPaging."));
2491	AssertMsg(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID."));
2492
2493	/*
2494	* Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
2495	* A change in the TLB flush count implies the host CPU is online after a suspend/resume.
2496	*/
2497	if ( pVCpu->hm.s.idLastCpu != pHostCpu->idCpu
2498	\|\| pVCpu->hm.s.cTlbFlushes != pHostCpu->cTlbFlushes)
2499	{
2500	pVCpu->hm.s.fForceTLBFlush = true;
2501	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2502	}
2503
2504	/* Check for explicit TLB flushes. */
2505	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2506	{
2507	pVCpu->hm.s.fForceTLBFlush = true;
2508	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2509	}
2510
2511	pVCpu->hm.s.idLastCpu = pHostCpu->idCpu;
2512	pVCpu->hm.s.cTlbFlushes = pHostCpu->cTlbFlushes;
2513
2514	if (pVCpu->hm.s.fForceTLBFlush)
2515	{
2516	hmR0VmxFlushEpt(pVCpu, pVmcsInfo, pVCpu->CTX_SUFF(pVM)->hm.s.vmx.enmTlbFlushEpt);
2517	pVCpu->hm.s.fForceTLBFlush = false;
2518	}
2519	}
2520
2521
2522	/**
2523	* Flushes the tagged-TLB entries for VPID CPUs as necessary.
2524	*
2525	* @param pHostCpu The HM physical-CPU structure.
2526	* @param pVCpu The cross context virtual CPU structure.
2527	*
2528	* @remarks Called with interrupts disabled.
2529	*/
2530	static void hmR0VmxFlushTaggedTlbVpid(PHMPHYSCPU pHostCpu, PVMCPU pVCpu)
2531	{
2532	AssertPtr(pVCpu);
2533	AssertPtr(pHostCpu);
2534	Assert(pHostCpu->idCpu != NIL_RTCPUID);
2535	AssertMsg(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked without VPID."));
2536	AssertMsg(!pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging"));
2537
2538	/*
2539	* Force a TLB flush for the first world switch if the current CPU differs from the one we
2540	* ran on last. If the TLB flush count changed, another VM (VCPU rather) has hit the ASID
2541	* limit while flushing the TLB or the host CPU is online after a suspend/resume, so we
2542	* cannot reuse the current ASID anymore.
2543	*/
2544	if ( pVCpu->hm.s.idLastCpu != pHostCpu->idCpu
2545	\|\| pVCpu->hm.s.cTlbFlushes != pHostCpu->cTlbFlushes)
2546	{
2547	pVCpu->hm.s.fForceTLBFlush = true;
2548	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2549	}
2550
2551	/* Check for explicit TLB flushes. */
2552	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2553	{
2554	/*
2555	* If we ever support VPID flush combinations other than ALL or SINGLE-context (see
2556	* hmR0VmxSetupTaggedTlb()) we would need to explicitly flush in this case (add an
2557	* fExplicitFlush = true here and change the pHostCpu->fFlushAsidBeforeUse check below to
2558	* include fExplicitFlush's too) - an obscure corner case.
2559	*/
2560	pVCpu->hm.s.fForceTLBFlush = true;
2561	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2562	}
2563
2564	PVM pVM = pVCpu->CTX_SUFF(pVM);
2565	pVCpu->hm.s.idLastCpu = pHostCpu->idCpu;
2566	if (pVCpu->hm.s.fForceTLBFlush)
2567	{
2568	++pHostCpu->uCurrentAsid;
2569	if (pHostCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
2570	{
2571	pHostCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0 */
2572	pHostCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
2573	pHostCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
2574	}
2575
2576	pVCpu->hm.s.fForceTLBFlush = false;
2577	pVCpu->hm.s.cTlbFlushes = pHostCpu->cTlbFlushes;
2578	pVCpu->hm.s.uCurrentAsid = pHostCpu->uCurrentAsid;
2579	if (pHostCpu->fFlushAsidBeforeUse)
2580	{
2581	if (pVM->hm.s.vmx.enmTlbFlushVpid == VMXTLBFLUSHVPID_SINGLE_CONTEXT)
2582	hmR0VmxFlushVpid(pVCpu, VMXTLBFLUSHVPID_SINGLE_CONTEXT, 0 /* GCPtr */);
2583	else if (pVM->hm.s.vmx.enmTlbFlushVpid == VMXTLBFLUSHVPID_ALL_CONTEXTS)
2584	{
2585	hmR0VmxFlushVpid(pVCpu, VMXTLBFLUSHVPID_ALL_CONTEXTS, 0 /* GCPtr */);
2586	pHostCpu->fFlushAsidBeforeUse = false;
2587	}
2588	else
2589	{
2590	/* hmR0VmxSetupTaggedTlb() ensures we never get here. Paranoia. */
2591	AssertMsgFailed(("Unsupported VPID-flush context type.\n"));
2592	}
2593	}
2594	}
2595
2596	AssertMsg(pVCpu->hm.s.cTlbFlushes == pHostCpu->cTlbFlushes,
2597	("Flush count mismatch for cpu %d (%u vs %u)\n", pHostCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pHostCpu->cTlbFlushes));
2598	AssertMsg(pHostCpu->uCurrentAsid >= 1 && pHostCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2599	("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pHostCpu->idCpu,
2600	pHostCpu->uCurrentAsid, pHostCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2601	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2602	("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pHostCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2603
2604	int rc = VMXWriteVmcs32(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
2605	AssertRC(rc);
2606	}
2607
2608
2609	/**
2610	* Flushes the guest TLB entry based on CPU capabilities.
2611	*
2612	* @param pHostCpu The HM physical-CPU structure.
2613	* @param pVCpu The cross context virtual CPU structure.
2614	* @param pVmcsInfo The VMCS info. object.
2615	*
2616	* @remarks Called with interrupts disabled.
2617	*/
2618	DECLINLINE(void) hmR0VmxFlushTaggedTlb(PHMPHYSCPU pHostCpu, PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
2619	{
2620	#ifdef HMVMX_ALWAYS_FLUSH_TLB
2621	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2622	#endif
2623	PVM pVM = pVCpu->CTX_SUFF(pVM);
2624	switch (pVM->hm.s.vmx.enmTlbFlushType)
2625	{
2626	case VMXTLBFLUSHTYPE_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pHostCpu, pVCpu, pVmcsInfo); break;
2627	case VMXTLBFLUSHTYPE_EPT: hmR0VmxFlushTaggedTlbEpt(pHostCpu, pVCpu, pVmcsInfo); break;
2628	case VMXTLBFLUSHTYPE_VPID: hmR0VmxFlushTaggedTlbVpid(pHostCpu, pVCpu); break;
2629	case VMXTLBFLUSHTYPE_NONE: hmR0VmxFlushTaggedTlbNone(pHostCpu, pVCpu); break;
2630	default:
2631	AssertMsgFailed(("Invalid flush-tag function identifier\n"));
2632	break;
2633	}
2634	/* Don't assert that VMCPU_FF_TLB_FLUSH should no longer be pending. It can be set by other EMTs. */
2635	}
2636
2637
2638	/**
2639	* Sets up the appropriate tagged TLB-flush level and handler for flushing guest
2640	* TLB entries from the host TLB before VM-entry.
2641	*
2642	* @returns VBox status code.
2643	* @param pVM The cross context VM structure.
2644	*/
2645	static int hmR0VmxSetupTaggedTlb(PVM pVM)
2646	{
2647	/*
2648	* Determine optimal flush type for nested paging.
2649	* We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup
2650	* unrestricted guest execution (see hmR3InitFinalizeR0()).
2651	*/
2652	if (pVM->hm.s.fNestedPaging)
2653	{
2654	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
2655	{
2656	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
2657	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_SINGLE_CONTEXT;
2658	else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
2659	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_ALL_CONTEXTS;
2660	else
2661	{
2662	/* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
2663	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
2664	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_FLUSH_TYPE_UNSUPPORTED;
2665	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2666	}
2667
2668	/* Make sure the write-back cacheable memory type for EPT is supported. */
2669	if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB)))
2670	{
2671	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
2672	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_MEM_TYPE_NOT_WB;
2673	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2674	}
2675
2676	/* EPT requires a page-walk length of 4. */
2677	if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_PAGE_WALK_LENGTH_4)))
2678	{
2679	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
2680	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_PAGE_WALK_LENGTH_UNSUPPORTED;
2681	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2682	}
2683	}
2684	else
2685	{
2686	/* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
2687	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
2688	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_INVEPT_UNAVAILABLE;
2689	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2690	}
2691	}
2692
2693	/*
2694	* Determine optimal flush type for VPID.
2695	*/
2696	if (pVM->hm.s.vmx.fVpid)
2697	{
2698	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
2699	{
2700	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
2701	pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_SINGLE_CONTEXT;
2702	else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
2703	pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_ALL_CONTEXTS;
2704	else
2705	{
2706	/* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
2707	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
2708	LogRelFunc(("Only INDIV_ADDR supported. Ignoring VPID.\n"));
2709	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
2710	LogRelFunc(("Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
2711	pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_NOT_SUPPORTED;
2712	pVM->hm.s.vmx.fVpid = false;
2713	}
2714	}
2715	else
2716	{
2717	/* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
2718	Log4Func(("VPID supported without INVEPT support. Ignoring VPID.\n"));
2719	pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_NOT_SUPPORTED;
2720	pVM->hm.s.vmx.fVpid = false;
2721	}
2722	}
2723
2724	/*
2725	* Setup the handler for flushing tagged-TLBs.
2726	*/
2727	if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
2728	pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_EPT_VPID;
2729	else if (pVM->hm.s.fNestedPaging)
2730	pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_EPT;
2731	else if (pVM->hm.s.vmx.fVpid)
2732	pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_VPID;
2733	else
2734	pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_NONE;
2735	return VINF_SUCCESS;
2736	}
2737
2738
2739	/**
2740	* Sets up the virtual-APIC page address for the VMCS.
2741	*
2742	* @returns VBox status code.
2743	* @param pVCpu The cross context virtual CPU structure.
2744	* @param pVmcsInfo The VMCS info. object.
2745	*/
2746	DECLINLINE(int) hmR0VmxSetupVmcsVirtApicAddr(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
2747	{
2748	NOREF(pVCpu);
2749	RTHCPHYS const HCPhysVirtApic = pVmcsInfo->HCPhysVirtApic;
2750	Assert(HCPhysVirtApic != NIL_RTHCPHYS);
2751	Assert(!(HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
2752	return VMXWriteVmcs64(VMX_VMCS64_CTRL_VIRT_APIC_PAGEADDR_FULL, HCPhysVirtApic);
2753	}
2754
2755
2756	/**
2757	* Sets up the MSR-bitmap address for the VMCS.
2758	*
2759	* @returns VBox status code.
2760	* @param pVCpu The cross context virtual CPU structure.
2761	* @param pVmcsInfo The VMCS info. object.
2762	*/
2763	DECLINLINE(int) hmR0VmxSetupVmcsMsrBitmapAddr(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
2764	{
2765	NOREF(pVCpu);
2766	RTHCPHYS const HCPhysMsrBitmap = pVmcsInfo->HCPhysMsrBitmap;
2767	Assert(HCPhysMsrBitmap != NIL_RTHCPHYS);
2768	Assert(!(HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
2769	return VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, HCPhysMsrBitmap);
2770	}
2771
2772
2773	/**
2774	* Sets up the APIC-access page address for the VMCS.
2775	*
2776	* @returns VBox status code.
2777	* @param pVCpu The cross context virtual CPU structure.
2778	*/
2779	DECLINLINE(int) hmR0VmxSetupVmcsApicAccessAddr(PVMCPU pVCpu)
2780	{
2781	RTHCPHYS const HCPhysApicAccess = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.HCPhysApicAccess;
2782	Assert(HCPhysApicAccess != NIL_RTHCPHYS);
2783	Assert(!(HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
2784	return VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, HCPhysApicAccess);
2785	}
2786
2787
2788	/**
2789	* Sets up the VMCS link pointer for the VMCS.
2790	*
2791	* @returns VBox status code.
2792	* @param pVCpu The cross context virtual CPU structure.
2793	* @param pVmcsInfo The VMCS info. object.
2794	*/
2795	DECLINLINE(int) hmR0VmxSetupVmcsLinkPtr(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
2796	{
2797	NOREF(pVCpu);
2798	uint64_t const u64VmcsLinkPtr = pVmcsInfo->u64VmcsLinkPtr;
2799	Assert(u64VmcsLinkPtr == UINT64_C(0xffffffffffffffff)); /* Bits 63:0 MB1. */
2800	return VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, u64VmcsLinkPtr);
2801	}
2802
2803
2804	/**
2805	* Sets up the VM-entry MSR load, VM-exit MSR-store and VM-exit MSR-load addresses
2806	* in the VMCS.
2807	*
2808	* @returns VBox status code.
2809	* @param pVCpu The cross context virtual CPU structure.
2810	* @param pVmcsInfo The VMCS info. object.
2811	*/
2812	DECLINLINE(int) hmR0VmxSetupVmcsAutoLoadStoreMsrAddrs(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
2813	{
2814	NOREF(pVCpu);
2815	RTHCPHYS const HCPhysGuestMsrLoad = pVmcsInfo->HCPhysGuestMsrLoad;
2816	Assert(HCPhysGuestMsrLoad != NIL_RTHCPHYS);
2817	Assert(!(HCPhysGuestMsrLoad & 0xf)); /* Bits 3:0 MBZ. */
2818
2819	RTHCPHYS const HCPhysGuestMsrStore = pVmcsInfo->HCPhysGuestMsrStore;
2820	Assert(HCPhysGuestMsrStore != NIL_RTHCPHYS);
2821	Assert(!(HCPhysGuestMsrStore & 0xf)); /* Bits 3:0 MBZ. */
2822
2823	RTHCPHYS const HCPhysHostMsrLoad = pVmcsInfo->HCPhysHostMsrLoad;
2824	Assert(HCPhysHostMsrLoad != NIL_RTHCPHYS);
2825	Assert(!(HCPhysHostMsrLoad & 0xf)); /* Bits 3:0 MBZ. */
2826
2827	int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, HCPhysGuestMsrLoad);
2828	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, HCPhysGuestMsrStore);
2829	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, HCPhysHostMsrLoad);
2830	AssertRCReturn(rc, rc);
2831	return VINF_SUCCESS;
2832	}
2833
2834
2835	/**
2836	* Sets up MSR permissions in the MSR bitmap of a VMCS info. object.
2837	*
2838	* @param pVCpu The cross context virtual CPU structure.
2839	* @param pVmcsInfo The VMCS info. object.
2840	* @param fIsNstGstVmcs Whether this is a nested-guest VMCS.
2841	*/
2842	static void hmR0VmxSetupVmcsMsrPermissions(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, bool fIsNstGstVmcs)
2843	{
2844	Assert(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS);
2845
2846	/*
2847	* The guest can access the following MSRs (read, write) without causing
2848	* VM-exits; they are loaded/stored automatically using fields in the VMCS.
2849	*/
2850	PVM pVM = pVCpu->CTX_SUFF(pVM);
2851	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_IA32_SYSENTER_CS, VMXMSRPM_ALLOW_RD \| VMXMSRPM_ALLOW_WR);
2852	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_IA32_SYSENTER_ESP, VMXMSRPM_ALLOW_RD \| VMXMSRPM_ALLOW_WR);
2853	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_IA32_SYSENTER_EIP, VMXMSRPM_ALLOW_RD \| VMXMSRPM_ALLOW_WR);
2854	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_K8_GS_BASE, VMXMSRPM_ALLOW_RD \| VMXMSRPM_ALLOW_WR);
2855	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_K8_FS_BASE, VMXMSRPM_ALLOW_RD \| VMXMSRPM_ALLOW_WR);
2856
2857	#ifdef VBOX_STRICT
2858	/** @todo NSTVMX: Remove this later. */
2859	uint32_t fMsrpm = HMGetVmxMsrPermission(pVmcsInfo->pvMsrBitmap, MSR_IA32_SYSENTER_CS);
2860	Assert((fMsrpm & (VMXMSRPM_ALLOW_RD \| VMXMSRPM_ALLOW_WR)) == (VMXMSRPM_ALLOW_RD \| VMXMSRPM_ALLOW_WR));
2861
2862	fMsrpm = HMGetVmxMsrPermission(pVmcsInfo->pvMsrBitmap, MSR_K8_GS_BASE);
2863	Assert((fMsrpm & (VMXMSRPM_ALLOW_RD \| VMXMSRPM_ALLOW_WR)) == (VMXMSRPM_ALLOW_RD \| VMXMSRPM_ALLOW_WR));
2864	#endif
2865
2866	/*
2867	* The IA32_PRED_CMD and IA32_FLUSH_CMD MSRs are write-only and has no state
2868	* associated with then. We never need to intercept access (writes need to be
2869	* executed without causing a VM-exit, reads will #GP fault anyway).
2870	*
2871	* The IA32_SPEC_CTRL MSR is read/write and has state. We allow the guest to
2872	* read/write them. We swap the the guest/host MSR value using the
2873	* auto-load/store MSR area.
2874	*/
2875	if (pVM->cpum.ro.GuestFeatures.fIbpb)
2876	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_IA32_PRED_CMD, VMXMSRPM_ALLOW_RD \| VMXMSRPM_ALLOW_WR);
2877	if (pVM->cpum.ro.GuestFeatures.fFlushCmd)
2878	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_IA32_FLUSH_CMD, VMXMSRPM_ALLOW_RD \| VMXMSRPM_ALLOW_WR);
2879	if (pVM->cpum.ro.GuestFeatures.fIbrs)
2880	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_IA32_SPEC_CTRL, VMXMSRPM_ALLOW_RD \| VMXMSRPM_ALLOW_WR);
2881
2882	/*
2883	* IA32_EFER MSR is always intercepted, see @bugref{9180#c37}.
2884	*/
2885
2886	#if HC_ARCH_BITS == 64
2887	/*
2888	* Allow full read/write access for the following MSRs (mandatory for VT-x)
2889	* required for 64-bit guests.
2890	*/
2891	if (pVM->hm.s.fAllow64BitGuests)
2892	{
2893	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_K8_LSTAR, VMXMSRPM_ALLOW_RD \| VMXMSRPM_ALLOW_WR);
2894	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_K6_STAR, VMXMSRPM_ALLOW_RD \| VMXMSRPM_ALLOW_WR);
2895	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_K8_SF_MASK, VMXMSRPM_ALLOW_RD \| VMXMSRPM_ALLOW_WR);
2896	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_K8_KERNEL_GS_BASE, VMXMSRPM_ALLOW_RD \| VMXMSRPM_ALLOW_WR);
2897
2898	# ifdef VBOX_STRICT
2899	fMsrpm = HMGetVmxMsrPermission(pVmcsInfo->pvMsrBitmap, MSR_K8_GS_BASE);
2900	Assert((fMsrpm & (VMXMSRPM_ALLOW_RD \| VMXMSRPM_ALLOW_WR)) == (VMXMSRPM_ALLOW_RD \| VMXMSRPM_ALLOW_WR));
2901	# endif
2902	}
2903	#endif
2904	}
2905
2906
2907	/**
2908	* Sets up pin-based VM-execution controls in the VMCS.
2909	*
2910	* @returns VBox status code.
2911	* @param pVCpu The cross context virtual CPU structure.
2912	* @param pVmcsInfo The VMCS info. object.
2913	*/
2914	static int hmR0VmxSetupVmcsPinCtls(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
2915	{
2916	PVM pVM = pVCpu->CTX_SUFF(pVM);
2917	uint32_t fVal = pVM->hm.s.vmx.Msrs.PinCtls.n.allowed0; /* Bits set here must always be set. */
2918	uint32_t const fZap = pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1; /* Bits cleared here must always be cleared. */
2919
2920	fVal \|= VMX_PIN_CTLS_EXT_INT_EXIT /* External interrupts cause a VM-exit. */
2921	\| VMX_PIN_CTLS_NMI_EXIT; /* Non-maskable interrupts (NMIs) cause a VM-exit. */
2922
2923	if (pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1 & VMX_PIN_CTLS_VIRT_NMI)
2924	fVal \|= VMX_PIN_CTLS_VIRT_NMI; /* Use virtual NMIs and virtual-NMI blocking features. */
2925
2926	/* Enable the VMX-preemption timer. */
2927	if (pVM->hm.s.vmx.fUsePreemptTimer)
2928	{
2929	Assert(pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1 & VMX_PIN_CTLS_PREEMPT_TIMER);
2930	fVal \|= VMX_PIN_CTLS_PREEMPT_TIMER;
2931	}
2932
2933	#if 0
2934	/* Enable posted-interrupt processing. */
2935	if (pVM->hm.s.fPostedIntrs)
2936	{
2937	Assert(pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1 & VMX_PIN_CTLS_POSTED_INT);
2938	Assert(pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_ACK_EXT_INT);
2939	fVal \|= VMX_PIN_CTL_POSTED_INT;
2940	}
2941	#endif
2942
2943	if ((fVal & fZap) != fVal)
2944	{
2945	LogRelFunc(("Invalid pin-based VM-execution controls combo! Cpu=%#RX32 fVal=%#RX32 fZap=%#RX32\n",
2946	pVM->hm.s.vmx.Msrs.PinCtls.n.allowed0, fVal, fZap));
2947	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PIN_EXEC;
2948	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2949	}
2950
2951	/* Commit it to the VMCS and update our cache. */
2952	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, fVal);
2953	AssertRCReturn(rc, rc);
2954	pVmcsInfo->u32PinCtls = fVal;
2955
2956	return VINF_SUCCESS;
2957	}
2958
2959
2960	/**
2961	* Sets up secondary processor-based VM-execution controls in the VMCS.
2962	*
2963	* @returns VBox status code.
2964	* @param pVCpu The cross context virtual CPU structure.
2965	* @param pVmcsInfo The VMCS info. object.
2966	*/
2967	static int hmR0VmxSetupVmcsProcCtls2(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
2968	{
2969	PVM pVM = pVCpu->CTX_SUFF(pVM);
2970	uint32_t fVal = pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed0; /* Bits set here must be set in the VMCS. */
2971	uint32_t const fZap = pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2972
2973	/* WBINVD causes a VM-exit. */
2974	if (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_WBINVD_EXIT)
2975	fVal \|= VMX_PROC_CTLS2_WBINVD_EXIT;
2976
2977	/* Enable EPT (aka nested-paging). */
2978	if (pVM->hm.s.fNestedPaging)
2979	fVal \|= VMX_PROC_CTLS2_EPT;
2980
2981	/* Enable the INVPCID instruction if supported by the hardware and we expose
2982	it to the guest. Without this, guest executing INVPCID would cause a #UD. */
2983	if ( (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_INVPCID)
2984	&& pVM->cpum.ro.GuestFeatures.fInvpcid)
2985	fVal \|= VMX_PROC_CTLS2_INVPCID;
2986
2987	/* Enable VPID. */
2988	if (pVM->hm.s.vmx.fVpid)
2989	fVal \|= VMX_PROC_CTLS2_VPID;
2990
2991	/* Enable unrestricted guest execution. */
2992	if (pVM->hm.s.vmx.fUnrestrictedGuest)
2993	fVal \|= VMX_PROC_CTLS2_UNRESTRICTED_GUEST;
2994
2995	#if 0
2996	if (pVM->hm.s.fVirtApicRegs)
2997	{
2998	/* Enable APIC-register virtualization. */
2999	Assert(pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_APIC_REG_VIRT);
3000	fVal \|= VMX_PROC_CTLS2_APIC_REG_VIRT;
3001
3002	/* Enable virtual-interrupt delivery. */
3003	Assert(pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_INTR_DELIVERY);
3004	fVal \|= VMX_PROC_CTLS2_VIRT_INTR_DELIVERY;
3005	}
3006	#endif
3007
3008	/* Virtualize-APIC accesses if supported by the CPU. The virtual-APIC page is where the TPR shadow resides. */
3009	/** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
3010	* done dynamically. */
3011	if (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
3012	{
3013	fVal \|= VMX_PROC_CTLS2_VIRT_APIC_ACCESS;
3014	int rc = hmR0VmxSetupVmcsApicAccessAddr(pVCpu);
3015	AssertRCReturn(rc, rc);
3016	}
3017
3018	/* Enable the RDTSCP instruction if supported by the hardware and we expose
3019	it to the guest. Without this, guest executing RDTSCP would cause a #UD. */
3020	if ( (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_RDTSCP)
3021	&& pVM->cpum.ro.GuestFeatures.fRdTscP)
3022	fVal \|= VMX_PROC_CTLS2_RDTSCP;
3023
3024	/* Enable Pause-Loop exiting. */
3025	if ( pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_PAUSE_LOOP_EXIT
3026	&& pVM->hm.s.vmx.cPleGapTicks
3027	&& pVM->hm.s.vmx.cPleWindowTicks)
3028	{
3029	fVal \|= VMX_PROC_CTLS2_PAUSE_LOOP_EXIT;
3030
3031	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_GAP, pVM->hm.s.vmx.cPleGapTicks);
3032	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_WINDOW, pVM->hm.s.vmx.cPleWindowTicks);
3033	AssertRCReturn(rc, rc);
3034	}
3035
3036	if ((fVal & fZap) != fVal)
3037	{
3038	LogRelFunc(("Invalid secondary processor-based VM-execution controls combo! cpu=%#RX32 fVal=%#RX32 fZap=%#RX32\n",
3039	pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed0, fVal, fZap));
3040	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC2;
3041	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3042	}
3043
3044	/* Commit it to the VMCS and update our cache. */
3045	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, fVal);
3046	AssertRCReturn(rc, rc);
3047	pVmcsInfo->u32ProcCtls2 = fVal;
3048
3049	return VINF_SUCCESS;
3050	}
3051
3052
3053	/**
3054	* Sets up processor-based VM-execution controls in the VMCS.
3055	*
3056	* @returns VBox status code.
3057	* @param pVCpu The cross context virtual CPU structure.
3058	* @param pVmcsInfo The VMCS info. object.
3059	*/
3060	static int hmR0VmxSetupVmcsProcCtls(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
3061	{
3062	PVM pVM = pVCpu->CTX_SUFF(pVM);
3063
3064	uint32_t fVal = pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed0; /* Bits set here must be set in the VMCS. */
3065	uint32_t const fZap = pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3066
3067	fVal \|= VMX_PROC_CTLS_HLT_EXIT /* HLT causes a VM-exit. */
3068	\| VMX_PROC_CTLS_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
3069	\| VMX_PROC_CTLS_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
3070	\| VMX_PROC_CTLS_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
3071	\| VMX_PROC_CTLS_RDPMC_EXIT /* RDPMC causes a VM-exit. */
3072	\| VMX_PROC_CTLS_MONITOR_EXIT /* MONITOR causes a VM-exit. */
3073	\| VMX_PROC_CTLS_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
3074
3075	/* We toggle VMX_PROC_CTLS_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
3076	if ( !(pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_MOV_DR_EXIT)
3077	\|\| (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed0 & VMX_PROC_CTLS_MOV_DR_EXIT))
3078	{
3079	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_MOV_DRX_EXIT;
3080	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3081	}
3082
3083	/* Without nested paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
3084	if (!pVM->hm.s.fNestedPaging)
3085	{
3086	Assert(!pVM->hm.s.vmx.fUnrestrictedGuest);
3087	fVal \|= VMX_PROC_CTLS_INVLPG_EXIT
3088	\| VMX_PROC_CTLS_CR3_LOAD_EXIT
3089	\| VMX_PROC_CTLS_CR3_STORE_EXIT;
3090	}
3091
3092	/* Use TPR shadowing if supported by the CPU. */
3093	if ( PDMHasApic(pVM)
3094	&& pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_TPR_SHADOW)
3095	{
3096	fVal \|= VMX_PROC_CTLS_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
3097	/* CR8 writes cause a VM-exit based on TPR threshold. */
3098	Assert(!(fVal & VMX_PROC_CTLS_CR8_STORE_EXIT));
3099	Assert(!(fVal & VMX_PROC_CTLS_CR8_LOAD_EXIT));
3100	int rc = hmR0VmxSetupVmcsVirtApicAddr(pVCpu, pVmcsInfo);
3101	AssertRCReturn(rc, rc);
3102	}
3103	else
3104	{
3105	/* Some 32-bit CPUs do not support CR8 load/store exiting as MOV CR8 is
3106	invalid on 32-bit Intel CPUs. Set this control only for 64-bit guests. */
3107	if (pVM->hm.s.fAllow64BitGuests)
3108	{
3109	fVal \|= VMX_PROC_CTLS_CR8_STORE_EXIT /* CR8 reads cause a VM-exit. */
3110	\| VMX_PROC_CTLS_CR8_LOAD_EXIT; /* CR8 writes cause a VM-exit. */
3111	}
3112	}
3113
3114	/* Use MSR-bitmaps if supported by the CPU. */
3115	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
3116	{
3117	fVal \|= VMX_PROC_CTLS_USE_MSR_BITMAPS;
3118	int rc = hmR0VmxSetupVmcsMsrBitmapAddr(pVCpu, pVmcsInfo);
3119	AssertRCReturn(rc, rc);
3120	}
3121
3122	/* Use the secondary processor-based VM-execution controls if supported by the CPU. */
3123	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
3124	fVal \|= VMX_PROC_CTLS_USE_SECONDARY_CTLS;
3125
3126	if ((fVal & fZap) != fVal)
3127	{
3128	LogRelFunc(("Invalid processor-based VM-execution controls combo! cpu=%#RX32 fVal=%#RX32 fZap=%#RX32\n",
3129	pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed0, fVal, fZap));
3130	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC;
3131	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3132	}
3133
3134	/* Commit it to the VMCS and update our cache. */
3135	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, fVal);
3136	AssertRCReturn(rc, rc);
3137	pVmcsInfo->u32ProcCtls = fVal;
3138
3139	/* Set up MSR permissions that don't change through the lifetime of the VM. */
3140	hmR0VmxSetupVmcsMsrPermissions(pVCpu, pVmcsInfo, false /* fIsNstGstVmcs */);
3141
3142	/* Set up secondary processor-based VM-execution controls if the CPU supports it. */
3143	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
3144	return hmR0VmxSetupVmcsProcCtls2(pVCpu, pVmcsInfo);
3145
3146	/* Sanity check, should not really happen. */
3147	if (RT_LIKELY(!pVM->hm.s.vmx.fUnrestrictedGuest))
3148	{ /* likely */ }
3149	else
3150	{
3151	pVCpu->hm.s.u32HMError = VMX_UFC_INVALID_UX_COMBO;
3152	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3153	}
3154
3155	/* Old CPUs without secondary processor-based VM-execution controls would end up here. */
3156	return VINF_SUCCESS;
3157	}
3158
3159
3160	/**
3161	* Sets up miscellaneous (everything other than Pin, Processor and secondary
3162	* Processor-based VM-execution) control fields in the VMCS.
3163	*
3164	* @returns VBox status code.
3165	* @param pVCpu The cross context virtual CPU structure.
3166	* @param pVmcsInfo The VMCS info. object.
3167	*/
3168	static int hmR0VmxSetupVmcsMiscCtls(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
3169	{
3170	/* Set the auto-load/store MSR area addresses in the VMCS. */
3171	int rc = hmR0VmxSetupVmcsAutoLoadStoreMsrAddrs(pVCpu, pVmcsInfo);
3172	if (RT_SUCCESS(rc))
3173	{
3174	/* Set the VMCS link pointer in the VMCS. */
3175	rc = hmR0VmxSetupVmcsLinkPtr(pVCpu, pVmcsInfo);
3176	if (RT_SUCCESS(rc))
3177	{
3178	/* Set the CR0/CR4 guest/host mask. */
3179	uint64_t const u64Cr0Mask = hmR0VmxGetFixedCr0Mask(pVCpu);
3180	uint64_t const u64Cr4Mask = hmR0VmxGetFixedCr4Mask(pVCpu);
3181	rc = VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, u64Cr0Mask);
3182	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, u64Cr4Mask);
3183	if (RT_SUCCESS(rc))
3184	{
3185	pVmcsInfo->u64Cr0Mask = u64Cr0Mask;
3186	pVmcsInfo->u64Cr4Mask = u64Cr4Mask;
3187	return VINF_SUCCESS;
3188	}
3189	LogRelFunc(("Failed to initialize VMCS CR0/CR4 guest/host mask. rc=%Rrc\n", rc));
3190	}
3191	else
3192	LogRelFunc(("Failed to initialize VMCS link pointer. rc=%Rrc\n", rc));
3193	}
3194	else
3195	LogRelFunc(("Failed to initialize VMCS auto-load/store MSR addresses. rc=%Rrc\n", rc));
3196	return rc;
3197	}
3198
3199
3200	/**
3201	* Sets up the initial exception bitmap in the VMCS based on static conditions.
3202	*
3203	* We shall setup those exception intercepts that don't change during the
3204	* lifetime of the VM here. The rest are done dynamically while loading the
3205	* guest state.
3206	*
3207	* @returns VBox status code.
3208	* @param pVCpu The cross context virtual CPU structure.
3209	* @param pVmcsInfo The VMCS info. object.
3210	*/
3211	static int hmR0VmxSetupVmcsXcptBitmap(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
3212	{
3213	/*
3214	* The following exceptions are always intercepted:
3215	*
3216	* #AC - To prevent the guest from hanging the CPU.
3217	* #DB - To maintain the DR6 state even when intercepting DRx reads/writes and
3218	* recursive #DBs can cause a CPU hang.
3219	* #PF - To sync our shadow page tables when nested-paging is not used.
3220	*/
3221	bool const fNestedPaging = pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging;
3222	uint32_t const uXcptBitmap = RT_BIT(X86_XCPT_AC)
3223	\| RT_BIT(X86_XCPT_DB)
3224	\| (fNestedPaging ? 0 : RT_BIT(X86_XCPT_PF));
3225
3226	/* Commit it to the VMCS. */
3227	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
3228	AssertRCReturn(rc, rc);
3229
3230	/* Update our cache of the exception bitmap. */
3231	pVmcsInfo->u32XcptBitmap = uXcptBitmap;
3232	return VINF_SUCCESS;
3233	}
3234
3235
3236	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
3237	/**
3238	* Sets up the VMCS for executing a nested-guest using hardware-assisted VMX.
3239	*
3240	* @returns VBox status code.
3241	* @param pVCpu The cross context virtual CPU structure.
3242	* @param pVmcsInfo The VMCS info. object.
3243	*/
3244	static int hmR0VmxSetupVmcsCtlsNested(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
3245	{
3246	PVM pVM = pVCpu->CTX_SUFF(pVM);
3247	int rc = hmR0VmxSetupVmcsLinkPtr(pVmcsInfo);
3248	if (RT_SUCCESS(rc))
3249	{
3250	rc = hmR0VmxSetupVmcsAutoLoadStoreMsrAddrs(pVmcsInfo);
3251	if (RT_SUCCESS(rc))
3252	{
3253	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
3254	rc = hmR0VmxSetupVmcsMsrBitmapAddr(pVmcsInfo);
3255	if (RT_SUCCESS(rc))
3256	{
3257	if (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
3258	rc = hmR0VmxSetupVmcsApicAccessAddr(pVCpu);
3259	if (RT_SUCCESS(rc))
3260	return VINF_SUCCESS;
3261
3262	LogRelFunc(("Failed to set up the APIC-access address in the nested-guest VMCS. rc=%Rrc\n", rc));
3263	}
3264	else
3265	LogRelFunc(("Failed to set up the MSR-bitmap address in the nested-guest VMCS. rc=%Rrc\n", rc));
3266	}
3267	else
3268	LogRelFunc(("Failed to set up the VMCS link pointer in the nested-guest VMCS. rc=%Rrc\n", rc));
3269	}
3270	else
3271	LogRelFunc(("Failed to set up the auto-load/store MSR addresses in the nested-guest VMCS. rc=%Rrc\n", rc));
3272
3273	return rc;
3274	}
3275	#endif
3276
3277
3278	/**
3279	* Sets up the VMCS for executing a guest (or nested-guest) using hardware-assisted
3280	* VMX.
3281	*
3282	* @returns VBox status code.
3283	* @param pVCpu The cross context virtual CPU structure.
3284	* @param pVmcsInfo The VMCS info. object.
3285	* @param fIsNstGstVmcs Whether this is a nested-guest VMCS.
3286	*/
3287	static int hmR0VmxSetupVmcs(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, bool fIsNstGstVmcs)
3288	{
3289	Assert(pVmcsInfo);
3290	Assert(pVmcsInfo->pvVmcs);
3291	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
3292
3293	/* Set the CPU specified revision identifier at the beginning of the VMCS structure. */
3294	PVM pVM = pVCpu->CTX_SUFF(pVM);
3295	(uint32_t )pVmcsInfo->pvVmcs = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_ID);
3296	const char * const pszVmcs = fIsNstGstVmcs ? "nested-guest VMCS" : "guest VMCS";
3297
3298	LogFlowFunc(("\n"));
3299
3300	/*
3301	* Initialize the VMCS using VMCLEAR before loading the VMCS.
3302	* See Intel spec. 31.6 "Preparation And Launching A Virtual Machine".
3303	*/
3304	int rc = hmR0VmxClearVmcs(pVmcsInfo);
3305	if (RT_SUCCESS(rc))
3306	{
3307	rc = hmR0VmxLoadVmcs(pVmcsInfo);
3308	if (RT_SUCCESS(rc))
3309	{
3310	if (!fIsNstGstVmcs)
3311	{
3312	rc = hmR0VmxSetupVmcsPinCtls(pVCpu, pVmcsInfo);
3313	if (RT_SUCCESS(rc))
3314	{
3315	rc = hmR0VmxSetupVmcsProcCtls(pVCpu, pVmcsInfo);
3316	if (RT_SUCCESS(rc))
3317	{
3318	rc = hmR0VmxSetupVmcsMiscCtls(pVCpu, pVmcsInfo);
3319	if (RT_SUCCESS(rc))
3320	{
3321	rc = hmR0VmxSetupVmcsXcptBitmap(pVCpu, pVmcsInfo);
3322	if (RT_SUCCESS(rc))
3323	{ /* likely */ }
3324	else
3325	LogRelFunc(("Failed to initialize exception bitmap. rc=%Rrc\n", rc));
3326	}
3327	else
3328	LogRelFunc(("Failed to setup miscellaneous controls. rc=%Rrc\n", rc));
3329	}
3330	else
3331	LogRelFunc(("Failed to setup processor-based VM-execution controls. rc=%Rrc\n", rc));
3332	}
3333	else
3334	LogRelFunc(("Failed to setup pin-based controls. rc=%Rrc\n", rc));
3335	}
3336	else
3337	{
3338	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
3339	rc = hmR0VmxSetupVmcsCtlsNested(pVCpu, pVmcsInfo);
3340	if (RT_SUCCESS(rc))
3341	{ /* likely */ }
3342	else
3343	LogRelFunc(("Failed to initialize nested-guest VMCS. rc=%Rrc\n", rc));
3344	#else
3345	AssertFailed();
3346	#endif
3347	}
3348	}
3349	else
3350	LogRelFunc(("Failed to load the %s. rc=%Rrc\n", rc, pszVmcs));
3351	}
3352	else
3353	LogRelFunc(("Failed to clear the %s. rc=%Rrc\n", rc, pszVmcs));
3354
3355	/* Sync any CPU internal VMCS data back into our VMCS in memory. */
3356	if (RT_SUCCESS(rc))
3357	{
3358	rc = hmR0VmxClearVmcs(pVmcsInfo);
3359	if (RT_SUCCESS(rc))
3360	{ /* likely */ }
3361	else
3362	LogRelFunc(("Failed to clear the %s post setup. rc=%Rrc\n", rc, pszVmcs));
3363	}
3364
3365	/*
3366	* Update the last-error record both for failures and success, so we
3367	* can propagate the status code back to ring-3 for diagnostics.
3368	*/
3369	hmR0VmxUpdateErrorRecord(pVCpu, rc);
3370	NOREF(pszVmcs);
3371	return rc;
3372	}
3373
3374
3375	/**
3376	* Does global VT-x initialization (called during module initialization).
3377	*
3378	* @returns VBox status code.
3379	*/
3380	VMMR0DECL(int) VMXR0GlobalInit(void)
3381	{
3382	#ifdef HMVMX_USE_FUNCTION_TABLE
3383	AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
3384	# ifdef VBOX_STRICT
3385	for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
3386	Assert(g_apfnVMExitHandlers[i]);
3387	# endif
3388	#endif
3389	return VINF_SUCCESS;
3390	}
3391
3392
3393	/**
3394	* Does global VT-x termination (called during module termination).
3395	*/
3396	VMMR0DECL(void) VMXR0GlobalTerm()
3397	{
3398	/* Nothing to do currently. */
3399	}
3400
3401
3402	/**
3403	* Sets up and activates VT-x on the current CPU.
3404	*
3405	* @returns VBox status code.
3406	* @param pHostCpu The HM physical-CPU structure.
3407	* @param pVM The cross context VM structure. Can be
3408	* NULL after a host resume operation.
3409	* @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
3410	* fEnabledByHost is @c true).
3411	* @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
3412	* @a fEnabledByHost is @c true).
3413	* @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
3414	* enable VT-x on the host.
3415	* @param pHwvirtMsrs Pointer to the hardware-virtualization MSRs.
3416	*/
3417	VMMR0DECL(int) VMXR0EnableCpu(PHMPHYSCPU pHostCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost,
3418	PCSUPHWVIRTMSRS pHwvirtMsrs)
3419	{
3420	Assert(pHostCpu);
3421	Assert(pHwvirtMsrs);
3422	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
3423
3424	/* Enable VT-x if it's not already enabled by the host. */
3425	if (!fEnabledByHost)
3426	{
3427	int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
3428	if (RT_FAILURE(rc))
3429	return rc;
3430	}
3431
3432	/*
3433	* Flush all EPT tagged-TLB entries (in case VirtualBox or any other hypervisor have been
3434	* using EPTPs) so we don't retain any stale guest-physical mappings which won't get
3435	* invalidated when flushing by VPID.
3436	*/
3437	if (pHwvirtMsrs->u.vmx.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
3438	{
3439	hmR0VmxFlushEpt(NULL /* pVCpu /, NULL / pVmcsInfo */, VMXTLBFLUSHEPT_ALL_CONTEXTS);
3440	pHostCpu->fFlushAsidBeforeUse = false;
3441	}
3442	else
3443	pHostCpu->fFlushAsidBeforeUse = true;
3444
3445	/* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
3446	++pHostCpu->cTlbFlushes;
3447
3448	return VINF_SUCCESS;
3449	}
3450
3451
3452	/**
3453	* Deactivates VT-x on the current CPU.
3454	*
3455	* @returns VBox status code.
3456	* @param pvCpuPage Pointer to the VMXON region.
3457	* @param HCPhysCpuPage Physical address of the VMXON region.
3458	*
3459	* @remarks This function should never be called when SUPR0EnableVTx() or
3460	* similar was used to enable VT-x on the host.
3461	*/
3462	VMMR0DECL(int) VMXR0DisableCpu(void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
3463	{
3464	RT_NOREF2(pvCpuPage, HCPhysCpuPage);
3465
3466	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
3467	return hmR0VmxLeaveRootMode();
3468	}
3469
3470
3471	/**
3472	* Does per-VM VT-x initialization.
3473	*
3474	* @returns VBox status code.
3475	* @param pVM The cross context VM structure.
3476	*/
3477	VMMR0DECL(int) VMXR0InitVM(PVM pVM)
3478	{
3479	LogFlowFunc(("pVM=%p\n", pVM));
3480
3481	int rc = hmR0VmxStructsAlloc(pVM);
3482	if (RT_FAILURE(rc))
3483	{
3484	LogRelFunc(("Failed to allocated VMX structures. rc=%Rrc\n", rc));
3485	return rc;
3486	}
3487
3488	return VINF_SUCCESS;
3489	}
3490
3491
3492	/**
3493	* Does per-VM VT-x termination.
3494	*
3495	* @returns VBox status code.
3496	* @param pVM The cross context VM structure.
3497	*/
3498	VMMR0DECL(int) VMXR0TermVM(PVM pVM)
3499	{
3500	LogFlowFunc(("pVM=%p\n", pVM));
3501
3502	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
3503	if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
3504	{
3505	Assert(pVM->hm.s.vmx.pvScratch);
3506	ASMMemZero32(pVM->hm.s.vmx.pvScratch, X86_PAGE_4K_SIZE);
3507	}
3508	#endif
3509	hmR0VmxStructsFree(pVM);
3510	return VINF_SUCCESS;
3511	}
3512
3513
3514	/**
3515	* Sets up the VM for execution using hardware-assisted VMX.
3516	* This function is only called once per-VM during initialization.
3517	*
3518	* @returns VBox status code.
3519	* @param pVM The cross context VM structure.
3520	*/
3521	VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
3522	{
3523	AssertPtrReturn(pVM, VERR_INVALID_PARAMETER);
3524	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
3525
3526	LogFlowFunc(("pVM=%p\n", pVM));
3527
3528	/*
3529	* At least verify if VMX is enabled, since we can't check if we're in
3530	* VMX root mode or not without causing a #GP.
3531	*/
3532	RTCCUINTREG const uHostCR4 = ASMGetCR4();
3533	if (RT_LIKELY(uHostCR4 & X86_CR4_VMXE))
3534	{ /* likely */ }
3535	else
3536	return VERR_VMX_NOT_IN_VMX_ROOT_MODE;
3537
3538	/*
3539	* Without unrestricted guest execution, pRealModeTSS and pNonPagingModeEPTPageTable must
3540	* always be allocated. We no longer support the highly unlikely case of unrestricted guest
3541	* without pRealModeTSS, see hmR3InitFinalizeR0Intel().
3542	*/
3543	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
3544	&& ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
3545	\|\| !pVM->hm.s.vmx.pRealModeTSS))
3546	{
3547	LogRelFunc(("Invalid real-on-v86 state.\n"));
3548	return VERR_INTERNAL_ERROR;
3549	}
3550
3551	/* Initialize these always, see hmR3InitFinalizeR0().*/
3552	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NONE;
3553	pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_NONE;
3554
3555	/* Setup the tagged-TLB flush handlers. */
3556	int rc = hmR0VmxSetupTaggedTlb(pVM);
3557	if (RT_FAILURE(rc))
3558	{
3559	LogRelFunc(("hmR0VmxSetupTaggedTlb failed! rc=%Rrc\n", rc));
3560	return rc;
3561	}
3562
3563	/* Check if we can use the VMCS controls for swapping the EFER MSR. */
3564	Assert(!pVM->hm.s.vmx.fSupportsVmcsEfer);
3565	#if HC_ARCH_BITS == 64
3566	if ( (pVM->hm.s.vmx.Msrs.EntryCtls.n.allowed1 & VMX_ENTRY_CTLS_LOAD_EFER_MSR)
3567	&& (pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_LOAD_EFER_MSR)
3568	&& (pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_SAVE_EFER_MSR))
3569	pVM->hm.s.vmx.fSupportsVmcsEfer = true;
3570	#endif
3571
3572	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
3573	{
3574	PVMCPU pVCpu = &pVM->aCpus[idCpu];
3575	Log4Func(("pVCpu=%p idCpu=%RU32\n", pVCpu, pVCpu->idCpu));
3576
3577	rc = hmR0VmxSetupVmcs(pVCpu, &pVCpu->hm.s.vmx.VmcsInfo, false /* fIsNstGstVmcs */);
3578	if (RT_SUCCESS(rc))
3579	{
3580	#if HC_ARCH_BITS == 32
3581	hmR0VmxInitVmcsReadCache(pVCpu);
3582	#endif
3583	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
3584	if (pVM->cpum.ro.GuestFeatures.fVmx)
3585	{
3586	rc = hmR0VmxSetupVmcs(pVCpu, &pVCpu->hm.s.vmx.VmcsInfoNstGst, true /* fIsNstGstVmcs */);
3587	if (RT_SUCCESS(rc))
3588	{ /* likely */ }
3589	else
3590	{
3591	LogRelFunc(("Nested-guest VMCS setup failed. rc=%Rrc\n", rc));
3592	return rc;
3593	}
3594	}
3595	#endif
3596	}
3597	else
3598	{
3599	LogRelFunc(("VMCS setup failed. rc=%Rrc\n", rc));
3600	return rc;
3601	}
3602	}
3603
3604	return VINF_SUCCESS;
3605	}
3606
3607
3608	#if HC_ARCH_BITS == 32
3609	# ifdef VBOX_ENABLE_64_BITS_GUESTS
3610	/**
3611	* Check if guest state allows safe use of 32-bit switcher again.
3612	*
3613	* Segment bases and protected mode structures must be 32-bit addressable
3614	* because the 32-bit switcher will ignore high dword when writing these VMCS
3615	* fields. See @bugref{8432} for details.
3616	*
3617	* @returns true if safe, false if must continue to use the 64-bit switcher.
3618	* @param pCtx Pointer to the guest-CPU context.
3619	*
3620	* @remarks No-long-jump zone!!!
3621	*/
3622	static bool hmR0VmxIs32BitSwitcherSafe(PCCPUMCTX pCtx)
3623	{
3624	if (pCtx->gdtr.pGdt & UINT64_C(0xffffffff00000000)) return false;
3625	if (pCtx->idtr.pIdt & UINT64_C(0xffffffff00000000)) return false;
3626	if (pCtx->ldtr.u64Base & UINT64_C(0xffffffff00000000)) return false;
3627	if (pCtx->tr.u64Base & UINT64_C(0xffffffff00000000)) return false;
3628	if (pCtx->es.u64Base & UINT64_C(0xffffffff00000000)) return false;
3629	if (pCtx->cs.u64Base & UINT64_C(0xffffffff00000000)) return false;
3630	if (pCtx->ss.u64Base & UINT64_C(0xffffffff00000000)) return false;
3631	if (pCtx->ds.u64Base & UINT64_C(0xffffffff00000000)) return false;
3632	if (pCtx->fs.u64Base & UINT64_C(0xffffffff00000000)) return false;
3633	if (pCtx->gs.u64Base & UINT64_C(0xffffffff00000000)) return false;
3634
3635	/* All good, bases are 32-bit. */
3636	return true;
3637	}
3638	# endif /* VBOX_ENABLE_64_BITS_GUESTS */
3639
3640	# ifdef VBOX_STRICT
3641	static bool hmR0VmxIsValidWriteField(uint32_t idxField)
3642	{
3643	switch (idxField)
3644	{
3645	case VMX_VMCS_GUEST_RIP:
3646	case VMX_VMCS_GUEST_RSP:
3647	case VMX_VMCS_GUEST_SYSENTER_EIP:
3648	case VMX_VMCS_GUEST_SYSENTER_ESP:
3649	case VMX_VMCS_GUEST_GDTR_BASE:
3650	case VMX_VMCS_GUEST_IDTR_BASE:
3651	case VMX_VMCS_GUEST_CS_BASE:
3652	case VMX_VMCS_GUEST_DS_BASE:
3653	case VMX_VMCS_GUEST_ES_BASE:
3654	case VMX_VMCS_GUEST_FS_BASE:
3655	case VMX_VMCS_GUEST_GS_BASE:
3656	case VMX_VMCS_GUEST_SS_BASE:
3657	case VMX_VMCS_GUEST_LDTR_BASE:
3658	case VMX_VMCS_GUEST_TR_BASE:
3659	case VMX_VMCS_GUEST_CR3:
3660	return true;
3661	}
3662	return false;
3663	}
3664
3665	static bool hmR0VmxIsValidReadField(uint32_t idxField)
3666	{
3667	switch (idxField)
3668	{
3669	/* Read-only fields. */
3670	case VMX_VMCS_RO_EXIT_QUALIFICATION:
3671	return true;
3672	}
3673	/* Remaining readable fields should also be writable. */
3674	return hmR0VmxIsValidWriteField(idxField);
3675	}
3676	# endif /* VBOX_STRICT */
3677
3678
3679	/**
3680	* Executes the specified handler in 64-bit mode.
3681	*
3682	* @returns VBox status code (no informational status codes).
3683	* @param pVCpu The cross context virtual CPU structure.
3684	* @param enmOp The operation to perform.
3685	* @param cParams Number of parameters.
3686	* @param paParam Array of 32-bit parameters.
3687	*/
3688	VMMR0DECL(int) VMXR0Execute64BitsHandler(PVMCPU pVCpu, HM64ON32OP enmOp, uint32_t cParams, uint32_t *paParam)
3689	{
3690	PVM pVM = pVCpu->CTX_SUFF(pVM);
3691	AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
3692	Assert(enmOp > HM64ON32OP_INVALID && enmOp < HM64ON32OP_END);
3693	Assert(pVCpu->hm.s.vmx.VmcsCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VmcsCache.Write.aField));
3694	Assert(pVCpu->hm.s.vmx.VmcsCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VmcsCache.Read.aField));
3695
3696	#ifdef VBOX_STRICT
3697	for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VmcsCache.Write.cValidEntries; i++)
3698	Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VmcsCache.Write.aField[i]));
3699
3700	for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VmcsCache.Read.cValidEntries; i++)
3701	Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VmcsCache.Read.aField[i]));
3702	#endif
3703
3704	/* Disable interrupts. */
3705	RTCCUINTREG fOldEFlags = ASMIntDisableFlags();
3706
3707	#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
3708	RTCPUID idHostCpu = RTMpCpuId();
3709	CPUMR0SetLApic(pVCpu, idHostCpu);
3710	#endif
3711
3712	/** @todo replace with hmR0VmxEnterRootMode() and hmR0VmxLeaveRootMode(). */
3713
3714	PCHMPHYSCPU pHostCpu = hmR0GetCurrentCpu();
3715	RTHCPHYS const HCPhysCpuPage = pHostCpu->HCPhysMemObj;
3716
3717	/* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
3718	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
3719	hmR0VmxClearVmcs(pVmcsInfo);
3720
3721	/* Leave VMX root mode and disable VMX. */
3722	VMXDisable();
3723	SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
3724
3725	CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
3726	CPUMSetHyperEIP(pVCpu, enmOp);
3727	for (int i = (int)cParams - 1; i >= 0; i--)
3728	CPUMPushHyper(pVCpu, paParam[i]);
3729
3730	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
3731
3732	/* Call the switcher. */
3733	int rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_UOFFSETOF_DYN(VM, aCpus[pVCpu->idCpu].cpum) - RT_UOFFSETOF(VM, cpum));
3734	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
3735
3736	/* Re-enable VMX to make sure the VMX instructions don't cause #UD faults. */
3737	SUPR0ChangeCR4(X86_CR4_VMXE, RTCCUINTREG_MAX);
3738
3739	/* Re-enter VMX root mode. */
3740	int rc2 = VMXEnable(HCPhysCpuPage);
3741	if (RT_FAILURE(rc2))
3742	{
3743	SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
3744	ASMSetFlags(fOldEFlags);
3745	pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
3746	return rc2;
3747	}
3748
3749	/* Restore the VMCS as the current VMCS. */
3750	rc2 = hmR0VmxLoadVmcs(pVmcsInfo);
3751	AssertRC(rc2);
3752	Assert(!(ASMGetFlags() & X86_EFL_IF));
3753	ASMSetFlags(fOldEFlags);
3754	return rc;
3755	}
3756
3757
3758	/**
3759	* Prepares for and executes VMLAUNCH (64-bit guests) for 32-bit hosts
3760	* supporting 64-bit guests.
3761	*
3762	* @returns VBox status code.
3763	* @param fResume Whether to VMLAUNCH or VMRESUME.
3764	* @param pCtx Pointer to the guest-CPU context.
3765	* @param pCache Pointer to the VMCS batch cache.
3766	* @param pVM The cross context VM structure.
3767	* @param pVCpu The cross context virtual CPU structure.
3768	*/
3769	DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMXVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
3770	{
3771	NOREF(fResume);
3772
3773	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
3774	PCHMPHYSCPU pHostCpu = hmR0GetCurrentCpu();
3775	RTHCPHYS const HCPhysCpuPage = pHostCpu->HCPhysMemObj;
3776
3777	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
3778	pCache->uPos = 1;
3779	pCache->interPD = PGMGetInterPaeCR3(pVM);
3780	pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
3781	#endif
3782
3783	#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
3784	pCache->TestIn.HCPhysCpuPage = 0;
3785	pCache->TestIn.HCPhysVmcs = 0;
3786	pCache->TestIn.pCache = 0;
3787	pCache->TestOut.HCPhysVmcs = 0;
3788	pCache->TestOut.pCache = 0;
3789	pCache->TestOut.pCtx = 0;
3790	pCache->TestOut.eflags = 0;
3791	#else
3792	NOREF(pCache);
3793	#endif
3794
3795	uint32_t aParam[10];
3796	aParam[0] = RT_LO_U32(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
3797	aParam[1] = RT_HI_U32(HCPhysCpuPage); /* Param 1: VMXON physical address - Hi. */
3798	aParam[2] = RT_LO_U32(pVmcsInfo->HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
3799	aParam[3] = RT_HI_U32(pVmcsInfo->HCPhysVmcs); /* Param 2: VMCS physical address - Hi. */
3800	aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VmcsCache);
3801	aParam[5] = 0;
3802	aParam[6] = VM_RC_ADDR(pVM, pVM);
3803	aParam[7] = 0;
3804	aParam[8] = VM_RC_ADDR(pVM, pVCpu);
3805	aParam[9] = 0;
3806
3807	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
3808	pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
3809	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
3810	#endif
3811	int rc = VMXR0Execute64BitsHandler(pVCpu, HM64ON32OP_VMXRCStartVM64, RT_ELEMENTS(aParam), &aParam[0]);
3812
3813	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
3814	Assert((uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
3815	Assert(pCtx->dr[4] == 10);
3816	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
3817	#endif
3818
3819	#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
3820	AssertMsg(pCache->TestIn.HCPhysCpuPage == HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
3821	AssertMsg(pCache->TestIn.HCPhysVmcs == pVmcsInfo->HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
3822	pVmcsInfo->HCPhysVmcs));
3823	AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
3824	pCache->TestOut.HCPhysVmcs));
3825	AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
3826	pCache->TestOut.pCache));
3827	AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VmcsCache),
3828	("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VmcsCache)));
3829	AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
3830	pCache->TestOut.pCtx));
3831	Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
3832	#endif
3833	NOREF(pCtx);
3834	return rc;
3835	}
3836	#endif
3837
3838
3839	/**
3840	* Saves the host control registers (CR0, CR3, CR4) into the host-state area in
3841	* the VMCS.
3842	*
3843	* @returns VBox status code.
3844	*/
3845	static int hmR0VmxExportHostControlRegs(void)
3846	{
3847	RTCCUINTREG uReg = ASMGetCR0();
3848	int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
3849	AssertRCReturn(rc, rc);
3850
3851	uReg = ASMGetCR3();
3852	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
3853	AssertRCReturn(rc, rc);
3854
3855	uReg = ASMGetCR4();
3856	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
3857	AssertRCReturn(rc, rc);
3858	return rc;
3859	}
3860
3861
3862	/**
3863	* Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
3864	* the host-state area in the VMCS.
3865	*
3866	* @returns VBox status code.
3867	* @param pVCpu The cross context virtual CPU structure.
3868	*/
3869	static int hmR0VmxExportHostSegmentRegs(PVMCPU pVCpu)
3870	{
3871	#if HC_ARCH_BITS == 64
3872	/**
3873	* Macro for adjusting host segment selectors to satisfy VT-x's VM-entry
3874	* requirements. See hmR0VmxExportHostSegmentRegs().
3875	*/
3876	# define VMXLOCAL_ADJUST_HOST_SEG(seg, selValue) \
3877	if ((selValue) & (X86_SEL_RPL \| X86_SEL_LDT)) \
3878	{ \
3879	bool fValidSelector = true; \
3880	if ((selValue) & X86_SEL_LDT) \
3881	{ \
3882	uint32_t uAttr = ASMGetSegAttr((selValue)); \
3883	fValidSelector = RT_BOOL(uAttr != UINT32_MAX && (uAttr & X86_DESC_P)); \
3884	} \
3885	if (fValidSelector) \
3886	{ \
3887	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_##seg; \
3888	pVCpu->hm.s.vmx.RestoreHost.uHostSel##seg = (selValue); \
3889	} \
3890	(selValue) = 0; \
3891	}
3892
3893	/*
3894	* If we've executed guest code using hardware-assisted VMX, the host-state bits
3895	* will be messed up. We should -not- save the messed up state without restoring
3896	* the original host-state, see @bugref{7240}.
3897	*
3898	* This apparently can happen (most likely the FPU changes), deal with it rather than
3899	* asserting. Was observed booting Solaris 10u10 32-bit guest.
3900	*/
3901	if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
3902	&& (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
3903	{
3904	Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags,
3905	pVCpu->idCpu));
3906	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
3907	}
3908	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
3909	#else
3910	RT_NOREF(pVCpu);
3911	#endif
3912
3913	/*
3914	* Host DS, ES, FS and GS segment registers.
3915	*/
3916	#if HC_ARCH_BITS == 64
3917	RTSEL uSelDS = ASMGetDS();
3918	RTSEL uSelES = ASMGetES();
3919	RTSEL uSelFS = ASMGetFS();
3920	RTSEL uSelGS = ASMGetGS();
3921	#else
3922	RTSEL uSelDS = 0;
3923	RTSEL uSelES = 0;
3924	RTSEL uSelFS = 0;
3925	RTSEL uSelGS = 0;
3926	#endif
3927
3928	/*
3929	* Host CS and SS segment registers.
3930	*/
3931	RTSEL uSelCS = ASMGetCS();
3932	RTSEL uSelSS = ASMGetSS();
3933
3934	/*
3935	* Host TR segment register.
3936	*/
3937	RTSEL uSelTR = ASMGetTR();
3938
3939	#if HC_ARCH_BITS == 64
3940	/*
3941	* Determine if the host segment registers are suitable for VT-x. Otherwise use zero to
3942	* gain VM-entry and restore them before we get preempted.
3943	*
3944	* See Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers".
3945	*/
3946	VMXLOCAL_ADJUST_HOST_SEG(DS, uSelDS);
3947	VMXLOCAL_ADJUST_HOST_SEG(ES, uSelES);
3948	VMXLOCAL_ADJUST_HOST_SEG(FS, uSelFS);
3949	VMXLOCAL_ADJUST_HOST_SEG(GS, uSelGS);
3950	# undef VMXLOCAL_ADJUST_HOST_SEG
3951	#endif
3952
3953	/* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
3954	Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
3955	Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
3956	Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
3957	Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
3958	Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
3959	Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
3960	Assert(!(uSelTR & X86_SEL_RPL)); Assert(!(uSelTR & X86_SEL_LDT));
3961	Assert(uSelCS);
3962	Assert(uSelTR);
3963
3964	/* Write these host selector fields into the host-state area in the VMCS. */
3965	int rc = VMXWriteVmcs32(VMX_VMCS16_HOST_CS_SEL, uSelCS);
3966	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_SS_SEL, uSelSS);
3967	#if HC_ARCH_BITS == 64
3968	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_DS_SEL, uSelDS);
3969	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_ES_SEL, uSelES);
3970	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_FS_SEL, uSelFS);
3971	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_GS_SEL, uSelGS);
3972	#else
3973	NOREF(uSelDS);
3974	NOREF(uSelES);
3975	NOREF(uSelFS);
3976	NOREF(uSelGS);
3977	#endif
3978	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_TR_SEL, uSelTR);
3979	AssertRCReturn(rc, rc);
3980
3981	/*
3982	* Host GDTR and IDTR.
3983	*/
3984	RTGDTR Gdtr;
3985	RTIDTR Idtr;
3986	RT_ZERO(Gdtr);
3987	RT_ZERO(Idtr);
3988	ASMGetGDTR(&Gdtr);
3989	ASMGetIDTR(&Idtr);
3990	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt);
3991	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt);
3992	AssertRCReturn(rc, rc);
3993
3994	#if HC_ARCH_BITS == 64
3995	/*
3996	* Determine if we need to manually need to restore the GDTR and IDTR limits as VT-x zaps
3997	* them to the maximum limit (0xffff) on every VM-exit.
3998	*/
3999	if (Gdtr.cbGdt != 0xffff)
4000	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_GDTR;
4001
4002	/*
4003	* IDT limit is effectively capped at 0xfff. (See Intel spec. 6.14.1 "64-Bit Mode IDT" and
4004	* Intel spec. 6.2 "Exception and Interrupt Vectors".) Therefore if the host has the limit
4005	* as 0xfff, VT-x bloating the limit to 0xffff shouldn't cause any different CPU behavior.
4006	* However, several hosts either insists on 0xfff being the limit (Windows Patch Guard) or
4007	* uses the limit for other purposes (darwin puts the CPU ID in there but botches sidt
4008	* alignment in at least one consumer). So, we're only allowing the IDTR.LIMIT to be left
4009	* at 0xffff on hosts where we are sure it won't cause trouble.
4010	*/
4011	# if defined(RT_OS_LINUX) \|\| defined(RT_OS_SOLARIS)
4012	if (Idtr.cbIdt < 0x0fff)
4013	# else
4014	if (Idtr.cbIdt != 0xffff)
4015	# endif
4016	{
4017	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_IDTR;
4018	AssertCompile(sizeof(Idtr) == sizeof(X86XDTR64));
4019	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostIdtr, &Idtr, sizeof(X86XDTR64));
4020	}
4021	#endif
4022
4023	/*
4024	* Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI
4025	* and RPL bits is effectively what the CPU does for "scaling by 8". TI is always 0 and
4026	* RPL should be too in most cases.
4027	*/
4028	AssertMsgReturn((uSelTR \| X86_SEL_RPL_LDT) <= Gdtr.cbGdt,
4029	("TR selector exceeds limit. TR=%RTsel cbGdt=%#x\n", uSelTR, Gdtr.cbGdt), VERR_VMX_INVALID_HOST_STATE);
4030
4031	PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
4032	#if HC_ARCH_BITS == 64
4033	uintptr_t const uTRBase = X86DESC64_BASE(pDesc);
4034
4035	/*
4036	* VT-x unconditionally restores the TR limit to 0x67 and type to 11 (32-bit busy TSS) on
4037	* all VM-exits. The type is the same for 64-bit busy TSS[1]. The limit needs manual
4038	* restoration if the host has something else. Task switching is not supported in 64-bit
4039	* mode[2], but the limit still matters as IOPM is supported in 64-bit mode. Restoring the
4040	* limit lazily while returning to ring-3 is safe because IOPM is not applicable in ring-0.
4041	*
4042	* [1] See Intel spec. 3.5 "System Descriptor Types".
4043	* [2] See Intel spec. 7.2.3 "TSS Descriptor in 64-bit mode".
4044	*/
4045	PVM pVM = pVCpu->CTX_SUFF(pVM);
4046	Assert(pDesc->System.u4Type == 11);
4047	if ( pDesc->System.u16LimitLow != 0x67
4048	\|\| pDesc->System.u4LimitHigh)
4049	{
4050	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_TR;
4051	/* If the host has made GDT read-only, we would need to temporarily toggle CR0.WP before writing the GDT. */
4052	if (pVM->hm.s.fHostKernelFeatures & SUPKERNELFEATURES_GDT_READ_ONLY)
4053	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_GDT_READ_ONLY;
4054	pVCpu->hm.s.vmx.RestoreHost.uHostSelTR = uSelTR;
4055	}
4056
4057	/*
4058	* Store the GDTR as we need it when restoring the GDT and while restoring the TR.
4059	*/
4060	if (pVCpu->hm.s.vmx.fRestoreHostFlags & (VMX_RESTORE_HOST_GDTR \| VMX_RESTORE_HOST_SEL_TR))
4061	{
4062	AssertCompile(sizeof(Gdtr) == sizeof(X86XDTR64));
4063	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
4064	if (pVM->hm.s.fHostKernelFeatures & SUPKERNELFEATURES_GDT_NEED_WRITABLE)
4065	{
4066	/* The GDT is read-only but the writable GDT is available. */
4067	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_GDT_NEED_WRITABLE;
4068	pVCpu->hm.s.vmx.RestoreHost.HostGdtrRw.cb = Gdtr.cbGdt;
4069	rc = SUPR0GetCurrentGdtRw(&pVCpu->hm.s.vmx.RestoreHost.HostGdtrRw.uAddr);
4070	AssertRCReturn(rc, rc);
4071	}
4072	}
4073	#else
4074	uintptr_t const uTRBase = X86DESC_BASE(pDesc);
4075	#endif
4076	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
4077	AssertRCReturn(rc, rc);
4078
4079	/*
4080	* Host FS base and GS base.
4081	*/
4082	#if HC_ARCH_BITS == 64
4083	uint64_t const u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
4084	uint64_t const u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
4085	rc = VMXWriteVmcs64(VMX_VMCS_HOST_FS_BASE, u64FSBase);
4086	rc \|= VMXWriteVmcs64(VMX_VMCS_HOST_GS_BASE, u64GSBase);
4087	AssertRCReturn(rc, rc);
4088
4089	/* Store the base if we have to restore FS or GS manually as we need to restore the base as well. */
4090	if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_FS)
4091	pVCpu->hm.s.vmx.RestoreHost.uHostFSBase = u64FSBase;
4092	if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_GS)
4093	pVCpu->hm.s.vmx.RestoreHost.uHostGSBase = u64GSBase;
4094	#endif
4095	return VINF_SUCCESS;
4096	}
4097
4098
4099	/**
4100	* Exports certain host MSRs in the VM-exit MSR-load area and some in the
4101	* host-state area of the VMCS.
4102	*
4103	* These MSRs will be automatically restored on the host after every successful
4104	* VM-exit.
4105	*
4106	* @returns VBox status code.
4107	* @param pVCpu The cross context virtual CPU structure.
4108	*
4109	* @remarks No-long-jump zone!!!
4110	*/
4111	static int hmR0VmxExportHostMsrs(PVMCPU pVCpu)
4112	{
4113	AssertPtr(pVCpu);
4114
4115	/*
4116	* Save MSRs that we restore lazily (due to preemption or transition to ring-3)
4117	* rather than swapping them on every VM-entry.
4118	*/
4119	hmR0VmxLazySaveHostMsrs(pVCpu);
4120
4121	/*
4122	* Host Sysenter MSRs.
4123	*/
4124	int rc = VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
4125	#if HC_ARCH_BITS == 32
4126	rc \|= VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
4127	rc \|= VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
4128	#else
4129	rc \|= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
4130	rc \|= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
4131	#endif
4132	AssertRCReturn(rc, rc);
4133
4134	/*
4135	* Host EFER MSR.
4136	*
4137	* If the CPU supports the newer VMCS controls for managing EFER, use it. Otherwise it's
4138	* done as part of auto-load/store MSR area in the VMCS, see hmR0VmxExportGuestMsrs().
4139	*/
4140	PVM pVM = pVCpu->CTX_SUFF(pVM);
4141	if (pVM->hm.s.vmx.fSupportsVmcsEfer)
4142	{
4143	rc = VMXWriteVmcs64(VMX_VMCS64_HOST_EFER_FULL, pVM->hm.s.vmx.u64HostMsrEfer);
4144	AssertRCReturn(rc, rc);
4145	}
4146
4147	/** @todo IA32_PERF_GLOBALCTRL, IA32_PAT also see
4148	* hmR0VmxExportGuestEntryExitCtls(). */
4149
4150	return VINF_SUCCESS;
4151	}
4152
4153
4154	/**
4155	* Figures out if we need to swap the EFER MSR which is particularly expensive.
4156	*
4157	* We check all relevant bits. For now, that's everything besides LMA/LME, as
4158	* these two bits are handled by VM-entry, see hmR0VMxExportGuestEntryExitCtls().
4159	*
4160	* @returns true if we need to load guest EFER, false otherwise.
4161	* @param pVCpu The cross context virtual CPU structure.
4162	*
4163	* @remarks Requires EFER, CR4.
4164	* @remarks No-long-jump zone!!!
4165	*/
4166	static bool hmR0VmxShouldSwapEferMsr(PVMCPU pVCpu)
4167	{
4168	#ifdef HMVMX_ALWAYS_SWAP_EFER
4169	RT_NOREF(pVCpu);
4170	return true;
4171	#else
4172	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4173	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
4174	/* For 32-bit hosts running 64-bit guests, we always swap EFER MSR in the world-switcher. Nothing to do here. */
4175	if (CPUMIsGuestInLongModeEx(pCtx))
4176	return false;
4177	#endif
4178
4179	PVM pVM = pVCpu->CTX_SUFF(pVM);
4180	uint64_t const u64HostEfer = pVM->hm.s.vmx.u64HostMsrEfer;
4181	uint64_t const u64GuestEfer = pCtx->msrEFER;
4182
4183	/*
4184	* For 64-bit guests, if EFER.SCE bit differs, we need to swap the EFER MSR
4185	* to ensure that the guest's SYSCALL behaviour isn't broken, see @bugref{7386}.
4186	*/
4187	if ( CPUMIsGuestInLongModeEx(pCtx)
4188	&& (u64GuestEfer & MSR_K6_EFER_SCE) != (u64HostEfer & MSR_K6_EFER_SCE))
4189	return true;
4190
4191	/*
4192	* If the guest uses PAE and EFER.NXE bit differs, we need to swap the EFER MSR
4193	* as it affects guest paging. 64-bit paging implies CR4.PAE as well.
4194	*
4195	* See Intel spec. 4.5 "IA-32e Paging".
4196	* See Intel spec. 4.1.1 "Three Paging Modes".
4197	*
4198	* Verify that we always intercept CR4.PAE and CR0.PG bits, so we don't need to
4199	* import CR4 and CR0 from the VMCS here as those bits are always up to date.
4200	*/
4201	Assert(hmR0VmxGetFixedCr4Mask(pVCpu) & X86_CR4_PAE);
4202	Assert(hmR0VmxGetFixedCr0Mask(pVCpu) & X86_CR0_PG);
4203	if ( (pCtx->cr4 & X86_CR4_PAE)
4204	&& (pCtx->cr0 & X86_CR0_PG)
4205	&& (u64GuestEfer & MSR_K6_EFER_NXE) != (u64HostEfer & MSR_K6_EFER_NXE))
4206	{
4207	/* Assert that host is NX capable. */
4208	Assert(pVCpu->CTX_SUFF(pVM)->cpum.ro.HostFeatures.fNoExecute);
4209	return true;
4210	}
4211
4212	return false;
4213	#endif
4214	}
4215
4216	/**
4217	* Exports the guest state with appropriate VM-entry and VM-exit controls in the
4218	* VMCS.
4219	*
4220	* This is typically required when the guest changes paging mode.
4221	*
4222	* @returns VBox status code.
4223	* @param pVCpu The cross context virtual CPU structure.
4224	* @param pVmxTransient The VMX-transient structure.
4225	*
4226	* @remarks Requires EFER.
4227	* @remarks No-long-jump zone!!!
4228	*/
4229	static int hmR0VmxExportGuestEntryExitCtls(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
4230	{
4231	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_ENTRY_EXIT_CTLS)
4232	{
4233	PVM pVM = pVCpu->CTX_SUFF(pVM);
4234	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
4235
4236	/*
4237	* VM-entry controls.
4238	*/
4239	{
4240	uint32_t fVal = pVM->hm.s.vmx.Msrs.EntryCtls.n.allowed0; /* Bits set here must be set in the VMCS. */
4241	uint32_t const fZap = pVM->hm.s.vmx.Msrs.EntryCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
4242
4243	/*
4244	* Load the guest debug controls (DR7 and IA32_DEBUGCTL MSR) on VM-entry.
4245	* The first VT-x capable CPUs only supported the 1-setting of this bit.
4246	*
4247	* For nested-guests, this is a mandatory VM-entry control. It's also
4248	* required because we do not want to leak host bits to the nested-guest.
4249	*/
4250	fVal \|= VMX_ENTRY_CTLS_LOAD_DEBUG;
4251
4252	/*
4253	* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry.
4254	*
4255	* For nested-guests, the "IA-32e mode guest" control we initialize with what is
4256	* required to get the nested-guest working with hardware-assisted VMX execution.
4257	* It depends on the nested-guest's IA32_EFER.LMA bit. Remember, a nested-hypervisor
4258	* can skip intercepting changes to the EFER MSR. This is why it it needs to be done
4259	* here rather than while merging the guest VMCS controls.
4260	*/
4261	if (CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx))
4262	fVal \|= VMX_ENTRY_CTLS_IA32E_MODE_GUEST;
4263	else
4264	Assert(!(fVal & VMX_ENTRY_CTLS_IA32E_MODE_GUEST));
4265
4266	/*
4267	* If the CPU supports the newer VMCS controls for managing guest/host EFER, use it.
4268	*
4269	* For nested-guests, we use the "load IA32_EFER" if the hardware supports it,
4270	* regardless of whether the nested-guest VMCS specifies it because we are free to
4271	* load whatever MSRs we require and we do not need to modify the guest visible copy
4272	* of the VM-entry MSR load area.
4273	*/
4274	if ( pVM->hm.s.vmx.fSupportsVmcsEfer
4275	&& hmR0VmxShouldSwapEferMsr(pVCpu))
4276	fVal \|= VMX_ENTRY_CTLS_LOAD_EFER_MSR;
4277	else
4278	Assert(!(fVal & VMX_ENTRY_CTLS_LOAD_EFER_MSR));
4279
4280	/*
4281	* The following should -not- be set (since we're not in SMM mode):
4282	* - VMX_ENTRY_CTLS_ENTRY_TO_SMM
4283	* - VMX_ENTRY_CTLS_DEACTIVATE_DUAL_MON
4284	*/
4285
4286	/** @todo VMX_ENTRY_CTLS_LOAD_PERF_MSR,
4287	* VMX_ENTRY_CTLS_LOAD_PAT_MSR. */
4288
4289	if ((fVal & fZap) == fVal)
4290	{ /* likely */ }
4291	else
4292	{
4293	Log4Func(("Invalid VM-entry controls combo! Cpu=%#RX32 fVal=%#RX32 fZap=%#RX32\n",
4294	pVM->hm.s.vmx.Msrs.EntryCtls.n.allowed0, fVal, fZap));
4295	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_ENTRY;
4296	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
4297	}
4298
4299	/* Commit it to the VMCS. */
4300	if (pVmcsInfo->u32EntryCtls != fVal)
4301	{
4302	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY, fVal);
4303	AssertRCReturn(rc, rc);
4304	pVmcsInfo->u32EntryCtls = fVal;
4305	}
4306	}
4307
4308	/*
4309	* VM-exit controls.
4310	*/
4311	{
4312	uint32_t fVal = pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed0; /* Bits set here must be set in the VMCS. */
4313	uint32_t const fZap = pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
4314
4315	/*
4316	* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only
4317	* supported the 1-setting of this bit.
4318	*
4319	* For nested-guests, we set the "save debug controls" as the converse
4320	* "load debug controls" is mandatory for nested-guests anyway.
4321	*/
4322	fVal \|= VMX_EXIT_CTLS_SAVE_DEBUG;
4323
4324	/*
4325	* Set the host long mode active (EFER.LMA) bit (which Intel calls
4326	* "Host address-space size") if necessary. On VM-exit, VT-x sets both the
4327	* host EFER.LMA and EFER.LME bit to this value. See assertion in
4328	* hmR0VmxExportHostMsrs().
4329	*
4330	* For nested-guests, we always set this bit as we do not support 32-bit
4331	* hosts.
4332	*/
4333	#if HC_ARCH_BITS == 64
4334	fVal \|= VMX_EXIT_CTLS_HOST_ADDR_SPACE_SIZE;
4335	#else
4336	Assert(!pVmxTransient->fIsNestedGuest);
4337	Assert( pVmcsInfo->pfnStartVM == VMXR0SwitcherStartVM64
4338	\|\| pVmcsInfo->pfnStartVM == VMXR0StartVM32);
4339	/* Set the host address-space size based on the switcher, not guest state. See @bugref{8432}. */
4340	if (pVmcsInfo->pfnStartVM == VMXR0SwitcherStartVM64)
4341	{
4342	/* The switcher returns to long mode, the EFER MSR is managed by the switcher. */
4343	fVal \|= VMX_EXIT_CTLS_HOST_ADDR_SPACE_SIZE;
4344	}
4345	else
4346	Assert(!(fVal & VMX_EXIT_CTLS_HOST_ADDR_SPACE_SIZE));
4347	#endif
4348
4349	/*
4350	* If the VMCS EFER MSR fields are supported by the hardware, we use it.
4351	*
4352	* For nested-guests, we should use the "save IA32_EFER" control if we also
4353	* used the "load IA32_EFER" control while exporting VM-entry controls.
4354	*/
4355	if ( pVM->hm.s.vmx.fSupportsVmcsEfer
4356	&& hmR0VmxShouldSwapEferMsr(pVCpu))
4357	{
4358	fVal \|= VMX_EXIT_CTLS_SAVE_EFER_MSR
4359	\| VMX_EXIT_CTLS_LOAD_EFER_MSR;
4360	}
4361
4362	/*
4363	* Enable saving of the VMX-preemption timer value on VM-exit.
4364	* For nested-guests, currently not exposed/used.
4365	*/
4366	if ( pVM->hm.s.vmx.fUsePreemptTimer
4367	&& (pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_SAVE_PREEMPT_TIMER))
4368	fVal \|= VMX_EXIT_CTLS_SAVE_PREEMPT_TIMER;
4369
4370	/* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
4371	Assert(!(fVal & VMX_EXIT_CTLS_ACK_EXT_INT));
4372
4373	/** @todo VMX_EXIT_CTLS_LOAD_PERF_MSR,
4374	* VMX_EXIT_CTLS_SAVE_PAT_MSR,
4375	* VMX_EXIT_CTLS_LOAD_PAT_MSR. */
4376
4377	if ((fVal & fZap) == fVal)
4378	{ /* likely */ }
4379	else
4380	{
4381	Log4Func(("Invalid VM-exit controls combo! cpu=%#RX32 fVal=%#RX32 fZap=%R#X32\n",
4382	pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed0, fVal, fZap));
4383	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_EXIT;
4384	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
4385	}
4386
4387	/* Commit it to the VMCS. */
4388	if (pVmcsInfo->u32ExitCtls != fVal)
4389	{
4390	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT, fVal);
4391	AssertRCReturn(rc, rc);
4392	pVmcsInfo->u32ExitCtls = fVal;
4393	}
4394	}
4395
4396	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_ENTRY_EXIT_CTLS);
4397	}
4398	return VINF_SUCCESS;
4399	}
4400
4401
4402	/**
4403	* Sets the TPR threshold in the VMCS.
4404	*
4405	* @returns VBox status code.
4406	* @param pVCpu The cross context virtual CPU structure.
4407	* @param pVmcsInfo The VMCS info. object.
4408	* @param u32TprThreshold The TPR threshold (task-priority class only).
4409	*/
4410	DECLINLINE(int) hmR0VmxApicSetTprThreshold(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, uint32_t u32TprThreshold)
4411	{
4412	Assert(!(u32TprThreshold & ~VMX_TPR_THRESHOLD_MASK)); /* Bits 31:4 MBZ. */
4413	Assert(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW);
4414	RT_NOREF2(pVCpu, pVmcsInfo);
4415	return VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
4416	}
4417
4418
4419	/**
4420	* Exports the guest APIC TPR state into the VMCS.
4421	*
4422	* @returns VBox status code.
4423	* @param pVCpu The cross context virtual CPU structure.
4424	* @param pVmxTransient The VMX-transient structure.
4425	*
4426	* @remarks No-long-jump zone!!!
4427	*/
4428	static int hmR0VmxExportGuestApicTpr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
4429	{
4430	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_APIC_TPR)
4431	{
4432	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_APIC_TPR);
4433
4434	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
4435	if (!pVmxTransient->fIsNestedGuest)
4436	{
4437	if ( PDMHasApic(pVCpu->CTX_SUFF(pVM))
4438	&& APICIsEnabled(pVCpu))
4439	{
4440	/*
4441	* Setup TPR shadowing.
4442	*/
4443	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
4444	{
4445	bool fPendingIntr = false;
4446	uint8_t u8Tpr = 0;
4447	uint8_t u8PendingIntr = 0;
4448	int rc = APICGetTpr(pVCpu, &u8Tpr, &fPendingIntr, &u8PendingIntr);
4449	AssertRCReturn(rc, rc);
4450
4451	/*
4452	* If there are interrupts pending but masked by the TPR, instruct VT-x to
4453	* cause a TPR-below-threshold VM-exit when the guest lowers its TPR below the
4454	* priority of the pending interrupt so we can deliver the interrupt. If there
4455	* are no interrupts pending, set threshold to 0 to not cause any
4456	* TPR-below-threshold VM-exits.
4457	*/
4458	Assert(pVmcsInfo->pbVirtApic);
4459	pVmcsInfo->pbVirtApic[XAPIC_OFF_TPR] = u8Tpr;
4460	uint32_t u32TprThreshold = 0;
4461	if (fPendingIntr)
4462	{
4463	/* Bits 3:0 of the TPR threshold field correspond to bits 7:4 of the TPR
4464	(which is the Task-Priority Class). */
4465	const uint8_t u8PendingPriority = u8PendingIntr >> 4;
4466	const uint8_t u8TprPriority = u8Tpr >> 4;
4467	if (u8PendingPriority <= u8TprPriority)
4468	u32TprThreshold = u8PendingPriority;
4469	}
4470
4471	rc = hmR0VmxApicSetTprThreshold(pVCpu, pVmcsInfo, u32TprThreshold);
4472	AssertRCReturn(rc, rc);
4473	}
4474	}
4475	}
4476	/* else: the TPR threshold has already been updated while merging the nested-guest VMCS. */
4477	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_APIC_TPR);
4478	}
4479	return VINF_SUCCESS;
4480	}
4481
4482
4483	/**
4484	* Gets the guest interruptibility-state.
4485	*
4486	* @returns Guest's interruptibility-state.
4487	* @param pVCpu The cross context virtual CPU structure.
4488	* @param pVmcsInfo The VMCS info. object.
4489	*
4490	* @remarks No-long-jump zone!!!
4491	*/
4492	static uint32_t hmR0VmxGetGuestIntrState(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
4493	{
4494	/*
4495	* Check if we should inhibit interrupt delivery due to instructions like STI and MOV SS.
4496	*/
4497	uint32_t fIntrState = 0;
4498	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
4499	{
4500	/* If inhibition is active, RIP and RFLAGS should've been updated
4501	(i.e. read previously from the VMCS or from ring-3). */
4502	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4503	#ifdef VBOX_STRICT
4504	uint64_t const fExtrn = ASMAtomicUoReadU64(&pCtx->fExtrn);
4505	RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
4506	AssertMsg(!(fExtrn & (CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS)), ("%#x\n", fExtrn));
4507	#endif
4508	if (pCtx->rip == EMGetInhibitInterruptsPC(pVCpu))
4509	{
4510	if (pCtx->eflags.Bits.u1IF)
4511	fIntrState = VMX_VMCS_GUEST_INT_STATE_BLOCK_STI;
4512	else
4513	fIntrState = VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS;
4514	}
4515	else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
4516	{
4517	/*
4518	* We can clear the inhibit force flag as even if we go back to the recompiler
4519	* without executing guest code in VT-x, the flag's condition to be cleared is
4520	* met and thus the cleared state is correct.
4521	*/
4522	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
4523	}
4524	}
4525
4526	/*
4527	* NMIs to the guest are blocked after an NMI is injected until the guest executes an IRET. We only
4528	* bother with virtual-NMI blocking when we have support for virtual NMIs in the CPU, otherwise
4529	* setting this would block host-NMIs and IRET will not clear the blocking.
4530	*
4531	* We always set NMI-exiting so when the host receives an NMI we get a VM-exit.
4532	*
4533	* See Intel spec. 26.6.1 "Interruptibility state". See @bugref{7445}.
4534	*/
4535	if ( (pVmcsInfo->u32PinCtls & VMX_PIN_CTLS_VIRT_NMI)
4536	&& CPUMIsGuestNmiBlocking(pVCpu))
4537	fIntrState \|= VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI;
4538
4539	return fIntrState;
4540	}
4541
4542
4543	/**
4544	* Exports the exception intercepts required for guest execution in the VMCS.
4545	*
4546	* @returns VBox status code.
4547	* @param pVCpu The cross context virtual CPU structure.
4548	* @param pVmxTransient The VMX-transient structure.
4549	*
4550	* @remarks No-long-jump zone!!!
4551	*/
4552	static int hmR0VmxExportGuestXcptIntercepts(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
4553	{
4554	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_GUEST_XCPT_INTERCEPTS)
4555	{
4556	/* When executing a nested-guest, we do not need to trap GIM hypercalls by intercepting #UD. */
4557	if ( !pVmxTransient->fIsNestedGuest
4558	&& pVCpu->hm.s.fGIMTrapXcptUD)
4559	hmR0VmxAddXcptIntercept(pVmxTransient, X86_XCPT_UD);
4560	else
4561	hmR0VmxRemoveXcptIntercept(pVCpu, pVmxTransient, X86_XCPT_UD);
4562
4563	/* Other exception intercepts are handled elsewhere, e.g. while exporting guest CR0. */
4564	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_GUEST_XCPT_INTERCEPTS);
4565	}
4566	return VINF_SUCCESS;
4567	}
4568
4569
4570	/**
4571	* Exports the guest's RIP into the guest-state area in the VMCS.
4572	*
4573	* @returns VBox status code.
4574	* @param pVCpu The cross context virtual CPU structure.
4575	*
4576	* @remarks No-long-jump zone!!!
4577	*/
4578	static int hmR0VmxExportGuestRip(PVMCPU pVCpu)
4579	{
4580	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RIP)
4581	{
4582	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RIP);
4583
4584	int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pVCpu->cpum.GstCtx.rip);
4585	AssertRCReturn(rc, rc);
4586
4587	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RIP);
4588	Log4Func(("rip=%#RX64\n", pVCpu->cpum.GstCtx.rip));
4589	}
4590	return VINF_SUCCESS;
4591	}
4592
4593
4594	/**
4595	* Exports the guest's RSP into the guest-state area in the VMCS.
4596	*
4597	* @returns VBox status code.
4598	* @param pVCpu The cross context virtual CPU structure.
4599	*
4600	* @remarks No-long-jump zone!!!
4601	*/
4602	static int hmR0VmxExportGuestRsp(PVMCPU pVCpu)
4603	{
4604	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RSP)
4605	{
4606	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RSP);
4607
4608	int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pVCpu->cpum.GstCtx.rsp);
4609	AssertRCReturn(rc, rc);
4610
4611	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RSP);
4612	}
4613	return VINF_SUCCESS;
4614	}
4615
4616
4617	/**
4618	* Exports the guest's RFLAGS into the guest-state area in the VMCS.
4619	*
4620	* @returns VBox status code.
4621	* @param pVCpu The cross context virtual CPU structure.
4622	* @param pVmxTransient The VMX-transient structure.
4623	*
4624	* @remarks No-long-jump zone!!!
4625	*/
4626	static int hmR0VmxExportGuestRflags(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
4627	{
4628	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RFLAGS)
4629	{
4630	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RFLAGS);
4631
4632	/* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
4633	Let us assert it as such and use 32-bit VMWRITE. */
4634	Assert(!RT_HI_U32(pVCpu->cpum.GstCtx.rflags.u64));
4635	X86EFLAGS fEFlags = pVCpu->cpum.GstCtx.eflags;
4636	Assert(fEFlags.u32 & X86_EFL_RA1_MASK);
4637	Assert(!(fEFlags.u32 & ~(X86_EFL_1 \| X86_EFL_LIVE_MASK)));
4638
4639	/*
4640	* If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so
4641	* we can restore them on VM-exit. Modify the real-mode guest's eflags so that VT-x
4642	* can run the real-mode guest code under Virtual 8086 mode.
4643	*/
4644	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
4645	if (pVmcsInfo->RealMode.fRealOnV86Active)
4646	{
4647	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
4648	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
4649	Assert(!pVmxTransient->fIsNestedGuest);
4650	pVmcsInfo->RealMode.Eflags.u32 = fEFlags.u32; /* Save the original eflags of the real-mode guest. */
4651	fEFlags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
4652	fEFlags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
4653	}
4654
4655	int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_RFLAGS, fEFlags.u32);
4656	AssertRCReturn(rc, rc);
4657
4658	/*
4659	* Setup pending debug exceptions if the guest is single-stepping using EFLAGS.TF.
4660	*
4661	* We must avoid setting any automatic debug exceptions delivery when single-stepping
4662	* through the hypervisor debugger using EFLAGS.TF.
4663	*/
4664	if ( !pVmxTransient->fIsNestedGuest
4665	&& !pVCpu->hm.s.fSingleInstruction
4666	&& fEFlags.Bits.u1TF)
4667	{
4668	/** @todo r=ramshankar: Warning!! We ASSUME EFLAGS.TF will not cleared on
4669	* premature trips to ring-3 esp since IEM does not yet handle it. */
4670	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, VMX_VMCS_GUEST_PENDING_DEBUG_XCPT_BS);
4671	AssertRCReturn(rc, rc);
4672	}
4673	/** @todo NSTVMX: Handling copying of VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS from
4674	* nested-guest VMCS. */
4675
4676	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RFLAGS);
4677	Log4Func(("EFlags=%#RX32\n", fEFlags.u32));
4678	}
4679	return VINF_SUCCESS;
4680	}
4681
4682
4683	/**
4684	* Exports the guest CR0 control register into the guest-state area in the VMCS.
4685	*
4686	* The guest FPU state is always pre-loaded hence we don't need to bother about
4687	* sharing FPU related CR0 bits between the guest and host.
4688	*
4689	* @returns VBox status code.
4690	* @param pVCpu The cross context virtual CPU structure.
4691	* @param pVmxTransient The VMX-transient structure.
4692	*
4693	* @remarks No-long-jump zone!!!
4694	*/
4695	static int hmR0VmxExportGuestCR0(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
4696	{
4697	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR0)
4698	{
4699	PVM pVM = pVCpu->CTX_SUFF(pVM);
4700	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
4701
4702	/*
4703	* Figure out fixed CR0 bits in VMX operation.
4704	*/
4705	uint64_t fSetCr0 = pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1;
4706	uint64_t const fZapCr0 = pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr0Fixed1;
4707	if (pVM->hm.s.vmx.fUnrestrictedGuest)
4708	fSetCr0 &= ~(uint64_t)(X86_CR0_PE \| X86_CR0_PG);
4709	else
4710	Assert((fSetCr0 & (X86_CR0_PE \| X86_CR0_PG)) == (X86_CR0_PE \| X86_CR0_PG));
4711
4712	if (!pVmxTransient->fIsNestedGuest)
4713	{
4714	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0);
4715	uint64_t u64GuestCr0 = pVCpu->cpum.GstCtx.cr0;
4716	uint64_t const u64ShadowCr0 = u64GuestCr0;
4717	Assert(!RT_HI_U32(u64GuestCr0));
4718
4719	/*
4720	* Setup VT-x's view of the guest CR0.
4721	*/
4722	uint32_t uProcCtls = pVmcsInfo->u32ProcCtls;
4723	if (pVM->hm.s.fNestedPaging)
4724	{
4725	if (CPUMIsGuestPagingEnabled(pVCpu))
4726	{
4727	/* The guest has paging enabled, let it access CR3 without causing a VM-exit if supported. */
4728	uProcCtls &= ~( VMX_PROC_CTLS_CR3_LOAD_EXIT
4729	\| VMX_PROC_CTLS_CR3_STORE_EXIT);
4730	}
4731	else
4732	{
4733	/* The guest doesn't have paging enabled, make CR3 access cause a VM-exit to update our shadow. */
4734	uProcCtls \|= VMX_PROC_CTLS_CR3_LOAD_EXIT
4735	\| VMX_PROC_CTLS_CR3_STORE_EXIT;
4736	}
4737
4738	/* If we have unrestricted guest execution, we never have to intercept CR3 reads. */
4739	if (pVM->hm.s.vmx.fUnrestrictedGuest)
4740	uProcCtls &= ~VMX_PROC_CTLS_CR3_STORE_EXIT;
4741	}
4742	else
4743	{
4744	/* Guest CPL 0 writes to its read-only pages should cause a #PF VM-exit. */
4745	u64GuestCr0 \|= X86_CR0_WP;
4746	}
4747
4748	/*
4749	* Guest FPU bits.
4750	*
4751	* Since we pre-load the guest FPU always before VM-entry there is no need to track lazy state
4752	* using CR0.TS.
4753	*
4754	* Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be
4755	* set on the first CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
4756	*/
4757	u64GuestCr0 \|= X86_CR0_NE;
4758
4759	/* If CR0.NE isn't set, we need to intercept #MF exceptions and report them to the guest differently. */
4760	bool const fInterceptMF = !(u64ShadowCr0 & X86_CR0_NE);
4761
4762	/*
4763	* Update exception intercepts.
4764	*/
4765	uint32_t uXcptBitmap = pVmcsInfo->u32XcptBitmap;
4766	if (pVmcsInfo->RealMode.fRealOnV86Active)
4767	{
4768	Assert(PDMVmmDevHeapIsEnabled(pVM));
4769	Assert(pVM->hm.s.vmx.pRealModeTSS);
4770	uXcptBitmap \|= HMVMX_REAL_MODE_XCPT_MASK;
4771	}
4772	else
4773	{
4774	/* For now, cleared here as mode-switches can happen outside HM/VT-x. See @bugref{7626#c11}. */
4775	uXcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
4776	if (fInterceptMF)
4777	uXcptBitmap \|= RT_BIT(X86_XCPT_MF);
4778	}
4779
4780	/* Additional intercepts for debugging, define these yourself explicitly. */
4781	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
4782	uXcptBitmap \|= 0
4783	\| RT_BIT(X86_XCPT_BP)
4784	\| RT_BIT(X86_XCPT_DE)
4785	\| RT_BIT(X86_XCPT_NM)
4786	\| RT_BIT(X86_XCPT_TS)
4787	\| RT_BIT(X86_XCPT_UD)
4788	\| RT_BIT(X86_XCPT_NP)
4789	\| RT_BIT(X86_XCPT_SS)
4790	\| RT_BIT(X86_XCPT_GP)
4791	\| RT_BIT(X86_XCPT_PF)
4792	\| RT_BIT(X86_XCPT_MF)
4793	;
4794	#elif defined(HMVMX_ALWAYS_TRAP_PF)
4795	uXcptBitmap \|= RT_BIT(X86_XCPT_PF);
4796	#endif
4797	if (pVCpu->hm.s.fTrapXcptGpForLovelyMesaDrv)
4798	uXcptBitmap \|= RT_BIT(X86_XCPT_GP);
4799	Assert(pVM->hm.s.fNestedPaging \|\| (uXcptBitmap & RT_BIT(X86_XCPT_PF)));
4800
4801	/* Apply the fixed CR0 bits and enable caching. */
4802	u64GuestCr0 \|= fSetCr0;
4803	u64GuestCr0 &= fZapCr0;
4804	u64GuestCr0 &= ~(uint64_t)(X86_CR0_CD \| X86_CR0_NW);
4805
4806	/* Commit the CR0 and related fields to the guest VMCS. */
4807	int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR0, u64GuestCr0);
4808	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, u64ShadowCr0);
4809	if (uProcCtls != pVmcsInfo->u32ProcCtls)
4810	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
4811	if (uXcptBitmap != pVmcsInfo->u32XcptBitmap)
4812	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
4813	AssertRCReturn(rc, rc);
4814
4815	/* Update our caches. */
4816	pVmcsInfo->u32ProcCtls = uProcCtls;
4817	pVmcsInfo->u32XcptBitmap = uXcptBitmap;
4818
4819	Log4Func(("cr0=%#RX64 shadow=%#RX64 set=%#RX64 zap=%#RX64\n", u64GuestCr0, u64ShadowCr0, fSetCr0, fZapCr0));
4820	}
4821	else
4822	{
4823	PCVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
4824	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0);
4825	uint64_t u64GuestCr0 = pVCpu->cpum.GstCtx.cr0;
4826	uint64_t const u64ShadowCr0 = pVmcsNstGst->u64Cr0ReadShadow.u;
4827	Assert(!RT_HI_U32(u64GuestCr0));
4828	Assert(u64GuestCr0 & X86_CR0_NE);
4829
4830	/* Apply the fixed CR0 bits and enable caching. */
4831	u64GuestCr0 \|= fSetCr0;
4832	u64GuestCr0 &= fZapCr0;
4833	u64GuestCr0 &= ~(uint64_t)(X86_CR0_CD \| X86_CR0_NW);
4834
4835	/* Commit the CR0 and CR0 read shadow to the nested-guest VMCS. */
4836	int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR0, u64GuestCr0);
4837	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, u64ShadowCr0);
4838	AssertRCReturn(rc, rc);
4839
4840	Log4Func(("cr0=%#RX64 shadow=%#RX64 set=%#RX64 zap=%#RX64\n", u64GuestCr0, u64ShadowCr0, fSetCr0, fZapCr0));
4841	}
4842
4843	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR0);
4844	}
4845
4846	return VINF_SUCCESS;
4847	}
4848
4849
4850	/**
4851	* Exports the guest control registers (CR3, CR4) into the guest-state area
4852	* in the VMCS.
4853	*
4854	* @returns VBox strict status code.
4855	* @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
4856	* without unrestricted guest access and the VMMDev is not presently
4857	* mapped (e.g. EFI32).
4858	*
4859	* @param pVCpu The cross context virtual CPU structure.
4860	* @param pVmxTransient The VMX-transient structure.
4861	*
4862	* @remarks No-long-jump zone!!!
4863	*/
4864	static VBOXSTRICTRC hmR0VmxExportGuestCR3AndCR4(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
4865	{
4866	int rc = VINF_SUCCESS;
4867	PVM pVM = pVCpu->CTX_SUFF(pVM);
4868
4869	/*
4870	* Guest CR2.
4871	* It's always loaded in the assembler code. Nothing to do here.
4872	*/
4873
4874	/*
4875	* Guest CR3.
4876	*/
4877	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR3)
4878	{
4879	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR3);
4880
4881	RTGCPHYS GCPhysGuestCR3 = NIL_RTGCPHYS;
4882	if (pVM->hm.s.fNestedPaging)
4883	{
4884	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
4885	pVmcsInfo->HCPhysEPTP = PGMGetHyperCR3(pVCpu);
4886
4887	/* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
4888	Assert(pVmcsInfo->HCPhysEPTP != NIL_RTHCPHYS);
4889	Assert(!(pVmcsInfo->HCPhysEPTP & UINT64_C(0xfff0000000000000)));
4890	Assert(!(pVmcsInfo->HCPhysEPTP & 0xfff));
4891
4892	/* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
4893	pVmcsInfo->HCPhysEPTP \|= VMX_EPT_MEMTYPE_WB
4894	\| (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
4895
4896	/* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
4897	AssertMsg( ((pVmcsInfo->HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
4898	&& ((pVmcsInfo->HCPhysEPTP >> 7) & 0x1f) == 0, /* Bits 7:11 MBZ. */
4899	("EPTP %#RX64\n", pVmcsInfo->HCPhysEPTP));
4900	AssertMsg( !((pVmcsInfo->HCPhysEPTP >> 6) & 0x01) /* Bit 6 (EPT accessed & dirty bit). */
4901	\|\| (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EPT_ACCESS_DIRTY),
4902	("EPTP accessed/dirty bit not supported by CPU but set %#RX64\n", pVmcsInfo->HCPhysEPTP));
4903
4904	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVmcsInfo->HCPhysEPTP);
4905	AssertRCReturn(rc, rc);
4906
4907	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4908	if ( pVM->hm.s.vmx.fUnrestrictedGuest
4909	\|\| CPUMIsGuestPagingEnabledEx(pCtx))
4910	{
4911	/* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
4912	if (CPUMIsGuestInPAEModeEx(pCtx))
4913	{
4914	rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
4915	AssertRCReturn(rc, rc);
4916	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u);
4917	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u);
4918	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u);
4919	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u);
4920	AssertRCReturn(rc, rc);
4921	}
4922
4923	/*
4924	* The guest's view of its CR3 is unblemished with nested paging when the
4925	* guest is using paging or we have unrestricted guest execution to handle
4926	* the guest when it's not using paging.
4927	*/
4928	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR3);
4929	GCPhysGuestCR3 = pCtx->cr3;
4930	}
4931	else
4932	{
4933	/*
4934	* The guest is not using paging, but the CPU (VT-x) has to. While the guest
4935	* thinks it accesses physical memory directly, we use our identity-mapped
4936	* page table to map guest-linear to guest-physical addresses. EPT takes care
4937	* of translating it to host-physical addresses.
4938	*/
4939	RTGCPHYS GCPhys;
4940	Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
4941
4942	/* We obtain it here every time as the guest could have relocated this PCI region. */
4943	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
4944	if (RT_SUCCESS(rc))
4945	{ /* likely */ }
4946	else if (rc == VERR_PDM_DEV_HEAP_R3_TO_GCPHYS)
4947	{
4948	Log4Func(("VERR_PDM_DEV_HEAP_R3_TO_GCPHYS -> VINF_EM_RESCHEDULE_REM\n"));
4949	return VINF_EM_RESCHEDULE_REM; /* We cannot execute now, switch to REM/IEM till the guest maps in VMMDev. */
4950	}
4951	else
4952	AssertMsgFailedReturn(("%Rrc\n", rc), rc);
4953
4954	GCPhysGuestCR3 = GCPhys;
4955	}
4956
4957	Log4Func(("u32GuestCr3=%#RGp (GstN)\n", GCPhysGuestCR3));
4958	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, GCPhysGuestCR3);
4959	AssertRCReturn(rc, rc);
4960	}
4961	else
4962	{
4963	/* Non-nested paging case, just use the hypervisor's CR3. */
4964	RTHCPHYS const HCPhysGuestCR3 = PGMGetHyperCR3(pVCpu);
4965
4966	Log4Func(("u32GuestCr3=%#RHv (HstN)\n", HCPhysGuestCR3));
4967	rc = VMXWriteVmcsHstN(VMX_VMCS_GUEST_CR3, HCPhysGuestCR3);
4968	AssertRCReturn(rc, rc);
4969	}
4970
4971	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR3);
4972	}
4973
4974	/*
4975	* Guest CR4.
4976	* ASSUMES this is done everytime we get in from ring-3! (XCR0)
4977	*/
4978	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR4)
4979	{
4980	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4981	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
4982	PCVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
4983
4984	/*
4985	* Figure out fixed CR4 bits in VMX operation.
4986	*/
4987	uint64_t const fSetCr4 = pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1;
4988	uint64_t const fZapCr4 = pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr4Fixed1;
4989
4990	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR4);
4991	uint64_t u64GuestCr4 = pCtx->cr4;
4992	uint64_t const u64ShadowCr4 = !pVmxTransient->fIsNestedGuest ? pCtx->cr4 : pVmcsNstGst->u64Cr4ReadShadow.u;
4993	Assert(!RT_HI_U32(u64GuestCr4));
4994
4995	/*
4996	* Setup VT-x's view of the guest CR4.
4997	*
4998	* If we're emulating real-mode using virtual-8086 mode, we want to redirect software
4999	* interrupts to the 8086 program interrupt handler. Clear the VME bit (the interrupt
5000	* redirection bitmap is already all 0, see hmR3InitFinalizeR0())
5001	*
5002	* See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
5003	*/
5004	if (pVmcsInfo->RealMode.fRealOnV86Active)
5005	{
5006	Assert(pVM->hm.s.vmx.pRealModeTSS);
5007	Assert(PDMVmmDevHeapIsEnabled(pVM));
5008	u64GuestCr4 &= ~(uint64_t)X86_CR4_VME;
5009	}
5010
5011	if (pVM->hm.s.fNestedPaging)
5012	{
5013	if ( !CPUMIsGuestPagingEnabledEx(pCtx)
5014	&& !pVM->hm.s.vmx.fUnrestrictedGuest)
5015	{
5016	/* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
5017	u64GuestCr4 \|= X86_CR4_PSE;
5018	/* Our identity mapping is a 32-bit page directory. */
5019	u64GuestCr4 &= ~(uint64_t)X86_CR4_PAE;
5020	}
5021	/* else use guest CR4.*/
5022	}
5023	else
5024	{
5025	Assert(!pVmxTransient->fIsNestedGuest);
5026
5027	/*
5028	* The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
5029	* paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
5030	*/
5031	switch (pVCpu->hm.s.enmShadowMode)
5032	{
5033	case PGMMODE_REAL: /* Real-mode. */
5034	case PGMMODE_PROTECTED: /* Protected mode without paging. */
5035	case PGMMODE_32_BIT: /* 32-bit paging. */
5036	{
5037	u64GuestCr4 &= ~(uint64_t)X86_CR4_PAE;
5038	break;
5039	}
5040
5041	case PGMMODE_PAE: /* PAE paging. */
5042	case PGMMODE_PAE_NX: /* PAE paging with NX. */
5043	{
5044	u64GuestCr4 \|= X86_CR4_PAE;
5045	break;
5046	}
5047
5048	case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
5049	case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
5050	#ifdef VBOX_ENABLE_64_BITS_GUESTS
5051	break;
5052	#endif
5053	default:
5054	AssertFailed();
5055	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
5056	}
5057	}
5058
5059	/* Apply the fixed CR4 bits (mainly CR4.VMXE). */
5060	u64GuestCr4 \|= fSetCr4;
5061	u64GuestCr4 &= fZapCr4;
5062
5063	/* Commit the CR4 and CR4 read shadow to the guest VMCS. */
5064	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR4, u64GuestCr4);
5065	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, u64ShadowCr4);
5066	AssertRCReturn(rc, rc);
5067
5068	/* Whether to save/load/restore XCR0 during world switch depends on CR4.OSXSAVE and host+guest XCR0. */
5069	pVCpu->hm.s.fLoadSaveGuestXcr0 = (pCtx->cr4 & X86_CR4_OSXSAVE) && pCtx->aXcr[0] != ASMGetXcr0();
5070
5071	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR4);
5072
5073	Log4Func(("cr4=%#RX64 shadow=%#RX64 set=%#RX64 zap=%#RX64)\n", u64GuestCr4, u64ShadowCr4, fSetCr4, fZapCr4));
5074	}
5075	return rc;
5076	}
5077
5078
5079	/**
5080	* Exports the guest debug registers into the guest-state area in the VMCS.
5081	* The guest debug bits are partially shared with the host (e.g. DR6, DR0-3).
5082	*
5083	* This also sets up whether \#DB and MOV DRx accesses cause VM-exits.
5084	*
5085	* @returns VBox status code.
5086	* @param pVCpu The cross context virtual CPU structure.
5087	* @param pVmxTransient The VMX-transient structure.
5088	*
5089	* @remarks No-long-jump zone!!!
5090	*/
5091	static int hmR0VmxExportSharedDebugState(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
5092	{
5093	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
5094
5095	/** @todo NSTVMX: Figure out what we want to do with nested-guest instruction
5096	* stepping. */
5097	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
5098	if (pVmxTransient->fIsNestedGuest)
5099	{
5100	int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, CPUMGetGuestDR7(pVCpu));
5101	AssertRCReturn(rc, rc);
5102	return VINF_SUCCESS;
5103	}
5104
5105	#ifdef VBOX_STRICT
5106	/* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
5107	if (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_DEBUG)
5108	{
5109	/* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
5110	Assert((pVCpu->cpum.GstCtx.dr[7] & (X86_DR7_MBZ_MASK \| X86_DR7_RAZ_MASK)) == 0);
5111	Assert((pVCpu->cpum.GstCtx.dr[7] & X86_DR7_RA1_MASK) == X86_DR7_RA1_MASK);
5112	}
5113	#endif
5114
5115	bool fSteppingDB = false;
5116	bool fInterceptMovDRx = false;
5117	uint32_t uProcCtls = pVmcsInfo->u32ProcCtls;
5118	if (pVCpu->hm.s.fSingleInstruction)
5119	{
5120	/* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
5121	PVM pVM = pVCpu->CTX_SUFF(pVM);
5122	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_MONITOR_TRAP_FLAG)
5123	{
5124	uProcCtls \|= VMX_PROC_CTLS_MONITOR_TRAP_FLAG;
5125	Assert(fSteppingDB == false);
5126	}
5127	else
5128	{
5129	pVCpu->cpum.GstCtx.eflags.u32 \|= X86_EFL_TF;
5130	pVCpu->hm.s.fCtxChanged \|= HM_CHANGED_GUEST_RFLAGS;
5131	pVCpu->hm.s.fClearTrapFlag = true;
5132	fSteppingDB = true;
5133	}
5134	}
5135
5136	uint32_t u32GuestDr7;
5137	if ( fSteppingDB
5138	\|\| (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
5139	{
5140	/*
5141	* Use the combined guest and host DRx values found in the hypervisor register set
5142	* because the hypervisor debugger has breakpoints active or someone is single stepping
5143	* on the host side without a monitor trap flag.
5144	*
5145	* Note! DBGF expects a clean DR6 state before executing guest code.
5146	*/
5147	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
5148	if ( CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx)
5149	&& !CPUMIsHyperDebugStateActivePending(pVCpu))
5150	{
5151	CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
5152	Assert(CPUMIsHyperDebugStateActivePending(pVCpu));
5153	Assert(!CPUMIsGuestDebugStateActivePending(pVCpu));
5154	}
5155	else
5156	#endif
5157	if (!CPUMIsHyperDebugStateActive(pVCpu))
5158	{
5159	CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
5160	Assert(CPUMIsHyperDebugStateActive(pVCpu));
5161	Assert(!CPUMIsGuestDebugStateActive(pVCpu));
5162	}
5163
5164	/* Update DR7 with the hypervisor value (other DRx registers are handled by CPUM one way or another). */
5165	u32GuestDr7 = (uint32_t)CPUMGetHyperDR7(pVCpu);
5166	pVCpu->hm.s.fUsingHyperDR7 = true;
5167	fInterceptMovDRx = true;
5168	}
5169	else
5170	{
5171	/*
5172	* If the guest has enabled debug registers, we need to load them prior to
5173	* executing guest code so they'll trigger at the right time.
5174	*/
5175	if (pVCpu->cpum.GstCtx.dr[7] & (X86_DR7_ENABLED_MASK \| X86_DR7_GD))
5176	{
5177	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
5178	if ( CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx)
5179	&& !CPUMIsGuestDebugStateActivePending(pVCpu))
5180	{
5181	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
5182	Assert(CPUMIsGuestDebugStateActivePending(pVCpu));
5183	Assert(!CPUMIsHyperDebugStateActivePending(pVCpu));
5184	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
5185	}
5186	else
5187	#endif
5188	if (!CPUMIsGuestDebugStateActive(pVCpu))
5189	{
5190	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
5191	Assert(CPUMIsGuestDebugStateActive(pVCpu));
5192	Assert(!CPUMIsHyperDebugStateActive(pVCpu));
5193	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
5194	}
5195	Assert(!fInterceptMovDRx);
5196	}
5197	/*
5198	* If no debugging enabled, we'll lazy load DR0-3. Unlike on AMD-V, we
5199	* must intercept #DB in order to maintain a correct DR6 guest value, and
5200	* because we need to intercept it to prevent nested #DBs from hanging the
5201	* CPU, we end up always having to intercept it. See hmR0VmxSetupVmcsXcptBitmap().
5202	*/
5203	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
5204	else if ( !CPUMIsGuestDebugStateActivePending(pVCpu)
5205	&& !CPUMIsGuestDebugStateActive(pVCpu))
5206	#else
5207	else if (!CPUMIsGuestDebugStateActive(pVCpu))
5208	#endif
5209	{
5210	fInterceptMovDRx = true;
5211	}
5212
5213	/* Update DR7 with the actual guest value. */
5214	u32GuestDr7 = pVCpu->cpum.GstCtx.dr[7];
5215	pVCpu->hm.s.fUsingHyperDR7 = false;
5216	}
5217
5218	if (fInterceptMovDRx)
5219	uProcCtls \|= VMX_PROC_CTLS_MOV_DR_EXIT;
5220	else
5221	uProcCtls &= ~VMX_PROC_CTLS_MOV_DR_EXIT;
5222
5223	/*
5224	* Update the processor-based VM-execution controls with the MOV-DRx intercepts and the
5225	* monitor-trap flag and update our cache.
5226	*/
5227	if (uProcCtls != pVmcsInfo->u32ProcCtls)
5228	{
5229	int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
5230	AssertRCReturn(rc2, rc2);
5231	pVmcsInfo->u32ProcCtls = uProcCtls;
5232	}
5233
5234	/*
5235	* Update guest DR7.
5236	*/
5237	int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, u32GuestDr7);
5238	AssertRCReturn(rc, rc);
5239
5240	/*
5241	* If we have forced EFLAGS.TF to be set because we're single-stepping in the hypervisor debugger,
5242	* we need to clear interrupt inhibition if any as otherwise it causes a VM-entry failure.
5243	*
5244	* See Intel spec. 26.3.1.5 "Checks on Guest Non-Register State".
5245	*/
5246	if (fSteppingDB)
5247	{
5248	Assert(pVCpu->hm.s.fSingleInstruction);
5249	Assert(pVCpu->cpum.GstCtx.eflags.Bits.u1TF);
5250
5251	uint32_t fIntrState = 0;
5252	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &fIntrState);
5253	AssertRCReturn(rc, rc);
5254
5255	if (fIntrState & (VMX_VMCS_GUEST_INT_STATE_BLOCK_STI \| VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS))
5256	{
5257	fIntrState &= ~(VMX_VMCS_GUEST_INT_STATE_BLOCK_STI \| VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS);
5258	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INT_STATE, fIntrState);
5259	AssertRCReturn(rc, rc);
5260	}
5261	}
5262
5263	return VINF_SUCCESS;
5264	}
5265
5266
5267	#ifdef VBOX_STRICT
5268	/**
5269	* Strict function to validate segment registers.
5270	*
5271	* @param pVCpu The cross context virtual CPU structure.
5272	* @param pVmcsInfo The VMCS info. object.
5273	*
5274	* @remarks Will import guest CR0 on strict builds during validation of
5275	* segments.
5276	*/
5277	static void hmR0VmxValidateSegmentRegs(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
5278	{
5279	/*
5280	* Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
5281	*
5282	* The reason we check for attribute value 0 in this function and not just the unusable bit is
5283	* because hmR0VmxExportGuestSegReg() only updates the VMCS' copy of the value with the
5284	* unusable bit and doesn't change the guest-context value.
5285	*/
5286	PVM pVM = pVCpu->CTX_SUFF(pVM);
5287	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
5288	hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_CR0);
5289	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
5290	&& ( !CPUMIsGuestInRealModeEx(pCtx)
5291	&& !CPUMIsGuestInV86ModeEx(pCtx)))
5292	{
5293	/* Protected mode checks */
5294	/* CS */
5295	Assert(pCtx->cs.Attr.n.u1Present);
5296	Assert(!(pCtx->cs.Attr.u & 0xf00));
5297	Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
5298	Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
5299	\|\| !(pCtx->cs.Attr.n.u1Granularity));
5300	Assert( !(pCtx->cs.u32Limit & 0xfff00000)
5301	\|\| (pCtx->cs.Attr.n.u1Granularity));
5302	/* CS cannot be loaded with NULL in protected mode. */
5303	Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE)); /** @todo is this really true even for 64-bit CS? */
5304	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
5305	Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
5306	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
5307	Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
5308	else
5309	AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
5310	/* SS */
5311	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
5312	Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
5313	if ( !(pCtx->cr0 & X86_CR0_PE)
5314	\|\| pCtx->cs.Attr.n.u4Type == 3)
5315	{
5316	Assert(!pCtx->ss.Attr.n.u2Dpl);
5317	}
5318	if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
5319	{
5320	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
5321	Assert(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7);
5322	Assert(pCtx->ss.Attr.n.u1Present);
5323	Assert(!(pCtx->ss.Attr.u & 0xf00));
5324	Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
5325	Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
5326	\|\| !(pCtx->ss.Attr.n.u1Granularity));
5327	Assert( !(pCtx->ss.u32Limit & 0xfff00000)
5328	\|\| (pCtx->ss.Attr.n.u1Granularity));
5329	}
5330	/* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxExportGuestSegReg(). */
5331	if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
5332	{
5333	Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
5334	Assert(pCtx->ds.Attr.n.u1Present);
5335	Assert(pCtx->ds.Attr.n.u4Type > 11 \|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
5336	Assert(!(pCtx->ds.Attr.u & 0xf00));
5337	Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
5338	Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
5339	\|\| !(pCtx->ds.Attr.n.u1Granularity));
5340	Assert( !(pCtx->ds.u32Limit & 0xfff00000)
5341	\|\| (pCtx->ds.Attr.n.u1Granularity));
5342	Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
5343	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
5344	}
5345	if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
5346	{
5347	Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
5348	Assert(pCtx->es.Attr.n.u1Present);
5349	Assert(pCtx->es.Attr.n.u4Type > 11 \|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
5350	Assert(!(pCtx->es.Attr.u & 0xf00));
5351	Assert(!(pCtx->es.Attr.u & 0xfffe0000));
5352	Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
5353	\|\| !(pCtx->es.Attr.n.u1Granularity));
5354	Assert( !(pCtx->es.u32Limit & 0xfff00000)
5355	\|\| (pCtx->es.Attr.n.u1Granularity));
5356	Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
5357	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
5358	}
5359	if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
5360	{
5361	Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
5362	Assert(pCtx->fs.Attr.n.u1Present);
5363	Assert(pCtx->fs.Attr.n.u4Type > 11 \|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
5364	Assert(!(pCtx->fs.Attr.u & 0xf00));
5365	Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
5366	Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
5367	\|\| !(pCtx->fs.Attr.n.u1Granularity));
5368	Assert( !(pCtx->fs.u32Limit & 0xfff00000)
5369	\|\| (pCtx->fs.Attr.n.u1Granularity));
5370	Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
5371	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
5372	}
5373	if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
5374	{
5375	Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
5376	Assert(pCtx->gs.Attr.n.u1Present);
5377	Assert(pCtx->gs.Attr.n.u4Type > 11 \|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
5378	Assert(!(pCtx->gs.Attr.u & 0xf00));
5379	Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
5380	Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
5381	\|\| !(pCtx->gs.Attr.n.u1Granularity));
5382	Assert( !(pCtx->gs.u32Limit & 0xfff00000)
5383	\|\| (pCtx->gs.Attr.n.u1Granularity));
5384	Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
5385	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
5386	}
5387	/* 64-bit capable CPUs. */
5388	# if HC_ARCH_BITS == 64
5389	Assert(!RT_HI_U32(pCtx->cs.u64Base));
5390	Assert(!pCtx->ss.Attr.u \|\| !RT_HI_U32(pCtx->ss.u64Base));
5391	Assert(!pCtx->ds.Attr.u \|\| !RT_HI_U32(pCtx->ds.u64Base));
5392	Assert(!pCtx->es.Attr.u \|\| !RT_HI_U32(pCtx->es.u64Base));
5393	# endif
5394	}
5395	else if ( CPUMIsGuestInV86ModeEx(pCtx)
5396	\|\| ( CPUMIsGuestInRealModeEx(pCtx)
5397	&& !pVM->hm.s.vmx.fUnrestrictedGuest))
5398	{
5399	/* Real and v86 mode checks. */
5400	/* hmR0VmxExportGuestSegReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
5401	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
5402	if (pVmcsInfo->RealMode.fRealOnV86Active)
5403	{
5404	u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3;
5405	u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
5406	}
5407	else
5408	{
5409	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
5410	u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
5411	}
5412
5413	/* CS */
5414	AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
5415	Assert(pCtx->cs.u32Limit == 0xffff);
5416	Assert(u32CSAttr == 0xf3);
5417	/* SS */
5418	Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
5419	Assert(pCtx->ss.u32Limit == 0xffff);
5420	Assert(u32SSAttr == 0xf3);
5421	/* DS */
5422	Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
5423	Assert(pCtx->ds.u32Limit == 0xffff);
5424	Assert(u32DSAttr == 0xf3);
5425	/* ES */
5426	Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
5427	Assert(pCtx->es.u32Limit == 0xffff);
5428	Assert(u32ESAttr == 0xf3);
5429	/* FS */
5430	Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
5431	Assert(pCtx->fs.u32Limit == 0xffff);
5432	Assert(u32FSAttr == 0xf3);
5433	/* GS */
5434	Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
5435	Assert(pCtx->gs.u32Limit == 0xffff);
5436	Assert(u32GSAttr == 0xf3);
5437	/* 64-bit capable CPUs. */
5438	# if HC_ARCH_BITS == 64
5439	Assert(!RT_HI_U32(pCtx->cs.u64Base));
5440	Assert(!u32SSAttr \|\| !RT_HI_U32(pCtx->ss.u64Base));
5441	Assert(!u32DSAttr \|\| !RT_HI_U32(pCtx->ds.u64Base));
5442	Assert(!u32ESAttr \|\| !RT_HI_U32(pCtx->es.u64Base));
5443	# endif
5444	}
5445	}
5446	#endif /* VBOX_STRICT */
5447
5448
5449	/**
5450	* Exports a guest segment register into the guest-state area in the VMCS.
5451	*
5452	* @returns VBox status code.
5453	* @param pVCpu The cross context virtual CPU structure.
5454	* @param pVmcsInfo The VMCS info. object.
5455	* @param iSegReg The segment register number (X86_SREG_XXX).
5456	* @param pSelReg Pointer to the segment selector.
5457	*
5458	* @remarks No-long-jump zone!!!
5459	*/
5460	static int hmR0VmxExportGuestSegReg(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo, uint8_t iSegReg, PCCPUMSELREG pSelReg)
5461	{
5462	Assert(iSegReg < X86_SREG_COUNT);
5463	uint32_t const idxSel = g_aVmcsSegSel[iSegReg];
5464	uint32_t const idxLimit = g_aVmcsSegLimit[iSegReg];
5465	uint32_t const idxBase = g_aVmcsSegBase[iSegReg];
5466	uint32_t const idxAttr = g_aVmcsSegAttr[iSegReg];
5467
5468	uint32_t u32Access = pSelReg->Attr.u;
5469	if (pVmcsInfo->RealMode.fRealOnV86Active)
5470	{
5471	/* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
5472	u32Access = 0xf3;
5473	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
5474	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
5475	RT_NOREF_PV(pVCpu);
5476	}
5477	else
5478	{
5479	/*
5480	* The way to differentiate between whether this is really a null selector or was just
5481	* a selector loaded with 0 in real-mode is using the segment attributes. A selector
5482	* loaded in real-mode with the value 0 is valid and usable in protected-mode and we
5483	* should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures
5484	* NULL selectors loaded in protected-mode have their attribute as 0.
5485	*/
5486	if (!u32Access)
5487	u32Access = X86DESCATTR_UNUSABLE;
5488	}
5489
5490	/* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
5491	AssertMsg((u32Access & X86DESCATTR_UNUSABLE) \|\| (u32Access & X86_SEL_TYPE_ACCESSED),
5492	("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
5493
5494	/*
5495	* Commit it to the VMCS.
5496	*/
5497	int rc = VMXWriteVmcs32(idxSel, pSelReg->Sel);
5498	rc \|= VMXWriteVmcs32(idxLimit, pSelReg->u32Limit);
5499	rc \|= VMXWriteVmcsGstN(idxBase, pSelReg->u64Base);
5500	rc \|= VMXWriteVmcs32(idxAttr, u32Access);
5501	AssertRCReturn(rc, rc);
5502	return rc;
5503	}
5504
5505
5506	/**
5507	* Exports the guest segment registers, GDTR, IDTR, LDTR, TR into the guest-state
5508	* area in the VMCS.
5509	*
5510	* @returns VBox status code.
5511	* @param pVCpu The cross context virtual CPU structure.
5512	* @param pVmxTransient The VMX-transient structure.
5513	*
5514	* @remarks Will import guest CR0 on strict builds during validation of
5515	* segments.
5516	* @remarks No-long-jump zone!!!
5517	*/
5518	static int hmR0VmxExportGuestSegRegsXdtr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
5519	{
5520	int rc = VERR_INTERNAL_ERROR_5;
5521	PVM pVM = pVCpu->CTX_SUFF(pVM);
5522	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
5523	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
5524
5525	/*
5526	* Guest Segment registers: CS, SS, DS, ES, FS, GS.
5527	*/
5528	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SREG_MASK)
5529	{
5530	#ifdef VBOX_WITH_REM
5531	if (!pVM->hm.s.vmx.fUnrestrictedGuest)
5532	{
5533	Assert(!pVmxTransient->fIsNestedGuest);
5534	Assert(pVM->hm.s.vmx.pRealModeTSS);
5535	AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
5536	if ( pVmcsInfo->fWasInRealMode
5537	&& PGMGetGuestMode(pVCpu) >= PGMMODE_PROTECTED)
5538	{
5539	/* Signal that the recompiler must flush its code-cache as the guest -may- rewrite code it will later execute
5540	in real-mode (e.g. OpenBSD 4.0) */
5541	REMFlushTBs(pVM);
5542	Log4Func(("Switch to protected mode detected!\n"));
5543	pVmcsInfo->fWasInRealMode = false;
5544	}
5545	}
5546	#endif
5547	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CS)
5548	{
5549	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CS);
5550	if (pVmcsInfo->RealMode.fRealOnV86Active)
5551	pVmcsInfo->RealMode.AttrCS.u = pCtx->cs.Attr.u;
5552	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_CS, &pCtx->cs);
5553	AssertRCReturn(rc, rc);
5554	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CS);
5555	}
5556
5557	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SS)
5558	{
5559	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SS);
5560	if (pVmcsInfo->RealMode.fRealOnV86Active)
5561	pVmcsInfo->RealMode.AttrSS.u = pCtx->ss.Attr.u;
5562	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_SS, &pCtx->ss);
5563	AssertRCReturn(rc, rc);
5564	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SS);
5565	}
5566
5567	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_DS)
5568	{
5569	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_DS);
5570	if (pVmcsInfo->RealMode.fRealOnV86Active)
5571	pVmcsInfo->RealMode.AttrDS.u = pCtx->ds.Attr.u;
5572	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_DS, &pCtx->ds);
5573	AssertRCReturn(rc, rc);
5574	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_DS);
5575	}
5576
5577	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_ES)
5578	{
5579	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_ES);
5580	if (pVmcsInfo->RealMode.fRealOnV86Active)
5581	pVmcsInfo->RealMode.AttrES.u = pCtx->es.Attr.u;
5582	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_ES, &pCtx->es);
5583	AssertRCReturn(rc, rc);
5584	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_ES);
5585	}
5586
5587	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_FS)
5588	{
5589	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_FS);
5590	if (pVmcsInfo->RealMode.fRealOnV86Active)
5591	pVmcsInfo->RealMode.AttrFS.u = pCtx->fs.Attr.u;
5592	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_FS, &pCtx->fs);
5593	AssertRCReturn(rc, rc);
5594	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_FS);
5595	}
5596
5597	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_GS)
5598	{
5599	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_GS);
5600	if (pVmcsInfo->RealMode.fRealOnV86Active)
5601	pVmcsInfo->RealMode.AttrGS.u = pCtx->gs.Attr.u;
5602	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_GS, &pCtx->gs);
5603	AssertRCReturn(rc, rc);
5604	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_GS);
5605	}
5606
5607	#ifdef VBOX_STRICT
5608	hmR0VmxValidateSegmentRegs(pVCpu, pVmcsInfo);
5609	#endif
5610	Log4Func(("cs={%#04x base=%#RX64 limit=%#RX32 attr=%#RX32}\n", pCtx->cs.Sel, pCtx->cs.u64Base, pCtx->cs.u32Limit,
5611	pCtx->cs.Attr.u));
5612	}
5613
5614	/*
5615	* Guest TR.
5616	*/
5617	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_TR)
5618	{
5619	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_TR);
5620
5621	/*
5622	* Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is
5623	* achieved using the interrupt redirection bitmap (all bits cleared to let the guest
5624	* handle INT-n's) in the TSS. See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
5625	*/
5626	uint16_t u16Sel;
5627	uint32_t u32Limit;
5628	uint64_t u64Base;
5629	uint32_t u32AccessRights;
5630	if (!pVmcsInfo->RealMode.fRealOnV86Active)
5631	{
5632	u16Sel = pCtx->tr.Sel;
5633	u32Limit = pCtx->tr.u32Limit;
5634	u64Base = pCtx->tr.u64Base;
5635	u32AccessRights = pCtx->tr.Attr.u;
5636	}
5637	else
5638	{
5639	Assert(!pVmxTransient->fIsNestedGuest);
5640	Assert(pVM->hm.s.vmx.pRealModeTSS);
5641	Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMCanExecuteGuest() -XXX- what about inner loop changes? */
5642
5643	/* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
5644	RTGCPHYS GCPhys;
5645	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
5646	AssertRCReturn(rc, rc);
5647
5648	X86DESCATTR DescAttr;
5649	DescAttr.u = 0;
5650	DescAttr.n.u1Present = 1;
5651	DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
5652
5653	u16Sel = 0;
5654	u32Limit = HM_VTX_TSS_SIZE;
5655	u64Base = GCPhys;
5656	u32AccessRights = DescAttr.u;
5657	}
5658
5659	/* Validate. */
5660	Assert(!(u16Sel & RT_BIT(2)));
5661	AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
5662	\|\| (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
5663	AssertMsg(!(u32AccessRights & X86DESCATTR_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
5664	Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
5665	Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
5666	Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
5667	Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
5668	Assert( (u32Limit & 0xfff) == 0xfff
5669	\|\| !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
5670	Assert( !(pCtx->tr.u32Limit & 0xfff00000)
5671	\|\| (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
5672
5673	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_TR_SEL, u16Sel);
5674	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit);
5675	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights);
5676	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base);
5677	AssertRCReturn(rc, rc);
5678
5679	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_TR);
5680	Log4Func(("tr base=%#RX64 limit=%#RX32\n", pCtx->tr.u64Base, pCtx->tr.u32Limit));
5681	}
5682
5683	/*
5684	* Guest GDTR.
5685	*/
5686	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_GDTR)
5687	{
5688	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_GDTR);
5689
5690	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pCtx->gdtr.cbGdt);
5691	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pCtx->gdtr.pGdt);
5692	AssertRCReturn(rc, rc);
5693
5694	/* Validate. */
5695	Assert(!(pCtx->gdtr.cbGdt & 0xffff0000)); /* Bits 31:16 MBZ. */
5696
5697	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_GDTR);
5698	Log4Func(("gdtr base=%#RX64 limit=%#RX32\n", pCtx->gdtr.pGdt, pCtx->gdtr.cbGdt));
5699	}
5700
5701	/*
5702	* Guest LDTR.
5703	*/
5704	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_LDTR)
5705	{
5706	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_LDTR);
5707
5708	/* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
5709	uint32_t u32Access;
5710	if ( !pVmxTransient->fIsNestedGuest
5711	&& !pCtx->ldtr.Attr.u)
5712	u32Access = X86DESCATTR_UNUSABLE;
5713	else
5714	u32Access = pCtx->ldtr.Attr.u;
5715
5716	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_LDTR_SEL, pCtx->ldtr.Sel);
5717	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pCtx->ldtr.u32Limit);
5718	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access);
5719	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pCtx->ldtr.u64Base);
5720	AssertRCReturn(rc, rc);
5721
5722	/* Validate. */
5723	if (!(u32Access & X86DESCATTR_UNUSABLE))
5724	{
5725	Assert(!(pCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
5726	Assert(pCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
5727	Assert(!pCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
5728	Assert(pCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
5729	Assert(!pCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
5730	Assert(!(pCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
5731	Assert( (pCtx->ldtr.u32Limit & 0xfff) == 0xfff
5732	\|\| !pCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
5733	Assert( !(pCtx->ldtr.u32Limit & 0xfff00000)
5734	\|\| pCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
5735	}
5736
5737	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_LDTR);
5738	Log4Func(("ldtr base=%#RX64 limit=%#RX32\n", pCtx->ldtr.u64Base, pCtx->ldtr.u32Limit));
5739	}
5740
5741	/*
5742	* Guest IDTR.
5743	*/
5744	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_IDTR)
5745	{
5746	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_IDTR);
5747
5748	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pCtx->idtr.cbIdt);
5749	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pCtx->idtr.pIdt);
5750	AssertRCReturn(rc, rc);
5751
5752	/* Validate. */
5753	Assert(!(pCtx->idtr.cbIdt & 0xffff0000)); /* Bits 31:16 MBZ. */
5754
5755	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_IDTR);
5756	Log4Func(("idtr base=%#RX64 limit=%#RX32\n", pCtx->idtr.pIdt, pCtx->idtr.cbIdt));
5757	}
5758
5759	return VINF_SUCCESS;
5760	}
5761
5762
5763	/**
5764	* Exports certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
5765	* areas.
5766	*
5767	* These MSRs will automatically be loaded to the host CPU on every successful
5768	* VM-entry and stored from the host CPU on every successful VM-exit.
5769	*
5770	* We creates/updates MSR slots for the host MSRs in the VM-exit MSR-load area. The
5771	* actual host MSR values are not- updated here for performance reasons. See
5772	* hmR0VmxExportHostMsrs().
5773	*
5774	* We also exports the guest sysenter MSRs into the guest-state area in the VMCS.
5775	*
5776	* @returns VBox status code.
5777	* @param pVCpu The cross context virtual CPU structure.
5778	* @param pVmxTransient The VMX-transient structure.
5779	*
5780	* @remarks No-long-jump zone!!!
5781	*/
5782	static int hmR0VmxExportGuestMsrs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
5783	{
5784	AssertPtr(pVCpu);
5785	AssertPtr(pVmxTransient);
5786
5787	PVM pVM = pVCpu->CTX_SUFF(pVM);
5788	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
5789
5790	/*
5791	* MSRs that we use the auto-load/store MSR area in the VMCS.
5792	* For 64-bit hosts, we load/restore them lazily, see hmR0VmxLazyLoadGuestMsrs().
5793	* The host MSR values are updated when it's safe in hmR0VmxLazySaveHostMsrs().
5794	*
5795	* For nested-guests, the guests MSRs from the VM-entry MSR-load area are already
5796	* loaded (into the guest-CPU context) by the VMLAUNCH/VMRESUME instruction
5797	* emulation, nothing to do here.
5798	*/
5799	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_GUEST_AUTO_MSRS)
5800	{
5801	if ( !pVmxTransient->fIsNestedGuest
5802	&& pVM->hm.s.fAllow64BitGuests)
5803	{
5804	#if HC_ARCH_BITS == 32
5805	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SYSCALL_MSRS \| CPUMCTX_EXTRN_KERNEL_GS_BASE);
5806	Assert(!pVmxTransient->fIsNestedGuest);
5807
5808	int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K8_LSTAR, pCtx->msrLSTAR, true, false);
5809	rc \|= hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K6_STAR, pCtx->msrSTAR, true, false);
5810	rc \|= hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K8_SF_MASK, pCtx->msrSFMASK, true, false);
5811	rc \|= hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K8_KERNEL_GS_BASE, pCtx->msrKERNELGSBASE, true, false);
5812	AssertRCReturn(rc, rc);
5813	#endif
5814	}
5815	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_GUEST_AUTO_MSRS);
5816	}
5817
5818	/*
5819	* Guest Sysenter MSRs.
5820	*/
5821	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_MSR_MASK)
5822	{
5823	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SYSENTER_MSRS);
5824
5825	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_CS_MSR)
5826	{
5827	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pCtx->SysEnter.cs);
5828	AssertRCReturn(rc, rc);
5829	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_CS_MSR);
5830	}
5831
5832	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_EIP_MSR)
5833	{
5834	int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pCtx->SysEnter.eip);
5835	AssertRCReturn(rc, rc);
5836	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_EIP_MSR);
5837	}
5838
5839	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_ESP_MSR)
5840	{
5841	int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pCtx->SysEnter.esp);
5842	AssertRCReturn(rc, rc);
5843	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_ESP_MSR);
5844	}
5845	}
5846
5847	/*
5848	* Guest/host EFER MSR.
5849	*/
5850	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_EFER_MSR)
5851	{
5852	/* Whether we are using the VMCS to swap the EFER MSR must have been
5853	determined earlier while exporting VM-entry/VM-exit controls. */
5854	Assert(!(ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_ENTRY_EXIT_CTLS));
5855	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_EFER);
5856
5857	if (hmR0VmxShouldSwapEferMsr(pVCpu))
5858	{
5859	/*
5860	* If the CPU supports VMCS controls for swapping EFER, use it. Otherwise, we have no option
5861	* but to use the auto-load store MSR area in the VMCS for swapping EFER. See @bugref{7368}.
5862	*/
5863	if (pVM->hm.s.vmx.fSupportsVmcsEfer)
5864	{
5865	int rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_EFER_FULL, pCtx->msrEFER);
5866	AssertRCReturn(rc, rc);
5867	}
5868	else
5869	{
5870	/*
5871	* We shall use the auto-load/store MSR area only for loading the EFER MSR but we must
5872	* continue to intercept guest read and write accesses to it, see @bugref{7386#c16}.
5873	*/
5874	int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K6_EFER, pCtx->msrEFER,
5875	false /* fSetReadWrite /, false / fUpdateHostMsr */);
5876	AssertRCReturn(rc, rc);
5877	}
5878	}
5879	else if (!pVM->hm.s.vmx.fSupportsVmcsEfer)
5880	hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K6_EFER);
5881
5882	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_EFER_MSR);
5883	}
5884
5885	/*
5886	* Other MSRs.
5887	* Speculation Control (R/W).
5888	*/
5889	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_OTHER_MSRS)
5890	{
5891	HMVMX_CPUMCTX_ASSERT(pVCpu, HM_CHANGED_GUEST_OTHER_MSRS);
5892	if (pVM->cpum.ro.GuestFeatures.fIbrs)
5893	{
5894	int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_IA32_SPEC_CTRL, CPUMGetGuestSpecCtrl(pVCpu),
5895	false /* fSetReadWrite /, false / fUpdateHostMsr */);
5896	AssertRCReturn(rc, rc);
5897	}
5898	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_OTHER_MSRS);
5899	}
5900
5901	return VINF_SUCCESS;
5902	}
5903
5904
5905	/**
5906	* Selects up the appropriate function to run guest code.
5907	*
5908	* @returns VBox status code.
5909	* @param pVCpu The cross context virtual CPU structure.
5910	* @param pVmxTransient The VMX-transient structure.
5911	*
5912	* @remarks No-long-jump zone!!!
5913	*/
5914	static int hmR0VmxSelectVMRunHandler(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
5915	{
5916	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
5917	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
5918
5919	if (CPUMIsGuestInLongModeEx(pCtx))
5920	{
5921	#ifndef VBOX_ENABLE_64_BITS_GUESTS
5922	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
5923	#endif
5924	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
5925	#if HC_ARCH_BITS == 32
5926	/* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
5927	if (pVmcsInfo->pfnStartVM != VMXR0SwitcherStartVM64)
5928	{
5929	#ifdef VBOX_STRICT
5930	if (pVmcsInfo->pfnStartVM != NULL) /* Very first VM-entry would have saved host-state already, ignore it. */
5931	{
5932	/* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
5933	uint64_t const fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
5934	RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
5935	AssertMsg(fCtxChanged & (HM_CHANGED_VMX_ENTRY_EXIT_CTLS \| HM_CHANGED_GUEST_EFER_MSR),
5936	("fCtxChanged=%#RX64\n", fCtxChanged));
5937	}
5938	#endif
5939	pVmcsInfo->pfnStartVM = VMXR0SwitcherStartVM64;
5940
5941	/* Mark that we've switched to 64-bit handler, we can't safely switch back to 32-bit for
5942	the rest of the VM run (until VM reset). See @bugref{8432#c7}. */
5943	pVmcsInfo->fSwitchedTo64on32 = true;
5944	Log4Func(("Selected 64-bit switcher\n"));
5945	}
5946	#else
5947	/* 64-bit host. */
5948	pVmcsInfo->pfnStartVM = VMXR0StartVM64;
5949	#endif
5950	}
5951	else
5952	{
5953	/* Guest is not in long mode, use the 32-bit handler. */
5954	#if HC_ARCH_BITS == 32
5955	if ( pVmcsInfo->pfnStartVM != VMXR0StartVM32
5956	&& !pVmcsInfo->fSwitchedTo64on32 /* If set, guest mode change does not imply switcher change. */
5957	&& pVmcsInfo->pfnStartVM != NULL) /* Very first VM-entry would have saved host-state already, ignore it. */
5958	{
5959	# ifdef VBOX_STRICT
5960	/* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
5961	uint64_t const fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
5962	RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
5963	AssertMsg(fCtxChanged & (HM_CHANGED_VMX_ENTRY_EXIT_CTLS \| HM_CHANGED_GUEST_EFER_MSR),
5964	("fCtxChanged=%#RX64\n", fCtxChanged));
5965	# endif
5966	}
5967	# ifdef VBOX_ENABLE_64_BITS_GUESTS
5968	/*
5969	* Keep using the 64-bit switcher even though we're in 32-bit because of bad Intel
5970	* design, see @bugref{8432#c7}. If real-on-v86 mode is active, clear the 64-bit
5971	* switcher flag now because we know the guest is in a sane state where it's safe
5972	* to use the 32-bit switcher. Otherwise, check the guest state if it's safe to use
5973	* the much faster 32-bit switcher again.
5974	*/
5975	if (!pVmcsInfo->fSwitchedTo64on32)
5976	{
5977	if (pVmcsInfo->pfnStartVM != VMXR0StartVM32)
5978	Log4Func(("Selected 32-bit switcher\n"));
5979	pVmcsInfo->pfnStartVM = VMXR0StartVM32;
5980	}
5981	else
5982	{
5983	Assert(pVmcsInfo->pfnStartVM == VMXR0SwitcherStartVM64);
5984	if ( pVmcsInfo->RealMode.fRealOnV86Active
5985	\|\| hmR0VmxIs32BitSwitcherSafe(pCtx))
5986	{
5987	pVmcsInfo->fSwitchedTo64on32 = false;
5988	pVmcsInfo->pfnStartVM = VMXR0StartVM32;
5989	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_EFER_MSR
5990	\| HM_CHANGED_VMX_ENTRY_EXIT_CTLS
5991	\| HM_CHANGED_HOST_CONTEXT);
5992	Log4Func(("Selected 32-bit switcher (safe)\n"));
5993	}
5994	}
5995	# else
5996	pVmcsInfo->pfnStartVM = VMXR0StartVM32;
5997	# endif
5998	#else
5999	pVmcsInfo->pfnStartVM = VMXR0StartVM32;
6000	#endif
6001	}
6002	Assert(pVmcsInfo->pfnStartVM);
6003	return VINF_SUCCESS;
6004	}
6005
6006
6007	/**
6008	* Wrapper for running the guest code in VT-x.
6009	*
6010	* @returns VBox status code, no informational status codes.
6011	* @param pVCpu The cross context virtual CPU structure.
6012	* @param pVmxTransient The VMX-transient structure.
6013	*
6014	* @remarks No-long-jump zone!!!
6015	*/
6016	DECLINLINE(int) hmR0VmxRunGuest(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
6017	{
6018	/* Mark that HM is the keeper of all guest-CPU registers now that we're going to execute guest code. */
6019	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6020	pCtx->fExtrn \|= HMVMX_CPUMCTX_EXTRN_ALL \| CPUMCTX_EXTRN_KEEPER_HM;
6021
6022	/** @todo Add stats for VMRESUME vs VMLAUNCH. */
6023
6024	/*
6025	* 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses
6026	* floating-point operations using SSE instructions. Some XMM registers (XMM6-XMM15) are
6027	* callee-saved and thus the need for this XMM wrapper.
6028	*
6029	* See MSDN "Configuring Programs for 64-bit/x64 Software Conventions / Register Usage".
6030	*/
6031	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
6032	bool const fResumeVM = RT_BOOL(pVmcsInfo->fVmcsState & VMX_V_VMCS_LAUNCH_STATE_LAUNCHED);
6033	PVM pVM = pVCpu->CTX_SUFF(pVM);
6034	#ifdef VBOX_WITH_KERNEL_USING_XMM
6035	int rc = hmR0VMXStartVMWrapXMM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VmcsCache, pVM, pVCpu, pVmcsInfo->pfnStartVM);
6036	#else
6037	int rc = pVmcsInfo->pfnStartVM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VmcsCache, pVM, pVCpu);
6038	#endif
6039	AssertMsg(rc <= VINF_SUCCESS, ("%Rrc\n", rc));
6040	return rc;
6041	}
6042
6043
6044	/**
6045	* Reports world-switch error and dumps some useful debug info.
6046	*
6047	* @param pVCpu The cross context virtual CPU structure.
6048	* @param rcVMRun The return code from VMLAUNCH/VMRESUME.
6049	* @param pVmxTransient The VMX-transient structure (only
6050	* exitReason updated).
6051	*/
6052	static void hmR0VmxReportWorldSwitchError(PVMCPU pVCpu, int rcVMRun, PVMXTRANSIENT pVmxTransient)
6053	{
6054	Assert(pVCpu);
6055	Assert(pVmxTransient);
6056	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
6057
6058	Log4Func(("VM-entry failure: %Rrc\n", rcVMRun));
6059	switch (rcVMRun)
6060	{
6061	case VERR_VMX_INVALID_VMXON_PTR:
6062	AssertFailed();
6063	break;
6064	case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
6065	case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
6066	{
6067	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.LastError.u32ExitReason);
6068	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
6069	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
6070	AssertRC(rc);
6071
6072	pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
6073	/* LastError.idCurrentCpu was already updated in hmR0VmxPreRunGuestCommitted().
6074	Cannot do it here as we may have been long preempted. */
6075
6076	#ifdef VBOX_STRICT
6077	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
6078	Log4(("uExitReason %#RX32 (VmxTransient %#RX16)\n", pVCpu->hm.s.vmx.LastError.u32ExitReason,
6079	pVmxTransient->uExitReason));
6080	Log4(("Exit Qualification %#RX64\n", pVmxTransient->uExitQual));
6081	Log4(("InstrError %#RX32\n", pVCpu->hm.s.vmx.LastError.u32InstrError));
6082	if (pVCpu->hm.s.vmx.LastError.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
6083	Log4(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.LastError.u32InstrError]));
6084	else
6085	Log4(("InstrError Desc. Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
6086	Log4(("Entered host CPU %u\n", pVCpu->hm.s.vmx.LastError.idEnteredCpu));
6087	Log4(("Current host CPU %u\n", pVCpu->hm.s.vmx.LastError.idCurrentCpu));
6088
6089	/* VMX control bits. */
6090	uint32_t u32Val;
6091	uint64_t u64Val;
6092	RTHCUINTREG uHCReg;
6093	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val); AssertRC(rc);
6094	Log4(("VMX_VMCS32_CTRL_PIN_EXEC %#RX32\n", u32Val));
6095	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val); AssertRC(rc);
6096	Log4(("VMX_VMCS32_CTRL_PROC_EXEC %#RX32\n", u32Val));
6097	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
6098	{
6099	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val); AssertRC(rc);
6100	Log4(("VMX_VMCS32_CTRL_PROC_EXEC2 %#RX32\n", u32Val));
6101	}
6102	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val); AssertRC(rc);
6103	Log4(("VMX_VMCS32_CTRL_ENTRY %#RX32\n", u32Val));
6104	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val); AssertRC(rc);
6105	Log4(("VMX_VMCS32_CTRL_EXIT %#RX32\n", u32Val));
6106	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
6107	Log4(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
6108	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
6109	Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
6110	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
6111	Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
6112	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
6113	Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
6114	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
6115	Log4(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
6116	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
6117	Log4(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
6118	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
6119	Log4(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
6120	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
6121	Log4(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
6122	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
6123	Log4(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
6124	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
6125	Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
6126	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
6127	Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
6128	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
6129	Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
6130	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
6131	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
6132	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
6133	Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
6134	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
6135	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
6136	if (pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging)
6137	{
6138	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
6139	Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
6140	}
6141
6142	/* Guest bits. */
6143	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val); AssertRC(rc);
6144	Log4(("Old Guest Rip %#RX64 New %#RX64\n", pVCpu->cpum.GstCtx.rip, u64Val));
6145	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val); AssertRC(rc);
6146	Log4(("Old Guest Rsp %#RX64 New %#RX64\n", pVCpu->cpum.GstCtx.rsp, u64Val));
6147	rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val); AssertRC(rc);
6148	Log4(("Old Guest Rflags %#RX32 New %#RX32\n", pVCpu->cpum.GstCtx.eflags.u32, u32Val));
6149	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid)
6150	{
6151	rc = VMXReadVmcs32(VMX_VMCS16_VPID, &u32Val); AssertRC(rc);
6152	Log4(("VMX_VMCS16_VPID %u\n", u32Val));
6153	}
6154
6155	/* Host bits. */
6156	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
6157	Log4(("Host CR0 %#RHr\n", uHCReg));
6158	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
6159	Log4(("Host CR3 %#RHr\n", uHCReg));
6160	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
6161	Log4(("Host CR4 %#RHr\n", uHCReg));
6162
6163	RTGDTR HostGdtr;
6164	PCX86DESCHC pDesc;
6165	ASMGetGDTR(&HostGdtr);
6166	rc = VMXReadVmcs32(VMX_VMCS16_HOST_CS_SEL, &u32Val); AssertRC(rc);
6167	Log4(("Host CS %#08x\n", u32Val));
6168	if (u32Val < HostGdtr.cbGdt)
6169	{
6170	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
6171	hmR0DumpDescriptor(pDesc, u32Val, "CS: ");
6172	}
6173
6174	rc = VMXReadVmcs32(VMX_VMCS16_HOST_DS_SEL, &u32Val); AssertRC(rc);
6175	Log4(("Host DS %#08x\n", u32Val));
6176	if (u32Val < HostGdtr.cbGdt)
6177	{
6178	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
6179	hmR0DumpDescriptor(pDesc, u32Val, "DS: ");
6180	}
6181
6182	rc = VMXReadVmcs32(VMX_VMCS16_HOST_ES_SEL, &u32Val); AssertRC(rc);
6183	Log4(("Host ES %#08x\n", u32Val));
6184	if (u32Val < HostGdtr.cbGdt)
6185	{
6186	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
6187	hmR0DumpDescriptor(pDesc, u32Val, "ES: ");
6188	}
6189
6190	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FS_SEL, &u32Val); AssertRC(rc);
6191	Log4(("Host FS %#08x\n", u32Val));
6192	if (u32Val < HostGdtr.cbGdt)
6193	{
6194	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
6195	hmR0DumpDescriptor(pDesc, u32Val, "FS: ");
6196	}
6197
6198	rc = VMXReadVmcs32(VMX_VMCS16_HOST_GS_SEL, &u32Val); AssertRC(rc);
6199	Log4(("Host GS %#08x\n", u32Val));
6200	if (u32Val < HostGdtr.cbGdt)
6201	{
6202	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
6203	hmR0DumpDescriptor(pDesc, u32Val, "GS: ");
6204	}
6205
6206	rc = VMXReadVmcs32(VMX_VMCS16_HOST_SS_SEL, &u32Val); AssertRC(rc);
6207	Log4(("Host SS %#08x\n", u32Val));
6208	if (u32Val < HostGdtr.cbGdt)
6209	{
6210	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
6211	hmR0DumpDescriptor(pDesc, u32Val, "SS: ");
6212	}
6213
6214	rc = VMXReadVmcs32(VMX_VMCS16_HOST_TR_SEL, &u32Val); AssertRC(rc);
6215	Log4(("Host TR %#08x\n", u32Val));
6216	if (u32Val < HostGdtr.cbGdt)
6217	{
6218	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
6219	hmR0DumpDescriptor(pDesc, u32Val, "TR: ");
6220	}
6221
6222	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
6223	Log4(("Host TR Base %#RHv\n", uHCReg));
6224	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
6225	Log4(("Host GDTR Base %#RHv\n", uHCReg));
6226	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
6227	Log4(("Host IDTR Base %#RHv\n", uHCReg));
6228	rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
6229	Log4(("Host SYSENTER CS %#08x\n", u32Val));
6230	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
6231	Log4(("Host SYSENTER EIP %#RHv\n", uHCReg));
6232	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
6233	Log4(("Host SYSENTER ESP %#RHv\n", uHCReg));
6234	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
6235	Log4(("Host RSP %#RHv\n", uHCReg));
6236	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
6237	Log4(("Host RIP %#RHv\n", uHCReg));
6238	# if HC_ARCH_BITS == 64
6239	Log4(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
6240	Log4(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
6241	Log4(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
6242	Log4(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
6243	Log4(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
6244	Log4(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
6245	# endif
6246	#endif /* VBOX_STRICT */
6247	break;
6248	}
6249
6250	default:
6251	/* Impossible */
6252	AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
6253	break;
6254	}
6255	}
6256
6257
6258	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
6259	# ifndef VMX_USE_CACHED_VMCS_ACCESSES
6260	# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
6261	# endif
6262
6263	/**
6264	* Initialize the VMCS-Read cache.
6265	*
6266	* The VMCS cache is used for 32-bit hosts running 64-bit guests (except 32-bit
6267	* Darwin which runs with 64-bit paging in 32-bit mode) for 64-bit fields that
6268	* cannot be accessed in 32-bit mode. Some 64-bit fields -can- be accessed
6269	* (those that have a 32-bit FULL & HIGH part).
6270	*
6271	* @param pVCpu The cross context virtual CPU structure.
6272	*/
6273	static void hmR0VmxInitVmcsReadCache(PVMCPU pVCpu)
6274	{
6275	#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
6276	do { \
6277	Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
6278	pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
6279	pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
6280	++cReadFields; \
6281	} while (0)
6282
6283	PVMXVMCSCACHE pCache = &pVCpu->hm.s.vmx.VmcsCache;
6284	uint32_t cReadFields = 0;
6285
6286	/*
6287	* Don't remove the #if 0'd fields in this code. They're listed here for consistency
6288	* and serve to indicate exceptions to the rules.
6289	*/
6290
6291	/* Guest-natural selector base fields. */
6292	#if 0
6293	/* These are 32-bit in practice. See Intel spec. 2.5 "Control Registers". */
6294	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
6295	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
6296	#endif
6297	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
6298	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
6299	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
6300	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
6301	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
6302	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
6303	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
6304	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
6305	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
6306	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
6307	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
6308	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
6309	#if 0
6310	/* Unused natural width guest-state fields. */
6311	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS);
6312	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in nested paging case */
6313	#endif
6314	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
6315	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
6316
6317	/* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for
6318	these 64-bit fields (using "FULL" and "HIGH" fields). */
6319	#if 0
6320	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
6321	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
6322	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
6323	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
6324	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
6325	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
6326	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
6327	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
6328	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
6329	#endif
6330
6331	/* Natural width guest-state fields. */
6332	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
6333	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_GUEST_LINEAR_ADDR);
6334
6335	if (pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging)
6336	{
6337	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
6338	AssertMsg(cReadFields == VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields,
6339	VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX));
6340	pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
6341	}
6342	else
6343	{
6344	AssertMsg(cReadFields == VMX_VMCS_MAX_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields, VMX_VMCS_MAX_CACHE_IDX));
6345	pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
6346	}
6347
6348	#undef VMXLOCAL_INIT_READ_CACHE_FIELD
6349	}
6350
6351
6352	/**
6353	* Writes a field into the VMCS. This can either directly invoke a VMWRITE or
6354	* queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
6355	* darwin, running 64-bit guests).
6356	*
6357	* @returns VBox status code.
6358	* @param pVCpu The cross context virtual CPU structure.
6359	* @param idxField The VMCS field encoding.
6360	* @param u64Val 16, 32 or 64-bit value.
6361	*/
6362	VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
6363	{
6364	int rc;
6365	switch (idxField)
6366	{
6367	/*
6368	* These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
6369	*/
6370	/* 64-bit Control fields. */
6371	case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
6372	case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
6373	case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
6374	case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
6375	case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
6376	case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
6377	case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
6378	case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
6379	case VMX_VMCS64_CTRL_VIRT_APIC_PAGEADDR_FULL:
6380	case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
6381	case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
6382	case VMX_VMCS64_CTRL_EPTP_FULL:
6383	case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
6384	/* 64-bit Guest-state fields. */
6385	case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
6386	case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
6387	case VMX_VMCS64_GUEST_PAT_FULL:
6388	case VMX_VMCS64_GUEST_EFER_FULL:
6389	case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
6390	case VMX_VMCS64_GUEST_PDPTE0_FULL:
6391	case VMX_VMCS64_GUEST_PDPTE1_FULL:
6392	case VMX_VMCS64_GUEST_PDPTE2_FULL:
6393	case VMX_VMCS64_GUEST_PDPTE3_FULL:
6394	/* 64-bit Host-state fields. */
6395	case VMX_VMCS64_HOST_PAT_FULL:
6396	case VMX_VMCS64_HOST_EFER_FULL:
6397	case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
6398	{
6399	rc = VMXWriteVmcs32(idxField, RT_LO_U32(u64Val));
6400	rc \|= VMXWriteVmcs32(idxField + 1, RT_HI_U32(u64Val));
6401	break;
6402	}
6403
6404	/*
6405	* These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
6406	* values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
6407	*/
6408	/* Natural-width Guest-state fields. */
6409	case VMX_VMCS_GUEST_CR3:
6410	case VMX_VMCS_GUEST_ES_BASE:
6411	case VMX_VMCS_GUEST_CS_BASE:
6412	case VMX_VMCS_GUEST_SS_BASE:
6413	case VMX_VMCS_GUEST_DS_BASE:
6414	case VMX_VMCS_GUEST_FS_BASE:
6415	case VMX_VMCS_GUEST_GS_BASE:
6416	case VMX_VMCS_GUEST_LDTR_BASE:
6417	case VMX_VMCS_GUEST_TR_BASE:
6418	case VMX_VMCS_GUEST_GDTR_BASE:
6419	case VMX_VMCS_GUEST_IDTR_BASE:
6420	case VMX_VMCS_GUEST_RSP:
6421	case VMX_VMCS_GUEST_RIP:
6422	case VMX_VMCS_GUEST_SYSENTER_ESP:
6423	case VMX_VMCS_GUEST_SYSENTER_EIP:
6424	{
6425	if (!(RT_HI_U32(u64Val)))
6426	{
6427	/* If this field is 64-bit, VT-x will zero out the top bits. */
6428	rc = VMXWriteVmcs32(idxField, RT_LO_U32(u64Val));
6429	}
6430	else
6431	{
6432	/* Assert that only the 32->64 switcher case should ever come here. */
6433	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests);
6434	rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
6435	}
6436	break;
6437	}
6438
6439	default:
6440	{
6441	AssertMsgFailed(("VMXWriteVmcs64Ex: Invalid field %#RX32 (pVCpu=%p u64Val=%#RX64)\n", idxField, pVCpu, u64Val));
6442	rc = VERR_INVALID_PARAMETER;
6443	break;
6444	}
6445	}
6446	AssertRCReturn(rc, rc);
6447	return rc;
6448	}
6449
6450
6451	/**
6452	* Queue up a VMWRITE by using the VMCS write cache.
6453	* This is only used on 32-bit hosts (except darwin) for 64-bit guests.
6454	*
6455	* @param pVCpu The cross context virtual CPU structure.
6456	* @param idxField The VMCS field encoding.
6457	* @param u64Val 16, 32 or 64-bit value.
6458	*/
6459	VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
6460	{
6461	AssertPtr(pVCpu);
6462	PVMXVMCSCACHE pCache = &pVCpu->hm.s.vmx.VmcsCache;
6463
6464	AssertMsgReturn(pCache->Write.cValidEntries < VMX_VMCS_CACHE_MAX_ENTRY - 1,
6465	("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
6466
6467	/* Make sure there are no duplicates. */
6468	for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
6469	{
6470	if (pCache->Write.aField[i] == idxField)
6471	{
6472	pCache->Write.aFieldVal[i] = u64Val;
6473	return VINF_SUCCESS;
6474	}
6475	}
6476
6477	pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
6478	pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
6479	pCache->Write.cValidEntries++;
6480	return VINF_SUCCESS;
6481	}
6482	#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) */
6483
6484
6485	/**
6486	* Sets up the usage of TSC-offsetting and updates the VMCS.
6487	*
6488	* If offsetting is not possible, cause VM-exits on RDTSC(P)s. Also sets up the
6489	* VMX-preemption timer.
6490	*
6491	* @returns VBox status code.
6492	* @param pVCpu The cross context virtual CPU structure.
6493	* @param pVmxTransient The VMX-transient structure.
6494	*
6495	* @remarks No-long-jump zone!!!
6496	*/
6497	static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
6498	{
6499	bool fOffsettedTsc;
6500	bool fParavirtTsc;
6501	uint64_t uTscOffset;
6502	PVM pVM = pVCpu->CTX_SUFF(pVM);
6503	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);;
6504
6505	if (pVM->hm.s.vmx.fUsePreemptTimer)
6506	{
6507	uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVM, pVCpu, &uTscOffset, &fOffsettedTsc, &fParavirtTsc);
6508
6509	/* Make sure the returned values have sane upper and lower boundaries. */
6510	uint64_t u64CpuHz = SUPGetCpuHzFromGipBySetIndex(g_pSUPGlobalInfoPage, pVCpu->iHostCpuSet);
6511	cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
6512	cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
6513	cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
6514
6515	/** @todo r=ramshankar: We need to find a way to integrate nested-guest
6516	* preemption timers here. We probably need to clamp the preemption timer,
6517	* after converting the timer value to the host. */
6518	uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
6519	int rc = VMXWriteVmcs32(VMX_VMCS32_PREEMPT_TIMER_VALUE, cPreemptionTickCount);
6520	AssertRC(rc);
6521	}
6522	else
6523	fOffsettedTsc = TMCpuTickCanUseRealTSC(pVM, pVCpu, &uTscOffset, &fParavirtTsc);
6524
6525	if (fParavirtTsc)
6526	{
6527	/* Currently neither Hyper-V nor KVM need to update their paravirt. TSC
6528	information before every VM-entry, hence disable it for performance sake. */
6529	#if 0
6530	int rc = GIMR0UpdateParavirtTsc(pVM, 0 /* u64Offset */);
6531	AssertRC(rc);
6532	#endif
6533	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscParavirt);
6534	}
6535
6536	uint32_t uProcCtls = pVmcsInfo->u32ProcCtls;
6537	if ( fOffsettedTsc
6538	&& RT_LIKELY(!pVCpu->hm.s.fDebugWantRdTscExit))
6539	{
6540	if (pVmxTransient->fIsNestedGuest)
6541	uTscOffset = CPUMApplyNestedGuestTscOffset(pVCpu, uTscOffset);
6542	if (pVmcsInfo->u64TscOffset != uTscOffset)
6543	{
6544	int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, uTscOffset);
6545	AssertRC(rc);
6546	pVmcsInfo->u64TscOffset = uTscOffset;
6547	}
6548
6549	if (uProcCtls & VMX_PROC_CTLS_RDTSC_EXIT)
6550	{
6551	uProcCtls &= ~VMX_PROC_CTLS_RDTSC_EXIT;
6552	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
6553	AssertRC(rc);
6554	pVmcsInfo->u32ProcCtls = uProcCtls;
6555	}
6556	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
6557	}
6558	else
6559	{
6560	/* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
6561	if (!(uProcCtls & VMX_PROC_CTLS_RDTSC_EXIT))
6562	{
6563	uProcCtls \|= VMX_PROC_CTLS_RDTSC_EXIT;
6564	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
6565	AssertRC(rc);
6566	pVmcsInfo->u32ProcCtls = uProcCtls;
6567	}
6568	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
6569	}
6570	}
6571
6572
6573	/**
6574	* Gets the IEM exception flags for the specified vector and IDT vectoring /
6575	* VM-exit interruption info type.
6576	*
6577	* @returns The IEM exception flags.
6578	* @param uVector The event vector.
6579	* @param uVmxEventType The VMX event type.
6580	*
6581	* @remarks This function currently only constructs flags required for
6582	* IEMEvaluateRecursiveXcpt and not the complete flags (e.g, error-code
6583	* and CR2 aspects of an exception are not included).
6584	*/
6585	static uint32_t hmR0VmxGetIemXcptFlags(uint8_t uVector, uint32_t uVmxEventType)
6586	{
6587	uint32_t fIemXcptFlags;
6588	switch (uVmxEventType)
6589	{
6590	case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
6591	case VMX_IDT_VECTORING_INFO_TYPE_NMI:
6592	fIemXcptFlags = IEM_XCPT_FLAGS_T_CPU_XCPT;
6593	break;
6594
6595	case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
6596	fIemXcptFlags = IEM_XCPT_FLAGS_T_EXT_INT;
6597	break;
6598
6599	case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
6600	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT \| IEM_XCPT_FLAGS_ICEBP_INSTR;
6601	break;
6602
6603	case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT:
6604	{
6605	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
6606	if (uVector == X86_XCPT_BP)
6607	fIemXcptFlags \|= IEM_XCPT_FLAGS_BP_INSTR;
6608	else if (uVector == X86_XCPT_OF)
6609	fIemXcptFlags \|= IEM_XCPT_FLAGS_OF_INSTR;
6610	else
6611	{
6612	fIemXcptFlags = 0;
6613	AssertMsgFailed(("Unexpected vector for software exception. uVector=%#x", uVector));
6614	}
6615	break;
6616	}
6617
6618	case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
6619	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
6620	break;
6621
6622	default:
6623	fIemXcptFlags = 0;
6624	AssertMsgFailed(("Unexpected vector type! uVmxEventType=%#x uVector=%#x", uVmxEventType, uVector));
6625	break;
6626	}
6627	return fIemXcptFlags;
6628	}
6629
6630
6631	/**
6632	* Sets an event as a pending event to be injected into the guest.
6633	*
6634	* @param pVCpu The cross context virtual CPU structure.
6635	* @param u32IntInfo The VM-entry interruption-information field.
6636	* @param cbInstr The VM-entry instruction length in bytes (for software
6637	* interrupts, exceptions and privileged software
6638	* exceptions).
6639	* @param u32ErrCode The VM-entry exception error code.
6640	* @param GCPtrFaultAddress The fault-address (CR2) in case it's a
6641	* page-fault.
6642	*/
6643	DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPU pVCpu, uint32_t u32IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
6644	RTGCUINTPTR GCPtrFaultAddress)
6645	{
6646	Assert(!pVCpu->hm.s.Event.fPending);
6647	pVCpu->hm.s.Event.fPending = true;
6648	pVCpu->hm.s.Event.u64IntInfo = u32IntInfo;
6649	pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
6650	pVCpu->hm.s.Event.cbInstr = cbInstr;
6651	pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
6652	}
6653
6654
6655	/**
6656	* Sets an external interrupt as pending-for-injection into the VM.
6657	*
6658	* @param pVCpu The cross context virtual CPU structure.
6659	* @param u8Interrupt The external interrupt vector.
6660	*/
6661	DECLINLINE(void) hmR0VmxSetPendingExtInt(PVMCPU pVCpu, uint8_t u8Interrupt)
6662	{
6663	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_EXIT_INT_INFO_VECTOR, u8Interrupt)
6664	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_ENTRY_INT_INFO_TYPE_EXT_INT)
6665	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
6666	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
6667	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
6668	}
6669
6670
6671	/**
6672	* Sets an NMI (\#NMI) exception as pending-for-injection into the VM.
6673	*
6674	* @param pVCpu The cross context virtual CPU structure.
6675	*/
6676	DECLINLINE(void) hmR0VmxSetPendingXcptNmi(PVMCPU pVCpu)
6677	{
6678	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_NMI)
6679	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_ENTRY_INT_INFO_TYPE_NMI)
6680	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
6681	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
6682	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
6683	}
6684
6685
6686	/**
6687	* Sets a double-fault (\#DF) exception as pending-for-injection into the VM.
6688	*
6689	* @param pVCpu The cross context virtual CPU structure.
6690	*/
6691	DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPU pVCpu)
6692	{
6693	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_DF)
6694	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
6695	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 1)
6696	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
6697	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
6698	}
6699
6700
6701	/**
6702	* Sets an invalid-opcode (\#UD) exception as pending-for-injection into the VM.
6703	*
6704	* @param pVCpu The cross context virtual CPU structure.
6705	*/
6706	DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPU pVCpu)
6707	{
6708	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_UD)
6709	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
6710	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
6711	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
6712	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
6713	}
6714
6715
6716	/**
6717	* Sets a debug (\#DB) exception as pending-for-injection into the VM.
6718	*
6719	* @param pVCpu The cross context virtual CPU structure.
6720	*/
6721	DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPU pVCpu)
6722	{
6723	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_DB)
6724	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
6725	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
6726	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
6727	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
6728	}
6729
6730
6731	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
6732	/**
6733	* Sets a general-protection (\#GP) exception as pending-for-injection into the VM.
6734	*
6735	* @param pVCpu The cross context virtual CPU structure.
6736	* @param u32ErrCode The error code for the general-protection exception.
6737	*/
6738	DECLINLINE(void) hmR0VmxSetPendingXcptGP(PVMCPU pVCpu, uint32_t u32ErrCode)
6739	{
6740	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_GP)
6741	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
6742	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 1)
6743	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
6744	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, u32ErrCode, 0 / GCPtrFaultAddress */);
6745	}
6746
6747
6748	/**
6749	* Sets a stack (\#SS) exception as pending-for-injection into the VM.
6750	*
6751	* @param pVCpu The cross context virtual CPU structure.
6752	* @param u32ErrCode The error code for the stack exception.
6753	*/
6754	DECLINLINE(void) hmR0VmxSetPendingXcptSS(PVMCPU pVCpu, uint32_t u32ErrCode)
6755	{
6756	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_SS)
6757	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
6758	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 1)
6759	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
6760	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, u32ErrCode, 0 / GCPtrFaultAddress */);
6761	}
6762
6763
6764	/**
6765	* Decodes the memory operand of an instruction that caused a VM-exit.
6766	*
6767	* The VM-exit qualification field provides the displacement field for memory
6768	* operand instructions, if any.
6769	*
6770	* @returns Strict VBox status code (i.e. informational status codes too).
6771	* @retval VINF_SUCCESS if the operand was successfully decoded.
6772	* @retval VINF_HM_PENDING_XCPT if an exception was raised while decoding the
6773	* operand.
6774	* @param pVCpu The cross context virtual CPU structure.
6775	* @param uExitInstrInfo The VM-exit instruction information field.
6776	* @param enmMemAccess The memory operand's access type (read or write).
6777	* @param GCPtrDisp The instruction displacement field, if any. For
6778	* RIP-relative addressing pass RIP + displacement here.
6779	* @param pGCPtrMem Where to store the effective destination memory address.
6780	*/
6781	static VBOXSTRICTRC hmR0VmxDecodeMemOperand(PVMCPU pVCpu, uint32_t uExitInstrInfo, RTGCPTR GCPtrDisp, VMXMEMACCESS enmMemAccess,
6782	PRTGCPTR pGCPtrMem)
6783	{
6784	Assert(pGCPtrMem);
6785	Assert(!CPUMIsGuestInRealOrV86Mode(pVCpu));
6786	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK \| CPUMCTX_EXTRN_EFER
6787	\| CPUMCTX_EXTRN_CR0);
6788
6789	static uint64_t const s_auAddrSizeMasks[] = { UINT64_C(0xffff), UINT64_C(0xffffffff), UINT64_C(0xffffffffffffffff) };
6790	static uint64_t const s_auAccessSizeMasks[] = { sizeof(uint16_t), sizeof(uint32_t), sizeof(uint64_t) };
6791	AssertCompile(RT_ELEMENTS(s_auAccessSizeMasks) == RT_ELEMENTS(s_auAddrSizeMasks));
6792
6793	VMXEXITINSTRINFO ExitInstrInfo;
6794	ExitInstrInfo.u = uExitInstrInfo;
6795	uint8_t const uAddrSize = ExitInstrInfo.All.u3AddrSize;
6796	uint8_t const iSegReg = ExitInstrInfo.All.iSegReg;
6797	bool const fIdxRegValid = !ExitInstrInfo.All.fIdxRegInvalid;
6798	uint8_t const iIdxReg = ExitInstrInfo.All.iIdxReg;
6799	uint8_t const uScale = ExitInstrInfo.All.u2Scaling;
6800	bool const fBaseRegValid = !ExitInstrInfo.All.fBaseRegInvalid;
6801	uint8_t const iBaseReg = ExitInstrInfo.All.iBaseReg;
6802	bool const fIsMemOperand = !ExitInstrInfo.All.fIsRegOperand;
6803	bool const fIsLongMode = CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx);
6804
6805	/*
6806	* Validate instruction information.
6807	* This shouldn't happen on real hardware but useful while testing our nested hardware-virtualization code.
6808	*/
6809	AssertLogRelMsgReturn(uAddrSize < RT_ELEMENTS(s_auAddrSizeMasks),
6810	("Invalid address size. ExitInstrInfo=%#RX32\n", ExitInstrInfo.u), VERR_VMX_IPE_1);
6811	AssertLogRelMsgReturn(iSegReg < X86_SREG_COUNT,
6812	("Invalid segment register. ExitInstrInfo=%#RX32\n", ExitInstrInfo.u), VERR_VMX_IPE_2);
6813	AssertLogRelMsgReturn(fIsMemOperand,
6814	("Expected memory operand. ExitInstrInfo=%#RX32\n", ExitInstrInfo.u), VERR_VMX_IPE_3);
6815
6816	/*
6817	* Compute the complete effective address.
6818	*
6819	* See AMD instruction spec. 1.4.2 "SIB Byte Format"
6820	* See AMD spec. 4.5.2 "Segment Registers".
6821	*/
6822	RTGCPTR GCPtrMem = GCPtrDisp;
6823	if (fBaseRegValid)
6824	GCPtrMem += pVCpu->cpum.GstCtx.aGRegs[iBaseReg].u64;
6825	if (fIdxRegValid)
6826	GCPtrMem += pVCpu->cpum.GstCtx.aGRegs[iIdxReg].u64 << uScale;
6827
6828	RTGCPTR const GCPtrOff = GCPtrMem;
6829	if ( !fIsLongMode
6830	\|\| iSegReg >= X86_SREG_FS)
6831	GCPtrMem += pVCpu->cpum.GstCtx.aSRegs[iSegReg].u64Base;
6832	GCPtrMem &= s_auAddrSizeMasks[uAddrSize];
6833
6834	/*
6835	* Validate effective address.
6836	* See AMD spec. 4.5.3 "Segment Registers in 64-Bit Mode".
6837	*/
6838	uint8_t const cbAccess = s_auAccessSizeMasks[uAddrSize];
6839	Assert(cbAccess > 0);
6840	if (fIsLongMode)
6841	{
6842	if (X86_IS_CANONICAL(GCPtrMem))
6843	{
6844	*pGCPtrMem = GCPtrMem;
6845	return VINF_SUCCESS;
6846	}
6847
6848	/** @todo r=ramshankar: We should probably raise \#SS or \#GP. See AMD spec. 4.12.2
6849	* "Data Limit Checks in 64-bit Mode". */
6850	Log4Func(("Long mode effective address is not canonical GCPtrMem=%#RX64\n", GCPtrMem));
6851	hmR0VmxSetPendingXcptGP(pVCpu, 0);
6852	return VINF_HM_PENDING_XCPT;
6853	}
6854
6855	/*
6856	* This is a watered down version of iemMemApplySegment().
6857	* Parts that are not applicable for VMX instructions like real-or-v8086 mode
6858	* and segment CPL/DPL checks are skipped.
6859	*/
6860	RTGCPTR32 const GCPtrFirst32 = (RTGCPTR32)GCPtrOff;
6861	RTGCPTR32 const GCPtrLast32 = GCPtrFirst32 + cbAccess - 1;
6862	PCCPUMSELREG pSel = &pVCpu->cpum.GstCtx.aSRegs[iSegReg];
6863
6864	/* Check if the segment is present and usable. */
6865	if ( pSel->Attr.n.u1Present
6866	&& !pSel->Attr.n.u1Unusable)
6867	{
6868	Assert(pSel->Attr.n.u1DescType);
6869	if (!(pSel->Attr.n.u4Type & X86_SEL_TYPE_CODE))
6870	{
6871	/* Check permissions for the data segment. */
6872	if ( enmMemAccess == VMXMEMACCESS_WRITE
6873	&& !(pSel->Attr.n.u4Type & X86_SEL_TYPE_WRITE))
6874	{
6875	Log4Func(("Data segment access invalid. iSegReg=%#x Attr=%#RX32\n", iSegReg, pSel->Attr.u));
6876	hmR0VmxSetPendingXcptGP(pVCpu, iSegReg);
6877	return VINF_HM_PENDING_XCPT;
6878	}
6879
6880	/* Check limits if it's a normal data segment. */
6881	if (!(pSel->Attr.n.u4Type & X86_SEL_TYPE_DOWN))
6882	{
6883	if ( GCPtrFirst32 > pSel->u32Limit
6884	\|\| GCPtrLast32 > pSel->u32Limit)
6885	{
6886	Log4Func(("Data segment limit exceeded."
6887	"iSegReg=%#x GCPtrFirst32=%#RX32 GCPtrLast32=%#RX32 u32Limit=%#RX32\n", iSegReg, GCPtrFirst32,
6888	GCPtrLast32, pSel->u32Limit));
6889	if (iSegReg == X86_SREG_SS)
6890	hmR0VmxSetPendingXcptSS(pVCpu, 0);
6891	else
6892	hmR0VmxSetPendingXcptGP(pVCpu, 0);
6893	return VINF_HM_PENDING_XCPT;
6894	}
6895	}
6896	else
6897	{
6898	/* Check limits if it's an expand-down data segment.
6899	Note! The upper boundary is defined by the B bit, not the G bit! */
6900	if ( GCPtrFirst32 < pSel->u32Limit + UINT32_C(1)
6901	\|\| GCPtrLast32 > (pSel->Attr.n.u1DefBig ? UINT32_MAX : UINT32_C(0xffff)))
6902	{
6903	Log4Func(("Expand-down data segment limit exceeded."
6904	"iSegReg=%#x GCPtrFirst32=%#RX32 GCPtrLast32=%#RX32 u32Limit=%#RX32\n", iSegReg, GCPtrFirst32,
6905	GCPtrLast32, pSel->u32Limit));
6906	if (iSegReg == X86_SREG_SS)
6907	hmR0VmxSetPendingXcptSS(pVCpu, 0);
6908	else
6909	hmR0VmxSetPendingXcptGP(pVCpu, 0);
6910	return VINF_HM_PENDING_XCPT;
6911	}
6912	}
6913	}
6914	else
6915	{
6916	/* Check permissions for the code segment. */
6917	if ( enmMemAccess == VMXMEMACCESS_WRITE
6918	\|\| ( enmMemAccess == VMXMEMACCESS_READ
6919	&& !(pSel->Attr.n.u4Type & X86_SEL_TYPE_READ)))
6920	{
6921	Log4Func(("Code segment access invalid. Attr=%#RX32\n", pSel->Attr.u));
6922	Assert(!CPUMIsGuestInRealOrV86ModeEx(&pVCpu->cpum.GstCtx));
6923	hmR0VmxSetPendingXcptGP(pVCpu, 0);
6924	return VINF_HM_PENDING_XCPT;
6925	}
6926
6927	/* Check limits for the code segment (normal/expand-down not applicable for code segments). */
6928	if ( GCPtrFirst32 > pSel->u32Limit
6929	\|\| GCPtrLast32 > pSel->u32Limit)
6930	{
6931	Log4Func(("Code segment limit exceeded. GCPtrFirst32=%#RX32 GCPtrLast32=%#RX32 u32Limit=%#RX32\n",
6932	GCPtrFirst32, GCPtrLast32, pSel->u32Limit));
6933	if (iSegReg == X86_SREG_SS)
6934	hmR0VmxSetPendingXcptSS(pVCpu, 0);
6935	else
6936	hmR0VmxSetPendingXcptGP(pVCpu, 0);
6937	return VINF_HM_PENDING_XCPT;
6938	}
6939	}
6940	}
6941	else
6942	{
6943	Log4Func(("Not present or unusable segment. iSegReg=%#x Attr=%#RX32\n", iSegReg, pSel->Attr.u));
6944	hmR0VmxSetPendingXcptGP(pVCpu, 0);
6945	return VINF_HM_PENDING_XCPT;
6946	}
6947
6948	*pGCPtrMem = GCPtrMem;
6949	return VINF_SUCCESS;
6950	}
6951
6952
6953	/**
6954	* Perform the relevant VMX instruction checks for VM-exits that occurred due to the
6955	* guest attempting to execute a VMX instruction.
6956	*
6957	* @returns Strict VBox status code (i.e. informational status codes too).
6958	* @retval VINF_SUCCESS if we should continue handling the VM-exit.
6959	* @retval VINF_HM_PENDING_XCPT if an exception was raised.
6960	*
6961	* @param pVCpu The cross context virtual CPU structure.
6962	* @param uExitReason The VM-exit reason.
6963	*
6964	* @todo NSTVMX: Document other error codes when VM-exit is implemented.
6965	* @remarks No-long-jump zone!!!
6966	*/
6967	static VBOXSTRICTRC hmR0VmxCheckExitDueToVmxInstr(PVMCPU pVCpu, uint32_t uExitReason)
6968	{
6969	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_SS
6970	\| CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_EFER);
6971
6972	if ( CPUMIsGuestInRealOrV86ModeEx(&pVCpu->cpum.GstCtx)
6973	\|\| ( CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx)
6974	&& !CPUMIsGuestIn64BitCodeEx(&pVCpu->cpum.GstCtx)))
6975	{
6976	Log4Func(("In real/v86-mode or long-mode outside 64-bit code segment -> #UD\n"));
6977	hmR0VmxSetPendingXcptUD(pVCpu);
6978	return VINF_HM_PENDING_XCPT;
6979	}
6980
6981	if (uExitReason == VMX_EXIT_VMXON)
6982	{
6983	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR4);
6984
6985	/*
6986	* We check CR4.VMXE because it is required to be always set while in VMX operation
6987	* by physical CPUs and our CR4 read shadow is only consulted when executing specific
6988	* instructions (CLTS, LMSW, MOV CR, and SMSW) and thus doesn't affect CPU operation
6989	* otherwise (i.e. physical CPU won't automatically #UD if Cr4Shadow.VMXE is 0).
6990	*/
6991	if (!CPUMIsGuestVmxEnabled(&pVCpu->cpum.GstCtx))
6992	{
6993	Log4Func(("CR4.VMXE is not set -> #UD\n"));
6994	hmR0VmxSetPendingXcptUD(pVCpu);
6995	return VINF_HM_PENDING_XCPT;
6996	}
6997	}
6998	else if (!CPUMIsGuestInVmxRootMode(&pVCpu->cpum.GstCtx))
6999	{
7000	/*
7001	* The guest has not entered VMX operation but attempted to execute a VMX instruction
7002	* (other than VMXON), we need to raise a #UD.
7003	*/
7004	Log4Func(("Not in VMX root mode -> #UD\n"));
7005	hmR0VmxSetPendingXcptUD(pVCpu);
7006	return VINF_HM_PENDING_XCPT;
7007	}
7008
7009	if (CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
7010	{
7011	/*
7012	* The nested-guest attempted to execute a VMX instruction, cause a VM-exit and let
7013	* the guest hypervisor deal with it.
7014	*/
7015	/** @todo NSTVMX: Trigger a VM-exit */
7016	}
7017
7018	/*
7019	* VMX instructions require CPL 0 except in VMX non-root mode where the VM-exit intercept
7020	* (above) takes preceedence over the CPL check.
7021	*/
7022	if (CPUMGetGuestCPL(pVCpu) > 0)
7023	{
7024	Log4Func(("CPL > 0 -> #GP(0)\n"));
7025	hmR0VmxSetPendingXcptGP(pVCpu, 0);
7026	return VINF_HM_PENDING_XCPT;
7027	}
7028
7029	return VINF_SUCCESS;
7030	}
7031	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
7032
7033
7034	static void hmR0VmxFixUnusableSegRegAttr(PVMCPU pVCpu, PCPUMSELREG pSelReg, uint32_t idxSel)
7035	{
7036	Assert(pSelReg->Attr.u & X86DESCATTR_UNUSABLE);
7037
7038	/*
7039	* If VT-x marks the segment as unusable, most other bits remain undefined:
7040	* - For CS the L, D and G bits have meaning.
7041	* - For SS the DPL has meaning (it -is- the CPL for Intel and VBox).
7042	* - For the remaining data segments no bits are defined.
7043	*
7044	* The present bit and the unusable bit has been observed to be set at the
7045	* same time (the selector was supposed to be invalid as we started executing
7046	* a V8086 interrupt in ring-0).
7047	*
7048	* What should be important for the rest of the VBox code, is that the P bit is
7049	* cleared. Some of the other VBox code recognizes the unusable bit, but
7050	* AMD-V certainly don't, and REM doesn't really either. So, to be on the
7051	* safe side here, we'll strip off P and other bits we don't care about. If
7052	* any code breaks because Attr.u != 0 when Sel < 4, it should be fixed.
7053	*
7054	* See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
7055	*/
7056	#ifdef VBOX_STRICT
7057	uint32_t const uAttr = pSelReg->Attr.u;
7058	#endif
7059
7060	/* Masking off: X86DESCATTR_P, X86DESCATTR_LIMIT_HIGH, and X86DESCATTR_AVL. The latter two are really irrelevant. */
7061	pSelReg->Attr.u &= X86DESCATTR_UNUSABLE \| X86DESCATTR_L \| X86DESCATTR_D \| X86DESCATTR_G
7062	\| X86DESCATTR_DPL \| X86DESCATTR_TYPE \| X86DESCATTR_DT;
7063
7064	#ifdef VBOX_STRICT
7065	VMMRZCallRing3Disable(pVCpu);
7066	Log4Func(("Unusable %#x: sel=%#x attr=%#x -> %#x\n", idxSel, pSelReg->Sel, uAttr, pSelReg->Attr.u));
7067	# ifdef DEBUG_bird
7068	AssertMsg((uAttr & ~X86DESCATTR_P) == pSelReg->Attr.u,
7069	("%#x: %#x != %#x (sel=%#x base=%#llx limit=%#x)\n",
7070	idxSel, uAttr, pSelReg->Attr.u, pSelReg->Sel, pSelReg->u64Base, pSelReg->u32Limit));
7071	# endif
7072	NOREF(uAttr);
7073	VMMRZCallRing3Enable(pVCpu);
7074	#else
7075	RT_NOREF2(pVCpu, idxSel);
7076	#endif
7077	}
7078
7079
7080	/**
7081	* Imports a guest segment register from the current VMCS into the guest-CPU
7082	* context.
7083	*
7084	* @returns VBox status code.
7085	* @param pVCpu The cross context virtual CPU structure.
7086	* @param iSegReg The segment register number (X86_SREG_XXX).
7087	*
7088	* @remarks Called with interrupts and/or preemption disabled, try not to assert and
7089	* do not log!
7090	*/
7091	static int hmR0VmxImportGuestSegReg(PVMCPU pVCpu, uint8_t iSegReg)
7092	{
7093	Assert(iSegReg < X86_SREG_COUNT);
7094
7095	uint32_t const idxSel = g_aVmcsSegSel[iSegReg];
7096	uint32_t const idxLimit = g_aVmcsSegLimit[iSegReg];
7097	uint32_t const idxAttr = g_aVmcsSegAttr[iSegReg];
7098	#ifdef VMX_USE_CACHED_VMCS_ACCESSES
7099	uint32_t const idxBase = g_aVmcsCacheSegBase[iSegReg];
7100	#else
7101	uint32_t const idxBase = g_aVmcsSegBase[iSegReg];
7102	#endif
7103	uint64_t u64Base;
7104	uint32_t u32Sel, u32Limit, u32Attr;
7105	int rc = VMXReadVmcs32(idxSel, &u32Sel);
7106	rc \|= VMXReadVmcs32(idxLimit, &u32Limit);
7107	rc \|= VMXReadVmcs32(idxAttr, &u32Attr);
7108	rc \|= VMXReadVmcsGstNByIdxVal(idxBase, &u64Base);
7109	if (RT_SUCCESS(rc))
7110	{
7111	PCPUMSELREG pSelReg = &pVCpu->cpum.GstCtx.aSRegs[iSegReg];
7112	pSelReg->Sel = u32Sel;
7113	pSelReg->ValidSel = u32Sel;
7114	pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
7115	pSelReg->u32Limit = u32Limit;
7116	pSelReg->u64Base = u64Base;
7117	pSelReg->Attr.u = u32Attr;
7118	if (u32Attr & X86DESCATTR_UNUSABLE)
7119	hmR0VmxFixUnusableSegRegAttr(pVCpu, pSelReg, idxSel);
7120	}
7121	return rc;
7122	}
7123
7124
7125	/**
7126	* Imports the guest LDTR from the current VMCS into the guest-CPU context.
7127	*
7128	* @returns VBox status code.
7129	* @param pVCpu The cross context virtual CPU structure.
7130	*
7131	* @remarks Called with interrupts and/or preemption disabled, try not to assert and
7132	* do not log!
7133	*/
7134	static int hmR0VmxImportGuestLdtr(PVMCPU pVCpu)
7135	{
7136	uint64_t u64Base;
7137	uint32_t u32Sel, u32Limit, u32Attr;
7138	int rc = VMXReadVmcs32(VMX_VMCS16_GUEST_LDTR_SEL, &u32Sel);
7139	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, &u32Limit);
7140	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, &u32Attr);
7141	rc \|= VMXReadVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, &u64Base);
7142
7143	if (RT_SUCCESS(rc))
7144	{
7145	pVCpu->cpum.GstCtx.ldtr.Sel = u32Sel;
7146	pVCpu->cpum.GstCtx.ldtr.ValidSel = u32Sel;
7147	pVCpu->cpum.GstCtx.ldtr.fFlags = CPUMSELREG_FLAGS_VALID;
7148	pVCpu->cpum.GstCtx.ldtr.u32Limit = u32Limit;
7149	pVCpu->cpum.GstCtx.ldtr.u64Base = u64Base;
7150	pVCpu->cpum.GstCtx.ldtr.Attr.u = u32Attr;
7151	if (u32Attr & X86DESCATTR_UNUSABLE)
7152	hmR0VmxFixUnusableSegRegAttr(pVCpu, &pVCpu->cpum.GstCtx.ldtr, VMX_VMCS16_GUEST_LDTR_SEL);
7153	}
7154	return rc;
7155	}
7156
7157
7158	/**
7159	* Imports the guest TR from the current VMCS into the guest-CPU context.
7160	*
7161	* @returns VBox status code.
7162	* @param pVCpu The cross context virtual CPU structure.
7163	*
7164	* @remarks Called with interrupts and/or preemption disabled, try not to assert and
7165	* do not log!
7166	*/
7167	static int hmR0VmxImportGuestTr(PVMCPU pVCpu)
7168	{
7169	uint32_t u32Sel, u32Limit, u32Attr;
7170	uint64_t u64Base;
7171	int rc = VMXReadVmcs32(VMX_VMCS16_GUEST_TR_SEL, &u32Sel);
7172	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, &u32Limit);
7173	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, &u32Attr);
7174	rc \|= VMXReadVmcsGstN(VMX_VMCS_GUEST_TR_BASE, &u64Base);
7175	AssertRCReturn(rc, rc);
7176
7177	pVCpu->cpum.GstCtx.tr.Sel = u32Sel;
7178	pVCpu->cpum.GstCtx.tr.ValidSel = u32Sel;
7179	pVCpu->cpum.GstCtx.tr.fFlags = CPUMSELREG_FLAGS_VALID;
7180	pVCpu->cpum.GstCtx.tr.u32Limit = u32Limit;
7181	pVCpu->cpum.GstCtx.tr.u64Base = u64Base;
7182	pVCpu->cpum.GstCtx.tr.Attr.u = u32Attr;
7183	/* TR is the only selector that can never be unusable. */
7184	Assert(!(u32Attr & X86DESCATTR_UNUSABLE));
7185	return VINF_SUCCESS;
7186	}
7187
7188
7189	/**
7190	* Imports the guest RIP from the VMCS back into the guest-CPU context.
7191	*
7192	* @returns VBox status code.
7193	* @param pVCpu The cross context virtual CPU structure.
7194	*
7195	* @remarks Called with interrupts and/or preemption disabled, should not assert!
7196	* @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
7197	* instead!!!
7198	*/
7199	static int hmR0VmxImportGuestRip(PVMCPU pVCpu)
7200	{
7201	uint64_t u64Val;
7202	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7203	if (pCtx->fExtrn & CPUMCTX_EXTRN_RIP)
7204	{
7205	int rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val);
7206	if (RT_SUCCESS(rc))
7207	{
7208	pCtx->rip = u64Val;
7209	EMR0HistoryUpdatePC(pVCpu, pCtx->rip, false);
7210	pCtx->fExtrn &= ~CPUMCTX_EXTRN_RIP;
7211	}
7212	return rc;
7213	}
7214	return VINF_SUCCESS;
7215	}
7216
7217
7218	/**
7219	* Imports the guest RFLAGS from the VMCS back into the guest-CPU context.
7220	*
7221	* @returns VBox status code.
7222	* @param pVCpu The cross context virtual CPU structure.
7223	* @param pVmcsInfo The VMCS info. object.
7224	*
7225	* @remarks Called with interrupts and/or preemption disabled, should not assert!
7226	* @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
7227	* instead!!!
7228	*/
7229	static int hmR0VmxImportGuestRFlags(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
7230	{
7231	uint32_t u32Val;
7232	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7233	if (pCtx->fExtrn & CPUMCTX_EXTRN_RFLAGS)
7234	{
7235	int rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val);
7236	if (RT_SUCCESS(rc))
7237	{
7238	pCtx->eflags.u32 = u32Val;
7239
7240	/* Restore eflags for real-on-v86-mode hack. */
7241	if (pVmcsInfo->RealMode.fRealOnV86Active)
7242	{
7243	pCtx->eflags.Bits.u1VM = 0;
7244	pCtx->eflags.Bits.u2IOPL = pVmcsInfo->RealMode.Eflags.Bits.u2IOPL;
7245	}
7246	}
7247	pCtx->fExtrn &= ~CPUMCTX_EXTRN_RFLAGS;
7248	return rc;
7249	}
7250	return VINF_SUCCESS;
7251	}
7252
7253
7254	/**
7255	* Imports the guest interruptibility-state from the VMCS back into the guest-CPU
7256	* context.
7257	*
7258	* @returns VBox status code.
7259	* @param pVCpu The cross context virtual CPU structure.
7260	* @param pVmcsInfo The VMCS info. object.
7261	*
7262	* @remarks Called with interrupts and/or preemption disabled, try not to assert and
7263	* do not log!
7264	* @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
7265	* instead!!!
7266	*/
7267	static int hmR0VmxImportGuestIntrState(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
7268	{
7269	uint32_t u32Val;
7270	int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &u32Val);
7271	if (RT_SUCCESS(rc))
7272	{
7273	if (!u32Val)
7274	{
7275	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
7276	{
7277	rc = hmR0VmxImportGuestRip(pVCpu);
7278	rc \|= hmR0VmxImportGuestRFlags(pVCpu, pVmcsInfo);
7279	AssertRCReturn(rc, rc);
7280	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
7281	}
7282
7283	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_BLOCK_NMIS))
7284	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
7285	}
7286	else
7287	{
7288	/*
7289	* We must import RIP here to set our EM interrupt-inhibited state.
7290	* We also import RFLAGS as our code that evaluates pending interrupts
7291	* before VM-entry requires it.
7292	*/
7293	rc = hmR0VmxImportGuestRip(pVCpu);
7294	rc \|= hmR0VmxImportGuestRFlags(pVCpu, pVmcsInfo);
7295	if (RT_SUCCESS(rc))
7296	{
7297	if (u32Val & ( VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS
7298	\| VMX_VMCS_GUEST_INT_STATE_BLOCK_STI))
7299	{
7300	EMSetInhibitInterruptsPC(pVCpu, pVCpu->cpum.GstCtx.rip);
7301	}
7302	else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
7303	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
7304
7305	if (u32Val & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI)
7306	{
7307	if (!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_BLOCK_NMIS))
7308	VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
7309	}
7310	else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_BLOCK_NMIS))
7311	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
7312	}
7313	}
7314	}
7315	return rc;
7316	}
7317
7318
7319	/**
7320	* Worker for VMXR0ImportStateOnDemand.
7321	*
7322	* @returns VBox status code.
7323	* @param pVCpu The cross context virtual CPU structure.
7324	* @param pVmcsInfo The VMCS info. object.
7325	* @param fWhat What to import, CPUMCTX_EXTRN_XXX.
7326	*/
7327	static int hmR0VmxImportGuestState(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo, uint64_t fWhat)
7328	{
7329	#define VMXLOCAL_BREAK_RC(a_rc) \
7330	if (RT_SUCCESS(a_rc)) \
7331	{ } \
7332	else \
7333	break
7334
7335	int rc = VINF_SUCCESS;
7336	PVM pVM = pVCpu->CTX_SUFF(pVM);
7337	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7338	uint64_t u64Val;
7339	uint32_t u32Val;
7340
7341	STAM_PROFILE_ADV_START(& pVCpu->hm.s.StatImportGuestState, x);
7342
7343	/*
7344	* We disable interrupts to make the updating of the state and in particular
7345	* the fExtrn modification atomic wrt to preemption hooks.
7346	*/
7347	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
7348
7349	fWhat &= pCtx->fExtrn;
7350	if (fWhat)
7351	{
7352	do
7353	{
7354	if (fWhat & CPUMCTX_EXTRN_RIP)
7355	{
7356	rc = hmR0VmxImportGuestRip(pVCpu);
7357	VMXLOCAL_BREAK_RC(rc);
7358	}
7359
7360	if (fWhat & CPUMCTX_EXTRN_RFLAGS)
7361	{
7362	rc = hmR0VmxImportGuestRFlags(pVCpu, pVmcsInfo);
7363	VMXLOCAL_BREAK_RC(rc);
7364	}
7365
7366	if (fWhat & CPUMCTX_EXTRN_HM_VMX_INT_STATE)
7367	{
7368	rc = hmR0VmxImportGuestIntrState(pVCpu, pVmcsInfo);
7369	VMXLOCAL_BREAK_RC(rc);
7370	}
7371
7372	if (fWhat & CPUMCTX_EXTRN_RSP)
7373	{
7374	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val);
7375	VMXLOCAL_BREAK_RC(rc);
7376	pCtx->rsp = u64Val;
7377	}
7378
7379	if (fWhat & CPUMCTX_EXTRN_SREG_MASK)
7380	{
7381	bool const fRealOnV86Active = pVmcsInfo->RealMode.fRealOnV86Active;
7382	if (fWhat & CPUMCTX_EXTRN_CS)
7383	{
7384	rc \|= hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_CS);
7385	rc \|= hmR0VmxImportGuestRip(pVCpu);
7386	if (fRealOnV86Active)
7387	pCtx->cs.Attr.u = pVmcsInfo->RealMode.AttrCS.u;
7388	EMR0HistoryUpdatePC(pVCpu, pCtx->cs.u64Base + pCtx->rip, true /* fFlattened */);
7389	}
7390	if (fWhat & CPUMCTX_EXTRN_SS)
7391	{
7392	rc \|= hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_SS);
7393	if (fRealOnV86Active)
7394	pCtx->ss.Attr.u = pVmcsInfo->RealMode.AttrSS.u;
7395	}
7396	if (fWhat & CPUMCTX_EXTRN_DS)
7397	{
7398	rc \|= hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_DS);
7399	if (fRealOnV86Active)
7400	pCtx->ds.Attr.u = pVmcsInfo->RealMode.AttrDS.u;
7401	}
7402	if (fWhat & CPUMCTX_EXTRN_ES)
7403	{
7404	rc \|= hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_ES);
7405	if (fRealOnV86Active)
7406	pCtx->es.Attr.u = pVmcsInfo->RealMode.AttrES.u;
7407	}
7408	if (fWhat & CPUMCTX_EXTRN_FS)
7409	{
7410	rc \|= hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_FS);
7411	if (fRealOnV86Active)
7412	pCtx->fs.Attr.u = pVmcsInfo->RealMode.AttrFS.u;
7413	}
7414	if (fWhat & CPUMCTX_EXTRN_GS)
7415	{
7416	rc \|= hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_GS);
7417	if (fRealOnV86Active)
7418	pCtx->gs.Attr.u = pVmcsInfo->RealMode.AttrGS.u;
7419	}
7420	VMXLOCAL_BREAK_RC(rc);
7421	}
7422
7423	if (fWhat & CPUMCTX_EXTRN_TABLE_MASK)
7424	{
7425	if (fWhat & CPUMCTX_EXTRN_LDTR)
7426	rc \|= hmR0VmxImportGuestLdtr(pVCpu);
7427
7428	if (fWhat & CPUMCTX_EXTRN_GDTR)
7429	{
7430	rc \|= VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
7431	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
7432	pCtx->gdtr.pGdt = u64Val;
7433	pCtx->gdtr.cbGdt = u32Val;
7434	}
7435
7436	/* Guest IDTR. */
7437	if (fWhat & CPUMCTX_EXTRN_IDTR)
7438	{
7439	rc \|= VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
7440	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
7441	pCtx->idtr.pIdt = u64Val;
7442	pCtx->idtr.cbIdt = u32Val;
7443	}
7444
7445	/* Guest TR. */
7446	if (fWhat & CPUMCTX_EXTRN_TR)
7447	{
7448	/* Real-mode emulation using virtual-8086 mode has the fake TSS (pRealModeTSS) in TR,
7449	don't need to import that one. */
7450	if (!pVmcsInfo->RealMode.fRealOnV86Active)
7451	rc \|= hmR0VmxImportGuestTr(pVCpu);
7452	}
7453	VMXLOCAL_BREAK_RC(rc);
7454	}
7455
7456	if (fWhat & CPUMCTX_EXTRN_DR7)
7457	{
7458	if (!pVCpu->hm.s.fUsingHyperDR7)
7459	{
7460	/* Upper 32-bits are always zero. See Intel spec. 2.7.3 "Loading and Storing Debug Registers". */
7461	rc = VMXReadVmcs32(VMX_VMCS_GUEST_DR7, &u32Val);
7462	VMXLOCAL_BREAK_RC(rc);
7463	pCtx->dr[7] = u32Val;
7464	}
7465	}
7466
7467	if (fWhat & CPUMCTX_EXTRN_SYSENTER_MSRS)
7468	{
7469	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &pCtx->SysEnter.eip);
7470	rc \|= VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &pCtx->SysEnter.esp);
7471	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val);
7472	pCtx->SysEnter.cs = u32Val;
7473	VMXLOCAL_BREAK_RC(rc);
7474	}
7475
7476	#if HC_ARCH_BITS == 64
7477	if (fWhat & CPUMCTX_EXTRN_KERNEL_GS_BASE)
7478	{
7479	if ( pVM->hm.s.fAllow64BitGuests
7480	&& (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
7481	pCtx->msrKERNELGSBASE = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
7482	}
7483
7484	if (fWhat & CPUMCTX_EXTRN_SYSCALL_MSRS)
7485	{
7486	if ( pVM->hm.s.fAllow64BitGuests
7487	&& (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
7488	{
7489	pCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR);
7490	pCtx->msrSTAR = ASMRdMsr(MSR_K6_STAR);
7491	pCtx->msrSFMASK = ASMRdMsr(MSR_K8_SF_MASK);
7492	}
7493	}
7494	#endif
7495
7496	if ( (fWhat & (CPUMCTX_EXTRN_TSC_AUX \| CPUMCTX_EXTRN_OTHER_MSRS))
7497	#if HC_ARCH_BITS == 32
7498	\|\| (fWhat & (CPUMCTX_EXTRN_KERNEL_GS_BASE \| CPUMCTX_EXTRN_SYSCALL_MSRS))
7499	#endif
7500	)
7501	{
7502	PCVMXAUTOMSR pMsr = (PCVMXAUTOMSR)pVmcsInfo->pvGuestMsrStore;
7503	uint32_t const cMsrs = pVmcsInfo->cExitMsrStore;
7504	Assert(cMsrs == 0 \|\| pMsr != NULL);
7505	for (uint32_t i = 0; i < cMsrs; i++, pMsr++)
7506	{
7507	switch (pMsr->u32Msr)
7508	{
7509	#if HC_ARCH_BITS == 32
7510	case MSR_K8_LSTAR: pCtx->msrLSTAR = pMsr->u64Value; break;
7511	case MSR_K6_STAR: pCtx->msrSTAR = pMsr->u64Value; break;
7512	case MSR_K8_SF_MASK: pCtx->msrSFMASK = pMsr->u64Value; break;
7513	case MSR_K8_KERNEL_GS_BASE: pCtx->msrKERNELGSBASE = pMsr->u64Value; break;
7514	#endif
7515	case MSR_IA32_SPEC_CTRL: CPUMSetGuestSpecCtrl(pVCpu, pMsr->u64Value); break;
7516	case MSR_K8_TSC_AUX: CPUMSetGuestTscAux(pVCpu, pMsr->u64Value); break;
7517	case MSR_K6_EFER: /* Can't be changed without causing a VM-exit */ break;
7518
7519	default:
7520	{
7521	pVCpu->hm.s.u32HMError = pMsr->u32Msr;
7522	ASMSetFlags(fEFlags);
7523	AssertMsgFailed(("Unexpected MSR in auto-load/store area. idMsr=%#RX32 cMsrs=%u\n", pMsr->u32Msr,
7524	cMsrs));
7525	return VERR_HM_UNEXPECTED_LD_ST_MSR;
7526	}
7527	}
7528	}
7529	}
7530
7531	if (fWhat & CPUMCTX_EXTRN_CR_MASK)
7532	{
7533	uint64_t u64Shadow;
7534	if (fWhat & CPUMCTX_EXTRN_CR0)
7535	{
7536	/** @todo r=ramshankar: We only read 32-bits here for legacy/convenience reasons,
7537	* remove when we drop 32-bit host w/ 64-bit host support, see
7538	* @bugref{9180#c39}. */
7539	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val);
7540	#if HC_ARCH_BITS == 32
7541	uint32_t u32Shadow;
7542	rc \|= VMXReadVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, &u32Shadow);
7543	u64Shadow = u32Shadow;
7544	#else
7545	rc \|= VMXReadVmcs64(VMX_VMCS_CTRL_CR0_READ_SHADOW, &u64Shadow);
7546	#endif
7547	VMXLOCAL_BREAK_RC(rc);
7548	u64Val = u32Val;
7549	u64Val = (u64Val & ~pVmcsInfo->u64Cr0Mask)
7550	\| (u64Shadow & pVmcsInfo->u64Cr0Mask);
7551	VMMRZCallRing3Disable(pVCpu); /* May call into PGM which has Log statements. */
7552	CPUMSetGuestCR0(pVCpu, u64Val);
7553	VMMRZCallRing3Enable(pVCpu);
7554	}
7555
7556	if (fWhat & CPUMCTX_EXTRN_CR4)
7557	{
7558	/** @todo r=ramshankar: We only read 32-bits here for legacy/convenience reasons,
7559	* remove when we drop 32-bit host w/ 64-bit host support, see
7560	* @bugref{9180#c39}. */
7561	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32Val);
7562	#if HC_ARCH_BITS == 32
7563	uint32_t u32Shadow;
7564	rc \|= VMXReadVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, &u32Shadow);
7565	u64Shadow = u32Shadow;
7566	#else
7567	rc \|= VMXReadVmcs64(VMX_VMCS_CTRL_CR4_READ_SHADOW, &u64Shadow);
7568	#endif
7569	VMXLOCAL_BREAK_RC(rc);
7570	u64Val = u32Val;
7571	u64Val = (u64Val & ~pVmcsInfo->u64Cr4Mask)
7572	\| (u64Shadow & pVmcsInfo->u64Cr4Mask);
7573	pCtx->cr4 = u64Val;
7574	}
7575
7576	if (fWhat & CPUMCTX_EXTRN_CR3)
7577	{
7578	/* CR0.PG bit changes are always intercepted, so it's up to date. */
7579	if ( pVM->hm.s.vmx.fUnrestrictedGuest
7580	\|\| ( pVM->hm.s.fNestedPaging
7581	&& CPUMIsGuestPagingEnabledEx(pCtx)))
7582	{
7583	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &u64Val);
7584	if (pCtx->cr3 != u64Val)
7585	{
7586	pCtx->cr3 = u64Val;
7587	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
7588	}
7589
7590	/* If the guest is in PAE mode, sync back the PDPE's into the guest state.
7591	Note: CR4.PAE, CR0.PG, EFER MSR changes are always intercepted, so they're up to date. */
7592	if (CPUMIsGuestInPAEModeEx(pCtx))
7593	{
7594	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u);
7595	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u);
7596	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u);
7597	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u);
7598	VMXLOCAL_BREAK_RC(rc);
7599	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
7600	}
7601	}
7602	}
7603	}
7604	} while (0);
7605
7606	if (RT_SUCCESS(rc))
7607	{
7608	/* Update fExtrn. */
7609	pCtx->fExtrn &= ~fWhat;
7610
7611	/* If everything has been imported, clear the HM keeper bit. */
7612	if (!(pCtx->fExtrn & HMVMX_CPUMCTX_EXTRN_ALL))
7613	{
7614	pCtx->fExtrn &= ~CPUMCTX_EXTRN_KEEPER_HM;
7615	Assert(!pCtx->fExtrn);
7616	}
7617	}
7618	}
7619	else
7620	AssertMsg(!pCtx->fExtrn \|\| (pCtx->fExtrn & HMVMX_CPUMCTX_EXTRN_ALL), ("%#RX64\n", pCtx->fExtrn));
7621
7622	ASMSetFlags(fEFlags);
7623
7624	STAM_PROFILE_ADV_STOP(& pVCpu->hm.s.StatImportGuestState, x);
7625
7626	if (RT_SUCCESS(rc))
7627	{ /* likely */ }
7628	else
7629	return rc;
7630
7631	/*
7632	* Honor any pending CR3 updates.
7633	*
7634	* Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> hmR0VmxCallRing3Callback()
7635	* -> VMMRZCallRing3Disable() -> hmR0VmxImportGuestState() -> Sets VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
7636	* -> continue with VM-exit handling -> hmR0VmxImportGuestState() and here we are.
7637	*
7638	* The reason for such complicated handling is because VM-exits that call into PGM expect CR3 to be up-to-date and thus
7639	* if any CR3-saves -before- the VM-exit (longjmp) postponed the CR3 update via the force-flag, any VM-exit handler that
7640	* calls into PGM when it re-saves CR3 will end up here and we call PGMUpdateCR3(). This is why the code below should
7641	* -NOT- check if CPUMCTX_EXTRN_CR3 is set!
7642	*
7643	* The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
7644	*/
7645	if (VMMRZCallRing3IsEnabled(pVCpu))
7646	{
7647	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
7648	{
7649	Assert(!(ASMAtomicUoReadU64(&pCtx->fExtrn) & CPUMCTX_EXTRN_CR3));
7650	PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
7651	}
7652
7653	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
7654	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
7655
7656	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
7657	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
7658	}
7659
7660	return VINF_SUCCESS;
7661	#undef VMXLOCAL_BREAK_RC
7662	}
7663
7664
7665	/**
7666	* Saves the guest state from the VMCS into the guest-CPU context.
7667	*
7668	* @returns VBox status code.
7669	* @param pVCpu The cross context virtual CPU structure.
7670	* @param fWhat What to import, CPUMCTX_EXTRN_XXX.
7671	*/
7672	VMMR0DECL(int) VMXR0ImportStateOnDemand(PVMCPU pVCpu, uint64_t fWhat)
7673	{
7674	PCVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
7675	return hmR0VmxImportGuestState(pVCpu, pVmcsInfo, fWhat);
7676	}
7677
7678
7679	/**
7680	* Check per-VM and per-VCPU force flag actions that require us to go back to
7681	* ring-3 for one reason or another.
7682	*
7683	* @returns Strict VBox status code (i.e. informational status codes too)
7684	* @retval VINF_SUCCESS if we don't have any actions that require going back to
7685	* ring-3.
7686	* @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
7687	* @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
7688	* interrupts)
7689	* @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
7690	* all EMTs to be in ring-3.
7691	* @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
7692	* @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
7693	* to the EM loop.
7694	*
7695	* @param pVCpu The cross context virtual CPU structure.
7696	* @param fStepping Whether we are single-stepping the guest using the
7697	* hypervisor debugger.
7698	*/
7699	static VBOXSTRICTRC hmR0VmxCheckForceFlags(PVMCPU pVCpu, bool fStepping)
7700	{
7701	Assert(VMMRZCallRing3IsEnabled(pVCpu));
7702
7703	/*
7704	* Update pending interrupts into the APIC's IRR.
7705	*/
7706	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_UPDATE_APIC))
7707	APICUpdatePendingInterrupts(pVCpu);
7708
7709	/*
7710	* Anything pending? Should be more likely than not if we're doing a good job.
7711	*/
7712	PVM pVM = pVCpu->CTX_SUFF(pVM);
7713	if ( !fStepping
7714	? !VM_FF_IS_ANY_SET(pVM, VM_FF_HP_R0_PRE_HM_MASK)
7715	&& !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HP_R0_PRE_HM_MASK)
7716	: !VM_FF_IS_ANY_SET(pVM, VM_FF_HP_R0_PRE_HM_STEP_MASK)
7717	&& !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
7718	return VINF_SUCCESS;
7719
7720	/* Pending PGM C3 sync. */
7721	if (VMCPU_FF_IS_ANY_SET(pVCpu,VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
7722	{
7723	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7724	Assert(!(ASMAtomicUoReadU64(&pCtx->fExtrn) & (CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_CR3 \| CPUMCTX_EXTRN_CR4)));
7725	VBOXSTRICTRC rcStrict2 = PGMSyncCR3(pVCpu, pCtx->cr0, pCtx->cr3, pCtx->cr4,
7726	VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
7727	if (rcStrict2 != VINF_SUCCESS)
7728	{
7729	AssertRC(VBOXSTRICTRC_VAL(rcStrict2));
7730	Log4Func(("PGMSyncCR3 forcing us back to ring-3. rc2=%d\n", VBOXSTRICTRC_VAL(rcStrict2)));
7731	return rcStrict2;
7732	}
7733	}
7734
7735	/* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
7736	if ( VM_FF_IS_ANY_SET(pVM, VM_FF_HM_TO_R3_MASK)
7737	\|\| VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
7738	{
7739	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
7740	int rc2 = RT_LIKELY(!VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_RAW_TO_R3 : VINF_EM_NO_MEMORY;
7741	Log4Func(("HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc2));
7742	return rc2;
7743	}
7744
7745	/* Pending VM request packets, such as hardware interrupts. */
7746	if ( VM_FF_IS_SET(pVM, VM_FF_REQUEST)
7747	\|\| VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_REQUEST))
7748	{
7749	Log4Func(("Pending VM request forcing us back to ring-3\n"));
7750	return VINF_EM_PENDING_REQUEST;
7751	}
7752
7753	/* Pending PGM pool flushes. */
7754	if (VM_FF_IS_SET(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
7755	{
7756	Log4Func(("PGM pool flush pending forcing us back to ring-3\n"));
7757	return VINF_PGM_POOL_FLUSH_PENDING;
7758	}
7759
7760	/* Pending DMA requests. */
7761	if (VM_FF_IS_SET(pVM, VM_FF_PDM_DMA))
7762	{
7763	Log4Func(("Pending DMA request forcing us back to ring-3\n"));
7764	return VINF_EM_RAW_TO_R3;
7765	}
7766
7767	return VINF_SUCCESS;
7768	}
7769
7770
7771	/**
7772	* Converts any TRPM trap into a pending HM event. This is typically used when
7773	* entering from ring-3 (not longjmp returns).
7774	*
7775	* @param pVCpu The cross context virtual CPU structure.
7776	*/
7777	static void hmR0VmxTrpmTrapToPendingEvent(PVMCPU pVCpu)
7778	{
7779	Assert(TRPMHasTrap(pVCpu));
7780	Assert(!pVCpu->hm.s.Event.fPending);
7781
7782	uint8_t uVector;
7783	TRPMEVENT enmTrpmEvent;
7784	RTGCUINT uErrCode;
7785	RTGCUINTPTR GCPtrFaultAddress;
7786	uint8_t cbInstr;
7787
7788	int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
7789	AssertRC(rc);
7790
7791	/* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntInfo. */
7792	uint32_t u32IntInfo = uVector \| VMX_EXIT_INT_INFO_VALID;
7793	if (enmTrpmEvent == TRPM_TRAP)
7794	{
7795	/** @todo r=ramshankar: TRPM currently offers no way to determine a \#DB that was
7796	* generated using INT1 (ICEBP). */
7797	switch (uVector)
7798	{
7799	case X86_XCPT_NMI:
7800	u32IntInfo \|= (VMX_EXIT_INT_INFO_TYPE_NMI << VMX_EXIT_INT_INFO_TYPE_SHIFT);
7801	break;
7802
7803	case X86_XCPT_BP:
7804	case X86_XCPT_OF:
7805	u32IntInfo \|= (VMX_EXIT_INT_INFO_TYPE_SW_XCPT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
7806	break;
7807
7808	case X86_XCPT_PF:
7809	case X86_XCPT_DF:
7810	case X86_XCPT_TS:
7811	case X86_XCPT_NP:
7812	case X86_XCPT_SS:
7813	case X86_XCPT_GP:
7814	case X86_XCPT_AC:
7815	u32IntInfo \|= VMX_EXIT_INT_INFO_ERROR_CODE_VALID;
7816	RT_FALL_THRU();
7817	default:
7818	u32IntInfo \|= (VMX_EXIT_INT_INFO_TYPE_HW_XCPT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
7819	break;
7820	}
7821	}
7822	else if (enmTrpmEvent == TRPM_HARDWARE_INT)
7823	u32IntInfo \|= (VMX_EXIT_INT_INFO_TYPE_EXT_INT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
7824	else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
7825	{
7826	switch (uVector)
7827	{
7828	case X86_XCPT_BP:
7829	case X86_XCPT_OF:
7830	u32IntInfo \|= (VMX_EXIT_INT_INFO_TYPE_SW_XCPT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
7831	break;
7832
7833	default:
7834	Assert(uVector == X86_XCPT_DB);
7835	u32IntInfo \|= (VMX_EXIT_INT_INFO_TYPE_SW_INT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
7836	break;
7837	}
7838	}
7839	else
7840	AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
7841
7842	rc = TRPMResetTrap(pVCpu);
7843	AssertRC(rc);
7844	Log4(("TRPM->HM event: u32IntInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
7845	u32IntInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
7846
7847	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, uErrCode, GCPtrFaultAddress);
7848	}
7849
7850
7851	/**
7852	* Converts the pending HM event into a TRPM trap.
7853	*
7854	* @param pVCpu The cross context virtual CPU structure.
7855	*/
7856	static void hmR0VmxPendingEventToTrpmTrap(PVMCPU pVCpu)
7857	{
7858	Assert(pVCpu->hm.s.Event.fPending);
7859
7860	uint32_t uVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
7861	uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVCpu->hm.s.Event.u64IntInfo);
7862	bool fErrorCodeValid = VMX_IDT_VECTORING_INFO_IS_ERROR_CODE_VALID(pVCpu->hm.s.Event.u64IntInfo);
7863	uint32_t uErrorCode = pVCpu->hm.s.Event.u32ErrCode;
7864
7865	/* If a trap was already pending, we did something wrong! */
7866	Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo /, NULL / pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
7867
7868	/** @todo Use HMVmxEventToTrpmEventType() later. */
7869	TRPMEVENT enmTrapType;
7870	switch (uVectorType)
7871	{
7872	case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
7873	enmTrapType = TRPM_HARDWARE_INT;
7874	break;
7875
7876	case VMX_IDT_VECTORING_INFO_TYPE_NMI:
7877	case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
7878	enmTrapType = TRPM_TRAP;
7879	break;
7880
7881	case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT: /* #DB (INT1/ICEBP). */
7882	Assert(uVector == X86_XCPT_DB);
7883	enmTrapType = TRPM_SOFTWARE_INT;
7884	break;
7885
7886	case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: /* #BP (INT3) and #OF (INTO) */
7887	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF);
7888	enmTrapType = TRPM_SOFTWARE_INT;
7889	break;
7890
7891	case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
7892	enmTrapType = TRPM_SOFTWARE_INT;
7893	break;
7894
7895	default:
7896	AssertMsgFailed(("Invalid trap type %#x\n", uVectorType));
7897	enmTrapType = TRPM_32BIT_HACK;
7898	break;
7899	}
7900
7901	Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
7902
7903	int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
7904	AssertRC(rc);
7905
7906	if (fErrorCodeValid)
7907	TRPMSetErrorCode(pVCpu, uErrorCode);
7908
7909	if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
7910	&& uVector == X86_XCPT_PF)
7911	TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
7912	else if (enmTrapType == TRPM_SOFTWARE_INT)
7913	TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
7914
7915	/* We're now done converting the pending event. */
7916	pVCpu->hm.s.Event.fPending = false;
7917	}
7918
7919
7920	/**
7921	* Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
7922	* cause a VM-exit as soon as the guest is in a state to receive interrupts.
7923	*
7924	* @param pVCpu The cross context virtual CPU structure.
7925	* @param pVmcsInfo The VMCS info. object.
7926	*/
7927	static void hmR0VmxSetIntWindowExitVmcs(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
7928	{
7929	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_INT_WINDOW_EXIT)
7930	{
7931	if (!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_INT_WINDOW_EXIT))
7932	{
7933	pVmcsInfo->u32ProcCtls \|= VMX_PROC_CTLS_INT_WINDOW_EXIT;
7934	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
7935	AssertRC(rc);
7936	}
7937	} /* else we will deliver interrupts whenever the guest Vm-exits next and is in a state to receive the interrupt. */
7938	}
7939
7940
7941	/**
7942	* Clears the interrupt-window exiting control in the VMCS.
7943	*
7944	* @param pVmcsInfo The VMCS info. object.
7945	*/
7946	DECLINLINE(int) hmR0VmxClearIntWindowExitVmcs(PVMXVMCSINFO pVmcsInfo)
7947	{
7948	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_INT_WINDOW_EXIT)
7949	{
7950	pVmcsInfo->u32ProcCtls &= ~VMX_PROC_CTLS_INT_WINDOW_EXIT;
7951	return VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
7952	}
7953	return VINF_SUCCESS;
7954	}
7955
7956
7957	/**
7958	* Sets the NMI-window exiting control in the VMCS which instructs VT-x to
7959	* cause a VM-exit as soon as the guest is in a state to receive NMIs.
7960	*
7961	* @param pVCpu The cross context virtual CPU structure.
7962	* @param pVmcsInfo The VMCS info. object.
7963	*/
7964	static void hmR0VmxSetNmiWindowExitVmcs(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
7965	{
7966	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_NMI_WINDOW_EXIT)
7967	{
7968	if (!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT))
7969	{
7970	pVmcsInfo->u32ProcCtls \|= VMX_PROC_CTLS_NMI_WINDOW_EXIT;
7971	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
7972	AssertRC(rc);
7973	Log4Func(("Setup NMI-window exiting\n"));
7974	}
7975	} /* else we will deliver NMIs whenever we VM-exit next, even possibly nesting NMIs. Can't be helped on ancient CPUs. */
7976	}
7977
7978
7979	/**
7980	* Clears the NMI-window exiting control in the VMCS.
7981	*
7982	* @param pVmcsInfo The VMCS info. object.
7983	*/
7984	DECLINLINE(int) hmR0VmxClearNmiWindowExitVmcs(PVMXVMCSINFO pVmcsInfo)
7985	{
7986	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT)
7987	{
7988	pVmcsInfo->u32ProcCtls &= ~VMX_PROC_CTLS_NMI_WINDOW_EXIT;
7989	return VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
7990	}
7991	return VINF_SUCCESS;
7992	}
7993
7994
7995	/**
7996	* Does the necessary state syncing before returning to ring-3 for any reason
7997	* (longjmp, preemption, voluntary exits to ring-3) from VT-x.
7998	*
7999	* @returns VBox status code.
8000	* @param pVCpu The cross context virtual CPU structure.
8001	* @param fImportState Whether to import the guest state from the VMCS back
8002	* to the guest-CPU context.
8003	*
8004	* @remarks No-long-jmp zone!!!
8005	*/
8006	static int hmR0VmxLeave(PVMCPU pVCpu, bool fImportState)
8007	{
8008	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8009	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8010
8011	RTCPUID idCpu = RTMpCpuId();
8012	Log4Func(("HostCpuId=%u\n", idCpu));
8013
8014	/*
8015	* !!! IMPORTANT !!!
8016	* If you modify code here, check whether hmR0VmxCallRing3Callback() needs to be updated too.
8017	*/
8018
8019	/* Save the guest state if necessary. */
8020	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
8021	if (fImportState)
8022	{
8023	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
8024	AssertRCReturn(rc, rc);
8025	}
8026
8027	/* Restore host FPU state if necessary. We will resync on next R0 reentry. */
8028	CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
8029	Assert(!CPUMIsGuestFPUStateActive(pVCpu));
8030
8031	/* Restore host debug registers if necessary. We will resync on next R0 reentry. */
8032	#ifdef VBOX_STRICT
8033	if (CPUMIsHyperDebugStateActive(pVCpu))
8034	Assert(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_MOV_DR_EXIT);
8035	#endif
8036	CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
8037	Assert(!CPUMIsGuestDebugStateActive(pVCpu) && !CPUMIsGuestDebugStateActivePending(pVCpu));
8038	Assert(!CPUMIsHyperDebugStateActive(pVCpu) && !CPUMIsHyperDebugStateActivePending(pVCpu));
8039
8040	#if HC_ARCH_BITS == 64
8041	/* Restore host-state bits that VT-x only restores partially. */
8042	if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
8043	&& (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
8044	{
8045	Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags, idCpu));
8046	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
8047	}
8048	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
8049	#endif
8050
8051	/* Restore the lazy host MSRs as we're leaving VT-x context. */
8052	if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
8053	{
8054	/* We shouldn't restore the host MSRs without saving the guest MSRs first. */
8055	if (!fImportState)
8056	{
8057	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_KERNEL_GS_BASE \| CPUMCTX_EXTRN_SYSCALL_MSRS);
8058	AssertRCReturn(rc, rc);
8059	}
8060	hmR0VmxLazyRestoreHostMsrs(pVCpu);
8061	Assert(!pVCpu->hm.s.vmx.fLazyMsrs);
8062	}
8063	else
8064	pVCpu->hm.s.vmx.fLazyMsrs = 0;
8065
8066	/* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
8067	pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs = false;
8068
8069	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
8070	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatImportGuestState);
8071	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExportGuestState);
8072	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatPreExit);
8073	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitHandling);
8074	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitIO);
8075	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitMovCRx);
8076	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitXcptNmi);
8077	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
8078
8079	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
8080
8081	/** @todo This partially defeats the purpose of having preemption hooks.
8082	* The problem is, deregistering the hooks should be moved to a place that
8083	* lasts until the EMT is about to be destroyed not everytime while leaving HM
8084	* context.
8085	*/
8086	int rc = hmR0VmxClearVmcs(pVmcsInfo);
8087	AssertRCReturn(rc, rc);
8088
8089	Log4Func(("Cleared Vmcs. HostCpuId=%u\n", idCpu));
8090	NOREF(idCpu);
8091	return VINF_SUCCESS;
8092	}
8093
8094
8095	/**
8096	* Leaves the VT-x session.
8097	*
8098	* @returns VBox status code.
8099	* @param pVCpu The cross context virtual CPU structure.
8100	*
8101	* @remarks No-long-jmp zone!!!
8102	*/
8103	static int hmR0VmxLeaveSession(PVMCPU pVCpu)
8104	{
8105	HM_DISABLE_PREEMPT(pVCpu);
8106	HMVMX_ASSERT_CPU_SAFE(pVCpu);
8107	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8108	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8109
8110	/* When thread-context hooks are used, we can avoid doing the leave again if we had been preempted before
8111	and done this from the VMXR0ThreadCtxCallback(). */
8112	if (!pVCpu->hm.s.fLeaveDone)
8113	{
8114	int rc2 = hmR0VmxLeave(pVCpu, true /* fImportState */);
8115	AssertRCReturnStmt(rc2, HM_RESTORE_PREEMPT(), rc2);
8116	pVCpu->hm.s.fLeaveDone = true;
8117	}
8118	Assert(!pVCpu->cpum.GstCtx.fExtrn);
8119
8120	/*
8121	* !!! IMPORTANT !!!
8122	* If you modify code here, make sure to check whether hmR0VmxCallRing3Callback() needs to be updated too.
8123	*/
8124
8125	/* Deregister hook now that we've left HM context before re-enabling preemption. */
8126	/** @todo Deregistering here means we need to VMCLEAR always
8127	* (longjmp/exit-to-r3) in VT-x which is not efficient, eliminate need
8128	* for calling VMMR0ThreadCtxHookDisable here! */
8129	VMMR0ThreadCtxHookDisable(pVCpu);
8130
8131	/* Leave HM context. This takes care of local init (term). */
8132	int rc = HMR0LeaveCpu(pVCpu);
8133
8134	HM_RESTORE_PREEMPT();
8135	return rc;
8136	}
8137
8138
8139	/**
8140	* Does the necessary state syncing before doing a longjmp to ring-3.
8141	*
8142	* @returns VBox status code.
8143	* @param pVCpu The cross context virtual CPU structure.
8144	*
8145	* @remarks No-long-jmp zone!!!
8146	*/
8147	DECLINLINE(int) hmR0VmxLongJmpToRing3(PVMCPU pVCpu)
8148	{
8149	return hmR0VmxLeaveSession(pVCpu);
8150	}
8151
8152
8153	/**
8154	* Take necessary actions before going back to ring-3.
8155	*
8156	* An action requires us to go back to ring-3. This function does the necessary
8157	* steps before we can safely return to ring-3. This is not the same as longjmps
8158	* to ring-3, this is voluntary and prepares the guest so it may continue
8159	* executing outside HM (recompiler/IEM).
8160	*
8161	* @returns VBox status code.
8162	* @param pVCpu The cross context virtual CPU structure.
8163	* @param rcExit The reason for exiting to ring-3. Can be
8164	* VINF_VMM_UNKNOWN_RING3_CALL.
8165	*/
8166	static int hmR0VmxExitToRing3(PVMCPU pVCpu, VBOXSTRICTRC rcExit)
8167	{
8168	Assert(pVCpu);
8169	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
8170
8171	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
8172	if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
8173	{
8174	VMXGetCurrentVmcs(&pVCpu->hm.s.vmx.LastError.HCPhysCurrentVmcs);
8175	pVCpu->hm.s.vmx.LastError.u32VmcsRev = (uint32_t )pVmcsInfo->pvVmcs;
8176	pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
8177	/* LastError.idCurrentCpu was updated in hmR0VmxPreRunGuestCommitted(). */
8178	}
8179
8180	/* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
8181	VMMRZCallRing3Disable(pVCpu);
8182	Log4Func(("rcExit=%d\n", VBOXSTRICTRC_VAL(rcExit)));
8183
8184	/*
8185	* Convert any pending HM events back to TRPM due to premature exits to ring-3.
8186	* We need to do this only on returns to ring-3 and not for longjmps to ring3.
8187	*
8188	* This is because execution may continue from ring-3 and we would need to inject
8189	* the event from there (hence place it back in TRPM).
8190	*/
8191	if (pVCpu->hm.s.Event.fPending)
8192	{
8193	hmR0VmxPendingEventToTrpmTrap(pVCpu);
8194	Assert(!pVCpu->hm.s.Event.fPending);
8195
8196	/* Clear the events from the VMCS. */
8197	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, 0);
8198	AssertRCReturn(rc, rc);
8199	}
8200	#ifdef VBOX_STRICT
8201	else
8202	{
8203	/*
8204	* Ensure we don't accidentally clear a pending HM event without clearing the VMCS.
8205	* This can be pretty hard to debug otherwise, interrupts might get injected twice
8206	* occasionally, see @bugref{9180#c42}.
8207	*/
8208	uint32_t uEntryIntInfo;
8209	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &uEntryIntInfo);
8210	AssertRC(rc);
8211	Assert(!VMX_ENTRY_INT_INFO_IS_VALID(uEntryIntInfo));
8212	}
8213	#endif
8214
8215	/*
8216	* Clear the interrupt-window and NMI-window VMCS controls as we could have got
8217	* a VM-exit with higher priority than interrupt-window or NMI-window VM-exits
8218	* (e.g. TPR below threshold).
8219	*/
8220	int rc = hmR0VmxClearIntWindowExitVmcs(pVmcsInfo);
8221	rc \|= hmR0VmxClearNmiWindowExitVmcs(pVmcsInfo);
8222	AssertRCReturn(rc, rc);
8223
8224	/* If we're emulating an instruction, we shouldn't have any TRPM traps pending
8225	and if we're injecting an event we should have a TRPM trap pending. */
8226	AssertMsg(rcExit != VINF_EM_RAW_INJECT_TRPM_EVENT \|\| TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
8227	#ifndef DEBUG_bird /* Triggered after firing an NMI against NT4SP1, possibly a triple fault in progress. */
8228	AssertMsg(rcExit != VINF_EM_RAW_EMULATE_INSTR \|\| !TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
8229	#endif
8230
8231	/* Save guest state and restore host state bits. */
8232	rc = hmR0VmxLeaveSession(pVCpu);
8233	AssertRCReturn(rc, rc);
8234	STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
8235
8236	/* Thread-context hooks are unregistered at this point!!! */
8237
8238	/* Sync recompiler state. */
8239	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
8240	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
8241	\| CPUM_CHANGED_LDTR
8242	\| CPUM_CHANGED_GDTR
8243	\| CPUM_CHANGED_IDTR
8244	\| CPUM_CHANGED_TR
8245	\| CPUM_CHANGED_HIDDEN_SEL_REGS);
8246	if ( pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging
8247	&& CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx))
8248	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
8249
8250	Assert(!pVCpu->hm.s.fClearTrapFlag);
8251
8252	/* Update the exit-to-ring 3 reason. */
8253	pVCpu->hm.s.rcLastExitToR3 = VBOXSTRICTRC_VAL(rcExit);
8254
8255	/* On our way back from ring-3 reload the guest state if there is a possibility of it being changed. */
8256	if ( rcExit != VINF_EM_RAW_INTERRUPT
8257	\|\| CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
8258	{
8259	Assert(!(pVCpu->cpum.GstCtx.fExtrn & HMVMX_CPUMCTX_EXTRN_ALL));
8260	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
8261	}
8262
8263	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
8264
8265	/* We do -not- want any longjmp notifications after this! We must return to ring-3 ASAP. */
8266	VMMRZCallRing3RemoveNotification(pVCpu);
8267	VMMRZCallRing3Enable(pVCpu);
8268
8269	return rc;
8270	}
8271
8272
8273	/**
8274	* VMMRZCallRing3() callback wrapper which saves the guest state before we
8275	* longjump to ring-3 and possibly get preempted.
8276	*
8277	* @returns VBox status code.
8278	* @param pVCpu The cross context virtual CPU structure.
8279	* @param enmOperation The operation causing the ring-3 longjump.
8280	* @param pvUser User argument, currently unused, NULL.
8281	*/
8282	static DECLCALLBACK(int) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
8283	{
8284	RT_NOREF(pvUser);
8285	if (enmOperation == VMMCALLRING3_VM_R0_ASSERTION)
8286	{
8287	/*
8288	* !!! IMPORTANT !!!
8289	* If you modify code here, check whether hmR0VmxLeave() and hmR0VmxLeaveSession() needs to be updated too.
8290	* This is a stripped down version which gets out ASAP, trying to not trigger any further assertions.
8291	*/
8292	VMMRZCallRing3RemoveNotification(pVCpu);
8293	VMMRZCallRing3Disable(pVCpu);
8294	RTTHREADPREEMPTSTATE PreemptState = RTTHREADPREEMPTSTATE_INITIALIZER;
8295	RTThreadPreemptDisable(&PreemptState);
8296
8297	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
8298	hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
8299	CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
8300	CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
8301
8302	#if HC_ARCH_BITS == 64
8303	/* Restore host-state bits that VT-x only restores partially. */
8304	if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
8305	&& (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
8306	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
8307	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
8308	#endif
8309
8310	/* Restore the lazy host MSRs as we're leaving VT-x context. */
8311	if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
8312	hmR0VmxLazyRestoreHostMsrs(pVCpu);
8313
8314	/* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
8315	pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs = false;
8316	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
8317
8318	/* Clear the current VMCS data back to memory. */
8319	hmR0VmxClearVmcs(pVmcsInfo);
8320
8321	/** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
8322	VMMR0ThreadCtxHookDisable(pVCpu);
8323	HMR0LeaveCpu(pVCpu);
8324	RTThreadPreemptRestore(&PreemptState);
8325	return VINF_SUCCESS;
8326	}
8327
8328	Assert(pVCpu);
8329	Assert(pvUser);
8330	Assert(VMMRZCallRing3IsEnabled(pVCpu));
8331	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
8332
8333	VMMRZCallRing3Disable(pVCpu);
8334	Assert(VMMR0IsLogFlushDisabled(pVCpu));
8335
8336	Log4Func((" -> hmR0VmxLongJmpToRing3 enmOperation=%d\n", enmOperation));
8337
8338	int rc = hmR0VmxLongJmpToRing3(pVCpu);
8339	AssertRCReturn(rc, rc);
8340
8341	VMMRZCallRing3Enable(pVCpu);
8342	return VINF_SUCCESS;
8343	}
8344
8345
8346	/**
8347	* Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
8348	* stack.
8349	*
8350	* @returns Strict VBox status code (i.e. informational status codes too).
8351	* @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
8352	* @param pVCpu The cross context virtual CPU structure.
8353	* @param uValue The value to push to the guest stack.
8354	*/
8355	static VBOXSTRICTRC hmR0VmxRealModeGuestStackPush(PVMCPU pVCpu, uint16_t uValue)
8356	{
8357	/*
8358	* The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
8359	* virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
8360	* See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
8361	*/
8362	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
8363	if (pCtx->sp == 1)
8364	return VINF_EM_RESET;
8365	pCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
8366	int rc = PGMPhysSimpleWriteGCPhys(pVCpu->CTX_SUFF(pVM), pCtx->ss.u64Base + pCtx->sp, &uValue, sizeof(uint16_t));
8367	AssertRC(rc);
8368	return rc;
8369	}
8370
8371
8372	/**
8373	* Injects an event into the guest upon VM-entry by updating the relevant fields
8374	* in the VM-entry area in the VMCS.
8375	*
8376	* @returns Strict VBox status code (i.e. informational status codes too).
8377	* @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
8378	* @retval VINF_EM_RESET if event injection resulted in a triple-fault.
8379	*
8380	* @param pVCpu The cross context virtual CPU structure.
8381	* @param pVmxTransient The VMX-transient structure.
8382	* @param pEvent The event being injected.
8383	* @param pfIntrState Pointer to the VT-x guest-interruptibility-state.
8384	* This will be updated if necessary. This cannot not
8385	* be NULL.
8386	* @param fStepping Whether we're single-stepping guest execution and
8387	* should return VINF_EM_DBG_STEPPED if the event is
8388	* injected directly (registers modified by us, not by
8389	* hardware on VM-entry).
8390	*/
8391	static VBOXSTRICTRC hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PCHMEVENT pEvent, bool fStepping,
8392	uint32_t *pfIntrState)
8393	{
8394	/* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
8395	AssertMsg(!RT_HI_U32(pEvent->u64IntInfo), ("%#RX64\n", pEvent->u64IntInfo));
8396	Assert(pfIntrState);
8397
8398	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
8399	uint32_t u32IntInfo = pEvent->u64IntInfo;
8400	uint32_t const u32ErrCode = pEvent->u32ErrCode;
8401	uint32_t const cbInstr = pEvent->cbInstr;
8402	RTGCUINTPTR const GCPtrFault = pEvent->GCPtrFaultAddress;
8403	uint32_t const uVector = VMX_ENTRY_INT_INFO_VECTOR(u32IntInfo);
8404	uint32_t const uIntType = VMX_ENTRY_INT_INFO_TYPE(u32IntInfo);
8405
8406	#ifdef VBOX_STRICT
8407	/*
8408	* Validate the error-code-valid bit for hardware exceptions.
8409	* No error codes for exceptions in real-mode.
8410	*
8411	* See Intel spec. 20.1.4 "Interrupt and Exception Handling"
8412	*/
8413	if ( uIntType == VMX_EXIT_INT_INFO_TYPE_HW_XCPT
8414	&& !CPUMIsGuestInRealModeEx(pCtx))
8415	{
8416	switch (uVector)
8417	{
8418	case X86_XCPT_PF:
8419	case X86_XCPT_DF:
8420	case X86_XCPT_TS:
8421	case X86_XCPT_NP:
8422	case X86_XCPT_SS:
8423	case X86_XCPT_GP:
8424	case X86_XCPT_AC:
8425	AssertMsg(VMX_ENTRY_INT_INFO_IS_ERROR_CODE_VALID(u32IntInfo),
8426	("Error-code-valid bit not set for exception that has an error code uVector=%#x\n", uVector));
8427	RT_FALL_THRU();
8428	default:
8429	break;
8430	}
8431	}
8432
8433	/* Cannot inject an NMI when block-by-MOV SS is in effect. */
8434	Assert( uIntType != VMX_EXIT_INT_INFO_TYPE_NMI
8435	\|\| !(*pfIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS));
8436	#endif
8437
8438	STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
8439
8440	/*
8441	* Hardware interrupts & exceptions cannot be delivered through the software interrupt
8442	* redirection bitmap to the real mode task in virtual-8086 mode. We must jump to the
8443	* interrupt handler in the (real-mode) guest.
8444	*
8445	* See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode".
8446	* See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
8447	*/
8448	if (CPUMIsGuestInRealModeEx(pCtx)) /* CR0.PE bit changes are always intercepted, so it's up to date. */
8449	{
8450	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest)
8451	{
8452	/*
8453	* For CPUs with unrestricted guest execution enabled and with the guest
8454	* in real-mode, we must not set the deliver-error-code bit.
8455	*
8456	* See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
8457	*/
8458	u32IntInfo &= ~VMX_ENTRY_INT_INFO_ERROR_CODE_VALID;
8459	}
8460	else
8461	{
8462	PVM pVM = pVCpu->CTX_SUFF(pVM);
8463	Assert(PDMVmmDevHeapIsEnabled(pVM));
8464	Assert(pVM->hm.s.vmx.pRealModeTSS);
8465	Assert(!CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx));
8466
8467	/* We require RIP, RSP, RFLAGS, CS, IDTR, import them. */
8468	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
8469	int rc2 = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_SREG_MASK \| CPUMCTX_EXTRN_TABLE_MASK
8470	\| CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_RFLAGS);
8471	AssertRCReturn(rc2, rc2);
8472
8473	/* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
8474	size_t const cbIdtEntry = sizeof(X86IDTR16);
8475	if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pCtx->idtr.cbIdt)
8476	{
8477	/* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
8478	if (uVector == X86_XCPT_DF)
8479	return VINF_EM_RESET;
8480
8481	/* If we're injecting a #GP with no valid IDT entry, inject a double-fault.
8482	No error codes for exceptions in real-mode. */
8483	if (uVector == X86_XCPT_GP)
8484	{
8485	uint32_t const uXcptDfInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_DF)
8486	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_ENTRY_INT_INFO_TYPE_HW_XCPT)
8487	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
8488	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
8489	HMEVENT EventXcptDf;
8490	RT_ZERO(EventXcptDf);
8491	EventXcptDf.u64IntInfo = uXcptDfInfo;
8492	return hmR0VmxInjectEventVmcs(pVCpu, pVmxTransient, &EventXcptDf, fStepping, pfIntrState);
8493	}
8494
8495	/*
8496	* If we're injecting an event with no valid IDT entry, inject a #GP.
8497	* No error codes for exceptions in real-mode.
8498	*
8499	* See Intel spec. 20.1.4 "Interrupt and Exception Handling"
8500	*/
8501	uint32_t const uXcptGpInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_GP)
8502	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_ENTRY_INT_INFO_TYPE_HW_XCPT)
8503	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
8504	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
8505	HMEVENT EventXcptGp;
8506	RT_ZERO(EventXcptGp);
8507	EventXcptGp.u64IntInfo = uXcptGpInfo;
8508	return hmR0VmxInjectEventVmcs(pVCpu, pVmxTransient, &EventXcptGp, fStepping, pfIntrState);
8509	}
8510
8511	/* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
8512	uint16_t uGuestIp = pCtx->ip;
8513	if (uIntType == VMX_ENTRY_INT_INFO_TYPE_SW_XCPT)
8514	{
8515	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF);
8516	/* #BP and #OF are both benign traps, we need to resume the next instruction. */
8517	uGuestIp = pCtx->ip + (uint16_t)cbInstr;
8518	}
8519	else if (uIntType == VMX_ENTRY_INT_INFO_TYPE_SW_INT)
8520	uGuestIp = pCtx->ip + (uint16_t)cbInstr;
8521
8522	/* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
8523	X86IDTR16 IdtEntry;
8524	RTGCPHYS const GCPhysIdtEntry = (RTGCPHYS)pCtx->idtr.pIdt + uVector * cbIdtEntry;
8525	rc2 = PGMPhysSimpleReadGCPhys(pVM, &IdtEntry, GCPhysIdtEntry, cbIdtEntry);
8526	AssertRCReturn(rc2, rc2);
8527
8528	/* Construct the stack frame for the interrupt/exception handler. */
8529	VBOXSTRICTRC rcStrict;
8530	rcStrict = hmR0VmxRealModeGuestStackPush(pVCpu, pCtx->eflags.u32);
8531	if (rcStrict == VINF_SUCCESS)
8532	{
8533	rcStrict = hmR0VmxRealModeGuestStackPush(pVCpu, pCtx->cs.Sel);
8534	if (rcStrict == VINF_SUCCESS)
8535	rcStrict = hmR0VmxRealModeGuestStackPush(pVCpu, uGuestIp);
8536	}
8537
8538	/* Clear the required eflag bits and jump to the interrupt/exception handler. */
8539	if (rcStrict == VINF_SUCCESS)
8540	{
8541	pCtx->eflags.u32 &= ~(X86_EFL_IF \| X86_EFL_TF \| X86_EFL_RF \| X86_EFL_AC);
8542	pCtx->rip = IdtEntry.offSel;
8543	pCtx->cs.Sel = IdtEntry.uSel;
8544	pCtx->cs.ValidSel = IdtEntry.uSel;
8545	pCtx->cs.u64Base = IdtEntry.uSel << cbIdtEntry;
8546	if ( uIntType == VMX_ENTRY_INT_INFO_TYPE_HW_XCPT
8547	&& uVector == X86_XCPT_PF)
8548	pCtx->cr2 = GCPtrFault;
8549
8550	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CS \| HM_CHANGED_GUEST_CR2
8551	\| HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS
8552	\| HM_CHANGED_GUEST_RSP);
8553
8554	/*
8555	* If we delivered a hardware exception (other than an NMI) and if there was
8556	* block-by-STI in effect, we should clear it.
8557	*/
8558	if (*pfIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
8559	{
8560	Assert( uIntType != VMX_ENTRY_INT_INFO_TYPE_NMI
8561	&& uIntType != VMX_ENTRY_INT_INFO_TYPE_EXT_INT);
8562	Log4Func(("Clearing inhibition due to STI\n"));
8563	*pfIntrState &= ~VMX_VMCS_GUEST_INT_STATE_BLOCK_STI;
8564	}
8565
8566	Log4(("Injected real-mode: u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x Eflags=%#x CS:EIP=%04x:%04x\n",
8567	u32IntInfo, u32ErrCode, cbInstr, pCtx->eflags.u, pCtx->cs.Sel, pCtx->eip));
8568
8569	/*
8570	* The event has been truly dispatched to the guest. Mark it as no longer pending so
8571	* we don't attempt to undo it if we are returning to ring-3 before executing guest code.
8572	*/
8573	pVCpu->hm.s.Event.fPending = false;
8574
8575	/* If we're stepping and we've changed cs:rip above, bail out of the VMX R0 execution loop. */
8576	if (fStepping)
8577	rcStrict = VINF_EM_DBG_STEPPED;
8578	}
8579	AssertMsg(rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_EM_RESET \|\| (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
8580	("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
8581	return rcStrict;
8582	}
8583	}
8584
8585	/*
8586	* Validate.
8587	*/
8588	Assert(VMX_ENTRY_INT_INFO_IS_VALID(u32IntInfo)); /* Bit 31 (Valid bit) must be set by caller. */
8589	Assert(!(u32IntInfo & VMX_BF_ENTRY_INT_INFO_RSVD_12_30_MASK)); /* Bits 30:12 MBZ. */
8590
8591	/*
8592	* Inject the event into the VMCS.
8593	*/
8594	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntInfo);
8595	if (VMX_ENTRY_INT_INFO_IS_ERROR_CODE_VALID(u32IntInfo))
8596	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
8597	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
8598	AssertRCReturn(rc, rc);
8599
8600	/*
8601	* Update guest CR2 if this is a page-fault.
8602	*/
8603	if ( VMX_ENTRY_INT_INFO_TYPE(u32IntInfo) == VMX_EXIT_INT_INFO_TYPE_HW_XCPT
8604	&& uVector == X86_XCPT_PF)
8605	pCtx->cr2 = GCPtrFault;
8606
8607	Log4(("Injecting u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x CR2=%#RX64\n", u32IntInfo, u32ErrCode, cbInstr, pCtx->cr2));
8608	return VINF_SUCCESS;
8609	}
8610
8611
8612	/**
8613	* Evaluates the event to be delivered to the guest and sets it as the pending
8614	* event.
8615	*
8616	* @returns Strict VBox status code (i.e. informational status codes too).
8617	* @param pVCpu The cross context virtual CPU structure.
8618	* @param pVmxTransient The VMX-transient structure.
8619	* @param pfIntrState Where to store the VT-x guest-interruptibility state.
8620	*/
8621	static VBOXSTRICTRC hmR0VmxEvaluatePendingEvent(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t *pfIntrState)
8622	{
8623	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
8624	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
8625
8626	/* Get the current interruptibility-state of the guest and then figure out what can be injected. */
8627	uint32_t const fIntrState = hmR0VmxGetGuestIntrState(pVCpu, pVmcsInfo);
8628	bool const fBlockMovSS = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS);
8629	bool const fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI);
8630	bool const fBlockNmi = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI);
8631
8632	Assert(!fBlockSti \|\| !(ASMAtomicUoReadU64(&pCtx->fExtrn) & CPUMCTX_EXTRN_RFLAGS));
8633	Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
8634	Assert(!fBlockSti \|\| pCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
8635	Assert(!TRPMHasTrap(pVCpu));
8636	Assert(pfIntrState);
8637
8638	*pfIntrState = fIntrState;
8639
8640	/*
8641	* Toggling of interrupt force-flags here is safe since we update TRPM on premature exits
8642	* to ring-3 before executing guest code, see hmR0VmxExitToRing3(). We must NOT restore these force-flags.
8643	*/
8644	/** @todo SMI. SMIs take priority over NMIs. */
8645	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_NMI)) /* NMI. NMIs take priority over regular interrupts. */
8646	{
8647	/* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
8648	if ( !pVCpu->hm.s.Event.fPending
8649	&& !fBlockNmi
8650	&& !fBlockSti
8651	&& !fBlockMovSS)
8652	{
8653	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
8654	if ( pVmxTransient->fIsNestedGuest
8655	&& CPUMIsGuestVmxPinCtlsSet(pVCpu, pCtx, VMX_PIN_CTLS_NMI_EXIT))
8656	return IEMExecVmxVmexitNmi(pVCpu);
8657	#endif
8658	hmR0VmxSetPendingXcptNmi(pVCpu);
8659	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
8660	Log4Func(("Pending NMI\n"));
8661	}
8662	else
8663	hmR0VmxSetNmiWindowExitVmcs(pVCpu, pVmcsInfo);
8664	}
8665	/*
8666	* Check if the guest can receive external interrupts (PIC/APIC). Once PDMGetInterrupt() returns
8667	* a valid interrupt we -must- deliver the interrupt. We can no longer re-request it from the APIC.
8668	*/
8669	else if ( VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC)
8670	&& !pVCpu->hm.s.fSingleInstruction)
8671	{
8672	Assert(!DBGFIsStepping(pVCpu));
8673	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_RFLAGS);
8674	AssertRCReturn(rc, rc);
8675	bool const fBlockInt = !(pCtx->eflags.u32 & X86_EFL_IF);
8676	if ( !pVCpu->hm.s.Event.fPending
8677	&& !fBlockInt
8678	&& !fBlockSti
8679	&& !fBlockMovSS)
8680	{
8681	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
8682	if ( pVmxTransient->fIsNestedGuest
8683	&& CPUMIsGuestVmxPinCtlsSet(pVCpu, pCtx, VMX_PIN_CTLS_EXT_INT_EXIT))
8684	{
8685	VBOXSTRICTRC rcStrict = IEMExecVmxVmexitExtInt(pVCpu, 0/* uVector /, true / fIntPending */);
8686	if (rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE)
8687	return rcStrict;
8688	}
8689	#endif
8690	uint8_t u8Interrupt;
8691	rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
8692	if (RT_SUCCESS(rc))
8693	{
8694	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
8695	if ( pVmxTransient->fIsNestedGuest
8696	&& CPUMIsGuestVmxPinCtlsSet(pVCpu, pCtx, VMX_PIN_CTLS_EXT_INT_EXIT)
8697	&& CPUMIsGuestVmxExitCtlsSet(pVCpu, pCtx, VMX_EXIT_CTLS_ACK_EXT_INT))
8698	{
8699	VBOXSTRICTRC rcStrict = IEMExecVmxVmexitExtInt(pVCpu, u8Interrupt, false /* fIntPending */);
8700	if (rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE)
8701	return rcStrict;
8702	}
8703	#endif
8704	hmR0VmxSetPendingExtInt(pVCpu, u8Interrupt);
8705	Log4Func(("Pending external interrupt vector %#x\n", u8Interrupt));
8706	}
8707	else if (rc == VERR_APIC_INTR_MASKED_BY_TPR)
8708	{
8709	if ( !pVmxTransient->fIsNestedGuest
8710	&& (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW))
8711	hmR0VmxApicSetTprThreshold(pVCpu, pVmcsInfo, u8Interrupt >> 4);
8712	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchTprMaskedIrq);
8713
8714	/*
8715	* If the CPU doesn't have TPR shadowing, we will always get a VM-exit on TPR changes and
8716	* APICSetTpr() will end up setting the VMCPU_FF_INTERRUPT_APIC if required, so there is no
8717	* need to re-set this force-flag here.
8718	*/
8719	}
8720	else
8721	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
8722	}
8723	else
8724	hmR0VmxSetIntWindowExitVmcs(pVCpu, pVmcsInfo);
8725	}
8726
8727	return VINF_SUCCESS;
8728	}
8729
8730
8731	/**
8732	* Injects any pending events into the guest if the guest is in a state to
8733	* receive them.
8734	*
8735	* @returns Strict VBox status code (i.e. informational status codes too).
8736	* @param pVCpu The cross context virtual CPU structure.
8737	* @param pVmxTransient The VMX-transient structure.
8738	* @param fIntrState The VT-x guest-interruptibility state.
8739	* @param fStepping Whether we are single-stepping the guest using the
8740	* hypervisor debugger and should return
8741	* VINF_EM_DBG_STEPPED if the event was dispatched
8742	* directly.
8743	*/
8744	static VBOXSTRICTRC hmR0VmxInjectPendingEvent(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t fIntrState, bool fStepping)
8745	{
8746	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
8747	Assert(VMMRZCallRing3IsEnabled(pVCpu));
8748
8749	bool const fBlockMovSS = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS);
8750	bool const fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI);
8751
8752	Assert(!fBlockSti \|\| !(ASMAtomicUoReadU64(&pVCpu->cpum.GstCtx.fExtrn) & CPUMCTX_EXTRN_RFLAGS));
8753	Assert(!fBlockSti \|\| pVCpu->cpum.GstCtx.eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
8754	Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
8755	Assert(!TRPMHasTrap(pVCpu));
8756
8757	VBOXSTRICTRC rcStrict = VINF_SUCCESS;
8758	if (pVCpu->hm.s.Event.fPending)
8759	{
8760	/*
8761	* Do -not- clear any interrupt-window exiting control here. We might have an interrupt
8762	* pending even while injecting an event and in this case, we want a VM-exit as soon as
8763	* the guest is ready for the next interrupt, see @bugref{6208#c45}.
8764	*
8765	* See Intel spec. 26.6.5 "Interrupt-Window Exiting and Virtual-Interrupt Delivery".
8766	*/
8767	uint32_t const uIntType = VMX_ENTRY_INT_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
8768	#ifdef VBOX_STRICT
8769	if (uIntType == VMX_ENTRY_INT_INFO_TYPE_EXT_INT)
8770	{
8771	bool const fBlockInt = !(pVCpu->cpum.GstCtx.eflags.u32 & X86_EFL_IF);
8772	Assert(!fBlockInt);
8773	Assert(!fBlockSti);
8774	Assert(!fBlockMovSS);
8775	}
8776	else if (uIntType == VMX_ENTRY_INT_INFO_TYPE_NMI)
8777	{
8778	bool const fBlockNmi = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI);
8779	Assert(!fBlockSti);
8780	Assert(!fBlockMovSS);
8781	Assert(!fBlockNmi);
8782	}
8783	#endif
8784	Log4(("Injecting pending event vcpu[%RU32] u64IntInfo=%#RX64 Type=%#RX32\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntInfo,
8785	uIntType));
8786
8787	/*
8788	* Inject the event and get any changes to the guest-interruptibility state.
8789	*
8790	* The guest-interruptibility state may need to be updated if we inject the event
8791	* into the guest IDT ourselves (for real-on-v86 guest injecting software interrupts).
8792	*/
8793	rcStrict = hmR0VmxInjectEventVmcs(pVCpu, pVmxTransient, &pVCpu->hm.s.Event, fStepping, &fIntrState);
8794	AssertRCReturn(VBOXSTRICTRC_VAL(rcStrict), rcStrict);
8795
8796	if (uIntType == VMX_ENTRY_INT_INFO_TYPE_EXT_INT)
8797	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
8798	else
8799	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
8800	}
8801
8802	/*
8803	* Update the guest-interruptibility state.
8804	*
8805	* This is required for the real-on-v86 software interrupt injection case above, as well as
8806	* updates to the guest state from ring-3 or IEM/REM.
8807	*/
8808	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INT_STATE, fIntrState);
8809	AssertRCReturn(rc, rc);
8810
8811	/*
8812	* There's no need to clear the VM-entry interruption-information field here if we're not
8813	* injecting anything. VT-x clears the valid bit on every VM-exit.
8814	*
8815	* See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
8816	*/
8817
8818	Assert(rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_EM_RESET \|\| (rcStrict == VINF_EM_DBG_STEPPED && fStepping));
8819	NOREF(fBlockMovSS); NOREF(fBlockSti);
8820	return rcStrict;
8821	}
8822
8823
8824	/**
8825	* Enters the VT-x session.
8826	*
8827	* @returns VBox status code.
8828	* @param pVCpu The cross context virtual CPU structure.
8829	*/
8830	VMMR0DECL(int) VMXR0Enter(PVMCPU pVCpu)
8831	{
8832	AssertPtr(pVCpu);
8833	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fSupported);
8834	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8835
8836	LogFlowFunc(("pVCpu=%p\n", pVCpu));
8837	Assert((pVCpu->hm.s.fCtxChanged & (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
8838	== (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE));
8839
8840	#ifdef VBOX_STRICT
8841	/* At least verify VMX is enabled, since we can't check if we're in VMX root mode without #GP'ing. */
8842	RTCCUINTREG uHostCR4 = ASMGetCR4();
8843	if (!(uHostCR4 & X86_CR4_VMXE))
8844	{
8845	LogRelFunc(("X86_CR4_VMXE bit in CR4 is not set!\n"));
8846	return VERR_VMX_X86_CR4_VMXE_CLEARED;
8847	}
8848	#endif
8849
8850	/*
8851	* Load the appropriate VMCS as the current and active one.
8852	*/
8853	PVMXVMCSINFO pVmcsInfo;
8854	bool const fInNestedGuestMode = CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx);
8855	if (!fInNestedGuestMode)
8856	pVmcsInfo = &pVCpu->hm.s.vmx.VmcsInfo;
8857	else
8858	pVmcsInfo = &pVCpu->hm.s.vmx.VmcsInfoNstGst;
8859	int rc = hmR0VmxLoadVmcs(pVmcsInfo);
8860	if (RT_SUCCESS(rc))
8861	{
8862	pVCpu->hm.s.vmx.fSwitchedToNstGstVmcs = fInNestedGuestMode;
8863	pVCpu->hm.s.fLeaveDone = false;
8864	Log4Func(("Loaded Vmcs. HostCpuId=%u\n", RTMpCpuId()));
8865
8866	/*
8867	* Do the EMT scheduled L1D flush here if needed.
8868	*/
8869	if (pVCpu->CTX_SUFF(pVM)->hm.s.fL1dFlushOnSched)
8870	ASMWrMsr(MSR_IA32_FLUSH_CMD, MSR_IA32_FLUSH_CMD_F_L1D);
8871	}
8872	return rc;
8873	}
8874
8875
8876	/**
8877	* The thread-context callback (only on platforms which support it).
8878	*
8879	* @param enmEvent The thread-context event.
8880	* @param pVCpu The cross context virtual CPU structure.
8881	* @param fGlobalInit Whether global VT-x/AMD-V init. was used.
8882	* @thread EMT(pVCpu)
8883	*/
8884	VMMR0DECL(void) VMXR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPU pVCpu, bool fGlobalInit)
8885	{
8886	NOREF(fGlobalInit);
8887
8888	switch (enmEvent)
8889	{
8890	case RTTHREADCTXEVENT_OUT:
8891	{
8892	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8893	Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
8894	VMCPU_ASSERT_EMT(pVCpu);
8895
8896	/* No longjmps (logger flushes, locks) in this fragile context. */
8897	VMMRZCallRing3Disable(pVCpu);
8898	Log4Func(("Preempting: HostCpuId=%u\n", RTMpCpuId()));
8899
8900	/* Restore host-state (FPU, debug etc.) */
8901	if (!pVCpu->hm.s.fLeaveDone)
8902	{
8903	/*
8904	* Do -not- import the guest-state here as we might already be in the middle of importing
8905	* it, esp. bad if we're holding the PGM lock, see comment in hmR0VmxImportGuestState().
8906	*/
8907	hmR0VmxLeave(pVCpu, false /* fImportState */);
8908	pVCpu->hm.s.fLeaveDone = true;
8909	}
8910
8911	/* Leave HM context, takes care of local init (term). */
8912	int rc = HMR0LeaveCpu(pVCpu);
8913	AssertRC(rc);
8914
8915	/* Restore longjmp state. */
8916	VMMRZCallRing3Enable(pVCpu);
8917	STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreempt);
8918	break;
8919	}
8920
8921	case RTTHREADCTXEVENT_IN:
8922	{
8923	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8924	Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
8925	VMCPU_ASSERT_EMT(pVCpu);
8926
8927	/* No longjmps here, as we don't want to trigger preemption (& its hook) while resuming. */
8928	VMMRZCallRing3Disable(pVCpu);
8929	Log4Func(("Resumed: HostCpuId=%u\n", RTMpCpuId()));
8930
8931	/* Initialize the bare minimum state required for HM. This takes care of
8932	initializing VT-x if necessary (onlined CPUs, local init etc.) */
8933	int rc = hmR0EnterCpu(pVCpu);
8934	AssertRC(rc);
8935	Assert( (pVCpu->hm.s.fCtxChanged & (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
8936	== (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE));
8937
8938	/* Load the active VMCS as the current one. */
8939	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
8940	rc = hmR0VmxLoadVmcs(pVmcsInfo);
8941	AssertRC(rc);
8942	Log4Func(("Resumed: Loaded Vmcs. HostCpuId=%u\n", RTMpCpuId()));
8943	pVCpu->hm.s.fLeaveDone = false;
8944
8945	/* Do the EMT scheduled L1D flush if needed. */
8946	if (pVCpu->CTX_SUFF(pVM)->hm.s.fL1dFlushOnSched)
8947	ASMWrMsr(MSR_IA32_FLUSH_CMD, MSR_IA32_FLUSH_CMD_F_L1D);
8948
8949	/* Restore longjmp state. */
8950	VMMRZCallRing3Enable(pVCpu);
8951	break;
8952	}
8953
8954	default:
8955	break;
8956	}
8957	}
8958
8959
8960	/**
8961	* Exports the host state into the VMCS host-state area.
8962	* Sets up the VM-exit MSR-load area.
8963	*
8964	* The CPU state will be loaded from these fields on every successful VM-exit.
8965	*
8966	* @returns VBox status code.
8967	* @param pVCpu The cross context virtual CPU structure.
8968	*
8969	* @remarks No-long-jump zone!!!
8970	*/
8971	static int hmR0VmxExportHostState(PVMCPU pVCpu)
8972	{
8973	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8974
8975	int rc = VINF_SUCCESS;
8976	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT)
8977	{
8978	rc = hmR0VmxExportHostControlRegs();
8979	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8980
8981	rc = hmR0VmxExportHostSegmentRegs(pVCpu);
8982	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8983
8984	rc = hmR0VmxExportHostMsrs(pVCpu);
8985	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8986
8987	pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_HOST_CONTEXT;
8988	}
8989	return rc;
8990	}
8991
8992
8993	/**
8994	* Saves the host state in the VMCS host-state.
8995	*
8996	* @returns VBox status code.
8997	* @param pVCpu The cross context virtual CPU structure.
8998	*
8999	* @remarks No-long-jump zone!!!
9000	*/
9001	VMMR0DECL(int) VMXR0ExportHostState(PVMCPU pVCpu)
9002	{
9003	AssertPtr(pVCpu);
9004	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
9005
9006	/*
9007	* Export the host state here while entering HM context.
9008	* When thread-context hooks are used, we might get preempted and have to re-save the host
9009	* state but most of the time we won't be, so do it here before we disable interrupts.
9010	*/
9011	return hmR0VmxExportHostState(pVCpu);
9012	}
9013
9014
9015	/**
9016	* Exports the guest state into the VMCS guest-state area.
9017	*
9018	* The will typically be done before VM-entry when the guest-CPU state and the
9019	* VMCS state may potentially be out of sync.
9020	*
9021	* Sets up the VM-entry MSR-load and VM-exit MSR-store areas. Sets up the
9022	* VM-entry controls.
9023	* Sets up the appropriate VMX non-root function to execute guest code based on
9024	* the guest CPU mode.
9025	*
9026	* @returns VBox strict status code.
9027	* @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
9028	* without unrestricted guest execution and the VMMDev is not presently
9029	* mapped (e.g. EFI32).
9030	*
9031	* @param pVCpu The cross context virtual CPU structure.
9032	* @param pVmxTransient The VMX-transient structure.
9033	*
9034	* @remarks No-long-jump zone!!!
9035	*/
9036	static VBOXSTRICTRC hmR0VmxExportGuestState(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
9037	{
9038	AssertPtr(pVCpu);
9039	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
9040	LogFlowFunc(("pVCpu=%p\n", pVCpu));
9041
9042	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExportGuestState, x);
9043
9044	/*
9045	* Determine real-on-v86 mode.
9046	* Used when the guest is in real-mode and unrestricted guest execution is not used.
9047	*/
9048	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
9049	if ( pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest
9050	\|\| !CPUMIsGuestInRealModeEx(&pVCpu->cpum.GstCtx))
9051	pVmcsInfo->RealMode. fRealOnV86Active = false;
9052	else
9053	{
9054	Assert(!pVmxTransient->fIsNestedGuest);
9055	pVmcsInfo->RealMode.fRealOnV86Active = true;
9056	}
9057
9058	/*
9059	* Any ordering dependency among the sub-functions below must be explicitly stated using comments.
9060	* Ideally, assert that the cross-dependent bits are up-to-date at the point of using it.
9061	*/
9062	/** @todo r=ramshankar: Move hmR0VmxSelectVMRunHandler inside
9063	* hmR0VmxExportGuestEntryExitCtls and do it conditionally. There shouldn't
9064	* be a need to evaluate this everytime since I'm pretty sure we intercept
9065	* all guest paging mode changes. */
9066	int rc = hmR0VmxSelectVMRunHandler(pVCpu, pVmxTransient);
9067	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9068
9069	rc = hmR0VmxExportGuestEntryExitCtls(pVCpu, pVmxTransient);
9070	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9071
9072	rc = hmR0VmxExportGuestCR0(pVCpu, pVmxTransient);
9073	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9074
9075	VBOXSTRICTRC rcStrict = hmR0VmxExportGuestCR3AndCR4(pVCpu, pVmxTransient);
9076	if (rcStrict == VINF_SUCCESS)
9077	{ /* likely */ }
9078	else
9079	{
9080	Assert(rcStrict == VINF_EM_RESCHEDULE_REM \|\| RT_FAILURE_NP(rcStrict));
9081	return rcStrict;
9082	}
9083
9084	rc = hmR0VmxExportGuestSegRegsXdtr(pVCpu, pVmxTransient);
9085	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9086
9087	rc = hmR0VmxExportGuestMsrs(pVCpu, pVmxTransient);
9088	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9089
9090	rc = hmR0VmxExportGuestApicTpr(pVCpu, pVmxTransient);
9091	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9092
9093	rc = hmR0VmxExportGuestXcptIntercepts(pVCpu, pVmxTransient);
9094	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9095
9096	rc = hmR0VmxExportGuestRip(pVCpu);
9097	rc \|= hmR0VmxExportGuestRsp(pVCpu);
9098	rc \|= hmR0VmxExportGuestRflags(pVCpu, pVmxTransient);
9099	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9100
9101	/* Clear any bits that may be set but exported unconditionally or unused/reserved bits. */
9102	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~( (HM_CHANGED_GUEST_GPRS_MASK & ~HM_CHANGED_GUEST_RSP)
9103	\| HM_CHANGED_GUEST_CR2
9104	\| (HM_CHANGED_GUEST_DR_MASK & ~HM_CHANGED_GUEST_DR7)
9105	\| HM_CHANGED_GUEST_X87
9106	\| HM_CHANGED_GUEST_SSE_AVX
9107	\| HM_CHANGED_GUEST_OTHER_XSAVE
9108	\| HM_CHANGED_GUEST_XCRx
9109	\| HM_CHANGED_GUEST_KERNEL_GS_BASE /* Part of lazy or auto load-store MSRs. */
9110	\| HM_CHANGED_GUEST_SYSCALL_MSRS /* Part of lazy or auto load-store MSRs. */
9111	\| HM_CHANGED_GUEST_TSC_AUX
9112	\| HM_CHANGED_GUEST_OTHER_MSRS
9113	\| HM_CHANGED_GUEST_HWVIRT /* More accurate PLE handling someday? */
9114	\| (HM_CHANGED_KEEPER_STATE_MASK & ~HM_CHANGED_VMX_MASK)));
9115
9116	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExportGuestState, x);
9117	return rc;
9118	}
9119
9120
9121	/**
9122	* Exports the state shared between the host and guest into the VMCS.
9123	*
9124	* @param pVCpu The cross context virtual CPU structure.
9125	* @param pVmxTransient The VMX-transient structure.
9126	*
9127	* @remarks No-long-jump zone!!!
9128	*/
9129	static void hmR0VmxExportSharedState(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
9130	{
9131	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
9132	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
9133
9134	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_DR_MASK)
9135	{
9136	int rc = hmR0VmxExportSharedDebugState(pVCpu, pVmxTransient);
9137	AssertRC(rc);
9138	pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_GUEST_DR_MASK;
9139
9140	/* Loading shared debug bits might have changed eflags.TF bit for debugging purposes. */
9141	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_RFLAGS)
9142	{
9143	rc = hmR0VmxExportGuestRflags(pVCpu, pVmxTransient);
9144	AssertRC(rc);
9145	}
9146	}
9147
9148	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_GUEST_LAZY_MSRS)
9149	{
9150	hmR0VmxLazyLoadGuestMsrs(pVCpu);
9151	pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_VMX_GUEST_LAZY_MSRS;
9152	}
9153
9154	AssertMsg(!(pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE),
9155	("fCtxChanged=%#RX64\n", pVCpu->hm.s.fCtxChanged));
9156	}
9157
9158
9159	/**
9160	* Worker for loading the guest-state bits in the inner VT-x execution loop.
9161	*
9162	* @returns Strict VBox status code (i.e. informational status codes too).
9163	* @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
9164	* without unrestricted guest execution and the VMMDev is not presently
9165	* mapped (e.g. EFI32).
9166	*
9167	* @param pVCpu The cross context virtual CPU structure.
9168	* @param pVmxTransient The VMX-transient structure.
9169	*
9170	* @remarks No-long-jump zone!!!
9171	*/
9172	static VBOXSTRICTRC hmR0VmxExportGuestStateOptimal(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
9173	{
9174	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
9175	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
9176	Assert(VMMR0IsLogFlushDisabled(pVCpu));
9177
9178	#ifdef HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
9179	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
9180	#endif
9181
9182	/*
9183	* For many exits it's only RIP that changes and hence try to export it first
9184	* without going through a lot of change flag checks.
9185	*/
9186	VBOXSTRICTRC rcStrict;
9187	uint64_t fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
9188	RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
9189	if ((fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)) == HM_CHANGED_GUEST_RIP)
9190	{
9191	rcStrict = hmR0VmxExportGuestRip(pVCpu);
9192	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
9193	{ /* likely */}
9194	else
9195	AssertMsgFailedReturn(("Failed to export guest RIP! rc=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)), rcStrict);
9196	STAM_COUNTER_INC(&pVCpu->hm.s.StatExportMinimal);
9197	}
9198	else if (fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
9199	{
9200	rcStrict = hmR0VmxExportGuestState(pVCpu, pVmxTransient);
9201	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
9202	{ /* likely */}
9203	else
9204	{
9205	AssertMsg(rcStrict == VINF_EM_RESCHEDULE_REM, ("Failed to export guest state! rc=%Rrc\n",
9206	VBOXSTRICTRC_VAL(rcStrict)));
9207	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
9208	return rcStrict;
9209	}
9210	STAM_COUNTER_INC(&pVCpu->hm.s.StatExportFull);
9211	}
9212	else
9213	rcStrict = VINF_SUCCESS;
9214
9215	#ifdef VBOX_STRICT
9216	/* All the guest state bits should be loaded except maybe the host context and/or the shared host/guest bits. */
9217	fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
9218	RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
9219	AssertMsg(!(fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)),
9220	("fCtxChanged=%#RX64\n", fCtxChanged));
9221	#endif
9222	return rcStrict;
9223	}
9224
9225
9226	/**
9227	* Tries to determine what part of the guest-state VT-x has deemed as invalid
9228	* and update error record fields accordingly.
9229	*
9230	* @return VMX_IGS_* return codes.
9231	* @retval VMX_IGS_REASON_NOT_FOUND if this function could not find anything
9232	* wrong with the guest state.
9233	*
9234	* @param pVCpu The cross context virtual CPU structure.
9235	* @param pVmcsInfo The VMCS info. object.
9236	*
9237	* @remarks This function assumes our cache of the VMCS controls
9238	* are valid, i.e. hmR0VmxCheckVmcsCtls() succeeded.
9239	*/
9240	static uint32_t hmR0VmxCheckGuestState(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
9241	{
9242	#define HMVMX_ERROR_BREAK(err) { uError = (err); break; }
9243	#define HMVMX_CHECK_BREAK(expr, err) if (!(expr)) { \
9244	uError = (err); \
9245	break; \
9246	} else do { } while (0)
9247
9248	int rc;
9249	PVM pVM = pVCpu->CTX_SUFF(pVM);
9250	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
9251	uint32_t uError = VMX_IGS_ERROR;
9252	uint32_t u32Val;
9253	bool const fUnrestrictedGuest = pVM->hm.s.vmx.fUnrestrictedGuest;
9254
9255	do
9256	{
9257	/*
9258	* CR0.
9259	*/
9260	uint32_t fSetCr0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
9261	uint32_t const fZapCr0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
9262	/* Exceptions for unrestricted guest execution for fixed CR0 bits (PE, PG).
9263	See Intel spec. 26.3.1 "Checks on Guest Control Registers, Debug Registers and MSRs." */
9264	if (fUnrestrictedGuest)
9265	fSetCr0 &= ~(X86_CR0_PE \| X86_CR0_PG);
9266
9267	uint32_t u32GuestCr0;
9268	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32GuestCr0);
9269	AssertRCBreak(rc);
9270	HMVMX_CHECK_BREAK((u32GuestCr0 & fSetCr0) == fSetCr0, VMX_IGS_CR0_FIXED1);
9271	HMVMX_CHECK_BREAK(!(u32GuestCr0 & ~fZapCr0), VMX_IGS_CR0_FIXED0);
9272	if ( !fUnrestrictedGuest
9273	&& (u32GuestCr0 & X86_CR0_PG)
9274	&& !(u32GuestCr0 & X86_CR0_PE))
9275	{
9276	HMVMX_ERROR_BREAK(VMX_IGS_CR0_PG_PE_COMBO);
9277	}
9278
9279	/*
9280	* CR4.
9281	*/
9282	uint64_t const fSetCr4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
9283	uint64_t const fZapCr4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
9284
9285	uint32_t u32GuestCr4;
9286	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32GuestCr4);
9287	AssertRCBreak(rc);
9288	HMVMX_CHECK_BREAK((u32GuestCr4 & fSetCr4) == fSetCr4, VMX_IGS_CR4_FIXED1);
9289	HMVMX_CHECK_BREAK(!(u32GuestCr4 & ~fZapCr4), VMX_IGS_CR4_FIXED0);
9290
9291	/*
9292	* IA32_DEBUGCTL MSR.
9293	*/
9294	uint64_t u64Val;
9295	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, &u64Val);
9296	AssertRCBreak(rc);
9297	if ( (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_DEBUG)
9298	&& (u64Val & 0xfffffe3c)) /* Bits 31:9, bits 5:2 MBZ. */
9299	{
9300	HMVMX_ERROR_BREAK(VMX_IGS_DEBUGCTL_MSR_RESERVED);
9301	}
9302	uint64_t u64DebugCtlMsr = u64Val;
9303
9304	#ifdef VBOX_STRICT
9305	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
9306	AssertRCBreak(rc);
9307	Assert(u32Val == pVmcsInfo->u32EntryCtls);
9308	#endif
9309	bool const fLongModeGuest = RT_BOOL(pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_IA32E_MODE_GUEST);
9310
9311	/*
9312	* RIP and RFLAGS.
9313	*/
9314	uint32_t u32Eflags;
9315	#if HC_ARCH_BITS == 64
9316	rc = VMXReadVmcs64(VMX_VMCS_GUEST_RIP, &u64Val);
9317	AssertRCBreak(rc);
9318	/* pCtx->rip can be different than the one in the VMCS (e.g. run guest code and VM-exits that don't update it). */
9319	if ( !fLongModeGuest
9320	\|\| !pCtx->cs.Attr.n.u1Long)
9321	{
9322	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffff00000000)), VMX_IGS_LONGMODE_RIP_INVALID);
9323	}
9324	/** @todo If the processor supports N < 64 linear-address bits, bits 63:N
9325	* must be identical if the "IA-32e mode guest" VM-entry
9326	* control is 1 and CS.L is 1. No check applies if the
9327	* CPU supports 64 linear-address bits. */
9328
9329	/* Flags in pCtx can be different (real-on-v86 for instance). We are only concerned about the VMCS contents here. */
9330	rc = VMXReadVmcs64(VMX_VMCS_GUEST_RFLAGS, &u64Val);
9331	AssertRCBreak(rc);
9332	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffc08028)), /* Bit 63:22, Bit 15, 5, 3 MBZ. */
9333	VMX_IGS_RFLAGS_RESERVED);
9334	HMVMX_CHECK_BREAK((u64Val & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
9335	u32Eflags = u64Val;
9336	#else
9337	rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Eflags);
9338	AssertRCBreak(rc);
9339	HMVMX_CHECK_BREAK(!(u32Eflags & 0xffc08028), VMX_IGS_RFLAGS_RESERVED); /* Bit 31:22, Bit 15, 5, 3 MBZ. */
9340	HMVMX_CHECK_BREAK((u32Eflags & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
9341	#endif
9342
9343	if ( fLongModeGuest
9344	\|\| ( fUnrestrictedGuest
9345	&& !(u32GuestCr0 & X86_CR0_PE)))
9346	{
9347	HMVMX_CHECK_BREAK(!(u32Eflags & X86_EFL_VM), VMX_IGS_RFLAGS_VM_INVALID);
9348	}
9349
9350	uint32_t u32EntryInfo;
9351	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
9352	AssertRCBreak(rc);
9353	if ( VMX_ENTRY_INT_INFO_IS_VALID(u32EntryInfo)
9354	&& VMX_ENTRY_INT_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INT_INFO_TYPE_EXT_INT)
9355	{
9356	HMVMX_CHECK_BREAK(u32Eflags & X86_EFL_IF, VMX_IGS_RFLAGS_IF_INVALID);
9357	}
9358
9359	/*
9360	* 64-bit checks.
9361	*/
9362	#if HC_ARCH_BITS == 64
9363	if (fLongModeGuest)
9364	{
9365	HMVMX_CHECK_BREAK(u32GuestCr0 & X86_CR0_PG, VMX_IGS_CR0_PG_LONGMODE);
9366	HMVMX_CHECK_BREAK(u32GuestCr4 & X86_CR4_PAE, VMX_IGS_CR4_PAE_LONGMODE);
9367	}
9368
9369	if ( !fLongModeGuest
9370	&& (u32GuestCr4 & X86_CR4_PCIDE))
9371	{
9372	HMVMX_ERROR_BREAK(VMX_IGS_CR4_PCIDE);
9373	}
9374
9375	/** @todo CR3 field must be such that bits 63:52 and bits in the range
9376	* 51:32 beyond the processor's physical-address width are 0. */
9377
9378	if ( (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_DEBUG)
9379	&& (pCtx->dr[7] & X86_DR7_MBZ_MASK))
9380	{
9381	HMVMX_ERROR_BREAK(VMX_IGS_DR7_RESERVED);
9382	}
9383
9384	rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, &u64Val);
9385	AssertRCBreak(rc);
9386	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_ESP_NOT_CANONICAL);
9387
9388	rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, &u64Val);
9389	AssertRCBreak(rc);
9390	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_EIP_NOT_CANONICAL);
9391	#endif
9392
9393	/*
9394	* PERF_GLOBAL MSR.
9395	*/
9396	if (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_PERF_MSR)
9397	{
9398	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL, &u64Val);
9399	AssertRCBreak(rc);
9400	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffff8fffffffc)),
9401	VMX_IGS_PERF_GLOBAL_MSR_RESERVED); /* Bits 63:35, bits 31:2 MBZ. */
9402	}
9403
9404	/*
9405	* PAT MSR.
9406	*/
9407	if (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_PAT_MSR)
9408	{
9409	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PAT_FULL, &u64Val);
9410	AssertRCBreak(rc);
9411	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0x707070707070707)), VMX_IGS_PAT_MSR_RESERVED);
9412	for (unsigned i = 0; i < 8; i++)
9413	{
9414	uint8_t u8Val = (u64Val & 0xff);
9415	if ( u8Val != 0 /* UC */
9416	&& u8Val != 1 /* WC */
9417	&& u8Val != 4 /* WT */
9418	&& u8Val != 5 /* WP */
9419	&& u8Val != 6 /* WB */
9420	&& u8Val != 7 /* UC- */)
9421	{
9422	HMVMX_ERROR_BREAK(VMX_IGS_PAT_MSR_INVALID);
9423	}
9424	u64Val >>= 8;
9425	}
9426	}
9427
9428	/*
9429	* EFER MSR.
9430	*/
9431	if (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_EFER_MSR)
9432	{
9433	Assert(pVM->hm.s.vmx.fSupportsVmcsEfer);
9434	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_EFER_FULL, &u64Val);
9435	AssertRCBreak(rc);
9436	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffffffffff2fe)),
9437	VMX_IGS_EFER_MSR_RESERVED); /* Bits 63:12, bit 9, bits 7:1 MBZ. */
9438	HMVMX_CHECK_BREAK(RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL( pVmcsInfo->u32EntryCtls
9439	& VMX_ENTRY_CTLS_IA32E_MODE_GUEST),
9440	VMX_IGS_EFER_LMA_GUEST_MODE_MISMATCH);
9441	/** @todo r=ramshankar: Unrestricted check here is probably wrong, see
9442	* iemVmxVmentryCheckGuestState(). */
9443	HMVMX_CHECK_BREAK( fUnrestrictedGuest
9444	\|\| !(u32GuestCr0 & X86_CR0_PG)
9445	\|\| RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL(u64Val & MSR_K6_EFER_LME),
9446	VMX_IGS_EFER_LMA_LME_MISMATCH);
9447	}
9448
9449	/*
9450	* Segment registers.
9451	*/
9452	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
9453	\|\| !(pCtx->ldtr.Sel & X86_SEL_LDT), VMX_IGS_LDTR_TI_INVALID);
9454	if (!(u32Eflags & X86_EFL_VM))
9455	{
9456	/* CS */
9457	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1Present, VMX_IGS_CS_ATTR_P_INVALID);
9458	HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xf00), VMX_IGS_CS_ATTR_RESERVED);
9459	HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xfffe0000), VMX_IGS_CS_ATTR_RESERVED);
9460	HMVMX_CHECK_BREAK( (pCtx->cs.u32Limit & 0xfff) == 0xfff
9461	\|\| !(pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
9462	HMVMX_CHECK_BREAK( !(pCtx->cs.u32Limit & 0xfff00000)
9463	\|\| (pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
9464	/* CS cannot be loaded with NULL in protected mode. */
9465	HMVMX_CHECK_BREAK(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_CS_ATTR_UNUSABLE);
9466	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1DescType, VMX_IGS_CS_ATTR_S_INVALID);
9467	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
9468	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_UNEQUAL);
9469	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
9470	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_MISMATCH);
9471	else if (pVM->hm.s.vmx.fUnrestrictedGuest && pCtx->cs.Attr.n.u4Type == 3)
9472	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == 0, VMX_IGS_CS_ATTR_DPL_INVALID);
9473	else
9474	HMVMX_ERROR_BREAK(VMX_IGS_CS_ATTR_TYPE_INVALID);
9475
9476	/* SS */
9477	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9478	\|\| (pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL), VMX_IGS_SS_CS_RPL_UNEQUAL);
9479	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL), VMX_IGS_SS_ATTR_DPL_RPL_UNEQUAL);
9480	if ( !(pCtx->cr0 & X86_CR0_PE)
9481	\|\| pCtx->cs.Attr.n.u4Type == 3)
9482	{
9483	HMVMX_CHECK_BREAK(!pCtx->ss.Attr.n.u2Dpl, VMX_IGS_SS_ATTR_DPL_INVALID);
9484	}
9485	if (!(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
9486	{
9487	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7, VMX_IGS_SS_ATTR_TYPE_INVALID);
9488	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u1Present, VMX_IGS_SS_ATTR_P_INVALID);
9489	HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xf00), VMX_IGS_SS_ATTR_RESERVED);
9490	HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xfffe0000), VMX_IGS_SS_ATTR_RESERVED);
9491	HMVMX_CHECK_BREAK( (pCtx->ss.u32Limit & 0xfff) == 0xfff
9492	\|\| !(pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
9493	HMVMX_CHECK_BREAK( !(pCtx->ss.u32Limit & 0xfff00000)
9494	\|\| (pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
9495	}
9496
9497	/* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxExportGuestSReg(). */
9498	if (!(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
9499	{
9500	HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_DS_ATTR_A_INVALID);
9501	HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u1Present, VMX_IGS_DS_ATTR_P_INVALID);
9502	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9503	\|\| pCtx->ds.Attr.n.u4Type > 11
9504	\|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
9505	HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xf00), VMX_IGS_DS_ATTR_RESERVED);
9506	HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xfffe0000), VMX_IGS_DS_ATTR_RESERVED);
9507	HMVMX_CHECK_BREAK( (pCtx->ds.u32Limit & 0xfff) == 0xfff
9508	\|\| !(pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
9509	HMVMX_CHECK_BREAK( !(pCtx->ds.u32Limit & 0xfff00000)
9510	\|\| (pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
9511	HMVMX_CHECK_BREAK( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9512	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_DS_ATTR_TYPE_INVALID);
9513	}
9514	if (!(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
9515	{
9516	HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_ES_ATTR_A_INVALID);
9517	HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u1Present, VMX_IGS_ES_ATTR_P_INVALID);
9518	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9519	\|\| pCtx->es.Attr.n.u4Type > 11
9520	\|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
9521	HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xf00), VMX_IGS_ES_ATTR_RESERVED);
9522	HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xfffe0000), VMX_IGS_ES_ATTR_RESERVED);
9523	HMVMX_CHECK_BREAK( (pCtx->es.u32Limit & 0xfff) == 0xfff
9524	\|\| !(pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
9525	HMVMX_CHECK_BREAK( !(pCtx->es.u32Limit & 0xfff00000)
9526	\|\| (pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
9527	HMVMX_CHECK_BREAK( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9528	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_ES_ATTR_TYPE_INVALID);
9529	}
9530	if (!(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
9531	{
9532	HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_FS_ATTR_A_INVALID);
9533	HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u1Present, VMX_IGS_FS_ATTR_P_INVALID);
9534	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9535	\|\| pCtx->fs.Attr.n.u4Type > 11
9536	\|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL), VMX_IGS_FS_ATTR_DPL_RPL_UNEQUAL);
9537	HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xf00), VMX_IGS_FS_ATTR_RESERVED);
9538	HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xfffe0000), VMX_IGS_FS_ATTR_RESERVED);
9539	HMVMX_CHECK_BREAK( (pCtx->fs.u32Limit & 0xfff) == 0xfff
9540	\|\| !(pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
9541	HMVMX_CHECK_BREAK( !(pCtx->fs.u32Limit & 0xfff00000)
9542	\|\| (pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
9543	HMVMX_CHECK_BREAK( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9544	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_FS_ATTR_TYPE_INVALID);
9545	}
9546	if (!(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
9547	{
9548	HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_GS_ATTR_A_INVALID);
9549	HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u1Present, VMX_IGS_GS_ATTR_P_INVALID);
9550	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9551	\|\| pCtx->gs.Attr.n.u4Type > 11
9552	\|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL), VMX_IGS_GS_ATTR_DPL_RPL_UNEQUAL);
9553	HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xf00), VMX_IGS_GS_ATTR_RESERVED);
9554	HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xfffe0000), VMX_IGS_GS_ATTR_RESERVED);
9555	HMVMX_CHECK_BREAK( (pCtx->gs.u32Limit & 0xfff) == 0xfff
9556	\|\| !(pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
9557	HMVMX_CHECK_BREAK( !(pCtx->gs.u32Limit & 0xfff00000)
9558	\|\| (pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
9559	HMVMX_CHECK_BREAK( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9560	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_GS_ATTR_TYPE_INVALID);
9561	}
9562	/* 64-bit capable CPUs. */
9563	#if HC_ARCH_BITS == 64
9564	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
9565	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
9566	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
9567	\|\| X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
9568	HMVMX_CHECK_BREAK(!RT_HI_U32(pCtx->cs.u64Base), VMX_IGS_LONGMODE_CS_BASE_INVALID);
9569	HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->ss.u64Base),
9570	VMX_IGS_LONGMODE_SS_BASE_INVALID);
9571	HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->ds.u64Base),
9572	VMX_IGS_LONGMODE_DS_BASE_INVALID);
9573	HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->es.u64Base),
9574	VMX_IGS_LONGMODE_ES_BASE_INVALID);
9575	#endif
9576	}
9577	else
9578	{
9579	/* V86 mode checks. */
9580	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
9581	if (pVmcsInfo->RealMode.fRealOnV86Active)
9582	{
9583	u32CSAttr = 0xf3; u32SSAttr = 0xf3;
9584	u32DSAttr = 0xf3; u32ESAttr = 0xf3;
9585	u32FSAttr = 0xf3; u32GSAttr = 0xf3;
9586	}
9587	else
9588	{
9589	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u;
9590	u32DSAttr = pCtx->ds.Attr.u; u32ESAttr = pCtx->es.Attr.u;
9591	u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
9592	}
9593
9594	/* CS */
9595	HMVMX_CHECK_BREAK((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), VMX_IGS_V86_CS_BASE_INVALID);
9596	HMVMX_CHECK_BREAK(pCtx->cs.u32Limit == 0xffff, VMX_IGS_V86_CS_LIMIT_INVALID);
9597	HMVMX_CHECK_BREAK(u32CSAttr == 0xf3, VMX_IGS_V86_CS_ATTR_INVALID);
9598	/* SS */
9599	HMVMX_CHECK_BREAK((pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4), VMX_IGS_V86_SS_BASE_INVALID);
9600	HMVMX_CHECK_BREAK(pCtx->ss.u32Limit == 0xffff, VMX_IGS_V86_SS_LIMIT_INVALID);
9601	HMVMX_CHECK_BREAK(u32SSAttr == 0xf3, VMX_IGS_V86_SS_ATTR_INVALID);
9602	/* DS */
9603	HMVMX_CHECK_BREAK((pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4), VMX_IGS_V86_DS_BASE_INVALID);
9604	HMVMX_CHECK_BREAK(pCtx->ds.u32Limit == 0xffff, VMX_IGS_V86_DS_LIMIT_INVALID);
9605	HMVMX_CHECK_BREAK(u32DSAttr == 0xf3, VMX_IGS_V86_DS_ATTR_INVALID);
9606	/* ES */
9607	HMVMX_CHECK_BREAK((pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4), VMX_IGS_V86_ES_BASE_INVALID);
9608	HMVMX_CHECK_BREAK(pCtx->es.u32Limit == 0xffff, VMX_IGS_V86_ES_LIMIT_INVALID);
9609	HMVMX_CHECK_BREAK(u32ESAttr == 0xf3, VMX_IGS_V86_ES_ATTR_INVALID);
9610	/* FS */
9611	HMVMX_CHECK_BREAK((pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4), VMX_IGS_V86_FS_BASE_INVALID);
9612	HMVMX_CHECK_BREAK(pCtx->fs.u32Limit == 0xffff, VMX_IGS_V86_FS_LIMIT_INVALID);
9613	HMVMX_CHECK_BREAK(u32FSAttr == 0xf3, VMX_IGS_V86_FS_ATTR_INVALID);
9614	/* GS */
9615	HMVMX_CHECK_BREAK((pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4), VMX_IGS_V86_GS_BASE_INVALID);
9616	HMVMX_CHECK_BREAK(pCtx->gs.u32Limit == 0xffff, VMX_IGS_V86_GS_LIMIT_INVALID);
9617	HMVMX_CHECK_BREAK(u32GSAttr == 0xf3, VMX_IGS_V86_GS_ATTR_INVALID);
9618	/* 64-bit capable CPUs. */
9619	#if HC_ARCH_BITS == 64
9620	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
9621	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
9622	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
9623	\|\| X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
9624	HMVMX_CHECK_BREAK(!RT_HI_U32(pCtx->cs.u64Base), VMX_IGS_LONGMODE_CS_BASE_INVALID);
9625	HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->ss.u64Base),
9626	VMX_IGS_LONGMODE_SS_BASE_INVALID);
9627	HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->ds.u64Base),
9628	VMX_IGS_LONGMODE_DS_BASE_INVALID);
9629	HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->es.u64Base),
9630	VMX_IGS_LONGMODE_ES_BASE_INVALID);
9631	#endif
9632	}
9633
9634	/*
9635	* TR.
9636	*/
9637	HMVMX_CHECK_BREAK(!(pCtx->tr.Sel & X86_SEL_LDT), VMX_IGS_TR_TI_INVALID);
9638	/* 64-bit capable CPUs. */
9639	#if HC_ARCH_BITS == 64
9640	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->tr.u64Base), VMX_IGS_TR_BASE_NOT_CANONICAL);
9641	#endif
9642	if (fLongModeGuest)
9643	{
9644	HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u4Type == 11, /* 64-bit busy TSS. */
9645	VMX_IGS_LONGMODE_TR_ATTR_TYPE_INVALID);
9646	}
9647	else
9648	{
9649	HMVMX_CHECK_BREAK( pCtx->tr.Attr.n.u4Type == 3 /* 16-bit busy TSS. */
9650	\|\| pCtx->tr.Attr.n.u4Type == 11, /* 32-bit busy TSS.*/
9651	VMX_IGS_TR_ATTR_TYPE_INVALID);
9652	}
9653	HMVMX_CHECK_BREAK(!pCtx->tr.Attr.n.u1DescType, VMX_IGS_TR_ATTR_S_INVALID);
9654	HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u1Present, VMX_IGS_TR_ATTR_P_INVALID);
9655	HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & 0xf00), VMX_IGS_TR_ATTR_RESERVED); /* Bits 11:8 MBZ. */
9656	HMVMX_CHECK_BREAK( (pCtx->tr.u32Limit & 0xfff) == 0xfff
9657	\|\| !(pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
9658	HMVMX_CHECK_BREAK( !(pCtx->tr.u32Limit & 0xfff00000)
9659	\|\| (pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
9660	HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_TR_ATTR_UNUSABLE);
9661
9662	/*
9663	* GDTR and IDTR.
9664	*/
9665	#if HC_ARCH_BITS == 64
9666	rc = VMXReadVmcs64(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
9667	AssertRCBreak(rc);
9668	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_GDTR_BASE_NOT_CANONICAL);
9669
9670	rc = VMXReadVmcs64(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
9671	AssertRCBreak(rc);
9672	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_IDTR_BASE_NOT_CANONICAL);
9673	#endif
9674
9675	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
9676	AssertRCBreak(rc);
9677	HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_GDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
9678
9679	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
9680	AssertRCBreak(rc);
9681	HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_IDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
9682
9683	/*
9684	* Guest Non-Register State.
9685	*/
9686	/* Activity State. */
9687	uint32_t u32ActivityState;
9688	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, &u32ActivityState);
9689	AssertRCBreak(rc);
9690	HMVMX_CHECK_BREAK( !u32ActivityState
9691	\|\| (u32ActivityState & RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Misc, VMX_BF_MISC_ACTIVITY_STATES)),
9692	VMX_IGS_ACTIVITY_STATE_INVALID);
9693	HMVMX_CHECK_BREAK( !(pCtx->ss.Attr.n.u2Dpl)
9694	\|\| u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_HLT, VMX_IGS_ACTIVITY_STATE_HLT_INVALID);
9695	uint32_t u32IntrState;
9696	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &u32IntrState);
9697	AssertRCBreak(rc);
9698	if ( u32IntrState == VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS
9699	\|\| u32IntrState == VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
9700	{
9701	HMVMX_CHECK_BREAK(u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_ACTIVE, VMX_IGS_ACTIVITY_STATE_ACTIVE_INVALID);
9702	}
9703
9704	/** @todo Activity state and injecting interrupts. Left as a todo since we
9705	* currently don't use activity states but ACTIVE. */
9706
9707	HMVMX_CHECK_BREAK( !(pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_ENTRY_TO_SMM)
9708	\|\| u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_SIPI_WAIT, VMX_IGS_ACTIVITY_STATE_SIPI_WAIT_INVALID);
9709
9710	/* Guest interruptibility-state. */
9711	HMVMX_CHECK_BREAK(!(u32IntrState & 0xffffffe0), VMX_IGS_INTERRUPTIBILITY_STATE_RESERVED);
9712	HMVMX_CHECK_BREAK((u32IntrState & (VMX_VMCS_GUEST_INT_STATE_BLOCK_STI \| VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS))
9713	!= (VMX_VMCS_GUEST_INT_STATE_BLOCK_STI \| VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS),
9714	VMX_IGS_INTERRUPTIBILITY_STATE_STI_MOVSS_INVALID);
9715	HMVMX_CHECK_BREAK( (u32Eflags & X86_EFL_IF)
9716	\|\| !(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI),
9717	VMX_IGS_INTERRUPTIBILITY_STATE_STI_EFL_INVALID);
9718	if (VMX_ENTRY_INT_INFO_IS_VALID(u32EntryInfo))
9719	{
9720	if (VMX_ENTRY_INT_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INT_INFO_TYPE_EXT_INT)
9721	{
9722	HMVMX_CHECK_BREAK( !(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
9723	&& !(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS),
9724	VMX_IGS_INTERRUPTIBILITY_STATE_EXT_INT_INVALID);
9725	}
9726	else if (VMX_ENTRY_INT_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INT_INFO_TYPE_NMI)
9727	{
9728	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS),
9729	VMX_IGS_INTERRUPTIBILITY_STATE_MOVSS_INVALID);
9730	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI),
9731	VMX_IGS_INTERRUPTIBILITY_STATE_STI_INVALID);
9732	}
9733	}
9734	/** @todo Assumes the processor is not in SMM. */
9735	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI),
9736	VMX_IGS_INTERRUPTIBILITY_STATE_SMI_INVALID);
9737	HMVMX_CHECK_BREAK( !(pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_ENTRY_TO_SMM)
9738	\|\| (u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI),
9739	VMX_IGS_INTERRUPTIBILITY_STATE_SMI_SMM_INVALID);
9740	if ( (pVmcsInfo->u32PinCtls & VMX_PIN_CTLS_VIRT_NMI)
9741	&& VMX_ENTRY_INT_INFO_IS_VALID(u32EntryInfo)
9742	&& VMX_ENTRY_INT_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INT_INFO_TYPE_NMI)
9743	{
9744	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI),
9745	VMX_IGS_INTERRUPTIBILITY_STATE_NMI_INVALID);
9746	}
9747
9748	/* Pending debug exceptions. */
9749	#if HC_ARCH_BITS == 64
9750	rc = VMXReadVmcs64(VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, &u64Val);
9751	AssertRCBreak(rc);
9752	/* Bits 63:15, Bit 13, Bits 11:4 MBZ. */
9753	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffffaff0)), VMX_IGS_LONGMODE_PENDING_DEBUG_RESERVED);
9754	u32Val = u64Val; /* For pending debug exceptions checks below. */
9755	#else
9756	rc = VMXReadVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, &u32Val);
9757	AssertRCBreak(rc);
9758	/* Bits 31:15, Bit 13, Bits 11:4 MBZ. */
9759	HMVMX_CHECK_BREAK(!(u32Val & 0xffffaff0), VMX_IGS_PENDING_DEBUG_RESERVED);
9760	#endif
9761
9762	if ( (u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
9763	\|\| (u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS)
9764	\|\| u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_HLT)
9765	{
9766	if ( (u32Eflags & X86_EFL_TF)
9767	&& !(u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
9768	{
9769	/* Bit 14 is PendingDebug.BS. */
9770	HMVMX_CHECK_BREAK(u32Val & RT_BIT(14), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_SET);
9771	}
9772	if ( !(u32Eflags & X86_EFL_TF)
9773	\|\| (u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
9774	{
9775	/* Bit 14 is PendingDebug.BS. */
9776	HMVMX_CHECK_BREAK(!(u32Val & RT_BIT(14)), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_CLEAR);
9777	}
9778	}
9779
9780	/* VMCS link pointer. */
9781	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, &u64Val);
9782	AssertRCBreak(rc);
9783	if (u64Val != UINT64_C(0xffffffffffffffff))
9784	{
9785	HMVMX_CHECK_BREAK(!(u64Val & 0xfff), VMX_IGS_VMCS_LINK_PTR_RESERVED);
9786	/** @todo Bits beyond the processor's physical-address width MBZ. */
9787	/** @todo 32-bit located in memory referenced by value of this field (as a
9788	* physical address) must contain the processor's VMCS revision ID. */
9789	/** @todo SMM checks. */
9790	}
9791
9792	/** @todo Checks on Guest Page-Directory-Pointer-Table Entries when guest is
9793	* not using nested paging? */
9794	if ( pVM->hm.s.fNestedPaging
9795	&& !fLongModeGuest
9796	&& CPUMIsGuestInPAEModeEx(pCtx))
9797	{
9798	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &u64Val);
9799	AssertRCBreak(rc);
9800	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9801
9802	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &u64Val);
9803	AssertRCBreak(rc);
9804	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9805
9806	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &u64Val);
9807	AssertRCBreak(rc);
9808	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9809
9810	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &u64Val);
9811	AssertRCBreak(rc);
9812	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9813	}
9814
9815	/* Shouldn't happen but distinguish it from AssertRCBreak() errors. */
9816	if (uError == VMX_IGS_ERROR)
9817	uError = VMX_IGS_REASON_NOT_FOUND;
9818	} while (0);
9819
9820	pVCpu->hm.s.u32HMError = uError;
9821	return uError;
9822
9823	#undef HMVMX_ERROR_BREAK
9824	#undef HMVMX_CHECK_BREAK
9825	}
9826
9827
9828	/**
9829	* Setup the APIC-access page for virtualizing APIC access.
9830	*
9831	* This can cause a longjumps to R3 due to the acquisition of the PGM lock, hence
9832	* this not done as part of exporting guest state, see @bugref{8721}.
9833	*
9834	* @returns VBox status code.
9835	* @param pVCpu The cross context virtual CPU structure.
9836	*/
9837	static int hmR0VmxMapHCApicAccessPage(PVMCPU pVCpu)
9838	{
9839	PVM pVM = pVCpu->CTX_SUFF(pVM);
9840	uint64_t const u64MsrApicBase = APICGetBaseMsrNoCheck(pVCpu);
9841
9842	Assert(PDMHasApic(pVM));
9843	Assert(u64MsrApicBase);
9844
9845	RTGCPHYS const GCPhysApicBase = u64MsrApicBase & PAGE_BASE_GC_MASK;
9846	Log4Func(("Mappping HC APIC-access page at %#RGp\n", GCPhysApicBase));
9847
9848	/* Unalias any existing mapping. */
9849	int rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
9850	AssertRCReturn(rc, rc);
9851
9852	/* Map the HC APIC-access page in place of the MMIO page, also updates the shadow page tables if necessary. */
9853	Assert(pVM->hm.s.vmx.HCPhysApicAccess != NIL_RTHCPHYS);
9854	rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW \| X86_PTE_P);
9855	AssertRCReturn(rc, rc);
9856
9857	/* Update the per-VCPU cache of the APIC base MSR. */
9858	pVCpu->hm.s.vmx.u64GstMsrApicBase = u64MsrApicBase;
9859	return VINF_SUCCESS;
9860	}
9861
9862
9863	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
9864	/**
9865	* Merges the guest with the nested-guest MSR bitmap in preparation of executing the
9866	* nested-guest using hardware-assisted VMX.
9867	*
9868	* @param pVCpu The cross context virtual CPU structure.
9869	* @param pVmcsInfoNstGst The nested-guest VMCS info. object.
9870	* @param pVmcsInfoGst The guest VMCS info. object.
9871	*/
9872	static void hmR0VmxMergeMsrBitmapNested(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfoNstGst, PCVMXVMCSINFO pVmcsInfoGst)
9873	{
9874	uint64_t const pu64MsrBitmapNstGst = (uint64_t const )pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvMsrBitmap);
9875	uint64_t const pu64MsrBitmapGst = (uint64_t const )pVmcsInfoGst->pvMsrBitmap;
9876	uint64_t pu64MsrBitmap = (uint64_t )pVmcsInfoNstGst->pvMsrBitmap;
9877	Assert(pu64MsrBitmapNstGst);
9878	Assert(pu64MsrBitmapGst);
9879	Assert(pu64MsrBitmap);
9880
9881	/*
9882	* We merge the guest MSR bitmap with the nested-guest MSR bitmap such that any
9883	* MSR that is intercepted by the guest is also intercepted while executing the
9884	* nested-guest using hardware-assisted VMX.
9885	*/
9886	uint32_t const cbFrag = sizeof(uint64_t);
9887	uint32_t const cFrags = X86_PAGE_4K_SIZE / cbFrag;
9888	for (uint32_t i = 0; i <= cFrags; i++)
9889	pu64MsrBitmap[i] = pu64MsrBitmapNstGst[i] \| pu64MsrBitmapGst[i];
9890	}
9891
9892
9893	/**
9894	* Merges the guest VMCS in to the nested-guest VMCS controls in preparation of
9895	* hardware-assisted VMX execution of the nested-guest.
9896	*
9897	* For a guest, we don't modify these controls once we set up the VMCS.
9898	*
9899	* For nested-guests since the guest hypervisor provides these controls on every
9900	* nested-guest VM-entry and could potentially change them everytime we need to
9901	* merge them before every nested-guest VM-entry.
9902	*
9903	* @returns VBox status code.
9904	* @param pVCpu The cross context virtual CPU structure.
9905	*/
9906	static int hmR0VmxMergeVmcsNested(PVMCPU pVCpu)
9907	{
9908	PVM pVM = pVCpu->CTX_SUFF(pVM);
9909	PCVMXVMCSINFO pVmcsInfoGst = &pVCpu->hm.s.vmx.VmcsInfo;
9910	PCVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
9911	Assert(pVmcsNstGst);
9912
9913	/*
9914	* Merge the controls with the requirements of the guest VMCS.
9915	*
9916	* We do not need to validate the nested-guest VMX features specified in the
9917	* nested-guest VMCS with the features supported by the physical CPU as it's
9918	* already done by the VMLAUNCH/VMRESUME instruction emulation.
9919	*
9920	* This is because the VMX features exposed by CPUM (through CPUID/MSRs) to the
9921	* guest are derived from the VMX features supported by the physical CPU.
9922	*/
9923
9924	/* Pin-based VM-execution controls. */
9925	uint32_t const u32PinCtls = pVmcsNstGst->u32PinCtls \| pVmcsInfoGst->u32PinCtls;
9926
9927	/* Processor-based VM-execution controls. */
9928	uint32_t u32ProcCtls = (pVmcsNstGst->u32ProcCtls & ~VMX_PROC_CTLS_USE_IO_BITMAPS)
9929	\| (pVmcsInfoGst->u32ProcCtls & ~( VMX_PROC_CTLS_INT_WINDOW_EXIT
9930	\| VMX_PROC_CTLS_NMI_WINDOW_EXIT
9931	\| VMX_PROC_CTLS_USE_TPR_SHADOW
9932	\| VMX_PROC_CTLS_MONITOR_TRAP_FLAG));
9933
9934	/* Secondary processor-based VM-execution controls. */
9935	uint32_t const u32ProcCtls2 = (pVmcsNstGst->u32ProcCtls2 & ~VMX_PROC_CTLS2_VPID)
9936	\| (pVmcsInfoGst->u32ProcCtls2 & ~( VMX_PROC_CTLS2_VIRT_APIC_ACCESS
9937	\| VMX_PROC_CTLS2_INVPCID
9938	\| VMX_PROC_CTLS2_RDTSCP
9939	\| VMX_PROC_CTLS2_XSAVES_XRSTORS
9940	\| VMX_PROC_CTLS2_APIC_REG_VIRT
9941	\| VMX_PROC_CTLS2_VIRT_INT_DELIVERY
9942	\| VMX_PROC_CTLS2_VMFUNC));
9943
9944	/*
9945	* VM-entry controls:
9946	* These controls contains state that depends on the nested-guest state (primarily
9947	* EFER MSR) and is thus not constant through VMLAUNCH/VMRESUME and the nested-guest
9948	* VM-exit. Although the nested-hypervisor cannot change it, we need to in order to
9949	* properly continue executing the nested-guest if the EFER MSR changes but does not
9950	* cause a nested-guest VM-exits.
9951	*
9952	* VM-exit controls:
9953	* These controls specify the host state on return. We cannot use the controls from
9954	* the nested-hypervisor state as is as it would contain the guest state rather than
9955	* the host state. Since the host state is subject to change (e.g. preemption, trips
9956	* to ring-3, longjmp and rescheduling to a different host CPU) they are not constant
9957	* through VMLAUNCH/VMRESUME and the nested-guest VM-exit.
9958	*
9959	* VM-entry MSR-load:
9960	* The guest MSRs from the VM-entry MSR-load area are already loaded into the
9961	* guest-CPU context by the VMLAUNCH/VMRESUME instruction emulation.
9962	*
9963	* VM-exit MSR-store:
9964	* The VM-exit emulation will take care of populating the MSRs from the guest-CPU
9965	* context back into the VM-exit MSR-store area.
9966	*
9967	* VM-exit MSR-load areas:
9968	* This must contain the real host MSRs with hardware-assisted VMX execution. Hence,
9969	* we can entirely ignore what the nested-hypervisor wants to load here.
9970	*/
9971
9972	/*
9973	* Exception bitmap.
9974	*
9975	* We could remove #UD from the guest bitmap and merge it with the nested-guest
9976	* bitmap here (and avoid doing anything while exporting nested-guest state), but to
9977	* keep the code more flexible if intercepting exceptions become more dynamic in
9978	* the future we do it as part of exporting the nested-guest state.
9979	*/
9980	uint32_t const u32XcptBitmap = pVmcsNstGst->u32XcptBitmap \| pVmcsInfoGst->u32XcptBitmap;
9981
9982	/*
9983	* CR0/CR4 guest/host mask.
9984	*
9985	* Modifications by the nested-guest to CR0/CR4 bits owned by the host and the guest
9986	* must cause VM-exits, so we need to merge them here.
9987	*/
9988	uint64_t const u64Cr0Mask = pVmcsNstGst->u64Cr0Mask.u \| pVmcsInfoGst->u64Cr0Mask;
9989	uint64_t const u64Cr4Mask = pVmcsNstGst->u64Cr4Mask.u \| pVmcsInfoGst->u64Cr4Mask;
9990
9991	/*
9992	* Page-fault error-code mask and match.
9993	*
9994	* Although we require unrestricted guest execution (and thereby nested-paging) for
9995	* hardware-assisted VMX execution of nested-guests and thus the outer guest doesn't
9996	* normally intercept #PFs, it might intercept them for debugging purposes.
9997	*
9998	* If the outer guest is not intercepting #PFs, we can use the nested-guest #PF
9999	* filters. If the outer guest is intercepting #PFs we must intercept all #PFs.
10000	*/
10001	uint32_t u32XcptPFMask;
10002	uint32_t u32XcptPFMatch;
10003	if (!(pVmcsInfoGst->u32XcptBitmap & RT_BIT(X86_XCPT_PF)))
10004	{
10005	u32XcptPFMask = pVmcsNstGst->u32XcptPFMask;
10006	u32XcptPFMatch = pVmcsNstGst->u32XcptPFMatch;
10007	}
10008	else
10009	{
10010	u32XcptPFMask = 0;
10011	u32XcptPFMatch = 0;
10012	}
10013
10014	/*
10015	* Pause-Loop exiting.
10016	*/
10017	uint32_t const cPleGapTicks = RT_MIN(pVM->hm.s.vmx.cPleGapTicks, pVmcsNstGst->u32PleGap);
10018	uint32_t const cPleWindowTicks = RT_MIN(pVM->hm.s.vmx.cPleWindowTicks, pVmcsNstGst->u32PleWindow);
10019
10020	/*
10021	* I/O Bitmap.
10022	*
10023	* We do not use the I/O bitmap that may be provided by the guest hypervisor as we
10024	* always intercept all I/O port accesses.
10025	*/
10026	Assert(u32ProcCtls & VMX_PROC_CTLS_UNCOND_IO_EXIT);
10027
10028	/*
10029	* APIC-access page.
10030	*
10031	* The APIC-access page address has already been initialized while setting up the
10032	* nested-guest VMCS. In theory, even if the guest-physical address is invalid, it
10033	* should not be on any consequence to the host or to the guest for that matter, but
10034	* we only accept valid addresses verified by the VMLAUNCH/VMRESUME instruction
10035	* emulation to keep it simple.
10036	*/
10037
10038	/*
10039	* Virtual-APIC page and TPR threshold.
10040	*
10041	* We shall use the host-physical address of the virtual-APIC page in guest memory directly.
10042	* For this reason, we can access the virtual-APIC page of the nested-guest only using
10043	* PGM physical handlers as we must not assume a kernel virtual-address mapping exists and
10044	* requesting PGM for a mapping could be expensive/resource intensive (PGM mapping cache).
10045	*/
10046	RTHCPHYS HCPhysVirtApic = NIL_RTHCPHYS;
10047	uint32_t const u32TprThreshold = pVmcsNstGst->u32TprThreshold;
10048	if (u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
10049	{
10050	int rc = PGMPhysGCPhys2HCPhys(pVM, pVmcsNstGst->u64AddrVirtApic.u, &HCPhysVirtApic);
10051
10052	/*
10053	* If the guest hypervisor has loaded crap into the virtual-APIC page field
10054	* we would fail to obtain a valid host-physical address for its guest-physical
10055	* address.
10056	*
10057	* We currently do not support this scenario. Maybe in the future if there is a
10058	* pressing need we can explore making this particular set of conditions work.
10059	* Right now we just cause a VM-entry failure.
10060	*
10061	* This has already been checked by VMLAUNCH/VMRESUME instruction emulation,
10062	* so should not really failure at the moment.
10063	*/
10064	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
10065	}
10066	else
10067	{
10068	/*
10069	* We must make sure CR8 reads/write must cause VM-exits when TPR shadowing is not
10070	* used by the guest hypervisor. Preventing MMIO accesses to the physical APIC will
10071	* be taken care of by EPT/shadow paging.
10072	*/
10073	if (pVM->hm.s.fAllow64BitGuests)
10074	{
10075	u32ProcCtls \|= VMX_PROC_CTLS_CR8_STORE_EXIT
10076	\| VMX_PROC_CTLS_CR8_LOAD_EXIT;
10077	}
10078	}
10079
10080	/*
10081	* Validate basic assumptions.
10082	*/
10083	PVMXVMCSINFO pVmcsInfoNstGst = &pVCpu->hm.s.vmx.VmcsInfoNstGst;
10084	Assert(pVM->hm.s.vmx.fAllowUnrestricted);
10085	Assert(pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_SECONDARY_CTLS);
10086	Assert(hmGetVmxActiveVmcsInfo(pVCpu) == pVmcsInfoNstGst);
10087
10088	/*
10089	* Commit it to the nested-guest VMCS.
10090	*/
10091	int rc = VINF_SUCCESS;
10092	if (pVmcsInfoNstGst->u32PinCtls != u32PinCtls)
10093	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, u32PinCtls);
10094	if (pVmcsInfoNstGst->u32ProcCtls != u32ProcCtls)
10095	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, u32ProcCtls);
10096	if (pVmcsInfoNstGst->u32ProcCtls2 != u32ProcCtls2)
10097	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, u32ProcCtls2);
10098	if (pVmcsInfoNstGst->u32XcptBitmap != u32XcptBitmap)
10099	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
10100	if (pVmcsInfoNstGst->u64Cr0Mask != u64Cr0Mask)
10101	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, u64Cr0Mask);
10102	if (pVmcsInfoNstGst->u64Cr4Mask != u64Cr4Mask)
10103	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, u64Cr4Mask);
10104	if (pVmcsInfoNstGst->u32XcptPFMask != u32XcptPFMask)
10105	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, u32XcptPFMask);
10106	if (pVmcsInfoNstGst->u32XcptPFMatch != u32XcptPFMatch)
10107	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, u32XcptPFMatch);
10108	if ( !(u32ProcCtls & VMX_PROC_CTLS_PAUSE_EXIT)
10109	&& (u32ProcCtls2 & VMX_PROC_CTLS2_PAUSE_LOOP_EXIT))
10110	{
10111	Assert(pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_PAUSE_LOOP_EXIT);
10112	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_GAP, cPleGapTicks);
10113	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_WINDOW, cPleWindowTicks);
10114	}
10115	if (u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
10116	{
10117	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
10118	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_VIRT_APIC_PAGEADDR_FULL, HCPhysVirtApic);
10119	}
10120	AssertRCReturn(rc, rc);
10121
10122	/*
10123	* Update the nested-guest VMCS cache.
10124	*/
10125	pVmcsInfoNstGst->u32PinCtls = u32PinCtls;
10126	pVmcsInfoNstGst->u32ProcCtls = u32ProcCtls;
10127	pVmcsInfoNstGst->u32ProcCtls2 = u32ProcCtls2;
10128	pVmcsInfoNstGst->u32XcptBitmap = u32XcptBitmap;
10129	pVmcsInfoNstGst->u64Cr0Mask = u64Cr0Mask;
10130	pVmcsInfoNstGst->u64Cr4Mask = u64Cr4Mask;
10131	pVmcsInfoNstGst->u32XcptPFMask = u32XcptPFMask;
10132	pVmcsInfoNstGst->u32XcptPFMatch = u32XcptPFMatch;
10133	pVmcsInfoNstGst->HCPhysVirtApic = HCPhysVirtApic;
10134
10135	/*
10136	* MSR bitmap.
10137	*
10138	* The MSR bitmap address has already been initialized while setting up the
10139	* nested-guest VMCS, here we need to merge the MSR bitmaps.
10140	*/
10141	if (u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
10142	hmR0VmxMergeMsrBitmapNested(pVCpu, pVmcsInfoNstGst, pVmcsInfoGst);
10143
10144	return VINF_SUCCESS;
10145	}
10146	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
10147
10148
10149	/**
10150	* Does the preparations before executing guest code in VT-x.
10151	*
10152	* This may cause longjmps to ring-3 and may even result in rescheduling to the
10153	* recompiler/IEM. We must be cautious what we do here regarding committing
10154	* guest-state information into the VMCS assuming we assuredly execute the
10155	* guest in VT-x mode.
10156	*
10157	* If we fall back to the recompiler/IEM after updating the VMCS and clearing
10158	* the common-state (TRPM/forceflags), we must undo those changes so that the
10159	* recompiler/IEM can (and should) use them when it resumes guest execution.
10160	* Otherwise such operations must be done when we can no longer exit to ring-3.
10161	*
10162	* @returns Strict VBox status code (i.e. informational status codes too).
10163	* @retval VINF_SUCCESS if we can proceed with running the guest, interrupts
10164	* have been disabled.
10165	* @retval VINF_EM_RESET if a triple-fault occurs while injecting a
10166	* double-fault into the guest.
10167	* @retval VINF_EM_DBG_STEPPED if @a fStepping is true and an event was
10168	* dispatched directly.
10169	* @retval VINF_* scheduling changes, we have to go back to ring-3.
10170	*
10171	* @param pVCpu The cross context virtual CPU structure.
10172	* @param pVmxTransient The VMX-transient structure.
10173	* @param fStepping Whether we are single-stepping the guest in the
10174	* hypervisor debugger. Makes us ignore some of the reasons
10175	* for returning to ring-3, and return VINF_EM_DBG_STEPPED
10176	* if event dispatching took place.
10177	*/
10178	static VBOXSTRICTRC hmR0VmxPreRunGuest(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, bool fStepping)
10179	{
10180	Assert(VMMRZCallRing3IsEnabled(pVCpu));
10181
10182	#ifdef VBOX_WITH_NESTED_HWVIRT_ONLY_IN_IEM
10183	if (pVmxTransient->fIsNestedGuest)
10184	{
10185	RT_NOREF2(pVCpu, fStepping);
10186	Log2Func(("Rescheduling to IEM due to nested-hwvirt or forced IEM exec -> VINF_EM_RESCHEDULE_REM\n"));
10187	return VINF_EM_RESCHEDULE_REM;
10188	}
10189	#endif
10190
10191	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
10192	PGMRZDynMapFlushAutoSet(pVCpu);
10193	#endif
10194
10195	/*
10196	* Check and process force flag actions, some of which might require us to go back to ring-3.
10197	*/
10198	VBOXSTRICTRC rcStrict = hmR0VmxCheckForceFlags(pVCpu, fStepping);
10199	if (rcStrict == VINF_SUCCESS)
10200	{ /* FFs don't get set all the time. */ }
10201	else
10202	return rcStrict;
10203
10204	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
10205	/*
10206	* Switch to the nested-guest VMCS as we may have transitioned into executing
10207	* the nested-guest without leaving ring-0. Otherwise, if we came from ring-3
10208	* we would load the nested-guest VMCS while entering the VMX ring-0 session.
10209	*
10210	* We do this as late as possible to minimize (though not completely remove)
10211	* clearing/loading VMCS again due to premature trips to ring-3 above.
10212	*/
10213	if (pVmxTransient->fIsNestedGuest)
10214	{
10215	if (!pVCpu->hm.s.vmx.fSwitchedToNstGstVmcs)
10216	{
10217	/*
10218	* Ensure we have synced everything from the guest VMCS and also flag that
10219	* that we need to export the full (nested) guest-CPU context to the
10220	* nested-guest VMCS.
10221	*/
10222	HMVMX_CPUMCTX_ASSERT(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
10223	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_ALL_GUEST);
10224
10225	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
10226	int rc = hmR0VmxSwitchVmcs(&pVCpu->hm.s.vmx.VmcsInfo, &pVCpu->hm.s.vmx.VmcsInfoNstGst);
10227	if (RT_LIKELY(rc == VINF_SUCCESS))
10228	{
10229	pVCpu->hm.s.vmx.fSwitchedToNstGstVmcs = true;
10230	ASMSetFlags(fEFlags);
10231	pVmxTransient->pVmcsInfo = &pVCpu->hm.s.vmx.VmcsInfoNstGst;
10232
10233	/*
10234	* We use a different VM-exit MSR-store area for the nested-guest. Hence,
10235	* flag that we need to update the host MSR values there. Even if we decide
10236	* in the future to share the VM-exit MSR-store area page with the guest,
10237	* if its content differs, we would have to update the host MSRs anyway.
10238	*/
10239	pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs = false;
10240	Assert(!pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer); /** @todo NSTVMX: Paranoia remove later. */
10241	}
10242	else
10243	{
10244	ASMSetFlags(fEFlags);
10245	return rc;
10246	}
10247	}
10248
10249	/*
10250	* Merge guest VMCS controls with the nested-guest VMCS controls.
10251	*
10252	* Even if we have not executed the guest prior to this (e.g. when resuming
10253	* from a saved state), we should be okay with merging controls as we
10254	* initialize the guest VMCS controls as part of VM setup phase.
10255	*/
10256	if (!pVCpu->hm.s.vmx.fMergedNstGstCtls)
10257	{
10258	int rc = hmR0VmxMergeVmcsNested(pVCpu);
10259	AssertRCReturn(rc, rc);
10260	pVCpu->hm.s.vmx.fMergedNstGstCtls = true;
10261	}
10262	}
10263	#endif
10264
10265	/*
10266	* Virtualize memory-mapped accesses to the physical APIC (may take locks).
10267	* We look at the guest VMCS control here as we always set it when supported by
10268	* the physical CPU. Looking at the nested-guest control here would not be
10269	* possible because they are not merged yet.
10270	*/
10271	PVM pVM = pVCpu->CTX_SUFF(pVM);
10272	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
10273	if ( !pVCpu->hm.s.vmx.u64GstMsrApicBase
10274	&& (pVmcsInfo->u32ProcCtls2 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
10275	&& PDMHasApic(pVM))
10276	{
10277	int rc = hmR0VmxMapHCApicAccessPage(pVCpu);
10278	AssertRCReturn(rc, rc);
10279	}
10280
10281	/*
10282	* Evaluate events to be injected into the guest.
10283	*
10284	* Events in TRPM can be injected without inspecting the guest state.
10285	* If any new events (interrupts/NMI) are pending currently, we try to set up the
10286	* guest to cause a VM-exit the next time they are ready to receive the event.
10287	*/
10288	if (TRPMHasTrap(pVCpu))
10289	hmR0VmxTrpmTrapToPendingEvent(pVCpu);
10290
10291	uint32_t fIntrState;
10292	rcStrict = hmR0VmxEvaluatePendingEvent(pVCpu, pVmxTransient, &fIntrState);
10293
10294	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
10295	/*
10296	* While evaluating pending events if something failed (unlikely) or if we were
10297	* preparing to run a nested-guest but performed a nested-guest VM-exit, we should bail.
10298	*/
10299	if ( rcStrict != VINF_SUCCESS
10300	\|\| ( pVmxTransient->fIsNestedGuest
10301	&& !CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx)))
10302	return rcStrict;
10303	#endif
10304
10305	/*
10306	* Event injection may take locks (currently the PGM lock for real-on-v86 case) and thus
10307	* needs to be done with longjmps or interrupts + preemption enabled. Event injection might
10308	* also result in triple-faulting the VM.
10309	*
10310	* The above does not apply when executing a nested-guest (since unrestricted guest execution
10311	* is a requirement) regardless doing it avoid duplicating code elsewhere.
10312	*/
10313	rcStrict = hmR0VmxInjectPendingEvent(pVCpu, pVmxTransient, fIntrState, fStepping);
10314	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
10315	{ /* likely */ }
10316	else
10317	{
10318	AssertMsg(rcStrict == VINF_EM_RESET \|\| (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
10319	("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
10320	return rcStrict;
10321	}
10322
10323	/*
10324	* A longjump might result in importing CR3 even for VM-exits that don't necessarily
10325	* import CR3 themselves. We will need to update them here, as even as late as the above
10326	* hmR0VmxInjectPendingEvent() call may lazily import guest-CPU state on demand causing
10327	* the below force flags to be set.
10328	*/
10329	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
10330	{
10331	Assert(!(ASMAtomicUoReadU64(&pVCpu->cpum.GstCtx.fExtrn) & CPUMCTX_EXTRN_CR3));
10332	int rc2 = PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
10333	AssertMsgReturn(rc2 == VINF_SUCCESS \|\| rc2 == VINF_PGM_SYNC_CR3,
10334	("%Rrc\n", rc2), RT_FAILURE_NP(rc2) ? rc2 : VERR_IPE_UNEXPECTED_INFO_STATUS);
10335	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
10336	}
10337	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
10338	{
10339	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
10340	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
10341	}
10342
10343	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
10344	/* Paranoia. */
10345	Assert(!pVmxTransient->fIsNestedGuest \|\| CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx));
10346	#endif
10347
10348	/*
10349	* No longjmps to ring-3 from this point on!!!
10350	* Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
10351	* This also disables flushing of the R0-logger instance (if any).
10352	*/
10353	VMMRZCallRing3Disable(pVCpu);
10354
10355	/*
10356	* Export the guest state bits.
10357	*
10358	* We cannot perform longjmps while loading the guest state because we do not preserve the
10359	* host/guest state (although the VMCS will be preserved) across longjmps which can cause
10360	* CPU migration.
10361	*
10362	* If we are injecting events to a real-on-v86 mode guest, we would have updated RIP and some segment
10363	* registers. Hence, loading of the guest state needs to be done -after- injection of events.
10364	*/
10365	rcStrict = hmR0VmxExportGuestStateOptimal(pVCpu, pVmxTransient);
10366	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
10367	{ /* likely */ }
10368	else
10369	{
10370	VMMRZCallRing3Enable(pVCpu);
10371	return rcStrict;
10372	}
10373
10374	/*
10375	* We disable interrupts so that we don't miss any interrupts that would flag preemption
10376	* (IPI/timers etc.) when thread-context hooks aren't used and we've been running with
10377	* preemption disabled for a while. Since this is purely to aid the
10378	* RTThreadPreemptIsPending() code, it doesn't matter that it may temporarily reenable and
10379	* disable interrupt on NT.
10380	*
10381	* We need to check for force-flags that could've possible been altered since we last
10382	* checked them (e.g. by PDMGetInterrupt() leaving the PDM critical section,
10383	* see @bugref{6398}).
10384	*
10385	* We also check a couple of other force-flags as a last opportunity to get the EMT back
10386	* to ring-3 before executing guest code.
10387	*/
10388	pVmxTransient->fEFlags = ASMIntDisableFlags();
10389
10390	if ( ( !VM_FF_IS_ANY_SET(pVM, VM_FF_EMT_RENDEZVOUS \| VM_FF_TM_VIRTUAL_SYNC)
10391	&& !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
10392	\|\| ( fStepping /* Optimized for the non-stepping case, so a bit of unnecessary work when stepping. */
10393	&& !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HM_TO_R3_MASK & ~(VMCPU_FF_TIMER \| VMCPU_FF_PDM_CRITSECT))) )
10394	{
10395	if (!RTThreadPreemptIsPending(NIL_RTTHREAD))
10396	{
10397	pVCpu->hm.s.Event.fPending = false;
10398
10399	/*
10400	* We've injected any pending events. This is really the point of no return (to ring-3).
10401	*
10402	* Note! The caller expects to continue with interrupts & longjmps disabled on successful
10403	* returns from this function, so don't enable them here.
10404	*/
10405	return VINF_SUCCESS;
10406	}
10407
10408	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPendingHostIrq);
10409	rcStrict = VINF_EM_RAW_INTERRUPT;
10410	}
10411	else
10412	{
10413	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
10414	rcStrict = VINF_EM_RAW_TO_R3;
10415	}
10416
10417	ASMSetFlags(pVmxTransient->fEFlags);
10418	VMMRZCallRing3Enable(pVCpu);
10419
10420	return rcStrict;
10421	}
10422
10423
10424	/**
10425	* Final preparations before executing guest code using hardware-assisted VMX.
10426	*
10427	* We can no longer get preempted to a different host CPU and there are no returns
10428	* to ring-3. We ignore any errors that may happen from this point (e.g. VMWRITE
10429	* failures), this function is not intended to fail sans unrecoverable hardware
10430	* errors.
10431	*
10432	* @param pVCpu The cross context virtual CPU structure.
10433	* @param pVmxTransient The VMX-transient structure.
10434	*
10435	* @remarks Called with preemption disabled.
10436	* @remarks No-long-jump zone!!!
10437	*/
10438	static void hmR0VmxPreRunGuestCommitted(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
10439	{
10440	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
10441	Assert(VMMR0IsLogFlushDisabled(pVCpu));
10442	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
10443	Assert(!pVCpu->hm.s.Event.fPending);
10444
10445	/*
10446	* Indicate start of guest execution and where poking EMT out of guest-context is recognized.
10447	*/
10448	VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
10449	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
10450
10451	PVM pVM = pVCpu->CTX_SUFF(pVM);
10452	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
10453
10454	if (!CPUMIsGuestFPUStateActive(pVCpu))
10455	{
10456	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestFpuState, x);
10457	if (CPUMR0LoadGuestFPU(pVM, pVCpu) == VINF_CPUM_HOST_CR0_MODIFIED)
10458	pVCpu->hm.s.fCtxChanged \|= HM_CHANGED_HOST_CONTEXT;
10459	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestFpuState, x);
10460	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadGuestFpu);
10461	}
10462
10463	/*
10464	* Re-save the host state bits as we may've been preempted (only happens when
10465	* thread-context hooks are used or when the VM start function changes).
10466	* The 64-on-32 switcher saves the (64-bit) host state into the VMCS and if we
10467	* changed the switcher back to 32-bit, we must save the 32-bit host state here,
10468	* see @bugref{8432}.
10469	*
10470	* This may also happen when switching to/from a nested-guest VMCS without leaving
10471	* ring-0.
10472	*/
10473	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT)
10474	{
10475	int rc = hmR0VmxExportHostState(pVCpu);
10476	AssertRC(rc);
10477	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreemptExportHostState);
10478	}
10479	Assert(!(pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT));
10480
10481	/*
10482	* Export the state shared between host and guest (FPU, debug, lazy MSRs).
10483	*/
10484	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)
10485	hmR0VmxExportSharedState(pVCpu, pVmxTransient);
10486	AssertMsg(!pVCpu->hm.s.fCtxChanged, ("fCtxChanged=%#RX64\n", pVCpu->hm.s.fCtxChanged));
10487
10488	/*
10489	* Store status of the shared guest/host debug state at the time of VM-entry.
10490	*/
10491	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
10492	if (CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx))
10493	{
10494	pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActivePending(pVCpu);
10495	pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActivePending(pVCpu);
10496	}
10497	else
10498	#endif
10499	{
10500	pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActive(pVCpu);
10501	pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActive(pVCpu);
10502	}
10503
10504	/*
10505	* Always cache the TPR-shadow if the virtual-APIC page exists, thereby skipping
10506	* more than one conditional check. The post-run side of our code shall determine
10507	* if it needs to sync. the virtual APIC TPR with the TPR-shadow.
10508	*/
10509	if (pVmcsInfo->pbVirtApic)
10510	pVmxTransient->u8GuestTpr = pVmcsInfo->pbVirtApic[XAPIC_OFF_TPR];
10511
10512	/*
10513	* Update the host MSRs values in the VM-exit MSR-load area.
10514	*/
10515	if (!pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs)
10516	{
10517	if (pVmcsInfo->cExitMsrLoad > 0)
10518	hmR0VmxUpdateAutoLoadHostMsrs(pVCpu, pVmcsInfo);
10519	pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs = true;
10520	}
10521
10522	/*
10523	* Evaluate if we need to intercept guest RDTSC/P accesses. Set up the
10524	* VMX-preemption timer based on the next virtual sync clock deadline.
10525	*/
10526	PHMPHYSCPU pHostCpu = hmR0GetCurrentCpu();
10527	RTCPUID const idCurrentCpu = pHostCpu->idCpu;
10528	if ( !pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer
10529	\|\| idCurrentCpu != pVCpu->hm.s.idLastCpu)
10530	{
10531	hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVCpu, pVmxTransient);
10532	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = true;
10533	}
10534
10535	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB flushing, set this across the world switch. */
10536	hmR0VmxFlushTaggedTlb(pHostCpu, pVCpu, pVmcsInfo); /* Invalidate the appropriate guest entries from the TLB. */
10537	Assert(idCurrentCpu == pVCpu->hm.s.idLastCpu);
10538	pVCpu->hm.s.vmx.LastError.idCurrentCpu = idCurrentCpu; /* Update the error reporting info. with the current host CPU. */
10539
10540	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
10541
10542	TMNotifyStartOfExecution(pVCpu); /* Notify TM to resume its clocks when TSC is tied to execution,
10543	as we're about to start executing the guest . */
10544
10545	/*
10546	* Load the guest TSC_AUX MSR when we are not intercepting RDTSCP.
10547	*
10548	* This is done this late as updating the TSC offsetting/preemption timer above
10549	* figures out if we can skip intercepting RDTSCP by calculating the number of
10550	* host CPU ticks till the next virtual sync deadline (for the dynamic case).
10551	*/
10552	if (pVmcsInfo->u32ProcCtls2 & VMX_PROC_CTLS2_RDTSCP)
10553	{
10554	if (!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_RDTSC_EXIT))
10555	{
10556	hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_TSC_AUX);
10557	int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K8_TSC_AUX, CPUMGetGuestTscAux(pVCpu),
10558	true /* fSetReadWrite /, true / fUpdateHostMsr */);
10559	AssertRC(rc);
10560	}
10561	else
10562	hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K8_TSC_AUX);
10563	}
10564
10565	#ifdef VBOX_STRICT
10566	hmR0VmxCheckAutoLoadStoreMsrs(pVCpu, pVmcsInfo);
10567	hmR0VmxCheckHostEferMsr(pVCpu, pVmcsInfo);
10568	AssertRC(hmR0VmxCheckVmcsCtls(pVCpu, pVmcsInfo));
10569	#endif
10570
10571	#ifdef HMVMX_ALWAYS_CHECK_GUEST_STATE
10572	/** @todo r=ramshankar: We can now probably use iemVmxVmentryCheckGuestState here.
10573	* Add a PVMXMSRS parameter to it, so that IEM can look at the host MSRs. */
10574	uint32_t const uInvalidReason = hmR0VmxCheckGuestState(pVCpu, pVmcsInfo);
10575	if (uInvalidReason != VMX_IGS_REASON_NOT_FOUND)
10576	Log4(("hmR0VmxCheckGuestState returned %#x\n", uInvalidReason));
10577	#endif
10578	}
10579
10580
10581	/**
10582	* First C routine invoked after running guest code using hardware-assisted VMX.
10583	*
10584	* @param pVCpu The cross context virtual CPU structure.
10585	* @param pVmxTransient The VMX-transient structure.
10586	* @param rcVMRun Return code of VMLAUNCH/VMRESUME.
10587	*
10588	* @remarks Called with interrupts disabled, and returns with interrupts enabled!
10589	*
10590	* @remarks No-long-jump zone!!! This function will however re-enable longjmps
10591	* unconditionally when it is safe to do so.
10592	*/
10593	static void hmR0VmxPostRunGuest(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, int rcVMRun)
10594	{
10595	uint64_t const uHostTsc = ASMReadTSC(); /** @todo We can do a lot better here, see @bugref{9180#c38}. */
10596
10597	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB flushing. */
10598	ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for EMT poking. */
10599	pVCpu->hm.s.fCtxChanged = 0; /* Exits/longjmps to ring-3 requires saving the guest state. */
10600	pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
10601	pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
10602	pVmxTransient->fVectoringDoublePF = false; /* Vectoring double page-fault needs to be determined later. */
10603
10604	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
10605	if (!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_RDTSC_EXIT))
10606	{
10607	uint64_t uGstTsc;
10608	if (!pVmxTransient->fIsNestedGuest)
10609	uGstTsc = uHostTsc + pVmcsInfo->u64TscOffset;
10610	else
10611	{
10612	uint64_t const uNstGstTsc = uHostTsc + pVmcsInfo->u64TscOffset;
10613	uGstTsc = CPUMRemoveNestedGuestTscOffset(pVCpu, uNstGstTsc);
10614	}
10615	TMCpuTickSetLastSeen(pVCpu, uGstTsc); /* Update TM with the guest TSC. */
10616	}
10617
10618	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatPreExit, x);
10619	TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
10620	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
10621
10622	#if HC_ARCH_BITS == 64
10623	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_REQUIRED; /* Some host state messed up by VMX needs restoring. */
10624	#endif
10625	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
10626	/* The 64-on-32 switcher maintains VMCS-launch state on its own
10627	and we need to leave it alone here. */
10628	if (pVmcsInfo->pfnStartVM != VMXR0SwitcherStartVM64)
10629	pVmcsInfo->fVmcsState \|= VMX_V_VMCS_LAUNCH_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
10630	#else
10631	pVmcsInfo->fVmcsState \|= VMX_V_VMCS_LAUNCH_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
10632	#endif
10633	#ifdef VBOX_STRICT
10634	hmR0VmxCheckHostEferMsr(pVCpu, pVmcsInfo); /* Verify that the host EFER MSR wasn't modified. */
10635	#endif
10636	Assert(!ASMIntAreEnabled());
10637	ASMSetFlags(pVmxTransient->fEFlags); /* Enable interrupts. */
10638	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
10639
10640	/*
10641	* Save the basic VM-exit reason and check if the VM-entry failed.
10642	* See Intel spec. 24.9.1 "Basic VM-exit Information".
10643	*/
10644	uint32_t uExitReason;
10645	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
10646	AssertRC(rc);
10647	pVmxTransient->uExitReason = VMX_EXIT_REASON_BASIC(uExitReason);
10648	pVmxTransient->fVMEntryFailed = VMX_EXIT_REASON_HAS_ENTRY_FAILED(uExitReason);
10649
10650	/*
10651	* Check if VMLAUNCH/VMRESUME succeeded.
10652	* If this failed, we cause a guru meditation and cease further execution.
10653	*/
10654	if (RT_LIKELY(rcVMRun == VINF_SUCCESS))
10655	{
10656	/*
10657	* Update the VM-exit history array here even if the VM-entry failed due to:
10658	* - Invalid guest state.
10659	* - MSR loading.
10660	* - Machine-check event.
10661	*
10662	* In any of the above cases we will still have a "valid" VM-exit reason
10663	* despite @a fVMEntryFailed being false.
10664	*
10665	* See Intel spec. 26.7 "VM-Entry failures during or after loading guest state".
10666	*
10667	* Note! We don't have CS or RIP at this point. Will probably address that later
10668	* by amending the history entry added here.
10669	*/
10670	EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_VMX, pVmxTransient->uExitReason & EMEXIT_F_TYPE_MASK),
10671	UINT64_MAX, uHostTsc);
10672
10673	if (RT_LIKELY(!pVmxTransient->fVMEntryFailed))
10674	{
10675	VMMRZCallRing3Enable(pVCpu);
10676
10677	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
10678	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
10679
10680	#if defined(HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE) \|\| defined(HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE)
10681	rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
10682	AssertRC(rc);
10683	#elif defined(HMVMX_ALWAYS_SAVE_GUEST_RFLAGS)
10684	rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_RFLAGS);
10685	AssertRC(rc);
10686	#else
10687	/*
10688	* Import the guest-interruptibility state always as we need it while evaluating
10689	* injecting events on re-entry.
10690	*
10691	* We don't import CR0 (when unrestricted guest execution is unavailable) despite
10692	* checking for real-mode while exporting the state because all bits that cause
10693	* mode changes wrt CR0 are intercepted.
10694	*/
10695	rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_HM_VMX_INT_STATE);
10696	AssertRC(rc);
10697	#endif
10698
10699	/*
10700	* Sync the TPR shadow with our APIC state.
10701	*/
10702	if ( !pVmxTransient->fIsNestedGuest
10703	&& (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW))
10704	{
10705	Assert(pVmcsInfo->pbVirtApic);
10706	if (pVmxTransient->u8GuestTpr != pVmcsInfo->pbVirtApic[XAPIC_OFF_TPR])
10707	{
10708	rc = APICSetTpr(pVCpu, pVmcsInfo->pbVirtApic[XAPIC_OFF_TPR]);
10709	AssertRC(rc);
10710	ASMAtomicOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
10711	}
10712	}
10713
10714	Assert(VMMRZCallRing3IsEnabled(pVCpu));
10715	return;
10716	}
10717	}
10718	else
10719	Log4Func(("VM-entry failure: rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", rcVMRun, pVmxTransient->fVMEntryFailed));
10720
10721	VMMRZCallRing3Enable(pVCpu);
10722	}
10723
10724
10725	/**
10726	* Runs the guest code using hardware-assisted VMX the normal way.
10727	*
10728	* @returns VBox status code.
10729	* @param pVCpu The cross context virtual CPU structure.
10730	* @param pcLoops Pointer to the number of executed loops.
10731	*/
10732	static VBOXSTRICTRC hmR0VmxRunGuestCodeNormal(PVMCPU pVCpu, uint32_t *pcLoops)
10733	{
10734	uint32_t const cMaxResumeLoops = pVCpu->CTX_SUFF(pVM)->hm.s.cMaxResumeLoops;
10735	Assert(pcLoops);
10736	Assert(*pcLoops <= cMaxResumeLoops);
10737
10738	VMXTRANSIENT VmxTransient;
10739	RT_ZERO(VmxTransient);
10740	VmxTransient.pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
10741
10742	/* Paranoia. */
10743	Assert(VmxTransient.pVmcsInfo == &pVCpu->hm.s.vmx.VmcsInfo);
10744	Assert(!CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx));
10745
10746	VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
10747	for (;;)
10748	{
10749	Assert(!HMR0SuspendPending());
10750	HMVMX_ASSERT_CPU_SAFE(pVCpu);
10751	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
10752
10753	/*
10754	* Preparatory work for running nested-guest code, this may force us to
10755	* return to ring-3.
10756	*
10757	* Warning! This bugger disables interrupts on VINF_SUCCESS!
10758	*/
10759	rcStrict = hmR0VmxPreRunGuest(pVCpu, &VmxTransient, false /* fStepping */);
10760	if (rcStrict != VINF_SUCCESS)
10761	break;
10762
10763	/* Interrupts are disabled at this point! */
10764	hmR0VmxPreRunGuestCommitted(pVCpu, &VmxTransient);
10765	int rcRun = hmR0VmxRunGuest(pVCpu, &VmxTransient);
10766	hmR0VmxPostRunGuest(pVCpu, &VmxTransient, rcRun);
10767	/* Interrupts are re-enabled at this point! */
10768
10769	/*
10770	* Check for errors with running the VM (VMLAUNCH/VMRESUME).
10771	*/
10772	if (RT_SUCCESS(rcRun))
10773	{ /* very likely */ }
10774	else
10775	{
10776	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
10777	hmR0VmxReportWorldSwitchError(pVCpu, rcRun, &VmxTransient);
10778	return rcRun;
10779	}
10780
10781	/*
10782	* Profile the VM-exit.
10783	*/
10784	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
10785	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
10786	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
10787	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
10788	HMVMX_START_EXIT_DISPATCH_PROF();
10789
10790	VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, &pVCpu->cpum.GstCtx, VmxTransient.uExitReason);
10791
10792	/*
10793	* Handle the VM-exit.
10794	*/
10795	#ifdef HMVMX_USE_FUNCTION_TABLE
10796	rcStrict = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, &VmxTransient);
10797	#else
10798	rcStrict = hmR0VmxHandleExit(pVCpu, &VmxTransient);
10799	#endif
10800	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
10801	if (rcStrict == VINF_SUCCESS)
10802	{
10803	if (++(*pcLoops) <= cMaxResumeLoops)
10804	continue;
10805	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
10806	rcStrict = VINF_EM_RAW_INTERRUPT;
10807	}
10808	break;
10809	}
10810
10811	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
10812	return rcStrict;
10813	}
10814
10815	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
10816	/**
10817	* Runs the nested-guest code using hardware-assisted VMX.
10818	*
10819	* @returns VBox status code.
10820	* @param pVCpu The cross context virtual CPU structure.
10821	* @param pcLoops Pointer to the number of executed loops.
10822	*
10823	* @sa hmR0VmxRunGuestCodeNormal().
10824	*/
10825	static VBOXSTRICTRC hmR0VmxRunGuestCodeNested(PVMCPU pVCpu, uint32_t *pcLoops)
10826	{
10827	uint32_t const cMaxResumeLoops = pVCpu->CTX_SUFF(pVM)->hm.s.cMaxResumeLoops;
10828	Assert(pcLoops);
10829	Assert(*pcLoops <= cMaxResumeLoops);
10830
10831	VMXTRANSIENT VmxTransient;
10832	RT_ZERO(VmxTransient);
10833	VmxTransient.pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
10834	VmxTransient.fIsNestedGuest = true;
10835
10836	VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
10837	for (;;)
10838	{
10839	Assert(!HMR0SuspendPending());
10840	HMVMX_ASSERT_CPU_SAFE(pVCpu);
10841	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
10842
10843	/*
10844	* Preparatory work for running guest code, this may force us to
10845	* return to ring-3.
10846	*
10847	* Warning! This bugger disables interrupts on VINF_SUCCESS!
10848	*/
10849	rcStrict = hmR0VmxPreRunGuest(pVCpu, &VmxTransient, false /* fStepping */);
10850	if (rcStrict != VINF_SUCCESS)
10851	break;
10852
10853	/* Interrupts are disabled at this point! */
10854	hmR0VmxPreRunGuestCommitted(pVCpu, &VmxTransient);
10855	int rcRun = hmR0VmxRunGuest(pVCpu, &VmxTransient);
10856	hmR0VmxPostRunGuest(pVCpu, &VmxTransient, rcRun);
10857	/* Interrupts are re-enabled at this point! */
10858
10859	/*
10860	* Check for errors with running the VM (VMLAUNCH/VMRESUME).
10861	*/
10862	if (RT_SUCCESS(rcRun))
10863	{ /* very likely */ }
10864	else
10865	{
10866	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
10867	hmR0VmxReportWorldSwitchError(pVCpu, rcRun, &VmxTransient);
10868	return rcRun;
10869	}
10870
10871	/*
10872	* Profile the VM-exit.
10873	*/
10874	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
10875	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
10876	STAM_COUNTER_INC(&pVCpu->hm.s.paStatNestedExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
10877	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
10878	HMVMX_START_EXIT_DISPATCH_PROF();
10879
10880	VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, &pVCpu->cpum.GstCtx, VmxTransient.uExitReason);
10881
10882	/*
10883	* Handle the VM-exit.
10884	*/
10885	rcStrict = hmR0VmxHandleExitNested(pVCpu, &VmxTransient);
10886	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
10887	if ( rcStrict == VINF_SUCCESS
10888	&& CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
10889	{
10890	if (++(*pcLoops) <= cMaxResumeLoops)
10891	continue;
10892	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
10893	rcStrict = VINF_EM_RAW_INTERRUPT;
10894	}
10895	break;
10896	}
10897
10898	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
10899	return rcStrict;
10900	}
10901	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
10902
10903
10904	/** @name Execution loop for single stepping, DBGF events and expensive Dtrace
10905	* probes.
10906	*
10907	* The following few functions and associated structure contains the bloat
10908	* necessary for providing detailed debug events and dtrace probes as well as
10909	* reliable host side single stepping. This works on the principle of
10910	* "subclassing" the normal execution loop and workers. We replace the loop
10911	* method completely and override selected helpers to add necessary adjustments
10912	* to their core operation.
10913	*
10914	* The goal is to keep the "parent" code lean and mean, so as not to sacrifice
10915	* any performance for debug and analysis features.
10916	*
10917	* @{
10918	*/
10919
10920	/**
10921	* Transient per-VCPU debug state of VMCS and related info. we save/restore in
10922	* the debug run loop.
10923	*/
10924	typedef struct VMXRUNDBGSTATE
10925	{
10926	/** The RIP we started executing at. This is for detecting that we stepped. */
10927	uint64_t uRipStart;
10928	/** The CS we started executing with. */
10929	uint16_t uCsStart;
10930
10931	/** Whether we've actually modified the 1st execution control field. */
10932	bool fModifiedProcCtls : 1;
10933	/** Whether we've actually modified the 2nd execution control field. */
10934	bool fModifiedProcCtls2 : 1;
10935	/** Whether we've actually modified the exception bitmap. */
10936	bool fModifiedXcptBitmap : 1;
10937
10938	/** We desire the modified the CR0 mask to be cleared. */
10939	bool fClearCr0Mask : 1;
10940	/** We desire the modified the CR4 mask to be cleared. */
10941	bool fClearCr4Mask : 1;
10942	/** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC. */
10943	uint32_t fCpe1Extra;
10944	/** Stuff we do not want in VMX_VMCS32_CTRL_PROC_EXEC. */
10945	uint32_t fCpe1Unwanted;
10946	/** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC2. */
10947	uint32_t fCpe2Extra;
10948	/** Extra stuff we need in VMX_VMCS32_CTRL_EXCEPTION_BITMAP. */
10949	uint32_t bmXcptExtra;
10950	/** The sequence number of the Dtrace provider settings the state was
10951	* configured against. */
10952	uint32_t uDtraceSettingsSeqNo;
10953	/** VM-exits to check (one bit per VM-exit). */
10954	uint32_t bmExitsToCheck[3];
10955
10956	/** The initial VMX_VMCS32_CTRL_PROC_EXEC value (helps with restore). */
10957	uint32_t fProcCtlsInitial;
10958	/** The initial VMX_VMCS32_CTRL_PROC_EXEC2 value (helps with restore). */
10959	uint32_t fProcCtls2Initial;
10960	/** The initial VMX_VMCS32_CTRL_EXCEPTION_BITMAP value (helps with restore). */
10961	uint32_t bmXcptInitial;
10962	} VMXRUNDBGSTATE;
10963	AssertCompileMemberSize(VMXRUNDBGSTATE, bmExitsToCheck, (VMX_EXIT_MAX + 1 + 31) / 32 * 4);
10964	typedef VMXRUNDBGSTATE *PVMXRUNDBGSTATE;
10965
10966
10967	/**
10968	* Initializes the VMXRUNDBGSTATE structure.
10969	*
10970	* @param pVCpu The cross context virtual CPU structure of the
10971	* calling EMT.
10972	* @param pVmxTransient The VMX-transient structure.
10973	* @param pDbgState The debug state to initialize.
10974	*/
10975	static void hmR0VmxRunDebugStateInit(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState)
10976	{
10977	pDbgState->uRipStart = pVCpu->cpum.GstCtx.rip;
10978	pDbgState->uCsStart = pVCpu->cpum.GstCtx.cs.Sel;
10979
10980	pDbgState->fModifiedProcCtls = false;
10981	pDbgState->fModifiedProcCtls2 = false;
10982	pDbgState->fModifiedXcptBitmap = false;
10983	pDbgState->fClearCr0Mask = false;
10984	pDbgState->fClearCr4Mask = false;
10985	pDbgState->fCpe1Extra = 0;
10986	pDbgState->fCpe1Unwanted = 0;
10987	pDbgState->fCpe2Extra = 0;
10988	pDbgState->bmXcptExtra = 0;
10989	pDbgState->fProcCtlsInitial = pVmxTransient->pVmcsInfo->u32ProcCtls;
10990	pDbgState->fProcCtls2Initial = pVmxTransient->pVmcsInfo->u32ProcCtls2;
10991	pDbgState->bmXcptInitial = pVmxTransient->pVmcsInfo->u32XcptBitmap;
10992	}
10993
10994
10995	/**
10996	* Updates the VMSC fields with changes requested by @a pDbgState.
10997	*
10998	* This is performed after hmR0VmxPreRunGuestDebugStateUpdate as well
10999	* immediately before executing guest code, i.e. when interrupts are disabled.
11000	* We don't check status codes here as we cannot easily assert or return in the
11001	* latter case.
11002	*
11003	* @param pVCpu The cross context virtual CPU structure.
11004	* @param pVmxTransient The VMX-transient structure.
11005	* @param pDbgState The debug state.
11006	*/
11007	static void hmR0VmxPreRunGuestDebugStateApply(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState)
11008	{
11009	/*
11010	* Ensure desired flags in VMCS control fields are set.
11011	* (Ignoring write failure here, as we're committed and it's just debug extras.)
11012	*
11013	* Note! We load the shadow CR0 & CR4 bits when we flag the clearing, so
11014	* there should be no stale data in pCtx at this point.
11015	*/
11016	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
11017	if ( (pVmcsInfo->u32ProcCtls & pDbgState->fCpe1Extra) != pDbgState->fCpe1Extra
11018	\|\| (pVmcsInfo->u32ProcCtls & pDbgState->fCpe1Unwanted))
11019	{
11020	pVmcsInfo->u32ProcCtls \|= pDbgState->fCpe1Extra;
11021	pVmcsInfo->u32ProcCtls &= ~pDbgState->fCpe1Unwanted;
11022	VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
11023	Log6Func(("VMX_VMCS32_CTRL_PROC_EXEC: %#RX32\n", pVmcsInfo->u32ProcCtls));
11024	pDbgState->fModifiedProcCtls = true;
11025	}
11026
11027	if ((pVmcsInfo->u32ProcCtls2 & pDbgState->fCpe2Extra) != pDbgState->fCpe2Extra)
11028	{
11029	pVmcsInfo->u32ProcCtls2 \|= pDbgState->fCpe2Extra;
11030	VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pVmcsInfo->u32ProcCtls2);
11031	Log6Func(("VMX_VMCS32_CTRL_PROC_EXEC2: %#RX32\n", pVmcsInfo->u32ProcCtls2));
11032	pDbgState->fModifiedProcCtls2 = true;
11033	}
11034
11035	if ((pVmcsInfo->u32XcptBitmap & pDbgState->bmXcptExtra) != pDbgState->bmXcptExtra)
11036	{
11037	pVmcsInfo->u32XcptBitmap \|= pDbgState->bmXcptExtra;
11038	VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVmcsInfo->u32XcptBitmap);
11039	Log6Func(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP: %#RX32\n", pVmcsInfo->u32XcptBitmap));
11040	pDbgState->fModifiedXcptBitmap = true;
11041	}
11042
11043	if (pDbgState->fClearCr0Mask && pVmcsInfo->u64Cr0Mask != 0)
11044	{
11045	pVmcsInfo->u64Cr0Mask = 0;
11046	VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, 0);
11047	Log6Func(("VMX_VMCS_CTRL_CR0_MASK: 0\n"));
11048	}
11049
11050	if (pDbgState->fClearCr4Mask && pVmcsInfo->u64Cr4Mask != 0)
11051	{
11052	pVmcsInfo->u64Cr4Mask = 0;
11053	VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, 0);
11054	Log6Func(("VMX_VMCS_CTRL_CR4_MASK: 0\n"));
11055	}
11056
11057	NOREF(pVCpu);
11058	}
11059
11060
11061	/**
11062	* Restores VMCS fields that were changed by hmR0VmxPreRunGuestDebugStateApply for
11063	* re-entry next time around.
11064	*
11065	* @returns Strict VBox status code (i.e. informational status codes too).
11066	* @param pVCpu The cross context virtual CPU structure.
11067	* @param pVmxTransient The VMX-transient structure.
11068	* @param pDbgState The debug state.
11069	* @param rcStrict The return code from executing the guest using single
11070	* stepping.
11071	*/
11072	static VBOXSTRICTRC hmR0VmxRunDebugStateRevert(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState,
11073	VBOXSTRICTRC rcStrict)
11074	{
11075	/*
11076	* Restore VM-exit control settings as we may not reenter this function the
11077	* next time around.
11078	*/
11079	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
11080
11081	/* We reload the initial value, trigger what we can of recalculations the
11082	next time around. From the looks of things, that's all that's required atm. */
11083	if (pDbgState->fModifiedProcCtls)
11084	{
11085	if (!(pDbgState->fProcCtlsInitial & VMX_PROC_CTLS_MOV_DR_EXIT) && CPUMIsHyperDebugStateActive(pVCpu))
11086	pDbgState->fProcCtlsInitial \|= VMX_PROC_CTLS_MOV_DR_EXIT; /* Avoid assertion in hmR0VmxLeave */
11087	int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pDbgState->fProcCtlsInitial);
11088	AssertRCReturn(rc2, rc2);
11089	pVmcsInfo->u32ProcCtls = pDbgState->fProcCtlsInitial;
11090	}
11091
11092	/* We're currently the only ones messing with this one, so just restore the
11093	cached value and reload the field. */
11094	if ( pDbgState->fModifiedProcCtls2
11095	&& pVmcsInfo->u32ProcCtls2 != pDbgState->fProcCtls2Initial)
11096	{
11097	int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pDbgState->fProcCtls2Initial);
11098	AssertRCReturn(rc2, rc2);
11099	pVmcsInfo->u32ProcCtls2 = pDbgState->fProcCtls2Initial;
11100	}
11101
11102	/* If we've modified the exception bitmap, we restore it and trigger
11103	reloading and partial recalculation the next time around. */
11104	if (pDbgState->fModifiedXcptBitmap)
11105	pVmcsInfo->u32XcptBitmap = pDbgState->bmXcptInitial;
11106
11107	return rcStrict;
11108	}
11109
11110
11111	/**
11112	* Configures VM-exit controls for current DBGF and DTrace settings.
11113	*
11114	* This updates @a pDbgState and the VMCS execution control fields to reflect
11115	* the necessary VM-exits demanded by DBGF and DTrace.
11116	*
11117	* @param pVCpu The cross context virtual CPU structure.
11118	* @param pVmxTransient The VMX-transient structure. May update
11119	* fUpdatedTscOffsettingAndPreemptTimer.
11120	* @param pDbgState The debug state.
11121	*/
11122	static void hmR0VmxPreRunGuestDebugStateUpdate(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState)
11123	{
11124	/*
11125	* Take down the dtrace serial number so we can spot changes.
11126	*/
11127	pDbgState->uDtraceSettingsSeqNo = VBOXVMM_GET_SETTINGS_SEQ_NO();
11128	ASMCompilerBarrier();
11129
11130	/*
11131	* We'll rebuild most of the middle block of data members (holding the
11132	* current settings) as we go along here, so start by clearing it all.
11133	*/
11134	pDbgState->bmXcptExtra = 0;
11135	pDbgState->fCpe1Extra = 0;
11136	pDbgState->fCpe1Unwanted = 0;
11137	pDbgState->fCpe2Extra = 0;
11138	for (unsigned i = 0; i < RT_ELEMENTS(pDbgState->bmExitsToCheck); i++)
11139	pDbgState->bmExitsToCheck[i] = 0;
11140
11141	/*
11142	* Software interrupts (INT XXh) - no idea how to trigger these...
11143	*/
11144	PVM pVM = pVCpu->CTX_SUFF(pVM);
11145	if ( DBGF_IS_EVENT_ENABLED(pVM, DBGFEVENT_INTERRUPT_SOFTWARE)
11146	\|\| VBOXVMM_INT_SOFTWARE_ENABLED())
11147	{
11148	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
11149	}
11150
11151	/*
11152	* INT3 breakpoints - triggered by #BP exceptions.
11153	*/
11154	if (pVM->dbgf.ro.cEnabledInt3Breakpoints > 0)
11155	pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_BP);
11156
11157	/*
11158	* Exception bitmap and XCPT events+probes.
11159	*/
11160	for (int iXcpt = 0; iXcpt < (DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST + 1); iXcpt++)
11161	if (DBGF_IS_EVENT_ENABLED(pVM, (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + iXcpt)))
11162	pDbgState->bmXcptExtra \|= RT_BIT_32(iXcpt);
11163
11164	if (VBOXVMM_XCPT_DE_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_DE);
11165	if (VBOXVMM_XCPT_DB_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_DB);
11166	if (VBOXVMM_XCPT_BP_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_BP);
11167	if (VBOXVMM_XCPT_OF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_OF);
11168	if (VBOXVMM_XCPT_BR_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_BR);
11169	if (VBOXVMM_XCPT_UD_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_UD);
11170	if (VBOXVMM_XCPT_NM_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_NM);
11171	if (VBOXVMM_XCPT_DF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_DF);
11172	if (VBOXVMM_XCPT_TS_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_TS);
11173	if (VBOXVMM_XCPT_NP_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_NP);
11174	if (VBOXVMM_XCPT_SS_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_SS);
11175	if (VBOXVMM_XCPT_GP_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_GP);
11176	if (VBOXVMM_XCPT_PF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_PF);
11177	if (VBOXVMM_XCPT_MF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_MF);
11178	if (VBOXVMM_XCPT_AC_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_AC);
11179	if (VBOXVMM_XCPT_XF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_XF);
11180	if (VBOXVMM_XCPT_VE_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_VE);
11181	if (VBOXVMM_XCPT_SX_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_SX);
11182
11183	if (pDbgState->bmXcptExtra)
11184	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
11185
11186	/*
11187	* Process events and probes for VM-exits, making sure we get the wanted VM-exits.
11188	*
11189	* Note! This is the reverse of what hmR0VmxHandleExitDtraceEvents does.
11190	* So, when adding/changing/removing please don't forget to update it.
11191	*
11192	* Some of the macros are picking up local variables to save horizontal space,
11193	* (being able to see it in a table is the lesser evil here).
11194	*/
11195	#define IS_EITHER_ENABLED(a_pVM, a_EventSubName) \
11196	( DBGF_IS_EVENT_ENABLED(a_pVM, RT_CONCAT(DBGFEVENT_, a_EventSubName)) \
11197	\|\| RT_CONCAT3(VBOXVMM_, a_EventSubName, _ENABLED)() )
11198	#define SET_ONLY_XBM_IF_EITHER_EN(a_EventSubName, a_uExit) \
11199	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
11200	{ AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
11201	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
11202	} else do { } while (0)
11203	#define SET_CPE1_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec) \
11204	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
11205	{ \
11206	(pDbgState)->fCpe1Extra \|= (a_fCtrlProcExec); \
11207	AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
11208	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
11209	} else do { } while (0)
11210	#define SET_CPEU_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fUnwantedCtrlProcExec) \
11211	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
11212	{ \
11213	(pDbgState)->fCpe1Unwanted \|= (a_fUnwantedCtrlProcExec); \
11214	AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
11215	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
11216	} else do { } while (0)
11217	#define SET_CPE2_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec2) \
11218	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
11219	{ \
11220	(pDbgState)->fCpe2Extra \|= (a_fCtrlProcExec2); \
11221	AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
11222	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
11223	} else do { } while (0)
11224
11225	SET_ONLY_XBM_IF_EITHER_EN(EXIT_TASK_SWITCH, VMX_EXIT_TASK_SWITCH); /* unconditional */
11226	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_VIOLATION, VMX_EXIT_EPT_VIOLATION); /* unconditional */
11227	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_MISCONFIG, VMX_EXIT_EPT_MISCONFIG); /* unconditional (unless #VE) */
11228	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_ACCESS, VMX_EXIT_APIC_ACCESS); /* feature dependent, nothing to enable here */
11229	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_WRITE, VMX_EXIT_APIC_WRITE); /* feature dependent, nothing to enable here */
11230
11231	SET_ONLY_XBM_IF_EITHER_EN(INSTR_CPUID, VMX_EXIT_CPUID); /* unconditional */
11232	SET_ONLY_XBM_IF_EITHER_EN( EXIT_CPUID, VMX_EXIT_CPUID);
11233	SET_ONLY_XBM_IF_EITHER_EN(INSTR_GETSEC, VMX_EXIT_GETSEC); /* unconditional */
11234	SET_ONLY_XBM_IF_EITHER_EN( EXIT_GETSEC, VMX_EXIT_GETSEC);
11235	SET_CPE1_XBM_IF_EITHER_EN(INSTR_HALT, VMX_EXIT_HLT, VMX_PROC_CTLS_HLT_EXIT); /* paranoia */
11236	SET_ONLY_XBM_IF_EITHER_EN( EXIT_HALT, VMX_EXIT_HLT);
11237	SET_ONLY_XBM_IF_EITHER_EN(INSTR_INVD, VMX_EXIT_INVD); /* unconditional */
11238	SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVD, VMX_EXIT_INVD);
11239	SET_CPE1_XBM_IF_EITHER_EN(INSTR_INVLPG, VMX_EXIT_INVLPG, VMX_PROC_CTLS_INVLPG_EXIT);
11240	SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVLPG, VMX_EXIT_INVLPG);
11241	SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDPMC, VMX_EXIT_RDPMC, VMX_PROC_CTLS_RDPMC_EXIT);
11242	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDPMC, VMX_EXIT_RDPMC);
11243	SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSC, VMX_EXIT_RDTSC, VMX_PROC_CTLS_RDTSC_EXIT);
11244	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSC, VMX_EXIT_RDTSC);
11245	SET_ONLY_XBM_IF_EITHER_EN(INSTR_RSM, VMX_EXIT_RSM); /* unconditional */
11246	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RSM, VMX_EXIT_RSM);
11247	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMM_CALL, VMX_EXIT_VMCALL); /* unconditional */
11248	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMM_CALL, VMX_EXIT_VMCALL);
11249	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMCLEAR, VMX_EXIT_VMCLEAR); /* unconditional */
11250	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMCLEAR, VMX_EXIT_VMCLEAR);
11251	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH); /* unconditional */
11252	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH);
11253	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRLD, VMX_EXIT_VMPTRLD); /* unconditional */
11254	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRLD, VMX_EXIT_VMPTRLD);
11255	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRST, VMX_EXIT_VMPTRST); /* unconditional */
11256	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRST, VMX_EXIT_VMPTRST);
11257	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMREAD, VMX_EXIT_VMREAD); /* unconditional */
11258	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMREAD, VMX_EXIT_VMREAD);
11259	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMRESUME, VMX_EXIT_VMRESUME); /* unconditional */
11260	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMRESUME, VMX_EXIT_VMRESUME);
11261	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMWRITE, VMX_EXIT_VMWRITE); /* unconditional */
11262	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMWRITE, VMX_EXIT_VMWRITE);
11263	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXOFF, VMX_EXIT_VMXOFF); /* unconditional */
11264	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXOFF, VMX_EXIT_VMXOFF);
11265	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXON, VMX_EXIT_VMXON); /* unconditional */
11266	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXON, VMX_EXIT_VMXON);
11267
11268	if ( IS_EITHER_ENABLED(pVM, INSTR_CRX_READ)
11269	\|\| IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
11270	{
11271	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_CR4
11272	\| CPUMCTX_EXTRN_APIC_TPR);
11273	AssertRC(rc);
11274
11275	#if 0 /** @todo fix me */
11276	pDbgState->fClearCr0Mask = true;
11277	pDbgState->fClearCr4Mask = true;
11278	#endif
11279	if (IS_EITHER_ENABLED(pVM, INSTR_CRX_READ))
11280	pDbgState->fCpe1Extra \|= VMX_PROC_CTLS_CR3_STORE_EXIT \| VMX_PROC_CTLS_CR8_STORE_EXIT;
11281	if (IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
11282	pDbgState->fCpe1Extra \|= VMX_PROC_CTLS_CR3_LOAD_EXIT \| VMX_PROC_CTLS_CR8_LOAD_EXIT;
11283	pDbgState->fCpe1Unwanted \|= VMX_PROC_CTLS_USE_TPR_SHADOW; /* risky? */
11284	/* Note! We currently don't use VMX_VMCS32_CTRL_CR3_TARGET_COUNT. It would
11285	require clearing here and in the loop if we start using it. */
11286	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_CRX);
11287	}
11288	else
11289	{
11290	if (pDbgState->fClearCr0Mask)
11291	{
11292	pDbgState->fClearCr0Mask = false;
11293	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR0);
11294	}
11295	if (pDbgState->fClearCr4Mask)
11296	{
11297	pDbgState->fClearCr4Mask = false;
11298	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR4);
11299	}
11300	}
11301	SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_READ, VMX_EXIT_MOV_CRX);
11302	SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_WRITE, VMX_EXIT_MOV_CRX);
11303
11304	if ( IS_EITHER_ENABLED(pVM, INSTR_DRX_READ)
11305	\|\| IS_EITHER_ENABLED(pVM, INSTR_DRX_WRITE))
11306	{
11307	/** @todo later, need to fix handler as it assumes this won't usually happen. */
11308	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_DRX);
11309	}
11310	SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_READ, VMX_EXIT_MOV_DRX);
11311	SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_WRITE, VMX_EXIT_MOV_DRX);
11312
11313	SET_CPEU_XBM_IF_EITHER_EN(INSTR_RDMSR, VMX_EXIT_RDMSR, VMX_PROC_CTLS_USE_MSR_BITMAPS); /* risky clearing this? */
11314	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDMSR, VMX_EXIT_RDMSR);
11315	SET_CPEU_XBM_IF_EITHER_EN(INSTR_WRMSR, VMX_EXIT_WRMSR, VMX_PROC_CTLS_USE_MSR_BITMAPS);
11316	SET_ONLY_XBM_IF_EITHER_EN( EXIT_WRMSR, VMX_EXIT_WRMSR);
11317	SET_CPE1_XBM_IF_EITHER_EN(INSTR_MWAIT, VMX_EXIT_MWAIT, VMX_PROC_CTLS_MWAIT_EXIT); /* paranoia */
11318	SET_ONLY_XBM_IF_EITHER_EN( EXIT_MWAIT, VMX_EXIT_MWAIT);
11319	SET_CPE1_XBM_IF_EITHER_EN(INSTR_MONITOR, VMX_EXIT_MONITOR, VMX_PROC_CTLS_MONITOR_EXIT); /* paranoia */
11320	SET_ONLY_XBM_IF_EITHER_EN( EXIT_MONITOR, VMX_EXIT_MONITOR);
11321	#if 0 /** @todo too slow, fix handler. */
11322	SET_CPE1_XBM_IF_EITHER_EN(INSTR_PAUSE, VMX_EXIT_PAUSE, VMX_PROC_CTLS_PAUSE_EXIT);
11323	#endif
11324	SET_ONLY_XBM_IF_EITHER_EN( EXIT_PAUSE, VMX_EXIT_PAUSE);
11325
11326	if ( IS_EITHER_ENABLED(pVM, INSTR_SGDT)
11327	\|\| IS_EITHER_ENABLED(pVM, INSTR_SIDT)
11328	\|\| IS_EITHER_ENABLED(pVM, INSTR_LGDT)
11329	\|\| IS_EITHER_ENABLED(pVM, INSTR_LIDT))
11330	{
11331	pDbgState->fCpe2Extra \|= VMX_PROC_CTLS2_DESC_TABLE_EXIT;
11332	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_GDTR_IDTR_ACCESS);
11333	}
11334	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SGDT, VMX_EXIT_GDTR_IDTR_ACCESS);
11335	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SIDT, VMX_EXIT_GDTR_IDTR_ACCESS);
11336	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LGDT, VMX_EXIT_GDTR_IDTR_ACCESS);
11337	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LIDT, VMX_EXIT_GDTR_IDTR_ACCESS);
11338
11339	if ( IS_EITHER_ENABLED(pVM, INSTR_SLDT)
11340	\|\| IS_EITHER_ENABLED(pVM, INSTR_STR)
11341	\|\| IS_EITHER_ENABLED(pVM, INSTR_LLDT)
11342	\|\| IS_EITHER_ENABLED(pVM, INSTR_LTR))
11343	{
11344	pDbgState->fCpe2Extra \|= VMX_PROC_CTLS2_DESC_TABLE_EXIT;
11345	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_LDTR_TR_ACCESS);
11346	}
11347	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SLDT, VMX_EXIT_LDTR_TR_ACCESS);
11348	SET_ONLY_XBM_IF_EITHER_EN( EXIT_STR, VMX_EXIT_LDTR_TR_ACCESS);
11349	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LLDT, VMX_EXIT_LDTR_TR_ACCESS);
11350	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LTR, VMX_EXIT_LDTR_TR_ACCESS);
11351
11352	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVEPT, VMX_EXIT_INVEPT); /* unconditional */
11353	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVEPT, VMX_EXIT_INVEPT);
11354	SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSCP, VMX_EXIT_RDTSCP, VMX_PROC_CTLS_RDTSC_EXIT);
11355	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSCP, VMX_EXIT_RDTSCP);
11356	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVVPID, VMX_EXIT_INVVPID); /* unconditional */
11357	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVVPID, VMX_EXIT_INVVPID);
11358	SET_CPE2_XBM_IF_EITHER_EN(INSTR_WBINVD, VMX_EXIT_WBINVD, VMX_PROC_CTLS2_WBINVD_EXIT);
11359	SET_ONLY_XBM_IF_EITHER_EN( EXIT_WBINVD, VMX_EXIT_WBINVD);
11360	SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSETBV, VMX_EXIT_XSETBV); /* unconditional */
11361	SET_ONLY_XBM_IF_EITHER_EN( EXIT_XSETBV, VMX_EXIT_XSETBV);
11362	SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDRAND, VMX_EXIT_RDRAND, VMX_PROC_CTLS2_RDRAND_EXIT);
11363	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDRAND, VMX_EXIT_RDRAND);
11364	SET_CPE1_XBM_IF_EITHER_EN(INSTR_VMX_INVPCID, VMX_EXIT_INVPCID, VMX_PROC_CTLS_INVLPG_EXIT);
11365	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVPCID, VMX_EXIT_INVPCID);
11366	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMFUNC, VMX_EXIT_VMFUNC); /* unconditional for the current setup */
11367	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMFUNC, VMX_EXIT_VMFUNC);
11368	SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDSEED, VMX_EXIT_RDSEED, VMX_PROC_CTLS2_RDSEED_EXIT);
11369	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDSEED, VMX_EXIT_RDSEED);
11370	SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSAVES, VMX_EXIT_XSAVES); /* unconditional (enabled by host, guest cfg) */
11371	SET_ONLY_XBM_IF_EITHER_EN(EXIT_XSAVES, VMX_EXIT_XSAVES);
11372	SET_ONLY_XBM_IF_EITHER_EN(INSTR_XRSTORS, VMX_EXIT_XRSTORS); /* unconditional (enabled by host, guest cfg) */
11373	SET_ONLY_XBM_IF_EITHER_EN( EXIT_XRSTORS, VMX_EXIT_XRSTORS);
11374
11375	#undef IS_EITHER_ENABLED
11376	#undef SET_ONLY_XBM_IF_EITHER_EN
11377	#undef SET_CPE1_XBM_IF_EITHER_EN
11378	#undef SET_CPEU_XBM_IF_EITHER_EN
11379	#undef SET_CPE2_XBM_IF_EITHER_EN
11380
11381	/*
11382	* Sanitize the control stuff.
11383	*/
11384	pDbgState->fCpe2Extra &= pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1;
11385	if (pDbgState->fCpe2Extra)
11386	pDbgState->fCpe1Extra \|= VMX_PROC_CTLS_USE_SECONDARY_CTLS;
11387	pDbgState->fCpe1Extra &= pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1;
11388	pDbgState->fCpe1Unwanted &= ~pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed0;
11389	if (pVCpu->hm.s.fDebugWantRdTscExit != RT_BOOL(pDbgState->fCpe1Extra & VMX_PROC_CTLS_RDTSC_EXIT))
11390	{
11391	pVCpu->hm.s.fDebugWantRdTscExit ^= true;
11392	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
11393	}
11394
11395	Log6(("HM: debug state: cpe1=%#RX32 cpeu=%#RX32 cpe2=%#RX32%s%s\n",
11396	pDbgState->fCpe1Extra, pDbgState->fCpe1Unwanted, pDbgState->fCpe2Extra,
11397	pDbgState->fClearCr0Mask ? " clr-cr0" : "",
11398	pDbgState->fClearCr4Mask ? " clr-cr4" : ""));
11399	}
11400
11401
11402	/**
11403	* Fires off DBGF events and dtrace probes for a VM-exit, when it's
11404	* appropriate.
11405	*
11406	* The caller has checked the VM-exit against the
11407	* VMXRUNDBGSTATE::bmExitsToCheck bitmap. The caller has checked for NMIs
11408	* already, so we don't have to do that either.
11409	*
11410	* @returns Strict VBox status code (i.e. informational status codes too).
11411	* @param pVCpu The cross context virtual CPU structure.
11412	* @param pVmxTransient The VMX-transient structure.
11413	* @param uExitReason The VM-exit reason.
11414	*
11415	* @remarks The name of this function is displayed by dtrace, so keep it short
11416	* and to the point. No longer than 33 chars long, please.
11417	*/
11418	static VBOXSTRICTRC hmR0VmxHandleExitDtraceEvents(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t uExitReason)
11419	{
11420	/*
11421	* Translate the event into a DBGF event (enmEvent + uEventArg) and at the
11422	* same time check whether any corresponding Dtrace event is enabled (fDtrace).
11423	*
11424	* Note! This is the reverse operation of what hmR0VmxPreRunGuestDebugStateUpdate
11425	* does. Must add/change/remove both places. Same ordering, please.
11426	*
11427	* Added/removed events must also be reflected in the next section
11428	* where we dispatch dtrace events.
11429	*/
11430	bool fDtrace1 = false;
11431	bool fDtrace2 = false;
11432	DBGFEVENTTYPE enmEvent1 = DBGFEVENT_END;
11433	DBGFEVENTTYPE enmEvent2 = DBGFEVENT_END;
11434	uint32_t uEventArg = 0;
11435	#define SET_EXIT(a_EventSubName) \
11436	do { \
11437	enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
11438	fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
11439	} while (0)
11440	#define SET_BOTH(a_EventSubName) \
11441	do { \
11442	enmEvent1 = RT_CONCAT(DBGFEVENT_INSTR_, a_EventSubName); \
11443	enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
11444	fDtrace1 = RT_CONCAT3(VBOXVMM_INSTR_, a_EventSubName, _ENABLED)(); \
11445	fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
11446	} while (0)
11447	switch (uExitReason)
11448	{
11449	case VMX_EXIT_MTF:
11450	return hmR0VmxExitMtf(pVCpu, pVmxTransient);
11451
11452	case VMX_EXIT_XCPT_OR_NMI:
11453	{
11454	uint8_t const idxVector = VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo);
11455	switch (VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo))
11456	{
11457	case VMX_EXIT_INT_INFO_TYPE_HW_XCPT:
11458	case VMX_EXIT_INT_INFO_TYPE_SW_XCPT:
11459	case VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT:
11460	if (idxVector <= (unsigned)(DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST))
11461	{
11462	if (VMX_EXIT_INT_INFO_IS_ERROR_CODE_VALID(pVmxTransient->uExitIntInfo))
11463	{
11464	hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11465	uEventArg = pVmxTransient->uExitIntErrorCode;
11466	}
11467	enmEvent1 = (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + idxVector);
11468	switch (enmEvent1)
11469	{
11470	case DBGFEVENT_XCPT_DE: fDtrace1 = VBOXVMM_XCPT_DE_ENABLED(); break;
11471	case DBGFEVENT_XCPT_DB: fDtrace1 = VBOXVMM_XCPT_DB_ENABLED(); break;
11472	case DBGFEVENT_XCPT_BP: fDtrace1 = VBOXVMM_XCPT_BP_ENABLED(); break;
11473	case DBGFEVENT_XCPT_OF: fDtrace1 = VBOXVMM_XCPT_OF_ENABLED(); break;
11474	case DBGFEVENT_XCPT_BR: fDtrace1 = VBOXVMM_XCPT_BR_ENABLED(); break;
11475	case DBGFEVENT_XCPT_UD: fDtrace1 = VBOXVMM_XCPT_UD_ENABLED(); break;
11476	case DBGFEVENT_XCPT_NM: fDtrace1 = VBOXVMM_XCPT_NM_ENABLED(); break;
11477	case DBGFEVENT_XCPT_DF: fDtrace1 = VBOXVMM_XCPT_DF_ENABLED(); break;
11478	case DBGFEVENT_XCPT_TS: fDtrace1 = VBOXVMM_XCPT_TS_ENABLED(); break;
11479	case DBGFEVENT_XCPT_NP: fDtrace1 = VBOXVMM_XCPT_NP_ENABLED(); break;
11480	case DBGFEVENT_XCPT_SS: fDtrace1 = VBOXVMM_XCPT_SS_ENABLED(); break;
11481	case DBGFEVENT_XCPT_GP: fDtrace1 = VBOXVMM_XCPT_GP_ENABLED(); break;
11482	case DBGFEVENT_XCPT_PF: fDtrace1 = VBOXVMM_XCPT_PF_ENABLED(); break;
11483	case DBGFEVENT_XCPT_MF: fDtrace1 = VBOXVMM_XCPT_MF_ENABLED(); break;
11484	case DBGFEVENT_XCPT_AC: fDtrace1 = VBOXVMM_XCPT_AC_ENABLED(); break;
11485	case DBGFEVENT_XCPT_XF: fDtrace1 = VBOXVMM_XCPT_XF_ENABLED(); break;
11486	case DBGFEVENT_XCPT_VE: fDtrace1 = VBOXVMM_XCPT_VE_ENABLED(); break;
11487	case DBGFEVENT_XCPT_SX: fDtrace1 = VBOXVMM_XCPT_SX_ENABLED(); break;
11488	default: break;
11489	}
11490	}
11491	else
11492	AssertFailed();
11493	break;
11494
11495	case VMX_EXIT_INT_INFO_TYPE_SW_INT:
11496	uEventArg = idxVector;
11497	enmEvent1 = DBGFEVENT_INTERRUPT_SOFTWARE;
11498	fDtrace1 = VBOXVMM_INT_SOFTWARE_ENABLED();
11499	break;
11500	}
11501	break;
11502	}
11503
11504	case VMX_EXIT_TRIPLE_FAULT:
11505	enmEvent1 = DBGFEVENT_TRIPLE_FAULT;
11506	//fDtrace1 = VBOXVMM_EXIT_TRIPLE_FAULT_ENABLED();
11507	break;
11508	case VMX_EXIT_TASK_SWITCH: SET_EXIT(TASK_SWITCH); break;
11509	case VMX_EXIT_EPT_VIOLATION: SET_EXIT(VMX_EPT_VIOLATION); break;
11510	case VMX_EXIT_EPT_MISCONFIG: SET_EXIT(VMX_EPT_MISCONFIG); break;
11511	case VMX_EXIT_APIC_ACCESS: SET_EXIT(VMX_VAPIC_ACCESS); break;
11512	case VMX_EXIT_APIC_WRITE: SET_EXIT(VMX_VAPIC_WRITE); break;
11513
11514	/* Instruction specific VM-exits: */
11515	case VMX_EXIT_CPUID: SET_BOTH(CPUID); break;
11516	case VMX_EXIT_GETSEC: SET_BOTH(GETSEC); break;
11517	case VMX_EXIT_HLT: SET_BOTH(HALT); break;
11518	case VMX_EXIT_INVD: SET_BOTH(INVD); break;
11519	case VMX_EXIT_INVLPG: SET_BOTH(INVLPG); break;
11520	case VMX_EXIT_RDPMC: SET_BOTH(RDPMC); break;
11521	case VMX_EXIT_RDTSC: SET_BOTH(RDTSC); break;
11522	case VMX_EXIT_RSM: SET_BOTH(RSM); break;
11523	case VMX_EXIT_VMCALL: SET_BOTH(VMM_CALL); break;
11524	case VMX_EXIT_VMCLEAR: SET_BOTH(VMX_VMCLEAR); break;
11525	case VMX_EXIT_VMLAUNCH: SET_BOTH(VMX_VMLAUNCH); break;
11526	case VMX_EXIT_VMPTRLD: SET_BOTH(VMX_VMPTRLD); break;
11527	case VMX_EXIT_VMPTRST: SET_BOTH(VMX_VMPTRST); break;
11528	case VMX_EXIT_VMREAD: SET_BOTH(VMX_VMREAD); break;
11529	case VMX_EXIT_VMRESUME: SET_BOTH(VMX_VMRESUME); break;
11530	case VMX_EXIT_VMWRITE: SET_BOTH(VMX_VMWRITE); break;
11531	case VMX_EXIT_VMXOFF: SET_BOTH(VMX_VMXOFF); break;
11532	case VMX_EXIT_VMXON: SET_BOTH(VMX_VMXON); break;
11533	case VMX_EXIT_MOV_CRX:
11534	hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
11535	if (VMX_EXIT_QUAL_CRX_ACCESS(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_CRX_ACCESS_READ)
11536	SET_BOTH(CRX_READ);
11537	else
11538	SET_BOTH(CRX_WRITE);
11539	uEventArg = VMX_EXIT_QUAL_CRX_REGISTER(pVmxTransient->uExitQual);
11540	break;
11541	case VMX_EXIT_MOV_DRX:
11542	hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
11543	if ( VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQual)
11544	== VMX_EXIT_QUAL_DRX_DIRECTION_READ)
11545	SET_BOTH(DRX_READ);
11546	else
11547	SET_BOTH(DRX_WRITE);
11548	uEventArg = VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQual);
11549	break;
11550	case VMX_EXIT_RDMSR: SET_BOTH(RDMSR); break;
11551	case VMX_EXIT_WRMSR: SET_BOTH(WRMSR); break;
11552	case VMX_EXIT_MWAIT: SET_BOTH(MWAIT); break;
11553	case VMX_EXIT_MONITOR: SET_BOTH(MONITOR); break;
11554	case VMX_EXIT_PAUSE: SET_BOTH(PAUSE); break;
11555	case VMX_EXIT_GDTR_IDTR_ACCESS:
11556	hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
11557	switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_BF_XDTR_INSINFO_INSTR_ID))
11558	{
11559	case VMX_XDTR_INSINFO_II_SGDT: SET_BOTH(SGDT); break;
11560	case VMX_XDTR_INSINFO_II_SIDT: SET_BOTH(SIDT); break;
11561	case VMX_XDTR_INSINFO_II_LGDT: SET_BOTH(LGDT); break;
11562	case VMX_XDTR_INSINFO_II_LIDT: SET_BOTH(LIDT); break;
11563	}
11564	break;
11565
11566	case VMX_EXIT_LDTR_TR_ACCESS:
11567	hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
11568	switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_BF_YYTR_INSINFO_INSTR_ID))
11569	{
11570	case VMX_YYTR_INSINFO_II_SLDT: SET_BOTH(SLDT); break;
11571	case VMX_YYTR_INSINFO_II_STR: SET_BOTH(STR); break;
11572	case VMX_YYTR_INSINFO_II_LLDT: SET_BOTH(LLDT); break;
11573	case VMX_YYTR_INSINFO_II_LTR: SET_BOTH(LTR); break;
11574	}
11575	break;
11576
11577	case VMX_EXIT_INVEPT: SET_BOTH(VMX_INVEPT); break;
11578	case VMX_EXIT_RDTSCP: SET_BOTH(RDTSCP); break;
11579	case VMX_EXIT_INVVPID: SET_BOTH(VMX_INVVPID); break;
11580	case VMX_EXIT_WBINVD: SET_BOTH(WBINVD); break;
11581	case VMX_EXIT_XSETBV: SET_BOTH(XSETBV); break;
11582	case VMX_EXIT_RDRAND: SET_BOTH(RDRAND); break;
11583	case VMX_EXIT_INVPCID: SET_BOTH(VMX_INVPCID); break;
11584	case VMX_EXIT_VMFUNC: SET_BOTH(VMX_VMFUNC); break;
11585	case VMX_EXIT_RDSEED: SET_BOTH(RDSEED); break;
11586	case VMX_EXIT_XSAVES: SET_BOTH(XSAVES); break;
11587	case VMX_EXIT_XRSTORS: SET_BOTH(XRSTORS); break;
11588
11589	/* Events that aren't relevant at this point. */
11590	case VMX_EXIT_EXT_INT:
11591	case VMX_EXIT_INT_WINDOW:
11592	case VMX_EXIT_NMI_WINDOW:
11593	case VMX_EXIT_TPR_BELOW_THRESHOLD:
11594	case VMX_EXIT_PREEMPT_TIMER:
11595	case VMX_EXIT_IO_INSTR:
11596	break;
11597
11598	/* Errors and unexpected events. */
11599	case VMX_EXIT_INIT_SIGNAL:
11600	case VMX_EXIT_SIPI:
11601	case VMX_EXIT_IO_SMI:
11602	case VMX_EXIT_SMI:
11603	case VMX_EXIT_ERR_INVALID_GUEST_STATE:
11604	case VMX_EXIT_ERR_MSR_LOAD:
11605	case VMX_EXIT_ERR_MACHINE_CHECK:
11606	break;
11607
11608	default:
11609	AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
11610	break;
11611	}
11612	#undef SET_BOTH
11613	#undef SET_EXIT
11614
11615	/*
11616	* Dtrace tracepoints go first. We do them here at once so we don't
11617	* have to copy the guest state saving and stuff a few dozen times.
11618	* Down side is that we've got to repeat the switch, though this time
11619	* we use enmEvent since the probes are a subset of what DBGF does.
11620	*/
11621	if (fDtrace1 \|\| fDtrace2)
11622	{
11623	hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
11624	hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
11625	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
11626	switch (enmEvent1)
11627	{
11628	/** @todo consider which extra parameters would be helpful for each probe. */
11629	case DBGFEVENT_END: break;
11630	case DBGFEVENT_XCPT_DE: VBOXVMM_XCPT_DE(pVCpu, pCtx); break;
11631	case DBGFEVENT_XCPT_DB: VBOXVMM_XCPT_DB(pVCpu, pCtx, pCtx->dr[6]); break;
11632	case DBGFEVENT_XCPT_BP: VBOXVMM_XCPT_BP(pVCpu, pCtx); break;
11633	case DBGFEVENT_XCPT_OF: VBOXVMM_XCPT_OF(pVCpu, pCtx); break;
11634	case DBGFEVENT_XCPT_BR: VBOXVMM_XCPT_BR(pVCpu, pCtx); break;
11635	case DBGFEVENT_XCPT_UD: VBOXVMM_XCPT_UD(pVCpu, pCtx); break;
11636	case DBGFEVENT_XCPT_NM: VBOXVMM_XCPT_NM(pVCpu, pCtx); break;
11637	case DBGFEVENT_XCPT_DF: VBOXVMM_XCPT_DF(pVCpu, pCtx); break;
11638	case DBGFEVENT_XCPT_TS: VBOXVMM_XCPT_TS(pVCpu, pCtx, uEventArg); break;
11639	case DBGFEVENT_XCPT_NP: VBOXVMM_XCPT_NP(pVCpu, pCtx, uEventArg); break;
11640	case DBGFEVENT_XCPT_SS: VBOXVMM_XCPT_SS(pVCpu, pCtx, uEventArg); break;
11641	case DBGFEVENT_XCPT_GP: VBOXVMM_XCPT_GP(pVCpu, pCtx, uEventArg); break;
11642	case DBGFEVENT_XCPT_PF: VBOXVMM_XCPT_PF(pVCpu, pCtx, uEventArg, pCtx->cr2); break;
11643	case DBGFEVENT_XCPT_MF: VBOXVMM_XCPT_MF(pVCpu, pCtx); break;
11644	case DBGFEVENT_XCPT_AC: VBOXVMM_XCPT_AC(pVCpu, pCtx); break;
11645	case DBGFEVENT_XCPT_XF: VBOXVMM_XCPT_XF(pVCpu, pCtx); break;
11646	case DBGFEVENT_XCPT_VE: VBOXVMM_XCPT_VE(pVCpu, pCtx); break;
11647	case DBGFEVENT_XCPT_SX: VBOXVMM_XCPT_SX(pVCpu, pCtx, uEventArg); break;
11648	case DBGFEVENT_INTERRUPT_SOFTWARE: VBOXVMM_INT_SOFTWARE(pVCpu, pCtx, (uint8_t)uEventArg); break;
11649	case DBGFEVENT_INSTR_CPUID: VBOXVMM_INSTR_CPUID(pVCpu, pCtx, pCtx->eax, pCtx->ecx); break;
11650	case DBGFEVENT_INSTR_GETSEC: VBOXVMM_INSTR_GETSEC(pVCpu, pCtx); break;
11651	case DBGFEVENT_INSTR_HALT: VBOXVMM_INSTR_HALT(pVCpu, pCtx); break;
11652	case DBGFEVENT_INSTR_INVD: VBOXVMM_INSTR_INVD(pVCpu, pCtx); break;
11653	case DBGFEVENT_INSTR_INVLPG: VBOXVMM_INSTR_INVLPG(pVCpu, pCtx); break;
11654	case DBGFEVENT_INSTR_RDPMC: VBOXVMM_INSTR_RDPMC(pVCpu, pCtx); break;
11655	case DBGFEVENT_INSTR_RDTSC: VBOXVMM_INSTR_RDTSC(pVCpu, pCtx); break;
11656	case DBGFEVENT_INSTR_RSM: VBOXVMM_INSTR_RSM(pVCpu, pCtx); break;
11657	case DBGFEVENT_INSTR_CRX_READ: VBOXVMM_INSTR_CRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
11658	case DBGFEVENT_INSTR_CRX_WRITE: VBOXVMM_INSTR_CRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
11659	case DBGFEVENT_INSTR_DRX_READ: VBOXVMM_INSTR_DRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
11660	case DBGFEVENT_INSTR_DRX_WRITE: VBOXVMM_INSTR_DRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
11661	case DBGFEVENT_INSTR_RDMSR: VBOXVMM_INSTR_RDMSR(pVCpu, pCtx, pCtx->ecx); break;
11662	case DBGFEVENT_INSTR_WRMSR: VBOXVMM_INSTR_WRMSR(pVCpu, pCtx, pCtx->ecx,
11663	RT_MAKE_U64(pCtx->eax, pCtx->edx)); break;
11664	case DBGFEVENT_INSTR_MWAIT: VBOXVMM_INSTR_MWAIT(pVCpu, pCtx); break;
11665	case DBGFEVENT_INSTR_MONITOR: VBOXVMM_INSTR_MONITOR(pVCpu, pCtx); break;
11666	case DBGFEVENT_INSTR_PAUSE: VBOXVMM_INSTR_PAUSE(pVCpu, pCtx); break;
11667	case DBGFEVENT_INSTR_SGDT: VBOXVMM_INSTR_SGDT(pVCpu, pCtx); break;
11668	case DBGFEVENT_INSTR_SIDT: VBOXVMM_INSTR_SIDT(pVCpu, pCtx); break;
11669	case DBGFEVENT_INSTR_LGDT: VBOXVMM_INSTR_LGDT(pVCpu, pCtx); break;
11670	case DBGFEVENT_INSTR_LIDT: VBOXVMM_INSTR_LIDT(pVCpu, pCtx); break;
11671	case DBGFEVENT_INSTR_SLDT: VBOXVMM_INSTR_SLDT(pVCpu, pCtx); break;
11672	case DBGFEVENT_INSTR_STR: VBOXVMM_INSTR_STR(pVCpu, pCtx); break;
11673	case DBGFEVENT_INSTR_LLDT: VBOXVMM_INSTR_LLDT(pVCpu, pCtx); break;
11674	case DBGFEVENT_INSTR_LTR: VBOXVMM_INSTR_LTR(pVCpu, pCtx); break;
11675	case DBGFEVENT_INSTR_RDTSCP: VBOXVMM_INSTR_RDTSCP(pVCpu, pCtx); break;
11676	case DBGFEVENT_INSTR_WBINVD: VBOXVMM_INSTR_WBINVD(pVCpu, pCtx); break;
11677	case DBGFEVENT_INSTR_XSETBV: VBOXVMM_INSTR_XSETBV(pVCpu, pCtx); break;
11678	case DBGFEVENT_INSTR_RDRAND: VBOXVMM_INSTR_RDRAND(pVCpu, pCtx); break;
11679	case DBGFEVENT_INSTR_RDSEED: VBOXVMM_INSTR_RDSEED(pVCpu, pCtx); break;
11680	case DBGFEVENT_INSTR_XSAVES: VBOXVMM_INSTR_XSAVES(pVCpu, pCtx); break;
11681	case DBGFEVENT_INSTR_XRSTORS: VBOXVMM_INSTR_XRSTORS(pVCpu, pCtx); break;
11682	case DBGFEVENT_INSTR_VMM_CALL: VBOXVMM_INSTR_VMM_CALL(pVCpu, pCtx); break;
11683	case DBGFEVENT_INSTR_VMX_VMCLEAR: VBOXVMM_INSTR_VMX_VMCLEAR(pVCpu, pCtx); break;
11684	case DBGFEVENT_INSTR_VMX_VMLAUNCH: VBOXVMM_INSTR_VMX_VMLAUNCH(pVCpu, pCtx); break;
11685	case DBGFEVENT_INSTR_VMX_VMPTRLD: VBOXVMM_INSTR_VMX_VMPTRLD(pVCpu, pCtx); break;
11686	case DBGFEVENT_INSTR_VMX_VMPTRST: VBOXVMM_INSTR_VMX_VMPTRST(pVCpu, pCtx); break;
11687	case DBGFEVENT_INSTR_VMX_VMREAD: VBOXVMM_INSTR_VMX_VMREAD(pVCpu, pCtx); break;
11688	case DBGFEVENT_INSTR_VMX_VMRESUME: VBOXVMM_INSTR_VMX_VMRESUME(pVCpu, pCtx); break;
11689	case DBGFEVENT_INSTR_VMX_VMWRITE: VBOXVMM_INSTR_VMX_VMWRITE(pVCpu, pCtx); break;
11690	case DBGFEVENT_INSTR_VMX_VMXOFF: VBOXVMM_INSTR_VMX_VMXOFF(pVCpu, pCtx); break;
11691	case DBGFEVENT_INSTR_VMX_VMXON: VBOXVMM_INSTR_VMX_VMXON(pVCpu, pCtx); break;
11692	case DBGFEVENT_INSTR_VMX_INVEPT: VBOXVMM_INSTR_VMX_INVEPT(pVCpu, pCtx); break;
11693	case DBGFEVENT_INSTR_VMX_INVVPID: VBOXVMM_INSTR_VMX_INVVPID(pVCpu, pCtx); break;
11694	case DBGFEVENT_INSTR_VMX_INVPCID: VBOXVMM_INSTR_VMX_INVPCID(pVCpu, pCtx); break;
11695	case DBGFEVENT_INSTR_VMX_VMFUNC: VBOXVMM_INSTR_VMX_VMFUNC(pVCpu, pCtx); break;
11696	default: AssertMsgFailed(("enmEvent1=%d uExitReason=%d\n", enmEvent1, uExitReason)); break;
11697	}
11698	switch (enmEvent2)
11699	{
11700	/** @todo consider which extra parameters would be helpful for each probe. */
11701	case DBGFEVENT_END: break;
11702	case DBGFEVENT_EXIT_TASK_SWITCH: VBOXVMM_EXIT_TASK_SWITCH(pVCpu, pCtx); break;
11703	case DBGFEVENT_EXIT_CPUID: VBOXVMM_EXIT_CPUID(pVCpu, pCtx, pCtx->eax, pCtx->ecx); break;
11704	case DBGFEVENT_EXIT_GETSEC: VBOXVMM_EXIT_GETSEC(pVCpu, pCtx); break;
11705	case DBGFEVENT_EXIT_HALT: VBOXVMM_EXIT_HALT(pVCpu, pCtx); break;
11706	case DBGFEVENT_EXIT_INVD: VBOXVMM_EXIT_INVD(pVCpu, pCtx); break;
11707	case DBGFEVENT_EXIT_INVLPG: VBOXVMM_EXIT_INVLPG(pVCpu, pCtx); break;
11708	case DBGFEVENT_EXIT_RDPMC: VBOXVMM_EXIT_RDPMC(pVCpu, pCtx); break;
11709	case DBGFEVENT_EXIT_RDTSC: VBOXVMM_EXIT_RDTSC(pVCpu, pCtx); break;
11710	case DBGFEVENT_EXIT_RSM: VBOXVMM_EXIT_RSM(pVCpu, pCtx); break;
11711	case DBGFEVENT_EXIT_CRX_READ: VBOXVMM_EXIT_CRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
11712	case DBGFEVENT_EXIT_CRX_WRITE: VBOXVMM_EXIT_CRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
11713	case DBGFEVENT_EXIT_DRX_READ: VBOXVMM_EXIT_DRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
11714	case DBGFEVENT_EXIT_DRX_WRITE: VBOXVMM_EXIT_DRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
11715	case DBGFEVENT_EXIT_RDMSR: VBOXVMM_EXIT_RDMSR(pVCpu, pCtx, pCtx->ecx); break;
11716	case DBGFEVENT_EXIT_WRMSR: VBOXVMM_EXIT_WRMSR(pVCpu, pCtx, pCtx->ecx,
11717	RT_MAKE_U64(pCtx->eax, pCtx->edx)); break;
11718	case DBGFEVENT_EXIT_MWAIT: VBOXVMM_EXIT_MWAIT(pVCpu, pCtx); break;
11719	case DBGFEVENT_EXIT_MONITOR: VBOXVMM_EXIT_MONITOR(pVCpu, pCtx); break;
11720	case DBGFEVENT_EXIT_PAUSE: VBOXVMM_EXIT_PAUSE(pVCpu, pCtx); break;
11721	case DBGFEVENT_EXIT_SGDT: VBOXVMM_EXIT_SGDT(pVCpu, pCtx); break;
11722	case DBGFEVENT_EXIT_SIDT: VBOXVMM_EXIT_SIDT(pVCpu, pCtx); break;
11723	case DBGFEVENT_EXIT_LGDT: VBOXVMM_EXIT_LGDT(pVCpu, pCtx); break;
11724	case DBGFEVENT_EXIT_LIDT: VBOXVMM_EXIT_LIDT(pVCpu, pCtx); break;
11725	case DBGFEVENT_EXIT_SLDT: VBOXVMM_EXIT_SLDT(pVCpu, pCtx); break;
11726	case DBGFEVENT_EXIT_STR: VBOXVMM_EXIT_STR(pVCpu, pCtx); break;
11727	case DBGFEVENT_EXIT_LLDT: VBOXVMM_EXIT_LLDT(pVCpu, pCtx); break;
11728	case DBGFEVENT_EXIT_LTR: VBOXVMM_EXIT_LTR(pVCpu, pCtx); break;
11729	case DBGFEVENT_EXIT_RDTSCP: VBOXVMM_EXIT_RDTSCP(pVCpu, pCtx); break;
11730	case DBGFEVENT_EXIT_WBINVD: VBOXVMM_EXIT_WBINVD(pVCpu, pCtx); break;
11731	case DBGFEVENT_EXIT_XSETBV: VBOXVMM_EXIT_XSETBV(pVCpu, pCtx); break;
11732	case DBGFEVENT_EXIT_RDRAND: VBOXVMM_EXIT_RDRAND(pVCpu, pCtx); break;
11733	case DBGFEVENT_EXIT_RDSEED: VBOXVMM_EXIT_RDSEED(pVCpu, pCtx); break;
11734	case DBGFEVENT_EXIT_XSAVES: VBOXVMM_EXIT_XSAVES(pVCpu, pCtx); break;
11735	case DBGFEVENT_EXIT_XRSTORS: VBOXVMM_EXIT_XRSTORS(pVCpu, pCtx); break;
11736	case DBGFEVENT_EXIT_VMM_CALL: VBOXVMM_EXIT_VMM_CALL(pVCpu, pCtx); break;
11737	case DBGFEVENT_EXIT_VMX_VMCLEAR: VBOXVMM_EXIT_VMX_VMCLEAR(pVCpu, pCtx); break;
11738	case DBGFEVENT_EXIT_VMX_VMLAUNCH: VBOXVMM_EXIT_VMX_VMLAUNCH(pVCpu, pCtx); break;
11739	case DBGFEVENT_EXIT_VMX_VMPTRLD: VBOXVMM_EXIT_VMX_VMPTRLD(pVCpu, pCtx); break;
11740	case DBGFEVENT_EXIT_VMX_VMPTRST: VBOXVMM_EXIT_VMX_VMPTRST(pVCpu, pCtx); break;
11741	case DBGFEVENT_EXIT_VMX_VMREAD: VBOXVMM_EXIT_VMX_VMREAD(pVCpu, pCtx); break;
11742	case DBGFEVENT_EXIT_VMX_VMRESUME: VBOXVMM_EXIT_VMX_VMRESUME(pVCpu, pCtx); break;
11743	case DBGFEVENT_EXIT_VMX_VMWRITE: VBOXVMM_EXIT_VMX_VMWRITE(pVCpu, pCtx); break;
11744	case DBGFEVENT_EXIT_VMX_VMXOFF: VBOXVMM_EXIT_VMX_VMXOFF(pVCpu, pCtx); break;
11745	case DBGFEVENT_EXIT_VMX_VMXON: VBOXVMM_EXIT_VMX_VMXON(pVCpu, pCtx); break;
11746	case DBGFEVENT_EXIT_VMX_INVEPT: VBOXVMM_EXIT_VMX_INVEPT(pVCpu, pCtx); break;
11747	case DBGFEVENT_EXIT_VMX_INVVPID: VBOXVMM_EXIT_VMX_INVVPID(pVCpu, pCtx); break;
11748	case DBGFEVENT_EXIT_VMX_INVPCID: VBOXVMM_EXIT_VMX_INVPCID(pVCpu, pCtx); break;
11749	case DBGFEVENT_EXIT_VMX_VMFUNC: VBOXVMM_EXIT_VMX_VMFUNC(pVCpu, pCtx); break;
11750	case DBGFEVENT_EXIT_VMX_EPT_MISCONFIG: VBOXVMM_EXIT_VMX_EPT_MISCONFIG(pVCpu, pCtx); break;
11751	case DBGFEVENT_EXIT_VMX_EPT_VIOLATION: VBOXVMM_EXIT_VMX_EPT_VIOLATION(pVCpu, pCtx); break;
11752	case DBGFEVENT_EXIT_VMX_VAPIC_ACCESS: VBOXVMM_EXIT_VMX_VAPIC_ACCESS(pVCpu, pCtx); break;
11753	case DBGFEVENT_EXIT_VMX_VAPIC_WRITE: VBOXVMM_EXIT_VMX_VAPIC_WRITE(pVCpu, pCtx); break;
11754	default: AssertMsgFailed(("enmEvent2=%d uExitReason=%d\n", enmEvent2, uExitReason)); break;
11755	}
11756	}
11757
11758	/*
11759	* Fire of the DBGF event, if enabled (our check here is just a quick one,
11760	* the DBGF call will do a full check).
11761	*
11762	* Note! DBGF sets DBGFEVENT_INTERRUPT_SOFTWARE in the bitmap.
11763	* Note! If we have to events, we prioritize the first, i.e. the instruction
11764	* one, in order to avoid event nesting.
11765	*/
11766	PVM pVM = pVCpu->CTX_SUFF(pVM);
11767	if ( enmEvent1 != DBGFEVENT_END
11768	&& DBGF_IS_EVENT_ENABLED(pVM, enmEvent1))
11769	{
11770	hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
11771	VBOXSTRICTRC rcStrict = DBGFEventGenericWithArgs(pVM, pVCpu, enmEvent1, DBGFEVENTCTX_HM, 1, uEventArg);
11772	if (rcStrict != VINF_SUCCESS)
11773	return rcStrict;
11774	}
11775	else if ( enmEvent2 != DBGFEVENT_END
11776	&& DBGF_IS_EVENT_ENABLED(pVM, enmEvent2))
11777	{
11778	hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
11779	VBOXSTRICTRC rcStrict = DBGFEventGenericWithArgs(pVM, pVCpu, enmEvent2, DBGFEVENTCTX_HM, 1, uEventArg);
11780	if (rcStrict != VINF_SUCCESS)
11781	return rcStrict;
11782	}
11783
11784	return VINF_SUCCESS;
11785	}
11786
11787
11788	/**
11789	* Single-stepping VM-exit filtering.
11790	*
11791	* This is preprocessing the VM-exits and deciding whether we've gotten far
11792	* enough to return VINF_EM_DBG_STEPPED already. If not, normal VM-exit
11793	* handling is performed.
11794	*
11795	* @returns Strict VBox status code (i.e. informational status codes too).
11796	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
11797	* @param pVmxTransient The VMX-transient structure.
11798	* @param pDbgState The debug state.
11799	*/
11800	DECLINLINE(VBOXSTRICTRC) hmR0VmxRunDebugHandleExit(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState)
11801	{
11802	/*
11803	* Expensive (saves context) generic dtrace VM-exit probe.
11804	*/
11805	uint32_t const uExitReason = pVmxTransient->uExitReason;
11806	if (!VBOXVMM_R0_HMVMX_VMEXIT_ENABLED())
11807	{ /* more likely */ }
11808	else
11809	{
11810	hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
11811	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
11812	AssertRC(rc);
11813	VBOXVMM_R0_HMVMX_VMEXIT(pVCpu, &pVCpu->cpum.GstCtx, pVmxTransient->uExitReason, pVmxTransient->uExitQual);
11814	}
11815
11816	/*
11817	* Check for host NMI, just to get that out of the way.
11818	*/
11819	if (uExitReason != VMX_EXIT_XCPT_OR_NMI)
11820	{ /* normally likely */ }
11821	else
11822	{
11823	int rc2 = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11824	AssertRCReturn(rc2, rc2);
11825	uint32_t uIntType = VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo);
11826	if (uIntType == VMX_EXIT_INT_INFO_TYPE_NMI)
11827	return hmR0VmxExitXcptOrNmi(pVCpu, pVmxTransient);
11828	}
11829
11830	/*
11831	* Check for single stepping event if we're stepping.
11832	*/
11833	if (pVCpu->hm.s.fSingleInstruction)
11834	{
11835	switch (uExitReason)
11836	{
11837	case VMX_EXIT_MTF:
11838	return hmR0VmxExitMtf(pVCpu, pVmxTransient);
11839
11840	/* Various events: */
11841	case VMX_EXIT_XCPT_OR_NMI:
11842	case VMX_EXIT_EXT_INT:
11843	case VMX_EXIT_TRIPLE_FAULT:
11844	case VMX_EXIT_INT_WINDOW:
11845	case VMX_EXIT_NMI_WINDOW:
11846	case VMX_EXIT_TASK_SWITCH:
11847	case VMX_EXIT_TPR_BELOW_THRESHOLD:
11848	case VMX_EXIT_APIC_ACCESS:
11849	case VMX_EXIT_EPT_VIOLATION:
11850	case VMX_EXIT_EPT_MISCONFIG:
11851	case VMX_EXIT_PREEMPT_TIMER:
11852
11853	/* Instruction specific VM-exits: */
11854	case VMX_EXIT_CPUID:
11855	case VMX_EXIT_GETSEC:
11856	case VMX_EXIT_HLT:
11857	case VMX_EXIT_INVD:
11858	case VMX_EXIT_INVLPG:
11859	case VMX_EXIT_RDPMC:
11860	case VMX_EXIT_RDTSC:
11861	case VMX_EXIT_RSM:
11862	case VMX_EXIT_VMCALL:
11863	case VMX_EXIT_VMCLEAR:
11864	case VMX_EXIT_VMLAUNCH:
11865	case VMX_EXIT_VMPTRLD:
11866	case VMX_EXIT_VMPTRST:
11867	case VMX_EXIT_VMREAD:
11868	case VMX_EXIT_VMRESUME:
11869	case VMX_EXIT_VMWRITE:
11870	case VMX_EXIT_VMXOFF:
11871	case VMX_EXIT_VMXON:
11872	case VMX_EXIT_MOV_CRX:
11873	case VMX_EXIT_MOV_DRX:
11874	case VMX_EXIT_IO_INSTR:
11875	case VMX_EXIT_RDMSR:
11876	case VMX_EXIT_WRMSR:
11877	case VMX_EXIT_MWAIT:
11878	case VMX_EXIT_MONITOR:
11879	case VMX_EXIT_PAUSE:
11880	case VMX_EXIT_GDTR_IDTR_ACCESS:
11881	case VMX_EXIT_LDTR_TR_ACCESS:
11882	case VMX_EXIT_INVEPT:
11883	case VMX_EXIT_RDTSCP:
11884	case VMX_EXIT_INVVPID:
11885	case VMX_EXIT_WBINVD:
11886	case VMX_EXIT_XSETBV:
11887	case VMX_EXIT_RDRAND:
11888	case VMX_EXIT_INVPCID:
11889	case VMX_EXIT_VMFUNC:
11890	case VMX_EXIT_RDSEED:
11891	case VMX_EXIT_XSAVES:
11892	case VMX_EXIT_XRSTORS:
11893	{
11894	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
11895	AssertRCReturn(rc, rc);
11896	if ( pVCpu->cpum.GstCtx.rip != pDbgState->uRipStart
11897	\|\| pVCpu->cpum.GstCtx.cs.Sel != pDbgState->uCsStart)
11898	return VINF_EM_DBG_STEPPED;
11899	break;
11900	}
11901
11902	/* Errors and unexpected events: */
11903	case VMX_EXIT_INIT_SIGNAL:
11904	case VMX_EXIT_SIPI:
11905	case VMX_EXIT_IO_SMI:
11906	case VMX_EXIT_SMI:
11907	case VMX_EXIT_ERR_INVALID_GUEST_STATE:
11908	case VMX_EXIT_ERR_MSR_LOAD:
11909	case VMX_EXIT_ERR_MACHINE_CHECK:
11910	case VMX_EXIT_APIC_WRITE: /* Some talk about this being fault like, so I guess we must process it? */
11911	break;
11912
11913	default:
11914	AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
11915	break;
11916	}
11917	}
11918
11919	/*
11920	* Check for debugger event breakpoints and dtrace probes.
11921	*/
11922	if ( uExitReason < RT_ELEMENTS(pDbgState->bmExitsToCheck) * 32U
11923	&& ASMBitTest(pDbgState->bmExitsToCheck, uExitReason) )
11924	{
11925	VBOXSTRICTRC rcStrict = hmR0VmxHandleExitDtraceEvents(pVCpu, pVmxTransient, uExitReason);
11926	if (rcStrict != VINF_SUCCESS)
11927	return rcStrict;
11928	}
11929
11930	/*
11931	* Normal processing.
11932	*/
11933	#ifdef HMVMX_USE_FUNCTION_TABLE
11934	return g_apfnVMExitHandlers[uExitReason](pVCpu, pVmxTransient);
11935	#else
11936	return hmR0VmxHandleExit(pVCpu, pVmxTransient, uExitReason);
11937	#endif
11938	}
11939
11940
11941	/**
11942	* Single steps guest code using hardware-assisted VMX.
11943	*
11944	* This is -not- the same as the guest single-stepping itself (say using EFLAGS.TF)
11945	* but single-stepping through the hypervisor debugger.
11946	*
11947	* @returns Strict VBox status code (i.e. informational status codes too).
11948	* @param pVCpu The cross context virtual CPU structure.
11949	* @param pcLoops Pointer to the number of executed loops.
11950	*
11951	* @note Mostly the same as hmR0VmxRunGuestCodeNormal().
11952	*/
11953	static VBOXSTRICTRC hmR0VmxRunGuestCodeDebug(PVMCPU pVCpu, uint32_t *pcLoops)
11954	{
11955	uint32_t const cMaxResumeLoops = pVCpu->CTX_SUFF(pVM)->hm.s.cMaxResumeLoops;
11956	Assert(pcLoops);
11957	Assert(*pcLoops <= cMaxResumeLoops);
11958
11959	VMXTRANSIENT VmxTransient;
11960	RT_ZERO(VmxTransient);
11961	VmxTransient.pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
11962
11963	/* Set HMCPU indicators. */
11964	bool const fSavedSingleInstruction = pVCpu->hm.s.fSingleInstruction;
11965	pVCpu->hm.s.fSingleInstruction = pVCpu->hm.s.fSingleInstruction \|\| DBGFIsStepping(pVCpu);
11966	pVCpu->hm.s.fDebugWantRdTscExit = false;
11967	pVCpu->hm.s.fUsingDebugLoop = true;
11968
11969	/* State we keep to help modify and later restore the VMCS fields we alter, and for detecting steps. */
11970	VMXRUNDBGSTATE DbgState;
11971	hmR0VmxRunDebugStateInit(pVCpu, &VmxTransient, &DbgState);
11972	hmR0VmxPreRunGuestDebugStateUpdate(pVCpu, &VmxTransient, &DbgState);
11973
11974	/*
11975	* The loop.
11976	*/
11977	VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
11978	for (;;)
11979	{
11980	Assert(!HMR0SuspendPending());
11981	HMVMX_ASSERT_CPU_SAFE(pVCpu);
11982	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
11983	bool fStepping = pVCpu->hm.s.fSingleInstruction;
11984
11985	/* Set up VM-execution controls the next two can respond to. */
11986	hmR0VmxPreRunGuestDebugStateApply(pVCpu, &VmxTransient, &DbgState);
11987
11988	/*
11989	* Preparatory work for running guest code, this may force us to
11990	* return to ring-3.
11991	*
11992	* Warning! This bugger disables interrupts on VINF_SUCCESS!
11993	*/
11994	rcStrict = hmR0VmxPreRunGuest(pVCpu, &VmxTransient, fStepping);
11995	if (rcStrict != VINF_SUCCESS)
11996	break;
11997
11998	/* Interrupts are disabled at this point! */
11999	hmR0VmxPreRunGuestCommitted(pVCpu, &VmxTransient);
12000
12001	/* Override any obnoxious code in the above two calls. */
12002	hmR0VmxPreRunGuestDebugStateApply(pVCpu, &VmxTransient, &DbgState);
12003
12004	/*
12005	* Finally execute the guest.
12006	*/
12007	int rcRun = hmR0VmxRunGuest(pVCpu, &VmxTransient);
12008
12009	hmR0VmxPostRunGuest(pVCpu, &VmxTransient, rcRun);
12010	/* Interrupts are re-enabled at this point! */
12011
12012	/* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
12013	if (RT_SUCCESS(rcRun))
12014	{ /* very likely */ }
12015	else
12016	{
12017	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
12018	hmR0VmxReportWorldSwitchError(pVCpu, rcRun, &VmxTransient);
12019	return rcRun;
12020	}
12021
12022	/* Profile the VM-exit. */
12023	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
12024	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
12025	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
12026	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
12027	HMVMX_START_EXIT_DISPATCH_PROF();
12028
12029	VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, &pVCpu->cpum.GstCtx, VmxTransient.uExitReason);
12030
12031	/*
12032	* Handle the VM-exit - we quit earlier on certain VM-exits, see hmR0VmxHandleExitDebug().
12033	*/
12034	rcStrict = hmR0VmxRunDebugHandleExit(pVCpu, &VmxTransient, &DbgState);
12035	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
12036	if (rcStrict != VINF_SUCCESS)
12037	break;
12038	if (++(*pcLoops) > cMaxResumeLoops)
12039	{
12040	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
12041	rcStrict = VINF_EM_RAW_INTERRUPT;
12042	break;
12043	}
12044
12045	/*
12046	* Stepping: Did the RIP change, if so, consider it a single step.
12047	* Otherwise, make sure one of the TFs gets set.
12048	*/
12049	if (fStepping)
12050	{
12051	int rc = hmR0VmxImportGuestState(pVCpu, VmxTransient.pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
12052	AssertRC(rc);
12053	if ( pVCpu->cpum.GstCtx.rip != DbgState.uRipStart
12054	\|\| pVCpu->cpum.GstCtx.cs.Sel != DbgState.uCsStart)
12055	{
12056	rcStrict = VINF_EM_DBG_STEPPED;
12057	break;
12058	}
12059	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_DR7);
12060	}
12061
12062	/*
12063	* Update when dtrace settings changes (DBGF kicks us, so no need to check).
12064	*/
12065	if (VBOXVMM_GET_SETTINGS_SEQ_NO() != DbgState.uDtraceSettingsSeqNo)
12066	hmR0VmxPreRunGuestDebugStateUpdate(pVCpu, &VmxTransient, &DbgState);
12067	}
12068
12069	/*
12070	* Clear the X86_EFL_TF if necessary.
12071	*/
12072	if (pVCpu->hm.s.fClearTrapFlag)
12073	{
12074	int rc = hmR0VmxImportGuestState(pVCpu, VmxTransient.pVmcsInfo, CPUMCTX_EXTRN_RFLAGS);
12075	AssertRC(rc);
12076	pVCpu->hm.s.fClearTrapFlag = false;
12077	pVCpu->cpum.GstCtx.eflags.Bits.u1TF = 0;
12078	}
12079	/** @todo there seems to be issues with the resume flag when the monitor trap
12080	* flag is pending without being used. Seen early in bios init when
12081	* accessing APIC page in protected mode. */
12082
12083	/*
12084	* Restore VM-exit control settings as we may not re-enter this function the
12085	* next time around.
12086	*/
12087	rcStrict = hmR0VmxRunDebugStateRevert(pVCpu, &VmxTransient, &DbgState, rcStrict);
12088
12089	/* Restore HMCPU indicators. */
12090	pVCpu->hm.s.fUsingDebugLoop = false;
12091	pVCpu->hm.s.fDebugWantRdTscExit = false;
12092	pVCpu->hm.s.fSingleInstruction = fSavedSingleInstruction;
12093
12094	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
12095	return rcStrict;
12096	}
12097
12098
12099	/** @} */
12100
12101
12102	/**
12103	* Checks if any expensive dtrace probes are enabled and we should go to the
12104	* debug loop.
12105	*
12106	* @returns true if we should use debug loop, false if not.
12107	*/
12108	static bool hmR0VmxAnyExpensiveProbesEnabled(void)
12109	{
12110	/* It's probably faster to OR the raw 32-bit counter variables together.
12111	Since the variables are in an array and the probes are next to one
12112	another (more or less), we have good locality. So, better read
12113	eight-nine cache lines ever time and only have one conditional, than
12114	128+ conditionals, right? */
12115	return ( VBOXVMM_R0_HMVMX_VMEXIT_ENABLED_RAW() /* expensive too due to context */
12116	\| VBOXVMM_XCPT_DE_ENABLED_RAW()
12117	\| VBOXVMM_XCPT_DB_ENABLED_RAW()
12118	\| VBOXVMM_XCPT_BP_ENABLED_RAW()
12119	\| VBOXVMM_XCPT_OF_ENABLED_RAW()
12120	\| VBOXVMM_XCPT_BR_ENABLED_RAW()
12121	\| VBOXVMM_XCPT_UD_ENABLED_RAW()
12122	\| VBOXVMM_XCPT_NM_ENABLED_RAW()
12123	\| VBOXVMM_XCPT_DF_ENABLED_RAW()
12124	\| VBOXVMM_XCPT_TS_ENABLED_RAW()
12125	\| VBOXVMM_XCPT_NP_ENABLED_RAW()
12126	\| VBOXVMM_XCPT_SS_ENABLED_RAW()
12127	\| VBOXVMM_XCPT_GP_ENABLED_RAW()
12128	\| VBOXVMM_XCPT_PF_ENABLED_RAW()
12129	\| VBOXVMM_XCPT_MF_ENABLED_RAW()
12130	\| VBOXVMM_XCPT_AC_ENABLED_RAW()
12131	\| VBOXVMM_XCPT_XF_ENABLED_RAW()
12132	\| VBOXVMM_XCPT_VE_ENABLED_RAW()
12133	\| VBOXVMM_XCPT_SX_ENABLED_RAW()
12134	\| VBOXVMM_INT_SOFTWARE_ENABLED_RAW()
12135	\| VBOXVMM_INT_HARDWARE_ENABLED_RAW()
12136	) != 0
12137	\|\| ( VBOXVMM_INSTR_HALT_ENABLED_RAW()
12138	\| VBOXVMM_INSTR_MWAIT_ENABLED_RAW()
12139	\| VBOXVMM_INSTR_MONITOR_ENABLED_RAW()
12140	\| VBOXVMM_INSTR_CPUID_ENABLED_RAW()
12141	\| VBOXVMM_INSTR_INVD_ENABLED_RAW()
12142	\| VBOXVMM_INSTR_WBINVD_ENABLED_RAW()
12143	\| VBOXVMM_INSTR_INVLPG_ENABLED_RAW()
12144	\| VBOXVMM_INSTR_RDTSC_ENABLED_RAW()
12145	\| VBOXVMM_INSTR_RDTSCP_ENABLED_RAW()
12146	\| VBOXVMM_INSTR_RDPMC_ENABLED_RAW()
12147	\| VBOXVMM_INSTR_RDMSR_ENABLED_RAW()
12148	\| VBOXVMM_INSTR_WRMSR_ENABLED_RAW()
12149	\| VBOXVMM_INSTR_CRX_READ_ENABLED_RAW()
12150	\| VBOXVMM_INSTR_CRX_WRITE_ENABLED_RAW()
12151	\| VBOXVMM_INSTR_DRX_READ_ENABLED_RAW()
12152	\| VBOXVMM_INSTR_DRX_WRITE_ENABLED_RAW()
12153	\| VBOXVMM_INSTR_PAUSE_ENABLED_RAW()
12154	\| VBOXVMM_INSTR_XSETBV_ENABLED_RAW()
12155	\| VBOXVMM_INSTR_SIDT_ENABLED_RAW()
12156	\| VBOXVMM_INSTR_LIDT_ENABLED_RAW()
12157	\| VBOXVMM_INSTR_SGDT_ENABLED_RAW()
12158	\| VBOXVMM_INSTR_LGDT_ENABLED_RAW()
12159	\| VBOXVMM_INSTR_SLDT_ENABLED_RAW()
12160	\| VBOXVMM_INSTR_LLDT_ENABLED_RAW()
12161	\| VBOXVMM_INSTR_STR_ENABLED_RAW()
12162	\| VBOXVMM_INSTR_LTR_ENABLED_RAW()
12163	\| VBOXVMM_INSTR_GETSEC_ENABLED_RAW()
12164	\| VBOXVMM_INSTR_RSM_ENABLED_RAW()
12165	\| VBOXVMM_INSTR_RDRAND_ENABLED_RAW()
12166	\| VBOXVMM_INSTR_RDSEED_ENABLED_RAW()
12167	\| VBOXVMM_INSTR_XSAVES_ENABLED_RAW()
12168	\| VBOXVMM_INSTR_XRSTORS_ENABLED_RAW()
12169	\| VBOXVMM_INSTR_VMM_CALL_ENABLED_RAW()
12170	\| VBOXVMM_INSTR_VMX_VMCLEAR_ENABLED_RAW()
12171	\| VBOXVMM_INSTR_VMX_VMLAUNCH_ENABLED_RAW()
12172	\| VBOXVMM_INSTR_VMX_VMPTRLD_ENABLED_RAW()
12173	\| VBOXVMM_INSTR_VMX_VMPTRST_ENABLED_RAW()
12174	\| VBOXVMM_INSTR_VMX_VMREAD_ENABLED_RAW()
12175	\| VBOXVMM_INSTR_VMX_VMRESUME_ENABLED_RAW()
12176	\| VBOXVMM_INSTR_VMX_VMWRITE_ENABLED_RAW()
12177	\| VBOXVMM_INSTR_VMX_VMXOFF_ENABLED_RAW()
12178	\| VBOXVMM_INSTR_VMX_VMXON_ENABLED_RAW()
12179	\| VBOXVMM_INSTR_VMX_VMFUNC_ENABLED_RAW()
12180	\| VBOXVMM_INSTR_VMX_INVEPT_ENABLED_RAW()
12181	\| VBOXVMM_INSTR_VMX_INVVPID_ENABLED_RAW()
12182	\| VBOXVMM_INSTR_VMX_INVPCID_ENABLED_RAW()
12183	) != 0
12184	\|\| ( VBOXVMM_EXIT_TASK_SWITCH_ENABLED_RAW()
12185	\| VBOXVMM_EXIT_HALT_ENABLED_RAW()
12186	\| VBOXVMM_EXIT_MWAIT_ENABLED_RAW()
12187	\| VBOXVMM_EXIT_MONITOR_ENABLED_RAW()
12188	\| VBOXVMM_EXIT_CPUID_ENABLED_RAW()
12189	\| VBOXVMM_EXIT_INVD_ENABLED_RAW()
12190	\| VBOXVMM_EXIT_WBINVD_ENABLED_RAW()
12191	\| VBOXVMM_EXIT_INVLPG_ENABLED_RAW()
12192	\| VBOXVMM_EXIT_RDTSC_ENABLED_RAW()
12193	\| VBOXVMM_EXIT_RDTSCP_ENABLED_RAW()
12194	\| VBOXVMM_EXIT_RDPMC_ENABLED_RAW()
12195	\| VBOXVMM_EXIT_RDMSR_ENABLED_RAW()
12196	\| VBOXVMM_EXIT_WRMSR_ENABLED_RAW()
12197	\| VBOXVMM_EXIT_CRX_READ_ENABLED_RAW()
12198	\| VBOXVMM_EXIT_CRX_WRITE_ENABLED_RAW()
12199	\| VBOXVMM_EXIT_DRX_READ_ENABLED_RAW()
12200	\| VBOXVMM_EXIT_DRX_WRITE_ENABLED_RAW()
12201	\| VBOXVMM_EXIT_PAUSE_ENABLED_RAW()
12202	\| VBOXVMM_EXIT_XSETBV_ENABLED_RAW()
12203	\| VBOXVMM_EXIT_SIDT_ENABLED_RAW()
12204	\| VBOXVMM_EXIT_LIDT_ENABLED_RAW()
12205	\| VBOXVMM_EXIT_SGDT_ENABLED_RAW()
12206	\| VBOXVMM_EXIT_LGDT_ENABLED_RAW()
12207	\| VBOXVMM_EXIT_SLDT_ENABLED_RAW()
12208	\| VBOXVMM_EXIT_LLDT_ENABLED_RAW()
12209	\| VBOXVMM_EXIT_STR_ENABLED_RAW()
12210	\| VBOXVMM_EXIT_LTR_ENABLED_RAW()
12211	\| VBOXVMM_EXIT_GETSEC_ENABLED_RAW()
12212	\| VBOXVMM_EXIT_RSM_ENABLED_RAW()
12213	\| VBOXVMM_EXIT_RDRAND_ENABLED_RAW()
12214	\| VBOXVMM_EXIT_RDSEED_ENABLED_RAW()
12215	\| VBOXVMM_EXIT_XSAVES_ENABLED_RAW()
12216	\| VBOXVMM_EXIT_XRSTORS_ENABLED_RAW()
12217	\| VBOXVMM_EXIT_VMM_CALL_ENABLED_RAW()
12218	\| VBOXVMM_EXIT_VMX_VMCLEAR_ENABLED_RAW()
12219	\| VBOXVMM_EXIT_VMX_VMLAUNCH_ENABLED_RAW()
12220	\| VBOXVMM_EXIT_VMX_VMPTRLD_ENABLED_RAW()
12221	\| VBOXVMM_EXIT_VMX_VMPTRST_ENABLED_RAW()
12222	\| VBOXVMM_EXIT_VMX_VMREAD_ENABLED_RAW()
12223	\| VBOXVMM_EXIT_VMX_VMRESUME_ENABLED_RAW()
12224	\| VBOXVMM_EXIT_VMX_VMWRITE_ENABLED_RAW()
12225	\| VBOXVMM_EXIT_VMX_VMXOFF_ENABLED_RAW()
12226	\| VBOXVMM_EXIT_VMX_VMXON_ENABLED_RAW()
12227	\| VBOXVMM_EXIT_VMX_VMFUNC_ENABLED_RAW()
12228	\| VBOXVMM_EXIT_VMX_INVEPT_ENABLED_RAW()
12229	\| VBOXVMM_EXIT_VMX_INVVPID_ENABLED_RAW()
12230	\| VBOXVMM_EXIT_VMX_INVPCID_ENABLED_RAW()
12231	\| VBOXVMM_EXIT_VMX_EPT_VIOLATION_ENABLED_RAW()
12232	\| VBOXVMM_EXIT_VMX_EPT_MISCONFIG_ENABLED_RAW()
12233	\| VBOXVMM_EXIT_VMX_VAPIC_ACCESS_ENABLED_RAW()
12234	\| VBOXVMM_EXIT_VMX_VAPIC_WRITE_ENABLED_RAW()
12235	) != 0;
12236	}
12237
12238
12239	/**
12240	* Runs the guest using hardware-assisted VMX.
12241	*
12242	* @returns Strict VBox status code (i.e. informational status codes too).
12243	* @param pVCpu The cross context virtual CPU structure.
12244	*/
12245	VMMR0DECL(VBOXSTRICTRC) VMXR0RunGuestCode(PVMCPU pVCpu)
12246	{
12247	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
12248	Assert(VMMRZCallRing3IsEnabled(pVCpu));
12249	Assert(!ASMAtomicUoReadU64(&pCtx->fExtrn));
12250	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
12251
12252	VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pCtx);
12253
12254	VBOXSTRICTRC rcStrict;
12255	uint32_t cLoops = 0;
12256	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
12257	bool const fInNestedGuestMode = CPUMIsGuestInVmxNonRootMode(pCtx);
12258	#else
12259	bool const fInNestedGuestMode = false;
12260	#endif
12261	if (!fInNestedGuestMode)
12262	{
12263	if ( !pVCpu->hm.s.fUseDebugLoop
12264	&& (!VBOXVMM_ANY_PROBES_ENABLED() \|\| !hmR0VmxAnyExpensiveProbesEnabled())
12265	&& !DBGFIsStepping(pVCpu)
12266	&& !pVCpu->CTX_SUFF(pVM)->dbgf.ro.cEnabledInt3Breakpoints)
12267	rcStrict = hmR0VmxRunGuestCodeNormal(pVCpu, &cLoops);
12268	else
12269	rcStrict = hmR0VmxRunGuestCodeDebug(pVCpu, &cLoops);
12270	}
12271	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
12272	else
12273	rcStrict = VINF_VMX_VMLAUNCH_VMRESUME;
12274
12275	if (rcStrict == VINF_VMX_VMLAUNCH_VMRESUME)
12276	rcStrict = hmR0VmxRunGuestCodeNested(pVCpu, &cLoops);
12277	#endif
12278
12279	if (rcStrict == VERR_EM_INTERPRETER)
12280	rcStrict = VINF_EM_RAW_EMULATE_INSTR;
12281	else if (rcStrict == VINF_EM_RESET)
12282	rcStrict = VINF_EM_TRIPLE_FAULT;
12283
12284	int rc2 = hmR0VmxExitToRing3(pVCpu, rcStrict);
12285	if (RT_FAILURE(rc2))
12286	{
12287	pVCpu->hm.s.u32HMError = (uint32_t)VBOXSTRICTRC_VAL(rcStrict);
12288	rcStrict = rc2;
12289	}
12290	Assert(!ASMAtomicUoReadU64(&pCtx->fExtrn));
12291	Assert(!VMMRZCallRing3IsNotificationSet(pVCpu));
12292	return rcStrict;
12293	}
12294
12295
12296	#ifndef HMVMX_USE_FUNCTION_TABLE
12297	/**
12298	* Handles a guest VM-exit from hardware-assisted VMX execution.
12299	*
12300	* @returns Strict VBox status code (i.e. informational status codes too).
12301	* @param pVCpu The cross context virtual CPU structure.
12302	* @param pVmxTransient The VMX-transient structure.
12303	*/
12304	DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12305	{
12306	#ifdef DEBUG_ramshankar
12307	#define VMEXIT_CALL_RET(a_fSave, a_CallExpr) \
12308	do { \
12309	if (a_fSave != 0) \
12310	hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL); \
12311	VBOXSTRICTRC rcStrict = a_CallExpr; \
12312	if (a_fSave != 0) \
12313	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST); \
12314	return rcStrict; \
12315	} while (0)
12316	#else
12317	# define VMEXIT_CALL_RET(a_fSave, a_CallExpr) return a_CallExpr
12318	#endif
12319	uint32_t const rcReason = pVmxTransient->uExitReason;
12320	switch (rcReason)
12321	{
12322	case VMX_EXIT_EPT_MISCONFIG: VMEXIT_CALL_RET(0, hmR0VmxExitEptMisconfig(pVCpu, pVmxTransient));
12323	case VMX_EXIT_EPT_VIOLATION: VMEXIT_CALL_RET(0, hmR0VmxExitEptViolation(pVCpu, pVmxTransient));
12324	case VMX_EXIT_IO_INSTR: VMEXIT_CALL_RET(0, hmR0VmxExitIoInstr(pVCpu, pVmxTransient));
12325	case VMX_EXIT_CPUID: VMEXIT_CALL_RET(0, hmR0VmxExitCpuid(pVCpu, pVmxTransient));
12326	case VMX_EXIT_RDTSC: VMEXIT_CALL_RET(0, hmR0VmxExitRdtsc(pVCpu, pVmxTransient));
12327	case VMX_EXIT_RDTSCP: VMEXIT_CALL_RET(0, hmR0VmxExitRdtscp(pVCpu, pVmxTransient));
12328	case VMX_EXIT_APIC_ACCESS: VMEXIT_CALL_RET(0, hmR0VmxExitApicAccess(pVCpu, pVmxTransient));
12329	case VMX_EXIT_XCPT_OR_NMI: VMEXIT_CALL_RET(0, hmR0VmxExitXcptOrNmi(pVCpu, pVmxTransient));
12330	case VMX_EXIT_MOV_CRX: VMEXIT_CALL_RET(0, hmR0VmxExitMovCRx(pVCpu, pVmxTransient));
12331	case VMX_EXIT_EXT_INT: VMEXIT_CALL_RET(0, hmR0VmxExitExtInt(pVCpu, pVmxTransient));
12332	case VMX_EXIT_INT_WINDOW: VMEXIT_CALL_RET(0, hmR0VmxExitIntWindow(pVCpu, pVmxTransient));
12333	case VMX_EXIT_TPR_BELOW_THRESHOLD: VMEXIT_CALL_RET(0, hmR0VmxExitTprBelowThreshold(pVCpu, pVmxTransient));
12334	case VMX_EXIT_MWAIT: VMEXIT_CALL_RET(0, hmR0VmxExitMwait(pVCpu, pVmxTransient));
12335	case VMX_EXIT_MONITOR: VMEXIT_CALL_RET(0, hmR0VmxExitMonitor(pVCpu, pVmxTransient));
12336	case VMX_EXIT_TASK_SWITCH: VMEXIT_CALL_RET(0, hmR0VmxExitTaskSwitch(pVCpu, pVmxTransient));
12337	case VMX_EXIT_PREEMPT_TIMER: VMEXIT_CALL_RET(0, hmR0VmxExitPreemptTimer(pVCpu, pVmxTransient));
12338	case VMX_EXIT_RDMSR: VMEXIT_CALL_RET(0, hmR0VmxExitRdmsr(pVCpu, pVmxTransient));
12339	case VMX_EXIT_WRMSR: VMEXIT_CALL_RET(0, hmR0VmxExitWrmsr(pVCpu, pVmxTransient));
12340	case VMX_EXIT_VMCALL: VMEXIT_CALL_RET(0, hmR0VmxExitVmcall(pVCpu, pVmxTransient));
12341	case VMX_EXIT_MOV_DRX: VMEXIT_CALL_RET(0, hmR0VmxExitMovDRx(pVCpu, pVmxTransient));
12342	case VMX_EXIT_HLT: VMEXIT_CALL_RET(0, hmR0VmxExitHlt(pVCpu, pVmxTransient));
12343	case VMX_EXIT_INVD: VMEXIT_CALL_RET(0, hmR0VmxExitInvd(pVCpu, pVmxTransient));
12344	case VMX_EXIT_INVLPG: VMEXIT_CALL_RET(0, hmR0VmxExitInvlpg(pVCpu, pVmxTransient));
12345	case VMX_EXIT_RSM: VMEXIT_CALL_RET(0, hmR0VmxExitRsm(pVCpu, pVmxTransient));
12346	case VMX_EXIT_MTF: VMEXIT_CALL_RET(0, hmR0VmxExitMtf(pVCpu, pVmxTransient));
12347	case VMX_EXIT_PAUSE: VMEXIT_CALL_RET(0, hmR0VmxExitPause(pVCpu, pVmxTransient));
12348	case VMX_EXIT_GDTR_IDTR_ACCESS: VMEXIT_CALL_RET(0, hmR0VmxExitXdtrAccess(pVCpu, pVmxTransient));
12349	case VMX_EXIT_LDTR_TR_ACCESS: VMEXIT_CALL_RET(0, hmR0VmxExitXdtrAccess(pVCpu, pVmxTransient));
12350	case VMX_EXIT_WBINVD: VMEXIT_CALL_RET(0, hmR0VmxExitWbinvd(pVCpu, pVmxTransient));
12351	case VMX_EXIT_XSETBV: VMEXIT_CALL_RET(0, hmR0VmxExitXsetbv(pVCpu, pVmxTransient));
12352	case VMX_EXIT_RDRAND: VMEXIT_CALL_RET(0, hmR0VmxExitRdrand(pVCpu, pVmxTransient));
12353	case VMX_EXIT_INVPCID: VMEXIT_CALL_RET(0, hmR0VmxExitInvpcid(pVCpu, pVmxTransient));
12354	case VMX_EXIT_GETSEC: VMEXIT_CALL_RET(0, hmR0VmxExitGetsec(pVCpu, pVmxTransient));
12355	case VMX_EXIT_RDPMC: VMEXIT_CALL_RET(0, hmR0VmxExitRdpmc(pVCpu, pVmxTransient));
12356	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
12357	case VMX_EXIT_VMCLEAR: VMEXIT_CALL_RET(0, hmR0VmxExitVmclear(pVCpu, pVmxTransient));
12358	case VMX_EXIT_VMLAUNCH: VMEXIT_CALL_RET(0, hmR0VmxExitVmlaunch(pVCpu, pVmxTransient));
12359	case VMX_EXIT_VMPTRLD: VMEXIT_CALL_RET(0, hmR0VmxExitVmptrld(pVCpu, pVmxTransient));
12360	case VMX_EXIT_VMPTRST: VMEXIT_CALL_RET(0, hmR0VmxExitVmptrst(pVCpu, pVmxTransient));
12361	case VMX_EXIT_VMREAD: VMEXIT_CALL_RET(0, hmR0VmxExitVmread(pVCpu, pVmxTransient));
12362	case VMX_EXIT_VMRESUME: VMEXIT_CALL_RET(0, hmR0VmxExitVmwrite(pVCpu, pVmxTransient));
12363	case VMX_EXIT_VMWRITE: VMEXIT_CALL_RET(0, hmR0VmxExitVmresume(pVCpu, pVmxTransient));
12364	case VMX_EXIT_VMXOFF: VMEXIT_CALL_RET(0, hmR0VmxExitVmxoff(pVCpu, pVmxTransient));
12365	case VMX_EXIT_VMXON: VMEXIT_CALL_RET(0, hmR0VmxExitVmxon(pVCpu, pVmxTransient));
12366	#else
12367	case VMX_EXIT_VMCLEAR:
12368	case VMX_EXIT_VMLAUNCH:
12369	case VMX_EXIT_VMPTRLD:
12370	case VMX_EXIT_VMPTRST:
12371	case VMX_EXIT_VMREAD:
12372	case VMX_EXIT_VMRESUME:
12373	case VMX_EXIT_VMWRITE:
12374	case VMX_EXIT_VMXOFF:
12375	case VMX_EXIT_VMXON:
12376	return hmR0VmxExitSetPendingXcptUD(pVCpu, pVmxTransient);
12377	#endif
12378
12379	case VMX_EXIT_TRIPLE_FAULT: return hmR0VmxExitTripleFault(pVCpu, pVmxTransient);
12380	case VMX_EXIT_NMI_WINDOW: return hmR0VmxExitNmiWindow(pVCpu, pVmxTransient);
12381	case VMX_EXIT_INIT_SIGNAL: return hmR0VmxExitInitSignal(pVCpu, pVmxTransient);
12382	case VMX_EXIT_SIPI: return hmR0VmxExitSipi(pVCpu, pVmxTransient);
12383	case VMX_EXIT_IO_SMI: return hmR0VmxExitIoSmi(pVCpu, pVmxTransient);
12384	case VMX_EXIT_SMI: return hmR0VmxExitSmi(pVCpu, pVmxTransient);
12385	case VMX_EXIT_ERR_MSR_LOAD: return hmR0VmxExitErrMsrLoad(pVCpu, pVmxTransient);
12386	case VMX_EXIT_ERR_INVALID_GUEST_STATE: return hmR0VmxExitErrInvalidGuestState(pVCpu, pVmxTransient);
12387	case VMX_EXIT_ERR_MACHINE_CHECK: return hmR0VmxExitErrMachineCheck(pVCpu, pVmxTransient);
12388
12389	case VMX_EXIT_INVEPT:
12390	case VMX_EXIT_INVVPID:
12391	case VMX_EXIT_VMFUNC:
12392	case VMX_EXIT_XSAVES:
12393	case VMX_EXIT_XRSTORS:
12394	return hmR0VmxExitSetPendingXcptUD(pVCpu, pVmxTransient);
12395
12396	case VMX_EXIT_ENCLS:
12397	case VMX_EXIT_RDSEED:
12398	case VMX_EXIT_PML_FULL:
12399	default:
12400	return hmR0VmxExitErrUndefined(pVCpu, pVmxTransient);
12401	}
12402	#undef VMEXIT_CALL_RET
12403	}
12404	#endif /* !HMVMX_USE_FUNCTION_TABLE */
12405
12406
12407	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
12408	/**
12409	* Handles a nested-guest VM-exit from hardware-assisted VMX execution.
12410	*
12411	* @returns Strict VBox status code (i.e. informational status codes too).
12412	* @param pVCpu The cross context virtual CPU structure.
12413	* @param pVmxTransient The VMX-transient structure.
12414	*/
12415	DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExitNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12416	{
12417	uint32_t const rcReason = pVmxTransient->uExitReason;
12418	switch (rcReason)
12419	{
12420	case VMX_EXIT_EPT_MISCONFIG:
12421	case VMX_EXIT_EPT_VIOLATION:
12422	case VMX_EXIT_IO_INSTR:
12423	case VMX_EXIT_CPUID:
12424	case VMX_EXIT_RDTSC:
12425	case VMX_EXIT_RDTSCP:
12426	case VMX_EXIT_APIC_ACCESS:
12427	case VMX_EXIT_XCPT_OR_NMI:
12428	case VMX_EXIT_MOV_CRX:
12429	case VMX_EXIT_EXT_INT:
12430	case VMX_EXIT_INT_WINDOW:
12431	case VMX_EXIT_TPR_BELOW_THRESHOLD:
12432	case VMX_EXIT_MWAIT:
12433	case VMX_EXIT_MONITOR:
12434	case VMX_EXIT_TASK_SWITCH:
12435	case VMX_EXIT_PREEMPT_TIMER:
12436	case VMX_EXIT_RDMSR:
12437	case VMX_EXIT_WRMSR:
12438	case VMX_EXIT_VMCALL:
12439	case VMX_EXIT_MOV_DRX:
12440	case VMX_EXIT_HLT:
12441	case VMX_EXIT_INVD:
12442	case VMX_EXIT_INVLPG:
12443	case VMX_EXIT_RSM:
12444	case VMX_EXIT_MTF:
12445	case VMX_EXIT_PAUSE:
12446	case VMX_EXIT_GDTR_IDTR_ACCESS:
12447	case VMX_EXIT_LDTR_TR_ACCESS:
12448	case VMX_EXIT_WBINVD:
12449	case VMX_EXIT_XSETBV:
12450	case VMX_EXIT_RDRAND:
12451	case VMX_EXIT_INVPCID:
12452	case VMX_EXIT_GETSEC:
12453	case VMX_EXIT_RDPMC:
12454	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
12455	case VMX_EXIT_VMCLEAR:
12456	case VMX_EXIT_VMLAUNCH:
12457	case VMX_EXIT_VMPTRLD:
12458	case VMX_EXIT_VMPTRST:
12459	case VMX_EXIT_VMREAD:
12460	case VMX_EXIT_VMRESUME:
12461	case VMX_EXIT_VMWRITE:
12462	case VMX_EXIT_VMXOFF:
12463	case VMX_EXIT_VMXON:
12464	#endif
12465	case VMX_EXIT_TRIPLE_FAULT:
12466	case VMX_EXIT_NMI_WINDOW:
12467	case VMX_EXIT_INIT_SIGNAL:
12468	case VMX_EXIT_SIPI:
12469	case VMX_EXIT_IO_SMI:
12470	case VMX_EXIT_SMI:
12471	case VMX_EXIT_ERR_MSR_LOAD:
12472	case VMX_EXIT_ERR_INVALID_GUEST_STATE:
12473	case VMX_EXIT_ERR_MACHINE_CHECK:
12474
12475	case VMX_EXIT_INVEPT:
12476	case VMX_EXIT_INVVPID:
12477	case VMX_EXIT_VMFUNC:
12478	case VMX_EXIT_XSAVES:
12479	case VMX_EXIT_XRSTORS:
12480
12481	case VMX_EXIT_ENCLS:
12482	case VMX_EXIT_RDSEED:
12483	case VMX_EXIT_PML_FULL:
12484	default:
12485	return hmR0VmxExitErrUndefined(pVCpu, pVmxTransient);
12486	}
12487	#undef VMEXIT_CALL_RET
12488	}
12489	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
12490
12491
12492	#ifdef VBOX_STRICT
12493	/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
12494	# define HMVMX_ASSERT_PREEMPT_CPUID_VAR() \
12495	RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
12496
12497	# define HMVMX_ASSERT_PREEMPT_CPUID() \
12498	do { \
12499	RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
12500	AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
12501	} while (0)
12502
12503	# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
12504	do { \
12505	AssertPtr((a_pVCpu)); \
12506	AssertPtr((a_pVmxTransient)); \
12507	Assert((a_pVmxTransient)->fVMEntryFailed == false); \
12508	Assert((a_pVmxTransient)->pVmcsInfo); \
12509	Assert(ASMIntAreEnabled()); \
12510	HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu); \
12511	HMVMX_ASSERT_PREEMPT_CPUID_VAR(); \
12512	Log4Func(("vcpu[%RU32] -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v\n", (a_pVCpu)->idCpu)); \
12513	HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu); \
12514	if (VMMR0IsLogFlushDisabled((a_pVCpu))) \
12515	HMVMX_ASSERT_PREEMPT_CPUID(); \
12516	HMVMX_STOP_EXIT_DISPATCH_PROF(); \
12517	} while (0)
12518
12519	# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
12520	do { \
12521	Log4Func(("\n")); \
12522	} while (0)
12523	#else
12524	# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
12525	do { \
12526	HMVMX_STOP_EXIT_DISPATCH_PROF(); \
12527	NOREF((a_pVCpu)); NOREF((a_pVmxTransient)); \
12528	} while (0)
12529	# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) do { } while (0)
12530	#endif
12531
12532
12533	/**
12534	* Advances the guest RIP by the specified number of bytes.
12535	*
12536	* @param pVCpu The cross context virtual CPU structure.
12537	* @param cbInstr Number of bytes to advance the RIP by.
12538	*
12539	* @remarks No-long-jump zone!!!
12540	*/
12541	DECLINLINE(void) hmR0VmxAdvanceGuestRipBy(PVMCPU pVCpu, uint32_t cbInstr)
12542	{
12543	/* Advance the RIP. */
12544	pVCpu->cpum.GstCtx.rip += cbInstr;
12545	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
12546
12547	/* Update interrupt inhibition. */
12548	if ( VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS)
12549	&& pVCpu->cpum.GstCtx.rip != EMGetInhibitInterruptsPC(pVCpu))
12550	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
12551	}
12552
12553
12554	/**
12555	* Advances the guest RIP after reading it from the VMCS.
12556	*
12557	* @returns VBox status code, no informational status codes.
12558	* @param pVCpu The cross context virtual CPU structure.
12559	* @param pVmxTransient The VMX-transient structure.
12560	*
12561	* @remarks No-long-jump zone!!!
12562	*/
12563	static int hmR0VmxAdvanceGuestRip(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12564	{
12565	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12566	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS);
12567	AssertRCReturn(rc, rc);
12568
12569	hmR0VmxAdvanceGuestRipBy(pVCpu, pVmxTransient->cbInstr);
12570	return VINF_SUCCESS;
12571	}
12572
12573
12574	/**
12575	* Handle a condition that occurred while delivering an event through the guest
12576	* IDT.
12577	*
12578	* @returns Strict VBox status code (i.e. informational status codes too).
12579	* @retval VINF_SUCCESS if we should continue handling the VM-exit.
12580	* @retval VINF_HM_DOUBLE_FAULT if a \#DF condition was detected and we ought
12581	* to continue execution of the guest which will delivery the \#DF.
12582	* @retval VINF_EM_RESET if we detected a triple-fault condition.
12583	* @retval VERR_EM_GUEST_CPU_HANG if we detected a guest CPU hang.
12584	*
12585	* @param pVCpu The cross context virtual CPU structure.
12586	* @param pVmxTransient The VMX-transient structure.
12587	*
12588	* @remarks No-long-jump zone!!!
12589	*/
12590	static VBOXSTRICTRC hmR0VmxCheckExitDueToEventDelivery(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12591	{
12592	uint32_t const uExitVector = VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo);
12593
12594	int rc2 = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
12595	rc2 \|= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
12596	AssertRCReturn(rc2, rc2);
12597
12598	VBOXSTRICTRC rcStrict = VINF_SUCCESS;
12599	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
12600	if (VMX_IDT_VECTORING_INFO_IS_VALID(pVmxTransient->uIdtVectoringInfo))
12601	{
12602	uint32_t const uIdtVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
12603	uint32_t const uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
12604
12605	/*
12606	* If the event was a software interrupt (generated with INT n) or a software exception
12607	* (generated by INT3/INTO) or a privileged software exception (generated by INT1), we
12608	* can handle the VM-exit and continue guest execution which will re-execute the
12609	* instruction rather than re-injecting the exception, as that can cause premature
12610	* trips to ring-3 before injection and involve TRPM which currently has no way of
12611	* storing that these exceptions were caused by these instructions (ICEBP's #DB poses
12612	* the problem).
12613	*/
12614	IEMXCPTRAISE enmRaise;
12615	IEMXCPTRAISEINFO fRaiseInfo;
12616	if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
12617	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
12618	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
12619	{
12620	enmRaise = IEMXCPTRAISE_REEXEC_INSTR;
12621	fRaiseInfo = IEMXCPTRAISEINFO_NONE;
12622	}
12623	else if (VMX_EXIT_INT_INFO_IS_VALID(pVmxTransient->uExitIntInfo))
12624	{
12625	uint32_t const uExitVectorType = VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo);
12626	uint32_t const fIdtVectorFlags = hmR0VmxGetIemXcptFlags(uIdtVector, uIdtVectorType);
12627	uint32_t const fExitVectorFlags = hmR0VmxGetIemXcptFlags(uExitVector, uExitVectorType);
12628	/** @todo Make AssertMsgReturn as just AssertMsg later. */
12629	AssertMsgReturn(uExitVectorType == VMX_EXIT_INT_INFO_TYPE_HW_XCPT,
12630	("Unexpected VM-exit interruption vector type %#x!\n", uExitVectorType), VERR_VMX_IPE_5);
12631
12632	enmRaise = IEMEvaluateRecursiveXcpt(pVCpu, fIdtVectorFlags, uIdtVector, fExitVectorFlags, uExitVector, &fRaiseInfo);
12633
12634	/* Determine a vectoring #PF condition, see comment in hmR0VmxExitXcptPF(). */
12635	if (fRaiseInfo & (IEMXCPTRAISEINFO_EXT_INT_PF \| IEMXCPTRAISEINFO_NMI_PF))
12636	{
12637	pVmxTransient->fVectoringPF = true;
12638	enmRaise = IEMXCPTRAISE_PREV_EVENT;
12639	}
12640	}
12641	else
12642	{
12643	/*
12644	* If an exception or hardware interrupt delivery caused an EPT violation/misconfig or APIC access
12645	* VM-exit, then the VM-exit interruption-information will not be valid and we end up here.
12646	* It is sufficient to reflect the original event to the guest after handling the VM-exit.
12647	*/
12648	Assert( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
12649	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
12650	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT);
12651	enmRaise = IEMXCPTRAISE_PREV_EVENT;
12652	fRaiseInfo = IEMXCPTRAISEINFO_NONE;
12653	}
12654
12655	/*
12656	* On CPUs that support Virtual NMIs, if this VM-exit (be it an exception or EPT violation/misconfig
12657	* etc.) occurred while delivering the NMI, we need to clear the block-by-NMI field in the guest
12658	* interruptibility-state before re-delivering the NMI after handling the VM-exit. Otherwise the
12659	* subsequent VM-entry would fail.
12660	*
12661	* See Intel spec. 30.7.1.2 "Resuming Guest Software after Handling an Exception". See @bugref{7445}.
12662	*/
12663	if ( VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_BLOCK_NMIS)
12664	&& uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
12665	&& ( enmRaise == IEMXCPTRAISE_PREV_EVENT
12666	\|\| (fRaiseInfo & IEMXCPTRAISEINFO_NMI_PF))
12667	&& (pVmcsInfo->u32PinCtls & VMX_PIN_CTLS_VIRT_NMI))
12668	{
12669	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
12670	}
12671
12672	switch (enmRaise)
12673	{
12674	case IEMXCPTRAISE_CURRENT_XCPT:
12675	{
12676	Log4Func(("IDT: Pending secondary Xcpt: uIdtVectoringInfo=%#RX64 uExitIntInfo=%#RX64\n",
12677	pVmxTransient->uIdtVectoringInfo, pVmxTransient->uExitIntInfo));
12678	Assert(rcStrict == VINF_SUCCESS);
12679	break;
12680	}
12681
12682	case IEMXCPTRAISE_PREV_EVENT:
12683	{
12684	uint32_t u32ErrCode;
12685	if (VMX_IDT_VECTORING_INFO_IS_ERROR_CODE_VALID(pVmxTransient->uIdtVectoringInfo))
12686	{
12687	rc2 = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
12688	AssertRCReturn(rc2, rc2);
12689	u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
12690	}
12691	else
12692	u32ErrCode = 0;
12693
12694	/* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF, see hmR0VmxExitXcptPF(). */
12695	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
12696	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
12697	0 /* cbInstr */, u32ErrCode, pVCpu->cpum.GstCtx.cr2);
12698
12699	Log4Func(("IDT: Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->hm.s.Event.u64IntInfo,
12700	pVCpu->hm.s.Event.u32ErrCode));
12701	Assert(rcStrict == VINF_SUCCESS);
12702	break;
12703	}
12704
12705	case IEMXCPTRAISE_REEXEC_INSTR:
12706	Assert(rcStrict == VINF_SUCCESS);
12707	break;
12708
12709	case IEMXCPTRAISE_DOUBLE_FAULT:
12710	{
12711	/*
12712	* Determing a vectoring double #PF condition. Used later, when PGM evaluates the
12713	* second #PF as a guest #PF (and not a shadow #PF) and needs to be converted into a #DF.
12714	*/
12715	if (fRaiseInfo & IEMXCPTRAISEINFO_PF_PF)
12716	{
12717	pVmxTransient->fVectoringDoublePF = true;
12718	Log4Func(("IDT: Vectoring double #PF %#RX64 cr2=%#RX64\n", pVCpu->hm.s.Event.u64IntInfo,
12719	pVCpu->cpum.GstCtx.cr2));
12720	rcStrict = VINF_SUCCESS;
12721	}
12722	else
12723	{
12724	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
12725	hmR0VmxSetPendingXcptDF(pVCpu);
12726	Log4Func(("IDT: Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->hm.s.Event.u64IntInfo,
12727	uIdtVector, uExitVector));
12728	rcStrict = VINF_HM_DOUBLE_FAULT;
12729	}
12730	break;
12731	}
12732
12733	case IEMXCPTRAISE_TRIPLE_FAULT:
12734	{
12735	Log4Func(("IDT: Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", uIdtVector, uExitVector));
12736	rcStrict = VINF_EM_RESET;
12737	break;
12738	}
12739
12740	case IEMXCPTRAISE_CPU_HANG:
12741	{
12742	Log4Func(("IDT: Bad guest! Entering CPU hang. fRaiseInfo=%#x\n", fRaiseInfo));
12743	rcStrict = VERR_EM_GUEST_CPU_HANG;
12744	break;
12745	}
12746
12747	default:
12748	{
12749	AssertMsgFailed(("IDT: vcpu[%RU32] Unexpected/invalid value! enmRaise=%#x\n", pVCpu->idCpu, enmRaise));
12750	rcStrict = VERR_VMX_IPE_2;
12751	break;
12752	}
12753	}
12754	}
12755	else if ( VMX_EXIT_INT_INFO_IS_VALID(pVmxTransient->uExitIntInfo)
12756	&& VMX_EXIT_INT_INFO_IS_NMI_UNBLOCK_IRET(pVmxTransient->uExitIntInfo)
12757	&& uExitVector != X86_XCPT_DF
12758	&& (pVmcsInfo->u32PinCtls & VMX_PIN_CTLS_VIRT_NMI))
12759	{
12760	/*
12761	* Execution of IRET caused this fault when NMI blocking was in effect (i.e we're in the guest NMI handler).
12762	* We need to set the block-by-NMI field so that NMIs remain blocked until the IRET execution is restarted.
12763	* See Intel spec. 30.7.1.2 "Resuming guest software after handling an exception".
12764	*/
12765	if (!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_BLOCK_NMIS))
12766	{
12767	Log4Func(("Setting VMCPU_FF_BLOCK_NMIS. fValid=%RTbool uExitReason=%u\n",
12768	VMX_EXIT_INT_INFO_IS_VALID(pVmxTransient->uExitIntInfo), pVmxTransient->uExitReason));
12769	VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
12770	}
12771	}
12772
12773	Assert( rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_HM_DOUBLE_FAULT
12774	\|\| rcStrict == VINF_EM_RESET \|\| rcStrict == VERR_EM_GUEST_CPU_HANG);
12775	return rcStrict;
12776	}
12777
12778
12779	/** @name VM-exit handlers.
12780	* @{
12781	*/
12782	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
12783	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
12784	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
12785
12786	/**
12787	* VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
12788	*/
12789	HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12790	{
12791	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12792	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
12793	/* Windows hosts (32-bit and 64-bit) have DPC latency issues. See @bugref{6853}. */
12794	if (VMMR0ThreadCtxHookIsEnabled(pVCpu))
12795	return VINF_SUCCESS;
12796	return VINF_EM_RAW_INTERRUPT;
12797	}
12798
12799
12800	/**
12801	* VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI).
12802	*/
12803	HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmi(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12804	{
12805	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12806	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
12807
12808	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
12809	int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
12810	AssertRCReturn(rc, rc);
12811
12812	uint32_t uIntType = VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo);
12813	Assert( !(pVmcsInfo->u32ExitCtls & VMX_EXIT_CTLS_ACK_EXT_INT)
12814	&& uIntType != VMX_EXIT_INT_INFO_TYPE_EXT_INT);
12815	Assert(VMX_EXIT_INT_INFO_IS_VALID(pVmxTransient->uExitIntInfo));
12816
12817	if (uIntType == VMX_EXIT_INT_INFO_TYPE_NMI)
12818	{
12819	/*
12820	* This cannot be a guest NMI as the only way for the guest to receive an NMI is if we
12821	* injected it ourselves and anything we inject is not going to cause a VM-exit directly
12822	* for the event being injected[1]. Go ahead and dispatch the NMI to the host[2].
12823	*
12824	* [1] -- See Intel spec. 27.2.3 "Information for VM Exits During Event Delivery".
12825	* [2] -- See Intel spec. 27.5.5 "Updating Non-Register State".
12826	*/
12827	VMXDispatchHostNmi();
12828	STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGC);
12829	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
12830	return VINF_SUCCESS;
12831	}
12832
12833	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
12834	VBOXSTRICTRC rcStrictRc1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
12835	if (RT_UNLIKELY(rcStrictRc1 == VINF_SUCCESS))
12836	{ /* likely */ }
12837	else
12838	{
12839	if (rcStrictRc1 == VINF_HM_DOUBLE_FAULT)
12840	rcStrictRc1 = VINF_SUCCESS;
12841	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
12842	return rcStrictRc1;
12843	}
12844
12845	uint32_t uExitIntInfo = pVmxTransient->uExitIntInfo;
12846	uint32_t uVector = VMX_EXIT_INT_INFO_VECTOR(uExitIntInfo);
12847	switch (uIntType)
12848	{
12849	case VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT: /* Privileged software exception. (#DB from ICEBP) */
12850	Assert(uVector == X86_XCPT_DB);
12851	RT_FALL_THRU();
12852	case VMX_EXIT_INT_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
12853	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF \|\| uIntType == VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT);
12854	RT_FALL_THRU();
12855	case VMX_EXIT_INT_INFO_TYPE_HW_XCPT:
12856	{
12857	/*
12858	* If there's any exception caused as a result of event injection, the resulting
12859	* secondary/final execption will be pending, we shall continue guest execution
12860	* after injecting the event. The page-fault case is complicated and we manually
12861	* handle any currently pending event in hmR0VmxExitXcptPF.
12862	*/
12863	if (!pVCpu->hm.s.Event.fPending)
12864	{ /* likely */ }
12865	else if (uVector != X86_XCPT_PF)
12866	{
12867	rc = VINF_SUCCESS;
12868	break;
12869	}
12870
12871	switch (uVector)
12872	{
12873	case X86_XCPT_PF: rc = hmR0VmxExitXcptPF(pVCpu, pVmxTransient); break;
12874	case X86_XCPT_GP: rc = hmR0VmxExitXcptGP(pVCpu, pVmxTransient); break;
12875	case X86_XCPT_MF: rc = hmR0VmxExitXcptMF(pVCpu, pVmxTransient); break;
12876	case X86_XCPT_DB: rc = hmR0VmxExitXcptDB(pVCpu, pVmxTransient); break;
12877	case X86_XCPT_BP: rc = hmR0VmxExitXcptBP(pVCpu, pVmxTransient); break;
12878	case X86_XCPT_AC: rc = hmR0VmxExitXcptAC(pVCpu, pVmxTransient); break;
12879
12880	case X86_XCPT_NM: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
12881	rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
12882	case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF);
12883	rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
12884	case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
12885	rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
12886	case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
12887	rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
12888	case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
12889	rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
12890	case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
12891	rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
12892	case X86_XCPT_TS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestTS);
12893	rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
12894	default:
12895	{
12896	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
12897	if (pVmcsInfo->RealMode.fRealOnV86Active)
12898	{
12899	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
12900	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
12901	Assert(CPUMIsGuestInRealModeEx(&pVCpu->cpum.GstCtx));
12902
12903	rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_CR0);
12904	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12905	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
12906	AssertRCReturn(rc, rc);
12907	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(uExitIntInfo),
12908	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode,
12909	0 /* GCPtrFaultAddress */);
12910	}
12911	else
12912	{
12913	AssertMsgFailed(("Unexpected VM-exit caused by exception %#x\n", uVector));
12914	pVCpu->hm.s.u32HMError = uVector;
12915	rc = VERR_VMX_UNEXPECTED_EXCEPTION;
12916	}
12917	break;
12918	}
12919	}
12920	break;
12921	}
12922
12923	default:
12924	{
12925	pVCpu->hm.s.u32HMError = uExitIntInfo;
12926	rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE;
12927	AssertMsgFailed(("Unexpected interruption info %#x\n", VMX_EXIT_INT_INFO_TYPE(uExitIntInfo)));
12928	break;
12929	}
12930	}
12931	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
12932	return rc;
12933	}
12934
12935
12936	/**
12937	* VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
12938	*/
12939	HMVMX_EXIT_NSRC_DECL hmR0VmxExitIntWindow(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12940	{
12941	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12942
12943	/* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
12944	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
12945	int rc = hmR0VmxClearIntWindowExitVmcs(pVmcsInfo);
12946	AssertRCReturn(rc, rc);
12947
12948	/* Evaluate and deliver pending events and resume guest execution. */
12949	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
12950	return VINF_SUCCESS;
12951	}
12952
12953
12954	/**
12955	* VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
12956	*/
12957	HMVMX_EXIT_NSRC_DECL hmR0VmxExitNmiWindow(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12958	{
12959	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12960
12961	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
12962	if (RT_UNLIKELY(!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT))) /** @todo NSTVMX: Turn this into an assertion. */
12963	{
12964	AssertMsgFailed(("Unexpected NMI-window exit.\n"));
12965	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
12966	}
12967
12968	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_BLOCK_NMIS));
12969
12970	/*
12971	* If block-by-STI is set when we get this VM-exit, it means the CPU doesn't block NMIs following STI.
12972	* It is therefore safe to unblock STI and deliver the NMI ourselves. See @bugref{7445}.
12973	*/
12974	uint32_t fIntrState;
12975	int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &fIntrState);
12976	AssertRCReturn(rc, rc);
12977	Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS));
12978	if (fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
12979	{
12980	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
12981	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
12982
12983	fIntrState &= ~VMX_VMCS_GUEST_INT_STATE_BLOCK_STI;
12984	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INT_STATE, fIntrState);
12985	AssertRCReturn(rc, rc);
12986	}
12987
12988	/* Indicate that we no longer need to VM-exit when the guest is ready to receive NMIs, it is now ready */
12989	rc = hmR0VmxClearNmiWindowExitVmcs(pVmcsInfo);
12990	AssertRCReturn(rc, rc);
12991
12992	/* Evaluate and deliver pending events and resume guest execution. */
12993	return VINF_SUCCESS;
12994	}
12995
12996
12997	/**
12998	* VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
12999	*/
13000	HMVMX_EXIT_NSRC_DECL hmR0VmxExitWbinvd(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13001	{
13002	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13003	return hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
13004	}
13005
13006
13007	/**
13008	* VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
13009	*/
13010	HMVMX_EXIT_NSRC_DECL hmR0VmxExitInvd(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13011	{
13012	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13013	return hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
13014	}
13015
13016
13017	/**
13018	* VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
13019	*/
13020	HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13021	{
13022	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13023
13024	/*
13025	* Get the state we need and update the exit history entry.
13026	*/
13027	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13028	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13029	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
13030	AssertRCReturn(rc, rc);
13031
13032	VBOXSTRICTRC rcStrict;
13033	PCEMEXITREC pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
13034	EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_CPUID),
13035	pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
13036	if (!pExitRec)
13037	{
13038	/*
13039	* Regular CPUID instruction execution.
13040	*/
13041	rcStrict = IEMExecDecodedCpuid(pVCpu, pVmxTransient->cbInstr);
13042	if (rcStrict == VINF_SUCCESS)
13043	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
13044	else if (rcStrict == VINF_IEM_RAISED_XCPT)
13045	{
13046	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13047	rcStrict = VINF_SUCCESS;
13048	}
13049	}
13050	else
13051	{
13052	/*
13053	* Frequent exit or something needing probing. Get state and call EMHistoryExec.
13054	*/
13055	int rc2 = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
13056	AssertRCReturn(rc2, rc2);
13057
13058	Log4(("CpuIdExit/%u: %04x:%08RX64: %#x/%#x -> EMHistoryExec\n",
13059	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.eax, pVCpu->cpum.GstCtx.ecx));
13060
13061	rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
13062	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
13063
13064	Log4(("CpuIdExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
13065	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
13066	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
13067	}
13068	return rcStrict;
13069	}
13070
13071
13072	/**
13073	* VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
13074	*/
13075	HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13076	{
13077	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13078
13079	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13080	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_CR4);
13081	AssertRCReturn(rc, rc);
13082
13083	if (pVCpu->cpum.GstCtx.cr4 & X86_CR4_SMXE)
13084	return VINF_EM_RAW_EMULATE_INSTR;
13085
13086	AssertMsgFailed(("hmR0VmxExitGetsec: unexpected VM-exit when CR4.SMXE is 0.\n"));
13087	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
13088	}
13089
13090
13091	/**
13092	* VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
13093	*/
13094	HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13095	{
13096	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13097
13098	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13099	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
13100	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13101	AssertRCReturn(rc, rc);
13102
13103	VBOXSTRICTRC rcStrict = IEMExecDecodedRdtsc(pVCpu, pVmxTransient->cbInstr);
13104	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
13105	{
13106	/* If we get a spurious VM-exit when TSC offsetting is enabled,
13107	we must reset offsetting on VM-entry. See @bugref{6634}. */
13108	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TSC_OFFSETTING)
13109	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
13110	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
13111	}
13112	else if (rcStrict == VINF_IEM_RAISED_XCPT)
13113	{
13114	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13115	rcStrict = VINF_SUCCESS;
13116	}
13117	return rcStrict;
13118	}
13119
13120
13121	/**
13122	* VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
13123	*/
13124	HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13125	{
13126	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13127
13128	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13129	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK \| CPUMCTX_EXTRN_TSC_AUX);
13130	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13131	AssertRCReturn(rc, rc);
13132
13133	VBOXSTRICTRC rcStrict = IEMExecDecodedRdtscp(pVCpu, pVmxTransient->cbInstr);
13134	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
13135	{
13136	/* If we get a spurious VM-exit when TSC offsetting is enabled,
13137	we must reset offsetting on VM-reentry. See @bugref{6634}. */
13138	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TSC_OFFSETTING)
13139	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
13140	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
13141	}
13142	else if (rcStrict == VINF_IEM_RAISED_XCPT)
13143	{
13144	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13145	rcStrict = VINF_SUCCESS;
13146	}
13147	return rcStrict;
13148	}
13149
13150
13151	/**
13152	* VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
13153	*/
13154	HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13155	{
13156	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13157
13158	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13159	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_CR4 \| CPUMCTX_EXTRN_CR0
13160	\| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_SS);
13161	AssertRCReturn(rc, rc);
13162
13163	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
13164	rc = EMInterpretRdpmc(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx));
13165	if (RT_LIKELY(rc == VINF_SUCCESS))
13166	{
13167	rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
13168	Assert(pVmxTransient->cbInstr == 2);
13169	}
13170	else
13171	{
13172	AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
13173	rc = VERR_EM_INTERPRETER;
13174	}
13175	return rc;
13176	}
13177
13178
13179	/**
13180	* VM-exit handler for VMCALL (VMX_EXIT_VMCALL). Unconditional VM-exit.
13181	*/
13182	HMVMX_EXIT_DECL hmR0VmxExitVmcall(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13183	{
13184	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13185
13186	VBOXSTRICTRC rcStrict = VERR_VMX_IPE_3;
13187	if (EMAreHypercallInstructionsEnabled(pVCpu))
13188	{
13189	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13190	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_CR0
13191	\| CPUMCTX_EXTRN_SS \| CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_EFER);
13192	AssertRCReturn(rc, rc);
13193
13194	/* Perform the hypercall. */
13195	rcStrict = GIMHypercall(pVCpu, &pVCpu->cpum.GstCtx);
13196	if (rcStrict == VINF_SUCCESS)
13197	{
13198	rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
13199	AssertRCReturn(rc, rc);
13200	}
13201	else
13202	Assert( rcStrict == VINF_GIM_R3_HYPERCALL
13203	\|\| rcStrict == VINF_GIM_HYPERCALL_CONTINUING
13204	\|\| RT_FAILURE(rcStrict));
13205
13206	/* If the hypercall changes anything other than guest's general-purpose registers,
13207	we would need to reload the guest changed bits here before VM-entry. */
13208	}
13209	else
13210	Log4Func(("Hypercalls not enabled\n"));
13211
13212	/* If hypercalls are disabled or the hypercall failed for some reason, raise #UD and continue. */
13213	if (RT_FAILURE(rcStrict))
13214	{
13215	hmR0VmxSetPendingXcptUD(pVCpu);
13216	rcStrict = VINF_SUCCESS;
13217	}
13218
13219	return rcStrict;
13220	}
13221
13222
13223	/**
13224	* VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
13225	*/
13226	HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13227	{
13228	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13229	Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging \|\| pVCpu->hm.s.fUsingDebugLoop);
13230
13231	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13232	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
13233	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13234	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
13235	AssertRCReturn(rc, rc);
13236
13237	VBOXSTRICTRC rcStrict = IEMExecDecodedInvlpg(pVCpu, pVmxTransient->cbInstr, pVmxTransient->uExitQual);
13238
13239	if (rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_PGM_SYNC_CR3)
13240	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
13241	else if (rcStrict == VINF_IEM_RAISED_XCPT)
13242	{
13243	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13244	rcStrict = VINF_SUCCESS;
13245	}
13246	else
13247	AssertMsgFailed(("Unexpected IEMExecDecodedInvlpg(%#RX64) sttus: %Rrc\n", pVmxTransient->uExitQual,
13248	VBOXSTRICTRC_VAL(rcStrict)));
13249	return rcStrict;
13250	}
13251
13252
13253	/**
13254	* VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
13255	*/
13256	HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13257	{
13258	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13259
13260	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13261	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_SS);
13262	AssertRCReturn(rc, rc);
13263
13264	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
13265	rc = EMInterpretMonitor(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx));
13266	if (RT_LIKELY(rc == VINF_SUCCESS))
13267	rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
13268	else
13269	{
13270	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMonitor: EMInterpretMonitor failed with %Rrc\n", rc));
13271	rc = VERR_EM_INTERPRETER;
13272	}
13273	return rc;
13274	}
13275
13276
13277	/**
13278	* VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
13279	*/
13280	HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13281	{
13282	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13283
13284	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13285	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_SS);
13286	AssertRCReturn(rc, rc);
13287
13288	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
13289	VBOXSTRICTRC rc2 = EMInterpretMWait(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx));
13290	rc = VBOXSTRICTRC_VAL(rc2);
13291	if (RT_LIKELY( rc == VINF_SUCCESS
13292	\|\| rc == VINF_EM_HALT))
13293	{
13294	int rc3 = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
13295	AssertRCReturn(rc3, rc3);
13296
13297	if ( rc == VINF_EM_HALT
13298	&& EMMonitorWaitShouldContinue(pVCpu, pCtx))
13299	rc = VINF_SUCCESS;
13300	}
13301	else
13302	{
13303	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMwait: EMInterpretMWait failed with %Rrc\n", rc));
13304	rc = VERR_EM_INTERPRETER;
13305	}
13306	AssertMsg(rc == VINF_SUCCESS \|\| rc == VINF_EM_HALT \|\| rc == VERR_EM_INTERPRETER,
13307	("hmR0VmxExitMwait: failed, invalid error code %Rrc\n", rc));
13308	return rc;
13309	}
13310
13311
13312	/**
13313	* VM-exit handler for RSM (VMX_EXIT_RSM). Unconditional VM-exit.
13314	*/
13315	HMVMX_EXIT_NSRC_DECL hmR0VmxExitRsm(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13316	{
13317	/*
13318	* Execution of RSM outside of SMM mode causes #UD regardless of VMX root or VMX non-root
13319	* mode. In theory, we should never get this VM-exit. This can happen only if dual-monitor
13320	* treatment of SMI and VMX is enabled, which can (only?) be done by executing VMCALL in
13321	* VMX root operation. If we get here, something funny is going on.
13322	*
13323	* See Intel spec. 33.15.5 "Enabling the Dual-Monitor Treatment".
13324	*/
13325	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13326	AssertMsgFailed(("Unexpected RSM VM-exit\n"));
13327	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
13328	}
13329
13330
13331	/**
13332	* VM-exit handler for SMI (VMX_EXIT_SMI). Unconditional VM-exit.
13333	*/
13334	HMVMX_EXIT_NSRC_DECL hmR0VmxExitSmi(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13335	{
13336	/*
13337	* This can only happen if we support dual-monitor treatment of SMI, which can be activated
13338	* by executing VMCALL in VMX root operation. Only an STM (SMM transfer monitor) would get
13339	* this VM-exit when we (the executive monitor) execute a VMCALL in VMX root mode or receive
13340	* an SMI. If we get here, something funny is going on.
13341	*
13342	* See Intel spec. 33.15.6 "Activating the Dual-Monitor Treatment"
13343	* See Intel spec. 25.3 "Other Causes of VM-Exits"
13344	*/
13345	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13346	AssertMsgFailed(("Unexpected SMI VM-exit\n"));
13347	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
13348	}
13349
13350
13351	/**
13352	* VM-exit handler for IO SMI (VMX_EXIT_IO_SMI). Unconditional VM-exit.
13353	*/
13354	HMVMX_EXIT_NSRC_DECL hmR0VmxExitIoSmi(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13355	{
13356	/* Same treatment as VMX_EXIT_SMI. See comment in hmR0VmxExitSmi(). */
13357	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13358	AssertMsgFailed(("Unexpected IO SMI VM-exit\n"));
13359	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
13360	}
13361
13362
13363	/**
13364	* VM-exit handler for SIPI (VMX_EXIT_SIPI). Conditional VM-exit.
13365	*/
13366	HMVMX_EXIT_NSRC_DECL hmR0VmxExitSipi(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13367	{
13368	/*
13369	* SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest activity state is used.
13370	* We don't make use of it as our guests don't have direct access to the host LAPIC.
13371	* See Intel spec. 25.3 "Other Causes of VM-exits".
13372	*/
13373	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13374	AssertMsgFailed(("Unexpected SIPI VM-exit\n"));
13375	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
13376	}
13377
13378
13379	/**
13380	* VM-exit handler for INIT signal (VMX_EXIT_INIT_SIGNAL). Unconditional
13381	* VM-exit.
13382	*/
13383	HMVMX_EXIT_NSRC_DECL hmR0VmxExitInitSignal(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13384	{
13385	/*
13386	* INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM.
13387	* See Intel spec. 33.14.1 Default Treatment of SMI Delivery" and Intel spec. 29.3 "VMX Instructions" for "VMXON".
13388	*
13389	* It is -NOT- blocked in VMX non-root operation so we can, in theory, still get these VM-exits.
13390	* See Intel spec. "23.8 Restrictions on VMX operation".
13391	*/
13392	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13393	return VINF_SUCCESS;
13394	}
13395
13396
13397	/**
13398	* VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
13399	* VM-exit.
13400	*/
13401	HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13402	{
13403	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13404	return VINF_EM_RESET;
13405	}
13406
13407
13408	/**
13409	* VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
13410	*/
13411	HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13412	{
13413	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13414
13415	int rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
13416	AssertRCReturn(rc, rc);
13417
13418	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RFLAGS); /* Advancing the RIP above should've imported eflags. */
13419	if (EMShouldContinueAfterHalt(pVCpu, &pVCpu->cpum.GstCtx)) /* Requires eflags. */
13420	rc = VINF_SUCCESS;
13421	else
13422	rc = VINF_EM_HALT;
13423
13424	if (rc != VINF_SUCCESS)
13425	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHltToR3);
13426	return rc;
13427	}
13428
13429
13430	/**
13431	* VM-exit handler for instructions that result in a \#UD exception delivered to
13432	* the guest.
13433	*/
13434	HMVMX_EXIT_NSRC_DECL hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13435	{
13436	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13437	hmR0VmxSetPendingXcptUD(pVCpu);
13438	return VINF_SUCCESS;
13439	}
13440
13441
13442	/**
13443	* VM-exit handler for expiry of the VMX-preemption timer.
13444	*/
13445	HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13446	{
13447	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13448
13449	/* If the VMX-preemption timer has expired, reinitialize the preemption timer on next VM-entry. */
13450	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
13451
13452	/* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
13453	PVM pVM = pVCpu->CTX_SUFF(pVM);
13454	bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
13455	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
13456	return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
13457	}
13458
13459
13460	/**
13461	* VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
13462	*/
13463	HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13464	{
13465	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13466
13467	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13468	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13469	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK \| CPUMCTX_EXTRN_CR4);
13470	AssertRCReturn(rc, rc);
13471
13472	VBOXSTRICTRC rcStrict = IEMExecDecodedXsetbv(pVCpu, pVmxTransient->cbInstr);
13473	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, rcStrict != VINF_IEM_RAISED_XCPT ? HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS
13474	: HM_CHANGED_RAISED_XCPT_MASK);
13475
13476	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
13477	pVCpu->hm.s.fLoadSaveGuestXcr0 = (pCtx->cr4 & X86_CR4_OSXSAVE) && pCtx->aXcr[0] != ASMGetXcr0();
13478
13479	return rcStrict;
13480	}
13481
13482
13483	/**
13484	* VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
13485	*/
13486	HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13487	{
13488	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13489	/** @todo Use VM-exit instruction information. */
13490	return VERR_EM_INTERPRETER;
13491	}
13492
13493
13494	/**
13495	* VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE).
13496	* Error VM-exit.
13497	*/
13498	HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13499	{
13500	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13501	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
13502	AssertRCReturn(rc, rc);
13503
13504	rc = hmR0VmxCheckVmcsCtls(pVCpu, pVmcsInfo);
13505	if (RT_FAILURE(rc))
13506	return rc;
13507
13508	uint32_t const uInvalidReason = hmR0VmxCheckGuestState(pVCpu, pVmcsInfo);
13509	NOREF(uInvalidReason);
13510
13511	#ifdef VBOX_STRICT
13512	uint32_t fIntrState;
13513	RTHCUINTREG uHCReg;
13514	uint64_t u64Val;
13515	uint32_t u32Val;
13516	rc = hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
13517	rc \|= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
13518	rc \|= hmR0VmxReadEntryInstrLenVmcs(pVmxTransient);
13519	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &fIntrState);
13520	AssertRCReturn(rc, rc);
13521
13522	Log4(("uInvalidReason %u\n", uInvalidReason));
13523	Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntInfo));
13524	Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
13525	Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
13526	Log4(("VMX_VMCS32_GUEST_INT_STATE %#RX32\n", fIntrState));
13527
13528	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val); AssertRC(rc);
13529	Log4(("VMX_VMCS_GUEST_CR0 %#RX32\n", u32Val));
13530	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
13531	Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
13532	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
13533	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
13534	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
13535	Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
13536	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
13537	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
13538	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
13539	Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
13540
13541	hmR0DumpRegs(pVCpu);
13542	#endif
13543
13544	return VERR_VMX_INVALID_GUEST_STATE;
13545	}
13546
13547
13548	/**
13549	* VM-exit handler for VM-entry failure due to an MSR-load
13550	* (VMX_EXIT_ERR_MSR_LOAD). Error VM-exit.
13551	*/
13552	HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13553	{
13554	AssertMsgFailed(("Unexpected MSR-load exit\n"));
13555	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
13556	}
13557
13558
13559	/**
13560	* VM-exit handler for VM-entry failure due to a machine-check event
13561	* (VMX_EXIT_ERR_MACHINE_CHECK). Error VM-exit.
13562	*/
13563	HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13564	{
13565	AssertMsgFailed(("Unexpected machine-check event exit\n"));
13566	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
13567	}
13568
13569
13570	/**
13571	* VM-exit handler for all undefined reasons. Should never ever happen.. in
13572	* theory.
13573	*/
13574	HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrUndefined(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13575	{
13576	RT_NOREF2(pVCpu, pVmxTransient);
13577	AssertMsgFailed(("Huh!? Undefined VM-exit reason %d\n", pVmxTransient->uExitReason));
13578	return VERR_VMX_UNDEFINED_EXIT_CODE;
13579	}
13580
13581
13582	/**
13583	* VM-exit handler for XDTR (LGDT, SGDT, LIDT, SIDT) accesses
13584	* (VMX_EXIT_GDTR_IDTR_ACCESS) and LDT and TR access (LLDT, LTR, SLDT, STR).
13585	* Conditional VM-exit.
13586	*/
13587	HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13588	{
13589	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13590
13591	/* By default, we don't enable VMX_PROC_CTLS2_DESCRIPTOR_TABLE_EXIT. */
13592	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitXdtrAccess);
13593	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13594	if (pVmcsInfo->u32ProcCtls2 & VMX_PROC_CTLS2_DESC_TABLE_EXIT)
13595	return VERR_EM_INTERPRETER;
13596	AssertMsgFailed(("Unexpected XDTR access\n"));
13597	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
13598	}
13599
13600
13601	/**
13602	* VM-exit handler for RDRAND (VMX_EXIT_RDRAND). Conditional VM-exit.
13603	*/
13604	HMVMX_EXIT_DECL hmR0VmxExitRdrand(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13605	{
13606	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13607
13608	/* By default, we don't enable VMX_PROC_CTLS2_RDRAND_EXIT. */
13609	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13610	if (pVmcsInfo->u32ProcCtls2 & VMX_PROC_CTLS2_RDRAND_EXIT)
13611	return VERR_EM_INTERPRETER;
13612	AssertMsgFailed(("Unexpected RDRAND exit\n"));
13613	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
13614	}
13615
13616
13617	/**
13618	* VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
13619	*/
13620	HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13621	{
13622	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13623
13624	/** @todo Optimize this: We currently drag in in the whole MSR state
13625	* (CPUMCTX_EXTRN_ALL_MSRS) here. We should optimize this to only get
13626	* MSRs required. That would require changes to IEM and possibly CPUM too.
13627	* (Should probably do it lazy fashion from CPUMAllMsrs.cpp). */
13628	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13629	uint32_t const idMsr = pVCpu->cpum.GstCtx.ecx;
13630	uint64_t fImport = IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| CPUMCTX_EXTRN_ALL_MSRS;
13631	switch (idMsr)
13632	{
13633	case MSR_K8_FS_BASE: fImport \|= CPUMCTX_EXTRN_FS; break;
13634	case MSR_K8_GS_BASE: fImport \|= CPUMCTX_EXTRN_GS; break;
13635	}
13636
13637	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13638	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, fImport);
13639	AssertRCReturn(rc, rc);
13640
13641	Log4Func(("ecx=%#RX32\n", idMsr));
13642
13643	#ifdef VBOX_STRICT
13644	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
13645	{
13646	if ( hmR0VmxIsAutoLoadGuestMsr(pVmcsInfo, idMsr)
13647	&& idMsr != MSR_K6_EFER)
13648	{
13649	AssertMsgFailed(("Unexpected RDMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n", idMsr));
13650	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
13651	}
13652	if (hmR0VmxIsLazyGuestMsr(pVCpu, idMsr))
13653	{
13654	Assert(pVmcsInfo->pvMsrBitmap);
13655	uint32_t fMsrpm = HMGetVmxMsrPermission(pVmcsInfo->pvMsrBitmap, idMsr);
13656	if (fMsrpm & VMXMSRPM_ALLOW_RD)
13657	{
13658	AssertMsgFailed(("Unexpected RDMSR for a passthru lazy-restore MSR. ecx=%#RX32\n", idMsr));
13659	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
13660	}
13661	}
13662	}
13663	#endif
13664
13665	VBOXSTRICTRC rcStrict = IEMExecDecodedRdmsr(pVCpu, pVmxTransient->cbInstr);
13666	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
13667	if (rcStrict == VINF_SUCCESS)
13668	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS
13669	\| HM_CHANGED_GUEST_RAX \| HM_CHANGED_GUEST_RDX);
13670	else if (rcStrict == VINF_IEM_RAISED_XCPT)
13671	{
13672	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13673	rcStrict = VINF_SUCCESS;
13674	}
13675	else
13676	AssertMsg(rcStrict == VINF_CPUM_R3_MSR_READ, ("Unexpected IEMExecDecodedRdmsr rc (%Rrc)\n", VBOXSTRICTRC_VAL(rcStrict)));
13677
13678	return rcStrict;
13679	}
13680
13681
13682	/**
13683	* VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
13684	*/
13685	HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13686	{
13687	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13688
13689	/** @todo Optimize this: We currently drag in in the whole MSR state
13690	* (CPUMCTX_EXTRN_ALL_MSRS) here. We should optimize this to only get
13691	* MSRs required. That would require changes to IEM and possibly CPUM too.
13692	* (Should probably do it lazy fashion from CPUMAllMsrs.cpp). */
13693	uint32_t const idMsr = pVCpu->cpum.GstCtx.ecx;
13694	uint64_t fImport = IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| CPUMCTX_EXTRN_ALL_MSRS;
13695
13696	/*
13697	* The FS and GS base MSRs are not part of the above all-MSRs mask.
13698	* Although we don't need to fetch the base as it will be overwritten shortly, while
13699	* loading guest-state we would also load the entire segment register including limit
13700	* and attributes and thus we need to load them here.
13701	*/
13702	switch (idMsr)
13703	{
13704	case MSR_K8_FS_BASE: fImport \|= CPUMCTX_EXTRN_FS; break;
13705	case MSR_K8_GS_BASE: fImport \|= CPUMCTX_EXTRN_GS; break;
13706	}
13707
13708	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13709	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13710	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, fImport);
13711	AssertRCReturn(rc, rc);
13712
13713	Log4Func(("ecx=%#RX32 edx:eax=%#RX32:%#RX32\n", idMsr, pVCpu->cpum.GstCtx.edx, pVCpu->cpum.GstCtx.eax));
13714
13715	VBOXSTRICTRC rcStrict = IEMExecDecodedWrmsr(pVCpu, pVmxTransient->cbInstr);
13716	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
13717
13718	if (rcStrict == VINF_SUCCESS)
13719	{
13720	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
13721
13722	/* If this is an X2APIC WRMSR access, update the APIC state as well. */
13723	if ( idMsr == MSR_IA32_APICBASE
13724	\|\| ( idMsr >= MSR_IA32_X2APIC_START
13725	&& idMsr <= MSR_IA32_X2APIC_END))
13726	{
13727	/*
13728	* We've already saved the APIC related guest-state (TPR) in post-run phase.
13729	* When full APIC register virtualization is implemented we'll have to make
13730	* sure APIC state is saved from the VMCS before IEM changes it.
13731	*/
13732	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
13733	}
13734	else if (idMsr == MSR_IA32_TSC) /* Windows 7 does this during bootup. See @bugref{6398}. */
13735	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
13736	else if (idMsr == MSR_K6_EFER)
13737	{
13738	/*
13739	* If the guest touches the EFER MSR we need to update the VM-Entry and VM-Exit controls
13740	* as well, even if it is -not- touching bits that cause paging mode changes (LMA/LME).
13741	* We care about the other bits as well, SCE and NXE. See @bugref{7368}.
13742	*/
13743	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_EFER_MSR \| HM_CHANGED_VMX_ENTRY_EXIT_CTLS);
13744	}
13745
13746	/* Update MSRs that are part of the VMCS and auto-load/store area when MSR-bitmaps are not supported. */
13747	if (!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS))
13748	{
13749	switch (idMsr)
13750	{
13751	case MSR_IA32_SYSENTER_CS: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_CS_MSR); break;
13752	case MSR_IA32_SYSENTER_EIP: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_EIP_MSR); break;
13753	case MSR_IA32_SYSENTER_ESP: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_ESP_MSR); break;
13754	case MSR_K8_FS_BASE: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_FS); break;
13755	case MSR_K8_GS_BASE: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_GS); break;
13756	case MSR_K6_EFER: /* Nothing to do, already handled above. */ break;
13757	default:
13758	{
13759	if (hmR0VmxIsAutoLoadGuestMsr(pVmcsInfo, idMsr))
13760	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
13761	else if (hmR0VmxIsLazyGuestMsr(pVCpu, idMsr))
13762	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_VMX_GUEST_LAZY_MSRS);
13763	break;
13764	}
13765	}
13766	}
13767	#ifdef VBOX_STRICT
13768	else
13769	{
13770	/* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
13771	switch (idMsr)
13772	{
13773	case MSR_IA32_SYSENTER_CS:
13774	case MSR_IA32_SYSENTER_EIP:
13775	case MSR_IA32_SYSENTER_ESP:
13776	case MSR_K8_FS_BASE:
13777	case MSR_K8_GS_BASE:
13778	{
13779	uint32_t fMsrpm = HMGetVmxMsrPermission(pVmcsInfo->pvMsrBitmap, idMsr);
13780	Assert(fMsrpm == (VMXMSRPM_ALLOW_RD \| VMXMSRPM_ALLOW_WR));
13781
13782	uint32_t u32Proc;
13783	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Proc);
13784	AssertRC(rc);
13785	Assert(u32Proc == pVmcsInfo->u32ProcCtls);
13786	Assert(u32Proc & VMX_PROC_CTLS_USE_MSR_BITMAPS);
13787
13788	AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32 fMsrpm=%#RX32\n", idMsr, fMsrpm));
13789	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
13790	}
13791
13792	/* Writes to MSRs in auto-load/store area/swapped MSRs, shouldn't cause VM-exits with MSR-bitmaps. */
13793	default:
13794	{
13795	if (hmR0VmxIsAutoLoadGuestMsr(pVmcsInfo, idMsr))
13796	{
13797	/* EFER MSR writes are always intercepted. */
13798	if (idMsr != MSR_K6_EFER)
13799	{
13800	AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
13801	idMsr));
13802	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
13803	}
13804	}
13805
13806	if (hmR0VmxIsLazyGuestMsr(pVCpu, idMsr))
13807	{
13808	Assert(pVmcsInfo->pvMsrBitmap);
13809	uint32_t fMsrpm = HMGetVmxMsrPermission(pVmcsInfo->pvMsrBitmap, idMsr);
13810	if (fMsrpm & VMXMSRPM_ALLOW_WR)
13811	{
13812	AssertMsgFailed(("Unexpected WRMSR for passthru, lazy-restore MSR. ecx=%#RX32\n", idMsr));
13813	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
13814	}
13815	}
13816	break;
13817	}
13818	}
13819	}
13820	#endif /* VBOX_STRICT */
13821	}
13822	else if (rcStrict == VINF_IEM_RAISED_XCPT)
13823	{
13824	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13825	rcStrict = VINF_SUCCESS;
13826	}
13827	else
13828	AssertMsg(rcStrict == VINF_CPUM_R3_MSR_WRITE, ("Unexpected IEMExecDecodedWrmsr rc (%Rrc)\n", VBOXSTRICTRC_VAL(rcStrict)));
13829
13830	return rcStrict;
13831	}
13832
13833
13834	/**
13835	* VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
13836	*/
13837	HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13838	{
13839	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13840	/** @todo The guest has likely hit a contended spinlock. We might want to
13841	* poke a schedule different guest VCPU. */
13842	return VINF_EM_RAW_INTERRUPT;
13843	}
13844
13845
13846	/**
13847	* VM-exit handler for when the TPR value is lowered below the specified
13848	* threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
13849	*/
13850	HMVMX_EXIT_NSRC_DECL hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13851	{
13852	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13853	Assert(pVmxTransient->pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW);
13854
13855	/*
13856	* The TPR shadow would've been synced with the APIC TPR in the post-run phase.
13857	* We'll re-evaluate pending interrupts and inject them before the next VM
13858	* entry so we can just continue execution here.
13859	*/
13860	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
13861	return VINF_SUCCESS;
13862	}
13863
13864
13865	/**
13866	* VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
13867	* VM-exit.
13868	*
13869	* @retval VINF_SUCCESS when guest execution can continue.
13870	* @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
13871	* @retval VERR_EM_INTERPRETER when something unexpected happened, fallback to
13872	* interpreter.
13873	*/
13874	HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13875	{
13876	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13877	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
13878
13879	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13880	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
13881	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13882	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
13883	AssertRCReturn(rc, rc);
13884
13885	VBOXSTRICTRC rcStrict;
13886	PVM pVM = pVCpu->CTX_SUFF(pVM);
13887	RTGCUINTPTR const uExitQual = pVmxTransient->uExitQual;
13888	uint32_t const uAccessType = VMX_EXIT_QUAL_CRX_ACCESS(uExitQual);
13889	switch (uAccessType)
13890	{
13891	case VMX_EXIT_QUAL_CRX_ACCESS_WRITE: /* MOV to CRx */
13892	{
13893	uint32_t const uOldCr0 = pVCpu->cpum.GstCtx.cr0;
13894	rcStrict = IEMExecDecodedMovCRxWrite(pVCpu, pVmxTransient->cbInstr, VMX_EXIT_QUAL_CRX_REGISTER(uExitQual),
13895	VMX_EXIT_QUAL_CRX_GENREG(uExitQual));
13896	AssertMsg( rcStrict == VINF_SUCCESS
13897	\|\| rcStrict == VINF_IEM_RAISED_XCPT
13898	\|\| rcStrict == VINF_PGM_SYNC_CR3, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
13899
13900	switch (VMX_EXIT_QUAL_CRX_REGISTER(uExitQual))
13901	{
13902	case 0:
13903	{
13904	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged,
13905	HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR0);
13906	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR0Write);
13907	Log4Func(("CR0 write rcStrict=%Rrc CR0=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cr0));
13908
13909	/*
13910	* This is a kludge for handling switches back to real mode when we try to use
13911	* V86 mode to run real mode code directly. Problem is that V86 mode cannot
13912	* deal with special selector values, so we have to return to ring-3 and run
13913	* there till the selector values are V86 mode compatible.
13914	*
13915	* Note! Using VINF_EM_RESCHEDULE_REM here rather than VINF_EM_RESCHEDULE since the
13916	* latter is an alias for VINF_IEM_RAISED_XCPT which is converted to VINF_SUCCESs
13917	* at the end of this function.
13918	*/
13919	if ( rc == VINF_SUCCESS
13920	&& !pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest
13921	&& CPUMIsGuestInRealModeEx(&pVCpu->cpum.GstCtx)
13922	&& (uOldCr0 & X86_CR0_PE)
13923	&& !(pVCpu->cpum.GstCtx.cr0 & X86_CR0_PE) )
13924	{
13925	/** @todo check selectors rather than returning all the time. */
13926	Log4Func(("CR0 write, back to real mode -> VINF_EM_RESCHEDULE_REM\n"));
13927	rcStrict = VINF_EM_RESCHEDULE_REM;
13928	}
13929	break;
13930	}
13931
13932	case 2:
13933	{
13934	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR2Write);
13935	/* Nothing to do here, CR2 it's not part of the VMCS. */
13936	break;
13937	}
13938
13939	case 3:
13940	{
13941	Assert( !pVM->hm.s.fNestedPaging
13942	\|\| !CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx)
13943	\|\| pVCpu->hm.s.fUsingDebugLoop);
13944	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR3Write);
13945	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged,
13946	HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR3);
13947	Log4Func(("CR3 write rcStrict=%Rrc CR3=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cr3));
13948	break;
13949	}
13950
13951	case 4:
13952	{
13953	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR4Write);
13954	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged,
13955	HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR4);
13956	Log4Func(("CR4 write rc=%Rrc CR4=%#RX64 fLoadSaveGuestXcr0=%u\n", VBOXSTRICTRC_VAL(rcStrict),
13957	pVCpu->cpum.GstCtx.cr4, pVCpu->hm.s.fLoadSaveGuestXcr0));
13958	break;
13959	}
13960
13961	case 8:
13962	{
13963	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR8Write);
13964	Assert(!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW));
13965	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged,
13966	HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_APIC_TPR);
13967	break;
13968	}
13969	default:
13970	AssertMsgFailed(("Invalid CRx register %#x\n", VMX_EXIT_QUAL_CRX_REGISTER(uExitQual)));
13971	break;
13972	}
13973	break;
13974	}
13975
13976	case VMX_EXIT_QUAL_CRX_ACCESS_READ: /* MOV from CRx */
13977	{
13978	Assert( !pVM->hm.s.fNestedPaging
13979	\|\| !CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx)
13980	\|\| pVCpu->hm.s.fUsingDebugLoop
13981	\|\| VMX_EXIT_QUAL_CRX_REGISTER(uExitQual) != 3);
13982	/* CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
13983	Assert( VMX_EXIT_QUAL_CRX_REGISTER(uExitQual) != 8
13984	\|\| !(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW));
13985
13986	rcStrict = IEMExecDecodedMovCRxRead(pVCpu, pVmxTransient->cbInstr, VMX_EXIT_QUAL_CRX_GENREG(uExitQual),
13987	VMX_EXIT_QUAL_CRX_REGISTER(uExitQual));
13988	AssertMsg( rcStrict == VINF_SUCCESS
13989	\|\| rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
13990	#ifdef VBOX_WITH_STATISTICS
13991	switch (VMX_EXIT_QUAL_CRX_REGISTER(uExitQual))
13992	{
13993	case 0: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR0Read); break;
13994	case 2: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR2Read); break;
13995	case 3: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR3Read); break;
13996	case 4: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR4Read); break;
13997	case 8: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR8Read); break;
13998	}
13999	#endif
14000	Log4Func(("CR%d Read access rcStrict=%Rrc\n", VMX_EXIT_QUAL_CRX_REGISTER(uExitQual),
14001	VBOXSTRICTRC_VAL(rcStrict)));
14002	if (VMX_EXIT_QUAL_CRX_GENREG(uExitQual) == X86_GREG_xSP)
14003	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_RSP);
14004	else
14005	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
14006	break;
14007	}
14008
14009	case VMX_EXIT_QUAL_CRX_ACCESS_CLTS: /* CLTS (Clear Task-Switch Flag in CR0) */
14010	{
14011	rcStrict = IEMExecDecodedClts(pVCpu, pVmxTransient->cbInstr);
14012	AssertMsg( rcStrict == VINF_SUCCESS
14013	\|\| rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
14014
14015	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR0);
14016	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
14017	Log4Func(("CLTS rcStrict=%d\n", VBOXSTRICTRC_VAL(rcStrict)));
14018	break;
14019	}
14020
14021	case VMX_EXIT_QUAL_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
14022	{
14023	/* Note! LMSW cannot clear CR0.PE, so no fRealOnV86Active kludge needed here. */
14024	rc = hmR0VmxReadGuestLinearAddrVmcs(pVCpu, pVmxTransient);
14025	AssertRCReturn(rc, rc);
14026	rcStrict = IEMExecDecodedLmsw(pVCpu, pVmxTransient->cbInstr, VMX_EXIT_QUAL_CRX_LMSW_DATA(uExitQual),
14027	pVmxTransient->uGuestLinearAddr);
14028	AssertMsg( rcStrict == VINF_SUCCESS
14029	\|\| rcStrict == VINF_IEM_RAISED_XCPT
14030	, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
14031
14032	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR0);
14033	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
14034	Log4Func(("LMSW rcStrict=%d\n", VBOXSTRICTRC_VAL(rcStrict)));
14035	break;
14036	}
14037
14038	default:
14039	AssertMsgFailedReturn(("Invalid access-type in Mov CRx VM-exit qualification %#x\n", uAccessType),
14040	VERR_VMX_UNEXPECTED_EXCEPTION);
14041	}
14042
14043	Assert( (pVCpu->hm.s.fCtxChanged & (HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS))
14044	== (HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS));
14045	if (rcStrict == VINF_IEM_RAISED_XCPT)
14046	{
14047	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
14048	rcStrict = VINF_SUCCESS;
14049	}
14050
14051	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
14052	NOREF(pVM);
14053	return rcStrict;
14054	}
14055
14056
14057	/**
14058	* VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
14059	* VM-exit.
14060	*/
14061	HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14062	{
14063	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14064	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
14065
14066	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
14067	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14068	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
14069	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14070	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK \| CPUMCTX_EXTRN_SREG_MASK
14071	\| CPUMCTX_EXTRN_EFER);
14072	/* EFER MSR also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
14073	AssertRCReturn(rc, rc);
14074
14075	/* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
14076	uint32_t uIOPort = VMX_EXIT_QUAL_IO_PORT(pVmxTransient->uExitQual);
14077	uint8_t uIOWidth = VMX_EXIT_QUAL_IO_WIDTH(pVmxTransient->uExitQual);
14078	bool fIOWrite = (VMX_EXIT_QUAL_IO_DIRECTION(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_IO_DIRECTION_OUT);
14079	bool fIOString = VMX_EXIT_QUAL_IO_IS_STRING(pVmxTransient->uExitQual);
14080	bool fGstStepping = RT_BOOL(pCtx->eflags.Bits.u1TF);
14081	bool fDbgStepping = pVCpu->hm.s.fSingleInstruction;
14082	AssertReturn(uIOWidth <= 3 && uIOWidth != 2, VERR_VMX_IPE_1);
14083
14084	/*
14085	* Update exit history to see if this exit can be optimized.
14086	*/
14087	VBOXSTRICTRC rcStrict;
14088	PCEMEXITREC pExitRec = NULL;
14089	if ( !fGstStepping
14090	&& !fDbgStepping)
14091	pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
14092	!fIOString
14093	? !fIOWrite
14094	? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_READ)
14095	: EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_WRITE)
14096	: !fIOWrite
14097	? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_STR_READ)
14098	: EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_STR_WRITE),
14099	pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
14100	if (!pExitRec)
14101	{
14102	/* I/O operation lookup arrays. */
14103	static uint32_t const s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses. */
14104	static uint32_t const s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving result in AL/AX/EAX. */
14105	uint32_t const cbValue = s_aIOSizes[uIOWidth];
14106	uint32_t const cbInstr = pVmxTransient->cbInstr;
14107	bool fUpdateRipAlready = false; /* ugly hack, should be temporary. */
14108	PVM pVM = pVCpu->CTX_SUFF(pVM);
14109	if (fIOString)
14110	{
14111	/*
14112	* INS/OUTS - I/O String instruction.
14113	*
14114	* Use instruction-information if available, otherwise fall back on
14115	* interpreting the instruction.
14116	*/
14117	Log4Func(("cs:rip=%#04x:%#RX64 %#06x/%u %c str\n", pCtx->cs.Sel, pCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
14118	AssertReturn(pCtx->dx == uIOPort, VERR_VMX_IPE_2);
14119	bool const fInsOutsInfo = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_INS_OUTS);
14120	if (fInsOutsInfo)
14121	{
14122	int rc2 = hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
14123	AssertRCReturn(rc2, rc2);
14124	AssertReturn(pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize <= 2, VERR_VMX_IPE_3);
14125	AssertCompile(IEMMODE_16BIT == 0 && IEMMODE_32BIT == 1 && IEMMODE_64BIT == 2);
14126	IEMMODE const enmAddrMode = (IEMMODE)pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize;
14127	bool const fRep = VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQual);
14128	if (fIOWrite)
14129	rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, fRep, cbInstr,
14130	pVmxTransient->ExitInstrInfo.StrIo.iSegReg, true /fIoChecked/);
14131	else
14132	{
14133	/*
14134	* The segment prefix for INS cannot be overridden and is always ES. We can safely assume X86_SREG_ES.
14135	* Hence "iSegReg" field is undefined in the instruction-information field in VT-x for INS.
14136	* See Intel Instruction spec. for "INS".
14137	* See Intel spec. Table 27-8 "Format of the VM-Exit Instruction-Information Field as Used for INS and OUTS".
14138	*/
14139	rcStrict = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, fRep, cbInstr, true /fIoChecked/);
14140	}
14141	}
14142	else
14143	rcStrict = IEMExecOne(pVCpu);
14144
14145	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
14146	fUpdateRipAlready = true;
14147	}
14148	else
14149	{
14150	/*
14151	* IN/OUT - I/O instruction.
14152	*/
14153	Log4Func(("cs:rip=%04x:%08RX64 %#06x/%u %c\n", pCtx->cs.Sel, pCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
14154	uint32_t const uAndVal = s_aIOOpAnd[uIOWidth];
14155	Assert(!VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQual));
14156	if (fIOWrite)
14157	{
14158	rcStrict = IOMIOPortWrite(pVM, pVCpu, uIOPort, pCtx->eax & uAndVal, cbValue);
14159	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
14160	if ( rcStrict == VINF_IOM_R3_IOPORT_WRITE
14161	&& !pCtx->eflags.Bits.u1TF)
14162	rcStrict = EMRZSetPendingIoPortWrite(pVCpu, uIOPort, cbInstr, cbValue, pCtx->eax & uAndVal);
14163	}
14164	else
14165	{
14166	uint32_t u32Result = 0;
14167	rcStrict = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbValue);
14168	if (IOM_SUCCESS(rcStrict))
14169	{
14170	/* Save result of I/O IN instr. in AL/AX/EAX. */
14171	pCtx->eax = (pCtx->eax & ~uAndVal) \| (u32Result & uAndVal);
14172	}
14173	if ( rcStrict == VINF_IOM_R3_IOPORT_READ
14174	&& !pCtx->eflags.Bits.u1TF)
14175	rcStrict = EMRZSetPendingIoPortRead(pVCpu, uIOPort, cbInstr, cbValue);
14176	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
14177	}
14178	}
14179
14180	if (IOM_SUCCESS(rcStrict))
14181	{
14182	if (!fUpdateRipAlready)
14183	{
14184	hmR0VmxAdvanceGuestRipBy(pVCpu, cbInstr);
14185	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
14186	}
14187
14188	/*
14189	* INS/OUTS with REP prefix updates RFLAGS, can be observed with triple-fault guru
14190	* while booting Fedora 17 64-bit guest.
14191	*
14192	* See Intel Instruction reference for REP/REPE/REPZ/REPNE/REPNZ.
14193	*/
14194	if (fIOString)
14195	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RFLAGS);
14196
14197	/*
14198	* If any I/O breakpoints are armed, we need to check if one triggered
14199	* and take appropriate action.
14200	* Note that the I/O breakpoint type is undefined if CR4.DE is 0.
14201	*/
14202	rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_DR7);
14203	AssertRCReturn(rc, rc);
14204
14205	/** @todo Optimize away the DBGFBpIsHwIoArmed call by having DBGF tell the
14206	* execution engines about whether hyper BPs and such are pending. */
14207	uint32_t const uDr7 = pCtx->dr[7];
14208	if (RT_UNLIKELY( ( (uDr7 & X86_DR7_ENABLED_MASK)
14209	&& X86_DR7_ANY_RW_IO(uDr7)
14210	&& (pCtx->cr4 & X86_CR4_DE))
14211	\|\| DBGFBpIsHwIoArmed(pVM)))
14212	{
14213	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
14214
14215	/* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
14216	VMMRZCallRing3Disable(pVCpu);
14217	HM_DISABLE_PREEMPT(pVCpu);
14218
14219	bool fIsGuestDbgActive = CPUMR0DebugStateMaybeSaveGuest(pVCpu, true /* fDr6 */);
14220
14221	VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, pCtx, uIOPort, cbValue);
14222	if (rcStrict2 == VINF_EM_RAW_GUEST_TRAP)
14223	{
14224	/* Raise #DB. */
14225	if (fIsGuestDbgActive)
14226	ASMSetDR6(pCtx->dr[6]);
14227	if (pCtx->dr[7] != uDr7)
14228	pVCpu->hm.s.fCtxChanged \|= HM_CHANGED_GUEST_DR7;
14229
14230	hmR0VmxSetPendingXcptDB(pVCpu);
14231	}
14232	/* rcStrict is VINF_SUCCESS, VINF_IOM_R3_IOPORT_COMMIT_WRITE, or in [VINF_EM_FIRST..VINF_EM_LAST],
14233	however we can ditch VINF_IOM_R3_IOPORT_COMMIT_WRITE as it has VMCPU_FF_IOM as backup. */
14234	else if ( rcStrict2 != VINF_SUCCESS
14235	&& (rcStrict == VINF_SUCCESS \|\| rcStrict2 < rcStrict))
14236	rcStrict = rcStrict2;
14237	AssertCompile(VINF_EM_LAST < VINF_IOM_R3_IOPORT_COMMIT_WRITE);
14238
14239	HM_RESTORE_PREEMPT();
14240	VMMRZCallRing3Enable(pVCpu);
14241	}
14242	}
14243
14244	#ifdef VBOX_STRICT
14245	if ( rcStrict == VINF_IOM_R3_IOPORT_READ
14246	\|\| rcStrict == VINF_EM_PENDING_R3_IOPORT_READ)
14247	Assert(!fIOWrite);
14248	else if ( rcStrict == VINF_IOM_R3_IOPORT_WRITE
14249	\|\| rcStrict == VINF_IOM_R3_IOPORT_COMMIT_WRITE
14250	\|\| rcStrict == VINF_EM_PENDING_R3_IOPORT_WRITE)
14251	Assert(fIOWrite);
14252	else
14253	{
14254	# if 0 /** @todo r=bird: This is missing a bunch of VINF_EM_FIRST..VINF_EM_LAST
14255	* statuses, that the VMM device and some others may return. See
14256	* IOM_SUCCESS() for guidance. */
14257	AssertMsg( RT_FAILURE(rcStrict)
14258	\|\| rcStrict == VINF_SUCCESS
14259	\|\| rcStrict == VINF_EM_RAW_EMULATE_INSTR
14260	\|\| rcStrict == VINF_EM_DBG_BREAKPOINT
14261	\|\| rcStrict == VINF_EM_RAW_GUEST_TRAP
14262	\|\| rcStrict == VINF_EM_RAW_TO_R3
14263	\|\| rcStrict == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
14264	# endif
14265	}
14266	#endif
14267	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
14268	}
14269	else
14270	{
14271	/*
14272	* Frequent exit or something needing probing. Get state and call EMHistoryExec.
14273	*/
14274	int rc2 = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
14275	AssertRCReturn(rc2, rc2);
14276	STAM_COUNTER_INC(!fIOString ? fIOWrite ? &pVCpu->hm.s.StatExitIOWrite : &pVCpu->hm.s.StatExitIORead
14277	: fIOWrite ? &pVCpu->hm.s.StatExitIOStringWrite : &pVCpu->hm.s.StatExitIOStringRead);
14278	Log4(("IOExit/%u: %04x:%08RX64: %s%s%s %#x LB %u -> EMHistoryExec\n",
14279	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
14280	VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQual) ? "REP " : "",
14281	fIOWrite ? "OUT" : "IN", fIOString ? "S" : "", uIOPort, uIOWidth));
14282
14283	rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
14284	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
14285
14286	Log4(("IOExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
14287	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
14288	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
14289	}
14290	return rcStrict;
14291	}
14292
14293
14294	/**
14295	* VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
14296	* VM-exit.
14297	*/
14298	HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14299	{
14300	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14301
14302	/* Check if this task-switch occurred while delivery an event through the guest IDT. */
14303	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
14304	AssertRCReturn(rc, rc);
14305	if (VMX_EXIT_QUAL_TASK_SWITCH_TYPE(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_TASK_SWITCH_TYPE_IDT)
14306	{
14307	rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
14308	AssertRCReturn(rc, rc);
14309	if (VMX_IDT_VECTORING_INFO_IS_VALID(pVmxTransient->uIdtVectoringInfo))
14310	{
14311	uint32_t uErrCode;
14312	RTGCUINTPTR GCPtrFaultAddress;
14313	uint32_t const uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
14314	uint32_t const uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
14315	bool const fErrorCodeValid = VMX_IDT_VECTORING_INFO_IS_ERROR_CODE_VALID(pVmxTransient->uIdtVectoringInfo);
14316	if (fErrorCodeValid)
14317	{
14318	rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
14319	AssertRCReturn(rc, rc);
14320	uErrCode = pVmxTransient->uIdtVectoringErrorCode;
14321	}
14322	else
14323	uErrCode = 0;
14324
14325	if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
14326	&& uVector == X86_XCPT_PF)
14327	GCPtrFaultAddress = pVCpu->cpum.GstCtx.cr2;
14328	else
14329	GCPtrFaultAddress = 0;
14330
14331	rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14332	AssertRCReturn(rc, rc);
14333
14334	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
14335	pVmxTransient->cbInstr, uErrCode, GCPtrFaultAddress);
14336
14337	Log4Func(("Pending event. uIntType=%#x uVector=%#x\n", uIntType, uVector));
14338	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
14339	return VINF_EM_RAW_INJECT_TRPM_EVENT;
14340	}
14341	}
14342
14343	/* Fall back to the interpreter to emulate the task-switch. */
14344	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
14345	return VERR_EM_INTERPRETER;
14346	}
14347
14348
14349	/**
14350	* VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
14351	*/
14352	HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14353	{
14354	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14355
14356	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14357	pVmcsInfo->u32ProcCtls &= ~VMX_PROC_CTLS_MONITOR_TRAP_FLAG;
14358	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
14359	AssertRCReturn(rc, rc);
14360	return VINF_EM_DBG_STEPPED;
14361	}
14362
14363
14364	/**
14365	* VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
14366	*/
14367	HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14368	{
14369	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14370	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
14371
14372	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
14373	VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
14374	if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
14375	{
14376	/* For some crazy guest, if an event delivery causes an APIC-access VM-exit, go to instruction emulation. */
14377	if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
14378	{
14379	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
14380	return VINF_EM_RAW_INJECT_TRPM_EVENT;
14381	}
14382	}
14383	else
14384	{
14385	if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
14386	rcStrict1 = VINF_SUCCESS;
14387	return rcStrict1;
14388	}
14389
14390	/* IOMMIOPhysHandler() below may call into IEM, save the necessary state. */
14391	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14392	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
14393	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
14394	AssertRCReturn(rc, rc);
14395
14396	/* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
14397	uint32_t uAccessType = VMX_EXIT_QUAL_APIC_ACCESS_TYPE(pVmxTransient->uExitQual);
14398	VBOXSTRICTRC rcStrict2;
14399	switch (uAccessType)
14400	{
14401	case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
14402	case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
14403	{
14404	AssertMsg( !(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
14405	\|\| VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQual) != XAPIC_OFF_TPR,
14406	("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
14407
14408	RTGCPHYS GCPhys = pVCpu->hm.s.vmx.u64GstMsrApicBase; /* Always up-to-date, as it is not part of the VMCS. */
14409	GCPhys &= PAGE_BASE_GC_MASK;
14410	GCPhys += VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQual);
14411	PVM pVM = pVCpu->CTX_SUFF(pVM);
14412	Log4Func(("Linear access uAccessType=%#x GCPhys=%#RGp Off=%#x\n", uAccessType, GCPhys,
14413	VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQual)));
14414
14415	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
14416	rcStrict2 = IOMMMIOPhysHandler(pVM, pVCpu,
14417	uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ ? 0 : X86_TRAP_PF_RW,
14418	CPUMCTX2CORE(pCtx), GCPhys);
14419	Log4Func(("IOMMMIOPhysHandler returned %Rrc\n", VBOXSTRICTRC_VAL(rcStrict2)));
14420	if ( rcStrict2 == VINF_SUCCESS
14421	\|\| rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
14422	\|\| rcStrict2 == VERR_PAGE_NOT_PRESENT)
14423	{
14424	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
14425	\| HM_CHANGED_GUEST_APIC_TPR);
14426	rcStrict2 = VINF_SUCCESS;
14427	}
14428	break;
14429	}
14430
14431	default:
14432	Log4Func(("uAccessType=%#x\n", uAccessType));
14433	rcStrict2 = VINF_EM_RAW_EMULATE_INSTR;
14434	break;
14435	}
14436
14437	if (rcStrict2 != VINF_SUCCESS)
14438	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchApicAccessToR3);
14439	return rcStrict2;
14440	}
14441
14442
14443	/**
14444	* VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
14445	* VM-exit.
14446	*/
14447	HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14448	{
14449	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14450
14451	/* We should -not- get this VM-exit if the guest's debug registers were active. */
14452	if (pVmxTransient->fWasGuestDebugStateActive)
14453	{
14454	AssertMsgFailed(("Unexpected MOV DRx exit\n"));
14455	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
14456	}
14457
14458	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14459	if ( !pVCpu->hm.s.fSingleInstruction
14460	&& !pVmxTransient->fWasHyperDebugStateActive)
14461	{
14462	Assert(!DBGFIsStepping(pVCpu));
14463	Assert(pVmcsInfo->u32XcptBitmap & RT_BIT(X86_XCPT_DB));
14464
14465	/* Don't intercept MOV DRx any more. */
14466	pVmcsInfo->u32ProcCtls &= ~VMX_PROC_CTLS_MOV_DR_EXIT;
14467	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
14468	AssertRCReturn(rc, rc);
14469
14470	/* We're playing with the host CPU state here, make sure we can't preempt or longjmp. */
14471	VMMRZCallRing3Disable(pVCpu);
14472	HM_DISABLE_PREEMPT(pVCpu);
14473
14474	/* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
14475	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
14476	Assert(CPUMIsGuestDebugStateActive(pVCpu) \|\| HC_ARCH_BITS == 32);
14477
14478	HM_RESTORE_PREEMPT();
14479	VMMRZCallRing3Enable(pVCpu);
14480
14481	#ifdef VBOX_WITH_STATISTICS
14482	rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
14483	AssertRCReturn(rc, rc);
14484	if (VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_DRX_DIRECTION_WRITE)
14485	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
14486	else
14487	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
14488	#endif
14489	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
14490	return VINF_SUCCESS;
14491	}
14492
14493	/*
14494	* EMInterpretDRx[Write\|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER MSR, CS.
14495	* The EFER MSR is always up-to-date.
14496	* Update the segment registers and DR7 from the CPU.
14497	*/
14498	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
14499	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
14500	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_SREG_MASK \| CPUMCTX_EXTRN_DR7);
14501	AssertRCReturn(rc, rc);
14502	Log4Func(("cs:rip=%#04x:%#RX64\n", pCtx->cs.Sel, pCtx->rip));
14503
14504	PVM pVM = pVCpu->CTX_SUFF(pVM);
14505	if (VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_DRX_DIRECTION_WRITE)
14506	{
14507	rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pCtx),
14508	VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQual),
14509	VMX_EXIT_QUAL_DRX_GENREG(pVmxTransient->uExitQual));
14510	if (RT_SUCCESS(rc))
14511	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_DR7);
14512	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
14513	}
14514	else
14515	{
14516	rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pCtx),
14517	VMX_EXIT_QUAL_DRX_GENREG(pVmxTransient->uExitQual),
14518	VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQual));
14519	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
14520	}
14521
14522	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER);
14523	if (RT_SUCCESS(rc))
14524	{
14525	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
14526	AssertRCReturn(rc2, rc2);
14527	return VINF_SUCCESS;
14528	}
14529	return rc;
14530	}
14531
14532
14533	/**
14534	* VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
14535	* Conditional VM-exit.
14536	*/
14537	HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14538	{
14539	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14540	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
14541
14542	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
14543	VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
14544	if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
14545	{
14546	/* If event delivery causes an EPT misconfig (MMIO), go back to instruction emulation as otherwise
14547	injecting the original pending event would most likely cause the same EPT misconfig VM-exit. */
14548	if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
14549	{
14550	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
14551	return VINF_EM_RAW_INJECT_TRPM_EVENT;
14552	}
14553	}
14554	else
14555	{
14556	if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
14557	rcStrict1 = VINF_SUCCESS;
14558	return rcStrict1;
14559	}
14560
14561	/*
14562	* Get sufficent state and update the exit history entry.
14563	*/
14564	RTGCPHYS GCPhys;
14565	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14566	int rc = VMXReadVmcs64(VMX_VMCS64_RO_GUEST_PHYS_ADDR_FULL, &GCPhys);
14567	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
14568	AssertRCReturn(rc, rc);
14569
14570	VBOXSTRICTRC rcStrict;
14571	PCEMEXITREC pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
14572	EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_MMIO),
14573	pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
14574	if (!pExitRec)
14575	{
14576	/*
14577	* If we succeed, resume guest execution.
14578	* If we fail in interpreting the instruction because we couldn't get the guest physical address
14579	* of the page containing the instruction via the guest's page tables (we would invalidate the guest page
14580	* in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
14581	* weird case. See @bugref{6043}.
14582	*/
14583	PVM pVM = pVCpu->CTX_SUFF(pVM);
14584	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
14585	rcStrict = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pCtx), GCPhys, UINT32_MAX);
14586	Log4Func(("At %#RGp RIP=%#RX64 rc=%Rrc\n", GCPhys, pCtx->rip, VBOXSTRICTRC_VAL(rcStrict)));
14587	if ( rcStrict == VINF_SUCCESS
14588	\|\| rcStrict == VERR_PAGE_TABLE_NOT_PRESENT
14589	\|\| rcStrict == VERR_PAGE_NOT_PRESENT)
14590	{
14591	/* Successfully handled MMIO operation. */
14592	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
14593	\| HM_CHANGED_GUEST_APIC_TPR);
14594	rcStrict = VINF_SUCCESS;
14595	}
14596	}
14597	else
14598	{
14599	/*
14600	* Frequent exit or something needing probing. Get state and call EMHistoryExec.
14601	*/
14602	int rc2 = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
14603	AssertRCReturn(rc2, rc2);
14604
14605	Log4(("EptMisscfgExit/%u: %04x:%08RX64: %RGp -> EMHistoryExec\n",
14606	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, GCPhys));
14607
14608	rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
14609	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
14610
14611	Log4(("EptMisscfgExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
14612	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
14613	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
14614	}
14615	return VBOXSTRICTRC_TODO(rcStrict);
14616	}
14617
14618
14619	/**
14620	* VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
14621	* VM-exit.
14622	*/
14623	HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14624	{
14625	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14626	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
14627
14628	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
14629	VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
14630	if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
14631	{
14632	/* In the unlikely case that the EPT violation happened as a result of delivering an event, log it. */
14633	if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
14634	Log4Func(("EPT violation with an event pending u64IntInfo=%#RX64\n", pVCpu->hm.s.Event.u64IntInfo));
14635	}
14636	else
14637	{
14638	if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
14639	rcStrict1 = VINF_SUCCESS;
14640	return rcStrict1;
14641	}
14642
14643	RTGCPHYS GCPhys;
14644	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14645	int rc = VMXReadVmcs64(VMX_VMCS64_RO_GUEST_PHYS_ADDR_FULL, &GCPhys);
14646	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
14647	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
14648	AssertRCReturn(rc, rc);
14649
14650	/* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
14651	AssertMsg(((pVmxTransient->uExitQual >> 7) & 3) != 2, ("%#RX64", pVmxTransient->uExitQual));
14652
14653	RTGCUINT uErrorCode = 0;
14654	if (pVmxTransient->uExitQual & VMX_EXIT_QUAL_EPT_INSTR_FETCH)
14655	uErrorCode \|= X86_TRAP_PF_ID;
14656	if (pVmxTransient->uExitQual & VMX_EXIT_QUAL_EPT_DATA_WRITE)
14657	uErrorCode \|= X86_TRAP_PF_RW;
14658	if (pVmxTransient->uExitQual & VMX_EXIT_QUAL_EPT_ENTRY_PRESENT)
14659	uErrorCode \|= X86_TRAP_PF_P;
14660
14661	TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
14662
14663
14664	/* Handle the pagefault trap for the nested shadow table. */
14665	PVM pVM = pVCpu->CTX_SUFF(pVM);
14666	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
14667
14668	Log4Func(("EPT violation %#x at %#RX64 ErrorCode %#x cs:rip=%#04x:%#RX64\n", pVmxTransient->uExitQual, GCPhys, uErrorCode,
14669	pCtx->cs.Sel, pCtx->rip));
14670
14671	VBOXSTRICTRC rcStrict2 = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pCtx), GCPhys);
14672	TRPMResetTrap(pVCpu);
14673
14674	/* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
14675	if ( rcStrict2 == VINF_SUCCESS
14676	\|\| rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
14677	\|\| rcStrict2 == VERR_PAGE_NOT_PRESENT)
14678	{
14679	/* Successfully synced our nested page tables. */
14680	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
14681	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS);
14682	return VINF_SUCCESS;
14683	}
14684
14685	Log4Func(("EPT return to ring-3 rcStrict2=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict2)));
14686	return rcStrict2;
14687	}
14688
14689	/** @} */
14690
14691	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
14692	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= VM-exit exception handlers =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
14693	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
14694
14695	/**
14696	* VM-exit exception handler for \#MF (Math Fault: floating point exception).
14697	*/
14698	static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14699	{
14700	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14701	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
14702
14703	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CR0);
14704	AssertRCReturn(rc, rc);
14705
14706	if (!(pVCpu->cpum.GstCtx.cr0 & X86_CR0_NE))
14707	{
14708	/* Convert a #MF into a FERR -> IRQ 13. See @bugref{6117}. */
14709	rc = PDMIsaSetIrq(pVCpu->CTX_SUFF(pVM), 13, 1, 0 /* uTagSrc */);
14710
14711	/** @todo r=ramshankar: The Intel spec. does -not- specify that this VM-exit
14712	* provides VM-exit instruction length. If this causes problem later,
14713	* disassemble the instruction like it's done on AMD-V. */
14714	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
14715	AssertRCReturn(rc2, rc2);
14716	return rc;
14717	}
14718
14719	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
14720	pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
14721	return rc;
14722	}
14723
14724
14725	/**
14726	* VM-exit exception handler for \#BP (Breakpoint exception).
14727	*/
14728	static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14729	{
14730	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14731	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
14732
14733	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
14734	AssertRCReturn(rc, rc);
14735
14736	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
14737	rc = DBGFRZTrap03Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx));
14738	if (rc == VINF_EM_RAW_GUEST_TRAP)
14739	{
14740	rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
14741	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14742	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
14743	AssertRCReturn(rc, rc);
14744
14745	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
14746	pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
14747	}
14748
14749	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RAW_GUEST_TRAP \|\| rc == VINF_EM_DBG_BREAKPOINT);
14750	return rc;
14751	}
14752
14753
14754	/**
14755	* VM-exit exception handler for \#AC (alignment check exception).
14756	*/
14757	static int hmR0VmxExitXcptAC(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14758	{
14759	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14760
14761	/*
14762	* Re-inject it. We'll detect any nesting before getting here.
14763	*/
14764	int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
14765	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14766	AssertRCReturn(rc, rc);
14767	Assert(ASMAtomicUoReadU32(&pVmxTransient->fVmcsFieldsRead) & HMVMX_READ_EXIT_INTERRUPTION_INFO);
14768
14769	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
14770	pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
14771	return VINF_SUCCESS;
14772	}
14773
14774
14775	/**
14776	* VM-exit exception handler for \#DB (Debug exception).
14777	*/
14778	static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14779	{
14780	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14781	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
14782
14783	/*
14784	* Get the DR6-like values from the VM-exit qualification and pass it to DBGF
14785	* for processing.
14786	*/
14787	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
14788
14789	/* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
14790	uint64_t uDR6 = X86_DR6_INIT_VAL;
14791	uDR6 \|= (pVmxTransient->uExitQual & (X86_DR6_B0 \| X86_DR6_B1 \| X86_DR6_B2 \| X86_DR6_B3 \| X86_DR6_BD \| X86_DR6_BS));
14792
14793	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
14794	rc = DBGFRZTrap01Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx), uDR6, pVCpu->hm.s.fSingleInstruction);
14795	Log6Func(("rc=%Rrc\n", rc));
14796	if (rc == VINF_EM_RAW_GUEST_TRAP)
14797	{
14798	/*
14799	* The exception was for the guest. Update DR6, DR7.GD and
14800	* IA32_DEBUGCTL.LBR before forwarding it.
14801	* (See Intel spec. 27.1 "Architectural State before a VM-Exit".)
14802	*/
14803	VMMRZCallRing3Disable(pVCpu);
14804	HM_DISABLE_PREEMPT(pVCpu);
14805
14806	pCtx->dr[6] &= ~X86_DR6_B_MASK;
14807	pCtx->dr[6] \|= uDR6;
14808	if (CPUMIsGuestDebugStateActive(pVCpu))
14809	ASMSetDR6(pCtx->dr[6]);
14810
14811	HM_RESTORE_PREEMPT();
14812	VMMRZCallRing3Enable(pVCpu);
14813
14814	rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_DR7);
14815	AssertRCReturn(rc, rc);
14816
14817	/* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
14818	pCtx->dr[7] &= ~X86_DR7_GD;
14819
14820	/* Paranoia. */
14821	pCtx->dr[7] &= ~X86_DR7_RAZ_MASK;
14822	pCtx->dr[7] \|= X86_DR7_RA1_MASK;
14823
14824	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)pCtx->dr[7]);
14825	AssertRCReturn(rc, rc);
14826
14827	/*
14828	* Raise #DB in the guest.
14829	*
14830	* It is important to reflect exactly what the VM-exit gave us (preserving the
14831	* interruption-type) rather than use hmR0VmxSetPendingXcptDB() as the #DB could've
14832	* been raised while executing ICEBP (INT1) and not the regular #DB. Thus it may
14833	* trigger different handling in the CPU (like skipping DPL checks), see @bugref{6398}.
14834	*
14835	* Intel re-documented ICEBP/INT1 on May 2018 previously documented as part of
14836	* Intel 386, see Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
14837	*/
14838	rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
14839	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14840	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
14841	AssertRCReturn(rc, rc);
14842	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
14843	pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
14844	return VINF_SUCCESS;
14845	}
14846
14847	/*
14848	* Not a guest trap, must be a hypervisor related debug event then.
14849	* Update DR6 in case someone is interested in it.
14850	*/
14851	AssertMsg(rc == VINF_EM_DBG_STEPPED \|\| rc == VINF_EM_DBG_BREAKPOINT, ("%Rrc\n", rc));
14852	AssertReturn(pVmxTransient->fWasHyperDebugStateActive, VERR_HM_IPE_5);
14853	CPUMSetHyperDR6(pVCpu, uDR6);
14854
14855	return rc;
14856	}
14857
14858
14859	/**
14860	* Hacks its way around the lovely mesa driver's backdoor accesses.
14861	*
14862	* @sa hmR0SvmHandleMesaDrvGp
14863	*/
14864	static int hmR0VmxHandleMesaDrvGp(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PCPUMCTX pCtx)
14865	{
14866	LogFunc(("cs:rip=%#04x:%#RX64 rcx=%#RX64 rbx=%#RX64\n", pCtx->cs.Sel, pCtx->rip, pCtx->rcx, pCtx->rbx));
14867	RT_NOREF(pCtx);
14868
14869	/* For now we'll just skip the instruction. */
14870	return hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
14871	}
14872
14873
14874	/**
14875	* Checks if the \#GP'ing instruction is the mesa driver doing it's lovely
14876	* backdoor logging w/o checking what it is running inside.
14877	*
14878	* This recognizes an "IN EAX,DX" instruction executed in flat ring-3, with the
14879	* backdoor port and magic numbers loaded in registers.
14880	*
14881	* @returns true if it is, false if it isn't.
14882	* @sa hmR0SvmIsMesaDrvGp
14883	*/
14884	DECLINLINE(bool) hmR0VmxIsMesaDrvGp(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PCPUMCTX pCtx)
14885	{
14886	/* 0xed: IN eAX,dx */
14887	uint8_t abInstr[1];
14888	if (pVmxTransient->cbInstr != sizeof(abInstr))
14889	return false;
14890
14891	/* Check that it is #GP(0). */
14892	if (pVmxTransient->uExitIntErrorCode != 0)
14893	return false;
14894
14895	/* Check magic and port. */
14896	Assert(!(pCtx->fExtrn & (CPUMCTX_EXTRN_RAX \| CPUMCTX_EXTRN_RDX \| CPUMCTX_EXTRN_RCX)));
14897	/Log(("hmR0VmxIsMesaDrvGp: rax=%RX64 rdx=%RX64\n", pCtx->rax, pCtx->rdx));/
14898	if (pCtx->rax != UINT32_C(0x564d5868))
14899	return false;
14900	if (pCtx->dx != UINT32_C(0x5658))
14901	return false;
14902
14903	/* Flat ring-3 CS. */
14904	AssertCompile(HMVMX_CPUMCTX_EXTRN_ALL & CPUMCTX_EXTRN_CS);
14905	Assert(!(pCtx->fExtrn & CPUMCTX_EXTRN_CS));
14906	/Log(("hmR0VmxIsMesaDrvGp: cs.Attr.n.u2Dpl=%d base=%Rx64\n", pCtx->cs.Attr.n.u2Dpl, pCtx->cs.u64Base));/
14907	if (pCtx->cs.Attr.n.u2Dpl != 3)
14908	return false;
14909	if (pCtx->cs.u64Base != 0)
14910	return false;
14911
14912	/* Check opcode. */
14913	AssertCompile(HMVMX_CPUMCTX_EXTRN_ALL & CPUMCTX_EXTRN_RIP);
14914	Assert(!(pCtx->fExtrn & CPUMCTX_EXTRN_RIP));
14915	int rc = PGMPhysSimpleReadGCPtr(pVCpu, abInstr, pCtx->rip, sizeof(abInstr));
14916	/Log(("hmR0VmxIsMesaDrvGp: PGMPhysSimpleReadGCPtr -> %Rrc %#x\n", rc, abInstr[0]));/
14917	if (RT_FAILURE(rc))
14918	return false;
14919	if (abInstr[0] != 0xed)
14920	return false;
14921
14922	return true;
14923	}
14924
14925
14926	/**
14927	* VM-exit exception handler for \#GP (General-protection exception).
14928	*
14929	* @remarks Requires pVmxTransient->uExitIntInfo to be up-to-date.
14930	*/
14931	static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14932	{
14933	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14934	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
14935
14936	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
14937	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14938	if (pVmcsInfo->RealMode.fRealOnV86Active)
14939	{ /* likely */ }
14940	else
14941	{
14942	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
14943	Assert(pVCpu->hm.s.fUsingDebugLoop \|\| pVCpu->hm.s.fTrapXcptGpForLovelyMesaDrv);
14944	#endif
14945	/* If the guest is not in real-mode or we have unrestricted guest execution support, reflect #GP to the guest. */
14946	int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
14947	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
14948	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14949	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
14950	AssertRCReturn(rc, rc);
14951	Log4Func(("Gst: cs:rip=%#04x:%#RX64 ErrorCode=%#x cr0=%#RX64 cpl=%u tr=%#04x\n", pCtx->cs.Sel, pCtx->rip,
14952	pVmxTransient->uExitIntErrorCode, pCtx->cr0, CPUMGetGuestCPL(pVCpu), pCtx->tr.Sel));
14953
14954	if ( !pVCpu->hm.s.fTrapXcptGpForLovelyMesaDrv
14955	\|\| !hmR0VmxIsMesaDrvGp(pVCpu, pVmxTransient, pCtx))
14956	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
14957	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
14958	else
14959	rc = hmR0VmxHandleMesaDrvGp(pVCpu, pVmxTransient, pCtx);
14960	return rc;
14961	}
14962
14963	Assert(CPUMIsGuestInRealModeEx(pCtx));
14964	Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
14965
14966	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
14967	AssertRCReturn(rc, rc);
14968
14969	VBOXSTRICTRC rcStrict = IEMExecOne(pVCpu);
14970	if (rcStrict == VINF_SUCCESS)
14971	{
14972	if (!CPUMIsGuestInRealModeEx(pCtx))
14973	{
14974	/*
14975	* The guest is no longer in real-mode, check if we can continue executing the
14976	* guest using hardware-assisted VMX. Otherwise, fall back to emulation.
14977	*/
14978	pVmcsInfo->RealMode.fRealOnV86Active = false;
14979	if (HMCanExecuteVmxGuest(pVCpu, pCtx))
14980	{
14981	Log4Func(("Mode changed but guest still suitable for executing using hardware-assisted VMX\n"));
14982	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
14983	}
14984	else
14985	{
14986	Log4Func(("Mode changed -> VINF_EM_RESCHEDULE\n"));
14987	rcStrict = VINF_EM_RESCHEDULE;
14988	}
14989	}
14990	else
14991	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
14992	}
14993	else if (rcStrict == VINF_IEM_RAISED_XCPT)
14994	{
14995	rcStrict = VINF_SUCCESS;
14996	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
14997	}
14998	return VBOXSTRICTRC_VAL(rcStrict);
14999	}
15000
15001
15002	/**
15003	* VM-exit exception handler wrapper for generic exceptions. Simply re-injects
15004	* the exception reported in the VMX transient structure back into the VM.
15005	*
15006	* @remarks Requires uExitIntInfo in the VMX transient structure to be
15007	* up-to-date.
15008	*/
15009	static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15010	{
15011	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15012	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
15013	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
15014	AssertMsg(pVCpu->hm.s.fUsingDebugLoop \|\| pVmcsInfo->RealMode.fRealOnV86Active,
15015	("uVector=%#x u32XcptBitmap=%#X32\n",
15016	VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo), pVmcsInfo->u32XcptBitmap));
15017	NOREF(pVmcsInfo);
15018	#endif
15019
15020	/* Re-inject the exception into the guest. This cannot be a double-fault condition which would have been handled in
15021	hmR0VmxCheckExitDueToEventDelivery(). */
15022	int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
15023	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15024	AssertRCReturn(rc, rc);
15025	Assert(ASMAtomicUoReadU32(&pVmxTransient->fVmcsFieldsRead) & HMVMX_READ_EXIT_INTERRUPTION_INFO);
15026
15027	#ifdef DEBUG_ramshankar
15028	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
15029	Log(("hmR0VmxExitXcptGeneric: Reinjecting Xcpt. uVector=%#x cs:rip=%#04x:%#RX64\n",
15030	VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo), pCtx->cs.Sel, pCtx->rip));
15031	#endif
15032
15033	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
15034	pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
15035	return VINF_SUCCESS;
15036	}
15037
15038
15039	/**
15040	* VM-exit exception handler for \#PF (Page-fault exception).
15041	*/
15042	static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15043	{
15044	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15045	PVM pVM = pVCpu->CTX_SUFF(pVM);
15046	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15047	rc \|= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
15048	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
15049	AssertRCReturn(rc, rc);
15050
15051	if (!pVM->hm.s.fNestedPaging)
15052	{ /* likely */ }
15053	else
15054	{
15055	#if !defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) && !defined(HMVMX_ALWAYS_TRAP_PF)
15056	Assert(pVCpu->hm.s.fUsingDebugLoop);
15057	#endif
15058	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
15059	if (RT_LIKELY(!pVmxTransient->fVectoringDoublePF))
15060	{
15061	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), 0 /* cbInstr */,
15062	pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQual);
15063	}
15064	else
15065	{
15066	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
15067	hmR0VmxSetPendingXcptDF(pVCpu);
15068	Log4Func(("Pending #DF due to vectoring #PF w/ NestedPaging\n"));
15069	}
15070	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
15071	return rc;
15072	}
15073
15074	/* If it's a vectoring #PF, emulate injecting the original event injection as PGMTrap0eHandler() is incapable
15075	of differentiating between instruction emulation and event injection that caused a #PF. See @bugref{6607}. */
15076	if (pVmxTransient->fVectoringPF)
15077	{
15078	Assert(pVCpu->hm.s.Event.fPending);
15079	return VINF_EM_RAW_INJECT_TRPM_EVENT;
15080	}
15081
15082	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
15083	rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
15084	AssertRCReturn(rc, rc);
15085
15086	Log4Func(("#PF: cr2=%#RX64 cs:rip=%#04x:%#RX64 uErrCode %#RX32 cr3=%#RX64\n", pVmxTransient->uExitQual, pCtx->cs.Sel,
15087	pCtx->rip, pVmxTransient->uExitIntErrorCode, pCtx->cr3));
15088
15089	TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQual, (RTGCUINT)pVmxTransient->uExitIntErrorCode);
15090	rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntErrorCode, CPUMCTX2CORE(pCtx), (RTGCPTR)pVmxTransient->uExitQual);
15091
15092	Log4Func(("#PF: rc=%Rrc\n", rc));
15093	if (rc == VINF_SUCCESS)
15094	{
15095	/*
15096	* This is typically a shadow page table sync or a MMIO instruction. But we may have
15097	* emulated something like LTR or a far jump. Any part of the CPU context may have changed.
15098	*/
15099	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
15100	TRPMResetTrap(pVCpu);
15101	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
15102	return rc;
15103	}
15104
15105	if (rc == VINF_EM_RAW_GUEST_TRAP)
15106	{
15107	if (!pVmxTransient->fVectoringDoublePF)
15108	{
15109	/* It's a guest page fault and needs to be reflected to the guest. */
15110	uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
15111	TRPMResetTrap(pVCpu);
15112	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
15113	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), 0 /* cbInstr */,
15114	uGstErrorCode, pVmxTransient->uExitQual);
15115	}
15116	else
15117	{
15118	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
15119	TRPMResetTrap(pVCpu);
15120	pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF to replace it with #DF. */
15121	hmR0VmxSetPendingXcptDF(pVCpu);
15122	Log4Func(("#PF: Pending #DF due to vectoring #PF\n"));
15123	}
15124
15125	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
15126	return VINF_SUCCESS;
15127	}
15128
15129	TRPMResetTrap(pVCpu);
15130	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
15131	return rc;
15132	}
15133
15134	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
15135	/** @name VMX instruction handlers.
15136	* @{
15137	*/
15138	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
15139	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VMX instructions VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
15140	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
15141
15142	/**
15143	* VM-exit handler for VMCLEAR (VMX_EXIT_VMCLEAR). Unconditional VM-exit.
15144	*/
15145	HMVMX_EXIT_DECL hmR0VmxExitVmclear(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15146	{
15147	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15148
15149	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15150	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
15151	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
15152	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
15153	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15154	AssertRCReturn(rc, rc);
15155
15156	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15157
15158	VMXVEXITINFO ExitInfo;
15159	RT_ZERO(ExitInfo);
15160	ExitInfo.uReason = pVmxTransient->uExitReason;
15161	ExitInfo.u64Qual = pVmxTransient->uExitQual;
15162	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
15163	ExitInfo.cbInstr = pVmxTransient->cbInstr;
15164	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
15165
15166	VBOXSTRICTRC rcStrict = IEMExecDecodedVmclear(pVCpu, &ExitInfo);
15167	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15168	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_HWVIRT);
15169	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15170	{
15171	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15172	rcStrict = VINF_SUCCESS;
15173	}
15174	return rcStrict;
15175	}
15176
15177
15178	/**
15179	* VM-exit handler for VMLAUNCH (VMX_EXIT_VMLAUNCH). Unconditional VM-exit.
15180	*/
15181	HMVMX_EXIT_DECL hmR0VmxExitVmlaunch(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15182	{
15183	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15184
15185	/* Import the entire VMCS state for now as we would be switching VMCS on successful VMLAUNCH,
15186	otherwise we could import just IEM_CPUMCTX_EXTRN_VMX_VMENTRY_MASK. */
15187	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15188	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
15189	AssertRCReturn(rc, rc);
15190
15191	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15192
15193	VBOXSTRICTRC rcStrict = IEMExecDecodedVmlaunchVmresume(pVCpu, pVmxTransient->cbInstr, VMXINSTRID_VMLAUNCH);
15194	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15195	{
15196	rcStrict = VINF_VMX_VMLAUNCH_VMRESUME;
15197	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
15198	}
15199	Assert(rcStrict != VINF_IEM_RAISED_XCPT);
15200	return rcStrict;
15201	}
15202
15203
15204	/**
15205	* VM-exit handler for VMPTRLD (VMX_EXIT_VMPTRLD). Unconditional VM-exit.
15206	*/
15207	HMVMX_EXIT_DECL hmR0VmxExitVmptrld(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15208	{
15209	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15210
15211	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15212	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
15213	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
15214	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
15215	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15216	AssertRCReturn(rc, rc);
15217
15218	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15219
15220	VMXVEXITINFO ExitInfo;
15221	RT_ZERO(ExitInfo);
15222	ExitInfo.uReason = pVmxTransient->uExitReason;
15223	ExitInfo.u64Qual = pVmxTransient->uExitQual;
15224	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
15225	ExitInfo.cbInstr = pVmxTransient->cbInstr;
15226	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
15227
15228	VBOXSTRICTRC rcStrict = IEMExecDecodedVmptrld(pVCpu, &ExitInfo);
15229	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15230	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_HWVIRT);
15231	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15232	{
15233	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15234	rcStrict = VINF_SUCCESS;
15235	}
15236	return rcStrict;
15237	}
15238
15239
15240	/**
15241	* VM-exit handler for VMPTRST (VMX_EXIT_VMPTRST). Unconditional VM-exit.
15242	*/
15243	HMVMX_EXIT_DECL hmR0VmxExitVmptrst(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15244	{
15245	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15246
15247	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15248	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
15249	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
15250	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
15251	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15252	AssertRCReturn(rc, rc);
15253
15254	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15255
15256	VMXVEXITINFO ExitInfo;
15257	RT_ZERO(ExitInfo);
15258	ExitInfo.uReason = pVmxTransient->uExitReason;
15259	ExitInfo.u64Qual = pVmxTransient->uExitQual;
15260	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
15261	ExitInfo.cbInstr = pVmxTransient->cbInstr;
15262	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_WRITE, &ExitInfo.GCPtrEffAddr);
15263
15264	VBOXSTRICTRC rcStrict = IEMExecDecodedVmptrst(pVCpu, &ExitInfo);
15265	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15266	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
15267	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15268	{
15269	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15270	rcStrict = VINF_SUCCESS;
15271	}
15272	return rcStrict;
15273	}
15274
15275
15276	/**
15277	* VM-exit handler for VMREAD (VMX_EXIT_VMREAD). Unconditional VM-exit.
15278	*/
15279	HMVMX_EXIT_DECL hmR0VmxExitVmread(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15280	{
15281	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15282
15283	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15284	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
15285	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
15286	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
15287	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15288	AssertRCReturn(rc, rc);
15289
15290	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15291
15292	VMXVEXITINFO ExitInfo;
15293	RT_ZERO(ExitInfo);
15294	ExitInfo.uReason = pVmxTransient->uExitReason;
15295	ExitInfo.u64Qual = pVmxTransient->uExitQual;
15296	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
15297	ExitInfo.cbInstr = pVmxTransient->cbInstr;
15298	if (!ExitInfo.InstrInfo.VmreadVmwrite.fIsRegOperand)
15299	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_WRITE, &ExitInfo.GCPtrEffAddr);
15300
15301	VBOXSTRICTRC rcStrict = IEMExecDecodedVmread(pVCpu, &ExitInfo);
15302	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15303	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
15304	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15305	{
15306	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15307	rcStrict = VINF_SUCCESS;
15308	}
15309	return rcStrict;
15310	}
15311
15312
15313	/**
15314	* VM-exit handler for VMRESUME (VMX_EXIT_VMRESUME). Unconditional VM-exit.
15315	*/
15316	HMVMX_EXIT_DECL hmR0VmxExitVmresume(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15317	{
15318	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15319
15320	/* Import the entire VMCS state for now as we would be switching VMCS on successful VMRESUME,
15321	otherwise we could import just IEM_CPUMCTX_EXTRN_VMX_VMENTRY_MASK. */
15322	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15323	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
15324	AssertRCReturn(rc, rc);
15325
15326	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15327
15328	VBOXSTRICTRC rcStrict = IEMExecDecodedVmlaunchVmresume(pVCpu, pVmxTransient->cbInstr, VMXINSTRID_VMRESUME);
15329	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15330	{
15331	rcStrict = VINF_VMX_VMLAUNCH_VMRESUME;
15332	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
15333	}
15334	Assert(rcStrict != VINF_IEM_RAISED_XCPT);
15335	return rcStrict;
15336	}
15337
15338
15339	/**
15340	* VM-exit handler for VMWRITE (VMX_EXIT_VMWRITE). Unconditional VM-exit.
15341	*/
15342	HMVMX_EXIT_DECL hmR0VmxExitVmwrite(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15343	{
15344	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15345
15346	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15347	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
15348	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
15349	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
15350	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15351	AssertRCReturn(rc, rc);
15352
15353	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15354
15355	VMXVEXITINFO ExitInfo;
15356	RT_ZERO(ExitInfo);
15357	ExitInfo.uReason = pVmxTransient->uExitReason;
15358	ExitInfo.u64Qual = pVmxTransient->uExitQual;
15359	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
15360	ExitInfo.cbInstr = pVmxTransient->cbInstr;
15361	if (!ExitInfo.InstrInfo.VmreadVmwrite.fIsRegOperand)
15362	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
15363
15364	VBOXSTRICTRC rcStrict = IEMExecDecodedVmwrite(pVCpu, &ExitInfo);
15365	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15366	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_HWVIRT);
15367	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15368	{
15369	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15370	rcStrict = VINF_SUCCESS;
15371	}
15372	return rcStrict;
15373	}
15374
15375
15376	/**
15377	* VM-exit handler for VMXOFF (VMX_EXIT_VMXOFF). Unconditional VM-exit.
15378	*/
15379	HMVMX_EXIT_DECL hmR0VmxExitVmxoff(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15380	{
15381	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15382
15383	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15384	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CR4
15385	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
15386	AssertRCReturn(rc, rc);
15387
15388	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15389
15390	VBOXSTRICTRC rcStrict = IEMExecDecodedVmxoff(pVCpu, pVmxTransient->cbInstr);
15391	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15392	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_HWVIRT);
15393	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15394	{
15395	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15396	rcStrict = VINF_SUCCESS;
15397	}
15398	return rcStrict;
15399	}
15400
15401
15402	/**
15403	* VM-exit handler for VMXON (VMX_EXIT_VMXON). Unconditional VM-exit.
15404	*/
15405	HMVMX_EXIT_DECL hmR0VmxExitVmxon(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15406	{
15407	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15408
15409	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15410	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
15411	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
15412	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
15413	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15414	AssertRCReturn(rc, rc);
15415
15416	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15417
15418	VMXVEXITINFO ExitInfo;
15419	RT_ZERO(ExitInfo);
15420	ExitInfo.uReason = pVmxTransient->uExitReason;
15421	ExitInfo.u64Qual = pVmxTransient->uExitQual;
15422	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
15423	ExitInfo.cbInstr = pVmxTransient->cbInstr;
15424	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
15425
15426	VBOXSTRICTRC rcStrict = IEMExecDecodedVmxon(pVCpu, &ExitInfo);
15427	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15428	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_HWVIRT);
15429	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15430	{
15431	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15432	rcStrict = VINF_SUCCESS;
15433	}
15434	return rcStrict;
15435	}
15436
15437	/** @} */
15438	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
15439

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 78229

Download in other formats: