HMVMXR0.cpp@ 78232

Last change on this file since 78232 was 78232, checked in by vboxsync, 6 years ago
VMM/HMVMXR0: Nested VMX: bugref:9180 Build fix.
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 641.9 KB

Line
1	/* $Id: HMVMXR0.cpp 78232 2019-04-20 22:27:09Z vboxsync $ */
2	/** @file
3	* HM VMX (Intel VT-x) - Host Context Ring-0.
4	*/
5
6	/*
7	* Copyright (C) 2012-2019 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18
19	/*********************************************************************************************************************************
20	* Header Files *
21	*********************************************************************************************************************************/
22	#define LOG_GROUP LOG_GROUP_HM
23	#define VMCPU_INCL_CPUM_GST_CTX
24	#include <iprt/x86.h>
25	#include <iprt/asm-amd64-x86.h>
26	#include <iprt/thread.h>
27
28	#include <VBox/vmm/pdmapi.h>
29	#include <VBox/vmm/dbgf.h>
30	#include <VBox/vmm/iem.h>
31	#include <VBox/vmm/iom.h>
32	#include <VBox/vmm/selm.h>
33	#include <VBox/vmm/tm.h>
34	#include <VBox/vmm/em.h>
35	#include <VBox/vmm/gim.h>
36	#include <VBox/vmm/apic.h>
37	#ifdef VBOX_WITH_REM
38	# include <VBox/vmm/rem.h>
39	#endif
40	#include "HMInternal.h"
41	#include <VBox/vmm/vm.h>
42	#include <VBox/vmm/hmvmxinline.h>
43	#include "HMVMXR0.h"
44	#include "dtrace/VBoxVMM.h"
45
46	# define HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
47	#ifdef DEBUG_ramshankar
48	# define HMVMX_ALWAYS_SAVE_GUEST_RFLAGS
49	# define HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE
50	# define HMVMX_ALWAYS_CHECK_GUEST_STATE
51	# define HMVMX_ALWAYS_TRAP_ALL_XCPTS
52	# define HMVMX_ALWAYS_TRAP_PF
53	# define HMVMX_ALWAYS_FLUSH_TLB
54	# define HMVMX_ALWAYS_SWAP_EFER
55	#endif
56
57
58	/*********************************************************************************************************************************
59	* Defined Constants And Macros *
60	*********************************************************************************************************************************/
61	/** Use the function table. */
62	#define HMVMX_USE_FUNCTION_TABLE
63
64	/** Determine which tagged-TLB flush handler to use. */
65	#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
66	#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
67	#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
68	#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
69
70	/** @name HMVMX_READ_XXX
71	* Flags to skip redundant reads of some common VMCS fields that are not part of
72	* the guest-CPU or VCPU state but are needed while handling VM-exits.
73	*/
74	#define HMVMX_READ_IDT_VECTORING_INFO RT_BIT_32(0)
75	#define HMVMX_READ_IDT_VECTORING_ERROR_CODE RT_BIT_32(1)
76	#define HMVMX_READ_EXIT_QUALIFICATION RT_BIT_32(2)
77	#define HMVMX_READ_EXIT_INSTR_LEN RT_BIT_32(3)
78	#define HMVMX_READ_EXIT_INTERRUPTION_INFO RT_BIT_32(4)
79	#define HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE RT_BIT_32(5)
80	#define HMVMX_READ_EXIT_INSTR_INFO RT_BIT_32(6)
81	#define HMVMX_READ_GUEST_LINEAR_ADDR RT_BIT_32(7)
82	/** @} */
83
84	/**
85	* Subset of the guest-CPU state that is kept by VMX R0 code while executing the
86	* guest using hardware-assisted VMX.
87	*
88	* This excludes state like GPRs (other than RSP) which are always are
89	* swapped and restored across the world-switch and also registers like EFER,
90	* MSR which cannot be modified by the guest without causing a VM-exit.
91	*/
92	#define HMVMX_CPUMCTX_EXTRN_ALL ( CPUMCTX_EXTRN_RIP \
93	\| CPUMCTX_EXTRN_RFLAGS \
94	\| CPUMCTX_EXTRN_RSP \
95	\| CPUMCTX_EXTRN_SREG_MASK \
96	\| CPUMCTX_EXTRN_TABLE_MASK \
97	\| CPUMCTX_EXTRN_KERNEL_GS_BASE \
98	\| CPUMCTX_EXTRN_SYSCALL_MSRS \
99	\| CPUMCTX_EXTRN_SYSENTER_MSRS \
100	\| CPUMCTX_EXTRN_TSC_AUX \
101	\| CPUMCTX_EXTRN_OTHER_MSRS \
102	\| CPUMCTX_EXTRN_CR0 \
103	\| CPUMCTX_EXTRN_CR3 \
104	\| CPUMCTX_EXTRN_CR4 \
105	\| CPUMCTX_EXTRN_DR7 \
106	\| CPUMCTX_EXTRN_HM_VMX_MASK)
107
108	/**
109	* Exception bitmap mask for real-mode guests (real-on-v86).
110	*
111	* We need to intercept all exceptions manually except:
112	* - \#AC and \#DB are always intercepted to prevent the CPU from deadlocking
113	* due to bugs in Intel CPUs.
114	* - \#PF need not be intercepted even in real-mode if we have nested paging
115	* support.
116	*/
117	#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) /* always: \| RT_BIT(X86_XCPT_DB) */ \| RT_BIT(X86_XCPT_NMI) \
118	\| RT_BIT(X86_XCPT_BP) \| RT_BIT(X86_XCPT_OF) \| RT_BIT(X86_XCPT_BR) \
119	\| RT_BIT(X86_XCPT_UD) \| RT_BIT(X86_XCPT_NM) \| RT_BIT(X86_XCPT_DF) \
120	\| RT_BIT(X86_XCPT_CO_SEG_OVERRUN) \| RT_BIT(X86_XCPT_TS) \| RT_BIT(X86_XCPT_NP) \
121	\| RT_BIT(X86_XCPT_SS) \| RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
122	\| RT_BIT(X86_XCPT_MF) /* always: \| RT_BIT(X86_XCPT_AC) */ \| RT_BIT(X86_XCPT_MC) \
123	\| RT_BIT(X86_XCPT_XF))
124
125	/** Maximum VM-instruction error number. */
126	#define HMVMX_INSTR_ERROR_MAX 28
127
128	/** Profiling macro. */
129	#ifdef HM_PROFILE_EXIT_DISPATCH
130	# define HMVMX_START_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed)
131	# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitDispatch, ed)
132	#else
133	# define HMVMX_START_EXIT_DISPATCH_PROF() do { } while (0)
134	# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
135	#endif
136
137	/** Assert that preemption is disabled or covered by thread-context hooks. */
138	#define HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu) Assert( VMMR0ThreadCtxHookIsEnabled((a_pVCpu)) \
139	\|\| !RTThreadPreemptIsEnabled(NIL_RTTHREAD))
140
141	/** Assert that we haven't migrated CPUs when thread-context hooks are not
142	* used. */
143	#define HMVMX_ASSERT_CPU_SAFE(a_pVCpu) AssertMsg( VMMR0ThreadCtxHookIsEnabled((a_pVCpu)) \
144	\|\| (a_pVCpu)->hm.s.idEnteredCpu == RTMpCpuId(), \
145	("Illegal migration! Entered on CPU %u Current %u\n", \
146	(a_pVCpu)->hm.s.idEnteredCpu, RTMpCpuId()))
147
148	/** Asserts that the given CPUMCTX_EXTRN_XXX bits are present in the guest-CPU
149	* context. */
150	#define HMVMX_CPUMCTX_ASSERT(a_pVCpu, a_fExtrnMbz) AssertMsg(!((a_pVCpu)->cpum.GstCtx.fExtrn & (a_fExtrnMbz)), \
151	("fExtrn=%#RX64 fExtrnMbz=%#RX64\n", \
152	(a_pVCpu)->cpum.GstCtx.fExtrn, (a_fExtrnMbz)))
153
154	/** Helper macro for VM-exit handlers called unexpectedly. */
155	#define HMVMX_UNEXPECTED_EXIT_RET(a_pVCpu, a_pVmxTransient) \
156	do { \
157	(a_pVCpu)->hm.s.u32HMError = (a_pVmxTransient)->uExitReason; \
158	return VERR_VMX_UNEXPECTED_EXIT; \
159	} while (0)
160
161	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
162	/** Macro that does the necessary privilege checks and intercepted VM-exits for
163	* guests that attempted to execute a VMX instruction. */
164	# define HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(a_pVCpu, a_uExitReason) \
165	do \
166	{ \
167	VBOXSTRICTRC rcStrictTmp = hmR0VmxCheckExitDueToVmxInstr((a_pVCpu), (a_uExitReason)); \
168	if (rcStrictTmp == VINF_SUCCESS) \
169	{ /* likely */ } \
170	else if (rcStrictTmp == VINF_HM_PENDING_XCPT) \
171	{ \
172	Assert((a_pVCpu)->hm.s.Event.fPending); \
173	Log4Func(("Privilege checks failed -> %#x\n", VMX_ENTRY_INT_INFO_VECTOR((a_pVCpu)->hm.s.Event.u64IntInfo))); \
174	return VINF_SUCCESS; \
175	} \
176	else \
177	{ \
178	int rcTmp = VBOXSTRICTRC_VAL(rcStrictTmp); \
179	AssertMsgFailedReturn(("Unexpected failure. rc=%Rrc", rcTmp), rcTmp); \
180	} \
181	} while (0)
182
183	/** Macro that decodes a memory operand for an instruction VM-exit. */
184	# define HMVMX_DECODE_MEM_OPERAND(a_pVCpu, a_uExitInstrInfo, a_uExitQual, a_enmMemAccess, a_pGCPtrEffAddr) \
185	do \
186	{ \
187	VBOXSTRICTRC rcStrictTmp = hmR0VmxDecodeMemOperand((a_pVCpu), (a_uExitInstrInfo), (a_uExitQual), (a_enmMemAccess), \
188	(a_pGCPtrEffAddr)); \
189	if (rcStrictTmp == VINF_SUCCESS) \
190	{ /* likely */ } \
191	else if (rcStrictTmp == VINF_HM_PENDING_XCPT) \
192	{ \
193	uint8_t const uXcptTmp = VMX_ENTRY_INT_INFO_VECTOR((a_pVCpu)->hm.s.Event.u64IntInfo); \
194	Log4Func(("Memory operand decoding failed, raising xcpt %#x\n", uXcptTmp)); \
195	NOREF(uXcptTmp); \
196	return VINF_SUCCESS; \
197	} \
198	else \
199	{ \
200	Log4Func(("hmR0VmxDecodeMemOperand failed. rc=%Rrc\n", VBOXSTRICTRC_VAL(rcStrictTmp))); \
201	return rcStrictTmp; \
202	} \
203	} while (0)
204
205	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
206
207
208	/*********************************************************************************************************************************
209	* Structures and Typedefs *
210	*********************************************************************************************************************************/
211	/**
212	* VMX transient state.
213	*
214	* A state structure for holding miscellaneous information across
215	* VMX non-root operation and restored after the transition.
216	*/
217	typedef struct VMXTRANSIENT
218	{
219	/** The host's rflags/eflags. */
220	RTCCUINTREG fEFlags;
221	#if HC_ARCH_BITS == 32
222	uint32_t u32Alignment0;
223	#endif
224	/** The guest's TPR value used for TPR shadowing. */
225	uint8_t u8GuestTpr;
226	/** Alignment. */
227	uint8_t abAlignment0[7];
228
229	/** The basic VM-exit reason. */
230	uint16_t uExitReason;
231	/** Alignment. */
232	uint16_t u16Alignment0;
233	/** The VM-exit interruption error code. */
234	uint32_t uExitIntErrorCode;
235	/** The VM-exit exit code qualification. */
236	uint64_t uExitQual;
237	/** The Guest-linear address. */
238	uint64_t uGuestLinearAddr;
239
240	/** The VM-exit interruption-information field. */
241	uint32_t uExitIntInfo;
242	/** The VM-exit instruction-length field. */
243	uint32_t cbInstr;
244	/** The VM-exit instruction-information field. */
245	VMXEXITINSTRINFO ExitInstrInfo;
246	/** Whether the VM-entry failed or not. */
247	bool fVMEntryFailed;
248	/** Whether we are currently executing a nested-guest. */
249	bool fIsNestedGuest;
250	/** Alignment. */
251	uint8_t abAlignment1[2];
252
253	/** The VM-entry interruption-information field. */
254	uint32_t uEntryIntInfo;
255	/** The VM-entry exception error code field. */
256	uint32_t uEntryXcptErrorCode;
257	/** The VM-entry instruction length field. */
258	uint32_t cbEntryInstr;
259
260	/** IDT-vectoring information field. */
261	uint32_t uIdtVectoringInfo;
262	/** IDT-vectoring error code. */
263	uint32_t uIdtVectoringErrorCode;
264
265	/** Mask of currently read VMCS fields; HMVMX_READ_XXX. */
266	uint32_t fVmcsFieldsRead;
267
268	/** Whether the guest debug state was active at the time of VM-exit. */
269	bool fWasGuestDebugStateActive;
270	/** Whether the hyper debug state was active at the time of VM-exit. */
271	bool fWasHyperDebugStateActive;
272	/** Whether TSC-offsetting and VMX-preemption timer was updated before VM-entry. */
273	bool fUpdatedTscOffsettingAndPreemptTimer;
274	/** Whether the VM-exit was caused by a page-fault during delivery of a
275	* contributory exception or a page-fault. */
276	bool fVectoringDoublePF;
277	/** Whether the VM-exit was caused by a page-fault during delivery of an
278	* external interrupt or NMI. */
279	bool fVectoringPF;
280	bool afAlignment0[3];
281
282	/** The VMCS info. object. */
283	PVMXVMCSINFO pVmcsInfo;
284	} VMXTRANSIENT;
285	AssertCompileMemberAlignment(VMXTRANSIENT, uExitReason, sizeof(uint64_t));
286	AssertCompileMemberAlignment(VMXTRANSIENT, uExitIntInfo, sizeof(uint64_t));
287	AssertCompileMemberAlignment(VMXTRANSIENT, uEntryIntInfo, sizeof(uint64_t));
288	AssertCompileMemberAlignment(VMXTRANSIENT, fWasGuestDebugStateActive, sizeof(uint64_t));
289	AssertCompileMemberAlignment(VMXTRANSIENT, pVmcsInfo, sizeof(uint64_t));
290	AssertCompileMemberSize(VMXTRANSIENT, ExitInstrInfo, sizeof(uint32_t));
291	/** Pointer to VMX transient state. */
292	typedef VMXTRANSIENT *PVMXTRANSIENT;
293
294	/**
295	* Memory operand read or write access.
296	*/
297	typedef enum VMXMEMACCESS
298	{
299	VMXMEMACCESS_READ = 0,
300	VMXMEMACCESS_WRITE = 1
301	} VMXMEMACCESS;
302
303	/**
304	* VMX VM-exit handler.
305	*
306	* @returns Strict VBox status code (i.e. informational status codes too).
307	* @param pVCpu The cross context virtual CPU structure.
308	* @param pVmxTransient The VMX-transient structure.
309	*/
310	#ifndef HMVMX_USE_FUNCTION_TABLE
311	typedef VBOXSTRICTRC FNVMXEXITHANDLER(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
312	#else
313	typedef DECLCALLBACK(VBOXSTRICTRC) FNVMXEXITHANDLER(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
314	/** Pointer to VM-exit handler. */
315	typedef FNVMXEXITHANDLER *PFNVMXEXITHANDLER;
316	#endif
317
318	/**
319	* VMX VM-exit handler, non-strict status code.
320	*
321	* This is generally the same as FNVMXEXITHANDLER, the NSRC bit is just FYI.
322	*
323	* @returns VBox status code, no informational status code returned.
324	* @param pVCpu The cross context virtual CPU structure.
325	* @param pVmxTransient The VMX-transient structure.
326	*
327	* @remarks This is not used on anything returning VERR_EM_INTERPRETER as the
328	* use of that status code will be replaced with VINF_EM_SOMETHING
329	* later when switching over to IEM.
330	*/
331	#ifndef HMVMX_USE_FUNCTION_TABLE
332	typedef int FNVMXEXITHANDLERNSRC(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
333	#else
334	typedef FNVMXEXITHANDLER FNVMXEXITHANDLERNSRC;
335	#endif
336
337
338	/*********************************************************************************************************************************
339	* Internal Functions *
340	*********************************************************************************************************************************/
341	#ifndef HMVMX_USE_FUNCTION_TABLE
342	DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
343	# define HMVMX_EXIT_DECL DECLINLINE(VBOXSTRICTRC)
344	# define HMVMX_EXIT_NSRC_DECL DECLINLINE(int)
345	#else
346	# define HMVMX_EXIT_DECL static DECLCALLBACK(VBOXSTRICTRC)
347	# define HMVMX_EXIT_NSRC_DECL HMVMX_EXIT_DECL
348	#endif
349	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
350	DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExitNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
351	#endif
352
353	static int hmR0VmxImportGuestState(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo, uint64_t fWhat);
354	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
355	static void hmR0VmxInitVmcsReadCache(PVMCPU pVCpu);
356	#endif
357
358	/** @name VM-exit handlers.
359	* @{
360	*/
361	static FNVMXEXITHANDLER hmR0VmxExitXcptOrNmi;
362	static FNVMXEXITHANDLER hmR0VmxExitExtInt;
363	static FNVMXEXITHANDLER hmR0VmxExitTripleFault;
364	static FNVMXEXITHANDLERNSRC hmR0VmxExitInitSignal;
365	static FNVMXEXITHANDLERNSRC hmR0VmxExitSipi;
366	static FNVMXEXITHANDLERNSRC hmR0VmxExitIoSmi;
367	static FNVMXEXITHANDLERNSRC hmR0VmxExitSmi;
368	static FNVMXEXITHANDLERNSRC hmR0VmxExitIntWindow;
369	static FNVMXEXITHANDLERNSRC hmR0VmxExitNmiWindow;
370	static FNVMXEXITHANDLER hmR0VmxExitTaskSwitch;
371	static FNVMXEXITHANDLER hmR0VmxExitCpuid;
372	static FNVMXEXITHANDLER hmR0VmxExitGetsec;
373	static FNVMXEXITHANDLER hmR0VmxExitHlt;
374	static FNVMXEXITHANDLERNSRC hmR0VmxExitInvd;
375	static FNVMXEXITHANDLER hmR0VmxExitInvlpg;
376	static FNVMXEXITHANDLER hmR0VmxExitRdpmc;
377	static FNVMXEXITHANDLER hmR0VmxExitVmcall;
378	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
379	static FNVMXEXITHANDLER hmR0VmxExitVmclear;
380	static FNVMXEXITHANDLER hmR0VmxExitVmlaunch;
381	static FNVMXEXITHANDLER hmR0VmxExitVmptrld;
382	static FNVMXEXITHANDLER hmR0VmxExitVmptrst;
383	static FNVMXEXITHANDLER hmR0VmxExitVmread;
384	static FNVMXEXITHANDLER hmR0VmxExitVmresume;
385	static FNVMXEXITHANDLER hmR0VmxExitVmwrite;
386	static FNVMXEXITHANDLER hmR0VmxExitVmxoff;
387	static FNVMXEXITHANDLER hmR0VmxExitVmxon;
388	#endif
389	static FNVMXEXITHANDLER hmR0VmxExitRdtsc;
390	static FNVMXEXITHANDLERNSRC hmR0VmxExitRsm;
391	static FNVMXEXITHANDLERNSRC hmR0VmxExitSetPendingXcptUD;
392	static FNVMXEXITHANDLER hmR0VmxExitMovCRx;
393	static FNVMXEXITHANDLER hmR0VmxExitMovDRx;
394	static FNVMXEXITHANDLER hmR0VmxExitIoInstr;
395	static FNVMXEXITHANDLER hmR0VmxExitRdmsr;
396	static FNVMXEXITHANDLER hmR0VmxExitWrmsr;
397	static FNVMXEXITHANDLERNSRC hmR0VmxExitErrInvalidGuestState;
398	static FNVMXEXITHANDLERNSRC hmR0VmxExitErrMsrLoad;
399	static FNVMXEXITHANDLERNSRC hmR0VmxExitErrUndefined;
400	static FNVMXEXITHANDLER hmR0VmxExitMwait;
401	static FNVMXEXITHANDLER hmR0VmxExitMtf;
402	static FNVMXEXITHANDLER hmR0VmxExitMonitor;
403	static FNVMXEXITHANDLER hmR0VmxExitPause;
404	static FNVMXEXITHANDLERNSRC hmR0VmxExitErrMachineCheck;
405	static FNVMXEXITHANDLERNSRC hmR0VmxExitTprBelowThreshold;
406	static FNVMXEXITHANDLER hmR0VmxExitApicAccess;
407	static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
408	static FNVMXEXITHANDLER hmR0VmxExitEptViolation;
409	static FNVMXEXITHANDLER hmR0VmxExitEptMisconfig;
410	static FNVMXEXITHANDLER hmR0VmxExitRdtscp;
411	static FNVMXEXITHANDLER hmR0VmxExitPreemptTimer;
412	static FNVMXEXITHANDLERNSRC hmR0VmxExitWbinvd;
413	static FNVMXEXITHANDLER hmR0VmxExitXsetbv;
414	static FNVMXEXITHANDLER hmR0VmxExitRdrand;
415	static FNVMXEXITHANDLER hmR0VmxExitInvpcid;
416	/** @} */
417
418	/** @name Helpers for hardware exceptions VM-exit handlers.
419	* @{
420	*/
421	static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
422	static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
423	static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
424	static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
425	static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
426	static int hmR0VmxExitXcptAC(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
427	static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
428	/** @} */
429
430
431	/*********************************************************************************************************************************
432	* Global Variables *
433	*********************************************************************************************************************************/
434	#ifdef VMX_USE_CACHED_VMCS_ACCESSES
435	static const uint32_t g_aVmcsCacheSegBase[] =
436	{
437	VMX_VMCS_GUEST_ES_BASE_CACHE_IDX,
438	VMX_VMCS_GUEST_CS_BASE_CACHE_IDX,
439	VMX_VMCS_GUEST_SS_BASE_CACHE_IDX,
440	VMX_VMCS_GUEST_DS_BASE_CACHE_IDX,
441	VMX_VMCS_GUEST_FS_BASE_CACHE_IDX,
442	VMX_VMCS_GUEST_GS_BASE_CACHE_IDX
443	};
444	AssertCompile(RT_ELEMENTS(g_aVmcsCacheSegBase) == X86_SREG_COUNT);
445	#endif
446	static const uint32_t g_aVmcsSegBase[] =
447	{
448	VMX_VMCS_GUEST_ES_BASE,
449	VMX_VMCS_GUEST_CS_BASE,
450	VMX_VMCS_GUEST_SS_BASE,
451	VMX_VMCS_GUEST_DS_BASE,
452	VMX_VMCS_GUEST_FS_BASE,
453	VMX_VMCS_GUEST_GS_BASE
454	};
455	static const uint32_t g_aVmcsSegSel[] =
456	{
457	VMX_VMCS16_GUEST_ES_SEL,
458	VMX_VMCS16_GUEST_CS_SEL,
459	VMX_VMCS16_GUEST_SS_SEL,
460	VMX_VMCS16_GUEST_DS_SEL,
461	VMX_VMCS16_GUEST_FS_SEL,
462	VMX_VMCS16_GUEST_GS_SEL
463	};
464	static const uint32_t g_aVmcsSegLimit[] =
465	{
466	VMX_VMCS32_GUEST_ES_LIMIT,
467	VMX_VMCS32_GUEST_CS_LIMIT,
468	VMX_VMCS32_GUEST_SS_LIMIT,
469	VMX_VMCS32_GUEST_DS_LIMIT,
470	VMX_VMCS32_GUEST_FS_LIMIT,
471	VMX_VMCS32_GUEST_GS_LIMIT
472	};
473	static const uint32_t g_aVmcsSegAttr[] =
474	{
475	VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS,
476	VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS,
477	VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS,
478	VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS,
479	VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS,
480	VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS
481	};
482	AssertCompile(RT_ELEMENTS(g_aVmcsSegSel) == X86_SREG_COUNT);
483	AssertCompile(RT_ELEMENTS(g_aVmcsSegLimit) == X86_SREG_COUNT);
484	AssertCompile(RT_ELEMENTS(g_aVmcsSegBase) == X86_SREG_COUNT);
485	AssertCompile(RT_ELEMENTS(g_aVmcsSegAttr) == X86_SREG_COUNT);
486
487	#ifdef HMVMX_USE_FUNCTION_TABLE
488	/**
489	* VMX_EXIT dispatch table.
490	*/
491	static const PFNVMXEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
492	{
493	/* 00 VMX_EXIT_XCPT_OR_NMI */ hmR0VmxExitXcptOrNmi,
494	/* 01 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
495	/* 02 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
496	/* 03 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitInitSignal,
497	/* 04 VMX_EXIT_SIPI */ hmR0VmxExitSipi,
498	/* 05 VMX_EXIT_IO_SMI */ hmR0VmxExitIoSmi,
499	/* 06 VMX_EXIT_SMI */ hmR0VmxExitSmi,
500	/* 07 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
501	/* 08 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
502	/* 09 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
503	/* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
504	/* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
505	/* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
506	/* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
507	/* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
508	/* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
509	/* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
510	/* 17 VMX_EXIT_RSM */ hmR0VmxExitRsm,
511	/* 18 VMX_EXIT_VMCALL */ hmR0VmxExitVmcall,
512	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
513	/* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitVmclear,
514	/* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitVmlaunch,
515	/* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitVmptrld,
516	/* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitVmptrst,
517	/* 23 VMX_EXIT_VMREAD */ hmR0VmxExitVmread,
518	/* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitVmresume,
519	/* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitVmwrite,
520	/* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitVmxoff,
521	/* 27 VMX_EXIT_VMXON */ hmR0VmxExitVmxon,
522	#else
523	/* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
524	/* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
525	/* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
526	/* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
527	/* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
528	/* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
529	/* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
530	/* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
531	/* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
532	#endif
533	/* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
534	/* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
535	/* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
536	/* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
537	/* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
538	/* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
539	/* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrMsrLoad,
540	/* 35 UNDEFINED */ hmR0VmxExitErrUndefined,
541	/* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
542	/* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
543	/* 38 UNDEFINED */ hmR0VmxExitErrUndefined,
544	/* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
545	/* 40 UNDEFINED */ hmR0VmxExitPause,
546	/* 41 VMX_EXIT_PAUSE */ hmR0VmxExitErrMachineCheck,
547	/* 42 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUndefined,
548	/* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
549	/* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
550	/* 45 UNDEFINED */ hmR0VmxExitErrUndefined,
551	/* 46 VMX_EXIT_GDTR_IDTR_ACCESS */ hmR0VmxExitXdtrAccess,
552	/* 47 VMX_EXIT_LDTR_TR_ACCESS */ hmR0VmxExitXdtrAccess,
553	/* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
554	/* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
555	/* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
556	/* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
557	/* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
558	/* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
559	/* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
560	/* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
561	/* 56 VMX_EXIT_APIC_WRITE */ hmR0VmxExitErrUndefined,
562	/* 57 VMX_EXIT_RDRAND */ hmR0VmxExitRdrand,
563	/* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
564	/* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitSetPendingXcptUD,
565	/* 60 VMX_EXIT_ENCLS */ hmR0VmxExitErrUndefined,
566	/* 61 VMX_EXIT_RDSEED / hmR0VmxExitErrUndefined, / only spurious exits, so undefined */
567	/* 62 VMX_EXIT_PML_FULL */ hmR0VmxExitErrUndefined,
568	/* 63 VMX_EXIT_XSAVES */ hmR0VmxExitSetPendingXcptUD,
569	/* 64 VMX_EXIT_XRSTORS */ hmR0VmxExitSetPendingXcptUD,
570	};
571	#endif /* HMVMX_USE_FUNCTION_TABLE */
572
573	#if defined(VBOX_STRICT) && defined(LOG_ENABLED)
574	static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
575	{
576	/* 0 */ "(Not Used)",
577	/* 1 */ "VMCALL executed in VMX root operation.",
578	/* 2 */ "VMCLEAR with invalid physical address.",
579	/* 3 */ "VMCLEAR with VMXON pointer.",
580	/* 4 */ "VMLAUNCH with non-clear VMCS.",
581	/* 5 */ "VMRESUME with non-launched VMCS.",
582	/* 6 */ "VMRESUME after VMXOFF",
583	/* 7 */ "VM-entry with invalid control fields.",
584	/* 8 */ "VM-entry with invalid host state fields.",
585	/* 9 */ "VMPTRLD with invalid physical address.",
586	/* 10 */ "VMPTRLD with VMXON pointer.",
587	/* 11 */ "VMPTRLD with incorrect revision identifier.",
588	/* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
589	/* 13 */ "VMWRITE to read-only VMCS component.",
590	/* 14 */ "(Not Used)",
591	/* 15 */ "VMXON executed in VMX root operation.",
592	/* 16 */ "VM-entry with invalid executive-VMCS pointer.",
593	/* 17 */ "VM-entry with non-launched executing VMCS.",
594	/* 18 */ "VM-entry with executive-VMCS pointer not VMXON pointer.",
595	/* 19 */ "VMCALL with non-clear VMCS.",
596	/* 20 */ "VMCALL with invalid VM-exit control fields.",
597	/* 21 */ "(Not Used)",
598	/* 22 */ "VMCALL with incorrect MSEG revision identifier.",
599	/* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
600	/* 24 */ "VMCALL with invalid SMM-monitor features.",
601	/* 25 */ "VM-entry with invalid VM-execution control fields in executive VMCS.",
602	/* 26 */ "VM-entry with events blocked by MOV SS.",
603	/* 27 */ "(Not Used)",
604	/* 28 */ "Invalid operand to INVEPT/INVVPID."
605	};
606	#endif /* VBOX_STRICT */
607
608
609	/**
610	* Get the CR0 guest/host mask that does not change through the lifetime of a VM.
611	*
612	* Any bit set in this mask is owned by the host/hypervisor and would cause a
613	* VM-exit when modified by the guest.
614	*
615	* @returns The static CR0 guest/host mask.
616	* @param pVCpu The cross context virtual CPU structure.
617	*/
618	DECL_FORCE_INLINE(uint64_t) hmR0VmxGetFixedCr0Mask(PVMCPU pVCpu)
619	{
620	/*
621	* Modifications to CR0 bits that VT-x ignores saving/restoring (CD, ET, NW) and
622	* to CR0 bits that we require for shadow paging (PG) by the guest must cause VM-exits.
623	*/
624	/** @todo Avoid intercepting CR0.PE with unrestricted guest execution. Fix PGM
625	* enmGuestMode to be in-sync with the current mode. See @bugref{6398}
626	* and @bugref{6944}. */
627	PVM pVM = pVCpu->CTX_SUFF(pVM);
628	return ( X86_CR0_PE
629	\| X86_CR0_NE
630	\| (pVM->hm.s.fNestedPaging ? 0 : X86_CR0_WP)
631	\| X86_CR0_PG
632	\| X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
633	\| X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
634	\| X86_CR0_NW); /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
635	}
636
637
638	/**
639	* Gets the CR4 guest/host mask that does not change through the lifetime of a VM.
640	*
641	* Any bit set in this mask is owned by the host/hypervisor and would cause a
642	* VM-exit when modified by the guest.
643	*
644	* @returns The static CR4 guest/host mask.
645	* @param pVCpu The cross context virtual CPU structure.
646	*/
647	DECL_FORCE_INLINE(uint64_t) hmR0VmxGetFixedCr4Mask(PVMCPU pVCpu)
648	{
649	/*
650	* We need to look at the host features here (for e.g. OSXSAVE, PCID) because
651	* these bits are reserved on hardware that does not support them. Since the
652	* CPU cannot refer to our virtual CPUID, we need to intercept CR4 changes to
653	* these bits and handle it depending on whether we expose them to the guest.
654	*/
655	PVM pVM = pVCpu->CTX_SUFF(pVM);
656	bool const fXSaveRstor = pVM->cpum.ro.HostFeatures.fXSaveRstor;
657	bool const fPcid = pVM->cpum.ro.HostFeatures.fPcid;
658	return ( X86_CR4_VMXE
659	\| X86_CR4_VME
660	\| X86_CR4_PAE
661	\| X86_CR4_PGE
662	\| X86_CR4_PSE
663	\| (fXSaveRstor ? X86_CR4_OSXSAVE : 0)
664	\| (fPcid ? X86_CR4_PCIDE : 0));
665	}
666
667
668	/**
669	* Returns whether the the VM-exit MSR-store area differs from the VM-exit MSR-load
670	* area.
671	*
672	* @returns @c true if it's different, @c false otherwise.
673	* @param pVmcsInfo The VMCS info. object.
674	*/
675	DECL_FORCE_INLINE(bool) hmR0VmxIsSeparateExitMsrStoreAreaVmcs(PCVMXVMCSINFO pVmcsInfo)
676	{
677	return RT_BOOL( pVmcsInfo->pvGuestMsrStore != pVmcsInfo->pvGuestMsrLoad
678	&& pVmcsInfo->pvGuestMsrStore);
679	}
680
681
682	/**
683	* Adds one or more exceptions to the exception bitmap and commits it to the current
684	* VMCS.
685	*
686	* @returns VBox status code.
687	* @param pVmxTransient The VMX-transient structure.
688	* @param uXcptMask The exception(s) to add.
689	*/
690	static int hmR0VmxAddXcptInterceptMask(PVMXTRANSIENT pVmxTransient, uint32_t uXcptMask)
691	{
692	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
693	uint32_t uXcptBitmap = pVmcsInfo->u32XcptBitmap;
694	if ((uXcptBitmap & uXcptMask) != uXcptMask)
695	{
696	uXcptBitmap \|= uXcptMask;
697	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
698	AssertRCReturn(rc, rc);
699	pVmcsInfo->u32XcptBitmap = uXcptBitmap;
700	}
701	return VINF_SUCCESS;
702	}
703
704
705	/**
706	* Adds an exception to the exception bitmap and commits it to the current VMCS.
707	*
708	* @returns VBox status code.
709	* @param pVmxTransient The VMX-transient structure.
710	* @param uXcpt The exception to add.
711	*/
712	static int hmR0VmxAddXcptIntercept(PVMXTRANSIENT pVmxTransient, uint8_t uXcpt)
713	{
714	Assert(uXcpt <= X86_XCPT_LAST);
715	return hmR0VmxAddXcptInterceptMask(pVmxTransient, RT_BIT_32(uXcpt));
716	}
717
718
719	/**
720	* Remove one or more exceptions from the exception bitmap and commits it to the
721	* current VMCS.
722	*
723	* This takes care of not removing the exception intercept if a nested-guest
724	* requires the exception to be intercepted.
725	*
726	* @returns VBox status code.
727	* @param pVCpu The cross context virtual CPU structure.
728	* @param pVmxTransient The VMX-transient structure.
729	* @param uXcptMask The exception(s) to remove.
730	*/
731	static int hmR0VmxRemoveXcptInterceptMask(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t uXcptMask)
732	{
733	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
734	uint32_t u32XcptBitmap = pVmcsInfo->u32XcptBitmap;
735	if (u32XcptBitmap & uXcptMask)
736	{
737	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
738	if (!pVmxTransient->fIsNestedGuest)
739	{ /* likely */ }
740	else
741	{
742	PCVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
743	uXcptMask &= ~pVmcsNstGst->u32XcptBitmap;
744	}
745	#endif
746	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
747	uXcptMask &= ~( RT_BIT(X86_XCPT_BP)
748	\| RT_BIT(X86_XCPT_DE)
749	\| RT_BIT(X86_XCPT_NM)
750	\| RT_BIT(X86_XCPT_TS)
751	\| RT_BIT(X86_XCPT_UD)
752	\| RT_BIT(X86_XCPT_NP)
753	\| RT_BIT(X86_XCPT_SS)
754	\| RT_BIT(X86_XCPT_GP)
755	\| RT_BIT(X86_XCPT_PF)
756	\| RT_BIT(X86_XCPT_MF));
757	#elif defined(HMVMX_ALWAYS_TRAP_PF)
758	uXcptMask &= ~RT_BIT(X86_XCPT_PF);
759	#endif
760	if (uXcptMask)
761	{
762	/* Validate we are not removing any essential exception intercepts. */
763	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging \|\| !(uXcptMask & RT_BIT(X86_XCPT_PF))); RT_NOREF(pVCpu);
764	Assert(!(uXcptMask & RT_BIT(X86_XCPT_DB)));
765	Assert(!(uXcptMask & RT_BIT(X86_XCPT_AC)));
766
767	/* Remove it from the exception bitmap. */
768	u32XcptBitmap &= ~uXcptMask;
769
770	/* Commit and update the cache if necessary. */
771	if (pVmcsInfo->u32XcptBitmap != u32XcptBitmap)
772	{
773	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
774	AssertRCReturn(rc, rc);
775	pVmcsInfo->u32XcptBitmap = u32XcptBitmap;
776	}
777	}
778	}
779	return VINF_SUCCESS;
780	}
781
782
783	/**
784	* Remove an exceptions from the exception bitmap and commits it to the current
785	* VMCS.
786	*
787	* @returns VBox status code.
788	* @param pVCpu The cross context virtual CPU structure.
789	* @param pVmxTransient The VMX-transient structure.
790	* @param uXcpt The exception to remove.
791	*/
792	static int hmR0VmxRemoveXcptIntercept(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint8_t uXcpt)
793	{
794	return hmR0VmxRemoveXcptInterceptMask(pVCpu, pVmxTransient, RT_BIT(uXcpt));
795	}
796
797
798	/**
799	* Loads the VMCS specified by the VMCS info. object.
800	*
801	* @returns VBox status code.
802	* @param pVmcsInfo The VMCS info. object.
803	*/
804	static int hmR0VmxLoadVmcs(PVMXVMCSINFO pVmcsInfo)
805	{
806	Assert(pVmcsInfo);
807	Assert(pVmcsInfo->HCPhysVmcs);
808	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
809
810	if (pVmcsInfo->fVmcsState & VMX_V_VMCS_LAUNCH_STATE_CLEAR)
811	{
812	int rc = VMXLoadVmcs(pVmcsInfo->HCPhysVmcs);
813	if (RT_SUCCESS(rc))
814	{
815	pVmcsInfo->fVmcsState \|= VMX_V_VMCS_LAUNCH_STATE_CURRENT;
816	return VINF_SUCCESS;
817	}
818	return rc;
819	}
820	return VERR_VMX_INVALID_VMCS_LAUNCH_STATE;
821	}
822
823
824	/**
825	* Clears the VMCS specified by the VMCS info. object.
826	*
827	* @returns VBox status code.
828	* @param pVmcsInfo The VMCS info. object.
829	*/
830	static int hmR0VmxClearVmcs(PVMXVMCSINFO pVmcsInfo)
831	{
832	Assert(pVmcsInfo);
833	Assert(pVmcsInfo->HCPhysVmcs);
834	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
835
836	int rc = VMXClearVmcs(pVmcsInfo->HCPhysVmcs);
837	if (RT_SUCCESS(rc))
838	pVmcsInfo->fVmcsState = VMX_V_VMCS_LAUNCH_STATE_CLEAR;
839	return rc;
840	}
841
842
843	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
844	/**
845	* Switches the current VMCS to the one specified.
846	*
847	* @returns VBox status code.
848	* @param pVmcsInfoFrom The VMCS info. object we are switching from.
849	* @param pVmcsInfoTo The VMCS info. object we are switching to.
850	*
851	* @remarks Called with interrupts disabled.
852	*/
853	static int hmR0VmxSwitchVmcs(PVMXVMCSINFO pVmcsInfoFrom, PVMXVMCSINFO pVmcsInfoTo)
854	{
855	Assert(pVmcsInfoFrom);
856	Assert(pVmcsInfoTo);
857
858	/*
859	* Clear the VMCS we are switching out if it has not already been cleared.
860	* This will sync any CPU internal data back to the VMCS.
861	*/
862	if (pVmcsInfoFrom->fVmcsState != VMX_V_VMCS_LAUNCH_STATE_CLEAR)
863	{
864	int rc = hmR0VmxClearVmcs(pVmcsInfoFrom);
865	if (RT_SUCCESS(rc))
866	{ /* likely */ }
867	else
868	return rc;
869	}
870
871	/*
872	* Clear the VMCS we are switching to if it has not already been cleared.
873	* This will initialize the VMCS launch state to "clear" required for loading it.
874	*
875	* See Intel spec. 31.6 "Preparation And Launching A Virtual Machine".
876	*/
877	if (pVmcsInfoTo->fVmcsState != VMX_V_VMCS_LAUNCH_STATE_CLEAR)
878	{
879	int rc = hmR0VmxClearVmcs(pVmcsInfoTo);
880	if (RT_SUCCESS(rc))
881	{ /* likely */ }
882	else
883	return rc;
884	}
885
886	/*
887	* Finally, load the VMCS we are switching to.
888	*/
889	return hmR0VmxLoadVmcs(pVmcsInfoTo);
890	}
891	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
892
893
894	/**
895	* Updates the VM's last error record.
896	*
897	* If there was a VMX instruction error, reads the error data from the VMCS and
898	* updates VCPU's last error record as well.
899	*
900	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
901	* Can be NULL if @a rc is not VERR_VMX_UNABLE_TO_START_VM or
902	* VERR_VMX_INVALID_VMCS_FIELD.
903	* @param rc The error code.
904	*/
905	static void hmR0VmxUpdateErrorRecord(PVMCPU pVCpu, int rc)
906	{
907	if ( rc == VERR_VMX_INVALID_VMCS_FIELD
908	\|\| rc == VERR_VMX_UNABLE_TO_START_VM)
909	{
910	AssertPtrReturnVoid(pVCpu);
911	VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
912	}
913	pVCpu->CTX_SUFF(pVM)->hm.s.rcInit = rc;
914	}
915
916
917	#ifdef VBOX_STRICT
918	/**
919	* Reads the VM-entry interruption-information field from the VMCS into the VMX
920	* transient structure.
921	*
922	* @returns VBox status code.
923	* @param pVmxTransient The VMX-transient structure.
924	*
925	* @remarks No-long-jump zone!!!
926	*/
927	DECLINLINE(int) hmR0VmxReadEntryIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
928	{
929	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntInfo);
930	AssertRCReturn(rc, rc);
931	return VINF_SUCCESS;
932	}
933
934
935	/**
936	* Reads the VM-entry exception error code field from the VMCS into
937	* the VMX transient structure.
938	*
939	* @returns VBox status code.
940	* @param pVmxTransient The VMX-transient structure.
941	*
942	* @remarks No-long-jump zone!!!
943	*/
944	DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
945	{
946	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
947	AssertRCReturn(rc, rc);
948	return VINF_SUCCESS;
949	}
950
951
952	/**
953	* Reads the VM-entry exception error code field from the VMCS into
954	* the VMX transient structure.
955	*
956	* @returns VBox status code.
957	* @param pVmxTransient The VMX-transient structure.
958	*
959	* @remarks No-long-jump zone!!!
960	*/
961	DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
962	{
963	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
964	AssertRCReturn(rc, rc);
965	return VINF_SUCCESS;
966	}
967	#endif /* VBOX_STRICT */
968
969
970	/**
971	* Reads the VM-exit interruption-information field from the VMCS into the VMX
972	* transient structure.
973	*
974	* @returns VBox status code.
975	* @param pVmxTransient The VMX-transient structure.
976	*/
977	DECLINLINE(int) hmR0VmxReadExitIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
978	{
979	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INTERRUPTION_INFO))
980	{
981	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntInfo);
982	AssertRCReturn(rc,rc);
983	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_INTERRUPTION_INFO;
984	}
985	return VINF_SUCCESS;
986	}
987
988
989	/**
990	* Reads the VM-exit interruption error code from the VMCS into the VMX
991	* transient structure.
992	*
993	* @returns VBox status code.
994	* @param pVmxTransient The VMX-transient structure.
995	*/
996	DECLINLINE(int) hmR0VmxReadExitIntErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
997	{
998	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE))
999	{
1000	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntErrorCode);
1001	AssertRCReturn(rc, rc);
1002	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE;
1003	}
1004	return VINF_SUCCESS;
1005	}
1006
1007
1008	/**
1009	* Reads the VM-exit instruction length field from the VMCS into the VMX
1010	* transient structure.
1011	*
1012	* @returns VBox status code.
1013	* @param pVmxTransient The VMX-transient structure.
1014	*/
1015	DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
1016	{
1017	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INSTR_LEN))
1018	{
1019	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
1020	AssertRCReturn(rc, rc);
1021	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_INSTR_LEN;
1022	}
1023	return VINF_SUCCESS;
1024	}
1025
1026
1027	/**
1028	* Reads the VM-exit instruction-information field from the VMCS into
1029	* the VMX transient structure.
1030	*
1031	* @returns VBox status code.
1032	* @param pVmxTransient The VMX-transient structure.
1033	*/
1034	DECLINLINE(int) hmR0VmxReadExitInstrInfoVmcs(PVMXTRANSIENT pVmxTransient)
1035	{
1036	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INSTR_INFO))
1037	{
1038	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->ExitInstrInfo.u);
1039	AssertRCReturn(rc, rc);
1040	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_INSTR_INFO;
1041	}
1042	return VINF_SUCCESS;
1043	}
1044
1045
1046	/**
1047	* Reads the VM-exit Qualification from the VMCS into the VMX transient structure.
1048	*
1049	* @returns VBox status code.
1050	* @param pVCpu The cross context virtual CPU structure of the
1051	* calling EMT. (Required for the VMCS cache case.)
1052	* @param pVmxTransient The VMX-transient structure.
1053	*/
1054	DECLINLINE(int) hmR0VmxReadExitQualVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
1055	{
1056	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_QUALIFICATION))
1057	{
1058	int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQual); NOREF(pVCpu);
1059	AssertRCReturn(rc, rc);
1060	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_EXIT_QUALIFICATION;
1061	}
1062	return VINF_SUCCESS;
1063	}
1064
1065
1066	/**
1067	* Reads the Guest-linear address from the VMCS into the VMX transient structure.
1068	*
1069	* @returns VBox status code.
1070	* @param pVCpu The cross context virtual CPU structure of the
1071	* calling EMT. (Required for the VMCS cache case.)
1072	* @param pVmxTransient The VMX-transient structure.
1073	*/
1074	DECLINLINE(int) hmR0VmxReadGuestLinearAddrVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
1075	{
1076	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_GUEST_LINEAR_ADDR))
1077	{
1078	int rc = VMXReadVmcsGstN(VMX_VMCS_RO_GUEST_LINEAR_ADDR, &pVmxTransient->uGuestLinearAddr); NOREF(pVCpu);
1079	AssertRCReturn(rc, rc);
1080	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_GUEST_LINEAR_ADDR;
1081	}
1082	return VINF_SUCCESS;
1083	}
1084
1085
1086	/**
1087	* Reads the IDT-vectoring information field from the VMCS into the VMX
1088	* transient structure.
1089	*
1090	* @returns VBox status code.
1091	* @param pVmxTransient The VMX-transient structure.
1092	*
1093	* @remarks No-long-jump zone!!!
1094	*/
1095	DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
1096	{
1097	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_IDT_VECTORING_INFO))
1098	{
1099	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_VECTORING_INFO, &pVmxTransient->uIdtVectoringInfo);
1100	AssertRCReturn(rc, rc);
1101	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_IDT_VECTORING_INFO;
1102	}
1103	return VINF_SUCCESS;
1104	}
1105
1106
1107	/**
1108	* Reads the IDT-vectoring error code from the VMCS into the VMX
1109	* transient structure.
1110	*
1111	* @returns VBox status code.
1112	* @param pVmxTransient The VMX-transient structure.
1113	*/
1114	DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
1115	{
1116	if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_IDT_VECTORING_ERROR_CODE))
1117	{
1118	int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_VECTORING_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
1119	AssertRCReturn(rc, rc);
1120	pVmxTransient->fVmcsFieldsRead \|= HMVMX_READ_IDT_VECTORING_ERROR_CODE;
1121	}
1122	return VINF_SUCCESS;
1123	}
1124
1125
1126	/**
1127	* Enters VMX root mode operation on the current CPU.
1128	*
1129	* @returns VBox status code.
1130	* @param pVM The cross context VM structure. Can be
1131	* NULL, after a resume.
1132	* @param HCPhysCpuPage Physical address of the VMXON region.
1133	* @param pvCpuPage Pointer to the VMXON region.
1134	*/
1135	static int hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
1136	{
1137	Assert(HCPhysCpuPage && HCPhysCpuPage != NIL_RTHCPHYS);
1138	Assert(RT_ALIGN_T(HCPhysCpuPage, _4K, RTHCPHYS) == HCPhysCpuPage);
1139	Assert(pvCpuPage);
1140	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1141
1142	if (pVM)
1143	{
1144	/* Write the VMCS revision identifier to the VMXON region. */
1145	(uint32_t )pvCpuPage = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_ID);
1146	}
1147
1148	/* Paranoid: Disable interrupts as, in theory, interrupt handlers might mess with CR4. */
1149	RTCCUINTREG fEFlags = ASMIntDisableFlags();
1150
1151	/* Enable the VMX bit in CR4 if necessary. */
1152	RTCCUINTREG uOldCr4 = SUPR0ChangeCR4(X86_CR4_VMXE, RTCCUINTREG_MAX);
1153
1154	/* Enter VMX root mode. */
1155	int rc = VMXEnable(HCPhysCpuPage);
1156	if (RT_FAILURE(rc))
1157	{
1158	if (!(uOldCr4 & X86_CR4_VMXE))
1159	SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
1160
1161	if (pVM)
1162	pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
1163	}
1164
1165	/* Restore interrupts. */
1166	ASMSetFlags(fEFlags);
1167	return rc;
1168	}
1169
1170
1171	/**
1172	* Exits VMX root mode operation on the current CPU.
1173	*
1174	* @returns VBox status code.
1175	*/
1176	static int hmR0VmxLeaveRootMode(void)
1177	{
1178	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1179
1180	/* Paranoid: Disable interrupts as, in theory, interrupts handlers might mess with CR4. */
1181	RTCCUINTREG fEFlags = ASMIntDisableFlags();
1182
1183	/* If we're for some reason not in VMX root mode, then don't leave it. */
1184	RTCCUINTREG uHostCR4 = ASMGetCR4();
1185
1186	int rc;
1187	if (uHostCR4 & X86_CR4_VMXE)
1188	{
1189	/* Exit VMX root mode and clear the VMX bit in CR4. */
1190	VMXDisable();
1191	SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
1192	rc = VINF_SUCCESS;
1193	}
1194	else
1195	rc = VERR_VMX_NOT_IN_VMX_ROOT_MODE;
1196
1197	/* Restore interrupts. */
1198	ASMSetFlags(fEFlags);
1199	return rc;
1200	}
1201
1202
1203	/**
1204	* Allocates and maps a physically contiguous page. The allocated page is
1205	* zero'd out (used by various VT-x structures).
1206	*
1207	* @returns IPRT status code.
1208	* @param pMemObj Pointer to the ring-0 memory object.
1209	* @param ppVirt Where to store the virtual address of the
1210	* allocation.
1211	* @param pHCPhys Where to store the physical address of the
1212	* allocation.
1213	*/
1214	static int hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
1215	{
1216	AssertPtr(pMemObj);
1217	AssertPtr(ppVirt);
1218	AssertPtr(pHCPhys);
1219	int rc = RTR0MemObjAllocCont(pMemObj, X86_PAGE_4K_SIZE, false /* fExecutable */);
1220	if (RT_FAILURE(rc))
1221	return rc;
1222	ppVirt = RTR0MemObjAddress(pMemObj);
1223	pHCPhys = RTR0MemObjGetPagePhysAddr(pMemObj, 0 /* iPage */);
1224	ASMMemZero32(*ppVirt, X86_PAGE_4K_SIZE);
1225	return VINF_SUCCESS;
1226	}
1227
1228
1229	/**
1230	* Frees and unmaps an allocated, physical page.
1231	*
1232	* @param pMemObj Pointer to the ring-0 memory object.
1233	* @param ppVirt Where to re-initialize the virtual address of
1234	* allocation as 0.
1235	* @param pHCPhys Where to re-initialize the physical address of the
1236	* allocation as 0.
1237	*/
1238	static void hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
1239	{
1240	AssertPtr(pMemObj);
1241	AssertPtr(ppVirt);
1242	AssertPtr(pHCPhys);
1243	/* NULL is valid, accepted and ignored by the free function below. */
1244	RTR0MemObjFree(pMemObj, true / fFreeMappings */);
1245	*pMemObj = NIL_RTR0MEMOBJ;
1246	*ppVirt = NULL;
1247	*pHCPhys = NIL_RTHCPHYS;
1248	}
1249
1250
1251	/**
1252	* Initializes a VMCS info. object.
1253	*
1254	* @param pVmcsInfo The VMCS info. object.
1255	*/
1256	static void hmR0VmxInitVmcsInfo(PVMXVMCSINFO pVmcsInfo)
1257	{
1258	RT_ZERO(*pVmcsInfo);
1259
1260	Assert(pVmcsInfo->hMemObjVmcs == NIL_RTR0MEMOBJ);
1261	Assert(pVmcsInfo->hMemObjMsrBitmap == NIL_RTR0MEMOBJ);
1262	Assert(pVmcsInfo->hMemObjGuestMsrLoad == NIL_RTR0MEMOBJ);
1263	Assert(pVmcsInfo->hMemObjGuestMsrStore == NIL_RTR0MEMOBJ);
1264	Assert(pVmcsInfo->hMemObjHostMsrLoad == NIL_RTR0MEMOBJ);
1265	pVmcsInfo->HCPhysVmcs = NIL_RTHCPHYS;
1266	pVmcsInfo->HCPhysMsrBitmap = NIL_RTHCPHYS;
1267	pVmcsInfo->HCPhysGuestMsrLoad = NIL_RTHCPHYS;
1268	pVmcsInfo->HCPhysGuestMsrStore = NIL_RTHCPHYS;
1269	pVmcsInfo->HCPhysHostMsrLoad = NIL_RTHCPHYS;
1270	pVmcsInfo->HCPhysVirtApic = NIL_RTHCPHYS;
1271	pVmcsInfo->HCPhysEPTP = NIL_RTHCPHYS;
1272	pVmcsInfo->u64VmcsLinkPtr = NIL_RTHCPHYS;
1273	}
1274
1275
1276	/**
1277	* Frees the VT-x structures for a VMCS info. object.
1278	*
1279	* @param pVM The cross context VM structure.
1280	* @param pVmcsInfo The VMCS info. object.
1281	*/
1282	static void hmR0VmxFreeVmcsInfo(PVM pVM, PVMXVMCSINFO pVmcsInfo)
1283	{
1284	hmR0VmxPageFree(&pVmcsInfo->hMemObjVmcs, &pVmcsInfo->pvVmcs, &pVmcsInfo->HCPhysVmcs);
1285
1286	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
1287	hmR0VmxPageFree(&pVmcsInfo->hMemObjMsrBitmap, &pVmcsInfo->pvMsrBitmap, &pVmcsInfo->HCPhysMsrBitmap);
1288
1289	hmR0VmxPageFree(&pVmcsInfo->hMemObjHostMsrLoad, &pVmcsInfo->pvHostMsrLoad, &pVmcsInfo->HCPhysHostMsrLoad);
1290	hmR0VmxPageFree(&pVmcsInfo->hMemObjGuestMsrLoad, &pVmcsInfo->pvGuestMsrLoad, &pVmcsInfo->HCPhysGuestMsrLoad);
1291	hmR0VmxPageFree(&pVmcsInfo->hMemObjGuestMsrStore, &pVmcsInfo->pvGuestMsrStore, &pVmcsInfo->HCPhysGuestMsrStore);
1292
1293	hmR0VmxInitVmcsInfo(pVmcsInfo);
1294	}
1295
1296
1297	/**
1298	* Allocates the VT-x structures for a VMCS info. object.
1299	*
1300	* @returns VBox status code.
1301	* @param pVCpu The cross context virtual CPU structure.
1302	* @param pVmcsInfo The VMCS info. object.
1303	* @param fIsNstGstVmcs Whether this is a nested-guest VMCS.
1304	*/
1305	static int hmR0VmxAllocVmcsInfo(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, bool fIsNstGstVmcs)
1306	{
1307	PVM pVM = pVCpu->CTX_SUFF(pVM);
1308
1309	/* Allocate the guest VM control structure (VMCS). */
1310	int rc = hmR0VmxPageAllocZ(&pVmcsInfo->hMemObjVmcs, &pVmcsInfo->pvVmcs, &pVmcsInfo->HCPhysVmcs);
1311	if (RT_SUCCESS(rc))
1312	{
1313	if (!fIsNstGstVmcs)
1314	{
1315	/* Get the allocated virtual-APIC page from the virtual APIC device. */
1316	if ( PDMHasApic(pVCpu->CTX_SUFF(pVM))
1317	&& (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_TPR_SHADOW))
1318	{
1319	rc = APICGetApicPageForCpu(pVCpu, &pVmcsInfo->HCPhysVirtApic, (PRTR0PTR)&pVmcsInfo->pbVirtApic,
1320	NULL /* pR3Ptr /, NULL / pRCPtr */);
1321	}
1322	}
1323	else
1324	{
1325	Assert(pVmcsInfo->HCPhysVirtApic == NIL_RTHCPHYS);
1326	Assert(!pVmcsInfo->pbVirtApic);
1327	}
1328
1329	if (RT_SUCCESS(rc))
1330	{
1331	/*
1332	* Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for
1333	* transparent accesses of specific MSRs.
1334	*
1335	* If the condition for enabling MSR bitmaps changes here, don't forget to
1336	* update HMIsMsrBitmapActive().
1337	*
1338	* We don't share MSR bitmaps between the guest and nested-guest as we then
1339	* don't need to care about carefully restoring the guest MSR bitmap.
1340	* The guest visible nested-guest MSR bitmap needs to remain unchanged.
1341	* Hence, allocate a separate MSR bitmap for the guest and nested-guest.
1342	*/
1343	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
1344	{
1345	rc = hmR0VmxPageAllocZ(&pVmcsInfo->hMemObjMsrBitmap, &pVmcsInfo->pvMsrBitmap, &pVmcsInfo->HCPhysMsrBitmap);
1346	if (RT_SUCCESS(rc))
1347	ASMMemFill32(pVmcsInfo->pvMsrBitmap, X86_PAGE_4K_SIZE, UINT32_C(0xffffffff));
1348	}
1349
1350	if (RT_SUCCESS(rc))
1351	{
1352	/*
1353	* Allocate the VM-entry MSR-load area for the guest MSRs.
1354	*
1355	* Similar to MSR-bitmaps, we do not share the auto MSR-load/store are between
1356	* the guest and nested-guest.
1357	*/
1358	rc = hmR0VmxPageAllocZ(&pVmcsInfo->hMemObjGuestMsrLoad, &pVmcsInfo->pvGuestMsrLoad,
1359	&pVmcsInfo->HCPhysGuestMsrLoad);
1360	if (RT_SUCCESS(rc))
1361	{
1362	/*
1363	* We use the same page for VM-entry MSR-load and VM-exit MSR store areas.
1364	* These contain the guest MSRs to load on VM-entry and store on VM-exit.
1365	*/
1366	Assert(pVmcsInfo->hMemObjGuestMsrStore == NIL_RTR0MEMOBJ);
1367	pVmcsInfo->pvGuestMsrStore = pVmcsInfo->pvGuestMsrLoad;
1368	pVmcsInfo->HCPhysGuestMsrStore = pVmcsInfo->HCPhysGuestMsrLoad;
1369
1370	/* Allocate the VM-exit MSR-load page for the host MSRs. */
1371	rc = hmR0VmxPageAllocZ(&pVmcsInfo->hMemObjHostMsrLoad, &pVmcsInfo->pvHostMsrLoad,
1372	&pVmcsInfo->HCPhysHostMsrLoad);
1373	}
1374	}
1375	}
1376	}
1377
1378	return rc;
1379	}
1380
1381
1382	/**
1383	* Free all VT-x structures for the VM.
1384	*
1385	* @returns IPRT status code.
1386	* @param pVM The cross context VM structure.
1387	*/
1388	static void hmR0VmxStructsFree(PVM pVM)
1389	{
1390	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
1391	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
1392	#endif
1393	hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
1394
1395	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
1396	{
1397	PVMCPU pVCpu = &pVM->aCpus[idCpu];
1398	PVMXVMCSINFO pVmcsInfo = &pVCpu->hm.s.vmx.VmcsInfo;
1399	hmR0VmxFreeVmcsInfo(pVM, pVmcsInfo);
1400	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1401	if (pVM->cpum.ro.GuestFeatures.fVmx)
1402	{
1403	pVmcsInfo = &pVCpu->hm.s.vmx.VmcsInfoNstGst;
1404	hmR0VmxFreeVmcsInfo(pVM, pVmcsInfo);
1405	}
1406	#endif
1407	}
1408	}
1409
1410
1411	/**
1412	* Allocate all VT-x structures for the VM.
1413	*
1414	* @returns IPRT status code.
1415	* @param pVM The cross context VM structure.
1416	*/
1417	static int hmR0VmxStructsAlloc(PVM pVM)
1418	{
1419	/*
1420	* Sanity check the VMCS size reported by the CPU as we assume 4KB allocations.
1421	* The VMCS size cannot be more than 4096 bytes.
1422	*
1423	* See Intel spec. Appendix A.1 "Basic VMX Information".
1424	*/
1425	uint32_t const cbVmcs = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_SIZE);
1426	if (cbVmcs <= X86_PAGE_4K_SIZE)
1427	{ /* likely */ }
1428	else
1429	{
1430	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_INVALID_VMCS_SIZE;
1431	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1432	}
1433
1434	/*
1435	* Initialize/check members up-front so we can cleanup en masse on allocation failures.
1436	*/
1437	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
1438	Assert(pVM->hm.s.vmx.hMemObjScratch == NIL_RTR0MEMOBJ);
1439	Assert(pVM->hm.s.vmx.pbScratch == NULL);
1440	pVM->hm.s.vmx.HCPhysScratch = NIL_RTHCPHYS;
1441	#endif
1442
1443	Assert(pVM->hm.s.vmx.hMemObjApicAccess == NIL_RTR0MEMOBJ);
1444	Assert(pVM->hm.s.vmx.pbApicAccess == NULL);
1445	pVM->hm.s.vmx.HCPhysApicAccess = NIL_RTHCPHYS;
1446
1447	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
1448	{
1449	PVMCPU pVCpu = &pVM->aCpus[idCpu];
1450	hmR0VmxInitVmcsInfo(&pVCpu->hm.s.vmx.VmcsInfo);
1451	hmR0VmxInitVmcsInfo(&pVCpu->hm.s.vmx.VmcsInfoNstGst);
1452	}
1453
1454	/*
1455	* Allocate per-VM VT-x structures.
1456	*/
1457	int rc = VINF_SUCCESS;
1458	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
1459	/* Allocate crash-dump magic scratch page. */
1460	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
1461	if (RT_FAILURE(rc))
1462	{
1463	hmR0VmxStructsFree(pVM);
1464	return rc;
1465	}
1466	strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
1467	(uint64_t )(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xdeadbeefdeadbeef);
1468	#endif
1469
1470	/* Allocate the APIC-access page for trapping APIC accesses from the guest. */
1471	if (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
1472	{
1473	rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
1474	&pVM->hm.s.vmx.HCPhysApicAccess);
1475	if (RT_FAILURE(rc))
1476	{
1477	hmR0VmxStructsFree(pVM);
1478	return rc;
1479	}
1480	}
1481
1482	/*
1483	* Initialize per-VCPU VT-x structures.
1484	*/
1485	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
1486	{
1487	/* Allocate the guest VMCS structures. */
1488	PVMCPU pVCpu = &pVM->aCpus[idCpu];
1489	rc = hmR0VmxAllocVmcsInfo(pVCpu, &pVCpu->hm.s.vmx.VmcsInfo, false /* fIsNstGstVmcs */);
1490	if (RT_SUCCESS(rc))
1491	{
1492	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1493	/* Allocate the nested-guest VMCS structures, when the VMX feature is exposed to the guest. */
1494	if (pVM->cpum.ro.GuestFeatures.fVmx)
1495	{
1496	rc = hmR0VmxAllocVmcsInfo(pVCpu, &pVCpu->hm.s.vmx.VmcsInfoNstGst, true /* fIsNstGstVmcs */);
1497	if (RT_SUCCESS(rc))
1498	{ /* likely */ }
1499	else
1500	break;
1501	}
1502	#endif
1503	}
1504	else
1505	break;
1506	}
1507
1508	if (RT_FAILURE(rc))
1509	{
1510	hmR0VmxStructsFree(pVM);
1511	return rc;
1512	}
1513
1514	return VINF_SUCCESS;
1515	}
1516
1517
1518	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1519	/**
1520	* Returns whether an MSR at the given MSR-bitmap offset is intercepted or not.
1521	*
1522	* @returns @c true if the MSR is intercepted, @c false otherwise.
1523	* @param pvMsrBitmap The MSR bitmap.
1524	* @param offMsr The MSR byte offset.
1525	* @param iBit The bit offset from the byte offset.
1526	*/
1527	DECLINLINE(bool) hmR0VmxIsMsrBitSet(const void *pvMsrBitmap, uint16_t offMsr, int32_t iBit)
1528	{
1529	uint8_t const * const pbMsrBitmap = (uint8_t const * const)pvMsrBitmap;
1530	Assert(pbMsrBitmap);
1531	Assert(offMsr + (iBit >> 3) <= X86_PAGE_4K_SIZE);
1532	return ASMBitTest(pbMsrBitmap + offMsr, iBit);
1533	}
1534	#endif
1535
1536
1537	/**
1538	* Sets the permission bits for the specified MSR in the given MSR bitmap.
1539	*
1540	* If the passed VMCS is a nested-guest VMCS, this function ensures that the
1541	* read/write intercept is cleared from the MSR bitmap used for hardware-assisted
1542	* VMX execution of the nested-guest, only if nested-guest is also not intercepting
1543	* the read/write access of this MSR.
1544	*
1545	* @param pVCpu The cross context virtual CPU structure.
1546	* @param pVmcsInfo The VMCS info. object.
1547	* @param fIsNstGstVmcs Whether this is a nested-guest VMCS.
1548	* @param idMsr The MSR value.
1549	* @param fMsrpm The MSR permissions (see VMXMSRPM_XXX). This must
1550	* include both a read -and- a write permission!
1551	*
1552	* @sa HMGetVmxMsrPermission.
1553	*/
1554	static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, bool fIsNstGstVmcs, uint32_t idMsr, uint32_t fMsrpm)
1555	{
1556	uint8_t pbMsrBitmap = (uint8_t )pVmcsInfo->pvMsrBitmap;
1557	Assert(pbMsrBitmap);
1558	Assert(VMXMSRPM_IS_FLAG_VALID(fMsrpm));
1559
1560	/*
1561	* MSR-bitmap Layout:
1562	* Byte index MSR range Interpreted as
1563	* 0x000 - 0x3ff 0x00000000 - 0x00001fff Low MSR read bits.
1564	* 0x400 - 0x7ff 0xc0000000 - 0xc0001fff High MSR read bits.
1565	* 0x800 - 0xbff 0x00000000 - 0x00001fff Low MSR write bits.
1566	* 0xc00 - 0xfff 0xc0000000 - 0xc0001fff High MSR write bits.
1567	*
1568	* A bit corresponding to an MSR within the above range causes a VM-exit
1569	* if the bit is 1 on executions of RDMSR/WRMSR. If an MSR falls out of
1570	* the MSR range, it always cause a VM-exit.
1571	*
1572	* See Intel spec. 24.6.9 "MSR-Bitmap Address".
1573	*/
1574	uint16_t const offBitmapRead = 0;
1575	uint16_t const offBitmapWrite = 0x800;
1576	uint16_t offMsr;
1577	int32_t iBit;
1578	if (idMsr <= UINT32_C(0x00001fff))
1579	{
1580	offMsr = 0;
1581	iBit = idMsr;
1582	}
1583	else if (idMsr - UINT32_C(0xc0000000) <= UINT32_C(0x00001fff))
1584	{
1585	offMsr = 0x400;
1586	iBit = idMsr - UINT32_C(0xc0000000);
1587	}
1588	else
1589	AssertMsgFailedReturnVoid(("Invalid MSR %#RX32\n", idMsr));
1590
1591	/*
1592	* Set the MSR read permission.
1593	*/
1594	uint16_t const offMsrRead = offBitmapRead + offMsr;
1595	Assert(offMsrRead + (iBit >> 3) < offBitmapWrite);
1596	if (fMsrpm & VMXMSRPM_ALLOW_RD)
1597	{
1598	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1599	bool const fClear = !fIsNstGstVmcs ? true
1600	: !hmR0VmxIsMsrBitSet(pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvMsrBitmap), offMsrRead, iBit);
1601	#else
1602	RT_NOREF2(pVCpu, fIsNstGstVmcs);
1603	bool const fClear = true;
1604	#endif
1605	if (fClear)
1606	ASMBitClear(pbMsrBitmap + offMsrRead, iBit);
1607	}
1608	else
1609	ASMBitSet(pbMsrBitmap + offMsrRead, iBit);
1610
1611	/*
1612	* Set the MSR write permission.
1613	*/
1614	uint16_t const offMsrWrite = offBitmapWrite + offMsr;
1615	Assert(offMsrWrite + (iBit >> 3) < X86_PAGE_4K_SIZE);
1616	if (fMsrpm & VMXMSRPM_ALLOW_WR)
1617	{
1618	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
1619	bool const fClear = !fIsNstGstVmcs ? true
1620	: !hmR0VmxIsMsrBitSet(pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvMsrBitmap), offMsrWrite, iBit);
1621	#else
1622	RT_NOREF2(pVCpu, fIsNstGstVmcs);
1623	bool const fClear = true;
1624	#endif
1625	if (fClear)
1626	ASMBitClear(pbMsrBitmap + offMsrWrite, iBit);
1627	}
1628	else
1629	ASMBitSet(pbMsrBitmap + offMsrWrite, iBit);
1630	}
1631
1632
1633	/**
1634	* Updates the VMCS with the number of effective MSRs in the auto-load/store MSR
1635	* area.
1636	*
1637	* @returns VBox status code.
1638	* @param pVCpu The cross context virtual CPU structure.
1639	* @param pVmcsInfo The VMCS info. object.
1640	* @param cMsrs The number of MSRs.
1641	*/
1642	static int hmR0VmxSetAutoLoadStoreMsrCount(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, uint32_t cMsrs)
1643	{
1644	/* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
1645	uint32_t const cMaxSupportedMsrs = VMX_MISC_MAX_MSRS(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc);
1646	if (RT_UNLIKELY(cMsrs >= cMaxSupportedMsrs))
1647	{
1648	LogRel(("Auto-load/store MSR count exceeded! cMsrs=%u Supported=%u.\n", cMsrs, cMaxSupportedMsrs));
1649	pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_GUEST_MSR_STORAGE;
1650	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1651	}
1652
1653	/* Commit the MSR counts to the VMCS and update the cache. */
1654	int rc = VINF_SUCCESS;
1655	if (pVmcsInfo->cEntryMsrLoad != cMsrs)
1656	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cMsrs);
1657	if (pVmcsInfo->cExitMsrStore != cMsrs)
1658	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cMsrs);
1659	if (pVmcsInfo->cExitMsrLoad != cMsrs)
1660	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cMsrs);
1661	AssertRCReturn(rc, rc);
1662
1663	pVmcsInfo->cEntryMsrLoad = cMsrs;
1664	pVmcsInfo->cExitMsrStore = cMsrs;
1665	pVmcsInfo->cExitMsrLoad = cMsrs;
1666
1667	return VINF_SUCCESS;
1668	}
1669
1670
1671	/**
1672	* Adds a new (or updates the value of an existing) guest/host MSR
1673	* pair to be swapped during the world-switch as part of the
1674	* auto-load/store MSR area in the VMCS.
1675	*
1676	* @returns VBox status code.
1677	* @param pVCpu The cross context virtual CPU structure.
1678	* @param pVmxTransient The VMX-transient structure.
1679	* @param idMsr The MSR.
1680	* @param uGuestMsrValue Value of the guest MSR.
1681	* @param fSetReadWrite Whether to set the guest read/write access of this
1682	* MSR (thus not causing a VM-exit).
1683	* @param fUpdateHostMsr Whether to update the value of the host MSR if
1684	* necessary.
1685	*/
1686	static int hmR0VmxAddAutoLoadStoreMsr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t idMsr, uint64_t uGuestMsrValue,
1687	bool fSetReadWrite, bool fUpdateHostMsr)
1688	{
1689	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
1690	bool const fIsNstGstVmcs = pVmxTransient->fIsNestedGuest;
1691	PVMXAUTOMSR pGuestMsrLoad = (PVMXAUTOMSR)pVmcsInfo->pvGuestMsrLoad;
1692	uint32_t cMsrs = pVmcsInfo->cEntryMsrLoad;
1693	uint32_t i;
1694
1695	/* Paranoia. */
1696	Assert(pGuestMsrLoad);
1697
1698	/* Check if the MSR already exists in the VM-entry MSR-load area. */
1699	for (i = 0; i < cMsrs; i++)
1700	{
1701	if (pGuestMsrLoad->u32Msr == idMsr)
1702	break;
1703	pGuestMsrLoad++;
1704	}
1705
1706	bool fAdded = false;
1707	if (i == cMsrs)
1708	{
1709	/* The MSR does not exist, bump the MSR coun to make room for the new MSR. */
1710	++cMsrs;
1711	int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, pVmcsInfo, cMsrs);
1712	AssertMsgRCReturn(rc, ("Insufficient space to add MSR to VM-entry MSR-load/store area %u\n", idMsr), rc);
1713
1714	/* Set the guest to read/write this MSR without causing VM-exits. */
1715	if ( fSetReadWrite
1716	&& (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS))
1717	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, idMsr, VMXMSRPM_ALLOW_RD \| VMXMSRPM_ALLOW_WR);
1718
1719	fAdded = true;
1720	}
1721
1722	/* Update the MSR value for the newly added or already existing MSR. */
1723	pGuestMsrLoad->u32Msr = idMsr;
1724	pGuestMsrLoad->u64Value = uGuestMsrValue;
1725
1726	/* Create the corresponding slot in the VM-exit MSR-store area if we use a different page. */
1727	if (hmR0VmxIsSeparateExitMsrStoreAreaVmcs(pVmcsInfo))
1728	{
1729	PVMXAUTOMSR pGuestMsrStore = (PVMXAUTOMSR)pVmcsInfo->pvGuestMsrStore;
1730	pGuestMsrStore += i;
1731	pGuestMsrStore->u32Msr = idMsr;
1732	pGuestMsrStore->u64Value = 0;
1733	}
1734
1735	/* Update the corresponding slot in the host MSR area. */
1736	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVmcsInfo->pvHostMsrLoad;
1737	Assert(pHostMsr != pVmcsInfo->pvGuestMsrLoad && pHostMsr != pVmcsInfo->pvGuestMsrStore);
1738	pHostMsr += i;
1739	pHostMsr->u32Msr = idMsr;
1740
1741	/*
1742	* Only if the caller requests to update the host MSR value AND we've newly added the
1743	* MSR to the host MSR area do we actually update the value. Otherwise, it will be
1744	* updated by hmR0VmxUpdateAutoLoadHostMsrs().
1745	*
1746	* We do this for performance reasons since reading MSRs may be quite expensive.
1747	*/
1748	if ( fAdded
1749	&& fUpdateHostMsr)
1750	{
1751	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1752	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1753	pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1754	}
1755	return VINF_SUCCESS;
1756	}
1757
1758
1759	/**
1760	* Removes a guest/host MSR pair to be swapped during the world-switch from the
1761	* auto-load/store MSR area in the VMCS.
1762	*
1763	* @returns VBox status code.
1764	* @param pVCpu The cross context virtual CPU structure.
1765	* @param pVmxTransient The VMX-transient structure.
1766	* @param idMsr The MSR.
1767	*/
1768	static int hmR0VmxRemoveAutoLoadStoreMsr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t idMsr)
1769	{
1770	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
1771	bool const fIsNstGstVmcs = pVmxTransient->fIsNestedGuest;
1772	PVMXAUTOMSR pGuestMsrLoad = (PVMXAUTOMSR)pVmcsInfo->pvGuestMsrLoad;
1773	uint32_t cMsrs = pVmcsInfo->cEntryMsrLoad;
1774
1775	bool const fSeparateExitMsrStorePage = hmR0VmxIsSeparateExitMsrStoreAreaVmcs(pVmcsInfo);
1776	for (uint32_t i = 0; i < cMsrs; i++)
1777	{
1778	/* Find the MSR. */
1779	if (pGuestMsrLoad->u32Msr == idMsr)
1780	{
1781	/* If it's the last MSR, simply reduce the count. */
1782	if (i == cMsrs - 1)
1783	{
1784	--cMsrs;
1785	break;
1786	}
1787
1788	/* Remove it by copying the last MSR in place of it, and reducing the count. */
1789	PVMXAUTOMSR pLastGuestMsrLoad = (PVMXAUTOMSR)pVmcsInfo->pvGuestMsrLoad;
1790	pLastGuestMsrLoad += cMsrs - 1;
1791	pGuestMsrLoad->u32Msr = pLastGuestMsrLoad->u32Msr;
1792	pGuestMsrLoad->u64Value = pLastGuestMsrLoad->u64Value;
1793
1794	/* Remove it from the VM-exit MSR-store area if we are using a different page. */
1795	if (fSeparateExitMsrStorePage)
1796	{
1797	PVMXAUTOMSR pGuestMsrStore = (PVMXAUTOMSR)pVmcsInfo->pvGuestMsrStore;
1798	PVMXAUTOMSR pLastGuestMsrStore = (PVMXAUTOMSR)pVmcsInfo->pvGuestMsrStore;
1799	pGuestMsrStore += i;
1800	pLastGuestMsrStore += cMsrs - 1;
1801	Assert(pGuestMsrStore->u32Msr == idMsr);
1802	pGuestMsrStore->u32Msr = pLastGuestMsrStore->u32Msr;
1803	pGuestMsrStore->u64Value = pLastGuestMsrStore->u64Value;
1804	}
1805
1806	/* Remove it from the VM-exit MSR-load area. */
1807	PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVmcsInfo->pvHostMsrLoad;
1808	PVMXAUTOMSR pLastHostMsr = (PVMXAUTOMSR)pVmcsInfo->pvHostMsrLoad;
1809	pHostMsr += i;
1810	pLastHostMsr += cMsrs - 1;
1811	Assert(pHostMsr->u32Msr == idMsr);
1812	pHostMsr->u32Msr = pLastHostMsr->u32Msr;
1813	pHostMsr->u64Value = pLastHostMsr->u64Value;
1814	--cMsrs;
1815	break;
1816	}
1817	pGuestMsrLoad++;
1818	}
1819
1820	/* Update the VMCS if the count changed (meaning the MSR was found). */
1821	if (cMsrs != pVmcsInfo->cEntryMsrLoad)
1822	{
1823	int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, pVmcsInfo, cMsrs);
1824	AssertRCReturn(rc, rc);
1825
1826	/* We're no longer swapping MSRs during the world-switch, intercept guest read/writes to them. */
1827	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
1828	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, idMsr, VMXMSRPM_EXIT_RD \| VMXMSRPM_EXIT_WR);
1829
1830	Log4Func(("Removed MSR %#RX32, cMsrs=%u\n", idMsr, cMsrs));
1831	return VINF_SUCCESS;
1832	}
1833
1834	return VERR_NOT_FOUND;
1835	}
1836
1837
1838	/**
1839	* Checks if the specified guest MSR is part of the VM-entry MSR-load area.
1840	*
1841	* @returns @c true if found, @c false otherwise.
1842	* @param pVmcsInfo The VMCS info. object.
1843	* @param idMsr The MSR to find.
1844	*/
1845	static bool hmR0VmxIsAutoLoadGuestMsr(PCVMXVMCSINFO pVmcsInfo, uint32_t idMsr)
1846	{
1847	PCVMXAUTOMSR pGuestMsrLoad = (PCVMXAUTOMSR)pVmcsInfo->pvGuestMsrLoad;
1848	uint32_t const cMsrs = pVmcsInfo->cEntryMsrLoad;
1849	for (uint32_t i = 0; i < cMsrs; i++)
1850	{
1851	if (pGuestMsrLoad->u32Msr == idMsr)
1852	return true;
1853	pGuestMsrLoad++;
1854	}
1855	return false;
1856	}
1857
1858
1859	/**
1860	* Updates the value of all host MSRs in the VM-exit MSR-load area.
1861	*
1862	* @param pVCpu The cross context virtual CPU structure.
1863	* @param pVmcsInfo The VMCS info. object.
1864	*
1865	* @remarks No-long-jump zone!!!
1866	*/
1867	static void hmR0VmxUpdateAutoLoadHostMsrs(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
1868	{
1869	PVMXAUTOMSR pHostMsrLoad = (PVMXAUTOMSR)pVmcsInfo->pvHostMsrLoad;
1870	uint32_t const cMsrs = pVmcsInfo->cExitMsrLoad;
1871
1872	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1873	Assert(pHostMsrLoad);
1874
1875	for (uint32_t i = 0; i < cMsrs; i++, pHostMsrLoad++)
1876	{
1877	/*
1878	* Performance hack for the host EFER MSR. We use the cached value rather than re-read it.
1879	* Strict builds will catch mismatches in hmR0VmxCheckAutoLoadStoreMsrs(). See @bugref{7368}.
1880	*/
1881	if (pHostMsrLoad->u32Msr == MSR_K6_EFER)
1882	pHostMsrLoad->u64Value = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostMsrEfer;
1883	else
1884	pHostMsrLoad->u64Value = ASMRdMsr(pHostMsrLoad->u32Msr);
1885	}
1886	}
1887
1888
1889	/**
1890	* Saves a set of host MSRs to allow read/write passthru access to the guest and
1891	* perform lazy restoration of the host MSRs while leaving VT-x.
1892	*
1893	* @param pVCpu The cross context virtual CPU structure.
1894	*
1895	* @remarks No-long-jump zone!!!
1896	*/
1897	static void hmR0VmxLazySaveHostMsrs(PVMCPU pVCpu)
1898	{
1899	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1900
1901	/*
1902	* Note: If you're adding MSRs here, make sure to update the MSR-bitmap accesses in hmR0VmxSetupVmcsProcCtls().
1903	*/
1904	if (!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST))
1905	{
1906	Assert(!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)); /* Guest MSRs better not be loaded now. */
1907	#if HC_ARCH_BITS == 64
1908	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1909	{
1910	pVCpu->hm.s.vmx.u64HostMsrLStar = ASMRdMsr(MSR_K8_LSTAR);
1911	pVCpu->hm.s.vmx.u64HostMsrStar = ASMRdMsr(MSR_K6_STAR);
1912	pVCpu->hm.s.vmx.u64HostMsrSfMask = ASMRdMsr(MSR_K8_SF_MASK);
1913	pVCpu->hm.s.vmx.u64HostMsrKernelGsBase = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1914	}
1915	#endif
1916	pVCpu->hm.s.vmx.fLazyMsrs \|= VMX_LAZY_MSRS_SAVED_HOST;
1917	}
1918	}
1919
1920
1921	/**
1922	* Checks whether the MSR belongs to the set of guest MSRs that we restore
1923	* lazily while leaving VT-x.
1924	*
1925	* @returns true if it does, false otherwise.
1926	* @param pVCpu The cross context virtual CPU structure.
1927	* @param idMsr The MSR to check.
1928	*/
1929	static bool hmR0VmxIsLazyGuestMsr(PVMCPU pVCpu, uint32_t idMsr)
1930	{
1931	NOREF(pVCpu);
1932	#if HC_ARCH_BITS == 64
1933	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1934	{
1935	switch (idMsr)
1936	{
1937	case MSR_K8_LSTAR:
1938	case MSR_K6_STAR:
1939	case MSR_K8_SF_MASK:
1940	case MSR_K8_KERNEL_GS_BASE:
1941	return true;
1942	}
1943	}
1944	#else
1945	RT_NOREF(pVCpu, idMsr);
1946	#endif
1947	return false;
1948	}
1949
1950
1951	/**
1952	* Loads a set of guests MSRs to allow read/passthru to the guest.
1953	*
1954	* The name of this function is slightly confusing. This function does NOT
1955	* postpone loading, but loads the MSR right now. "hmR0VmxLazy" is simply a
1956	* common prefix for functions dealing with "lazy restoration" of the shared
1957	* MSRs.
1958	*
1959	* @param pVCpu The cross context virtual CPU structure.
1960	*
1961	* @remarks No-long-jump zone!!!
1962	*/
1963	static void hmR0VmxLazyLoadGuestMsrs(PVMCPU pVCpu)
1964	{
1965	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1966	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1967
1968	Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1969	#if HC_ARCH_BITS == 64
1970	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1971	{
1972	/*
1973	* If the guest MSRs are not loaded -and- if all the guest MSRs are identical
1974	* to the MSRs on the CPU (which are the saved host MSRs, see assertion above) then
1975	* we can skip a few MSR writes.
1976	*
1977	* Otherwise, it implies either 1. they're not loaded, or 2. they're loaded but the
1978	* guest MSR values in the guest-CPU context might be different to what's currently
1979	* loaded in the CPU. In either case, we need to write the new guest MSR values to the
1980	* CPU, see @bugref{8728}.
1981	*/
1982	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
1983	if ( !(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1984	&& pCtx->msrKERNELGSBASE == pVCpu->hm.s.vmx.u64HostMsrKernelGsBase
1985	&& pCtx->msrLSTAR == pVCpu->hm.s.vmx.u64HostMsrLStar
1986	&& pCtx->msrSTAR == pVCpu->hm.s.vmx.u64HostMsrStar
1987	&& pCtx->msrSFMASK == pVCpu->hm.s.vmx.u64HostMsrSfMask)
1988	{
1989	#ifdef VBOX_STRICT
1990	Assert(ASMRdMsr(MSR_K8_KERNEL_GS_BASE) == pCtx->msrKERNELGSBASE);
1991	Assert(ASMRdMsr(MSR_K8_LSTAR) == pCtx->msrLSTAR);
1992	Assert(ASMRdMsr(MSR_K6_STAR) == pCtx->msrSTAR);
1993	Assert(ASMRdMsr(MSR_K8_SF_MASK) == pCtx->msrSFMASK);
1994	#endif
1995	}
1996	else
1997	{
1998	ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pCtx->msrKERNELGSBASE);
1999	ASMWrMsr(MSR_K8_LSTAR, pCtx->msrLSTAR);
2000	ASMWrMsr(MSR_K6_STAR, pCtx->msrSTAR);
2001	ASMWrMsr(MSR_K8_SF_MASK, pCtx->msrSFMASK);
2002	}
2003	}
2004	#endif
2005	pVCpu->hm.s.vmx.fLazyMsrs \|= VMX_LAZY_MSRS_LOADED_GUEST;
2006	}
2007
2008
2009	/**
2010	* Performs lazy restoration of the set of host MSRs if they were previously
2011	* loaded with guest MSR values.
2012	*
2013	* @param pVCpu The cross context virtual CPU structure.
2014	*
2015	* @remarks No-long-jump zone!!!
2016	* @remarks The guest MSRs should have been saved back into the guest-CPU
2017	* context by hmR0VmxImportGuestState()!!!
2018	*/
2019	static void hmR0VmxLazyRestoreHostMsrs(PVMCPU pVCpu)
2020	{
2021	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2022	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
2023
2024	if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
2025	{
2026	Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
2027	#if HC_ARCH_BITS == 64
2028	if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
2029	{
2030	ASMWrMsr(MSR_K8_LSTAR, pVCpu->hm.s.vmx.u64HostMsrLStar);
2031	ASMWrMsr(MSR_K6_STAR, pVCpu->hm.s.vmx.u64HostMsrStar);
2032	ASMWrMsr(MSR_K8_SF_MASK, pVCpu->hm.s.vmx.u64HostMsrSfMask);
2033	ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pVCpu->hm.s.vmx.u64HostMsrKernelGsBase);
2034	}
2035	#endif
2036	}
2037	pVCpu->hm.s.vmx.fLazyMsrs &= ~(VMX_LAZY_MSRS_LOADED_GUEST \| VMX_LAZY_MSRS_SAVED_HOST);
2038	}
2039
2040
2041	/**
2042	* Verifies that our cached values of the VMCS fields are all consistent with
2043	* what's actually present in the VMCS.
2044	*
2045	* @returns VBox status code.
2046	* @retval VINF_SUCCESS if all our caches match their respective VMCS fields.
2047	* @retval VERR_VMX_VMCS_FIELD_CACHE_INVALID if a cache field doesn't match the
2048	* VMCS content. HMCPU error-field is
2049	* updated, see VMX_VCI_XXX.
2050	* @param pVCpu The cross context virtual CPU structure.
2051	* @param pVmcsInfo The VMCS info. object.
2052	*/
2053	static int hmR0VmxCheckVmcsCtls(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
2054	{
2055	uint32_t u32Val;
2056	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
2057	AssertRCReturn(rc, rc);
2058	AssertMsgReturnStmt(pVmcsInfo->u32EntryCtls == u32Val,
2059	("Cache=%#RX32 VMCS=%#RX32\n", pVmcsInfo->u32EntryCtls, u32Val),
2060	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_ENTRY,
2061	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2062
2063	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val);
2064	AssertRCReturn(rc, rc);
2065	AssertMsgReturnStmt(pVmcsInfo->u32ExitCtls == u32Val,
2066	("Cache=%#RX32 VMCS=%#RX32\n", pVmcsInfo->u32ExitCtls, u32Val),
2067	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_EXIT,
2068	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2069
2070	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val);
2071	AssertRCReturn(rc, rc);
2072	AssertMsgReturnStmt(pVmcsInfo->u32PinCtls == u32Val,
2073	("Cache=%#RX32 VMCS=%#RX32\n", pVmcsInfo->u32PinCtls, u32Val),
2074	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_PIN_EXEC,
2075	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2076
2077	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val);
2078	AssertRCReturn(rc, rc);
2079	AssertMsgReturnStmt(pVmcsInfo->u32ProcCtls == u32Val,
2080	("Cache=%#RX32 VMCS=%#RX32\n", pVmcsInfo->u32ProcCtls, u32Val),
2081	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_PROC_EXEC,
2082	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2083
2084	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
2085	{
2086	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val);
2087	AssertRCReturn(rc, rc);
2088	AssertMsgReturnStmt(pVmcsInfo->u32ProcCtls2 == u32Val,
2089	("Cache=%#RX32 VMCS=%#RX32\n", pVmcsInfo->u32ProcCtls2, u32Val),
2090	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_PROC_EXEC2,
2091	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2092	}
2093
2094	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val);
2095	AssertRCReturn(rc, rc);
2096	AssertMsgReturnStmt(pVmcsInfo->u32XcptBitmap == u32Val,
2097	("Cache=%#RX32 VMCS=%#RX32\n", pVmcsInfo->u32XcptBitmap, u32Val),
2098	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_XCPT_BITMAP,
2099	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2100
2101	uint64_t u64Val;
2102	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, &u64Val);
2103	AssertRCReturn(rc, rc);
2104	AssertMsgReturnStmt(pVmcsInfo->u64TscOffset == u64Val,
2105	("Cache=%#RX64 VMCS=%#RX64\n", pVmcsInfo->u64TscOffset, u64Val),
2106	pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_TSC_OFFSET,
2107	VERR_VMX_VMCS_FIELD_CACHE_INVALID);
2108
2109	return VINF_SUCCESS;
2110	}
2111
2112
2113	#ifdef VBOX_STRICT
2114	/**
2115	* Verifies that our cached host EFER MSR value has not changed since we cached it.
2116	*
2117	* @param pVCpu The cross context virtual CPU structure.
2118	* @param pVmcsInfo The VMCS info. object.
2119	*/
2120	static void hmR0VmxCheckHostEferMsr(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
2121	{
2122	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2123
2124	if (pVmcsInfo->u32ExitCtls & VMX_EXIT_CTLS_LOAD_EFER_MSR)
2125	{
2126	uint64_t const uHostEferMsr = ASMRdMsr(MSR_K6_EFER);
2127	uint64_t const uHostEferMsrCache = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostMsrEfer;
2128	uint64_t uVmcsEferMsrVmcs;
2129	int rc = VMXReadVmcs64(VMX_VMCS64_HOST_EFER_FULL, &uVmcsEferMsrVmcs);
2130	AssertRC(rc);
2131
2132	AssertMsgReturnVoid(uHostEferMsr == uVmcsEferMsrVmcs,
2133	("EFER Host/VMCS mismatch! host=%#RX64 vmcs=%#RX64\n", uHostEferMsr, uVmcsEferMsrVmcs));
2134	AssertMsgReturnVoid(uHostEferMsr == uHostEferMsrCache,
2135	("EFER Host/Cache mismatch! host=%#RX64 cache=%#RX64\n", uHostEferMsr, uHostEferMsrCache));
2136	}
2137	}
2138
2139
2140	/**
2141	* Verifies whether the guest/host MSR pairs in the auto-load/store area in the
2142	* VMCS are correct.
2143	*
2144	* @param pVCpu The cross context virtual CPU structure.
2145	* @param pVmcsInfo The VMCS info. object.
2146	*/
2147	static void hmR0VmxCheckAutoLoadStoreMsrs(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
2148	{
2149	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2150
2151	/* Verify MSR counts in the VMCS are what we think it should be. */
2152	uint32_t cMsrs;
2153	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &cMsrs);
2154	AssertRC(rc);
2155	Assert(cMsrs == pVmcsInfo->cEntryMsrLoad);
2156
2157	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &cMsrs);
2158	AssertRC(rc);
2159	Assert(cMsrs == pVmcsInfo->cExitMsrStore);
2160
2161	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &cMsrs);
2162	AssertRC(rc);
2163	Assert(cMsrs == pVmcsInfo->cExitMsrLoad);
2164
2165	/* Verify the MSR counts do not exceed the maximum count supported by the hardware. */
2166	Assert(cMsrs < VMX_MISC_MAX_MSRS(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc));
2167
2168	PCVMXAUTOMSR pGuestMsrLoad = (PCVMXAUTOMSR)pVmcsInfo->pvGuestMsrLoad;
2169	PCVMXAUTOMSR pGuestMsrStore = (PCVMXAUTOMSR)pVmcsInfo->pvGuestMsrStore;
2170	PCVMXAUTOMSR pHostMsrLoad = (PCVMXAUTOMSR)pVmcsInfo->pvHostMsrLoad;
2171	bool const fSeparateExitMsrStorePage = hmR0VmxIsSeparateExitMsrStoreAreaVmcs(pVmcsInfo);
2172	for (uint32_t i = 0; i < cMsrs; i++)
2173	{
2174	/* Verify that the MSRs are paired properly and that the host MSR has the correct value. */
2175	if (fSeparateExitMsrStorePage)
2176	{
2177	AssertMsgReturnVoid(pGuestMsrLoad->u32Msr == pGuestMsrStore->u32Msr,
2178	("GuestMsrLoad=%#RX32 GuestMsrStore=%#RX32 cMsrs=%u\n",
2179	pGuestMsrLoad->u32Msr, pGuestMsrStore->u32Msr, cMsrs));
2180	}
2181
2182	AssertMsgReturnVoid(pHostMsrLoad->u32Msr == pGuestMsrLoad->u32Msr,
2183	("HostMsrLoad=%#RX32 GuestMsrLoad=%#RX32 cMsrs=%u\n",
2184	pHostMsrLoad->u32Msr, pGuestMsrLoad->u32Msr, cMsrs));
2185
2186	uint64_t const u64Msr = ASMRdMsr(pHostMsrLoad->u32Msr);
2187	AssertMsgReturnVoid(pHostMsrLoad->u64Value == u64Msr,
2188	("u32Msr=%#RX32 VMCS Value=%#RX64 ASMRdMsr=%#RX64 cMsrs=%u\n",
2189	pHostMsrLoad->u32Msr, pHostMsrLoad->u64Value, u64Msr, cMsrs));
2190
2191	/* Verify that the accesses are as expected in the MSR bitmap for auto-load/store MSRs. */
2192	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
2193	{
2194	uint32_t fMsrpm = HMGetVmxMsrPermission(pVmcsInfo->pvMsrBitmap, pGuestMsrLoad->u32Msr);
2195	if (pGuestMsrLoad->u32Msr == MSR_K6_EFER)
2196	{
2197	AssertMsgReturnVoid((fMsrpm & VMXMSRPM_EXIT_RD), ("Passthru read for EFER MSR!?\n"));
2198	AssertMsgReturnVoid((fMsrpm & VMXMSRPM_EXIT_WR), ("Passthru write for EFER MSR!?\n"));
2199	}
2200	else
2201	{
2202	AssertMsgReturnVoid((fMsrpm & (VMXMSRPM_ALLOW_RD \| VMXMSRPM_ALLOW_WR))
2203	== (VMXMSRPM_ALLOW_RD \| VMXMSRPM_ALLOW_WR),
2204	("u32Msr=%#RX32 cMsrs=%u No passthru read/write!\n", pGuestMsrLoad->u32Msr, cMsrs));
2205	}
2206	}
2207
2208	/* Move to the next MSR. */
2209	pHostMsrLoad++;
2210	pGuestMsrLoad++;
2211	pGuestMsrStore++;
2212	}
2213	}
2214	#endif /* VBOX_STRICT */
2215
2216
2217	/**
2218	* Flushes the TLB using EPT.
2219	*
2220	* @returns VBox status code.
2221	* @param pVCpu The cross context virtual CPU structure of the calling
2222	* EMT. Can be NULL depending on @a enmTlbFlush.
2223	* @param pVmcsInfo The VMCS info. object. Can be NULL depending on @a
2224	* enmTlbFlush.
2225	* @param enmTlbFlush Type of flush.
2226	*
2227	* @remarks Caller is responsible for making sure this function is called only
2228	* when NestedPaging is supported and providing @a enmTlbFlush that is
2229	* supported by the CPU.
2230	* @remarks Can be called with interrupts disabled.
2231	*/
2232	static void hmR0VmxFlushEpt(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo, VMXTLBFLUSHEPT enmTlbFlush)
2233	{
2234	uint64_t au64Descriptor[2];
2235	if (enmTlbFlush == VMXTLBFLUSHEPT_ALL_CONTEXTS)
2236	au64Descriptor[0] = 0;
2237	else
2238	{
2239	Assert(pVCpu);
2240	Assert(pVmcsInfo);
2241	au64Descriptor[0] = pVmcsInfo->HCPhysEPTP;
2242	}
2243	au64Descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
2244
2245	int rc = VMXR0InvEPT(enmTlbFlush, &au64Descriptor[0]);
2246	AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %#x %#RHp failed. rc=%Rrc\n", enmTlbFlush, au64Descriptor[0], rc));
2247
2248	if ( RT_SUCCESS(rc)
2249	&& pVCpu)
2250	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
2251	}
2252
2253
2254	/**
2255	* Flushes the TLB using VPID.
2256	*
2257	* @returns VBox status code.
2258	* @param pVCpu The cross context virtual CPU structure of the calling
2259	* EMT. Can be NULL depending on @a enmTlbFlush.
2260	* @param enmTlbFlush Type of flush.
2261	* @param GCPtr Virtual address of the page to flush (can be 0 depending
2262	* on @a enmTlbFlush).
2263	*
2264	* @remarks Can be called with interrupts disabled.
2265	*/
2266	static void hmR0VmxFlushVpid(PVMCPU pVCpu, VMXTLBFLUSHVPID enmTlbFlush, RTGCPTR GCPtr)
2267	{
2268	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid);
2269
2270	uint64_t au64Descriptor[2];
2271	if (enmTlbFlush == VMXTLBFLUSHVPID_ALL_CONTEXTS)
2272	{
2273	au64Descriptor[0] = 0;
2274	au64Descriptor[1] = 0;
2275	}
2276	else
2277	{
2278	AssertPtr(pVCpu);
2279	AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
2280	AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
2281	au64Descriptor[0] = pVCpu->hm.s.uCurrentAsid;
2282	au64Descriptor[1] = GCPtr;
2283	}
2284
2285	int rc = VMXR0InvVPID(enmTlbFlush, &au64Descriptor[0]);
2286	AssertMsg(rc == VINF_SUCCESS,
2287	("VMXR0InvVPID %#x %u %RGv failed with %Rrc\n", enmTlbFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
2288
2289	if ( RT_SUCCESS(rc)
2290	&& pVCpu)
2291	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
2292	NOREF(rc);
2293	}
2294
2295
2296	/**
2297	* Invalidates a guest page by guest virtual address. Only relevant for EPT/VPID,
2298	* otherwise there is nothing really to invalidate.
2299	*
2300	* @returns VBox status code.
2301	* @param pVCpu The cross context virtual CPU structure.
2302	* @param GCVirt Guest virtual address of the page to invalidate.
2303	*/
2304	VMMR0DECL(int) VMXR0InvalidatePage(PVMCPU pVCpu, RTGCPTR GCVirt)
2305	{
2306	AssertPtr(pVCpu);
2307	LogFlowFunc(("pVCpu=%p GCVirt=%RGv\n", pVCpu, GCVirt));
2308
2309	if (!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_TLB_FLUSH))
2310	{
2311	/*
2312	* We must invalidate the guest TLB entry in either case, we cannot ignore it even for
2313	* the EPT case. See @bugref{6043} and @bugref{6177}.
2314	*
2315	* Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*()
2316	* as this function maybe called in a loop with individual addresses.
2317	*/
2318	PVM pVM = pVCpu->CTX_SUFF(pVM);
2319	if (pVM->hm.s.vmx.fVpid)
2320	{
2321	bool fVpidFlush = RT_BOOL(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR);
2322
2323	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
2324	/*
2325	* Workaround Erratum BV75, AAJ159 and others that affect several Intel CPUs
2326	* where executing INVVPID outside 64-bit mode does not flush translations of
2327	* 64-bit linear addresses, see @bugref{6208#c72}.
2328	*/
2329	if (RT_HI_U32(GCVirt))
2330	fVpidFlush = false;
2331	#endif
2332
2333	if (fVpidFlush)
2334	{
2335	hmR0VmxFlushVpid(pVCpu, VMXTLBFLUSHVPID_INDIV_ADDR, GCVirt);
2336	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
2337	}
2338	else
2339	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2340	}
2341	else if (pVM->hm.s.fNestedPaging)
2342	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2343	}
2344
2345	return VINF_SUCCESS;
2346	}
2347
2348
2349	/**
2350	* Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
2351	* case where neither EPT nor VPID is supported by the CPU.
2352	*
2353	* @param pHostCpu The HM physical-CPU structure.
2354	* @param pVCpu The cross context virtual CPU structure.
2355	*
2356	* @remarks Called with interrupts disabled.
2357	*/
2358	static void hmR0VmxFlushTaggedTlbNone(PHMPHYSCPU pHostCpu, PVMCPU pVCpu)
2359	{
2360	AssertPtr(pVCpu);
2361	AssertPtr(pHostCpu);
2362
2363	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
2364
2365	Assert(pHostCpu->idCpu != NIL_RTCPUID);
2366	pVCpu->hm.s.idLastCpu = pHostCpu->idCpu;
2367	pVCpu->hm.s.cTlbFlushes = pHostCpu->cTlbFlushes;
2368	pVCpu->hm.s.fForceTLBFlush = false;
2369	return;
2370	}
2371
2372
2373	/**
2374	* Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
2375	*
2376	* @param pHostCpu The HM physical-CPU structure.
2377	* @param pVCpu The cross context virtual CPU structure.
2378	* @param pVmcsInfo The VMCS info. object.
2379	*
2380	* @remarks All references to "ASID" in this function pertains to "VPID" in Intel's
2381	* nomenclature. The reason is, to avoid confusion in compare statements
2382	* since the host-CPU copies are named "ASID".
2383	*
2384	* @remarks Called with interrupts disabled.
2385	*/
2386	static void hmR0VmxFlushTaggedTlbBoth(PHMPHYSCPU pHostCpu, PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
2387	{
2388	#ifdef VBOX_WITH_STATISTICS
2389	bool fTlbFlushed = false;
2390	# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { fTlbFlushed = true; } while (0)
2391	# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { \
2392	if (!fTlbFlushed) \
2393	STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch); \
2394	} while (0)
2395	#else
2396	# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { } while (0)
2397	# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { } while (0)
2398	#endif
2399
2400	AssertPtr(pVCpu);
2401	AssertPtr(pHostCpu);
2402	Assert(pHostCpu->idCpu != NIL_RTCPUID);
2403
2404	PVM pVM = pVCpu->CTX_SUFF(pVM);
2405	AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
2406	("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
2407	"fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
2408
2409	/*
2410	* Force a TLB flush for the first world-switch if the current CPU differs from the one we
2411	* ran on last. If the TLB flush count changed, another VM (VCPU rather) has hit the ASID
2412	* limit while flushing the TLB or the host CPU is online after a suspend/resume, so we
2413	* cannot reuse the current ASID anymore.
2414	*/
2415	if ( pVCpu->hm.s.idLastCpu != pHostCpu->idCpu
2416	\|\| pVCpu->hm.s.cTlbFlushes != pHostCpu->cTlbFlushes)
2417	{
2418	++pHostCpu->uCurrentAsid;
2419	if (pHostCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
2420	{
2421	pHostCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0. */
2422	pHostCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
2423	pHostCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
2424	}
2425
2426	pVCpu->hm.s.uCurrentAsid = pHostCpu->uCurrentAsid;
2427	pVCpu->hm.s.idLastCpu = pHostCpu->idCpu;
2428	pVCpu->hm.s.cTlbFlushes = pHostCpu->cTlbFlushes;
2429
2430	/*
2431	* Flush by EPT when we get rescheduled to a new host CPU to ensure EPT-only tagged mappings are also
2432	* invalidated. We don't need to flush-by-VPID here as flushing by EPT covers it. See @bugref{6568}.
2433	*/
2434	hmR0VmxFlushEpt(pVCpu, pVmcsInfo, pVM->hm.s.vmx.enmTlbFlushEpt);
2435	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2436	HMVMX_SET_TAGGED_TLB_FLUSHED();
2437	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
2438	}
2439	else if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH)) /* Check for explicit TLB flushes. */
2440	{
2441	/*
2442	* Changes to the EPT paging structure by VMM requires flushing-by-EPT as the CPU
2443	* creates guest-physical (ie. only EPT-tagged) mappings while traversing the EPT
2444	* tables when EPT is in use. Flushing-by-VPID will only flush linear (only
2445	* VPID-tagged) and combined (EPT+VPID tagged) mappings but not guest-physical
2446	* mappings, see @bugref{6568}.
2447	*
2448	* See Intel spec. 28.3.2 "Creating and Using Cached Translation Information".
2449	*/
2450	hmR0VmxFlushEpt(pVCpu, pVmcsInfo, pVM->hm.s.vmx.enmTlbFlushEpt);
2451	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2452	HMVMX_SET_TAGGED_TLB_FLUSHED();
2453	}
2454
2455	pVCpu->hm.s.fForceTLBFlush = false;
2456	HMVMX_UPDATE_FLUSH_SKIPPED_STAT();
2457
2458	Assert(pVCpu->hm.s.idLastCpu == pHostCpu->idCpu);
2459	Assert(pVCpu->hm.s.cTlbFlushes == pHostCpu->cTlbFlushes);
2460	AssertMsg(pVCpu->hm.s.cTlbFlushes == pHostCpu->cTlbFlushes,
2461	("Flush count mismatch for cpu %d (%u vs %u)\n", pHostCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pHostCpu->cTlbFlushes));
2462	AssertMsg(pHostCpu->uCurrentAsid >= 1 && pHostCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2463	("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pHostCpu->idCpu,
2464	pHostCpu->uCurrentAsid, pHostCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2465	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2466	("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pHostCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2467
2468	/* Update VMCS with the VPID. */
2469	int rc = VMXWriteVmcs32(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
2470	AssertRC(rc);
2471
2472	#undef HMVMX_SET_TAGGED_TLB_FLUSHED
2473	}
2474
2475
2476	/**
2477	* Flushes the tagged-TLB entries for EPT CPUs as necessary.
2478	*
2479	* @param pHostCpu The HM physical-CPU structure.
2480	* @param pVCpu The cross context virtual CPU structure.
2481	* @param pVmcsInfo The VMCS info. object.
2482	*
2483	* @remarks Called with interrupts disabled.
2484	*/
2485	static void hmR0VmxFlushTaggedTlbEpt(PHMPHYSCPU pHostCpu, PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
2486	{
2487	AssertPtr(pVCpu);
2488	AssertPtr(pHostCpu);
2489	Assert(pHostCpu->idCpu != NIL_RTCPUID);
2490	AssertMsg(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked without NestedPaging."));
2491	AssertMsg(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID."));
2492
2493	/*
2494	* Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
2495	* A change in the TLB flush count implies the host CPU is online after a suspend/resume.
2496	*/
2497	if ( pVCpu->hm.s.idLastCpu != pHostCpu->idCpu
2498	\|\| pVCpu->hm.s.cTlbFlushes != pHostCpu->cTlbFlushes)
2499	{
2500	pVCpu->hm.s.fForceTLBFlush = true;
2501	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2502	}
2503
2504	/* Check for explicit TLB flushes. */
2505	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2506	{
2507	pVCpu->hm.s.fForceTLBFlush = true;
2508	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2509	}
2510
2511	pVCpu->hm.s.idLastCpu = pHostCpu->idCpu;
2512	pVCpu->hm.s.cTlbFlushes = pHostCpu->cTlbFlushes;
2513
2514	if (pVCpu->hm.s.fForceTLBFlush)
2515	{
2516	hmR0VmxFlushEpt(pVCpu, pVmcsInfo, pVCpu->CTX_SUFF(pVM)->hm.s.vmx.enmTlbFlushEpt);
2517	pVCpu->hm.s.fForceTLBFlush = false;
2518	}
2519	}
2520
2521
2522	/**
2523	* Flushes the tagged-TLB entries for VPID CPUs as necessary.
2524	*
2525	* @param pHostCpu The HM physical-CPU structure.
2526	* @param pVCpu The cross context virtual CPU structure.
2527	*
2528	* @remarks Called with interrupts disabled.
2529	*/
2530	static void hmR0VmxFlushTaggedTlbVpid(PHMPHYSCPU pHostCpu, PVMCPU pVCpu)
2531	{
2532	AssertPtr(pVCpu);
2533	AssertPtr(pHostCpu);
2534	Assert(pHostCpu->idCpu != NIL_RTCPUID);
2535	AssertMsg(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked without VPID."));
2536	AssertMsg(!pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging"));
2537
2538	/*
2539	* Force a TLB flush for the first world switch if the current CPU differs from the one we
2540	* ran on last. If the TLB flush count changed, another VM (VCPU rather) has hit the ASID
2541	* limit while flushing the TLB or the host CPU is online after a suspend/resume, so we
2542	* cannot reuse the current ASID anymore.
2543	*/
2544	if ( pVCpu->hm.s.idLastCpu != pHostCpu->idCpu
2545	\|\| pVCpu->hm.s.cTlbFlushes != pHostCpu->cTlbFlushes)
2546	{
2547	pVCpu->hm.s.fForceTLBFlush = true;
2548	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2549	}
2550
2551	/* Check for explicit TLB flushes. */
2552	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2553	{
2554	/*
2555	* If we ever support VPID flush combinations other than ALL or SINGLE-context (see
2556	* hmR0VmxSetupTaggedTlb()) we would need to explicitly flush in this case (add an
2557	* fExplicitFlush = true here and change the pHostCpu->fFlushAsidBeforeUse check below to
2558	* include fExplicitFlush's too) - an obscure corner case.
2559	*/
2560	pVCpu->hm.s.fForceTLBFlush = true;
2561	STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2562	}
2563
2564	PVM pVM = pVCpu->CTX_SUFF(pVM);
2565	pVCpu->hm.s.idLastCpu = pHostCpu->idCpu;
2566	if (pVCpu->hm.s.fForceTLBFlush)
2567	{
2568	++pHostCpu->uCurrentAsid;
2569	if (pHostCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
2570	{
2571	pHostCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0 */
2572	pHostCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
2573	pHostCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
2574	}
2575
2576	pVCpu->hm.s.fForceTLBFlush = false;
2577	pVCpu->hm.s.cTlbFlushes = pHostCpu->cTlbFlushes;
2578	pVCpu->hm.s.uCurrentAsid = pHostCpu->uCurrentAsid;
2579	if (pHostCpu->fFlushAsidBeforeUse)
2580	{
2581	if (pVM->hm.s.vmx.enmTlbFlushVpid == VMXTLBFLUSHVPID_SINGLE_CONTEXT)
2582	hmR0VmxFlushVpid(pVCpu, VMXTLBFLUSHVPID_SINGLE_CONTEXT, 0 /* GCPtr */);
2583	else if (pVM->hm.s.vmx.enmTlbFlushVpid == VMXTLBFLUSHVPID_ALL_CONTEXTS)
2584	{
2585	hmR0VmxFlushVpid(pVCpu, VMXTLBFLUSHVPID_ALL_CONTEXTS, 0 /* GCPtr */);
2586	pHostCpu->fFlushAsidBeforeUse = false;
2587	}
2588	else
2589	{
2590	/* hmR0VmxSetupTaggedTlb() ensures we never get here. Paranoia. */
2591	AssertMsgFailed(("Unsupported VPID-flush context type.\n"));
2592	}
2593	}
2594	}
2595
2596	AssertMsg(pVCpu->hm.s.cTlbFlushes == pHostCpu->cTlbFlushes,
2597	("Flush count mismatch for cpu %d (%u vs %u)\n", pHostCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pHostCpu->cTlbFlushes));
2598	AssertMsg(pHostCpu->uCurrentAsid >= 1 && pHostCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2599	("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pHostCpu->idCpu,
2600	pHostCpu->uCurrentAsid, pHostCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2601	AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2602	("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pHostCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2603
2604	int rc = VMXWriteVmcs32(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
2605	AssertRC(rc);
2606	}
2607
2608
2609	/**
2610	* Flushes the guest TLB entry based on CPU capabilities.
2611	*
2612	* @param pHostCpu The HM physical-CPU structure.
2613	* @param pVCpu The cross context virtual CPU structure.
2614	* @param pVmcsInfo The VMCS info. object.
2615	*
2616	* @remarks Called with interrupts disabled.
2617	*/
2618	DECLINLINE(void) hmR0VmxFlushTaggedTlb(PHMPHYSCPU pHostCpu, PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
2619	{
2620	#ifdef HMVMX_ALWAYS_FLUSH_TLB
2621	VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2622	#endif
2623	PVM pVM = pVCpu->CTX_SUFF(pVM);
2624	switch (pVM->hm.s.vmx.enmTlbFlushType)
2625	{
2626	case VMXTLBFLUSHTYPE_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pHostCpu, pVCpu, pVmcsInfo); break;
2627	case VMXTLBFLUSHTYPE_EPT: hmR0VmxFlushTaggedTlbEpt(pHostCpu, pVCpu, pVmcsInfo); break;
2628	case VMXTLBFLUSHTYPE_VPID: hmR0VmxFlushTaggedTlbVpid(pHostCpu, pVCpu); break;
2629	case VMXTLBFLUSHTYPE_NONE: hmR0VmxFlushTaggedTlbNone(pHostCpu, pVCpu); break;
2630	default:
2631	AssertMsgFailed(("Invalid flush-tag function identifier\n"));
2632	break;
2633	}
2634	/* Don't assert that VMCPU_FF_TLB_FLUSH should no longer be pending. It can be set by other EMTs. */
2635	}
2636
2637
2638	/**
2639	* Sets up the appropriate tagged TLB-flush level and handler for flushing guest
2640	* TLB entries from the host TLB before VM-entry.
2641	*
2642	* @returns VBox status code.
2643	* @param pVM The cross context VM structure.
2644	*/
2645	static int hmR0VmxSetupTaggedTlb(PVM pVM)
2646	{
2647	/*
2648	* Determine optimal flush type for nested paging.
2649	* We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup
2650	* unrestricted guest execution (see hmR3InitFinalizeR0()).
2651	*/
2652	if (pVM->hm.s.fNestedPaging)
2653	{
2654	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
2655	{
2656	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
2657	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_SINGLE_CONTEXT;
2658	else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
2659	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_ALL_CONTEXTS;
2660	else
2661	{
2662	/* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
2663	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
2664	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_FLUSH_TYPE_UNSUPPORTED;
2665	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2666	}
2667
2668	/* Make sure the write-back cacheable memory type for EPT is supported. */
2669	if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB)))
2670	{
2671	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
2672	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_MEM_TYPE_NOT_WB;
2673	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2674	}
2675
2676	/* EPT requires a page-walk length of 4. */
2677	if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_PAGE_WALK_LENGTH_4)))
2678	{
2679	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
2680	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_PAGE_WALK_LENGTH_UNSUPPORTED;
2681	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2682	}
2683	}
2684	else
2685	{
2686	/* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
2687	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
2688	pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_INVEPT_UNAVAILABLE;
2689	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2690	}
2691	}
2692
2693	/*
2694	* Determine optimal flush type for VPID.
2695	*/
2696	if (pVM->hm.s.vmx.fVpid)
2697	{
2698	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
2699	{
2700	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
2701	pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_SINGLE_CONTEXT;
2702	else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
2703	pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_ALL_CONTEXTS;
2704	else
2705	{
2706	/* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
2707	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
2708	LogRelFunc(("Only INDIV_ADDR supported. Ignoring VPID.\n"));
2709	if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
2710	LogRelFunc(("Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
2711	pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_NOT_SUPPORTED;
2712	pVM->hm.s.vmx.fVpid = false;
2713	}
2714	}
2715	else
2716	{
2717	/* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
2718	Log4Func(("VPID supported without INVEPT support. Ignoring VPID.\n"));
2719	pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_NOT_SUPPORTED;
2720	pVM->hm.s.vmx.fVpid = false;
2721	}
2722	}
2723
2724	/*
2725	* Setup the handler for flushing tagged-TLBs.
2726	*/
2727	if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
2728	pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_EPT_VPID;
2729	else if (pVM->hm.s.fNestedPaging)
2730	pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_EPT;
2731	else if (pVM->hm.s.vmx.fVpid)
2732	pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_VPID;
2733	else
2734	pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_NONE;
2735	return VINF_SUCCESS;
2736	}
2737
2738
2739	/**
2740	* Sets up the virtual-APIC page address for the VMCS.
2741	*
2742	* @returns VBox status code.
2743	* @param pVCpu The cross context virtual CPU structure.
2744	* @param pVmcsInfo The VMCS info. object.
2745	*/
2746	DECLINLINE(int) hmR0VmxSetupVmcsVirtApicAddr(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
2747	{
2748	NOREF(pVCpu);
2749	RTHCPHYS const HCPhysVirtApic = pVmcsInfo->HCPhysVirtApic;
2750	Assert(HCPhysVirtApic != NIL_RTHCPHYS);
2751	Assert(!(HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
2752	return VMXWriteVmcs64(VMX_VMCS64_CTRL_VIRT_APIC_PAGEADDR_FULL, HCPhysVirtApic);
2753	}
2754
2755
2756	/**
2757	* Sets up the MSR-bitmap address for the VMCS.
2758	*
2759	* @returns VBox status code.
2760	* @param pVmcsInfo The VMCS info. object.
2761	*/
2762	DECLINLINE(int) hmR0VmxSetupVmcsMsrBitmapAddr(PCVMXVMCSINFO pVmcsInfo)
2763	{
2764	RTHCPHYS const HCPhysMsrBitmap = pVmcsInfo->HCPhysMsrBitmap;
2765	Assert(HCPhysMsrBitmap != NIL_RTHCPHYS);
2766	Assert(!(HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
2767	return VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, HCPhysMsrBitmap);
2768	}
2769
2770
2771	/**
2772	* Sets up the APIC-access page address for the VMCS.
2773	*
2774	* @returns VBox status code.
2775	* @param pVCpu The cross context virtual CPU structure.
2776	*/
2777	DECLINLINE(int) hmR0VmxSetupVmcsApicAccessAddr(PVMCPU pVCpu)
2778	{
2779	RTHCPHYS const HCPhysApicAccess = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.HCPhysApicAccess;
2780	Assert(HCPhysApicAccess != NIL_RTHCPHYS);
2781	Assert(!(HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
2782	return VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, HCPhysApicAccess);
2783	}
2784
2785
2786	/**
2787	* Sets up the VMCS link pointer for the VMCS.
2788	*
2789	* @returns VBox status code.
2790	* @param pVmcsInfo The VMCS info. object.
2791	*/
2792	DECLINLINE(int) hmR0VmxSetupVmcsLinkPtr(PVMXVMCSINFO pVmcsInfo)
2793	{
2794	uint64_t const u64VmcsLinkPtr = pVmcsInfo->u64VmcsLinkPtr;
2795	Assert(u64VmcsLinkPtr == UINT64_C(0xffffffffffffffff)); /* Bits 63:0 MB1. */
2796	return VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, u64VmcsLinkPtr);
2797	}
2798
2799
2800	/**
2801	* Sets up the VM-entry MSR load, VM-exit MSR-store and VM-exit MSR-load addresses
2802	* in the VMCS.
2803	*
2804	* @returns VBox status code.
2805	* @param pVmcsInfo The VMCS info. object.
2806	*/
2807	DECLINLINE(int) hmR0VmxSetupVmcsAutoLoadStoreMsrAddrs(PVMXVMCSINFO pVmcsInfo)
2808	{
2809	RTHCPHYS const HCPhysGuestMsrLoad = pVmcsInfo->HCPhysGuestMsrLoad;
2810	Assert(HCPhysGuestMsrLoad != NIL_RTHCPHYS);
2811	Assert(!(HCPhysGuestMsrLoad & 0xf)); /* Bits 3:0 MBZ. */
2812
2813	RTHCPHYS const HCPhysGuestMsrStore = pVmcsInfo->HCPhysGuestMsrStore;
2814	Assert(HCPhysGuestMsrStore != NIL_RTHCPHYS);
2815	Assert(!(HCPhysGuestMsrStore & 0xf)); /* Bits 3:0 MBZ. */
2816
2817	RTHCPHYS const HCPhysHostMsrLoad = pVmcsInfo->HCPhysHostMsrLoad;
2818	Assert(HCPhysHostMsrLoad != NIL_RTHCPHYS);
2819	Assert(!(HCPhysHostMsrLoad & 0xf)); /* Bits 3:0 MBZ. */
2820
2821	int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, HCPhysGuestMsrLoad);
2822	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, HCPhysGuestMsrStore);
2823	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, HCPhysHostMsrLoad);
2824	AssertRCReturn(rc, rc);
2825	return VINF_SUCCESS;
2826	}
2827
2828
2829	/**
2830	* Sets up MSR permissions in the MSR bitmap of a VMCS info. object.
2831	*
2832	* @param pVCpu The cross context virtual CPU structure.
2833	* @param pVmcsInfo The VMCS info. object.
2834	* @param fIsNstGstVmcs Whether this is a nested-guest VMCS.
2835	*/
2836	static void hmR0VmxSetupVmcsMsrPermissions(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, bool fIsNstGstVmcs)
2837	{
2838	Assert(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS);
2839
2840	/*
2841	* The guest can access the following MSRs (read, write) without causing
2842	* VM-exits; they are loaded/stored automatically using fields in the VMCS.
2843	*/
2844	PVM pVM = pVCpu->CTX_SUFF(pVM);
2845	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_IA32_SYSENTER_CS, VMXMSRPM_ALLOW_RD \| VMXMSRPM_ALLOW_WR);
2846	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_IA32_SYSENTER_ESP, VMXMSRPM_ALLOW_RD \| VMXMSRPM_ALLOW_WR);
2847	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_IA32_SYSENTER_EIP, VMXMSRPM_ALLOW_RD \| VMXMSRPM_ALLOW_WR);
2848	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_K8_GS_BASE, VMXMSRPM_ALLOW_RD \| VMXMSRPM_ALLOW_WR);
2849	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_K8_FS_BASE, VMXMSRPM_ALLOW_RD \| VMXMSRPM_ALLOW_WR);
2850
2851	#ifdef VBOX_STRICT
2852	/** @todo NSTVMX: Remove this later. */
2853	uint32_t fMsrpm = HMGetVmxMsrPermission(pVmcsInfo->pvMsrBitmap, MSR_IA32_SYSENTER_CS);
2854	Assert((fMsrpm & (VMXMSRPM_ALLOW_RD \| VMXMSRPM_ALLOW_WR)) == (VMXMSRPM_ALLOW_RD \| VMXMSRPM_ALLOW_WR));
2855
2856	fMsrpm = HMGetVmxMsrPermission(pVmcsInfo->pvMsrBitmap, MSR_K8_GS_BASE);
2857	Assert((fMsrpm & (VMXMSRPM_ALLOW_RD \| VMXMSRPM_ALLOW_WR)) == (VMXMSRPM_ALLOW_RD \| VMXMSRPM_ALLOW_WR));
2858	#endif
2859
2860	/*
2861	* The IA32_PRED_CMD and IA32_FLUSH_CMD MSRs are write-only and has no state
2862	* associated with then. We never need to intercept access (writes need to be
2863	* executed without causing a VM-exit, reads will #GP fault anyway).
2864	*
2865	* The IA32_SPEC_CTRL MSR is read/write and has state. We allow the guest to
2866	* read/write them. We swap the the guest/host MSR value using the
2867	* auto-load/store MSR area.
2868	*/
2869	if (pVM->cpum.ro.GuestFeatures.fIbpb)
2870	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_IA32_PRED_CMD, VMXMSRPM_ALLOW_RD \| VMXMSRPM_ALLOW_WR);
2871	if (pVM->cpum.ro.GuestFeatures.fFlushCmd)
2872	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_IA32_FLUSH_CMD, VMXMSRPM_ALLOW_RD \| VMXMSRPM_ALLOW_WR);
2873	if (pVM->cpum.ro.GuestFeatures.fIbrs)
2874	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_IA32_SPEC_CTRL, VMXMSRPM_ALLOW_RD \| VMXMSRPM_ALLOW_WR);
2875
2876	/*
2877	* IA32_EFER MSR is always intercepted, see @bugref{9180#c37}.
2878	*/
2879
2880	#if HC_ARCH_BITS == 64
2881	/*
2882	* Allow full read/write access for the following MSRs (mandatory for VT-x)
2883	* required for 64-bit guests.
2884	*/
2885	if (pVM->hm.s.fAllow64BitGuests)
2886	{
2887	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_K8_LSTAR, VMXMSRPM_ALLOW_RD \| VMXMSRPM_ALLOW_WR);
2888	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_K6_STAR, VMXMSRPM_ALLOW_RD \| VMXMSRPM_ALLOW_WR);
2889	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_K8_SF_MASK, VMXMSRPM_ALLOW_RD \| VMXMSRPM_ALLOW_WR);
2890	hmR0VmxSetMsrPermission(pVCpu, pVmcsInfo, fIsNstGstVmcs, MSR_K8_KERNEL_GS_BASE, VMXMSRPM_ALLOW_RD \| VMXMSRPM_ALLOW_WR);
2891
2892	# ifdef VBOX_STRICT
2893	fMsrpm = HMGetVmxMsrPermission(pVmcsInfo->pvMsrBitmap, MSR_K8_GS_BASE);
2894	Assert((fMsrpm & (VMXMSRPM_ALLOW_RD \| VMXMSRPM_ALLOW_WR)) == (VMXMSRPM_ALLOW_RD \| VMXMSRPM_ALLOW_WR));
2895	# endif
2896	}
2897	#endif
2898	}
2899
2900
2901	/**
2902	* Sets up pin-based VM-execution controls in the VMCS.
2903	*
2904	* @returns VBox status code.
2905	* @param pVCpu The cross context virtual CPU structure.
2906	* @param pVmcsInfo The VMCS info. object.
2907	*/
2908	static int hmR0VmxSetupVmcsPinCtls(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
2909	{
2910	PVM pVM = pVCpu->CTX_SUFF(pVM);
2911	uint32_t fVal = pVM->hm.s.vmx.Msrs.PinCtls.n.allowed0; /* Bits set here must always be set. */
2912	uint32_t const fZap = pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1; /* Bits cleared here must always be cleared. */
2913
2914	fVal \|= VMX_PIN_CTLS_EXT_INT_EXIT /* External interrupts cause a VM-exit. */
2915	\| VMX_PIN_CTLS_NMI_EXIT; /* Non-maskable interrupts (NMIs) cause a VM-exit. */
2916
2917	if (pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1 & VMX_PIN_CTLS_VIRT_NMI)
2918	fVal \|= VMX_PIN_CTLS_VIRT_NMI; /* Use virtual NMIs and virtual-NMI blocking features. */
2919
2920	/* Enable the VMX-preemption timer. */
2921	if (pVM->hm.s.vmx.fUsePreemptTimer)
2922	{
2923	Assert(pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1 & VMX_PIN_CTLS_PREEMPT_TIMER);
2924	fVal \|= VMX_PIN_CTLS_PREEMPT_TIMER;
2925	}
2926
2927	#if 0
2928	/* Enable posted-interrupt processing. */
2929	if (pVM->hm.s.fPostedIntrs)
2930	{
2931	Assert(pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1 & VMX_PIN_CTLS_POSTED_INT);
2932	Assert(pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_ACK_EXT_INT);
2933	fVal \|= VMX_PIN_CTL_POSTED_INT;
2934	}
2935	#endif
2936
2937	if ((fVal & fZap) != fVal)
2938	{
2939	LogRelFunc(("Invalid pin-based VM-execution controls combo! Cpu=%#RX32 fVal=%#RX32 fZap=%#RX32\n",
2940	pVM->hm.s.vmx.Msrs.PinCtls.n.allowed0, fVal, fZap));
2941	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PIN_EXEC;
2942	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2943	}
2944
2945	/* Commit it to the VMCS and update our cache. */
2946	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, fVal);
2947	AssertRCReturn(rc, rc);
2948	pVmcsInfo->u32PinCtls = fVal;
2949
2950	return VINF_SUCCESS;
2951	}
2952
2953
2954	/**
2955	* Sets up secondary processor-based VM-execution controls in the VMCS.
2956	*
2957	* @returns VBox status code.
2958	* @param pVCpu The cross context virtual CPU structure.
2959	* @param pVmcsInfo The VMCS info. object.
2960	*/
2961	static int hmR0VmxSetupVmcsProcCtls2(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
2962	{
2963	PVM pVM = pVCpu->CTX_SUFF(pVM);
2964	uint32_t fVal = pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed0; /* Bits set here must be set in the VMCS. */
2965	uint32_t const fZap = pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2966
2967	/* WBINVD causes a VM-exit. */
2968	if (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_WBINVD_EXIT)
2969	fVal \|= VMX_PROC_CTLS2_WBINVD_EXIT;
2970
2971	/* Enable EPT (aka nested-paging). */
2972	if (pVM->hm.s.fNestedPaging)
2973	fVal \|= VMX_PROC_CTLS2_EPT;
2974
2975	/* Enable the INVPCID instruction if supported by the hardware and we expose
2976	it to the guest. Without this, guest executing INVPCID would cause a #UD. */
2977	if ( (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_INVPCID)
2978	&& pVM->cpum.ro.GuestFeatures.fInvpcid)
2979	fVal \|= VMX_PROC_CTLS2_INVPCID;
2980
2981	/* Enable VPID. */
2982	if (pVM->hm.s.vmx.fVpid)
2983	fVal \|= VMX_PROC_CTLS2_VPID;
2984
2985	/* Enable unrestricted guest execution. */
2986	if (pVM->hm.s.vmx.fUnrestrictedGuest)
2987	fVal \|= VMX_PROC_CTLS2_UNRESTRICTED_GUEST;
2988
2989	#if 0
2990	if (pVM->hm.s.fVirtApicRegs)
2991	{
2992	/* Enable APIC-register virtualization. */
2993	Assert(pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_APIC_REG_VIRT);
2994	fVal \|= VMX_PROC_CTLS2_APIC_REG_VIRT;
2995
2996	/* Enable virtual-interrupt delivery. */
2997	Assert(pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_INTR_DELIVERY);
2998	fVal \|= VMX_PROC_CTLS2_VIRT_INTR_DELIVERY;
2999	}
3000	#endif
3001
3002	/* Virtualize-APIC accesses if supported by the CPU. The virtual-APIC page is where the TPR shadow resides. */
3003	/** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
3004	* done dynamically. */
3005	if (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
3006	{
3007	fVal \|= VMX_PROC_CTLS2_VIRT_APIC_ACCESS;
3008	int rc = hmR0VmxSetupVmcsApicAccessAddr(pVCpu);
3009	AssertRCReturn(rc, rc);
3010	}
3011
3012	/* Enable the RDTSCP instruction if supported by the hardware and we expose
3013	it to the guest. Without this, guest executing RDTSCP would cause a #UD. */
3014	if ( (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_RDTSCP)
3015	&& pVM->cpum.ro.GuestFeatures.fRdTscP)
3016	fVal \|= VMX_PROC_CTLS2_RDTSCP;
3017
3018	/* Enable Pause-Loop exiting. */
3019	if ( pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_PAUSE_LOOP_EXIT
3020	&& pVM->hm.s.vmx.cPleGapTicks
3021	&& pVM->hm.s.vmx.cPleWindowTicks)
3022	{
3023	fVal \|= VMX_PROC_CTLS2_PAUSE_LOOP_EXIT;
3024
3025	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_GAP, pVM->hm.s.vmx.cPleGapTicks);
3026	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_WINDOW, pVM->hm.s.vmx.cPleWindowTicks);
3027	AssertRCReturn(rc, rc);
3028	}
3029
3030	if ((fVal & fZap) != fVal)
3031	{
3032	LogRelFunc(("Invalid secondary processor-based VM-execution controls combo! cpu=%#RX32 fVal=%#RX32 fZap=%#RX32\n",
3033	pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed0, fVal, fZap));
3034	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC2;
3035	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3036	}
3037
3038	/* Commit it to the VMCS and update our cache. */
3039	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, fVal);
3040	AssertRCReturn(rc, rc);
3041	pVmcsInfo->u32ProcCtls2 = fVal;
3042
3043	return VINF_SUCCESS;
3044	}
3045
3046
3047	/**
3048	* Sets up processor-based VM-execution controls in the VMCS.
3049	*
3050	* @returns VBox status code.
3051	* @param pVCpu The cross context virtual CPU structure.
3052	* @param pVmcsInfo The VMCS info. object.
3053	*/
3054	static int hmR0VmxSetupVmcsProcCtls(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
3055	{
3056	PVM pVM = pVCpu->CTX_SUFF(pVM);
3057
3058	uint32_t fVal = pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed0; /* Bits set here must be set in the VMCS. */
3059	uint32_t const fZap = pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3060
3061	fVal \|= VMX_PROC_CTLS_HLT_EXIT /* HLT causes a VM-exit. */
3062	\| VMX_PROC_CTLS_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
3063	\| VMX_PROC_CTLS_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
3064	\| VMX_PROC_CTLS_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
3065	\| VMX_PROC_CTLS_RDPMC_EXIT /* RDPMC causes a VM-exit. */
3066	\| VMX_PROC_CTLS_MONITOR_EXIT /* MONITOR causes a VM-exit. */
3067	\| VMX_PROC_CTLS_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
3068
3069	/* We toggle VMX_PROC_CTLS_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
3070	if ( !(pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_MOV_DR_EXIT)
3071	\|\| (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed0 & VMX_PROC_CTLS_MOV_DR_EXIT))
3072	{
3073	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_MOV_DRX_EXIT;
3074	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3075	}
3076
3077	/* Without nested paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
3078	if (!pVM->hm.s.fNestedPaging)
3079	{
3080	Assert(!pVM->hm.s.vmx.fUnrestrictedGuest);
3081	fVal \|= VMX_PROC_CTLS_INVLPG_EXIT
3082	\| VMX_PROC_CTLS_CR3_LOAD_EXIT
3083	\| VMX_PROC_CTLS_CR3_STORE_EXIT;
3084	}
3085
3086	/* Use TPR shadowing if supported by the CPU. */
3087	if ( PDMHasApic(pVM)
3088	&& pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_TPR_SHADOW)
3089	{
3090	fVal \|= VMX_PROC_CTLS_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
3091	/* CR8 writes cause a VM-exit based on TPR threshold. */
3092	Assert(!(fVal & VMX_PROC_CTLS_CR8_STORE_EXIT));
3093	Assert(!(fVal & VMX_PROC_CTLS_CR8_LOAD_EXIT));
3094	int rc = hmR0VmxSetupVmcsVirtApicAddr(pVCpu, pVmcsInfo);
3095	AssertRCReturn(rc, rc);
3096	}
3097	else
3098	{
3099	/* Some 32-bit CPUs do not support CR8 load/store exiting as MOV CR8 is
3100	invalid on 32-bit Intel CPUs. Set this control only for 64-bit guests. */
3101	if (pVM->hm.s.fAllow64BitGuests)
3102	{
3103	fVal \|= VMX_PROC_CTLS_CR8_STORE_EXIT /* CR8 reads cause a VM-exit. */
3104	\| VMX_PROC_CTLS_CR8_LOAD_EXIT; /* CR8 writes cause a VM-exit. */
3105	}
3106	}
3107
3108	/* Use MSR-bitmaps if supported by the CPU. */
3109	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
3110	{
3111	fVal \|= VMX_PROC_CTLS_USE_MSR_BITMAPS;
3112	int rc = hmR0VmxSetupVmcsMsrBitmapAddr(pVmcsInfo);
3113	AssertRCReturn(rc, rc);
3114	}
3115
3116	/* Use the secondary processor-based VM-execution controls if supported by the CPU. */
3117	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
3118	fVal \|= VMX_PROC_CTLS_USE_SECONDARY_CTLS;
3119
3120	if ((fVal & fZap) != fVal)
3121	{
3122	LogRelFunc(("Invalid processor-based VM-execution controls combo! cpu=%#RX32 fVal=%#RX32 fZap=%#RX32\n",
3123	pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed0, fVal, fZap));
3124	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC;
3125	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3126	}
3127
3128	/* Commit it to the VMCS and update our cache. */
3129	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, fVal);
3130	AssertRCReturn(rc, rc);
3131	pVmcsInfo->u32ProcCtls = fVal;
3132
3133	/* Set up MSR permissions that don't change through the lifetime of the VM. */
3134	hmR0VmxSetupVmcsMsrPermissions(pVCpu, pVmcsInfo, false /* fIsNstGstVmcs */);
3135
3136	/* Set up secondary processor-based VM-execution controls if the CPU supports it. */
3137	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
3138	return hmR0VmxSetupVmcsProcCtls2(pVCpu, pVmcsInfo);
3139
3140	/* Sanity check, should not really happen. */
3141	if (RT_LIKELY(!pVM->hm.s.vmx.fUnrestrictedGuest))
3142	{ /* likely */ }
3143	else
3144	{
3145	pVCpu->hm.s.u32HMError = VMX_UFC_INVALID_UX_COMBO;
3146	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3147	}
3148
3149	/* Old CPUs without secondary processor-based VM-execution controls would end up here. */
3150	return VINF_SUCCESS;
3151	}
3152
3153
3154	/**
3155	* Sets up miscellaneous (everything other than Pin, Processor and secondary
3156	* Processor-based VM-execution) control fields in the VMCS.
3157	*
3158	* @returns VBox status code.
3159	* @param pVCpu The cross context virtual CPU structure.
3160	* @param pVmcsInfo The VMCS info. object.
3161	*/
3162	static int hmR0VmxSetupVmcsMiscCtls(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
3163	{
3164	/* Set the auto-load/store MSR area addresses in the VMCS. */
3165	int rc = hmR0VmxSetupVmcsAutoLoadStoreMsrAddrs(pVmcsInfo);
3166	if (RT_SUCCESS(rc))
3167	{
3168	/* Set the VMCS link pointer in the VMCS. */
3169	rc = hmR0VmxSetupVmcsLinkPtr(pVmcsInfo);
3170	if (RT_SUCCESS(rc))
3171	{
3172	/* Set the CR0/CR4 guest/host mask. */
3173	uint64_t const u64Cr0Mask = hmR0VmxGetFixedCr0Mask(pVCpu);
3174	uint64_t const u64Cr4Mask = hmR0VmxGetFixedCr4Mask(pVCpu);
3175	rc = VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, u64Cr0Mask);
3176	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, u64Cr4Mask);
3177	if (RT_SUCCESS(rc))
3178	{
3179	pVmcsInfo->u64Cr0Mask = u64Cr0Mask;
3180	pVmcsInfo->u64Cr4Mask = u64Cr4Mask;
3181	return VINF_SUCCESS;
3182	}
3183	LogRelFunc(("Failed to initialize VMCS CR0/CR4 guest/host mask. rc=%Rrc\n", rc));
3184	}
3185	else
3186	LogRelFunc(("Failed to initialize VMCS link pointer. rc=%Rrc\n", rc));
3187	}
3188	else
3189	LogRelFunc(("Failed to initialize VMCS auto-load/store MSR addresses. rc=%Rrc\n", rc));
3190	return rc;
3191	}
3192
3193
3194	/**
3195	* Sets up the initial exception bitmap in the VMCS based on static conditions.
3196	*
3197	* We shall setup those exception intercepts that don't change during the
3198	* lifetime of the VM here. The rest are done dynamically while loading the
3199	* guest state.
3200	*
3201	* @returns VBox status code.
3202	* @param pVCpu The cross context virtual CPU structure.
3203	* @param pVmcsInfo The VMCS info. object.
3204	*/
3205	static int hmR0VmxSetupVmcsXcptBitmap(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
3206	{
3207	/*
3208	* The following exceptions are always intercepted:
3209	*
3210	* #AC - To prevent the guest from hanging the CPU.
3211	* #DB - To maintain the DR6 state even when intercepting DRx reads/writes and
3212	* recursive #DBs can cause a CPU hang.
3213	* #PF - To sync our shadow page tables when nested-paging is not used.
3214	*/
3215	bool const fNestedPaging = pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging;
3216	uint32_t const uXcptBitmap = RT_BIT(X86_XCPT_AC)
3217	\| RT_BIT(X86_XCPT_DB)
3218	\| (fNestedPaging ? 0 : RT_BIT(X86_XCPT_PF));
3219
3220	/* Commit it to the VMCS. */
3221	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
3222	AssertRCReturn(rc, rc);
3223
3224	/* Update our cache of the exception bitmap. */
3225	pVmcsInfo->u32XcptBitmap = uXcptBitmap;
3226	return VINF_SUCCESS;
3227	}
3228
3229
3230	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
3231	/**
3232	* Sets up the VMCS for executing a nested-guest using hardware-assisted VMX.
3233	*
3234	* @returns VBox status code.
3235	* @param pVCpu The cross context virtual CPU structure.
3236	* @param pVmcsInfo The VMCS info. object.
3237	*/
3238	static int hmR0VmxSetupVmcsCtlsNested(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
3239	{
3240	PVM pVM = pVCpu->CTX_SUFF(pVM);
3241	int rc = hmR0VmxSetupVmcsLinkPtr(pVmcsInfo);
3242	if (RT_SUCCESS(rc))
3243	{
3244	rc = hmR0VmxSetupVmcsAutoLoadStoreMsrAddrs(pVmcsInfo);
3245	if (RT_SUCCESS(rc))
3246	{
3247	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
3248	rc = hmR0VmxSetupVmcsMsrBitmapAddr(pVmcsInfo);
3249	if (RT_SUCCESS(rc))
3250	{
3251	if (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
3252	rc = hmR0VmxSetupVmcsApicAccessAddr(pVCpu);
3253	if (RT_SUCCESS(rc))
3254	return VINF_SUCCESS;
3255
3256	LogRelFunc(("Failed to set up the APIC-access address in the nested-guest VMCS. rc=%Rrc\n", rc));
3257	}
3258	else
3259	LogRelFunc(("Failed to set up the MSR-bitmap address in the nested-guest VMCS. rc=%Rrc\n", rc));
3260	}
3261	else
3262	LogRelFunc(("Failed to set up the VMCS link pointer in the nested-guest VMCS. rc=%Rrc\n", rc));
3263	}
3264	else
3265	LogRelFunc(("Failed to set up the auto-load/store MSR addresses in the nested-guest VMCS. rc=%Rrc\n", rc));
3266
3267	return rc;
3268	}
3269	#endif
3270
3271
3272	/**
3273	* Sets up the VMCS for executing a guest (or nested-guest) using hardware-assisted
3274	* VMX.
3275	*
3276	* @returns VBox status code.
3277	* @param pVCpu The cross context virtual CPU structure.
3278	* @param pVmcsInfo The VMCS info. object.
3279	* @param fIsNstGstVmcs Whether this is a nested-guest VMCS.
3280	*/
3281	static int hmR0VmxSetupVmcs(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, bool fIsNstGstVmcs)
3282	{
3283	Assert(pVmcsInfo);
3284	Assert(pVmcsInfo->pvVmcs);
3285	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
3286
3287	/* Set the CPU specified revision identifier at the beginning of the VMCS structure. */
3288	PVM pVM = pVCpu->CTX_SUFF(pVM);
3289	(uint32_t )pVmcsInfo->pvVmcs = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_ID);
3290	const char * const pszVmcs = fIsNstGstVmcs ? "nested-guest VMCS" : "guest VMCS";
3291
3292	LogFlowFunc(("\n"));
3293
3294	/*
3295	* Initialize the VMCS using VMCLEAR before loading the VMCS.
3296	* See Intel spec. 31.6 "Preparation And Launching A Virtual Machine".
3297	*/
3298	int rc = hmR0VmxClearVmcs(pVmcsInfo);
3299	if (RT_SUCCESS(rc))
3300	{
3301	rc = hmR0VmxLoadVmcs(pVmcsInfo);
3302	if (RT_SUCCESS(rc))
3303	{
3304	if (!fIsNstGstVmcs)
3305	{
3306	rc = hmR0VmxSetupVmcsPinCtls(pVCpu, pVmcsInfo);
3307	if (RT_SUCCESS(rc))
3308	{
3309	rc = hmR0VmxSetupVmcsProcCtls(pVCpu, pVmcsInfo);
3310	if (RT_SUCCESS(rc))
3311	{
3312	rc = hmR0VmxSetupVmcsMiscCtls(pVCpu, pVmcsInfo);
3313	if (RT_SUCCESS(rc))
3314	{
3315	rc = hmR0VmxSetupVmcsXcptBitmap(pVCpu, pVmcsInfo);
3316	if (RT_SUCCESS(rc))
3317	{ /* likely */ }
3318	else
3319	LogRelFunc(("Failed to initialize exception bitmap. rc=%Rrc\n", rc));
3320	}
3321	else
3322	LogRelFunc(("Failed to setup miscellaneous controls. rc=%Rrc\n", rc));
3323	}
3324	else
3325	LogRelFunc(("Failed to setup processor-based VM-execution controls. rc=%Rrc\n", rc));
3326	}
3327	else
3328	LogRelFunc(("Failed to setup pin-based controls. rc=%Rrc\n", rc));
3329	}
3330	else
3331	{
3332	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
3333	rc = hmR0VmxSetupVmcsCtlsNested(pVCpu, pVmcsInfo);
3334	if (RT_SUCCESS(rc))
3335	{ /* likely */ }
3336	else
3337	LogRelFunc(("Failed to initialize nested-guest VMCS. rc=%Rrc\n", rc));
3338	#else
3339	AssertFailed();
3340	#endif
3341	}
3342	}
3343	else
3344	LogRelFunc(("Failed to load the %s. rc=%Rrc\n", rc, pszVmcs));
3345	}
3346	else
3347	LogRelFunc(("Failed to clear the %s. rc=%Rrc\n", rc, pszVmcs));
3348
3349	/* Sync any CPU internal VMCS data back into our VMCS in memory. */
3350	if (RT_SUCCESS(rc))
3351	{
3352	rc = hmR0VmxClearVmcs(pVmcsInfo);
3353	if (RT_SUCCESS(rc))
3354	{ /* likely */ }
3355	else
3356	LogRelFunc(("Failed to clear the %s post setup. rc=%Rrc\n", rc, pszVmcs));
3357	}
3358
3359	/*
3360	* Update the last-error record both for failures and success, so we
3361	* can propagate the status code back to ring-3 for diagnostics.
3362	*/
3363	hmR0VmxUpdateErrorRecord(pVCpu, rc);
3364	NOREF(pszVmcs);
3365	return rc;
3366	}
3367
3368
3369	/**
3370	* Does global VT-x initialization (called during module initialization).
3371	*
3372	* @returns VBox status code.
3373	*/
3374	VMMR0DECL(int) VMXR0GlobalInit(void)
3375	{
3376	#ifdef HMVMX_USE_FUNCTION_TABLE
3377	AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
3378	# ifdef VBOX_STRICT
3379	for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
3380	Assert(g_apfnVMExitHandlers[i]);
3381	# endif
3382	#endif
3383	return VINF_SUCCESS;
3384	}
3385
3386
3387	/**
3388	* Does global VT-x termination (called during module termination).
3389	*/
3390	VMMR0DECL(void) VMXR0GlobalTerm()
3391	{
3392	/* Nothing to do currently. */
3393	}
3394
3395
3396	/**
3397	* Sets up and activates VT-x on the current CPU.
3398	*
3399	* @returns VBox status code.
3400	* @param pHostCpu The HM physical-CPU structure.
3401	* @param pVM The cross context VM structure. Can be
3402	* NULL after a host resume operation.
3403	* @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
3404	* fEnabledByHost is @c true).
3405	* @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
3406	* @a fEnabledByHost is @c true).
3407	* @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
3408	* enable VT-x on the host.
3409	* @param pHwvirtMsrs Pointer to the hardware-virtualization MSRs.
3410	*/
3411	VMMR0DECL(int) VMXR0EnableCpu(PHMPHYSCPU pHostCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost,
3412	PCSUPHWVIRTMSRS pHwvirtMsrs)
3413	{
3414	Assert(pHostCpu);
3415	Assert(pHwvirtMsrs);
3416	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
3417
3418	/* Enable VT-x if it's not already enabled by the host. */
3419	if (!fEnabledByHost)
3420	{
3421	int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
3422	if (RT_FAILURE(rc))
3423	return rc;
3424	}
3425
3426	/*
3427	* Flush all EPT tagged-TLB entries (in case VirtualBox or any other hypervisor have been
3428	* using EPTPs) so we don't retain any stale guest-physical mappings which won't get
3429	* invalidated when flushing by VPID.
3430	*/
3431	if (pHwvirtMsrs->u.vmx.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
3432	{
3433	hmR0VmxFlushEpt(NULL /* pVCpu /, NULL / pVmcsInfo */, VMXTLBFLUSHEPT_ALL_CONTEXTS);
3434	pHostCpu->fFlushAsidBeforeUse = false;
3435	}
3436	else
3437	pHostCpu->fFlushAsidBeforeUse = true;
3438
3439	/* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
3440	++pHostCpu->cTlbFlushes;
3441
3442	return VINF_SUCCESS;
3443	}
3444
3445
3446	/**
3447	* Deactivates VT-x on the current CPU.
3448	*
3449	* @returns VBox status code.
3450	* @param pvCpuPage Pointer to the VMXON region.
3451	* @param HCPhysCpuPage Physical address of the VMXON region.
3452	*
3453	* @remarks This function should never be called when SUPR0EnableVTx() or
3454	* similar was used to enable VT-x on the host.
3455	*/
3456	VMMR0DECL(int) VMXR0DisableCpu(void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
3457	{
3458	RT_NOREF2(pvCpuPage, HCPhysCpuPage);
3459
3460	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
3461	return hmR0VmxLeaveRootMode();
3462	}
3463
3464
3465	/**
3466	* Does per-VM VT-x initialization.
3467	*
3468	* @returns VBox status code.
3469	* @param pVM The cross context VM structure.
3470	*/
3471	VMMR0DECL(int) VMXR0InitVM(PVM pVM)
3472	{
3473	LogFlowFunc(("pVM=%p\n", pVM));
3474
3475	int rc = hmR0VmxStructsAlloc(pVM);
3476	if (RT_FAILURE(rc))
3477	{
3478	LogRelFunc(("Failed to allocated VMX structures. rc=%Rrc\n", rc));
3479	return rc;
3480	}
3481
3482	return VINF_SUCCESS;
3483	}
3484
3485
3486	/**
3487	* Does per-VM VT-x termination.
3488	*
3489	* @returns VBox status code.
3490	* @param pVM The cross context VM structure.
3491	*/
3492	VMMR0DECL(int) VMXR0TermVM(PVM pVM)
3493	{
3494	LogFlowFunc(("pVM=%p\n", pVM));
3495
3496	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
3497	if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
3498	{
3499	Assert(pVM->hm.s.vmx.pvScratch);
3500	ASMMemZero32(pVM->hm.s.vmx.pvScratch, X86_PAGE_4K_SIZE);
3501	}
3502	#endif
3503	hmR0VmxStructsFree(pVM);
3504	return VINF_SUCCESS;
3505	}
3506
3507
3508	/**
3509	* Sets up the VM for execution using hardware-assisted VMX.
3510	* This function is only called once per-VM during initialization.
3511	*
3512	* @returns VBox status code.
3513	* @param pVM The cross context VM structure.
3514	*/
3515	VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
3516	{
3517	AssertPtrReturn(pVM, VERR_INVALID_PARAMETER);
3518	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
3519
3520	LogFlowFunc(("pVM=%p\n", pVM));
3521
3522	/*
3523	* At least verify if VMX is enabled, since we can't check if we're in
3524	* VMX root mode or not without causing a #GP.
3525	*/
3526	RTCCUINTREG const uHostCR4 = ASMGetCR4();
3527	if (RT_LIKELY(uHostCR4 & X86_CR4_VMXE))
3528	{ /* likely */ }
3529	else
3530	return VERR_VMX_NOT_IN_VMX_ROOT_MODE;
3531
3532	/*
3533	* Without unrestricted guest execution, pRealModeTSS and pNonPagingModeEPTPageTable must
3534	* always be allocated. We no longer support the highly unlikely case of unrestricted guest
3535	* without pRealModeTSS, see hmR3InitFinalizeR0Intel().
3536	*/
3537	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
3538	&& ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
3539	\|\| !pVM->hm.s.vmx.pRealModeTSS))
3540	{
3541	LogRelFunc(("Invalid real-on-v86 state.\n"));
3542	return VERR_INTERNAL_ERROR;
3543	}
3544
3545	/* Initialize these always, see hmR3InitFinalizeR0().*/
3546	pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NONE;
3547	pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_NONE;
3548
3549	/* Setup the tagged-TLB flush handlers. */
3550	int rc = hmR0VmxSetupTaggedTlb(pVM);
3551	if (RT_FAILURE(rc))
3552	{
3553	LogRelFunc(("hmR0VmxSetupTaggedTlb failed! rc=%Rrc\n", rc));
3554	return rc;
3555	}
3556
3557	/* Check if we can use the VMCS controls for swapping the EFER MSR. */
3558	Assert(!pVM->hm.s.vmx.fSupportsVmcsEfer);
3559	#if HC_ARCH_BITS == 64
3560	if ( (pVM->hm.s.vmx.Msrs.EntryCtls.n.allowed1 & VMX_ENTRY_CTLS_LOAD_EFER_MSR)
3561	&& (pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_LOAD_EFER_MSR)
3562	&& (pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_SAVE_EFER_MSR))
3563	pVM->hm.s.vmx.fSupportsVmcsEfer = true;
3564	#endif
3565
3566	for (VMCPUID idCpu = 0; idCpu < pVM->cCpus; idCpu++)
3567	{
3568	PVMCPU pVCpu = &pVM->aCpus[idCpu];
3569	Log4Func(("pVCpu=%p idCpu=%RU32\n", pVCpu, pVCpu->idCpu));
3570
3571	rc = hmR0VmxSetupVmcs(pVCpu, &pVCpu->hm.s.vmx.VmcsInfo, false /* fIsNstGstVmcs */);
3572	if (RT_SUCCESS(rc))
3573	{
3574	#if HC_ARCH_BITS == 32
3575	hmR0VmxInitVmcsReadCache(pVCpu);
3576	#endif
3577	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
3578	if (pVM->cpum.ro.GuestFeatures.fVmx)
3579	{
3580	rc = hmR0VmxSetupVmcs(pVCpu, &pVCpu->hm.s.vmx.VmcsInfoNstGst, true /* fIsNstGstVmcs */);
3581	if (RT_SUCCESS(rc))
3582	{ /* likely */ }
3583	else
3584	{
3585	LogRelFunc(("Nested-guest VMCS setup failed. rc=%Rrc\n", rc));
3586	return rc;
3587	}
3588	}
3589	#endif
3590	}
3591	else
3592	{
3593	LogRelFunc(("VMCS setup failed. rc=%Rrc\n", rc));
3594	return rc;
3595	}
3596	}
3597
3598	return VINF_SUCCESS;
3599	}
3600
3601
3602	#if HC_ARCH_BITS == 32
3603	# ifdef VBOX_ENABLE_64_BITS_GUESTS
3604	/**
3605	* Check if guest state allows safe use of 32-bit switcher again.
3606	*
3607	* Segment bases and protected mode structures must be 32-bit addressable
3608	* because the 32-bit switcher will ignore high dword when writing these VMCS
3609	* fields. See @bugref{8432} for details.
3610	*
3611	* @returns true if safe, false if must continue to use the 64-bit switcher.
3612	* @param pCtx Pointer to the guest-CPU context.
3613	*
3614	* @remarks No-long-jump zone!!!
3615	*/
3616	static bool hmR0VmxIs32BitSwitcherSafe(PCCPUMCTX pCtx)
3617	{
3618	if (pCtx->gdtr.pGdt & UINT64_C(0xffffffff00000000)) return false;
3619	if (pCtx->idtr.pIdt & UINT64_C(0xffffffff00000000)) return false;
3620	if (pCtx->ldtr.u64Base & UINT64_C(0xffffffff00000000)) return false;
3621	if (pCtx->tr.u64Base & UINT64_C(0xffffffff00000000)) return false;
3622	if (pCtx->es.u64Base & UINT64_C(0xffffffff00000000)) return false;
3623	if (pCtx->cs.u64Base & UINT64_C(0xffffffff00000000)) return false;
3624	if (pCtx->ss.u64Base & UINT64_C(0xffffffff00000000)) return false;
3625	if (pCtx->ds.u64Base & UINT64_C(0xffffffff00000000)) return false;
3626	if (pCtx->fs.u64Base & UINT64_C(0xffffffff00000000)) return false;
3627	if (pCtx->gs.u64Base & UINT64_C(0xffffffff00000000)) return false;
3628
3629	/* All good, bases are 32-bit. */
3630	return true;
3631	}
3632	# endif /* VBOX_ENABLE_64_BITS_GUESTS */
3633
3634	# ifdef VBOX_STRICT
3635	static bool hmR0VmxIsValidWriteField(uint32_t idxField)
3636	{
3637	switch (idxField)
3638	{
3639	case VMX_VMCS_GUEST_RIP:
3640	case VMX_VMCS_GUEST_RSP:
3641	case VMX_VMCS_GUEST_SYSENTER_EIP:
3642	case VMX_VMCS_GUEST_SYSENTER_ESP:
3643	case VMX_VMCS_GUEST_GDTR_BASE:
3644	case VMX_VMCS_GUEST_IDTR_BASE:
3645	case VMX_VMCS_GUEST_CS_BASE:
3646	case VMX_VMCS_GUEST_DS_BASE:
3647	case VMX_VMCS_GUEST_ES_BASE:
3648	case VMX_VMCS_GUEST_FS_BASE:
3649	case VMX_VMCS_GUEST_GS_BASE:
3650	case VMX_VMCS_GUEST_SS_BASE:
3651	case VMX_VMCS_GUEST_LDTR_BASE:
3652	case VMX_VMCS_GUEST_TR_BASE:
3653	case VMX_VMCS_GUEST_CR3:
3654	return true;
3655	}
3656	return false;
3657	}
3658
3659	static bool hmR0VmxIsValidReadField(uint32_t idxField)
3660	{
3661	switch (idxField)
3662	{
3663	/* Read-only fields. */
3664	case VMX_VMCS_RO_EXIT_QUALIFICATION:
3665	return true;
3666	}
3667	/* Remaining readable fields should also be writable. */
3668	return hmR0VmxIsValidWriteField(idxField);
3669	}
3670	# endif /* VBOX_STRICT */
3671
3672
3673	/**
3674	* Executes the specified handler in 64-bit mode.
3675	*
3676	* @returns VBox status code (no informational status codes).
3677	* @param pVCpu The cross context virtual CPU structure.
3678	* @param enmOp The operation to perform.
3679	* @param cParams Number of parameters.
3680	* @param paParam Array of 32-bit parameters.
3681	*/
3682	VMMR0DECL(int) VMXR0Execute64BitsHandler(PVMCPU pVCpu, HM64ON32OP enmOp, uint32_t cParams, uint32_t *paParam)
3683	{
3684	PVM pVM = pVCpu->CTX_SUFF(pVM);
3685	AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
3686	Assert(enmOp > HM64ON32OP_INVALID && enmOp < HM64ON32OP_END);
3687	Assert(pVCpu->hm.s.vmx.VmcsCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VmcsCache.Write.aField));
3688	Assert(pVCpu->hm.s.vmx.VmcsCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VmcsCache.Read.aField));
3689
3690	#ifdef VBOX_STRICT
3691	for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VmcsCache.Write.cValidEntries; i++)
3692	Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VmcsCache.Write.aField[i]));
3693
3694	for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VmcsCache.Read.cValidEntries; i++)
3695	Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VmcsCache.Read.aField[i]));
3696	#endif
3697
3698	/* Disable interrupts. */
3699	RTCCUINTREG fOldEFlags = ASMIntDisableFlags();
3700
3701	#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
3702	RTCPUID idHostCpu = RTMpCpuId();
3703	CPUMR0SetLApic(pVCpu, idHostCpu);
3704	#endif
3705
3706	/** @todo replace with hmR0VmxEnterRootMode() and hmR0VmxLeaveRootMode(). */
3707
3708	PCHMPHYSCPU pHostCpu = hmR0GetCurrentCpu();
3709	RTHCPHYS const HCPhysCpuPage = pHostCpu->HCPhysMemObj;
3710
3711	/* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
3712	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
3713	hmR0VmxClearVmcs(pVmcsInfo);
3714
3715	/* Leave VMX root mode and disable VMX. */
3716	VMXDisable();
3717	SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
3718
3719	CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
3720	CPUMSetHyperEIP(pVCpu, enmOp);
3721	for (int i = (int)cParams - 1; i >= 0; i--)
3722	CPUMPushHyper(pVCpu, paParam[i]);
3723
3724	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
3725
3726	/* Call the switcher. */
3727	int rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_UOFFSETOF_DYN(VM, aCpus[pVCpu->idCpu].cpum) - RT_UOFFSETOF(VM, cpum));
3728	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
3729
3730	/* Re-enable VMX to make sure the VMX instructions don't cause #UD faults. */
3731	SUPR0ChangeCR4(X86_CR4_VMXE, RTCCUINTREG_MAX);
3732
3733	/* Re-enter VMX root mode. */
3734	int rc2 = VMXEnable(HCPhysCpuPage);
3735	if (RT_FAILURE(rc2))
3736	{
3737	SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
3738	ASMSetFlags(fOldEFlags);
3739	pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
3740	return rc2;
3741	}
3742
3743	/* Restore the VMCS as the current VMCS. */
3744	rc2 = hmR0VmxLoadVmcs(pVmcsInfo);
3745	AssertRC(rc2);
3746	Assert(!(ASMGetFlags() & X86_EFL_IF));
3747	ASMSetFlags(fOldEFlags);
3748	return rc;
3749	}
3750
3751
3752	/**
3753	* Prepares for and executes VMLAUNCH (64-bit guests) for 32-bit hosts
3754	* supporting 64-bit guests.
3755	*
3756	* @returns VBox status code.
3757	* @param fResume Whether to VMLAUNCH or VMRESUME.
3758	* @param pCtx Pointer to the guest-CPU context.
3759	* @param pCache Pointer to the VMCS batch cache.
3760	* @param pVM The cross context VM structure.
3761	* @param pVCpu The cross context virtual CPU structure.
3762	*/
3763	DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMXVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
3764	{
3765	NOREF(fResume);
3766
3767	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
3768	PCHMPHYSCPU pHostCpu = hmR0GetCurrentCpu();
3769	RTHCPHYS const HCPhysCpuPage = pHostCpu->HCPhysMemObj;
3770
3771	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
3772	pCache->uPos = 1;
3773	pCache->interPD = PGMGetInterPaeCR3(pVM);
3774	pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
3775	#endif
3776
3777	#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
3778	pCache->TestIn.HCPhysCpuPage = 0;
3779	pCache->TestIn.HCPhysVmcs = 0;
3780	pCache->TestIn.pCache = 0;
3781	pCache->TestOut.HCPhysVmcs = 0;
3782	pCache->TestOut.pCache = 0;
3783	pCache->TestOut.pCtx = 0;
3784	pCache->TestOut.eflags = 0;
3785	#else
3786	NOREF(pCache);
3787	#endif
3788
3789	uint32_t aParam[10];
3790	aParam[0] = RT_LO_U32(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
3791	aParam[1] = RT_HI_U32(HCPhysCpuPage); /* Param 1: VMXON physical address - Hi. */
3792	aParam[2] = RT_LO_U32(pVmcsInfo->HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
3793	aParam[3] = RT_HI_U32(pVmcsInfo->HCPhysVmcs); /* Param 2: VMCS physical address - Hi. */
3794	aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VmcsCache);
3795	aParam[5] = 0;
3796	aParam[6] = VM_RC_ADDR(pVM, pVM);
3797	aParam[7] = 0;
3798	aParam[8] = VM_RC_ADDR(pVM, pVCpu);
3799	aParam[9] = 0;
3800
3801	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
3802	pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
3803	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
3804	#endif
3805	int rc = VMXR0Execute64BitsHandler(pVCpu, HM64ON32OP_VMXRCStartVM64, RT_ELEMENTS(aParam), &aParam[0]);
3806
3807	#ifdef VBOX_WITH_CRASHDUMP_MAGIC
3808	Assert((uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
3809	Assert(pCtx->dr[4] == 10);
3810	(uint32_t )(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
3811	#endif
3812
3813	#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
3814	AssertMsg(pCache->TestIn.HCPhysCpuPage == HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
3815	AssertMsg(pCache->TestIn.HCPhysVmcs == pVmcsInfo->HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
3816	pVmcsInfo->HCPhysVmcs));
3817	AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
3818	pCache->TestOut.HCPhysVmcs));
3819	AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
3820	pCache->TestOut.pCache));
3821	AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VmcsCache),
3822	("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VmcsCache)));
3823	AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
3824	pCache->TestOut.pCtx));
3825	Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
3826	#endif
3827	NOREF(pCtx);
3828	return rc;
3829	}
3830	#endif
3831
3832
3833	/**
3834	* Saves the host control registers (CR0, CR3, CR4) into the host-state area in
3835	* the VMCS.
3836	*
3837	* @returns VBox status code.
3838	*/
3839	static int hmR0VmxExportHostControlRegs(void)
3840	{
3841	RTCCUINTREG uReg = ASMGetCR0();
3842	int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
3843	AssertRCReturn(rc, rc);
3844
3845	uReg = ASMGetCR3();
3846	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
3847	AssertRCReturn(rc, rc);
3848
3849	uReg = ASMGetCR4();
3850	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
3851	AssertRCReturn(rc, rc);
3852	return rc;
3853	}
3854
3855
3856	/**
3857	* Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
3858	* the host-state area in the VMCS.
3859	*
3860	* @returns VBox status code.
3861	* @param pVCpu The cross context virtual CPU structure.
3862	*/
3863	static int hmR0VmxExportHostSegmentRegs(PVMCPU pVCpu)
3864	{
3865	#if HC_ARCH_BITS == 64
3866	/**
3867	* Macro for adjusting host segment selectors to satisfy VT-x's VM-entry
3868	* requirements. See hmR0VmxExportHostSegmentRegs().
3869	*/
3870	# define VMXLOCAL_ADJUST_HOST_SEG(seg, selValue) \
3871	if ((selValue) & (X86_SEL_RPL \| X86_SEL_LDT)) \
3872	{ \
3873	bool fValidSelector = true; \
3874	if ((selValue) & X86_SEL_LDT) \
3875	{ \
3876	uint32_t uAttr = ASMGetSegAttr((selValue)); \
3877	fValidSelector = RT_BOOL(uAttr != UINT32_MAX && (uAttr & X86_DESC_P)); \
3878	} \
3879	if (fValidSelector) \
3880	{ \
3881	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_##seg; \
3882	pVCpu->hm.s.vmx.RestoreHost.uHostSel##seg = (selValue); \
3883	} \
3884	(selValue) = 0; \
3885	}
3886
3887	/*
3888	* If we've executed guest code using hardware-assisted VMX, the host-state bits
3889	* will be messed up. We should -not- save the messed up state without restoring
3890	* the original host-state, see @bugref{7240}.
3891	*
3892	* This apparently can happen (most likely the FPU changes), deal with it rather than
3893	* asserting. Was observed booting Solaris 10u10 32-bit guest.
3894	*/
3895	if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
3896	&& (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
3897	{
3898	Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags,
3899	pVCpu->idCpu));
3900	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
3901	}
3902	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
3903	#else
3904	RT_NOREF(pVCpu);
3905	#endif
3906
3907	/*
3908	* Host DS, ES, FS and GS segment registers.
3909	*/
3910	#if HC_ARCH_BITS == 64
3911	RTSEL uSelDS = ASMGetDS();
3912	RTSEL uSelES = ASMGetES();
3913	RTSEL uSelFS = ASMGetFS();
3914	RTSEL uSelGS = ASMGetGS();
3915	#else
3916	RTSEL uSelDS = 0;
3917	RTSEL uSelES = 0;
3918	RTSEL uSelFS = 0;
3919	RTSEL uSelGS = 0;
3920	#endif
3921
3922	/*
3923	* Host CS and SS segment registers.
3924	*/
3925	RTSEL uSelCS = ASMGetCS();
3926	RTSEL uSelSS = ASMGetSS();
3927
3928	/*
3929	* Host TR segment register.
3930	*/
3931	RTSEL uSelTR = ASMGetTR();
3932
3933	#if HC_ARCH_BITS == 64
3934	/*
3935	* Determine if the host segment registers are suitable for VT-x. Otherwise use zero to
3936	* gain VM-entry and restore them before we get preempted.
3937	*
3938	* See Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers".
3939	*/
3940	VMXLOCAL_ADJUST_HOST_SEG(DS, uSelDS);
3941	VMXLOCAL_ADJUST_HOST_SEG(ES, uSelES);
3942	VMXLOCAL_ADJUST_HOST_SEG(FS, uSelFS);
3943	VMXLOCAL_ADJUST_HOST_SEG(GS, uSelGS);
3944	# undef VMXLOCAL_ADJUST_HOST_SEG
3945	#endif
3946
3947	/* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
3948	Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
3949	Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
3950	Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
3951	Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
3952	Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
3953	Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
3954	Assert(!(uSelTR & X86_SEL_RPL)); Assert(!(uSelTR & X86_SEL_LDT));
3955	Assert(uSelCS);
3956	Assert(uSelTR);
3957
3958	/* Write these host selector fields into the host-state area in the VMCS. */
3959	int rc = VMXWriteVmcs32(VMX_VMCS16_HOST_CS_SEL, uSelCS);
3960	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_SS_SEL, uSelSS);
3961	#if HC_ARCH_BITS == 64
3962	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_DS_SEL, uSelDS);
3963	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_ES_SEL, uSelES);
3964	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_FS_SEL, uSelFS);
3965	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_GS_SEL, uSelGS);
3966	#else
3967	NOREF(uSelDS);
3968	NOREF(uSelES);
3969	NOREF(uSelFS);
3970	NOREF(uSelGS);
3971	#endif
3972	rc \|= VMXWriteVmcs32(VMX_VMCS16_HOST_TR_SEL, uSelTR);
3973	AssertRCReturn(rc, rc);
3974
3975	/*
3976	* Host GDTR and IDTR.
3977	*/
3978	RTGDTR Gdtr;
3979	RTIDTR Idtr;
3980	RT_ZERO(Gdtr);
3981	RT_ZERO(Idtr);
3982	ASMGetGDTR(&Gdtr);
3983	ASMGetIDTR(&Idtr);
3984	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt);
3985	rc \|= VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt);
3986	AssertRCReturn(rc, rc);
3987
3988	#if HC_ARCH_BITS == 64
3989	/*
3990	* Determine if we need to manually need to restore the GDTR and IDTR limits as VT-x zaps
3991	* them to the maximum limit (0xffff) on every VM-exit.
3992	*/
3993	if (Gdtr.cbGdt != 0xffff)
3994	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_GDTR;
3995
3996	/*
3997	* IDT limit is effectively capped at 0xfff. (See Intel spec. 6.14.1 "64-Bit Mode IDT" and
3998	* Intel spec. 6.2 "Exception and Interrupt Vectors".) Therefore if the host has the limit
3999	* as 0xfff, VT-x bloating the limit to 0xffff shouldn't cause any different CPU behavior.
4000	* However, several hosts either insists on 0xfff being the limit (Windows Patch Guard) or
4001	* uses the limit for other purposes (darwin puts the CPU ID in there but botches sidt
4002	* alignment in at least one consumer). So, we're only allowing the IDTR.LIMIT to be left
4003	* at 0xffff on hosts where we are sure it won't cause trouble.
4004	*/
4005	# if defined(RT_OS_LINUX) \|\| defined(RT_OS_SOLARIS)
4006	if (Idtr.cbIdt < 0x0fff)
4007	# else
4008	if (Idtr.cbIdt != 0xffff)
4009	# endif
4010	{
4011	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_IDTR;
4012	AssertCompile(sizeof(Idtr) == sizeof(X86XDTR64));
4013	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostIdtr, &Idtr, sizeof(X86XDTR64));
4014	}
4015	#endif
4016
4017	/*
4018	* Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI
4019	* and RPL bits is effectively what the CPU does for "scaling by 8". TI is always 0 and
4020	* RPL should be too in most cases.
4021	*/
4022	AssertMsgReturn((uSelTR \| X86_SEL_RPL_LDT) <= Gdtr.cbGdt,
4023	("TR selector exceeds limit. TR=%RTsel cbGdt=%#x\n", uSelTR, Gdtr.cbGdt), VERR_VMX_INVALID_HOST_STATE);
4024
4025	PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
4026	#if HC_ARCH_BITS == 64
4027	uintptr_t const uTRBase = X86DESC64_BASE(pDesc);
4028
4029	/*
4030	* VT-x unconditionally restores the TR limit to 0x67 and type to 11 (32-bit busy TSS) on
4031	* all VM-exits. The type is the same for 64-bit busy TSS[1]. The limit needs manual
4032	* restoration if the host has something else. Task switching is not supported in 64-bit
4033	* mode[2], but the limit still matters as IOPM is supported in 64-bit mode. Restoring the
4034	* limit lazily while returning to ring-3 is safe because IOPM is not applicable in ring-0.
4035	*
4036	* [1] See Intel spec. 3.5 "System Descriptor Types".
4037	* [2] See Intel spec. 7.2.3 "TSS Descriptor in 64-bit mode".
4038	*/
4039	PVM pVM = pVCpu->CTX_SUFF(pVM);
4040	Assert(pDesc->System.u4Type == 11);
4041	if ( pDesc->System.u16LimitLow != 0x67
4042	\|\| pDesc->System.u4LimitHigh)
4043	{
4044	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_SEL_TR;
4045	/* If the host has made GDT read-only, we would need to temporarily toggle CR0.WP before writing the GDT. */
4046	if (pVM->hm.s.fHostKernelFeatures & SUPKERNELFEATURES_GDT_READ_ONLY)
4047	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_GDT_READ_ONLY;
4048	pVCpu->hm.s.vmx.RestoreHost.uHostSelTR = uSelTR;
4049	}
4050
4051	/*
4052	* Store the GDTR as we need it when restoring the GDT and while restoring the TR.
4053	*/
4054	if (pVCpu->hm.s.vmx.fRestoreHostFlags & (VMX_RESTORE_HOST_GDTR \| VMX_RESTORE_HOST_SEL_TR))
4055	{
4056	AssertCompile(sizeof(Gdtr) == sizeof(X86XDTR64));
4057	memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
4058	if (pVM->hm.s.fHostKernelFeatures & SUPKERNELFEATURES_GDT_NEED_WRITABLE)
4059	{
4060	/* The GDT is read-only but the writable GDT is available. */
4061	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_GDT_NEED_WRITABLE;
4062	pVCpu->hm.s.vmx.RestoreHost.HostGdtrRw.cb = Gdtr.cbGdt;
4063	rc = SUPR0GetCurrentGdtRw(&pVCpu->hm.s.vmx.RestoreHost.HostGdtrRw.uAddr);
4064	AssertRCReturn(rc, rc);
4065	}
4066	}
4067	#else
4068	uintptr_t const uTRBase = X86DESC_BASE(pDesc);
4069	#endif
4070	rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
4071	AssertRCReturn(rc, rc);
4072
4073	/*
4074	* Host FS base and GS base.
4075	*/
4076	#if HC_ARCH_BITS == 64
4077	uint64_t const u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
4078	uint64_t const u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
4079	rc = VMXWriteVmcs64(VMX_VMCS_HOST_FS_BASE, u64FSBase);
4080	rc \|= VMXWriteVmcs64(VMX_VMCS_HOST_GS_BASE, u64GSBase);
4081	AssertRCReturn(rc, rc);
4082
4083	/* Store the base if we have to restore FS or GS manually as we need to restore the base as well. */
4084	if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_FS)
4085	pVCpu->hm.s.vmx.RestoreHost.uHostFSBase = u64FSBase;
4086	if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_GS)
4087	pVCpu->hm.s.vmx.RestoreHost.uHostGSBase = u64GSBase;
4088	#endif
4089	return VINF_SUCCESS;
4090	}
4091
4092
4093	/**
4094	* Exports certain host MSRs in the VM-exit MSR-load area and some in the
4095	* host-state area of the VMCS.
4096	*
4097	* These MSRs will be automatically restored on the host after every successful
4098	* VM-exit.
4099	*
4100	* @returns VBox status code.
4101	* @param pVCpu The cross context virtual CPU structure.
4102	*
4103	* @remarks No-long-jump zone!!!
4104	*/
4105	static int hmR0VmxExportHostMsrs(PVMCPU pVCpu)
4106	{
4107	AssertPtr(pVCpu);
4108
4109	/*
4110	* Save MSRs that we restore lazily (due to preemption or transition to ring-3)
4111	* rather than swapping them on every VM-entry.
4112	*/
4113	hmR0VmxLazySaveHostMsrs(pVCpu);
4114
4115	/*
4116	* Host Sysenter MSRs.
4117	*/
4118	int rc = VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
4119	#if HC_ARCH_BITS == 32
4120	rc \|= VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
4121	rc \|= VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
4122	#else
4123	rc \|= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
4124	rc \|= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
4125	#endif
4126	AssertRCReturn(rc, rc);
4127
4128	/*
4129	* Host EFER MSR.
4130	*
4131	* If the CPU supports the newer VMCS controls for managing EFER, use it. Otherwise it's
4132	* done as part of auto-load/store MSR area in the VMCS, see hmR0VmxExportGuestMsrs().
4133	*/
4134	PVM pVM = pVCpu->CTX_SUFF(pVM);
4135	if (pVM->hm.s.vmx.fSupportsVmcsEfer)
4136	{
4137	rc = VMXWriteVmcs64(VMX_VMCS64_HOST_EFER_FULL, pVM->hm.s.vmx.u64HostMsrEfer);
4138	AssertRCReturn(rc, rc);
4139	}
4140
4141	/** @todo IA32_PERF_GLOBALCTRL, IA32_PAT also see
4142	* hmR0VmxExportGuestEntryExitCtls(). */
4143
4144	return VINF_SUCCESS;
4145	}
4146
4147
4148	/**
4149	* Figures out if we need to swap the EFER MSR which is particularly expensive.
4150	*
4151	* We check all relevant bits. For now, that's everything besides LMA/LME, as
4152	* these two bits are handled by VM-entry, see hmR0VMxExportGuestEntryExitCtls().
4153	*
4154	* @returns true if we need to load guest EFER, false otherwise.
4155	* @param pVCpu The cross context virtual CPU structure.
4156	*
4157	* @remarks Requires EFER, CR4.
4158	* @remarks No-long-jump zone!!!
4159	*/
4160	static bool hmR0VmxShouldSwapEferMsr(PVMCPU pVCpu)
4161	{
4162	#ifdef HMVMX_ALWAYS_SWAP_EFER
4163	RT_NOREF(pVCpu);
4164	return true;
4165	#else
4166	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4167	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
4168	/* For 32-bit hosts running 64-bit guests, we always swap EFER MSR in the world-switcher. Nothing to do here. */
4169	if (CPUMIsGuestInLongModeEx(pCtx))
4170	return false;
4171	#endif
4172
4173	PVM pVM = pVCpu->CTX_SUFF(pVM);
4174	uint64_t const u64HostEfer = pVM->hm.s.vmx.u64HostMsrEfer;
4175	uint64_t const u64GuestEfer = pCtx->msrEFER;
4176
4177	/*
4178	* For 64-bit guests, if EFER.SCE bit differs, we need to swap the EFER MSR
4179	* to ensure that the guest's SYSCALL behaviour isn't broken, see @bugref{7386}.
4180	*/
4181	if ( CPUMIsGuestInLongModeEx(pCtx)
4182	&& (u64GuestEfer & MSR_K6_EFER_SCE) != (u64HostEfer & MSR_K6_EFER_SCE))
4183	return true;
4184
4185	/*
4186	* If the guest uses PAE and EFER.NXE bit differs, we need to swap the EFER MSR
4187	* as it affects guest paging. 64-bit paging implies CR4.PAE as well.
4188	*
4189	* See Intel spec. 4.5 "IA-32e Paging".
4190	* See Intel spec. 4.1.1 "Three Paging Modes".
4191	*
4192	* Verify that we always intercept CR4.PAE and CR0.PG bits, so we don't need to
4193	* import CR4 and CR0 from the VMCS here as those bits are always up to date.
4194	*/
4195	Assert(hmR0VmxGetFixedCr4Mask(pVCpu) & X86_CR4_PAE);
4196	Assert(hmR0VmxGetFixedCr0Mask(pVCpu) & X86_CR0_PG);
4197	if ( (pCtx->cr4 & X86_CR4_PAE)
4198	&& (pCtx->cr0 & X86_CR0_PG)
4199	&& (u64GuestEfer & MSR_K6_EFER_NXE) != (u64HostEfer & MSR_K6_EFER_NXE))
4200	{
4201	/* Assert that host is NX capable. */
4202	Assert(pVCpu->CTX_SUFF(pVM)->cpum.ro.HostFeatures.fNoExecute);
4203	return true;
4204	}
4205
4206	return false;
4207	#endif
4208	}
4209
4210	/**
4211	* Exports the guest state with appropriate VM-entry and VM-exit controls in the
4212	* VMCS.
4213	*
4214	* This is typically required when the guest changes paging mode.
4215	*
4216	* @returns VBox status code.
4217	* @param pVCpu The cross context virtual CPU structure.
4218	* @param pVmxTransient The VMX-transient structure.
4219	*
4220	* @remarks Requires EFER.
4221	* @remarks No-long-jump zone!!!
4222	*/
4223	static int hmR0VmxExportGuestEntryExitCtls(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
4224	{
4225	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_ENTRY_EXIT_CTLS)
4226	{
4227	PVM pVM = pVCpu->CTX_SUFF(pVM);
4228	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
4229
4230	/*
4231	* VM-entry controls.
4232	*/
4233	{
4234	uint32_t fVal = pVM->hm.s.vmx.Msrs.EntryCtls.n.allowed0; /* Bits set here must be set in the VMCS. */
4235	uint32_t const fZap = pVM->hm.s.vmx.Msrs.EntryCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
4236
4237	/*
4238	* Load the guest debug controls (DR7 and IA32_DEBUGCTL MSR) on VM-entry.
4239	* The first VT-x capable CPUs only supported the 1-setting of this bit.
4240	*
4241	* For nested-guests, this is a mandatory VM-entry control. It's also
4242	* required because we do not want to leak host bits to the nested-guest.
4243	*/
4244	fVal \|= VMX_ENTRY_CTLS_LOAD_DEBUG;
4245
4246	/*
4247	* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry.
4248	*
4249	* For nested-guests, the "IA-32e mode guest" control we initialize with what is
4250	* required to get the nested-guest working with hardware-assisted VMX execution.
4251	* It depends on the nested-guest's IA32_EFER.LMA bit. Remember, a nested-hypervisor
4252	* can skip intercepting changes to the EFER MSR. This is why it it needs to be done
4253	* here rather than while merging the guest VMCS controls.
4254	*/
4255	if (CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx))
4256	fVal \|= VMX_ENTRY_CTLS_IA32E_MODE_GUEST;
4257	else
4258	Assert(!(fVal & VMX_ENTRY_CTLS_IA32E_MODE_GUEST));
4259
4260	/*
4261	* If the CPU supports the newer VMCS controls for managing guest/host EFER, use it.
4262	*
4263	* For nested-guests, we use the "load IA32_EFER" if the hardware supports it,
4264	* regardless of whether the nested-guest VMCS specifies it because we are free to
4265	* load whatever MSRs we require and we do not need to modify the guest visible copy
4266	* of the VM-entry MSR load area.
4267	*/
4268	if ( pVM->hm.s.vmx.fSupportsVmcsEfer
4269	&& hmR0VmxShouldSwapEferMsr(pVCpu))
4270	fVal \|= VMX_ENTRY_CTLS_LOAD_EFER_MSR;
4271	else
4272	Assert(!(fVal & VMX_ENTRY_CTLS_LOAD_EFER_MSR));
4273
4274	/*
4275	* The following should -not- be set (since we're not in SMM mode):
4276	* - VMX_ENTRY_CTLS_ENTRY_TO_SMM
4277	* - VMX_ENTRY_CTLS_DEACTIVATE_DUAL_MON
4278	*/
4279
4280	/** @todo VMX_ENTRY_CTLS_LOAD_PERF_MSR,
4281	* VMX_ENTRY_CTLS_LOAD_PAT_MSR. */
4282
4283	if ((fVal & fZap) == fVal)
4284	{ /* likely */ }
4285	else
4286	{
4287	Log4Func(("Invalid VM-entry controls combo! Cpu=%#RX32 fVal=%#RX32 fZap=%#RX32\n",
4288	pVM->hm.s.vmx.Msrs.EntryCtls.n.allowed0, fVal, fZap));
4289	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_ENTRY;
4290	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
4291	}
4292
4293	/* Commit it to the VMCS. */
4294	if (pVmcsInfo->u32EntryCtls != fVal)
4295	{
4296	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY, fVal);
4297	AssertRCReturn(rc, rc);
4298	pVmcsInfo->u32EntryCtls = fVal;
4299	}
4300	}
4301
4302	/*
4303	* VM-exit controls.
4304	*/
4305	{
4306	uint32_t fVal = pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed0; /* Bits set here must be set in the VMCS. */
4307	uint32_t const fZap = pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
4308
4309	/*
4310	* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only
4311	* supported the 1-setting of this bit.
4312	*
4313	* For nested-guests, we set the "save debug controls" as the converse
4314	* "load debug controls" is mandatory for nested-guests anyway.
4315	*/
4316	fVal \|= VMX_EXIT_CTLS_SAVE_DEBUG;
4317
4318	/*
4319	* Set the host long mode active (EFER.LMA) bit (which Intel calls
4320	* "Host address-space size") if necessary. On VM-exit, VT-x sets both the
4321	* host EFER.LMA and EFER.LME bit to this value. See assertion in
4322	* hmR0VmxExportHostMsrs().
4323	*
4324	* For nested-guests, we always set this bit as we do not support 32-bit
4325	* hosts.
4326	*/
4327	#if HC_ARCH_BITS == 64
4328	fVal \|= VMX_EXIT_CTLS_HOST_ADDR_SPACE_SIZE;
4329	#else
4330	Assert(!pVmxTransient->fIsNestedGuest);
4331	Assert( pVmcsInfo->pfnStartVM == VMXR0SwitcherStartVM64
4332	\|\| pVmcsInfo->pfnStartVM == VMXR0StartVM32);
4333	/* Set the host address-space size based on the switcher, not guest state. See @bugref{8432}. */
4334	if (pVmcsInfo->pfnStartVM == VMXR0SwitcherStartVM64)
4335	{
4336	/* The switcher returns to long mode, the EFER MSR is managed by the switcher. */
4337	fVal \|= VMX_EXIT_CTLS_HOST_ADDR_SPACE_SIZE;
4338	}
4339	else
4340	Assert(!(fVal & VMX_EXIT_CTLS_HOST_ADDR_SPACE_SIZE));
4341	#endif
4342
4343	/*
4344	* If the VMCS EFER MSR fields are supported by the hardware, we use it.
4345	*
4346	* For nested-guests, we should use the "save IA32_EFER" control if we also
4347	* used the "load IA32_EFER" control while exporting VM-entry controls.
4348	*/
4349	if ( pVM->hm.s.vmx.fSupportsVmcsEfer
4350	&& hmR0VmxShouldSwapEferMsr(pVCpu))
4351	{
4352	fVal \|= VMX_EXIT_CTLS_SAVE_EFER_MSR
4353	\| VMX_EXIT_CTLS_LOAD_EFER_MSR;
4354	}
4355
4356	/*
4357	* Enable saving of the VMX-preemption timer value on VM-exit.
4358	* For nested-guests, currently not exposed/used.
4359	*/
4360	if ( pVM->hm.s.vmx.fUsePreemptTimer
4361	&& (pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_SAVE_PREEMPT_TIMER))
4362	fVal \|= VMX_EXIT_CTLS_SAVE_PREEMPT_TIMER;
4363
4364	/* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
4365	Assert(!(fVal & VMX_EXIT_CTLS_ACK_EXT_INT));
4366
4367	/** @todo VMX_EXIT_CTLS_LOAD_PERF_MSR,
4368	* VMX_EXIT_CTLS_SAVE_PAT_MSR,
4369	* VMX_EXIT_CTLS_LOAD_PAT_MSR. */
4370
4371	if ((fVal & fZap) == fVal)
4372	{ /* likely */ }
4373	else
4374	{
4375	Log4Func(("Invalid VM-exit controls combo! cpu=%#RX32 fVal=%#RX32 fZap=%R#X32\n",
4376	pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed0, fVal, fZap));
4377	pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_EXIT;
4378	return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
4379	}
4380
4381	/* Commit it to the VMCS. */
4382	if (pVmcsInfo->u32ExitCtls != fVal)
4383	{
4384	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT, fVal);
4385	AssertRCReturn(rc, rc);
4386	pVmcsInfo->u32ExitCtls = fVal;
4387	}
4388	}
4389
4390	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_ENTRY_EXIT_CTLS);
4391	}
4392	return VINF_SUCCESS;
4393	}
4394
4395
4396	/**
4397	* Sets the TPR threshold in the VMCS.
4398	*
4399	* @returns VBox status code.
4400	* @param pVCpu The cross context virtual CPU structure.
4401	* @param pVmcsInfo The VMCS info. object.
4402	* @param u32TprThreshold The TPR threshold (task-priority class only).
4403	*/
4404	DECLINLINE(int) hmR0VmxApicSetTprThreshold(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo, uint32_t u32TprThreshold)
4405	{
4406	Assert(!(u32TprThreshold & ~VMX_TPR_THRESHOLD_MASK)); /* Bits 31:4 MBZ. */
4407	Assert(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW);
4408	RT_NOREF2(pVCpu, pVmcsInfo);
4409	return VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
4410	}
4411
4412
4413	/**
4414	* Exports the guest APIC TPR state into the VMCS.
4415	*
4416	* @returns VBox status code.
4417	* @param pVCpu The cross context virtual CPU structure.
4418	* @param pVmxTransient The VMX-transient structure.
4419	*
4420	* @remarks No-long-jump zone!!!
4421	*/
4422	static int hmR0VmxExportGuestApicTpr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
4423	{
4424	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_APIC_TPR)
4425	{
4426	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_APIC_TPR);
4427
4428	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
4429	if (!pVmxTransient->fIsNestedGuest)
4430	{
4431	if ( PDMHasApic(pVCpu->CTX_SUFF(pVM))
4432	&& APICIsEnabled(pVCpu))
4433	{
4434	/*
4435	* Setup TPR shadowing.
4436	*/
4437	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
4438	{
4439	bool fPendingIntr = false;
4440	uint8_t u8Tpr = 0;
4441	uint8_t u8PendingIntr = 0;
4442	int rc = APICGetTpr(pVCpu, &u8Tpr, &fPendingIntr, &u8PendingIntr);
4443	AssertRCReturn(rc, rc);
4444
4445	/*
4446	* If there are interrupts pending but masked by the TPR, instruct VT-x to
4447	* cause a TPR-below-threshold VM-exit when the guest lowers its TPR below the
4448	* priority of the pending interrupt so we can deliver the interrupt. If there
4449	* are no interrupts pending, set threshold to 0 to not cause any
4450	* TPR-below-threshold VM-exits.
4451	*/
4452	Assert(pVmcsInfo->pbVirtApic);
4453	pVmcsInfo->pbVirtApic[XAPIC_OFF_TPR] = u8Tpr;
4454	uint32_t u32TprThreshold = 0;
4455	if (fPendingIntr)
4456	{
4457	/* Bits 3:0 of the TPR threshold field correspond to bits 7:4 of the TPR
4458	(which is the Task-Priority Class). */
4459	const uint8_t u8PendingPriority = u8PendingIntr >> 4;
4460	const uint8_t u8TprPriority = u8Tpr >> 4;
4461	if (u8PendingPriority <= u8TprPriority)
4462	u32TprThreshold = u8PendingPriority;
4463	}
4464
4465	rc = hmR0VmxApicSetTprThreshold(pVCpu, pVmcsInfo, u32TprThreshold);
4466	AssertRCReturn(rc, rc);
4467	}
4468	}
4469	}
4470	/* else: the TPR threshold has already been updated while merging the nested-guest VMCS. */
4471	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_APIC_TPR);
4472	}
4473	return VINF_SUCCESS;
4474	}
4475
4476
4477	/**
4478	* Gets the guest interruptibility-state.
4479	*
4480	* @returns Guest's interruptibility-state.
4481	* @param pVCpu The cross context virtual CPU structure.
4482	* @param pVmcsInfo The VMCS info. object.
4483	*
4484	* @remarks No-long-jump zone!!!
4485	*/
4486	static uint32_t hmR0VmxGetGuestIntrState(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
4487	{
4488	/*
4489	* Check if we should inhibit interrupt delivery due to instructions like STI and MOV SS.
4490	*/
4491	uint32_t fIntrState = 0;
4492	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
4493	{
4494	/* If inhibition is active, RIP and RFLAGS should've been updated
4495	(i.e. read previously from the VMCS or from ring-3). */
4496	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4497	#ifdef VBOX_STRICT
4498	uint64_t const fExtrn = ASMAtomicUoReadU64(&pCtx->fExtrn);
4499	RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
4500	AssertMsg(!(fExtrn & (CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS)), ("%#x\n", fExtrn));
4501	#endif
4502	if (pCtx->rip == EMGetInhibitInterruptsPC(pVCpu))
4503	{
4504	if (pCtx->eflags.Bits.u1IF)
4505	fIntrState = VMX_VMCS_GUEST_INT_STATE_BLOCK_STI;
4506	else
4507	fIntrState = VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS;
4508	}
4509	else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
4510	{
4511	/*
4512	* We can clear the inhibit force flag as even if we go back to the recompiler
4513	* without executing guest code in VT-x, the flag's condition to be cleared is
4514	* met and thus the cleared state is correct.
4515	*/
4516	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
4517	}
4518	}
4519
4520	/*
4521	* NMIs to the guest are blocked after an NMI is injected until the guest executes an IRET. We only
4522	* bother with virtual-NMI blocking when we have support for virtual NMIs in the CPU, otherwise
4523	* setting this would block host-NMIs and IRET will not clear the blocking.
4524	*
4525	* We always set NMI-exiting so when the host receives an NMI we get a VM-exit.
4526	*
4527	* See Intel spec. 26.6.1 "Interruptibility state". See @bugref{7445}.
4528	*/
4529	if ( (pVmcsInfo->u32PinCtls & VMX_PIN_CTLS_VIRT_NMI)
4530	&& CPUMIsGuestNmiBlocking(pVCpu))
4531	fIntrState \|= VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI;
4532
4533	return fIntrState;
4534	}
4535
4536
4537	/**
4538	* Exports the exception intercepts required for guest execution in the VMCS.
4539	*
4540	* @returns VBox status code.
4541	* @param pVCpu The cross context virtual CPU structure.
4542	* @param pVmxTransient The VMX-transient structure.
4543	*
4544	* @remarks No-long-jump zone!!!
4545	*/
4546	static int hmR0VmxExportGuestXcptIntercepts(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
4547	{
4548	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_GUEST_XCPT_INTERCEPTS)
4549	{
4550	/* When executing a nested-guest, we do not need to trap GIM hypercalls by intercepting #UD. */
4551	if ( !pVmxTransient->fIsNestedGuest
4552	&& pVCpu->hm.s.fGIMTrapXcptUD)
4553	hmR0VmxAddXcptIntercept(pVmxTransient, X86_XCPT_UD);
4554	else
4555	hmR0VmxRemoveXcptIntercept(pVCpu, pVmxTransient, X86_XCPT_UD);
4556
4557	/* Other exception intercepts are handled elsewhere, e.g. while exporting guest CR0. */
4558	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_GUEST_XCPT_INTERCEPTS);
4559	}
4560	return VINF_SUCCESS;
4561	}
4562
4563
4564	/**
4565	* Exports the guest's RIP into the guest-state area in the VMCS.
4566	*
4567	* @returns VBox status code.
4568	* @param pVCpu The cross context virtual CPU structure.
4569	*
4570	* @remarks No-long-jump zone!!!
4571	*/
4572	static int hmR0VmxExportGuestRip(PVMCPU pVCpu)
4573	{
4574	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RIP)
4575	{
4576	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RIP);
4577
4578	int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pVCpu->cpum.GstCtx.rip);
4579	AssertRCReturn(rc, rc);
4580
4581	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RIP);
4582	Log4Func(("rip=%#RX64\n", pVCpu->cpum.GstCtx.rip));
4583	}
4584	return VINF_SUCCESS;
4585	}
4586
4587
4588	/**
4589	* Exports the guest's RSP into the guest-state area in the VMCS.
4590	*
4591	* @returns VBox status code.
4592	* @param pVCpu The cross context virtual CPU structure.
4593	*
4594	* @remarks No-long-jump zone!!!
4595	*/
4596	static int hmR0VmxExportGuestRsp(PVMCPU pVCpu)
4597	{
4598	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RSP)
4599	{
4600	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RSP);
4601
4602	int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pVCpu->cpum.GstCtx.rsp);
4603	AssertRCReturn(rc, rc);
4604
4605	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RSP);
4606	}
4607	return VINF_SUCCESS;
4608	}
4609
4610
4611	/**
4612	* Exports the guest's RFLAGS into the guest-state area in the VMCS.
4613	*
4614	* @returns VBox status code.
4615	* @param pVCpu The cross context virtual CPU structure.
4616	* @param pVmxTransient The VMX-transient structure.
4617	*
4618	* @remarks No-long-jump zone!!!
4619	*/
4620	static int hmR0VmxExportGuestRflags(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
4621	{
4622	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RFLAGS)
4623	{
4624	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RFLAGS);
4625
4626	/* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
4627	Let us assert it as such and use 32-bit VMWRITE. */
4628	Assert(!RT_HI_U32(pVCpu->cpum.GstCtx.rflags.u64));
4629	X86EFLAGS fEFlags = pVCpu->cpum.GstCtx.eflags;
4630	Assert(fEFlags.u32 & X86_EFL_RA1_MASK);
4631	Assert(!(fEFlags.u32 & ~(X86_EFL_1 \| X86_EFL_LIVE_MASK)));
4632
4633	/*
4634	* If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so
4635	* we can restore them on VM-exit. Modify the real-mode guest's eflags so that VT-x
4636	* can run the real-mode guest code under Virtual 8086 mode.
4637	*/
4638	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
4639	if (pVmcsInfo->RealMode.fRealOnV86Active)
4640	{
4641	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
4642	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
4643	Assert(!pVmxTransient->fIsNestedGuest);
4644	pVmcsInfo->RealMode.Eflags.u32 = fEFlags.u32; /* Save the original eflags of the real-mode guest. */
4645	fEFlags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
4646	fEFlags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
4647	}
4648
4649	int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_RFLAGS, fEFlags.u32);
4650	AssertRCReturn(rc, rc);
4651
4652	/*
4653	* Setup pending debug exceptions if the guest is single-stepping using EFLAGS.TF.
4654	*
4655	* We must avoid setting any automatic debug exceptions delivery when single-stepping
4656	* through the hypervisor debugger using EFLAGS.TF.
4657	*/
4658	if ( !pVmxTransient->fIsNestedGuest
4659	&& !pVCpu->hm.s.fSingleInstruction
4660	&& fEFlags.Bits.u1TF)
4661	{
4662	/** @todo r=ramshankar: Warning!! We ASSUME EFLAGS.TF will not cleared on
4663	* premature trips to ring-3 esp since IEM does not yet handle it. */
4664	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, VMX_VMCS_GUEST_PENDING_DEBUG_XCPT_BS);
4665	AssertRCReturn(rc, rc);
4666	}
4667	/** @todo NSTVMX: Handling copying of VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS from
4668	* nested-guest VMCS. */
4669
4670	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RFLAGS);
4671	Log4Func(("EFlags=%#RX32\n", fEFlags.u32));
4672	}
4673	return VINF_SUCCESS;
4674	}
4675
4676
4677	/**
4678	* Exports the guest CR0 control register into the guest-state area in the VMCS.
4679	*
4680	* The guest FPU state is always pre-loaded hence we don't need to bother about
4681	* sharing FPU related CR0 bits between the guest and host.
4682	*
4683	* @returns VBox status code.
4684	* @param pVCpu The cross context virtual CPU structure.
4685	* @param pVmxTransient The VMX-transient structure.
4686	*
4687	* @remarks No-long-jump zone!!!
4688	*/
4689	static int hmR0VmxExportGuestCR0(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
4690	{
4691	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR0)
4692	{
4693	PVM pVM = pVCpu->CTX_SUFF(pVM);
4694	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
4695
4696	/*
4697	* Figure out fixed CR0 bits in VMX operation.
4698	*/
4699	uint64_t fSetCr0 = pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1;
4700	uint64_t const fZapCr0 = pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr0Fixed1;
4701	if (pVM->hm.s.vmx.fUnrestrictedGuest)
4702	fSetCr0 &= ~(uint64_t)(X86_CR0_PE \| X86_CR0_PG);
4703	else
4704	Assert((fSetCr0 & (X86_CR0_PE \| X86_CR0_PG)) == (X86_CR0_PE \| X86_CR0_PG));
4705
4706	if (!pVmxTransient->fIsNestedGuest)
4707	{
4708	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0);
4709	uint64_t u64GuestCr0 = pVCpu->cpum.GstCtx.cr0;
4710	uint64_t const u64ShadowCr0 = u64GuestCr0;
4711	Assert(!RT_HI_U32(u64GuestCr0));
4712
4713	/*
4714	* Setup VT-x's view of the guest CR0.
4715	*/
4716	uint32_t uProcCtls = pVmcsInfo->u32ProcCtls;
4717	if (pVM->hm.s.fNestedPaging)
4718	{
4719	if (CPUMIsGuestPagingEnabled(pVCpu))
4720	{
4721	/* The guest has paging enabled, let it access CR3 without causing a VM-exit if supported. */
4722	uProcCtls &= ~( VMX_PROC_CTLS_CR3_LOAD_EXIT
4723	\| VMX_PROC_CTLS_CR3_STORE_EXIT);
4724	}
4725	else
4726	{
4727	/* The guest doesn't have paging enabled, make CR3 access cause a VM-exit to update our shadow. */
4728	uProcCtls \|= VMX_PROC_CTLS_CR3_LOAD_EXIT
4729	\| VMX_PROC_CTLS_CR3_STORE_EXIT;
4730	}
4731
4732	/* If we have unrestricted guest execution, we never have to intercept CR3 reads. */
4733	if (pVM->hm.s.vmx.fUnrestrictedGuest)
4734	uProcCtls &= ~VMX_PROC_CTLS_CR3_STORE_EXIT;
4735	}
4736	else
4737	{
4738	/* Guest CPL 0 writes to its read-only pages should cause a #PF VM-exit. */
4739	u64GuestCr0 \|= X86_CR0_WP;
4740	}
4741
4742	/*
4743	* Guest FPU bits.
4744	*
4745	* Since we pre-load the guest FPU always before VM-entry there is no need to track lazy state
4746	* using CR0.TS.
4747	*
4748	* Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be
4749	* set on the first CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
4750	*/
4751	u64GuestCr0 \|= X86_CR0_NE;
4752
4753	/* If CR0.NE isn't set, we need to intercept #MF exceptions and report them to the guest differently. */
4754	bool const fInterceptMF = !(u64ShadowCr0 & X86_CR0_NE);
4755
4756	/*
4757	* Update exception intercepts.
4758	*/
4759	uint32_t uXcptBitmap = pVmcsInfo->u32XcptBitmap;
4760	if (pVmcsInfo->RealMode.fRealOnV86Active)
4761	{
4762	Assert(PDMVmmDevHeapIsEnabled(pVM));
4763	Assert(pVM->hm.s.vmx.pRealModeTSS);
4764	uXcptBitmap \|= HMVMX_REAL_MODE_XCPT_MASK;
4765	}
4766	else
4767	{
4768	/* For now, cleared here as mode-switches can happen outside HM/VT-x. See @bugref{7626#c11}. */
4769	uXcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
4770	if (fInterceptMF)
4771	uXcptBitmap \|= RT_BIT(X86_XCPT_MF);
4772	}
4773
4774	/* Additional intercepts for debugging, define these yourself explicitly. */
4775	#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
4776	uXcptBitmap \|= 0
4777	\| RT_BIT(X86_XCPT_BP)
4778	\| RT_BIT(X86_XCPT_DE)
4779	\| RT_BIT(X86_XCPT_NM)
4780	\| RT_BIT(X86_XCPT_TS)
4781	\| RT_BIT(X86_XCPT_UD)
4782	\| RT_BIT(X86_XCPT_NP)
4783	\| RT_BIT(X86_XCPT_SS)
4784	\| RT_BIT(X86_XCPT_GP)
4785	\| RT_BIT(X86_XCPT_PF)
4786	\| RT_BIT(X86_XCPT_MF)
4787	;
4788	#elif defined(HMVMX_ALWAYS_TRAP_PF)
4789	uXcptBitmap \|= RT_BIT(X86_XCPT_PF);
4790	#endif
4791	if (pVCpu->hm.s.fTrapXcptGpForLovelyMesaDrv)
4792	uXcptBitmap \|= RT_BIT(X86_XCPT_GP);
4793	Assert(pVM->hm.s.fNestedPaging \|\| (uXcptBitmap & RT_BIT(X86_XCPT_PF)));
4794
4795	/* Apply the fixed CR0 bits and enable caching. */
4796	u64GuestCr0 \|= fSetCr0;
4797	u64GuestCr0 &= fZapCr0;
4798	u64GuestCr0 &= ~(uint64_t)(X86_CR0_CD \| X86_CR0_NW);
4799
4800	/* Commit the CR0 and related fields to the guest VMCS. */
4801	int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR0, u64GuestCr0);
4802	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, u64ShadowCr0);
4803	if (uProcCtls != pVmcsInfo->u32ProcCtls)
4804	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
4805	if (uXcptBitmap != pVmcsInfo->u32XcptBitmap)
4806	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
4807	AssertRCReturn(rc, rc);
4808
4809	/* Update our caches. */
4810	pVmcsInfo->u32ProcCtls = uProcCtls;
4811	pVmcsInfo->u32XcptBitmap = uXcptBitmap;
4812
4813	Log4Func(("cr0=%#RX64 shadow=%#RX64 set=%#RX64 zap=%#RX64\n", u64GuestCr0, u64ShadowCr0, fSetCr0, fZapCr0));
4814	}
4815	else
4816	{
4817	PCVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
4818	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0);
4819	uint64_t u64GuestCr0 = pVCpu->cpum.GstCtx.cr0;
4820	uint64_t const u64ShadowCr0 = pVmcsNstGst->u64Cr0ReadShadow.u;
4821	Assert(!RT_HI_U32(u64GuestCr0));
4822	Assert(u64GuestCr0 & X86_CR0_NE);
4823
4824	/* Apply the fixed CR0 bits and enable caching. */
4825	u64GuestCr0 \|= fSetCr0;
4826	u64GuestCr0 &= fZapCr0;
4827	u64GuestCr0 &= ~(uint64_t)(X86_CR0_CD \| X86_CR0_NW);
4828
4829	/* Commit the CR0 and CR0 read shadow to the nested-guest VMCS. */
4830	int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR0, u64GuestCr0);
4831	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, u64ShadowCr0);
4832	AssertRCReturn(rc, rc);
4833
4834	Log4Func(("cr0=%#RX64 shadow=%#RX64 set=%#RX64 zap=%#RX64\n", u64GuestCr0, u64ShadowCr0, fSetCr0, fZapCr0));
4835	}
4836
4837	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR0);
4838	}
4839
4840	return VINF_SUCCESS;
4841	}
4842
4843
4844	/**
4845	* Exports the guest control registers (CR3, CR4) into the guest-state area
4846	* in the VMCS.
4847	*
4848	* @returns VBox strict status code.
4849	* @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
4850	* without unrestricted guest access and the VMMDev is not presently
4851	* mapped (e.g. EFI32).
4852	*
4853	* @param pVCpu The cross context virtual CPU structure.
4854	* @param pVmxTransient The VMX-transient structure.
4855	*
4856	* @remarks No-long-jump zone!!!
4857	*/
4858	static VBOXSTRICTRC hmR0VmxExportGuestCR3AndCR4(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
4859	{
4860	int rc = VINF_SUCCESS;
4861	PVM pVM = pVCpu->CTX_SUFF(pVM);
4862
4863	/*
4864	* Guest CR2.
4865	* It's always loaded in the assembler code. Nothing to do here.
4866	*/
4867
4868	/*
4869	* Guest CR3.
4870	*/
4871	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR3)
4872	{
4873	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR3);
4874
4875	RTGCPHYS GCPhysGuestCR3 = NIL_RTGCPHYS;
4876	if (pVM->hm.s.fNestedPaging)
4877	{
4878	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
4879	pVmcsInfo->HCPhysEPTP = PGMGetHyperCR3(pVCpu);
4880
4881	/* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
4882	Assert(pVmcsInfo->HCPhysEPTP != NIL_RTHCPHYS);
4883	Assert(!(pVmcsInfo->HCPhysEPTP & UINT64_C(0xfff0000000000000)));
4884	Assert(!(pVmcsInfo->HCPhysEPTP & 0xfff));
4885
4886	/* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
4887	pVmcsInfo->HCPhysEPTP \|= VMX_EPT_MEMTYPE_WB
4888	\| (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
4889
4890	/* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
4891	AssertMsg( ((pVmcsInfo->HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
4892	&& ((pVmcsInfo->HCPhysEPTP >> 7) & 0x1f) == 0, /* Bits 7:11 MBZ. */
4893	("EPTP %#RX64\n", pVmcsInfo->HCPhysEPTP));
4894	AssertMsg( !((pVmcsInfo->HCPhysEPTP >> 6) & 0x01) /* Bit 6 (EPT accessed & dirty bit). */
4895	\|\| (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EPT_ACCESS_DIRTY),
4896	("EPTP accessed/dirty bit not supported by CPU but set %#RX64\n", pVmcsInfo->HCPhysEPTP));
4897
4898	rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVmcsInfo->HCPhysEPTP);
4899	AssertRCReturn(rc, rc);
4900
4901	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4902	if ( pVM->hm.s.vmx.fUnrestrictedGuest
4903	\|\| CPUMIsGuestPagingEnabledEx(pCtx))
4904	{
4905	/* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
4906	if (CPUMIsGuestInPAEModeEx(pCtx))
4907	{
4908	rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
4909	AssertRCReturn(rc, rc);
4910	rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u);
4911	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u);
4912	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u);
4913	rc \|= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u);
4914	AssertRCReturn(rc, rc);
4915	}
4916
4917	/*
4918	* The guest's view of its CR3 is unblemished with nested paging when the
4919	* guest is using paging or we have unrestricted guest execution to handle
4920	* the guest when it's not using paging.
4921	*/
4922	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR3);
4923	GCPhysGuestCR3 = pCtx->cr3;
4924	}
4925	else
4926	{
4927	/*
4928	* The guest is not using paging, but the CPU (VT-x) has to. While the guest
4929	* thinks it accesses physical memory directly, we use our identity-mapped
4930	* page table to map guest-linear to guest-physical addresses. EPT takes care
4931	* of translating it to host-physical addresses.
4932	*/
4933	RTGCPHYS GCPhys;
4934	Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
4935
4936	/* We obtain it here every time as the guest could have relocated this PCI region. */
4937	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
4938	if (RT_SUCCESS(rc))
4939	{ /* likely */ }
4940	else if (rc == VERR_PDM_DEV_HEAP_R3_TO_GCPHYS)
4941	{
4942	Log4Func(("VERR_PDM_DEV_HEAP_R3_TO_GCPHYS -> VINF_EM_RESCHEDULE_REM\n"));
4943	return VINF_EM_RESCHEDULE_REM; /* We cannot execute now, switch to REM/IEM till the guest maps in VMMDev. */
4944	}
4945	else
4946	AssertMsgFailedReturn(("%Rrc\n", rc), rc);
4947
4948	GCPhysGuestCR3 = GCPhys;
4949	}
4950
4951	Log4Func(("u32GuestCr3=%#RGp (GstN)\n", GCPhysGuestCR3));
4952	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, GCPhysGuestCR3);
4953	AssertRCReturn(rc, rc);
4954	}
4955	else
4956	{
4957	/* Non-nested paging case, just use the hypervisor's CR3. */
4958	RTHCPHYS const HCPhysGuestCR3 = PGMGetHyperCR3(pVCpu);
4959
4960	Log4Func(("u32GuestCr3=%#RHv (HstN)\n", HCPhysGuestCR3));
4961	rc = VMXWriteVmcsHstN(VMX_VMCS_GUEST_CR3, HCPhysGuestCR3);
4962	AssertRCReturn(rc, rc);
4963	}
4964
4965	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR3);
4966	}
4967
4968	/*
4969	* Guest CR4.
4970	* ASSUMES this is done everytime we get in from ring-3! (XCR0)
4971	*/
4972	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR4)
4973	{
4974	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4975	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
4976	PCVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
4977
4978	/*
4979	* Figure out fixed CR4 bits in VMX operation.
4980	*/
4981	uint64_t const fSetCr4 = pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1;
4982	uint64_t const fZapCr4 = pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr4Fixed1;
4983
4984	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR4);
4985	uint64_t u64GuestCr4 = pCtx->cr4;
4986	uint64_t const u64ShadowCr4 = !pVmxTransient->fIsNestedGuest ? pCtx->cr4 : pVmcsNstGst->u64Cr4ReadShadow.u;
4987	Assert(!RT_HI_U32(u64GuestCr4));
4988
4989	/*
4990	* Setup VT-x's view of the guest CR4.
4991	*
4992	* If we're emulating real-mode using virtual-8086 mode, we want to redirect software
4993	* interrupts to the 8086 program interrupt handler. Clear the VME bit (the interrupt
4994	* redirection bitmap is already all 0, see hmR3InitFinalizeR0())
4995	*
4996	* See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
4997	*/
4998	if (pVmcsInfo->RealMode.fRealOnV86Active)
4999	{
5000	Assert(pVM->hm.s.vmx.pRealModeTSS);
5001	Assert(PDMVmmDevHeapIsEnabled(pVM));
5002	u64GuestCr4 &= ~(uint64_t)X86_CR4_VME;
5003	}
5004
5005	if (pVM->hm.s.fNestedPaging)
5006	{
5007	if ( !CPUMIsGuestPagingEnabledEx(pCtx)
5008	&& !pVM->hm.s.vmx.fUnrestrictedGuest)
5009	{
5010	/* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
5011	u64GuestCr4 \|= X86_CR4_PSE;
5012	/* Our identity mapping is a 32-bit page directory. */
5013	u64GuestCr4 &= ~(uint64_t)X86_CR4_PAE;
5014	}
5015	/* else use guest CR4.*/
5016	}
5017	else
5018	{
5019	Assert(!pVmxTransient->fIsNestedGuest);
5020
5021	/*
5022	* The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
5023	* paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
5024	*/
5025	switch (pVCpu->hm.s.enmShadowMode)
5026	{
5027	case PGMMODE_REAL: /* Real-mode. */
5028	case PGMMODE_PROTECTED: /* Protected mode without paging. */
5029	case PGMMODE_32_BIT: /* 32-bit paging. */
5030	{
5031	u64GuestCr4 &= ~(uint64_t)X86_CR4_PAE;
5032	break;
5033	}
5034
5035	case PGMMODE_PAE: /* PAE paging. */
5036	case PGMMODE_PAE_NX: /* PAE paging with NX. */
5037	{
5038	u64GuestCr4 \|= X86_CR4_PAE;
5039	break;
5040	}
5041
5042	case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
5043	case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
5044	#ifdef VBOX_ENABLE_64_BITS_GUESTS
5045	break;
5046	#endif
5047	default:
5048	AssertFailed();
5049	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
5050	}
5051	}
5052
5053	/* Apply the fixed CR4 bits (mainly CR4.VMXE). */
5054	u64GuestCr4 \|= fSetCr4;
5055	u64GuestCr4 &= fZapCr4;
5056
5057	/* Commit the CR4 and CR4 read shadow to the guest VMCS. */
5058	rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR4, u64GuestCr4);
5059	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, u64ShadowCr4);
5060	AssertRCReturn(rc, rc);
5061
5062	/* Whether to save/load/restore XCR0 during world switch depends on CR4.OSXSAVE and host+guest XCR0. */
5063	pVCpu->hm.s.fLoadSaveGuestXcr0 = (pCtx->cr4 & X86_CR4_OSXSAVE) && pCtx->aXcr[0] != ASMGetXcr0();
5064
5065	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR4);
5066
5067	Log4Func(("cr4=%#RX64 shadow=%#RX64 set=%#RX64 zap=%#RX64)\n", u64GuestCr4, u64ShadowCr4, fSetCr4, fZapCr4));
5068	}
5069	return rc;
5070	}
5071
5072
5073	/**
5074	* Exports the guest debug registers into the guest-state area in the VMCS.
5075	* The guest debug bits are partially shared with the host (e.g. DR6, DR0-3).
5076	*
5077	* This also sets up whether \#DB and MOV DRx accesses cause VM-exits.
5078	*
5079	* @returns VBox status code.
5080	* @param pVCpu The cross context virtual CPU structure.
5081	* @param pVmxTransient The VMX-transient structure.
5082	*
5083	* @remarks No-long-jump zone!!!
5084	*/
5085	static int hmR0VmxExportSharedDebugState(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
5086	{
5087	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
5088
5089	/** @todo NSTVMX: Figure out what we want to do with nested-guest instruction
5090	* stepping. */
5091	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
5092	if (pVmxTransient->fIsNestedGuest)
5093	{
5094	int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, CPUMGetGuestDR7(pVCpu));
5095	AssertRCReturn(rc, rc);
5096	return VINF_SUCCESS;
5097	}
5098
5099	#ifdef VBOX_STRICT
5100	/* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
5101	if (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_DEBUG)
5102	{
5103	/* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
5104	Assert((pVCpu->cpum.GstCtx.dr[7] & (X86_DR7_MBZ_MASK \| X86_DR7_RAZ_MASK)) == 0);
5105	Assert((pVCpu->cpum.GstCtx.dr[7] & X86_DR7_RA1_MASK) == X86_DR7_RA1_MASK);
5106	}
5107	#endif
5108
5109	bool fSteppingDB = false;
5110	bool fInterceptMovDRx = false;
5111	uint32_t uProcCtls = pVmcsInfo->u32ProcCtls;
5112	if (pVCpu->hm.s.fSingleInstruction)
5113	{
5114	/* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
5115	PVM pVM = pVCpu->CTX_SUFF(pVM);
5116	if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_MONITOR_TRAP_FLAG)
5117	{
5118	uProcCtls \|= VMX_PROC_CTLS_MONITOR_TRAP_FLAG;
5119	Assert(fSteppingDB == false);
5120	}
5121	else
5122	{
5123	pVCpu->cpum.GstCtx.eflags.u32 \|= X86_EFL_TF;
5124	pVCpu->hm.s.fCtxChanged \|= HM_CHANGED_GUEST_RFLAGS;
5125	pVCpu->hm.s.fClearTrapFlag = true;
5126	fSteppingDB = true;
5127	}
5128	}
5129
5130	uint32_t u32GuestDr7;
5131	if ( fSteppingDB
5132	\|\| (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
5133	{
5134	/*
5135	* Use the combined guest and host DRx values found in the hypervisor register set
5136	* because the hypervisor debugger has breakpoints active or someone is single stepping
5137	* on the host side without a monitor trap flag.
5138	*
5139	* Note! DBGF expects a clean DR6 state before executing guest code.
5140	*/
5141	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
5142	if ( CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx)
5143	&& !CPUMIsHyperDebugStateActivePending(pVCpu))
5144	{
5145	CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
5146	Assert(CPUMIsHyperDebugStateActivePending(pVCpu));
5147	Assert(!CPUMIsGuestDebugStateActivePending(pVCpu));
5148	}
5149	else
5150	#endif
5151	if (!CPUMIsHyperDebugStateActive(pVCpu))
5152	{
5153	CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
5154	Assert(CPUMIsHyperDebugStateActive(pVCpu));
5155	Assert(!CPUMIsGuestDebugStateActive(pVCpu));
5156	}
5157
5158	/* Update DR7 with the hypervisor value (other DRx registers are handled by CPUM one way or another). */
5159	u32GuestDr7 = (uint32_t)CPUMGetHyperDR7(pVCpu);
5160	pVCpu->hm.s.fUsingHyperDR7 = true;
5161	fInterceptMovDRx = true;
5162	}
5163	else
5164	{
5165	/*
5166	* If the guest has enabled debug registers, we need to load them prior to
5167	* executing guest code so they'll trigger at the right time.
5168	*/
5169	if (pVCpu->cpum.GstCtx.dr[7] & (X86_DR7_ENABLED_MASK \| X86_DR7_GD))
5170	{
5171	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
5172	if ( CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx)
5173	&& !CPUMIsGuestDebugStateActivePending(pVCpu))
5174	{
5175	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
5176	Assert(CPUMIsGuestDebugStateActivePending(pVCpu));
5177	Assert(!CPUMIsHyperDebugStateActivePending(pVCpu));
5178	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
5179	}
5180	else
5181	#endif
5182	if (!CPUMIsGuestDebugStateActive(pVCpu))
5183	{
5184	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
5185	Assert(CPUMIsGuestDebugStateActive(pVCpu));
5186	Assert(!CPUMIsHyperDebugStateActive(pVCpu));
5187	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
5188	}
5189	Assert(!fInterceptMovDRx);
5190	}
5191	/*
5192	* If no debugging enabled, we'll lazy load DR0-3. Unlike on AMD-V, we
5193	* must intercept #DB in order to maintain a correct DR6 guest value, and
5194	* because we need to intercept it to prevent nested #DBs from hanging the
5195	* CPU, we end up always having to intercept it. See hmR0VmxSetupVmcsXcptBitmap().
5196	*/
5197	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
5198	else if ( !CPUMIsGuestDebugStateActivePending(pVCpu)
5199	&& !CPUMIsGuestDebugStateActive(pVCpu))
5200	#else
5201	else if (!CPUMIsGuestDebugStateActive(pVCpu))
5202	#endif
5203	{
5204	fInterceptMovDRx = true;
5205	}
5206
5207	/* Update DR7 with the actual guest value. */
5208	u32GuestDr7 = pVCpu->cpum.GstCtx.dr[7];
5209	pVCpu->hm.s.fUsingHyperDR7 = false;
5210	}
5211
5212	if (fInterceptMovDRx)
5213	uProcCtls \|= VMX_PROC_CTLS_MOV_DR_EXIT;
5214	else
5215	uProcCtls &= ~VMX_PROC_CTLS_MOV_DR_EXIT;
5216
5217	/*
5218	* Update the processor-based VM-execution controls with the MOV-DRx intercepts and the
5219	* monitor-trap flag and update our cache.
5220	*/
5221	if (uProcCtls != pVmcsInfo->u32ProcCtls)
5222	{
5223	int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
5224	AssertRCReturn(rc2, rc2);
5225	pVmcsInfo->u32ProcCtls = uProcCtls;
5226	}
5227
5228	/*
5229	* Update guest DR7.
5230	*/
5231	int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, u32GuestDr7);
5232	AssertRCReturn(rc, rc);
5233
5234	/*
5235	* If we have forced EFLAGS.TF to be set because we're single-stepping in the hypervisor debugger,
5236	* we need to clear interrupt inhibition if any as otherwise it causes a VM-entry failure.
5237	*
5238	* See Intel spec. 26.3.1.5 "Checks on Guest Non-Register State".
5239	*/
5240	if (fSteppingDB)
5241	{
5242	Assert(pVCpu->hm.s.fSingleInstruction);
5243	Assert(pVCpu->cpum.GstCtx.eflags.Bits.u1TF);
5244
5245	uint32_t fIntrState = 0;
5246	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &fIntrState);
5247	AssertRCReturn(rc, rc);
5248
5249	if (fIntrState & (VMX_VMCS_GUEST_INT_STATE_BLOCK_STI \| VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS))
5250	{
5251	fIntrState &= ~(VMX_VMCS_GUEST_INT_STATE_BLOCK_STI \| VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS);
5252	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INT_STATE, fIntrState);
5253	AssertRCReturn(rc, rc);
5254	}
5255	}
5256
5257	return VINF_SUCCESS;
5258	}
5259
5260
5261	#ifdef VBOX_STRICT
5262	/**
5263	* Strict function to validate segment registers.
5264	*
5265	* @param pVCpu The cross context virtual CPU structure.
5266	* @param pVmcsInfo The VMCS info. object.
5267	*
5268	* @remarks Will import guest CR0 on strict builds during validation of
5269	* segments.
5270	*/
5271	static void hmR0VmxValidateSegmentRegs(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
5272	{
5273	/*
5274	* Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
5275	*
5276	* The reason we check for attribute value 0 in this function and not just the unusable bit is
5277	* because hmR0VmxExportGuestSegReg() only updates the VMCS' copy of the value with the
5278	* unusable bit and doesn't change the guest-context value.
5279	*/
5280	PVM pVM = pVCpu->CTX_SUFF(pVM);
5281	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
5282	hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_CR0);
5283	if ( !pVM->hm.s.vmx.fUnrestrictedGuest
5284	&& ( !CPUMIsGuestInRealModeEx(pCtx)
5285	&& !CPUMIsGuestInV86ModeEx(pCtx)))
5286	{
5287	/* Protected mode checks */
5288	/* CS */
5289	Assert(pCtx->cs.Attr.n.u1Present);
5290	Assert(!(pCtx->cs.Attr.u & 0xf00));
5291	Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
5292	Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
5293	\|\| !(pCtx->cs.Attr.n.u1Granularity));
5294	Assert( !(pCtx->cs.u32Limit & 0xfff00000)
5295	\|\| (pCtx->cs.Attr.n.u1Granularity));
5296	/* CS cannot be loaded with NULL in protected mode. */
5297	Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE)); /** @todo is this really true even for 64-bit CS? */
5298	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
5299	Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
5300	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
5301	Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
5302	else
5303	AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
5304	/* SS */
5305	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
5306	Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
5307	if ( !(pCtx->cr0 & X86_CR0_PE)
5308	\|\| pCtx->cs.Attr.n.u4Type == 3)
5309	{
5310	Assert(!pCtx->ss.Attr.n.u2Dpl);
5311	}
5312	if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
5313	{
5314	Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
5315	Assert(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7);
5316	Assert(pCtx->ss.Attr.n.u1Present);
5317	Assert(!(pCtx->ss.Attr.u & 0xf00));
5318	Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
5319	Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
5320	\|\| !(pCtx->ss.Attr.n.u1Granularity));
5321	Assert( !(pCtx->ss.u32Limit & 0xfff00000)
5322	\|\| (pCtx->ss.Attr.n.u1Granularity));
5323	}
5324	/* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxExportGuestSegReg(). */
5325	if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
5326	{
5327	Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
5328	Assert(pCtx->ds.Attr.n.u1Present);
5329	Assert(pCtx->ds.Attr.n.u4Type > 11 \|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
5330	Assert(!(pCtx->ds.Attr.u & 0xf00));
5331	Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
5332	Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
5333	\|\| !(pCtx->ds.Attr.n.u1Granularity));
5334	Assert( !(pCtx->ds.u32Limit & 0xfff00000)
5335	\|\| (pCtx->ds.Attr.n.u1Granularity));
5336	Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
5337	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
5338	}
5339	if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
5340	{
5341	Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
5342	Assert(pCtx->es.Attr.n.u1Present);
5343	Assert(pCtx->es.Attr.n.u4Type > 11 \|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
5344	Assert(!(pCtx->es.Attr.u & 0xf00));
5345	Assert(!(pCtx->es.Attr.u & 0xfffe0000));
5346	Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
5347	\|\| !(pCtx->es.Attr.n.u1Granularity));
5348	Assert( !(pCtx->es.u32Limit & 0xfff00000)
5349	\|\| (pCtx->es.Attr.n.u1Granularity));
5350	Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
5351	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
5352	}
5353	if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
5354	{
5355	Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
5356	Assert(pCtx->fs.Attr.n.u1Present);
5357	Assert(pCtx->fs.Attr.n.u4Type > 11 \|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
5358	Assert(!(pCtx->fs.Attr.u & 0xf00));
5359	Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
5360	Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
5361	\|\| !(pCtx->fs.Attr.n.u1Granularity));
5362	Assert( !(pCtx->fs.u32Limit & 0xfff00000)
5363	\|\| (pCtx->fs.Attr.n.u1Granularity));
5364	Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
5365	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
5366	}
5367	if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
5368	{
5369	Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
5370	Assert(pCtx->gs.Attr.n.u1Present);
5371	Assert(pCtx->gs.Attr.n.u4Type > 11 \|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
5372	Assert(!(pCtx->gs.Attr.u & 0xf00));
5373	Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
5374	Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
5375	\|\| !(pCtx->gs.Attr.n.u1Granularity));
5376	Assert( !(pCtx->gs.u32Limit & 0xfff00000)
5377	\|\| (pCtx->gs.Attr.n.u1Granularity));
5378	Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
5379	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
5380	}
5381	/* 64-bit capable CPUs. */
5382	# if HC_ARCH_BITS == 64
5383	Assert(!RT_HI_U32(pCtx->cs.u64Base));
5384	Assert(!pCtx->ss.Attr.u \|\| !RT_HI_U32(pCtx->ss.u64Base));
5385	Assert(!pCtx->ds.Attr.u \|\| !RT_HI_U32(pCtx->ds.u64Base));
5386	Assert(!pCtx->es.Attr.u \|\| !RT_HI_U32(pCtx->es.u64Base));
5387	# endif
5388	}
5389	else if ( CPUMIsGuestInV86ModeEx(pCtx)
5390	\|\| ( CPUMIsGuestInRealModeEx(pCtx)
5391	&& !pVM->hm.s.vmx.fUnrestrictedGuest))
5392	{
5393	/* Real and v86 mode checks. */
5394	/* hmR0VmxExportGuestSegReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
5395	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
5396	if (pVmcsInfo->RealMode.fRealOnV86Active)
5397	{
5398	u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3;
5399	u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
5400	}
5401	else
5402	{
5403	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
5404	u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
5405	}
5406
5407	/* CS */
5408	AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
5409	Assert(pCtx->cs.u32Limit == 0xffff);
5410	Assert(u32CSAttr == 0xf3);
5411	/* SS */
5412	Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
5413	Assert(pCtx->ss.u32Limit == 0xffff);
5414	Assert(u32SSAttr == 0xf3);
5415	/* DS */
5416	Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
5417	Assert(pCtx->ds.u32Limit == 0xffff);
5418	Assert(u32DSAttr == 0xf3);
5419	/* ES */
5420	Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
5421	Assert(pCtx->es.u32Limit == 0xffff);
5422	Assert(u32ESAttr == 0xf3);
5423	/* FS */
5424	Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
5425	Assert(pCtx->fs.u32Limit == 0xffff);
5426	Assert(u32FSAttr == 0xf3);
5427	/* GS */
5428	Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
5429	Assert(pCtx->gs.u32Limit == 0xffff);
5430	Assert(u32GSAttr == 0xf3);
5431	/* 64-bit capable CPUs. */
5432	# if HC_ARCH_BITS == 64
5433	Assert(!RT_HI_U32(pCtx->cs.u64Base));
5434	Assert(!u32SSAttr \|\| !RT_HI_U32(pCtx->ss.u64Base));
5435	Assert(!u32DSAttr \|\| !RT_HI_U32(pCtx->ds.u64Base));
5436	Assert(!u32ESAttr \|\| !RT_HI_U32(pCtx->es.u64Base));
5437	# endif
5438	}
5439	}
5440	#endif /* VBOX_STRICT */
5441
5442
5443	/**
5444	* Exports a guest segment register into the guest-state area in the VMCS.
5445	*
5446	* @returns VBox status code.
5447	* @param pVCpu The cross context virtual CPU structure.
5448	* @param pVmcsInfo The VMCS info. object.
5449	* @param iSegReg The segment register number (X86_SREG_XXX).
5450	* @param pSelReg Pointer to the segment selector.
5451	*
5452	* @remarks No-long-jump zone!!!
5453	*/
5454	static int hmR0VmxExportGuestSegReg(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo, uint8_t iSegReg, PCCPUMSELREG pSelReg)
5455	{
5456	Assert(iSegReg < X86_SREG_COUNT);
5457	uint32_t const idxSel = g_aVmcsSegSel[iSegReg];
5458	uint32_t const idxLimit = g_aVmcsSegLimit[iSegReg];
5459	uint32_t const idxBase = g_aVmcsSegBase[iSegReg];
5460	uint32_t const idxAttr = g_aVmcsSegAttr[iSegReg];
5461
5462	uint32_t u32Access = pSelReg->Attr.u;
5463	if (pVmcsInfo->RealMode.fRealOnV86Active)
5464	{
5465	/* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
5466	u32Access = 0xf3;
5467	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
5468	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
5469	RT_NOREF_PV(pVCpu);
5470	}
5471	else
5472	{
5473	/*
5474	* The way to differentiate between whether this is really a null selector or was just
5475	* a selector loaded with 0 in real-mode is using the segment attributes. A selector
5476	* loaded in real-mode with the value 0 is valid and usable in protected-mode and we
5477	* should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures
5478	* NULL selectors loaded in protected-mode have their attribute as 0.
5479	*/
5480	if (!u32Access)
5481	u32Access = X86DESCATTR_UNUSABLE;
5482	}
5483
5484	/* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
5485	AssertMsg((u32Access & X86DESCATTR_UNUSABLE) \|\| (u32Access & X86_SEL_TYPE_ACCESSED),
5486	("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
5487
5488	/*
5489	* Commit it to the VMCS.
5490	*/
5491	int rc = VMXWriteVmcs32(idxSel, pSelReg->Sel);
5492	rc \|= VMXWriteVmcs32(idxLimit, pSelReg->u32Limit);
5493	rc \|= VMXWriteVmcsGstN(idxBase, pSelReg->u64Base);
5494	rc \|= VMXWriteVmcs32(idxAttr, u32Access);
5495	AssertRCReturn(rc, rc);
5496	return rc;
5497	}
5498
5499
5500	/**
5501	* Exports the guest segment registers, GDTR, IDTR, LDTR, TR into the guest-state
5502	* area in the VMCS.
5503	*
5504	* @returns VBox status code.
5505	* @param pVCpu The cross context virtual CPU structure.
5506	* @param pVmxTransient The VMX-transient structure.
5507	*
5508	* @remarks Will import guest CR0 on strict builds during validation of
5509	* segments.
5510	* @remarks No-long-jump zone!!!
5511	*/
5512	static int hmR0VmxExportGuestSegRegsXdtr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
5513	{
5514	int rc = VERR_INTERNAL_ERROR_5;
5515	PVM pVM = pVCpu->CTX_SUFF(pVM);
5516	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
5517	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
5518
5519	/*
5520	* Guest Segment registers: CS, SS, DS, ES, FS, GS.
5521	*/
5522	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SREG_MASK)
5523	{
5524	#ifdef VBOX_WITH_REM
5525	if (!pVM->hm.s.vmx.fUnrestrictedGuest)
5526	{
5527	Assert(!pVmxTransient->fIsNestedGuest);
5528	Assert(pVM->hm.s.vmx.pRealModeTSS);
5529	AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
5530	if ( pVmcsInfo->fWasInRealMode
5531	&& PGMGetGuestMode(pVCpu) >= PGMMODE_PROTECTED)
5532	{
5533	/* Signal that the recompiler must flush its code-cache as the guest -may- rewrite code it will later execute
5534	in real-mode (e.g. OpenBSD 4.0) */
5535	REMFlushTBs(pVM);
5536	Log4Func(("Switch to protected mode detected!\n"));
5537	pVmcsInfo->fWasInRealMode = false;
5538	}
5539	}
5540	#endif
5541	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CS)
5542	{
5543	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CS);
5544	if (pVmcsInfo->RealMode.fRealOnV86Active)
5545	pVmcsInfo->RealMode.AttrCS.u = pCtx->cs.Attr.u;
5546	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_CS, &pCtx->cs);
5547	AssertRCReturn(rc, rc);
5548	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CS);
5549	}
5550
5551	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SS)
5552	{
5553	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SS);
5554	if (pVmcsInfo->RealMode.fRealOnV86Active)
5555	pVmcsInfo->RealMode.AttrSS.u = pCtx->ss.Attr.u;
5556	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_SS, &pCtx->ss);
5557	AssertRCReturn(rc, rc);
5558	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SS);
5559	}
5560
5561	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_DS)
5562	{
5563	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_DS);
5564	if (pVmcsInfo->RealMode.fRealOnV86Active)
5565	pVmcsInfo->RealMode.AttrDS.u = pCtx->ds.Attr.u;
5566	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_DS, &pCtx->ds);
5567	AssertRCReturn(rc, rc);
5568	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_DS);
5569	}
5570
5571	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_ES)
5572	{
5573	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_ES);
5574	if (pVmcsInfo->RealMode.fRealOnV86Active)
5575	pVmcsInfo->RealMode.AttrES.u = pCtx->es.Attr.u;
5576	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_ES, &pCtx->es);
5577	AssertRCReturn(rc, rc);
5578	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_ES);
5579	}
5580
5581	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_FS)
5582	{
5583	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_FS);
5584	if (pVmcsInfo->RealMode.fRealOnV86Active)
5585	pVmcsInfo->RealMode.AttrFS.u = pCtx->fs.Attr.u;
5586	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_FS, &pCtx->fs);
5587	AssertRCReturn(rc, rc);
5588	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_FS);
5589	}
5590
5591	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_GS)
5592	{
5593	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_GS);
5594	if (pVmcsInfo->RealMode.fRealOnV86Active)
5595	pVmcsInfo->RealMode.AttrGS.u = pCtx->gs.Attr.u;
5596	rc = hmR0VmxExportGuestSegReg(pVCpu, pVmcsInfo, X86_SREG_GS, &pCtx->gs);
5597	AssertRCReturn(rc, rc);
5598	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_GS);
5599	}
5600
5601	#ifdef VBOX_STRICT
5602	hmR0VmxValidateSegmentRegs(pVCpu, pVmcsInfo);
5603	#endif
5604	Log4Func(("cs={%#04x base=%#RX64 limit=%#RX32 attr=%#RX32}\n", pCtx->cs.Sel, pCtx->cs.u64Base, pCtx->cs.u32Limit,
5605	pCtx->cs.Attr.u));
5606	}
5607
5608	/*
5609	* Guest TR.
5610	*/
5611	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_TR)
5612	{
5613	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_TR);
5614
5615	/*
5616	* Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is
5617	* achieved using the interrupt redirection bitmap (all bits cleared to let the guest
5618	* handle INT-n's) in the TSS. See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
5619	*/
5620	uint16_t u16Sel;
5621	uint32_t u32Limit;
5622	uint64_t u64Base;
5623	uint32_t u32AccessRights;
5624	if (!pVmcsInfo->RealMode.fRealOnV86Active)
5625	{
5626	u16Sel = pCtx->tr.Sel;
5627	u32Limit = pCtx->tr.u32Limit;
5628	u64Base = pCtx->tr.u64Base;
5629	u32AccessRights = pCtx->tr.Attr.u;
5630	}
5631	else
5632	{
5633	Assert(!pVmxTransient->fIsNestedGuest);
5634	Assert(pVM->hm.s.vmx.pRealModeTSS);
5635	Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMCanExecuteGuest() -XXX- what about inner loop changes? */
5636
5637	/* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
5638	RTGCPHYS GCPhys;
5639	rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
5640	AssertRCReturn(rc, rc);
5641
5642	X86DESCATTR DescAttr;
5643	DescAttr.u = 0;
5644	DescAttr.n.u1Present = 1;
5645	DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
5646
5647	u16Sel = 0;
5648	u32Limit = HM_VTX_TSS_SIZE;
5649	u64Base = GCPhys;
5650	u32AccessRights = DescAttr.u;
5651	}
5652
5653	/* Validate. */
5654	Assert(!(u16Sel & RT_BIT(2)));
5655	AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
5656	\|\| (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
5657	AssertMsg(!(u32AccessRights & X86DESCATTR_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
5658	Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
5659	Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
5660	Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
5661	Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
5662	Assert( (u32Limit & 0xfff) == 0xfff
5663	\|\| !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
5664	Assert( !(pCtx->tr.u32Limit & 0xfff00000)
5665	\|\| (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
5666
5667	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_TR_SEL, u16Sel);
5668	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit);
5669	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights);
5670	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base);
5671	AssertRCReturn(rc, rc);
5672
5673	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_TR);
5674	Log4Func(("tr base=%#RX64 limit=%#RX32\n", pCtx->tr.u64Base, pCtx->tr.u32Limit));
5675	}
5676
5677	/*
5678	* Guest GDTR.
5679	*/
5680	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_GDTR)
5681	{
5682	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_GDTR);
5683
5684	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pCtx->gdtr.cbGdt);
5685	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pCtx->gdtr.pGdt);
5686	AssertRCReturn(rc, rc);
5687
5688	/* Validate. */
5689	Assert(!(pCtx->gdtr.cbGdt & 0xffff0000)); /* Bits 31:16 MBZ. */
5690
5691	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_GDTR);
5692	Log4Func(("gdtr base=%#RX64 limit=%#RX32\n", pCtx->gdtr.pGdt, pCtx->gdtr.cbGdt));
5693	}
5694
5695	/*
5696	* Guest LDTR.
5697	*/
5698	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_LDTR)
5699	{
5700	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_LDTR);
5701
5702	/* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
5703	uint32_t u32Access;
5704	if ( !pVmxTransient->fIsNestedGuest
5705	&& !pCtx->ldtr.Attr.u)
5706	u32Access = X86DESCATTR_UNUSABLE;
5707	else
5708	u32Access = pCtx->ldtr.Attr.u;
5709
5710	rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_LDTR_SEL, pCtx->ldtr.Sel);
5711	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pCtx->ldtr.u32Limit);
5712	rc \|= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access);
5713	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pCtx->ldtr.u64Base);
5714	AssertRCReturn(rc, rc);
5715
5716	/* Validate. */
5717	if (!(u32Access & X86DESCATTR_UNUSABLE))
5718	{
5719	Assert(!(pCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
5720	Assert(pCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
5721	Assert(!pCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
5722	Assert(pCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
5723	Assert(!pCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
5724	Assert(!(pCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
5725	Assert( (pCtx->ldtr.u32Limit & 0xfff) == 0xfff
5726	\|\| !pCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
5727	Assert( !(pCtx->ldtr.u32Limit & 0xfff00000)
5728	\|\| pCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
5729	}
5730
5731	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_LDTR);
5732	Log4Func(("ldtr base=%#RX64 limit=%#RX32\n", pCtx->ldtr.u64Base, pCtx->ldtr.u32Limit));
5733	}
5734
5735	/*
5736	* Guest IDTR.
5737	*/
5738	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_IDTR)
5739	{
5740	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_IDTR);
5741
5742	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pCtx->idtr.cbIdt);
5743	rc \|= VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pCtx->idtr.pIdt);
5744	AssertRCReturn(rc, rc);
5745
5746	/* Validate. */
5747	Assert(!(pCtx->idtr.cbIdt & 0xffff0000)); /* Bits 31:16 MBZ. */
5748
5749	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_IDTR);
5750	Log4Func(("idtr base=%#RX64 limit=%#RX32\n", pCtx->idtr.pIdt, pCtx->idtr.cbIdt));
5751	}
5752
5753	return VINF_SUCCESS;
5754	}
5755
5756
5757	/**
5758	* Exports certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
5759	* areas.
5760	*
5761	* These MSRs will automatically be loaded to the host CPU on every successful
5762	* VM-entry and stored from the host CPU on every successful VM-exit.
5763	*
5764	* We creates/updates MSR slots for the host MSRs in the VM-exit MSR-load area. The
5765	* actual host MSR values are not- updated here for performance reasons. See
5766	* hmR0VmxExportHostMsrs().
5767	*
5768	* We also exports the guest sysenter MSRs into the guest-state area in the VMCS.
5769	*
5770	* @returns VBox status code.
5771	* @param pVCpu The cross context virtual CPU structure.
5772	* @param pVmxTransient The VMX-transient structure.
5773	*
5774	* @remarks No-long-jump zone!!!
5775	*/
5776	static int hmR0VmxExportGuestMsrs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
5777	{
5778	AssertPtr(pVCpu);
5779	AssertPtr(pVmxTransient);
5780
5781	PVM pVM = pVCpu->CTX_SUFF(pVM);
5782	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
5783
5784	/*
5785	* MSRs that we use the auto-load/store MSR area in the VMCS.
5786	* For 64-bit hosts, we load/restore them lazily, see hmR0VmxLazyLoadGuestMsrs().
5787	* The host MSR values are updated when it's safe in hmR0VmxLazySaveHostMsrs().
5788	*
5789	* For nested-guests, the guests MSRs from the VM-entry MSR-load area are already
5790	* loaded (into the guest-CPU context) by the VMLAUNCH/VMRESUME instruction
5791	* emulation, nothing to do here.
5792	*/
5793	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_GUEST_AUTO_MSRS)
5794	{
5795	if ( !pVmxTransient->fIsNestedGuest
5796	&& pVM->hm.s.fAllow64BitGuests)
5797	{
5798	#if HC_ARCH_BITS == 32
5799	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SYSCALL_MSRS \| CPUMCTX_EXTRN_KERNEL_GS_BASE);
5800	Assert(!pVmxTransient->fIsNestedGuest);
5801
5802	int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K8_LSTAR, pCtx->msrLSTAR, true, false);
5803	rc \|= hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K6_STAR, pCtx->msrSTAR, true, false);
5804	rc \|= hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K8_SF_MASK, pCtx->msrSFMASK, true, false);
5805	rc \|= hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K8_KERNEL_GS_BASE, pCtx->msrKERNELGSBASE, true, false);
5806	AssertRCReturn(rc, rc);
5807	#endif
5808	}
5809	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_GUEST_AUTO_MSRS);
5810	}
5811
5812	/*
5813	* Guest Sysenter MSRs.
5814	*/
5815	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_MSR_MASK)
5816	{
5817	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SYSENTER_MSRS);
5818
5819	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_CS_MSR)
5820	{
5821	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pCtx->SysEnter.cs);
5822	AssertRCReturn(rc, rc);
5823	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_CS_MSR);
5824	}
5825
5826	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_EIP_MSR)
5827	{
5828	int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pCtx->SysEnter.eip);
5829	AssertRCReturn(rc, rc);
5830	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_EIP_MSR);
5831	}
5832
5833	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_ESP_MSR)
5834	{
5835	int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pCtx->SysEnter.esp);
5836	AssertRCReturn(rc, rc);
5837	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_ESP_MSR);
5838	}
5839	}
5840
5841	/*
5842	* Guest/host EFER MSR.
5843	*/
5844	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_EFER_MSR)
5845	{
5846	/* Whether we are using the VMCS to swap the EFER MSR must have been
5847	determined earlier while exporting VM-entry/VM-exit controls. */
5848	Assert(!(ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_ENTRY_EXIT_CTLS));
5849	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_EFER);
5850
5851	if (hmR0VmxShouldSwapEferMsr(pVCpu))
5852	{
5853	/*
5854	* If the CPU supports VMCS controls for swapping EFER, use it. Otherwise, we have no option
5855	* but to use the auto-load store MSR area in the VMCS for swapping EFER. See @bugref{7368}.
5856	*/
5857	if (pVM->hm.s.vmx.fSupportsVmcsEfer)
5858	{
5859	int rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_EFER_FULL, pCtx->msrEFER);
5860	AssertRCReturn(rc, rc);
5861	}
5862	else
5863	{
5864	/*
5865	* We shall use the auto-load/store MSR area only for loading the EFER MSR but we must
5866	* continue to intercept guest read and write accesses to it, see @bugref{7386#c16}.
5867	*/
5868	int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K6_EFER, pCtx->msrEFER,
5869	false /* fSetReadWrite /, false / fUpdateHostMsr */);
5870	AssertRCReturn(rc, rc);
5871	}
5872	}
5873	else if (!pVM->hm.s.vmx.fSupportsVmcsEfer)
5874	hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K6_EFER);
5875
5876	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_EFER_MSR);
5877	}
5878
5879	/*
5880	* Other MSRs.
5881	* Speculation Control (R/W).
5882	*/
5883	if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_OTHER_MSRS)
5884	{
5885	HMVMX_CPUMCTX_ASSERT(pVCpu, HM_CHANGED_GUEST_OTHER_MSRS);
5886	if (pVM->cpum.ro.GuestFeatures.fIbrs)
5887	{
5888	int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_IA32_SPEC_CTRL, CPUMGetGuestSpecCtrl(pVCpu),
5889	false /* fSetReadWrite /, false / fUpdateHostMsr */);
5890	AssertRCReturn(rc, rc);
5891	}
5892	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_OTHER_MSRS);
5893	}
5894
5895	return VINF_SUCCESS;
5896	}
5897
5898
5899	/**
5900	* Selects up the appropriate function to run guest code.
5901	*
5902	* @returns VBox status code.
5903	* @param pVCpu The cross context virtual CPU structure.
5904	* @param pVmxTransient The VMX-transient structure.
5905	*
5906	* @remarks No-long-jump zone!!!
5907	*/
5908	static int hmR0VmxSelectVMRunHandler(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
5909	{
5910	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
5911	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
5912
5913	if (CPUMIsGuestInLongModeEx(pCtx))
5914	{
5915	#ifndef VBOX_ENABLE_64_BITS_GUESTS
5916	return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
5917	#endif
5918	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
5919	#if HC_ARCH_BITS == 32
5920	/* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
5921	if (pVmcsInfo->pfnStartVM != VMXR0SwitcherStartVM64)
5922	{
5923	#ifdef VBOX_STRICT
5924	if (pVmcsInfo->pfnStartVM != NULL) /* Very first VM-entry would have saved host-state already, ignore it. */
5925	{
5926	/* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
5927	uint64_t const fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
5928	RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
5929	AssertMsg(fCtxChanged & (HM_CHANGED_VMX_ENTRY_EXIT_CTLS \| HM_CHANGED_GUEST_EFER_MSR),
5930	("fCtxChanged=%#RX64\n", fCtxChanged));
5931	}
5932	#endif
5933	pVmcsInfo->pfnStartVM = VMXR0SwitcherStartVM64;
5934
5935	/* Mark that we've switched to 64-bit handler, we can't safely switch back to 32-bit for
5936	the rest of the VM run (until VM reset). See @bugref{8432#c7}. */
5937	pVmcsInfo->fSwitchedTo64on32 = true;
5938	Log4Func(("Selected 64-bit switcher\n"));
5939	}
5940	#else
5941	/* 64-bit host. */
5942	pVmcsInfo->pfnStartVM = VMXR0StartVM64;
5943	#endif
5944	}
5945	else
5946	{
5947	/* Guest is not in long mode, use the 32-bit handler. */
5948	#if HC_ARCH_BITS == 32
5949	if ( pVmcsInfo->pfnStartVM != VMXR0StartVM32
5950	&& !pVmcsInfo->fSwitchedTo64on32 /* If set, guest mode change does not imply switcher change. */
5951	&& pVmcsInfo->pfnStartVM != NULL) /* Very first VM-entry would have saved host-state already, ignore it. */
5952	{
5953	# ifdef VBOX_STRICT
5954	/* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
5955	uint64_t const fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
5956	RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
5957	AssertMsg(fCtxChanged & (HM_CHANGED_VMX_ENTRY_EXIT_CTLS \| HM_CHANGED_GUEST_EFER_MSR),
5958	("fCtxChanged=%#RX64\n", fCtxChanged));
5959	# endif
5960	}
5961	# ifdef VBOX_ENABLE_64_BITS_GUESTS
5962	/*
5963	* Keep using the 64-bit switcher even though we're in 32-bit because of bad Intel
5964	* design, see @bugref{8432#c7}. If real-on-v86 mode is active, clear the 64-bit
5965	* switcher flag now because we know the guest is in a sane state where it's safe
5966	* to use the 32-bit switcher. Otherwise, check the guest state if it's safe to use
5967	* the much faster 32-bit switcher again.
5968	*/
5969	if (!pVmcsInfo->fSwitchedTo64on32)
5970	{
5971	if (pVmcsInfo->pfnStartVM != VMXR0StartVM32)
5972	Log4Func(("Selected 32-bit switcher\n"));
5973	pVmcsInfo->pfnStartVM = VMXR0StartVM32;
5974	}
5975	else
5976	{
5977	Assert(pVmcsInfo->pfnStartVM == VMXR0SwitcherStartVM64);
5978	if ( pVmcsInfo->RealMode.fRealOnV86Active
5979	\|\| hmR0VmxIs32BitSwitcherSafe(pCtx))
5980	{
5981	pVmcsInfo->fSwitchedTo64on32 = false;
5982	pVmcsInfo->pfnStartVM = VMXR0StartVM32;
5983	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_EFER_MSR
5984	\| HM_CHANGED_VMX_ENTRY_EXIT_CTLS
5985	\| HM_CHANGED_HOST_CONTEXT);
5986	Log4Func(("Selected 32-bit switcher (safe)\n"));
5987	}
5988	}
5989	# else
5990	pVmcsInfo->pfnStartVM = VMXR0StartVM32;
5991	# endif
5992	#else
5993	pVmcsInfo->pfnStartVM = VMXR0StartVM32;
5994	#endif
5995	}
5996	Assert(pVmcsInfo->pfnStartVM);
5997	return VINF_SUCCESS;
5998	}
5999
6000
6001	/**
6002	* Wrapper for running the guest code in VT-x.
6003	*
6004	* @returns VBox status code, no informational status codes.
6005	* @param pVCpu The cross context virtual CPU structure.
6006	* @param pVmxTransient The VMX-transient structure.
6007	*
6008	* @remarks No-long-jump zone!!!
6009	*/
6010	DECLINLINE(int) hmR0VmxRunGuest(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
6011	{
6012	/* Mark that HM is the keeper of all guest-CPU registers now that we're going to execute guest code. */
6013	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6014	pCtx->fExtrn \|= HMVMX_CPUMCTX_EXTRN_ALL \| CPUMCTX_EXTRN_KEEPER_HM;
6015
6016	/** @todo Add stats for VMRESUME vs VMLAUNCH. */
6017
6018	/*
6019	* 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses
6020	* floating-point operations using SSE instructions. Some XMM registers (XMM6-XMM15) are
6021	* callee-saved and thus the need for this XMM wrapper.
6022	*
6023	* See MSDN "Configuring Programs for 64-bit/x64 Software Conventions / Register Usage".
6024	*/
6025	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
6026	bool const fResumeVM = RT_BOOL(pVmcsInfo->fVmcsState & VMX_V_VMCS_LAUNCH_STATE_LAUNCHED);
6027	PVM pVM = pVCpu->CTX_SUFF(pVM);
6028	#ifdef VBOX_WITH_KERNEL_USING_XMM
6029	int rc = hmR0VMXStartVMWrapXMM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VmcsCache, pVM, pVCpu, pVmcsInfo->pfnStartVM);
6030	#else
6031	int rc = pVmcsInfo->pfnStartVM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VmcsCache, pVM, pVCpu);
6032	#endif
6033	AssertMsg(rc <= VINF_SUCCESS, ("%Rrc\n", rc));
6034	return rc;
6035	}
6036
6037
6038	/**
6039	* Reports world-switch error and dumps some useful debug info.
6040	*
6041	* @param pVCpu The cross context virtual CPU structure.
6042	* @param rcVMRun The return code from VMLAUNCH/VMRESUME.
6043	* @param pVmxTransient The VMX-transient structure (only
6044	* exitReason updated).
6045	*/
6046	static void hmR0VmxReportWorldSwitchError(PVMCPU pVCpu, int rcVMRun, PVMXTRANSIENT pVmxTransient)
6047	{
6048	Assert(pVCpu);
6049	Assert(pVmxTransient);
6050	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
6051
6052	Log4Func(("VM-entry failure: %Rrc\n", rcVMRun));
6053	switch (rcVMRun)
6054	{
6055	case VERR_VMX_INVALID_VMXON_PTR:
6056	AssertFailed();
6057	break;
6058	case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
6059	case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
6060	{
6061	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.LastError.u32ExitReason);
6062	rc \|= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
6063	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
6064	AssertRC(rc);
6065
6066	pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
6067	/* LastError.idCurrentCpu was already updated in hmR0VmxPreRunGuestCommitted().
6068	Cannot do it here as we may have been long preempted. */
6069
6070	#ifdef VBOX_STRICT
6071	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
6072	Log4(("uExitReason %#RX32 (VmxTransient %#RX16)\n", pVCpu->hm.s.vmx.LastError.u32ExitReason,
6073	pVmxTransient->uExitReason));
6074	Log4(("Exit Qualification %#RX64\n", pVmxTransient->uExitQual));
6075	Log4(("InstrError %#RX32\n", pVCpu->hm.s.vmx.LastError.u32InstrError));
6076	if (pVCpu->hm.s.vmx.LastError.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
6077	Log4(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.LastError.u32InstrError]));
6078	else
6079	Log4(("InstrError Desc. Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
6080	Log4(("Entered host CPU %u\n", pVCpu->hm.s.vmx.LastError.idEnteredCpu));
6081	Log4(("Current host CPU %u\n", pVCpu->hm.s.vmx.LastError.idCurrentCpu));
6082
6083	/* VMX control bits. */
6084	uint32_t u32Val;
6085	uint64_t u64Val;
6086	RTHCUINTREG uHCReg;
6087	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val); AssertRC(rc);
6088	Log4(("VMX_VMCS32_CTRL_PIN_EXEC %#RX32\n", u32Val));
6089	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val); AssertRC(rc);
6090	Log4(("VMX_VMCS32_CTRL_PROC_EXEC %#RX32\n", u32Val));
6091	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
6092	{
6093	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val); AssertRC(rc);
6094	Log4(("VMX_VMCS32_CTRL_PROC_EXEC2 %#RX32\n", u32Val));
6095	}
6096	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val); AssertRC(rc);
6097	Log4(("VMX_VMCS32_CTRL_ENTRY %#RX32\n", u32Val));
6098	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val); AssertRC(rc);
6099	Log4(("VMX_VMCS32_CTRL_EXIT %#RX32\n", u32Val));
6100	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
6101	Log4(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
6102	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
6103	Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
6104	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
6105	Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
6106	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
6107	Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
6108	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
6109	Log4(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
6110	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
6111	Log4(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
6112	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
6113	Log4(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
6114	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
6115	Log4(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
6116	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
6117	Log4(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
6118	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
6119	Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
6120	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
6121	Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
6122	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
6123	Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
6124	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
6125	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
6126	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
6127	Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
6128	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
6129	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
6130	if (pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging)
6131	{
6132	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
6133	Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
6134	}
6135
6136	/* Guest bits. */
6137	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val); AssertRC(rc);
6138	Log4(("Old Guest Rip %#RX64 New %#RX64\n", pVCpu->cpum.GstCtx.rip, u64Val));
6139	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val); AssertRC(rc);
6140	Log4(("Old Guest Rsp %#RX64 New %#RX64\n", pVCpu->cpum.GstCtx.rsp, u64Val));
6141	rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val); AssertRC(rc);
6142	Log4(("Old Guest Rflags %#RX32 New %#RX32\n", pVCpu->cpum.GstCtx.eflags.u32, u32Val));
6143	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid)
6144	{
6145	rc = VMXReadVmcs32(VMX_VMCS16_VPID, &u32Val); AssertRC(rc);
6146	Log4(("VMX_VMCS16_VPID %u\n", u32Val));
6147	}
6148
6149	/* Host bits. */
6150	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
6151	Log4(("Host CR0 %#RHr\n", uHCReg));
6152	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
6153	Log4(("Host CR3 %#RHr\n", uHCReg));
6154	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
6155	Log4(("Host CR4 %#RHr\n", uHCReg));
6156
6157	RTGDTR HostGdtr;
6158	PCX86DESCHC pDesc;
6159	ASMGetGDTR(&HostGdtr);
6160	rc = VMXReadVmcs32(VMX_VMCS16_HOST_CS_SEL, &u32Val); AssertRC(rc);
6161	Log4(("Host CS %#08x\n", u32Val));
6162	if (u32Val < HostGdtr.cbGdt)
6163	{
6164	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
6165	hmR0DumpDescriptor(pDesc, u32Val, "CS: ");
6166	}
6167
6168	rc = VMXReadVmcs32(VMX_VMCS16_HOST_DS_SEL, &u32Val); AssertRC(rc);
6169	Log4(("Host DS %#08x\n", u32Val));
6170	if (u32Val < HostGdtr.cbGdt)
6171	{
6172	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
6173	hmR0DumpDescriptor(pDesc, u32Val, "DS: ");
6174	}
6175
6176	rc = VMXReadVmcs32(VMX_VMCS16_HOST_ES_SEL, &u32Val); AssertRC(rc);
6177	Log4(("Host ES %#08x\n", u32Val));
6178	if (u32Val < HostGdtr.cbGdt)
6179	{
6180	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
6181	hmR0DumpDescriptor(pDesc, u32Val, "ES: ");
6182	}
6183
6184	rc = VMXReadVmcs32(VMX_VMCS16_HOST_FS_SEL, &u32Val); AssertRC(rc);
6185	Log4(("Host FS %#08x\n", u32Val));
6186	if (u32Val < HostGdtr.cbGdt)
6187	{
6188	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
6189	hmR0DumpDescriptor(pDesc, u32Val, "FS: ");
6190	}
6191
6192	rc = VMXReadVmcs32(VMX_VMCS16_HOST_GS_SEL, &u32Val); AssertRC(rc);
6193	Log4(("Host GS %#08x\n", u32Val));
6194	if (u32Val < HostGdtr.cbGdt)
6195	{
6196	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
6197	hmR0DumpDescriptor(pDesc, u32Val, "GS: ");
6198	}
6199
6200	rc = VMXReadVmcs32(VMX_VMCS16_HOST_SS_SEL, &u32Val); AssertRC(rc);
6201	Log4(("Host SS %#08x\n", u32Val));
6202	if (u32Val < HostGdtr.cbGdt)
6203	{
6204	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
6205	hmR0DumpDescriptor(pDesc, u32Val, "SS: ");
6206	}
6207
6208	rc = VMXReadVmcs32(VMX_VMCS16_HOST_TR_SEL, &u32Val); AssertRC(rc);
6209	Log4(("Host TR %#08x\n", u32Val));
6210	if (u32Val < HostGdtr.cbGdt)
6211	{
6212	pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
6213	hmR0DumpDescriptor(pDesc, u32Val, "TR: ");
6214	}
6215
6216	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
6217	Log4(("Host TR Base %#RHv\n", uHCReg));
6218	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
6219	Log4(("Host GDTR Base %#RHv\n", uHCReg));
6220	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
6221	Log4(("Host IDTR Base %#RHv\n", uHCReg));
6222	rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
6223	Log4(("Host SYSENTER CS %#08x\n", u32Val));
6224	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
6225	Log4(("Host SYSENTER EIP %#RHv\n", uHCReg));
6226	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
6227	Log4(("Host SYSENTER ESP %#RHv\n", uHCReg));
6228	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
6229	Log4(("Host RSP %#RHv\n", uHCReg));
6230	rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
6231	Log4(("Host RIP %#RHv\n", uHCReg));
6232	# if HC_ARCH_BITS == 64
6233	Log4(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
6234	Log4(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
6235	Log4(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
6236	Log4(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
6237	Log4(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
6238	Log4(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
6239	# endif
6240	#endif /* VBOX_STRICT */
6241	break;
6242	}
6243
6244	default:
6245	/* Impossible */
6246	AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
6247	break;
6248	}
6249	}
6250
6251
6252	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
6253	# ifndef VMX_USE_CACHED_VMCS_ACCESSES
6254	# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
6255	# endif
6256
6257	/**
6258	* Initialize the VMCS-Read cache.
6259	*
6260	* The VMCS cache is used for 32-bit hosts running 64-bit guests (except 32-bit
6261	* Darwin which runs with 64-bit paging in 32-bit mode) for 64-bit fields that
6262	* cannot be accessed in 32-bit mode. Some 64-bit fields -can- be accessed
6263	* (those that have a 32-bit FULL & HIGH part).
6264	*
6265	* @param pVCpu The cross context virtual CPU structure.
6266	*/
6267	static void hmR0VmxInitVmcsReadCache(PVMCPU pVCpu)
6268	{
6269	#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
6270	do { \
6271	Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
6272	pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
6273	pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
6274	++cReadFields; \
6275	} while (0)
6276
6277	PVMXVMCSCACHE pCache = &pVCpu->hm.s.vmx.VmcsCache;
6278	uint32_t cReadFields = 0;
6279
6280	/*
6281	* Don't remove the #if 0'd fields in this code. They're listed here for consistency
6282	* and serve to indicate exceptions to the rules.
6283	*/
6284
6285	/* Guest-natural selector base fields. */
6286	#if 0
6287	/* These are 32-bit in practice. See Intel spec. 2.5 "Control Registers". */
6288	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
6289	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
6290	#endif
6291	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
6292	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
6293	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
6294	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
6295	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
6296	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
6297	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
6298	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
6299	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
6300	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
6301	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
6302	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
6303	#if 0
6304	/* Unused natural width guest-state fields. */
6305	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS);
6306	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in nested paging case */
6307	#endif
6308	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
6309	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
6310
6311	/* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for
6312	these 64-bit fields (using "FULL" and "HIGH" fields). */
6313	#if 0
6314	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
6315	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
6316	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
6317	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
6318	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
6319	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
6320	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
6321	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
6322	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
6323	#endif
6324
6325	/* Natural width guest-state fields. */
6326	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
6327	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_GUEST_LINEAR_ADDR);
6328
6329	if (pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging)
6330	{
6331	VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
6332	AssertMsg(cReadFields == VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields,
6333	VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX));
6334	pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
6335	}
6336	else
6337	{
6338	AssertMsg(cReadFields == VMX_VMCS_MAX_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields, VMX_VMCS_MAX_CACHE_IDX));
6339	pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
6340	}
6341
6342	#undef VMXLOCAL_INIT_READ_CACHE_FIELD
6343	}
6344
6345
6346	/**
6347	* Writes a field into the VMCS. This can either directly invoke a VMWRITE or
6348	* queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
6349	* darwin, running 64-bit guests).
6350	*
6351	* @returns VBox status code.
6352	* @param pVCpu The cross context virtual CPU structure.
6353	* @param idxField The VMCS field encoding.
6354	* @param u64Val 16, 32 or 64-bit value.
6355	*/
6356	VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
6357	{
6358	int rc;
6359	switch (idxField)
6360	{
6361	/*
6362	* These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
6363	*/
6364	/* 64-bit Control fields. */
6365	case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
6366	case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
6367	case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
6368	case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
6369	case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
6370	case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
6371	case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
6372	case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
6373	case VMX_VMCS64_CTRL_VIRT_APIC_PAGEADDR_FULL:
6374	case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
6375	case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
6376	case VMX_VMCS64_CTRL_EPTP_FULL:
6377	case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
6378	/* 64-bit Guest-state fields. */
6379	case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
6380	case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
6381	case VMX_VMCS64_GUEST_PAT_FULL:
6382	case VMX_VMCS64_GUEST_EFER_FULL:
6383	case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
6384	case VMX_VMCS64_GUEST_PDPTE0_FULL:
6385	case VMX_VMCS64_GUEST_PDPTE1_FULL:
6386	case VMX_VMCS64_GUEST_PDPTE2_FULL:
6387	case VMX_VMCS64_GUEST_PDPTE3_FULL:
6388	/* 64-bit Host-state fields. */
6389	case VMX_VMCS64_HOST_PAT_FULL:
6390	case VMX_VMCS64_HOST_EFER_FULL:
6391	case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
6392	{
6393	rc = VMXWriteVmcs32(idxField, RT_LO_U32(u64Val));
6394	rc \|= VMXWriteVmcs32(idxField + 1, RT_HI_U32(u64Val));
6395	break;
6396	}
6397
6398	/*
6399	* These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
6400	* values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
6401	*/
6402	/* Natural-width Guest-state fields. */
6403	case VMX_VMCS_GUEST_CR3:
6404	case VMX_VMCS_GUEST_ES_BASE:
6405	case VMX_VMCS_GUEST_CS_BASE:
6406	case VMX_VMCS_GUEST_SS_BASE:
6407	case VMX_VMCS_GUEST_DS_BASE:
6408	case VMX_VMCS_GUEST_FS_BASE:
6409	case VMX_VMCS_GUEST_GS_BASE:
6410	case VMX_VMCS_GUEST_LDTR_BASE:
6411	case VMX_VMCS_GUEST_TR_BASE:
6412	case VMX_VMCS_GUEST_GDTR_BASE:
6413	case VMX_VMCS_GUEST_IDTR_BASE:
6414	case VMX_VMCS_GUEST_RSP:
6415	case VMX_VMCS_GUEST_RIP:
6416	case VMX_VMCS_GUEST_SYSENTER_ESP:
6417	case VMX_VMCS_GUEST_SYSENTER_EIP:
6418	{
6419	if (!(RT_HI_U32(u64Val)))
6420	{
6421	/* If this field is 64-bit, VT-x will zero out the top bits. */
6422	rc = VMXWriteVmcs32(idxField, RT_LO_U32(u64Val));
6423	}
6424	else
6425	{
6426	/* Assert that only the 32->64 switcher case should ever come here. */
6427	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests);
6428	rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
6429	}
6430	break;
6431	}
6432
6433	default:
6434	{
6435	AssertMsgFailed(("VMXWriteVmcs64Ex: Invalid field %#RX32 (pVCpu=%p u64Val=%#RX64)\n", idxField, pVCpu, u64Val));
6436	rc = VERR_INVALID_PARAMETER;
6437	break;
6438	}
6439	}
6440	AssertRCReturn(rc, rc);
6441	return rc;
6442	}
6443
6444
6445	/**
6446	* Queue up a VMWRITE by using the VMCS write cache.
6447	* This is only used on 32-bit hosts (except darwin) for 64-bit guests.
6448	*
6449	* @param pVCpu The cross context virtual CPU structure.
6450	* @param idxField The VMCS field encoding.
6451	* @param u64Val 16, 32 or 64-bit value.
6452	*/
6453	VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
6454	{
6455	AssertPtr(pVCpu);
6456	PVMXVMCSCACHE pCache = &pVCpu->hm.s.vmx.VmcsCache;
6457
6458	AssertMsgReturn(pCache->Write.cValidEntries < VMX_VMCS_CACHE_MAX_ENTRY - 1,
6459	("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
6460
6461	/* Make sure there are no duplicates. */
6462	for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
6463	{
6464	if (pCache->Write.aField[i] == idxField)
6465	{
6466	pCache->Write.aFieldVal[i] = u64Val;
6467	return VINF_SUCCESS;
6468	}
6469	}
6470
6471	pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
6472	pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
6473	pCache->Write.cValidEntries++;
6474	return VINF_SUCCESS;
6475	}
6476	#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) */
6477
6478
6479	/**
6480	* Sets up the usage of TSC-offsetting and updates the VMCS.
6481	*
6482	* If offsetting is not possible, cause VM-exits on RDTSC(P)s. Also sets up the
6483	* VMX-preemption timer.
6484	*
6485	* @returns VBox status code.
6486	* @param pVCpu The cross context virtual CPU structure.
6487	* @param pVmxTransient The VMX-transient structure.
6488	*
6489	* @remarks No-long-jump zone!!!
6490	*/
6491	static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
6492	{
6493	bool fOffsettedTsc;
6494	bool fParavirtTsc;
6495	uint64_t uTscOffset;
6496	PVM pVM = pVCpu->CTX_SUFF(pVM);
6497	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);;
6498
6499	if (pVM->hm.s.vmx.fUsePreemptTimer)
6500	{
6501	uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVM, pVCpu, &uTscOffset, &fOffsettedTsc, &fParavirtTsc);
6502
6503	/* Make sure the returned values have sane upper and lower boundaries. */
6504	uint64_t u64CpuHz = SUPGetCpuHzFromGipBySetIndex(g_pSUPGlobalInfoPage, pVCpu->iHostCpuSet);
6505	cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
6506	cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
6507	cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
6508
6509	/** @todo r=ramshankar: We need to find a way to integrate nested-guest
6510	* preemption timers here. We probably need to clamp the preemption timer,
6511	* after converting the timer value to the host. */
6512	uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
6513	int rc = VMXWriteVmcs32(VMX_VMCS32_PREEMPT_TIMER_VALUE, cPreemptionTickCount);
6514	AssertRC(rc);
6515	}
6516	else
6517	fOffsettedTsc = TMCpuTickCanUseRealTSC(pVM, pVCpu, &uTscOffset, &fParavirtTsc);
6518
6519	if (fParavirtTsc)
6520	{
6521	/* Currently neither Hyper-V nor KVM need to update their paravirt. TSC
6522	information before every VM-entry, hence disable it for performance sake. */
6523	#if 0
6524	int rc = GIMR0UpdateParavirtTsc(pVM, 0 /* u64Offset */);
6525	AssertRC(rc);
6526	#endif
6527	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscParavirt);
6528	}
6529
6530	uint32_t uProcCtls = pVmcsInfo->u32ProcCtls;
6531	if ( fOffsettedTsc
6532	&& RT_LIKELY(!pVCpu->hm.s.fDebugWantRdTscExit))
6533	{
6534	if (pVmxTransient->fIsNestedGuest)
6535	uTscOffset = CPUMApplyNestedGuestTscOffset(pVCpu, uTscOffset);
6536	if (pVmcsInfo->u64TscOffset != uTscOffset)
6537	{
6538	int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, uTscOffset);
6539	AssertRC(rc);
6540	pVmcsInfo->u64TscOffset = uTscOffset;
6541	}
6542
6543	if (uProcCtls & VMX_PROC_CTLS_RDTSC_EXIT)
6544	{
6545	uProcCtls &= ~VMX_PROC_CTLS_RDTSC_EXIT;
6546	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
6547	AssertRC(rc);
6548	pVmcsInfo->u32ProcCtls = uProcCtls;
6549	}
6550	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
6551	}
6552	else
6553	{
6554	/* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
6555	if (!(uProcCtls & VMX_PROC_CTLS_RDTSC_EXIT))
6556	{
6557	uProcCtls \|= VMX_PROC_CTLS_RDTSC_EXIT;
6558	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
6559	AssertRC(rc);
6560	pVmcsInfo->u32ProcCtls = uProcCtls;
6561	}
6562	STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
6563	}
6564	}
6565
6566
6567	/**
6568	* Gets the IEM exception flags for the specified vector and IDT vectoring /
6569	* VM-exit interruption info type.
6570	*
6571	* @returns The IEM exception flags.
6572	* @param uVector The event vector.
6573	* @param uVmxEventType The VMX event type.
6574	*
6575	* @remarks This function currently only constructs flags required for
6576	* IEMEvaluateRecursiveXcpt and not the complete flags (e.g, error-code
6577	* and CR2 aspects of an exception are not included).
6578	*/
6579	static uint32_t hmR0VmxGetIemXcptFlags(uint8_t uVector, uint32_t uVmxEventType)
6580	{
6581	uint32_t fIemXcptFlags;
6582	switch (uVmxEventType)
6583	{
6584	case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
6585	case VMX_IDT_VECTORING_INFO_TYPE_NMI:
6586	fIemXcptFlags = IEM_XCPT_FLAGS_T_CPU_XCPT;
6587	break;
6588
6589	case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
6590	fIemXcptFlags = IEM_XCPT_FLAGS_T_EXT_INT;
6591	break;
6592
6593	case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
6594	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT \| IEM_XCPT_FLAGS_ICEBP_INSTR;
6595	break;
6596
6597	case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT:
6598	{
6599	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
6600	if (uVector == X86_XCPT_BP)
6601	fIemXcptFlags \|= IEM_XCPT_FLAGS_BP_INSTR;
6602	else if (uVector == X86_XCPT_OF)
6603	fIemXcptFlags \|= IEM_XCPT_FLAGS_OF_INSTR;
6604	else
6605	{
6606	fIemXcptFlags = 0;
6607	AssertMsgFailed(("Unexpected vector for software exception. uVector=%#x", uVector));
6608	}
6609	break;
6610	}
6611
6612	case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
6613	fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
6614	break;
6615
6616	default:
6617	fIemXcptFlags = 0;
6618	AssertMsgFailed(("Unexpected vector type! uVmxEventType=%#x uVector=%#x", uVmxEventType, uVector));
6619	break;
6620	}
6621	return fIemXcptFlags;
6622	}
6623
6624
6625	/**
6626	* Sets an event as a pending event to be injected into the guest.
6627	*
6628	* @param pVCpu The cross context virtual CPU structure.
6629	* @param u32IntInfo The VM-entry interruption-information field.
6630	* @param cbInstr The VM-entry instruction length in bytes (for software
6631	* interrupts, exceptions and privileged software
6632	* exceptions).
6633	* @param u32ErrCode The VM-entry exception error code.
6634	* @param GCPtrFaultAddress The fault-address (CR2) in case it's a
6635	* page-fault.
6636	*/
6637	DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPU pVCpu, uint32_t u32IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
6638	RTGCUINTPTR GCPtrFaultAddress)
6639	{
6640	Assert(!pVCpu->hm.s.Event.fPending);
6641	pVCpu->hm.s.Event.fPending = true;
6642	pVCpu->hm.s.Event.u64IntInfo = u32IntInfo;
6643	pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
6644	pVCpu->hm.s.Event.cbInstr = cbInstr;
6645	pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
6646	}
6647
6648
6649	/**
6650	* Sets an external interrupt as pending-for-injection into the VM.
6651	*
6652	* @param pVCpu The cross context virtual CPU structure.
6653	* @param u8Interrupt The external interrupt vector.
6654	*/
6655	DECLINLINE(void) hmR0VmxSetPendingExtInt(PVMCPU pVCpu, uint8_t u8Interrupt)
6656	{
6657	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_EXIT_INT_INFO_VECTOR, u8Interrupt)
6658	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_ENTRY_INT_INFO_TYPE_EXT_INT)
6659	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
6660	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
6661	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
6662	}
6663
6664
6665	/**
6666	* Sets an NMI (\#NMI) exception as pending-for-injection into the VM.
6667	*
6668	* @param pVCpu The cross context virtual CPU structure.
6669	*/
6670	DECLINLINE(void) hmR0VmxSetPendingXcptNmi(PVMCPU pVCpu)
6671	{
6672	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_NMI)
6673	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_ENTRY_INT_INFO_TYPE_NMI)
6674	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
6675	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
6676	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
6677	}
6678
6679
6680	/**
6681	* Sets a double-fault (\#DF) exception as pending-for-injection into the VM.
6682	*
6683	* @param pVCpu The cross context virtual CPU structure.
6684	*/
6685	DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPU pVCpu)
6686	{
6687	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_DF)
6688	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
6689	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 1)
6690	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
6691	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
6692	}
6693
6694
6695	/**
6696	* Sets an invalid-opcode (\#UD) exception as pending-for-injection into the VM.
6697	*
6698	* @param pVCpu The cross context virtual CPU structure.
6699	*/
6700	DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPU pVCpu)
6701	{
6702	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_UD)
6703	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
6704	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
6705	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
6706	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
6707	}
6708
6709
6710	/**
6711	* Sets a debug (\#DB) exception as pending-for-injection into the VM.
6712	*
6713	* @param pVCpu The cross context virtual CPU structure.
6714	*/
6715	DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPU pVCpu)
6716	{
6717	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_DB)
6718	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
6719	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
6720	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
6721	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, 0 / u32ErrCode /, 0 / GCPtrFaultAddress */);
6722	}
6723
6724
6725	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
6726	/**
6727	* Sets a general-protection (\#GP) exception as pending-for-injection into the VM.
6728	*
6729	* @param pVCpu The cross context virtual CPU structure.
6730	* @param u32ErrCode The error code for the general-protection exception.
6731	*/
6732	DECLINLINE(void) hmR0VmxSetPendingXcptGP(PVMCPU pVCpu, uint32_t u32ErrCode)
6733	{
6734	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_GP)
6735	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
6736	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 1)
6737	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
6738	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, u32ErrCode, 0 / GCPtrFaultAddress */);
6739	}
6740
6741
6742	/**
6743	* Sets a stack (\#SS) exception as pending-for-injection into the VM.
6744	*
6745	* @param pVCpu The cross context virtual CPU structure.
6746	* @param u32ErrCode The error code for the stack exception.
6747	*/
6748	DECLINLINE(void) hmR0VmxSetPendingXcptSS(PVMCPU pVCpu, uint32_t u32ErrCode)
6749	{
6750	uint32_t const u32IntInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_SS)
6751	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_EXIT_INT_INFO_TYPE_HW_XCPT)
6752	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 1)
6753	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
6754	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr /, u32ErrCode, 0 / GCPtrFaultAddress */);
6755	}
6756
6757
6758	/**
6759	* Decodes the memory operand of an instruction that caused a VM-exit.
6760	*
6761	* The VM-exit qualification field provides the displacement field for memory
6762	* operand instructions, if any.
6763	*
6764	* @returns Strict VBox status code (i.e. informational status codes too).
6765	* @retval VINF_SUCCESS if the operand was successfully decoded.
6766	* @retval VINF_HM_PENDING_XCPT if an exception was raised while decoding the
6767	* operand.
6768	* @param pVCpu The cross context virtual CPU structure.
6769	* @param uExitInstrInfo The VM-exit instruction information field.
6770	* @param enmMemAccess The memory operand's access type (read or write).
6771	* @param GCPtrDisp The instruction displacement field, if any. For
6772	* RIP-relative addressing pass RIP + displacement here.
6773	* @param pGCPtrMem Where to store the effective destination memory address.
6774	*/
6775	static VBOXSTRICTRC hmR0VmxDecodeMemOperand(PVMCPU pVCpu, uint32_t uExitInstrInfo, RTGCPTR GCPtrDisp, VMXMEMACCESS enmMemAccess,
6776	PRTGCPTR pGCPtrMem)
6777	{
6778	Assert(pGCPtrMem);
6779	Assert(!CPUMIsGuestInRealOrV86Mode(pVCpu));
6780	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK \| CPUMCTX_EXTRN_EFER
6781	\| CPUMCTX_EXTRN_CR0);
6782
6783	static uint64_t const s_auAddrSizeMasks[] = { UINT64_C(0xffff), UINT64_C(0xffffffff), UINT64_C(0xffffffffffffffff) };
6784	static uint64_t const s_auAccessSizeMasks[] = { sizeof(uint16_t), sizeof(uint32_t), sizeof(uint64_t) };
6785	AssertCompile(RT_ELEMENTS(s_auAccessSizeMasks) == RT_ELEMENTS(s_auAddrSizeMasks));
6786
6787	VMXEXITINSTRINFO ExitInstrInfo;
6788	ExitInstrInfo.u = uExitInstrInfo;
6789	uint8_t const uAddrSize = ExitInstrInfo.All.u3AddrSize;
6790	uint8_t const iSegReg = ExitInstrInfo.All.iSegReg;
6791	bool const fIdxRegValid = !ExitInstrInfo.All.fIdxRegInvalid;
6792	uint8_t const iIdxReg = ExitInstrInfo.All.iIdxReg;
6793	uint8_t const uScale = ExitInstrInfo.All.u2Scaling;
6794	bool const fBaseRegValid = !ExitInstrInfo.All.fBaseRegInvalid;
6795	uint8_t const iBaseReg = ExitInstrInfo.All.iBaseReg;
6796	bool const fIsMemOperand = !ExitInstrInfo.All.fIsRegOperand;
6797	bool const fIsLongMode = CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx);
6798
6799	/*
6800	* Validate instruction information.
6801	* This shouldn't happen on real hardware but useful while testing our nested hardware-virtualization code.
6802	*/
6803	AssertLogRelMsgReturn(uAddrSize < RT_ELEMENTS(s_auAddrSizeMasks),
6804	("Invalid address size. ExitInstrInfo=%#RX32\n", ExitInstrInfo.u), VERR_VMX_IPE_1);
6805	AssertLogRelMsgReturn(iSegReg < X86_SREG_COUNT,
6806	("Invalid segment register. ExitInstrInfo=%#RX32\n", ExitInstrInfo.u), VERR_VMX_IPE_2);
6807	AssertLogRelMsgReturn(fIsMemOperand,
6808	("Expected memory operand. ExitInstrInfo=%#RX32\n", ExitInstrInfo.u), VERR_VMX_IPE_3);
6809
6810	/*
6811	* Compute the complete effective address.
6812	*
6813	* See AMD instruction spec. 1.4.2 "SIB Byte Format"
6814	* See AMD spec. 4.5.2 "Segment Registers".
6815	*/
6816	RTGCPTR GCPtrMem = GCPtrDisp;
6817	if (fBaseRegValid)
6818	GCPtrMem += pVCpu->cpum.GstCtx.aGRegs[iBaseReg].u64;
6819	if (fIdxRegValid)
6820	GCPtrMem += pVCpu->cpum.GstCtx.aGRegs[iIdxReg].u64 << uScale;
6821
6822	RTGCPTR const GCPtrOff = GCPtrMem;
6823	if ( !fIsLongMode
6824	\|\| iSegReg >= X86_SREG_FS)
6825	GCPtrMem += pVCpu->cpum.GstCtx.aSRegs[iSegReg].u64Base;
6826	GCPtrMem &= s_auAddrSizeMasks[uAddrSize];
6827
6828	/*
6829	* Validate effective address.
6830	* See AMD spec. 4.5.3 "Segment Registers in 64-Bit Mode".
6831	*/
6832	uint8_t const cbAccess = s_auAccessSizeMasks[uAddrSize];
6833	Assert(cbAccess > 0);
6834	if (fIsLongMode)
6835	{
6836	if (X86_IS_CANONICAL(GCPtrMem))
6837	{
6838	*pGCPtrMem = GCPtrMem;
6839	return VINF_SUCCESS;
6840	}
6841
6842	/** @todo r=ramshankar: We should probably raise \#SS or \#GP. See AMD spec. 4.12.2
6843	* "Data Limit Checks in 64-bit Mode". */
6844	Log4Func(("Long mode effective address is not canonical GCPtrMem=%#RX64\n", GCPtrMem));
6845	hmR0VmxSetPendingXcptGP(pVCpu, 0);
6846	return VINF_HM_PENDING_XCPT;
6847	}
6848
6849	/*
6850	* This is a watered down version of iemMemApplySegment().
6851	* Parts that are not applicable for VMX instructions like real-or-v8086 mode
6852	* and segment CPL/DPL checks are skipped.
6853	*/
6854	RTGCPTR32 const GCPtrFirst32 = (RTGCPTR32)GCPtrOff;
6855	RTGCPTR32 const GCPtrLast32 = GCPtrFirst32 + cbAccess - 1;
6856	PCCPUMSELREG pSel = &pVCpu->cpum.GstCtx.aSRegs[iSegReg];
6857
6858	/* Check if the segment is present and usable. */
6859	if ( pSel->Attr.n.u1Present
6860	&& !pSel->Attr.n.u1Unusable)
6861	{
6862	Assert(pSel->Attr.n.u1DescType);
6863	if (!(pSel->Attr.n.u4Type & X86_SEL_TYPE_CODE))
6864	{
6865	/* Check permissions for the data segment. */
6866	if ( enmMemAccess == VMXMEMACCESS_WRITE
6867	&& !(pSel->Attr.n.u4Type & X86_SEL_TYPE_WRITE))
6868	{
6869	Log4Func(("Data segment access invalid. iSegReg=%#x Attr=%#RX32\n", iSegReg, pSel->Attr.u));
6870	hmR0VmxSetPendingXcptGP(pVCpu, iSegReg);
6871	return VINF_HM_PENDING_XCPT;
6872	}
6873
6874	/* Check limits if it's a normal data segment. */
6875	if (!(pSel->Attr.n.u4Type & X86_SEL_TYPE_DOWN))
6876	{
6877	if ( GCPtrFirst32 > pSel->u32Limit
6878	\|\| GCPtrLast32 > pSel->u32Limit)
6879	{
6880	Log4Func(("Data segment limit exceeded."
6881	"iSegReg=%#x GCPtrFirst32=%#RX32 GCPtrLast32=%#RX32 u32Limit=%#RX32\n", iSegReg, GCPtrFirst32,
6882	GCPtrLast32, pSel->u32Limit));
6883	if (iSegReg == X86_SREG_SS)
6884	hmR0VmxSetPendingXcptSS(pVCpu, 0);
6885	else
6886	hmR0VmxSetPendingXcptGP(pVCpu, 0);
6887	return VINF_HM_PENDING_XCPT;
6888	}
6889	}
6890	else
6891	{
6892	/* Check limits if it's an expand-down data segment.
6893	Note! The upper boundary is defined by the B bit, not the G bit! */
6894	if ( GCPtrFirst32 < pSel->u32Limit + UINT32_C(1)
6895	\|\| GCPtrLast32 > (pSel->Attr.n.u1DefBig ? UINT32_MAX : UINT32_C(0xffff)))
6896	{
6897	Log4Func(("Expand-down data segment limit exceeded."
6898	"iSegReg=%#x GCPtrFirst32=%#RX32 GCPtrLast32=%#RX32 u32Limit=%#RX32\n", iSegReg, GCPtrFirst32,
6899	GCPtrLast32, pSel->u32Limit));
6900	if (iSegReg == X86_SREG_SS)
6901	hmR0VmxSetPendingXcptSS(pVCpu, 0);
6902	else
6903	hmR0VmxSetPendingXcptGP(pVCpu, 0);
6904	return VINF_HM_PENDING_XCPT;
6905	}
6906	}
6907	}
6908	else
6909	{
6910	/* Check permissions for the code segment. */
6911	if ( enmMemAccess == VMXMEMACCESS_WRITE
6912	\|\| ( enmMemAccess == VMXMEMACCESS_READ
6913	&& !(pSel->Attr.n.u4Type & X86_SEL_TYPE_READ)))
6914	{
6915	Log4Func(("Code segment access invalid. Attr=%#RX32\n", pSel->Attr.u));
6916	Assert(!CPUMIsGuestInRealOrV86ModeEx(&pVCpu->cpum.GstCtx));
6917	hmR0VmxSetPendingXcptGP(pVCpu, 0);
6918	return VINF_HM_PENDING_XCPT;
6919	}
6920
6921	/* Check limits for the code segment (normal/expand-down not applicable for code segments). */
6922	if ( GCPtrFirst32 > pSel->u32Limit
6923	\|\| GCPtrLast32 > pSel->u32Limit)
6924	{
6925	Log4Func(("Code segment limit exceeded. GCPtrFirst32=%#RX32 GCPtrLast32=%#RX32 u32Limit=%#RX32\n",
6926	GCPtrFirst32, GCPtrLast32, pSel->u32Limit));
6927	if (iSegReg == X86_SREG_SS)
6928	hmR0VmxSetPendingXcptSS(pVCpu, 0);
6929	else
6930	hmR0VmxSetPendingXcptGP(pVCpu, 0);
6931	return VINF_HM_PENDING_XCPT;
6932	}
6933	}
6934	}
6935	else
6936	{
6937	Log4Func(("Not present or unusable segment. iSegReg=%#x Attr=%#RX32\n", iSegReg, pSel->Attr.u));
6938	hmR0VmxSetPendingXcptGP(pVCpu, 0);
6939	return VINF_HM_PENDING_XCPT;
6940	}
6941
6942	*pGCPtrMem = GCPtrMem;
6943	return VINF_SUCCESS;
6944	}
6945
6946
6947	/**
6948	* Perform the relevant VMX instruction checks for VM-exits that occurred due to the
6949	* guest attempting to execute a VMX instruction.
6950	*
6951	* @returns Strict VBox status code (i.e. informational status codes too).
6952	* @retval VINF_SUCCESS if we should continue handling the VM-exit.
6953	* @retval VINF_HM_PENDING_XCPT if an exception was raised.
6954	*
6955	* @param pVCpu The cross context virtual CPU structure.
6956	* @param uExitReason The VM-exit reason.
6957	*
6958	* @todo NSTVMX: Document other error codes when VM-exit is implemented.
6959	* @remarks No-long-jump zone!!!
6960	*/
6961	static VBOXSTRICTRC hmR0VmxCheckExitDueToVmxInstr(PVMCPU pVCpu, uint32_t uExitReason)
6962	{
6963	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_SS
6964	\| CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_EFER);
6965
6966	if ( CPUMIsGuestInRealOrV86ModeEx(&pVCpu->cpum.GstCtx)
6967	\|\| ( CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx)
6968	&& !CPUMIsGuestIn64BitCodeEx(&pVCpu->cpum.GstCtx)))
6969	{
6970	Log4Func(("In real/v86-mode or long-mode outside 64-bit code segment -> #UD\n"));
6971	hmR0VmxSetPendingXcptUD(pVCpu);
6972	return VINF_HM_PENDING_XCPT;
6973	}
6974
6975	if (uExitReason == VMX_EXIT_VMXON)
6976	{
6977	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR4);
6978
6979	/*
6980	* We check CR4.VMXE because it is required to be always set while in VMX operation
6981	* by physical CPUs and our CR4 read shadow is only consulted when executing specific
6982	* instructions (CLTS, LMSW, MOV CR, and SMSW) and thus doesn't affect CPU operation
6983	* otherwise (i.e. physical CPU won't automatically #UD if Cr4Shadow.VMXE is 0).
6984	*/
6985	if (!CPUMIsGuestVmxEnabled(&pVCpu->cpum.GstCtx))
6986	{
6987	Log4Func(("CR4.VMXE is not set -> #UD\n"));
6988	hmR0VmxSetPendingXcptUD(pVCpu);
6989	return VINF_HM_PENDING_XCPT;
6990	}
6991	}
6992	else if (!CPUMIsGuestInVmxRootMode(&pVCpu->cpum.GstCtx))
6993	{
6994	/*
6995	* The guest has not entered VMX operation but attempted to execute a VMX instruction
6996	* (other than VMXON), we need to raise a #UD.
6997	*/
6998	Log4Func(("Not in VMX root mode -> #UD\n"));
6999	hmR0VmxSetPendingXcptUD(pVCpu);
7000	return VINF_HM_PENDING_XCPT;
7001	}
7002
7003	if (CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
7004	{
7005	/*
7006	* The nested-guest attempted to execute a VMX instruction, cause a VM-exit and let
7007	* the guest hypervisor deal with it.
7008	*/
7009	/** @todo NSTVMX: Trigger a VM-exit */
7010	}
7011
7012	/*
7013	* VMX instructions require CPL 0 except in VMX non-root mode where the VM-exit intercept
7014	* (above) takes preceedence over the CPL check.
7015	*/
7016	if (CPUMGetGuestCPL(pVCpu) > 0)
7017	{
7018	Log4Func(("CPL > 0 -> #GP(0)\n"));
7019	hmR0VmxSetPendingXcptGP(pVCpu, 0);
7020	return VINF_HM_PENDING_XCPT;
7021	}
7022
7023	return VINF_SUCCESS;
7024	}
7025	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
7026
7027
7028	static void hmR0VmxFixUnusableSegRegAttr(PVMCPU pVCpu, PCPUMSELREG pSelReg, uint32_t idxSel)
7029	{
7030	Assert(pSelReg->Attr.u & X86DESCATTR_UNUSABLE);
7031
7032	/*
7033	* If VT-x marks the segment as unusable, most other bits remain undefined:
7034	* - For CS the L, D and G bits have meaning.
7035	* - For SS the DPL has meaning (it -is- the CPL for Intel and VBox).
7036	* - For the remaining data segments no bits are defined.
7037	*
7038	* The present bit and the unusable bit has been observed to be set at the
7039	* same time (the selector was supposed to be invalid as we started executing
7040	* a V8086 interrupt in ring-0).
7041	*
7042	* What should be important for the rest of the VBox code, is that the P bit is
7043	* cleared. Some of the other VBox code recognizes the unusable bit, but
7044	* AMD-V certainly don't, and REM doesn't really either. So, to be on the
7045	* safe side here, we'll strip off P and other bits we don't care about. If
7046	* any code breaks because Attr.u != 0 when Sel < 4, it should be fixed.
7047	*
7048	* See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
7049	*/
7050	#ifdef VBOX_STRICT
7051	uint32_t const uAttr = pSelReg->Attr.u;
7052	#endif
7053
7054	/* Masking off: X86DESCATTR_P, X86DESCATTR_LIMIT_HIGH, and X86DESCATTR_AVL. The latter two are really irrelevant. */
7055	pSelReg->Attr.u &= X86DESCATTR_UNUSABLE \| X86DESCATTR_L \| X86DESCATTR_D \| X86DESCATTR_G
7056	\| X86DESCATTR_DPL \| X86DESCATTR_TYPE \| X86DESCATTR_DT;
7057
7058	#ifdef VBOX_STRICT
7059	VMMRZCallRing3Disable(pVCpu);
7060	Log4Func(("Unusable %#x: sel=%#x attr=%#x -> %#x\n", idxSel, pSelReg->Sel, uAttr, pSelReg->Attr.u));
7061	# ifdef DEBUG_bird
7062	AssertMsg((uAttr & ~X86DESCATTR_P) == pSelReg->Attr.u,
7063	("%#x: %#x != %#x (sel=%#x base=%#llx limit=%#x)\n",
7064	idxSel, uAttr, pSelReg->Attr.u, pSelReg->Sel, pSelReg->u64Base, pSelReg->u32Limit));
7065	# endif
7066	VMMRZCallRing3Enable(pVCpu);
7067	NOREF(uAttr);
7068	#endif
7069	RT_NOREF2(pVCpu, idxSel);
7070	}
7071
7072
7073	/**
7074	* Imports a guest segment register from the current VMCS into the guest-CPU
7075	* context.
7076	*
7077	* @returns VBox status code.
7078	* @param pVCpu The cross context virtual CPU structure.
7079	* @param iSegReg The segment register number (X86_SREG_XXX).
7080	*
7081	* @remarks Called with interrupts and/or preemption disabled, try not to assert and
7082	* do not log!
7083	*/
7084	static int hmR0VmxImportGuestSegReg(PVMCPU pVCpu, uint8_t iSegReg)
7085	{
7086	Assert(iSegReg < X86_SREG_COUNT);
7087
7088	uint32_t const idxSel = g_aVmcsSegSel[iSegReg];
7089	uint32_t const idxLimit = g_aVmcsSegLimit[iSegReg];
7090	uint32_t const idxAttr = g_aVmcsSegAttr[iSegReg];
7091	#ifdef VMX_USE_CACHED_VMCS_ACCESSES
7092	uint32_t const idxBase = g_aVmcsCacheSegBase[iSegReg];
7093	#else
7094	uint32_t const idxBase = g_aVmcsSegBase[iSegReg];
7095	#endif
7096	uint64_t u64Base;
7097	uint32_t u32Sel, u32Limit, u32Attr;
7098	int rc = VMXReadVmcs32(idxSel, &u32Sel);
7099	rc \|= VMXReadVmcs32(idxLimit, &u32Limit);
7100	rc \|= VMXReadVmcs32(idxAttr, &u32Attr);
7101	rc \|= VMXReadVmcsGstNByIdxVal(idxBase, &u64Base);
7102	if (RT_SUCCESS(rc))
7103	{
7104	PCPUMSELREG pSelReg = &pVCpu->cpum.GstCtx.aSRegs[iSegReg];
7105	pSelReg->Sel = u32Sel;
7106	pSelReg->ValidSel = u32Sel;
7107	pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
7108	pSelReg->u32Limit = u32Limit;
7109	pSelReg->u64Base = u64Base;
7110	pSelReg->Attr.u = u32Attr;
7111	if (u32Attr & X86DESCATTR_UNUSABLE)
7112	hmR0VmxFixUnusableSegRegAttr(pVCpu, pSelReg, idxSel);
7113	}
7114	return rc;
7115	}
7116
7117
7118	/**
7119	* Imports the guest LDTR from the current VMCS into the guest-CPU context.
7120	*
7121	* @returns VBox status code.
7122	* @param pVCpu The cross context virtual CPU structure.
7123	*
7124	* @remarks Called with interrupts and/or preemption disabled, try not to assert and
7125	* do not log!
7126	*/
7127	static int hmR0VmxImportGuestLdtr(PVMCPU pVCpu)
7128	{
7129	uint64_t u64Base;
7130	uint32_t u32Sel, u32Limit, u32Attr;
7131	int rc = VMXReadVmcs32(VMX_VMCS16_GUEST_LDTR_SEL, &u32Sel);
7132	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, &u32Limit);
7133	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, &u32Attr);
7134	rc \|= VMXReadVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, &u64Base);
7135
7136	if (RT_SUCCESS(rc))
7137	{
7138	pVCpu->cpum.GstCtx.ldtr.Sel = u32Sel;
7139	pVCpu->cpum.GstCtx.ldtr.ValidSel = u32Sel;
7140	pVCpu->cpum.GstCtx.ldtr.fFlags = CPUMSELREG_FLAGS_VALID;
7141	pVCpu->cpum.GstCtx.ldtr.u32Limit = u32Limit;
7142	pVCpu->cpum.GstCtx.ldtr.u64Base = u64Base;
7143	pVCpu->cpum.GstCtx.ldtr.Attr.u = u32Attr;
7144	if (u32Attr & X86DESCATTR_UNUSABLE)
7145	hmR0VmxFixUnusableSegRegAttr(pVCpu, &pVCpu->cpum.GstCtx.ldtr, VMX_VMCS16_GUEST_LDTR_SEL);
7146	}
7147	return rc;
7148	}
7149
7150
7151	/**
7152	* Imports the guest TR from the current VMCS into the guest-CPU context.
7153	*
7154	* @returns VBox status code.
7155	* @param pVCpu The cross context virtual CPU structure.
7156	*
7157	* @remarks Called with interrupts and/or preemption disabled, try not to assert and
7158	* do not log!
7159	*/
7160	static int hmR0VmxImportGuestTr(PVMCPU pVCpu)
7161	{
7162	uint32_t u32Sel, u32Limit, u32Attr;
7163	uint64_t u64Base;
7164	int rc = VMXReadVmcs32(VMX_VMCS16_GUEST_TR_SEL, &u32Sel);
7165	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, &u32Limit);
7166	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, &u32Attr);
7167	rc \|= VMXReadVmcsGstN(VMX_VMCS_GUEST_TR_BASE, &u64Base);
7168	AssertRCReturn(rc, rc);
7169
7170	pVCpu->cpum.GstCtx.tr.Sel = u32Sel;
7171	pVCpu->cpum.GstCtx.tr.ValidSel = u32Sel;
7172	pVCpu->cpum.GstCtx.tr.fFlags = CPUMSELREG_FLAGS_VALID;
7173	pVCpu->cpum.GstCtx.tr.u32Limit = u32Limit;
7174	pVCpu->cpum.GstCtx.tr.u64Base = u64Base;
7175	pVCpu->cpum.GstCtx.tr.Attr.u = u32Attr;
7176	/* TR is the only selector that can never be unusable. */
7177	Assert(!(u32Attr & X86DESCATTR_UNUSABLE));
7178	return VINF_SUCCESS;
7179	}
7180
7181
7182	/**
7183	* Imports the guest RIP from the VMCS back into the guest-CPU context.
7184	*
7185	* @returns VBox status code.
7186	* @param pVCpu The cross context virtual CPU structure.
7187	*
7188	* @remarks Called with interrupts and/or preemption disabled, should not assert!
7189	* @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
7190	* instead!!!
7191	*/
7192	static int hmR0VmxImportGuestRip(PVMCPU pVCpu)
7193	{
7194	uint64_t u64Val;
7195	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7196	if (pCtx->fExtrn & CPUMCTX_EXTRN_RIP)
7197	{
7198	int rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val);
7199	if (RT_SUCCESS(rc))
7200	{
7201	pCtx->rip = u64Val;
7202	EMR0HistoryUpdatePC(pVCpu, pCtx->rip, false);
7203	pCtx->fExtrn &= ~CPUMCTX_EXTRN_RIP;
7204	}
7205	return rc;
7206	}
7207	return VINF_SUCCESS;
7208	}
7209
7210
7211	/**
7212	* Imports the guest RFLAGS from the VMCS back into the guest-CPU context.
7213	*
7214	* @returns VBox status code.
7215	* @param pVCpu The cross context virtual CPU structure.
7216	* @param pVmcsInfo The VMCS info. object.
7217	*
7218	* @remarks Called with interrupts and/or preemption disabled, should not assert!
7219	* @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
7220	* instead!!!
7221	*/
7222	static int hmR0VmxImportGuestRFlags(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
7223	{
7224	uint32_t u32Val;
7225	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7226	if (pCtx->fExtrn & CPUMCTX_EXTRN_RFLAGS)
7227	{
7228	int rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val);
7229	if (RT_SUCCESS(rc))
7230	{
7231	pCtx->eflags.u32 = u32Val;
7232
7233	/* Restore eflags for real-on-v86-mode hack. */
7234	if (pVmcsInfo->RealMode.fRealOnV86Active)
7235	{
7236	pCtx->eflags.Bits.u1VM = 0;
7237	pCtx->eflags.Bits.u2IOPL = pVmcsInfo->RealMode.Eflags.Bits.u2IOPL;
7238	}
7239	}
7240	pCtx->fExtrn &= ~CPUMCTX_EXTRN_RFLAGS;
7241	return rc;
7242	}
7243	return VINF_SUCCESS;
7244	}
7245
7246
7247	/**
7248	* Imports the guest interruptibility-state from the VMCS back into the guest-CPU
7249	* context.
7250	*
7251	* @returns VBox status code.
7252	* @param pVCpu The cross context virtual CPU structure.
7253	* @param pVmcsInfo The VMCS info. object.
7254	*
7255	* @remarks Called with interrupts and/or preemption disabled, try not to assert and
7256	* do not log!
7257	* @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
7258	* instead!!!
7259	*/
7260	static int hmR0VmxImportGuestIntrState(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
7261	{
7262	uint32_t u32Val;
7263	int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &u32Val);
7264	if (RT_SUCCESS(rc))
7265	{
7266	if (!u32Val)
7267	{
7268	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
7269	{
7270	rc = hmR0VmxImportGuestRip(pVCpu);
7271	rc \|= hmR0VmxImportGuestRFlags(pVCpu, pVmcsInfo);
7272	AssertRCReturn(rc, rc);
7273	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
7274	}
7275
7276	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_BLOCK_NMIS))
7277	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
7278	}
7279	else
7280	{
7281	/*
7282	* We must import RIP here to set our EM interrupt-inhibited state.
7283	* We also import RFLAGS as our code that evaluates pending interrupts
7284	* before VM-entry requires it.
7285	*/
7286	rc = hmR0VmxImportGuestRip(pVCpu);
7287	rc \|= hmR0VmxImportGuestRFlags(pVCpu, pVmcsInfo);
7288	if (RT_SUCCESS(rc))
7289	{
7290	if (u32Val & ( VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS
7291	\| VMX_VMCS_GUEST_INT_STATE_BLOCK_STI))
7292	{
7293	EMSetInhibitInterruptsPC(pVCpu, pVCpu->cpum.GstCtx.rip);
7294	}
7295	else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
7296	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
7297
7298	if (u32Val & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI)
7299	{
7300	if (!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_BLOCK_NMIS))
7301	VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
7302	}
7303	else if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_BLOCK_NMIS))
7304	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
7305	}
7306	}
7307	}
7308	return rc;
7309	}
7310
7311
7312	/**
7313	* Worker for VMXR0ImportStateOnDemand.
7314	*
7315	* @returns VBox status code.
7316	* @param pVCpu The cross context virtual CPU structure.
7317	* @param pVmcsInfo The VMCS info. object.
7318	* @param fWhat What to import, CPUMCTX_EXTRN_XXX.
7319	*/
7320	static int hmR0VmxImportGuestState(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo, uint64_t fWhat)
7321	{
7322	#define VMXLOCAL_BREAK_RC(a_rc) \
7323	if (RT_SUCCESS(a_rc)) \
7324	{ } \
7325	else \
7326	break
7327
7328	int rc = VINF_SUCCESS;
7329	PVM pVM = pVCpu->CTX_SUFF(pVM);
7330	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7331	uint64_t u64Val;
7332	uint32_t u32Val;
7333
7334	STAM_PROFILE_ADV_START(& pVCpu->hm.s.StatImportGuestState, x);
7335
7336	/*
7337	* We disable interrupts to make the updating of the state and in particular
7338	* the fExtrn modification atomic wrt to preemption hooks.
7339	*/
7340	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
7341
7342	fWhat &= pCtx->fExtrn;
7343	if (fWhat)
7344	{
7345	do
7346	{
7347	if (fWhat & CPUMCTX_EXTRN_RIP)
7348	{
7349	rc = hmR0VmxImportGuestRip(pVCpu);
7350	VMXLOCAL_BREAK_RC(rc);
7351	}
7352
7353	if (fWhat & CPUMCTX_EXTRN_RFLAGS)
7354	{
7355	rc = hmR0VmxImportGuestRFlags(pVCpu, pVmcsInfo);
7356	VMXLOCAL_BREAK_RC(rc);
7357	}
7358
7359	if (fWhat & CPUMCTX_EXTRN_HM_VMX_INT_STATE)
7360	{
7361	rc = hmR0VmxImportGuestIntrState(pVCpu, pVmcsInfo);
7362	VMXLOCAL_BREAK_RC(rc);
7363	}
7364
7365	if (fWhat & CPUMCTX_EXTRN_RSP)
7366	{
7367	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val);
7368	VMXLOCAL_BREAK_RC(rc);
7369	pCtx->rsp = u64Val;
7370	}
7371
7372	if (fWhat & CPUMCTX_EXTRN_SREG_MASK)
7373	{
7374	bool const fRealOnV86Active = pVmcsInfo->RealMode.fRealOnV86Active;
7375	if (fWhat & CPUMCTX_EXTRN_CS)
7376	{
7377	rc \|= hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_CS);
7378	rc \|= hmR0VmxImportGuestRip(pVCpu);
7379	if (fRealOnV86Active)
7380	pCtx->cs.Attr.u = pVmcsInfo->RealMode.AttrCS.u;
7381	EMR0HistoryUpdatePC(pVCpu, pCtx->cs.u64Base + pCtx->rip, true /* fFlattened */);
7382	}
7383	if (fWhat & CPUMCTX_EXTRN_SS)
7384	{
7385	rc \|= hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_SS);
7386	if (fRealOnV86Active)
7387	pCtx->ss.Attr.u = pVmcsInfo->RealMode.AttrSS.u;
7388	}
7389	if (fWhat & CPUMCTX_EXTRN_DS)
7390	{
7391	rc \|= hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_DS);
7392	if (fRealOnV86Active)
7393	pCtx->ds.Attr.u = pVmcsInfo->RealMode.AttrDS.u;
7394	}
7395	if (fWhat & CPUMCTX_EXTRN_ES)
7396	{
7397	rc \|= hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_ES);
7398	if (fRealOnV86Active)
7399	pCtx->es.Attr.u = pVmcsInfo->RealMode.AttrES.u;
7400	}
7401	if (fWhat & CPUMCTX_EXTRN_FS)
7402	{
7403	rc \|= hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_FS);
7404	if (fRealOnV86Active)
7405	pCtx->fs.Attr.u = pVmcsInfo->RealMode.AttrFS.u;
7406	}
7407	if (fWhat & CPUMCTX_EXTRN_GS)
7408	{
7409	rc \|= hmR0VmxImportGuestSegReg(pVCpu, X86_SREG_GS);
7410	if (fRealOnV86Active)
7411	pCtx->gs.Attr.u = pVmcsInfo->RealMode.AttrGS.u;
7412	}
7413	VMXLOCAL_BREAK_RC(rc);
7414	}
7415
7416	if (fWhat & CPUMCTX_EXTRN_TABLE_MASK)
7417	{
7418	if (fWhat & CPUMCTX_EXTRN_LDTR)
7419	rc \|= hmR0VmxImportGuestLdtr(pVCpu);
7420
7421	if (fWhat & CPUMCTX_EXTRN_GDTR)
7422	{
7423	rc \|= VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
7424	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
7425	pCtx->gdtr.pGdt = u64Val;
7426	pCtx->gdtr.cbGdt = u32Val;
7427	}
7428
7429	/* Guest IDTR. */
7430	if (fWhat & CPUMCTX_EXTRN_IDTR)
7431	{
7432	rc \|= VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
7433	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
7434	pCtx->idtr.pIdt = u64Val;
7435	pCtx->idtr.cbIdt = u32Val;
7436	}
7437
7438	/* Guest TR. */
7439	if (fWhat & CPUMCTX_EXTRN_TR)
7440	{
7441	/* Real-mode emulation using virtual-8086 mode has the fake TSS (pRealModeTSS) in TR,
7442	don't need to import that one. */
7443	if (!pVmcsInfo->RealMode.fRealOnV86Active)
7444	rc \|= hmR0VmxImportGuestTr(pVCpu);
7445	}
7446	VMXLOCAL_BREAK_RC(rc);
7447	}
7448
7449	if (fWhat & CPUMCTX_EXTRN_DR7)
7450	{
7451	if (!pVCpu->hm.s.fUsingHyperDR7)
7452	{
7453	/* Upper 32-bits are always zero. See Intel spec. 2.7.3 "Loading and Storing Debug Registers". */
7454	rc = VMXReadVmcs32(VMX_VMCS_GUEST_DR7, &u32Val);
7455	VMXLOCAL_BREAK_RC(rc);
7456	pCtx->dr[7] = u32Val;
7457	}
7458	}
7459
7460	if (fWhat & CPUMCTX_EXTRN_SYSENTER_MSRS)
7461	{
7462	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &pCtx->SysEnter.eip);
7463	rc \|= VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &pCtx->SysEnter.esp);
7464	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val);
7465	pCtx->SysEnter.cs = u32Val;
7466	VMXLOCAL_BREAK_RC(rc);
7467	}
7468
7469	#if HC_ARCH_BITS == 64
7470	if (fWhat & CPUMCTX_EXTRN_KERNEL_GS_BASE)
7471	{
7472	if ( pVM->hm.s.fAllow64BitGuests
7473	&& (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
7474	pCtx->msrKERNELGSBASE = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
7475	}
7476
7477	if (fWhat & CPUMCTX_EXTRN_SYSCALL_MSRS)
7478	{
7479	if ( pVM->hm.s.fAllow64BitGuests
7480	&& (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
7481	{
7482	pCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR);
7483	pCtx->msrSTAR = ASMRdMsr(MSR_K6_STAR);
7484	pCtx->msrSFMASK = ASMRdMsr(MSR_K8_SF_MASK);
7485	}
7486	}
7487	#endif
7488
7489	if ( (fWhat & (CPUMCTX_EXTRN_TSC_AUX \| CPUMCTX_EXTRN_OTHER_MSRS))
7490	#if HC_ARCH_BITS == 32
7491	\|\| (fWhat & (CPUMCTX_EXTRN_KERNEL_GS_BASE \| CPUMCTX_EXTRN_SYSCALL_MSRS))
7492	#endif
7493	)
7494	{
7495	PCVMXAUTOMSR pMsr = (PCVMXAUTOMSR)pVmcsInfo->pvGuestMsrStore;
7496	uint32_t const cMsrs = pVmcsInfo->cExitMsrStore;
7497	Assert(cMsrs == 0 \|\| pMsr != NULL);
7498	for (uint32_t i = 0; i < cMsrs; i++, pMsr++)
7499	{
7500	switch (pMsr->u32Msr)
7501	{
7502	#if HC_ARCH_BITS == 32
7503	case MSR_K8_LSTAR: pCtx->msrLSTAR = pMsr->u64Value; break;
7504	case MSR_K6_STAR: pCtx->msrSTAR = pMsr->u64Value; break;
7505	case MSR_K8_SF_MASK: pCtx->msrSFMASK = pMsr->u64Value; break;
7506	case MSR_K8_KERNEL_GS_BASE: pCtx->msrKERNELGSBASE = pMsr->u64Value; break;
7507	#endif
7508	case MSR_IA32_SPEC_CTRL: CPUMSetGuestSpecCtrl(pVCpu, pMsr->u64Value); break;
7509	case MSR_K8_TSC_AUX: CPUMSetGuestTscAux(pVCpu, pMsr->u64Value); break;
7510	case MSR_K6_EFER: /* Can't be changed without causing a VM-exit */ break;
7511
7512	default:
7513	{
7514	pVCpu->hm.s.u32HMError = pMsr->u32Msr;
7515	ASMSetFlags(fEFlags);
7516	AssertMsgFailed(("Unexpected MSR in auto-load/store area. idMsr=%#RX32 cMsrs=%u\n", pMsr->u32Msr,
7517	cMsrs));
7518	return VERR_HM_UNEXPECTED_LD_ST_MSR;
7519	}
7520	}
7521	}
7522	}
7523
7524	if (fWhat & CPUMCTX_EXTRN_CR_MASK)
7525	{
7526	uint64_t u64Shadow;
7527	if (fWhat & CPUMCTX_EXTRN_CR0)
7528	{
7529	/** @todo r=ramshankar: We only read 32-bits here for legacy/convenience reasons,
7530	* remove when we drop 32-bit host w/ 64-bit host support, see
7531	* @bugref{9180#c39}. */
7532	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val);
7533	#if HC_ARCH_BITS == 32
7534	uint32_t u32Shadow;
7535	rc \|= VMXReadVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, &u32Shadow);
7536	u64Shadow = u32Shadow;
7537	#else
7538	rc \|= VMXReadVmcs64(VMX_VMCS_CTRL_CR0_READ_SHADOW, &u64Shadow);
7539	#endif
7540	VMXLOCAL_BREAK_RC(rc);
7541	u64Val = u32Val;
7542	u64Val = (u64Val & ~pVmcsInfo->u64Cr0Mask)
7543	\| (u64Shadow & pVmcsInfo->u64Cr0Mask);
7544	VMMRZCallRing3Disable(pVCpu); /* May call into PGM which has Log statements. */
7545	CPUMSetGuestCR0(pVCpu, u64Val);
7546	VMMRZCallRing3Enable(pVCpu);
7547	}
7548
7549	if (fWhat & CPUMCTX_EXTRN_CR4)
7550	{
7551	/** @todo r=ramshankar: We only read 32-bits here for legacy/convenience reasons,
7552	* remove when we drop 32-bit host w/ 64-bit host support, see
7553	* @bugref{9180#c39}. */
7554	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32Val);
7555	#if HC_ARCH_BITS == 32
7556	uint32_t u32Shadow;
7557	rc \|= VMXReadVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, &u32Shadow);
7558	u64Shadow = u32Shadow;
7559	#else
7560	rc \|= VMXReadVmcs64(VMX_VMCS_CTRL_CR4_READ_SHADOW, &u64Shadow);
7561	#endif
7562	VMXLOCAL_BREAK_RC(rc);
7563	u64Val = u32Val;
7564	u64Val = (u64Val & ~pVmcsInfo->u64Cr4Mask)
7565	\| (u64Shadow & pVmcsInfo->u64Cr4Mask);
7566	pCtx->cr4 = u64Val;
7567	}
7568
7569	if (fWhat & CPUMCTX_EXTRN_CR3)
7570	{
7571	/* CR0.PG bit changes are always intercepted, so it's up to date. */
7572	if ( pVM->hm.s.vmx.fUnrestrictedGuest
7573	\|\| ( pVM->hm.s.fNestedPaging
7574	&& CPUMIsGuestPagingEnabledEx(pCtx)))
7575	{
7576	rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &u64Val);
7577	if (pCtx->cr3 != u64Val)
7578	{
7579	pCtx->cr3 = u64Val;
7580	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
7581	}
7582
7583	/* If the guest is in PAE mode, sync back the PDPE's into the guest state.
7584	Note: CR4.PAE, CR0.PG, EFER MSR changes are always intercepted, so they're up to date. */
7585	if (CPUMIsGuestInPAEModeEx(pCtx))
7586	{
7587	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u);
7588	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u);
7589	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u);
7590	rc \|= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u);
7591	VMXLOCAL_BREAK_RC(rc);
7592	VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
7593	}
7594	}
7595	}
7596	}
7597	} while (0);
7598
7599	if (RT_SUCCESS(rc))
7600	{
7601	/* Update fExtrn. */
7602	pCtx->fExtrn &= ~fWhat;
7603
7604	/* If everything has been imported, clear the HM keeper bit. */
7605	if (!(pCtx->fExtrn & HMVMX_CPUMCTX_EXTRN_ALL))
7606	{
7607	pCtx->fExtrn &= ~CPUMCTX_EXTRN_KEEPER_HM;
7608	Assert(!pCtx->fExtrn);
7609	}
7610	}
7611	}
7612	else
7613	AssertMsg(!pCtx->fExtrn \|\| (pCtx->fExtrn & HMVMX_CPUMCTX_EXTRN_ALL), ("%#RX64\n", pCtx->fExtrn));
7614
7615	ASMSetFlags(fEFlags);
7616
7617	STAM_PROFILE_ADV_STOP(& pVCpu->hm.s.StatImportGuestState, x);
7618
7619	if (RT_SUCCESS(rc))
7620	{ /* likely */ }
7621	else
7622	return rc;
7623
7624	/*
7625	* Honor any pending CR3 updates.
7626	*
7627	* Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> hmR0VmxCallRing3Callback()
7628	* -> VMMRZCallRing3Disable() -> hmR0VmxImportGuestState() -> Sets VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
7629	* -> continue with VM-exit handling -> hmR0VmxImportGuestState() and here we are.
7630	*
7631	* The reason for such complicated handling is because VM-exits that call into PGM expect CR3 to be up-to-date and thus
7632	* if any CR3-saves -before- the VM-exit (longjmp) postponed the CR3 update via the force-flag, any VM-exit handler that
7633	* calls into PGM when it re-saves CR3 will end up here and we call PGMUpdateCR3(). This is why the code below should
7634	* -NOT- check if CPUMCTX_EXTRN_CR3 is set!
7635	*
7636	* The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
7637	*/
7638	if (VMMRZCallRing3IsEnabled(pVCpu))
7639	{
7640	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
7641	{
7642	Assert(!(ASMAtomicUoReadU64(&pCtx->fExtrn) & CPUMCTX_EXTRN_CR3));
7643	PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
7644	}
7645
7646	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
7647	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
7648
7649	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
7650	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
7651	}
7652
7653	return VINF_SUCCESS;
7654	#undef VMXLOCAL_BREAK_RC
7655	}
7656
7657
7658	/**
7659	* Saves the guest state from the VMCS into the guest-CPU context.
7660	*
7661	* @returns VBox status code.
7662	* @param pVCpu The cross context virtual CPU structure.
7663	* @param fWhat What to import, CPUMCTX_EXTRN_XXX.
7664	*/
7665	VMMR0DECL(int) VMXR0ImportStateOnDemand(PVMCPU pVCpu, uint64_t fWhat)
7666	{
7667	PCVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
7668	return hmR0VmxImportGuestState(pVCpu, pVmcsInfo, fWhat);
7669	}
7670
7671
7672	/**
7673	* Check per-VM and per-VCPU force flag actions that require us to go back to
7674	* ring-3 for one reason or another.
7675	*
7676	* @returns Strict VBox status code (i.e. informational status codes too)
7677	* @retval VINF_SUCCESS if we don't have any actions that require going back to
7678	* ring-3.
7679	* @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
7680	* @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
7681	* interrupts)
7682	* @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
7683	* all EMTs to be in ring-3.
7684	* @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
7685	* @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
7686	* to the EM loop.
7687	*
7688	* @param pVCpu The cross context virtual CPU structure.
7689	* @param fStepping Whether we are single-stepping the guest using the
7690	* hypervisor debugger.
7691	*/
7692	static VBOXSTRICTRC hmR0VmxCheckForceFlags(PVMCPU pVCpu, bool fStepping)
7693	{
7694	Assert(VMMRZCallRing3IsEnabled(pVCpu));
7695
7696	/*
7697	* Update pending interrupts into the APIC's IRR.
7698	*/
7699	if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_UPDATE_APIC))
7700	APICUpdatePendingInterrupts(pVCpu);
7701
7702	/*
7703	* Anything pending? Should be more likely than not if we're doing a good job.
7704	*/
7705	PVM pVM = pVCpu->CTX_SUFF(pVM);
7706	if ( !fStepping
7707	? !VM_FF_IS_ANY_SET(pVM, VM_FF_HP_R0_PRE_HM_MASK)
7708	&& !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HP_R0_PRE_HM_MASK)
7709	: !VM_FF_IS_ANY_SET(pVM, VM_FF_HP_R0_PRE_HM_STEP_MASK)
7710	&& !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
7711	return VINF_SUCCESS;
7712
7713	/* Pending PGM C3 sync. */
7714	if (VMCPU_FF_IS_ANY_SET(pVCpu,VMCPU_FF_PGM_SYNC_CR3 \| VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
7715	{
7716	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7717	Assert(!(ASMAtomicUoReadU64(&pCtx->fExtrn) & (CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_CR3 \| CPUMCTX_EXTRN_CR4)));
7718	VBOXSTRICTRC rcStrict2 = PGMSyncCR3(pVCpu, pCtx->cr0, pCtx->cr3, pCtx->cr4,
7719	VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
7720	if (rcStrict2 != VINF_SUCCESS)
7721	{
7722	AssertRC(VBOXSTRICTRC_VAL(rcStrict2));
7723	Log4Func(("PGMSyncCR3 forcing us back to ring-3. rc2=%d\n", VBOXSTRICTRC_VAL(rcStrict2)));
7724	return rcStrict2;
7725	}
7726	}
7727
7728	/* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
7729	if ( VM_FF_IS_ANY_SET(pVM, VM_FF_HM_TO_R3_MASK)
7730	\|\| VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
7731	{
7732	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
7733	int rc2 = RT_LIKELY(!VM_FF_IS_SET(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_RAW_TO_R3 : VINF_EM_NO_MEMORY;
7734	Log4Func(("HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc2));
7735	return rc2;
7736	}
7737
7738	/* Pending VM request packets, such as hardware interrupts. */
7739	if ( VM_FF_IS_SET(pVM, VM_FF_REQUEST)
7740	\|\| VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_REQUEST))
7741	{
7742	Log4Func(("Pending VM request forcing us back to ring-3\n"));
7743	return VINF_EM_PENDING_REQUEST;
7744	}
7745
7746	/* Pending PGM pool flushes. */
7747	if (VM_FF_IS_SET(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
7748	{
7749	Log4Func(("PGM pool flush pending forcing us back to ring-3\n"));
7750	return VINF_PGM_POOL_FLUSH_PENDING;
7751	}
7752
7753	/* Pending DMA requests. */
7754	if (VM_FF_IS_SET(pVM, VM_FF_PDM_DMA))
7755	{
7756	Log4Func(("Pending DMA request forcing us back to ring-3\n"));
7757	return VINF_EM_RAW_TO_R3;
7758	}
7759
7760	return VINF_SUCCESS;
7761	}
7762
7763
7764	/**
7765	* Converts any TRPM trap into a pending HM event. This is typically used when
7766	* entering from ring-3 (not longjmp returns).
7767	*
7768	* @param pVCpu The cross context virtual CPU structure.
7769	*/
7770	static void hmR0VmxTrpmTrapToPendingEvent(PVMCPU pVCpu)
7771	{
7772	Assert(TRPMHasTrap(pVCpu));
7773	Assert(!pVCpu->hm.s.Event.fPending);
7774
7775	uint8_t uVector;
7776	TRPMEVENT enmTrpmEvent;
7777	RTGCUINT uErrCode;
7778	RTGCUINTPTR GCPtrFaultAddress;
7779	uint8_t cbInstr;
7780
7781	int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
7782	AssertRC(rc);
7783
7784	/* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntInfo. */
7785	uint32_t u32IntInfo = uVector \| VMX_EXIT_INT_INFO_VALID;
7786	if (enmTrpmEvent == TRPM_TRAP)
7787	{
7788	/** @todo r=ramshankar: TRPM currently offers no way to determine a \#DB that was
7789	* generated using INT1 (ICEBP). */
7790	switch (uVector)
7791	{
7792	case X86_XCPT_NMI:
7793	u32IntInfo \|= (VMX_EXIT_INT_INFO_TYPE_NMI << VMX_EXIT_INT_INFO_TYPE_SHIFT);
7794	break;
7795
7796	case X86_XCPT_BP:
7797	case X86_XCPT_OF:
7798	u32IntInfo \|= (VMX_EXIT_INT_INFO_TYPE_SW_XCPT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
7799	break;
7800
7801	case X86_XCPT_PF:
7802	case X86_XCPT_DF:
7803	case X86_XCPT_TS:
7804	case X86_XCPT_NP:
7805	case X86_XCPT_SS:
7806	case X86_XCPT_GP:
7807	case X86_XCPT_AC:
7808	u32IntInfo \|= VMX_EXIT_INT_INFO_ERROR_CODE_VALID;
7809	RT_FALL_THRU();
7810	default:
7811	u32IntInfo \|= (VMX_EXIT_INT_INFO_TYPE_HW_XCPT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
7812	break;
7813	}
7814	}
7815	else if (enmTrpmEvent == TRPM_HARDWARE_INT)
7816	u32IntInfo \|= (VMX_EXIT_INT_INFO_TYPE_EXT_INT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
7817	else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
7818	{
7819	switch (uVector)
7820	{
7821	case X86_XCPT_BP:
7822	case X86_XCPT_OF:
7823	u32IntInfo \|= (VMX_EXIT_INT_INFO_TYPE_SW_XCPT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
7824	break;
7825
7826	default:
7827	Assert(uVector == X86_XCPT_DB);
7828	u32IntInfo \|= (VMX_EXIT_INT_INFO_TYPE_SW_INT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
7829	break;
7830	}
7831	}
7832	else
7833	AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
7834
7835	rc = TRPMResetTrap(pVCpu);
7836	AssertRC(rc);
7837	Log4(("TRPM->HM event: u32IntInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
7838	u32IntInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
7839
7840	hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, uErrCode, GCPtrFaultAddress);
7841	}
7842
7843
7844	/**
7845	* Converts the pending HM event into a TRPM trap.
7846	*
7847	* @param pVCpu The cross context virtual CPU structure.
7848	*/
7849	static void hmR0VmxPendingEventToTrpmTrap(PVMCPU pVCpu)
7850	{
7851	Assert(pVCpu->hm.s.Event.fPending);
7852
7853	uint32_t uVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
7854	uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVCpu->hm.s.Event.u64IntInfo);
7855	bool fErrorCodeValid = VMX_IDT_VECTORING_INFO_IS_ERROR_CODE_VALID(pVCpu->hm.s.Event.u64IntInfo);
7856	uint32_t uErrorCode = pVCpu->hm.s.Event.u32ErrCode;
7857
7858	/* If a trap was already pending, we did something wrong! */
7859	Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo /, NULL / pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
7860
7861	/** @todo Use HMVmxEventToTrpmEventType() later. */
7862	TRPMEVENT enmTrapType;
7863	switch (uVectorType)
7864	{
7865	case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
7866	enmTrapType = TRPM_HARDWARE_INT;
7867	break;
7868
7869	case VMX_IDT_VECTORING_INFO_TYPE_NMI:
7870	case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
7871	enmTrapType = TRPM_TRAP;
7872	break;
7873
7874	case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT: /* #DB (INT1/ICEBP). */
7875	Assert(uVector == X86_XCPT_DB);
7876	enmTrapType = TRPM_SOFTWARE_INT;
7877	break;
7878
7879	case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: /* #BP (INT3) and #OF (INTO) */
7880	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF);
7881	enmTrapType = TRPM_SOFTWARE_INT;
7882	break;
7883
7884	case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
7885	enmTrapType = TRPM_SOFTWARE_INT;
7886	break;
7887
7888	default:
7889	AssertMsgFailed(("Invalid trap type %#x\n", uVectorType));
7890	enmTrapType = TRPM_32BIT_HACK;
7891	break;
7892	}
7893
7894	Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
7895
7896	int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
7897	AssertRC(rc);
7898
7899	if (fErrorCodeValid)
7900	TRPMSetErrorCode(pVCpu, uErrorCode);
7901
7902	if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
7903	&& uVector == X86_XCPT_PF)
7904	TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
7905	else if (enmTrapType == TRPM_SOFTWARE_INT)
7906	TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
7907
7908	/* We're now done converting the pending event. */
7909	pVCpu->hm.s.Event.fPending = false;
7910	}
7911
7912
7913	/**
7914	* Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
7915	* cause a VM-exit as soon as the guest is in a state to receive interrupts.
7916	*
7917	* @param pVCpu The cross context virtual CPU structure.
7918	* @param pVmcsInfo The VMCS info. object.
7919	*/
7920	static void hmR0VmxSetIntWindowExitVmcs(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
7921	{
7922	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_INT_WINDOW_EXIT)
7923	{
7924	if (!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_INT_WINDOW_EXIT))
7925	{
7926	pVmcsInfo->u32ProcCtls \|= VMX_PROC_CTLS_INT_WINDOW_EXIT;
7927	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
7928	AssertRC(rc);
7929	}
7930	} /* else we will deliver interrupts whenever the guest Vm-exits next and is in a state to receive the interrupt. */
7931	}
7932
7933
7934	/**
7935	* Clears the interrupt-window exiting control in the VMCS.
7936	*
7937	* @param pVmcsInfo The VMCS info. object.
7938	*/
7939	DECLINLINE(int) hmR0VmxClearIntWindowExitVmcs(PVMXVMCSINFO pVmcsInfo)
7940	{
7941	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_INT_WINDOW_EXIT)
7942	{
7943	pVmcsInfo->u32ProcCtls &= ~VMX_PROC_CTLS_INT_WINDOW_EXIT;
7944	return VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
7945	}
7946	return VINF_SUCCESS;
7947	}
7948
7949
7950	/**
7951	* Sets the NMI-window exiting control in the VMCS which instructs VT-x to
7952	* cause a VM-exit as soon as the guest is in a state to receive NMIs.
7953	*
7954	* @param pVCpu The cross context virtual CPU structure.
7955	* @param pVmcsInfo The VMCS info. object.
7956	*/
7957	static void hmR0VmxSetNmiWindowExitVmcs(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfo)
7958	{
7959	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_NMI_WINDOW_EXIT)
7960	{
7961	if (!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT))
7962	{
7963	pVmcsInfo->u32ProcCtls \|= VMX_PROC_CTLS_NMI_WINDOW_EXIT;
7964	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
7965	AssertRC(rc);
7966	Log4Func(("Setup NMI-window exiting\n"));
7967	}
7968	} /* else we will deliver NMIs whenever we VM-exit next, even possibly nesting NMIs. Can't be helped on ancient CPUs. */
7969	}
7970
7971
7972	/**
7973	* Clears the NMI-window exiting control in the VMCS.
7974	*
7975	* @param pVmcsInfo The VMCS info. object.
7976	*/
7977	DECLINLINE(int) hmR0VmxClearNmiWindowExitVmcs(PVMXVMCSINFO pVmcsInfo)
7978	{
7979	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT)
7980	{
7981	pVmcsInfo->u32ProcCtls &= ~VMX_PROC_CTLS_NMI_WINDOW_EXIT;
7982	return VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
7983	}
7984	return VINF_SUCCESS;
7985	}
7986
7987
7988	/**
7989	* Does the necessary state syncing before returning to ring-3 for any reason
7990	* (longjmp, preemption, voluntary exits to ring-3) from VT-x.
7991	*
7992	* @returns VBox status code.
7993	* @param pVCpu The cross context virtual CPU structure.
7994	* @param fImportState Whether to import the guest state from the VMCS back
7995	* to the guest-CPU context.
7996	*
7997	* @remarks No-long-jmp zone!!!
7998	*/
7999	static int hmR0VmxLeave(PVMCPU pVCpu, bool fImportState)
8000	{
8001	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8002	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8003
8004	RTCPUID idCpu = RTMpCpuId();
8005	Log4Func(("HostCpuId=%u\n", idCpu));
8006
8007	/*
8008	* !!! IMPORTANT !!!
8009	* If you modify code here, check whether hmR0VmxCallRing3Callback() needs to be updated too.
8010	*/
8011
8012	/* Save the guest state if necessary. */
8013	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
8014	if (fImportState)
8015	{
8016	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
8017	AssertRCReturn(rc, rc);
8018	}
8019
8020	/* Restore host FPU state if necessary. We will resync on next R0 reentry. */
8021	CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
8022	Assert(!CPUMIsGuestFPUStateActive(pVCpu));
8023
8024	/* Restore host debug registers if necessary. We will resync on next R0 reentry. */
8025	#ifdef VBOX_STRICT
8026	if (CPUMIsHyperDebugStateActive(pVCpu))
8027	Assert(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_MOV_DR_EXIT);
8028	#endif
8029	CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
8030	Assert(!CPUMIsGuestDebugStateActive(pVCpu) && !CPUMIsGuestDebugStateActivePending(pVCpu));
8031	Assert(!CPUMIsHyperDebugStateActive(pVCpu) && !CPUMIsHyperDebugStateActivePending(pVCpu));
8032
8033	#if HC_ARCH_BITS == 64
8034	/* Restore host-state bits that VT-x only restores partially. */
8035	if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
8036	&& (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
8037	{
8038	Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags, idCpu));
8039	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
8040	}
8041	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
8042	#endif
8043
8044	/* Restore the lazy host MSRs as we're leaving VT-x context. */
8045	if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
8046	{
8047	/* We shouldn't restore the host MSRs without saving the guest MSRs first. */
8048	if (!fImportState)
8049	{
8050	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_KERNEL_GS_BASE \| CPUMCTX_EXTRN_SYSCALL_MSRS);
8051	AssertRCReturn(rc, rc);
8052	}
8053	hmR0VmxLazyRestoreHostMsrs(pVCpu);
8054	Assert(!pVCpu->hm.s.vmx.fLazyMsrs);
8055	}
8056	else
8057	pVCpu->hm.s.vmx.fLazyMsrs = 0;
8058
8059	/* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
8060	pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs = false;
8061
8062	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
8063	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatImportGuestState);
8064	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExportGuestState);
8065	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatPreExit);
8066	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitHandling);
8067	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitIO);
8068	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitMovCRx);
8069	STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitXcptNmi);
8070	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
8071
8072	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
8073
8074	/** @todo This partially defeats the purpose of having preemption hooks.
8075	* The problem is, deregistering the hooks should be moved to a place that
8076	* lasts until the EMT is about to be destroyed not everytime while leaving HM
8077	* context.
8078	*/
8079	int rc = hmR0VmxClearVmcs(pVmcsInfo);
8080	AssertRCReturn(rc, rc);
8081
8082	Log4Func(("Cleared Vmcs. HostCpuId=%u\n", idCpu));
8083	NOREF(idCpu);
8084	return VINF_SUCCESS;
8085	}
8086
8087
8088	/**
8089	* Leaves the VT-x session.
8090	*
8091	* @returns VBox status code.
8092	* @param pVCpu The cross context virtual CPU structure.
8093	*
8094	* @remarks No-long-jmp zone!!!
8095	*/
8096	static int hmR0VmxLeaveSession(PVMCPU pVCpu)
8097	{
8098	HM_DISABLE_PREEMPT(pVCpu);
8099	HMVMX_ASSERT_CPU_SAFE(pVCpu);
8100	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8101	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8102
8103	/* When thread-context hooks are used, we can avoid doing the leave again if we had been preempted before
8104	and done this from the VMXR0ThreadCtxCallback(). */
8105	if (!pVCpu->hm.s.fLeaveDone)
8106	{
8107	int rc2 = hmR0VmxLeave(pVCpu, true /* fImportState */);
8108	AssertRCReturnStmt(rc2, HM_RESTORE_PREEMPT(), rc2);
8109	pVCpu->hm.s.fLeaveDone = true;
8110	}
8111	Assert(!pVCpu->cpum.GstCtx.fExtrn);
8112
8113	/*
8114	* !!! IMPORTANT !!!
8115	* If you modify code here, make sure to check whether hmR0VmxCallRing3Callback() needs to be updated too.
8116	*/
8117
8118	/* Deregister hook now that we've left HM context before re-enabling preemption. */
8119	/** @todo Deregistering here means we need to VMCLEAR always
8120	* (longjmp/exit-to-r3) in VT-x which is not efficient, eliminate need
8121	* for calling VMMR0ThreadCtxHookDisable here! */
8122	VMMR0ThreadCtxHookDisable(pVCpu);
8123
8124	/* Leave HM context. This takes care of local init (term). */
8125	int rc = HMR0LeaveCpu(pVCpu);
8126
8127	HM_RESTORE_PREEMPT();
8128	return rc;
8129	}
8130
8131
8132	/**
8133	* Does the necessary state syncing before doing a longjmp to ring-3.
8134	*
8135	* @returns VBox status code.
8136	* @param pVCpu The cross context virtual CPU structure.
8137	*
8138	* @remarks No-long-jmp zone!!!
8139	*/
8140	DECLINLINE(int) hmR0VmxLongJmpToRing3(PVMCPU pVCpu)
8141	{
8142	return hmR0VmxLeaveSession(pVCpu);
8143	}
8144
8145
8146	/**
8147	* Take necessary actions before going back to ring-3.
8148	*
8149	* An action requires us to go back to ring-3. This function does the necessary
8150	* steps before we can safely return to ring-3. This is not the same as longjmps
8151	* to ring-3, this is voluntary and prepares the guest so it may continue
8152	* executing outside HM (recompiler/IEM).
8153	*
8154	* @returns VBox status code.
8155	* @param pVCpu The cross context virtual CPU structure.
8156	* @param rcExit The reason for exiting to ring-3. Can be
8157	* VINF_VMM_UNKNOWN_RING3_CALL.
8158	*/
8159	static int hmR0VmxExitToRing3(PVMCPU pVCpu, VBOXSTRICTRC rcExit)
8160	{
8161	Assert(pVCpu);
8162	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
8163
8164	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
8165	if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
8166	{
8167	VMXGetCurrentVmcs(&pVCpu->hm.s.vmx.LastError.HCPhysCurrentVmcs);
8168	pVCpu->hm.s.vmx.LastError.u32VmcsRev = (uint32_t )pVmcsInfo->pvVmcs;
8169	pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
8170	/* LastError.idCurrentCpu was updated in hmR0VmxPreRunGuestCommitted(). */
8171	}
8172
8173	/* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
8174	VMMRZCallRing3Disable(pVCpu);
8175	Log4Func(("rcExit=%d\n", VBOXSTRICTRC_VAL(rcExit)));
8176
8177	/*
8178	* Convert any pending HM events back to TRPM due to premature exits to ring-3.
8179	* We need to do this only on returns to ring-3 and not for longjmps to ring3.
8180	*
8181	* This is because execution may continue from ring-3 and we would need to inject
8182	* the event from there (hence place it back in TRPM).
8183	*/
8184	if (pVCpu->hm.s.Event.fPending)
8185	{
8186	hmR0VmxPendingEventToTrpmTrap(pVCpu);
8187	Assert(!pVCpu->hm.s.Event.fPending);
8188
8189	/* Clear the events from the VMCS. */
8190	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, 0);
8191	AssertRCReturn(rc, rc);
8192	}
8193	#ifdef VBOX_STRICT
8194	else
8195	{
8196	/*
8197	* Ensure we don't accidentally clear a pending HM event without clearing the VMCS.
8198	* This can be pretty hard to debug otherwise, interrupts might get injected twice
8199	* occasionally, see @bugref{9180#c42}.
8200	*/
8201	uint32_t uEntryIntInfo;
8202	int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &uEntryIntInfo);
8203	AssertRC(rc);
8204	Assert(!VMX_ENTRY_INT_INFO_IS_VALID(uEntryIntInfo));
8205	}
8206	#endif
8207
8208	/*
8209	* Clear the interrupt-window and NMI-window VMCS controls as we could have got
8210	* a VM-exit with higher priority than interrupt-window or NMI-window VM-exits
8211	* (e.g. TPR below threshold).
8212	*/
8213	int rc = hmR0VmxClearIntWindowExitVmcs(pVmcsInfo);
8214	rc \|= hmR0VmxClearNmiWindowExitVmcs(pVmcsInfo);
8215	AssertRCReturn(rc, rc);
8216
8217	/* If we're emulating an instruction, we shouldn't have any TRPM traps pending
8218	and if we're injecting an event we should have a TRPM trap pending. */
8219	AssertMsg(rcExit != VINF_EM_RAW_INJECT_TRPM_EVENT \|\| TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
8220	#ifndef DEBUG_bird /* Triggered after firing an NMI against NT4SP1, possibly a triple fault in progress. */
8221	AssertMsg(rcExit != VINF_EM_RAW_EMULATE_INSTR \|\| !TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
8222	#endif
8223
8224	/* Save guest state and restore host state bits. */
8225	rc = hmR0VmxLeaveSession(pVCpu);
8226	AssertRCReturn(rc, rc);
8227	STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
8228
8229	/* Thread-context hooks are unregistered at this point!!! */
8230
8231	/* Sync recompiler state. */
8232	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
8233	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
8234	\| CPUM_CHANGED_LDTR
8235	\| CPUM_CHANGED_GDTR
8236	\| CPUM_CHANGED_IDTR
8237	\| CPUM_CHANGED_TR
8238	\| CPUM_CHANGED_HIDDEN_SEL_REGS);
8239	if ( pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging
8240	&& CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx))
8241	CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
8242
8243	Assert(!pVCpu->hm.s.fClearTrapFlag);
8244
8245	/* Update the exit-to-ring 3 reason. */
8246	pVCpu->hm.s.rcLastExitToR3 = VBOXSTRICTRC_VAL(rcExit);
8247
8248	/* On our way back from ring-3 reload the guest state if there is a possibility of it being changed. */
8249	if ( rcExit != VINF_EM_RAW_INTERRUPT
8250	\|\| CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
8251	{
8252	Assert(!(pVCpu->cpum.GstCtx.fExtrn & HMVMX_CPUMCTX_EXTRN_ALL));
8253	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
8254	}
8255
8256	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
8257
8258	/* We do -not- want any longjmp notifications after this! We must return to ring-3 ASAP. */
8259	VMMRZCallRing3RemoveNotification(pVCpu);
8260	VMMRZCallRing3Enable(pVCpu);
8261
8262	return rc;
8263	}
8264
8265
8266	/**
8267	* VMMRZCallRing3() callback wrapper which saves the guest state before we
8268	* longjump to ring-3 and possibly get preempted.
8269	*
8270	* @returns VBox status code.
8271	* @param pVCpu The cross context virtual CPU structure.
8272	* @param enmOperation The operation causing the ring-3 longjump.
8273	* @param pvUser User argument, currently unused, NULL.
8274	*/
8275	static DECLCALLBACK(int) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
8276	{
8277	RT_NOREF(pvUser);
8278	if (enmOperation == VMMCALLRING3_VM_R0_ASSERTION)
8279	{
8280	/*
8281	* !!! IMPORTANT !!!
8282	* If you modify code here, check whether hmR0VmxLeave() and hmR0VmxLeaveSession() needs to be updated too.
8283	* This is a stripped down version which gets out ASAP, trying to not trigger any further assertions.
8284	*/
8285	VMMRZCallRing3RemoveNotification(pVCpu);
8286	VMMRZCallRing3Disable(pVCpu);
8287	RTTHREADPREEMPTSTATE PreemptState = RTTHREADPREEMPTSTATE_INITIALIZER;
8288	RTThreadPreemptDisable(&PreemptState);
8289
8290	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
8291	hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
8292	CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
8293	CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
8294
8295	#if HC_ARCH_BITS == 64
8296	/* Restore host-state bits that VT-x only restores partially. */
8297	if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
8298	&& (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
8299	VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
8300	pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
8301	#endif
8302
8303	/* Restore the lazy host MSRs as we're leaving VT-x context. */
8304	if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
8305	hmR0VmxLazyRestoreHostMsrs(pVCpu);
8306
8307	/* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
8308	pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs = false;
8309	VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
8310
8311	/* Clear the current VMCS data back to memory. */
8312	hmR0VmxClearVmcs(pVmcsInfo);
8313
8314	/** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
8315	VMMR0ThreadCtxHookDisable(pVCpu);
8316	HMR0LeaveCpu(pVCpu);
8317	RTThreadPreemptRestore(&PreemptState);
8318	return VINF_SUCCESS;
8319	}
8320
8321	Assert(pVCpu);
8322	Assert(pvUser);
8323	Assert(VMMRZCallRing3IsEnabled(pVCpu));
8324	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
8325
8326	VMMRZCallRing3Disable(pVCpu);
8327	Assert(VMMR0IsLogFlushDisabled(pVCpu));
8328
8329	Log4Func((" -> hmR0VmxLongJmpToRing3 enmOperation=%d\n", enmOperation));
8330
8331	int rc = hmR0VmxLongJmpToRing3(pVCpu);
8332	AssertRCReturn(rc, rc);
8333
8334	VMMRZCallRing3Enable(pVCpu);
8335	return VINF_SUCCESS;
8336	}
8337
8338
8339	/**
8340	* Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
8341	* stack.
8342	*
8343	* @returns Strict VBox status code (i.e. informational status codes too).
8344	* @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
8345	* @param pVCpu The cross context virtual CPU structure.
8346	* @param uValue The value to push to the guest stack.
8347	*/
8348	static VBOXSTRICTRC hmR0VmxRealModeGuestStackPush(PVMCPU pVCpu, uint16_t uValue)
8349	{
8350	/*
8351	* The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
8352	* virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
8353	* See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
8354	*/
8355	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
8356	if (pCtx->sp == 1)
8357	return VINF_EM_RESET;
8358	pCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
8359	int rc = PGMPhysSimpleWriteGCPhys(pVCpu->CTX_SUFF(pVM), pCtx->ss.u64Base + pCtx->sp, &uValue, sizeof(uint16_t));
8360	AssertRC(rc);
8361	return rc;
8362	}
8363
8364
8365	/**
8366	* Injects an event into the guest upon VM-entry by updating the relevant fields
8367	* in the VM-entry area in the VMCS.
8368	*
8369	* @returns Strict VBox status code (i.e. informational status codes too).
8370	* @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
8371	* @retval VINF_EM_RESET if event injection resulted in a triple-fault.
8372	*
8373	* @param pVCpu The cross context virtual CPU structure.
8374	* @param pVmxTransient The VMX-transient structure.
8375	* @param pEvent The event being injected.
8376	* @param pfIntrState Pointer to the VT-x guest-interruptibility-state.
8377	* This will be updated if necessary. This cannot not
8378	* be NULL.
8379	* @param fStepping Whether we're single-stepping guest execution and
8380	* should return VINF_EM_DBG_STEPPED if the event is
8381	* injected directly (registers modified by us, not by
8382	* hardware on VM-entry).
8383	*/
8384	static VBOXSTRICTRC hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PCHMEVENT pEvent, bool fStepping,
8385	uint32_t *pfIntrState)
8386	{
8387	/* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
8388	AssertMsg(!RT_HI_U32(pEvent->u64IntInfo), ("%#RX64\n", pEvent->u64IntInfo));
8389	Assert(pfIntrState);
8390
8391	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
8392	uint32_t u32IntInfo = pEvent->u64IntInfo;
8393	uint32_t const u32ErrCode = pEvent->u32ErrCode;
8394	uint32_t const cbInstr = pEvent->cbInstr;
8395	RTGCUINTPTR const GCPtrFault = pEvent->GCPtrFaultAddress;
8396	uint32_t const uVector = VMX_ENTRY_INT_INFO_VECTOR(u32IntInfo);
8397	uint32_t const uIntType = VMX_ENTRY_INT_INFO_TYPE(u32IntInfo);
8398
8399	#ifdef VBOX_STRICT
8400	/*
8401	* Validate the error-code-valid bit for hardware exceptions.
8402	* No error codes for exceptions in real-mode.
8403	*
8404	* See Intel spec. 20.1.4 "Interrupt and Exception Handling"
8405	*/
8406	if ( uIntType == VMX_EXIT_INT_INFO_TYPE_HW_XCPT
8407	&& !CPUMIsGuestInRealModeEx(pCtx))
8408	{
8409	switch (uVector)
8410	{
8411	case X86_XCPT_PF:
8412	case X86_XCPT_DF:
8413	case X86_XCPT_TS:
8414	case X86_XCPT_NP:
8415	case X86_XCPT_SS:
8416	case X86_XCPT_GP:
8417	case X86_XCPT_AC:
8418	AssertMsg(VMX_ENTRY_INT_INFO_IS_ERROR_CODE_VALID(u32IntInfo),
8419	("Error-code-valid bit not set for exception that has an error code uVector=%#x\n", uVector));
8420	RT_FALL_THRU();
8421	default:
8422	break;
8423	}
8424	}
8425
8426	/* Cannot inject an NMI when block-by-MOV SS is in effect. */
8427	Assert( uIntType != VMX_EXIT_INT_INFO_TYPE_NMI
8428	\|\| !(*pfIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS));
8429	#endif
8430
8431	STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
8432
8433	/*
8434	* Hardware interrupts & exceptions cannot be delivered through the software interrupt
8435	* redirection bitmap to the real mode task in virtual-8086 mode. We must jump to the
8436	* interrupt handler in the (real-mode) guest.
8437	*
8438	* See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode".
8439	* See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
8440	*/
8441	if (CPUMIsGuestInRealModeEx(pCtx)) /* CR0.PE bit changes are always intercepted, so it's up to date. */
8442	{
8443	if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest)
8444	{
8445	/*
8446	* For CPUs with unrestricted guest execution enabled and with the guest
8447	* in real-mode, we must not set the deliver-error-code bit.
8448	*
8449	* See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
8450	*/
8451	u32IntInfo &= ~VMX_ENTRY_INT_INFO_ERROR_CODE_VALID;
8452	}
8453	else
8454	{
8455	PVM pVM = pVCpu->CTX_SUFF(pVM);
8456	Assert(PDMVmmDevHeapIsEnabled(pVM));
8457	Assert(pVM->hm.s.vmx.pRealModeTSS);
8458	Assert(!CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx));
8459
8460	/* We require RIP, RSP, RFLAGS, CS, IDTR, import them. */
8461	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
8462	int rc2 = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_SREG_MASK \| CPUMCTX_EXTRN_TABLE_MASK
8463	\| CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_RFLAGS);
8464	AssertRCReturn(rc2, rc2);
8465
8466	/* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
8467	size_t const cbIdtEntry = sizeof(X86IDTR16);
8468	if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pCtx->idtr.cbIdt)
8469	{
8470	/* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
8471	if (uVector == X86_XCPT_DF)
8472	return VINF_EM_RESET;
8473
8474	/* If we're injecting a #GP with no valid IDT entry, inject a double-fault.
8475	No error codes for exceptions in real-mode. */
8476	if (uVector == X86_XCPT_GP)
8477	{
8478	uint32_t const uXcptDfInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_DF)
8479	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_ENTRY_INT_INFO_TYPE_HW_XCPT)
8480	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
8481	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
8482	HMEVENT EventXcptDf;
8483	RT_ZERO(EventXcptDf);
8484	EventXcptDf.u64IntInfo = uXcptDfInfo;
8485	return hmR0VmxInjectEventVmcs(pVCpu, pVmxTransient, &EventXcptDf, fStepping, pfIntrState);
8486	}
8487
8488	/*
8489	* If we're injecting an event with no valid IDT entry, inject a #GP.
8490	* No error codes for exceptions in real-mode.
8491	*
8492	* See Intel spec. 20.1.4 "Interrupt and Exception Handling"
8493	*/
8494	uint32_t const uXcptGpInfo = RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VECTOR, X86_XCPT_GP)
8495	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_TYPE, VMX_ENTRY_INT_INFO_TYPE_HW_XCPT)
8496	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_ERR_CODE_VALID, 0)
8497	\| RT_BF_MAKE(VMX_BF_ENTRY_INT_INFO_VALID, 1);
8498	HMEVENT EventXcptGp;
8499	RT_ZERO(EventXcptGp);
8500	EventXcptGp.u64IntInfo = uXcptGpInfo;
8501	return hmR0VmxInjectEventVmcs(pVCpu, pVmxTransient, &EventXcptGp, fStepping, pfIntrState);
8502	}
8503
8504	/* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
8505	uint16_t uGuestIp = pCtx->ip;
8506	if (uIntType == VMX_ENTRY_INT_INFO_TYPE_SW_XCPT)
8507	{
8508	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF);
8509	/* #BP and #OF are both benign traps, we need to resume the next instruction. */
8510	uGuestIp = pCtx->ip + (uint16_t)cbInstr;
8511	}
8512	else if (uIntType == VMX_ENTRY_INT_INFO_TYPE_SW_INT)
8513	uGuestIp = pCtx->ip + (uint16_t)cbInstr;
8514
8515	/* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
8516	X86IDTR16 IdtEntry;
8517	RTGCPHYS const GCPhysIdtEntry = (RTGCPHYS)pCtx->idtr.pIdt + uVector * cbIdtEntry;
8518	rc2 = PGMPhysSimpleReadGCPhys(pVM, &IdtEntry, GCPhysIdtEntry, cbIdtEntry);
8519	AssertRCReturn(rc2, rc2);
8520
8521	/* Construct the stack frame for the interrupt/exception handler. */
8522	VBOXSTRICTRC rcStrict;
8523	rcStrict = hmR0VmxRealModeGuestStackPush(pVCpu, pCtx->eflags.u32);
8524	if (rcStrict == VINF_SUCCESS)
8525	{
8526	rcStrict = hmR0VmxRealModeGuestStackPush(pVCpu, pCtx->cs.Sel);
8527	if (rcStrict == VINF_SUCCESS)
8528	rcStrict = hmR0VmxRealModeGuestStackPush(pVCpu, uGuestIp);
8529	}
8530
8531	/* Clear the required eflag bits and jump to the interrupt/exception handler. */
8532	if (rcStrict == VINF_SUCCESS)
8533	{
8534	pCtx->eflags.u32 &= ~(X86_EFL_IF \| X86_EFL_TF \| X86_EFL_RF \| X86_EFL_AC);
8535	pCtx->rip = IdtEntry.offSel;
8536	pCtx->cs.Sel = IdtEntry.uSel;
8537	pCtx->cs.ValidSel = IdtEntry.uSel;
8538	pCtx->cs.u64Base = IdtEntry.uSel << cbIdtEntry;
8539	if ( uIntType == VMX_ENTRY_INT_INFO_TYPE_HW_XCPT
8540	&& uVector == X86_XCPT_PF)
8541	pCtx->cr2 = GCPtrFault;
8542
8543	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CS \| HM_CHANGED_GUEST_CR2
8544	\| HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS
8545	\| HM_CHANGED_GUEST_RSP);
8546
8547	/*
8548	* If we delivered a hardware exception (other than an NMI) and if there was
8549	* block-by-STI in effect, we should clear it.
8550	*/
8551	if (*pfIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
8552	{
8553	Assert( uIntType != VMX_ENTRY_INT_INFO_TYPE_NMI
8554	&& uIntType != VMX_ENTRY_INT_INFO_TYPE_EXT_INT);
8555	Log4Func(("Clearing inhibition due to STI\n"));
8556	*pfIntrState &= ~VMX_VMCS_GUEST_INT_STATE_BLOCK_STI;
8557	}
8558
8559	Log4(("Injected real-mode: u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x Eflags=%#x CS:EIP=%04x:%04x\n",
8560	u32IntInfo, u32ErrCode, cbInstr, pCtx->eflags.u, pCtx->cs.Sel, pCtx->eip));
8561
8562	/*
8563	* The event has been truly dispatched to the guest. Mark it as no longer pending so
8564	* we don't attempt to undo it if we are returning to ring-3 before executing guest code.
8565	*/
8566	pVCpu->hm.s.Event.fPending = false;
8567
8568	/* If we're stepping and we've changed cs:rip above, bail out of the VMX R0 execution loop. */
8569	if (fStepping)
8570	rcStrict = VINF_EM_DBG_STEPPED;
8571	}
8572	AssertMsg(rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_EM_RESET \|\| (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
8573	("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
8574	return rcStrict;
8575	}
8576	}
8577
8578	/*
8579	* Validate.
8580	*/
8581	Assert(VMX_ENTRY_INT_INFO_IS_VALID(u32IntInfo)); /* Bit 31 (Valid bit) must be set by caller. */
8582	Assert(!(u32IntInfo & VMX_BF_ENTRY_INT_INFO_RSVD_12_30_MASK)); /* Bits 30:12 MBZ. */
8583
8584	/*
8585	* Inject the event into the VMCS.
8586	*/
8587	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntInfo);
8588	if (VMX_ENTRY_INT_INFO_IS_ERROR_CODE_VALID(u32IntInfo))
8589	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
8590	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
8591	AssertRCReturn(rc, rc);
8592
8593	/*
8594	* Update guest CR2 if this is a page-fault.
8595	*/
8596	if ( VMX_ENTRY_INT_INFO_TYPE(u32IntInfo) == VMX_EXIT_INT_INFO_TYPE_HW_XCPT
8597	&& uVector == X86_XCPT_PF)
8598	pCtx->cr2 = GCPtrFault;
8599
8600	Log4(("Injecting u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x CR2=%#RX64\n", u32IntInfo, u32ErrCode, cbInstr, pCtx->cr2));
8601	return VINF_SUCCESS;
8602	}
8603
8604
8605	/**
8606	* Evaluates the event to be delivered to the guest and sets it as the pending
8607	* event.
8608	*
8609	* @returns Strict VBox status code (i.e. informational status codes too).
8610	* @param pVCpu The cross context virtual CPU structure.
8611	* @param pVmxTransient The VMX-transient structure.
8612	* @param pfIntrState Where to store the VT-x guest-interruptibility state.
8613	*/
8614	static VBOXSTRICTRC hmR0VmxEvaluatePendingEvent(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t *pfIntrState)
8615	{
8616	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
8617	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
8618
8619	/* Get the current interruptibility-state of the guest and then figure out what can be injected. */
8620	uint32_t const fIntrState = hmR0VmxGetGuestIntrState(pVCpu, pVmcsInfo);
8621	bool const fBlockMovSS = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS);
8622	bool const fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI);
8623	bool const fBlockNmi = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI);
8624
8625	Assert(!fBlockSti \|\| !(ASMAtomicUoReadU64(&pCtx->fExtrn) & CPUMCTX_EXTRN_RFLAGS));
8626	Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
8627	Assert(!fBlockSti \|\| pCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
8628	Assert(!TRPMHasTrap(pVCpu));
8629	Assert(pfIntrState);
8630
8631	*pfIntrState = fIntrState;
8632
8633	/*
8634	* Toggling of interrupt force-flags here is safe since we update TRPM on premature exits
8635	* to ring-3 before executing guest code, see hmR0VmxExitToRing3(). We must NOT restore these force-flags.
8636	*/
8637	/** @todo SMI. SMIs take priority over NMIs. */
8638	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INTERRUPT_NMI)) /* NMI. NMIs take priority over regular interrupts. */
8639	{
8640	/* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
8641	if ( !pVCpu->hm.s.Event.fPending
8642	&& !fBlockNmi
8643	&& !fBlockSti
8644	&& !fBlockMovSS)
8645	{
8646	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
8647	if ( pVmxTransient->fIsNestedGuest
8648	&& CPUMIsGuestVmxPinCtlsSet(pVCpu, pCtx, VMX_PIN_CTLS_NMI_EXIT))
8649	return IEMExecVmxVmexitNmi(pVCpu);
8650	#endif
8651	hmR0VmxSetPendingXcptNmi(pVCpu);
8652	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
8653	Log4Func(("Pending NMI\n"));
8654	}
8655	else
8656	hmR0VmxSetNmiWindowExitVmcs(pVCpu, pVmcsInfo);
8657	}
8658	/*
8659	* Check if the guest can receive external interrupts (PIC/APIC). Once PDMGetInterrupt() returns
8660	* a valid interrupt we -must- deliver the interrupt. We can no longer re-request it from the APIC.
8661	*/
8662	else if ( VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_INTERRUPT_APIC \| VMCPU_FF_INTERRUPT_PIC)
8663	&& !pVCpu->hm.s.fSingleInstruction)
8664	{
8665	Assert(!DBGFIsStepping(pVCpu));
8666	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_RFLAGS);
8667	AssertRCReturn(rc, rc);
8668	bool const fBlockInt = !(pCtx->eflags.u32 & X86_EFL_IF);
8669	if ( !pVCpu->hm.s.Event.fPending
8670	&& !fBlockInt
8671	&& !fBlockSti
8672	&& !fBlockMovSS)
8673	{
8674	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
8675	if ( pVmxTransient->fIsNestedGuest
8676	&& CPUMIsGuestVmxPinCtlsSet(pVCpu, pCtx, VMX_PIN_CTLS_EXT_INT_EXIT))
8677	{
8678	VBOXSTRICTRC rcStrict = IEMExecVmxVmexitExtInt(pVCpu, 0/* uVector /, true / fIntPending */);
8679	if (rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE)
8680	return rcStrict;
8681	}
8682	#endif
8683	uint8_t u8Interrupt;
8684	rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
8685	if (RT_SUCCESS(rc))
8686	{
8687	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
8688	if ( pVmxTransient->fIsNestedGuest
8689	&& CPUMIsGuestVmxPinCtlsSet(pVCpu, pCtx, VMX_PIN_CTLS_EXT_INT_EXIT)
8690	&& CPUMIsGuestVmxExitCtlsSet(pVCpu, pCtx, VMX_EXIT_CTLS_ACK_EXT_INT))
8691	{
8692	VBOXSTRICTRC rcStrict = IEMExecVmxVmexitExtInt(pVCpu, u8Interrupt, false /* fIntPending */);
8693	if (rcStrict != VINF_VMX_INTERCEPT_NOT_ACTIVE)
8694	return rcStrict;
8695	}
8696	#endif
8697	hmR0VmxSetPendingExtInt(pVCpu, u8Interrupt);
8698	Log4Func(("Pending external interrupt vector %#x\n", u8Interrupt));
8699	}
8700	else if (rc == VERR_APIC_INTR_MASKED_BY_TPR)
8701	{
8702	if ( !pVmxTransient->fIsNestedGuest
8703	&& (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW))
8704	hmR0VmxApicSetTprThreshold(pVCpu, pVmcsInfo, u8Interrupt >> 4);
8705	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchTprMaskedIrq);
8706
8707	/*
8708	* If the CPU doesn't have TPR shadowing, we will always get a VM-exit on TPR changes and
8709	* APICSetTpr() will end up setting the VMCPU_FF_INTERRUPT_APIC if required, so there is no
8710	* need to re-set this force-flag here.
8711	*/
8712	}
8713	else
8714	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
8715	}
8716	else
8717	hmR0VmxSetIntWindowExitVmcs(pVCpu, pVmcsInfo);
8718	}
8719
8720	return VINF_SUCCESS;
8721	}
8722
8723
8724	/**
8725	* Injects any pending events into the guest if the guest is in a state to
8726	* receive them.
8727	*
8728	* @returns Strict VBox status code (i.e. informational status codes too).
8729	* @param pVCpu The cross context virtual CPU structure.
8730	* @param pVmxTransient The VMX-transient structure.
8731	* @param fIntrState The VT-x guest-interruptibility state.
8732	* @param fStepping Whether we are single-stepping the guest using the
8733	* hypervisor debugger and should return
8734	* VINF_EM_DBG_STEPPED if the event was dispatched
8735	* directly.
8736	*/
8737	static VBOXSTRICTRC hmR0VmxInjectPendingEvent(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t fIntrState, bool fStepping)
8738	{
8739	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
8740	Assert(VMMRZCallRing3IsEnabled(pVCpu));
8741
8742	bool const fBlockMovSS = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS);
8743	bool const fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI);
8744
8745	Assert(!fBlockSti \|\| !(ASMAtomicUoReadU64(&pVCpu->cpum.GstCtx.fExtrn) & CPUMCTX_EXTRN_RFLAGS));
8746	Assert(!fBlockSti \|\| pVCpu->cpum.GstCtx.eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
8747	Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
8748	Assert(!TRPMHasTrap(pVCpu));
8749
8750	VBOXSTRICTRC rcStrict = VINF_SUCCESS;
8751	if (pVCpu->hm.s.Event.fPending)
8752	{
8753	/*
8754	* Do -not- clear any interrupt-window exiting control here. We might have an interrupt
8755	* pending even while injecting an event and in this case, we want a VM-exit as soon as
8756	* the guest is ready for the next interrupt, see @bugref{6208#c45}.
8757	*
8758	* See Intel spec. 26.6.5 "Interrupt-Window Exiting and Virtual-Interrupt Delivery".
8759	*/
8760	uint32_t const uIntType = VMX_ENTRY_INT_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
8761	#ifdef VBOX_STRICT
8762	if (uIntType == VMX_ENTRY_INT_INFO_TYPE_EXT_INT)
8763	{
8764	bool const fBlockInt = !(pVCpu->cpum.GstCtx.eflags.u32 & X86_EFL_IF);
8765	Assert(!fBlockInt);
8766	Assert(!fBlockSti);
8767	Assert(!fBlockMovSS);
8768	}
8769	else if (uIntType == VMX_ENTRY_INT_INFO_TYPE_NMI)
8770	{
8771	bool const fBlockNmi = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI);
8772	Assert(!fBlockSti);
8773	Assert(!fBlockMovSS);
8774	Assert(!fBlockNmi);
8775	}
8776	#endif
8777	Log4(("Injecting pending event vcpu[%RU32] u64IntInfo=%#RX64 Type=%#RX32\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntInfo,
8778	uIntType));
8779
8780	/*
8781	* Inject the event and get any changes to the guest-interruptibility state.
8782	*
8783	* The guest-interruptibility state may need to be updated if we inject the event
8784	* into the guest IDT ourselves (for real-on-v86 guest injecting software interrupts).
8785	*/
8786	rcStrict = hmR0VmxInjectEventVmcs(pVCpu, pVmxTransient, &pVCpu->hm.s.Event, fStepping, &fIntrState);
8787	AssertRCReturn(VBOXSTRICTRC_VAL(rcStrict), rcStrict);
8788
8789	if (uIntType == VMX_ENTRY_INT_INFO_TYPE_EXT_INT)
8790	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
8791	else
8792	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
8793	}
8794
8795	/*
8796	* Update the guest-interruptibility state.
8797	*
8798	* This is required for the real-on-v86 software interrupt injection case above, as well as
8799	* updates to the guest state from ring-3 or IEM/REM.
8800	*/
8801	int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INT_STATE, fIntrState);
8802	AssertRCReturn(rc, rc);
8803
8804	/*
8805	* There's no need to clear the VM-entry interruption-information field here if we're not
8806	* injecting anything. VT-x clears the valid bit on every VM-exit.
8807	*
8808	* See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
8809	*/
8810
8811	Assert(rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_EM_RESET \|\| (rcStrict == VINF_EM_DBG_STEPPED && fStepping));
8812	NOREF(fBlockMovSS); NOREF(fBlockSti);
8813	return rcStrict;
8814	}
8815
8816
8817	/**
8818	* Enters the VT-x session.
8819	*
8820	* @returns VBox status code.
8821	* @param pVCpu The cross context virtual CPU structure.
8822	*/
8823	VMMR0DECL(int) VMXR0Enter(PVMCPU pVCpu)
8824	{
8825	AssertPtr(pVCpu);
8826	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fSupported);
8827	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8828
8829	LogFlowFunc(("pVCpu=%p\n", pVCpu));
8830	Assert((pVCpu->hm.s.fCtxChanged & (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
8831	== (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE));
8832
8833	#ifdef VBOX_STRICT
8834	/* At least verify VMX is enabled, since we can't check if we're in VMX root mode without #GP'ing. */
8835	RTCCUINTREG uHostCR4 = ASMGetCR4();
8836	if (!(uHostCR4 & X86_CR4_VMXE))
8837	{
8838	LogRelFunc(("X86_CR4_VMXE bit in CR4 is not set!\n"));
8839	return VERR_VMX_X86_CR4_VMXE_CLEARED;
8840	}
8841	#endif
8842
8843	/*
8844	* Load the appropriate VMCS as the current and active one.
8845	*/
8846	PVMXVMCSINFO pVmcsInfo;
8847	bool const fInNestedGuestMode = CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx);
8848	if (!fInNestedGuestMode)
8849	pVmcsInfo = &pVCpu->hm.s.vmx.VmcsInfo;
8850	else
8851	pVmcsInfo = &pVCpu->hm.s.vmx.VmcsInfoNstGst;
8852	int rc = hmR0VmxLoadVmcs(pVmcsInfo);
8853	if (RT_SUCCESS(rc))
8854	{
8855	pVCpu->hm.s.vmx.fSwitchedToNstGstVmcs = fInNestedGuestMode;
8856	pVCpu->hm.s.fLeaveDone = false;
8857	Log4Func(("Loaded Vmcs. HostCpuId=%u\n", RTMpCpuId()));
8858
8859	/*
8860	* Do the EMT scheduled L1D flush here if needed.
8861	*/
8862	if (pVCpu->CTX_SUFF(pVM)->hm.s.fL1dFlushOnSched)
8863	ASMWrMsr(MSR_IA32_FLUSH_CMD, MSR_IA32_FLUSH_CMD_F_L1D);
8864	}
8865	return rc;
8866	}
8867
8868
8869	/**
8870	* The thread-context callback (only on platforms which support it).
8871	*
8872	* @param enmEvent The thread-context event.
8873	* @param pVCpu The cross context virtual CPU structure.
8874	* @param fGlobalInit Whether global VT-x/AMD-V init. was used.
8875	* @thread EMT(pVCpu)
8876	*/
8877	VMMR0DECL(void) VMXR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPU pVCpu, bool fGlobalInit)
8878	{
8879	NOREF(fGlobalInit);
8880
8881	switch (enmEvent)
8882	{
8883	case RTTHREADCTXEVENT_OUT:
8884	{
8885	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8886	Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
8887	VMCPU_ASSERT_EMT(pVCpu);
8888
8889	/* No longjmps (logger flushes, locks) in this fragile context. */
8890	VMMRZCallRing3Disable(pVCpu);
8891	Log4Func(("Preempting: HostCpuId=%u\n", RTMpCpuId()));
8892
8893	/* Restore host-state (FPU, debug etc.) */
8894	if (!pVCpu->hm.s.fLeaveDone)
8895	{
8896	/*
8897	* Do -not- import the guest-state here as we might already be in the middle of importing
8898	* it, esp. bad if we're holding the PGM lock, see comment in hmR0VmxImportGuestState().
8899	*/
8900	hmR0VmxLeave(pVCpu, false /* fImportState */);
8901	pVCpu->hm.s.fLeaveDone = true;
8902	}
8903
8904	/* Leave HM context, takes care of local init (term). */
8905	int rc = HMR0LeaveCpu(pVCpu);
8906	AssertRC(rc);
8907
8908	/* Restore longjmp state. */
8909	VMMRZCallRing3Enable(pVCpu);
8910	STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreempt);
8911	break;
8912	}
8913
8914	case RTTHREADCTXEVENT_IN:
8915	{
8916	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8917	Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
8918	VMCPU_ASSERT_EMT(pVCpu);
8919
8920	/* No longjmps here, as we don't want to trigger preemption (& its hook) while resuming. */
8921	VMMRZCallRing3Disable(pVCpu);
8922	Log4Func(("Resumed: HostCpuId=%u\n", RTMpCpuId()));
8923
8924	/* Initialize the bare minimum state required for HM. This takes care of
8925	initializing VT-x if necessary (onlined CPUs, local init etc.) */
8926	int rc = hmR0EnterCpu(pVCpu);
8927	AssertRC(rc);
8928	Assert( (pVCpu->hm.s.fCtxChanged & (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
8929	== (HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE));
8930
8931	/* Load the active VMCS as the current one. */
8932	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
8933	rc = hmR0VmxLoadVmcs(pVmcsInfo);
8934	AssertRC(rc);
8935	Log4Func(("Resumed: Loaded Vmcs. HostCpuId=%u\n", RTMpCpuId()));
8936	pVCpu->hm.s.fLeaveDone = false;
8937
8938	/* Do the EMT scheduled L1D flush if needed. */
8939	if (pVCpu->CTX_SUFF(pVM)->hm.s.fL1dFlushOnSched)
8940	ASMWrMsr(MSR_IA32_FLUSH_CMD, MSR_IA32_FLUSH_CMD_F_L1D);
8941
8942	/* Restore longjmp state. */
8943	VMMRZCallRing3Enable(pVCpu);
8944	break;
8945	}
8946
8947	default:
8948	break;
8949	}
8950	}
8951
8952
8953	/**
8954	* Exports the host state into the VMCS host-state area.
8955	* Sets up the VM-exit MSR-load area.
8956	*
8957	* The CPU state will be loaded from these fields on every successful VM-exit.
8958	*
8959	* @returns VBox status code.
8960	* @param pVCpu The cross context virtual CPU structure.
8961	*
8962	* @remarks No-long-jump zone!!!
8963	*/
8964	static int hmR0VmxExportHostState(PVMCPU pVCpu)
8965	{
8966	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8967
8968	int rc = VINF_SUCCESS;
8969	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT)
8970	{
8971	rc = hmR0VmxExportHostControlRegs();
8972	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8973
8974	rc = hmR0VmxExportHostSegmentRegs(pVCpu);
8975	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8976
8977	rc = hmR0VmxExportHostMsrs(pVCpu);
8978	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8979
8980	pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_HOST_CONTEXT;
8981	}
8982	return rc;
8983	}
8984
8985
8986	/**
8987	* Saves the host state in the VMCS host-state.
8988	*
8989	* @returns VBox status code.
8990	* @param pVCpu The cross context virtual CPU structure.
8991	*
8992	* @remarks No-long-jump zone!!!
8993	*/
8994	VMMR0DECL(int) VMXR0ExportHostState(PVMCPU pVCpu)
8995	{
8996	AssertPtr(pVCpu);
8997	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8998
8999	/*
9000	* Export the host state here while entering HM context.
9001	* When thread-context hooks are used, we might get preempted and have to re-save the host
9002	* state but most of the time we won't be, so do it here before we disable interrupts.
9003	*/
9004	return hmR0VmxExportHostState(pVCpu);
9005	}
9006
9007
9008	/**
9009	* Exports the guest state into the VMCS guest-state area.
9010	*
9011	* The will typically be done before VM-entry when the guest-CPU state and the
9012	* VMCS state may potentially be out of sync.
9013	*
9014	* Sets up the VM-entry MSR-load and VM-exit MSR-store areas. Sets up the
9015	* VM-entry controls.
9016	* Sets up the appropriate VMX non-root function to execute guest code based on
9017	* the guest CPU mode.
9018	*
9019	* @returns VBox strict status code.
9020	* @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
9021	* without unrestricted guest execution and the VMMDev is not presently
9022	* mapped (e.g. EFI32).
9023	*
9024	* @param pVCpu The cross context virtual CPU structure.
9025	* @param pVmxTransient The VMX-transient structure.
9026	*
9027	* @remarks No-long-jump zone!!!
9028	*/
9029	static VBOXSTRICTRC hmR0VmxExportGuestState(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
9030	{
9031	AssertPtr(pVCpu);
9032	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
9033	LogFlowFunc(("pVCpu=%p\n", pVCpu));
9034
9035	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExportGuestState, x);
9036
9037	/*
9038	* Determine real-on-v86 mode.
9039	* Used when the guest is in real-mode and unrestricted guest execution is not used.
9040	*/
9041	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
9042	if ( pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest
9043	\|\| !CPUMIsGuestInRealModeEx(&pVCpu->cpum.GstCtx))
9044	pVmcsInfo->RealMode. fRealOnV86Active = false;
9045	else
9046	{
9047	Assert(!pVmxTransient->fIsNestedGuest);
9048	pVmcsInfo->RealMode.fRealOnV86Active = true;
9049	}
9050
9051	/*
9052	* Any ordering dependency among the sub-functions below must be explicitly stated using comments.
9053	* Ideally, assert that the cross-dependent bits are up-to-date at the point of using it.
9054	*/
9055	/** @todo r=ramshankar: Move hmR0VmxSelectVMRunHandler inside
9056	* hmR0VmxExportGuestEntryExitCtls and do it conditionally. There shouldn't
9057	* be a need to evaluate this everytime since I'm pretty sure we intercept
9058	* all guest paging mode changes. */
9059	int rc = hmR0VmxSelectVMRunHandler(pVCpu, pVmxTransient);
9060	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9061
9062	rc = hmR0VmxExportGuestEntryExitCtls(pVCpu, pVmxTransient);
9063	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9064
9065	rc = hmR0VmxExportGuestCR0(pVCpu, pVmxTransient);
9066	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9067
9068	VBOXSTRICTRC rcStrict = hmR0VmxExportGuestCR3AndCR4(pVCpu, pVmxTransient);
9069	if (rcStrict == VINF_SUCCESS)
9070	{ /* likely */ }
9071	else
9072	{
9073	Assert(rcStrict == VINF_EM_RESCHEDULE_REM \|\| RT_FAILURE_NP(rcStrict));
9074	return rcStrict;
9075	}
9076
9077	rc = hmR0VmxExportGuestSegRegsXdtr(pVCpu, pVmxTransient);
9078	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9079
9080	rc = hmR0VmxExportGuestMsrs(pVCpu, pVmxTransient);
9081	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9082
9083	rc = hmR0VmxExportGuestApicTpr(pVCpu, pVmxTransient);
9084	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9085
9086	rc = hmR0VmxExportGuestXcptIntercepts(pVCpu, pVmxTransient);
9087	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9088
9089	rc = hmR0VmxExportGuestRip(pVCpu);
9090	rc \|= hmR0VmxExportGuestRsp(pVCpu);
9091	rc \|= hmR0VmxExportGuestRflags(pVCpu, pVmxTransient);
9092	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
9093
9094	/* Clear any bits that may be set but exported unconditionally or unused/reserved bits. */
9095	ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~( (HM_CHANGED_GUEST_GPRS_MASK & ~HM_CHANGED_GUEST_RSP)
9096	\| HM_CHANGED_GUEST_CR2
9097	\| (HM_CHANGED_GUEST_DR_MASK & ~HM_CHANGED_GUEST_DR7)
9098	\| HM_CHANGED_GUEST_X87
9099	\| HM_CHANGED_GUEST_SSE_AVX
9100	\| HM_CHANGED_GUEST_OTHER_XSAVE
9101	\| HM_CHANGED_GUEST_XCRx
9102	\| HM_CHANGED_GUEST_KERNEL_GS_BASE /* Part of lazy or auto load-store MSRs. */
9103	\| HM_CHANGED_GUEST_SYSCALL_MSRS /* Part of lazy or auto load-store MSRs. */
9104	\| HM_CHANGED_GUEST_TSC_AUX
9105	\| HM_CHANGED_GUEST_OTHER_MSRS
9106	\| HM_CHANGED_GUEST_HWVIRT /* More accurate PLE handling someday? */
9107	\| (HM_CHANGED_KEEPER_STATE_MASK & ~HM_CHANGED_VMX_MASK)));
9108
9109	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExportGuestState, x);
9110	return rc;
9111	}
9112
9113
9114	/**
9115	* Exports the state shared between the host and guest into the VMCS.
9116	*
9117	* @param pVCpu The cross context virtual CPU structure.
9118	* @param pVmxTransient The VMX-transient structure.
9119	*
9120	* @remarks No-long-jump zone!!!
9121	*/
9122	static void hmR0VmxExportSharedState(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
9123	{
9124	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
9125	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
9126
9127	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_DR_MASK)
9128	{
9129	int rc = hmR0VmxExportSharedDebugState(pVCpu, pVmxTransient);
9130	AssertRC(rc);
9131	pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_GUEST_DR_MASK;
9132
9133	/* Loading shared debug bits might have changed eflags.TF bit for debugging purposes. */
9134	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_RFLAGS)
9135	{
9136	rc = hmR0VmxExportGuestRflags(pVCpu, pVmxTransient);
9137	AssertRC(rc);
9138	}
9139	}
9140
9141	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_GUEST_LAZY_MSRS)
9142	{
9143	hmR0VmxLazyLoadGuestMsrs(pVCpu);
9144	pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_VMX_GUEST_LAZY_MSRS;
9145	}
9146
9147	AssertMsg(!(pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE),
9148	("fCtxChanged=%#RX64\n", pVCpu->hm.s.fCtxChanged));
9149	}
9150
9151
9152	/**
9153	* Worker for loading the guest-state bits in the inner VT-x execution loop.
9154	*
9155	* @returns Strict VBox status code (i.e. informational status codes too).
9156	* @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
9157	* without unrestricted guest execution and the VMMDev is not presently
9158	* mapped (e.g. EFI32).
9159	*
9160	* @param pVCpu The cross context virtual CPU structure.
9161	* @param pVmxTransient The VMX-transient structure.
9162	*
9163	* @remarks No-long-jump zone!!!
9164	*/
9165	static VBOXSTRICTRC hmR0VmxExportGuestStateOptimal(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
9166	{
9167	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
9168	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
9169	Assert(VMMR0IsLogFlushDisabled(pVCpu));
9170
9171	#ifdef HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
9172	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
9173	#endif
9174
9175	/*
9176	* For many exits it's only RIP that changes and hence try to export it first
9177	* without going through a lot of change flag checks.
9178	*/
9179	VBOXSTRICTRC rcStrict;
9180	uint64_t fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
9181	RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
9182	if ((fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)) == HM_CHANGED_GUEST_RIP)
9183	{
9184	rcStrict = hmR0VmxExportGuestRip(pVCpu);
9185	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
9186	{ /* likely */}
9187	else
9188	AssertMsgFailedReturn(("Failed to export guest RIP! rc=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)), rcStrict);
9189	STAM_COUNTER_INC(&pVCpu->hm.s.StatExportMinimal);
9190	}
9191	else if (fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
9192	{
9193	rcStrict = hmR0VmxExportGuestState(pVCpu, pVmxTransient);
9194	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
9195	{ /* likely */}
9196	else
9197	{
9198	AssertMsg(rcStrict == VINF_EM_RESCHEDULE_REM, ("Failed to export guest state! rc=%Rrc\n",
9199	VBOXSTRICTRC_VAL(rcStrict)));
9200	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
9201	return rcStrict;
9202	}
9203	STAM_COUNTER_INC(&pVCpu->hm.s.StatExportFull);
9204	}
9205	else
9206	rcStrict = VINF_SUCCESS;
9207
9208	#ifdef VBOX_STRICT
9209	/* All the guest state bits should be loaded except maybe the host context and/or the shared host/guest bits. */
9210	fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
9211	RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
9212	AssertMsg(!(fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)),
9213	("fCtxChanged=%#RX64\n", fCtxChanged));
9214	#endif
9215	return rcStrict;
9216	}
9217
9218
9219	/**
9220	* Tries to determine what part of the guest-state VT-x has deemed as invalid
9221	* and update error record fields accordingly.
9222	*
9223	* @return VMX_IGS_* return codes.
9224	* @retval VMX_IGS_REASON_NOT_FOUND if this function could not find anything
9225	* wrong with the guest state.
9226	*
9227	* @param pVCpu The cross context virtual CPU structure.
9228	* @param pVmcsInfo The VMCS info. object.
9229	*
9230	* @remarks This function assumes our cache of the VMCS controls
9231	* are valid, i.e. hmR0VmxCheckVmcsCtls() succeeded.
9232	*/
9233	static uint32_t hmR0VmxCheckGuestState(PVMCPU pVCpu, PCVMXVMCSINFO pVmcsInfo)
9234	{
9235	#define HMVMX_ERROR_BREAK(err) { uError = (err); break; }
9236	#define HMVMX_CHECK_BREAK(expr, err) if (!(expr)) { \
9237	uError = (err); \
9238	break; \
9239	} else do { } while (0)
9240
9241	int rc;
9242	PVM pVM = pVCpu->CTX_SUFF(pVM);
9243	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
9244	uint32_t uError = VMX_IGS_ERROR;
9245	uint32_t u32Val;
9246	bool const fUnrestrictedGuest = pVM->hm.s.vmx.fUnrestrictedGuest;
9247
9248	do
9249	{
9250	/*
9251	* CR0.
9252	*/
9253	uint32_t fSetCr0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
9254	uint32_t const fZapCr0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
9255	/* Exceptions for unrestricted guest execution for fixed CR0 bits (PE, PG).
9256	See Intel spec. 26.3.1 "Checks on Guest Control Registers, Debug Registers and MSRs." */
9257	if (fUnrestrictedGuest)
9258	fSetCr0 &= ~(X86_CR0_PE \| X86_CR0_PG);
9259
9260	uint32_t u32GuestCr0;
9261	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32GuestCr0);
9262	AssertRCBreak(rc);
9263	HMVMX_CHECK_BREAK((u32GuestCr0 & fSetCr0) == fSetCr0, VMX_IGS_CR0_FIXED1);
9264	HMVMX_CHECK_BREAK(!(u32GuestCr0 & ~fZapCr0), VMX_IGS_CR0_FIXED0);
9265	if ( !fUnrestrictedGuest
9266	&& (u32GuestCr0 & X86_CR0_PG)
9267	&& !(u32GuestCr0 & X86_CR0_PE))
9268	{
9269	HMVMX_ERROR_BREAK(VMX_IGS_CR0_PG_PE_COMBO);
9270	}
9271
9272	/*
9273	* CR4.
9274	*/
9275	uint64_t const fSetCr4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
9276	uint64_t const fZapCr4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 \| pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
9277
9278	uint32_t u32GuestCr4;
9279	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32GuestCr4);
9280	AssertRCBreak(rc);
9281	HMVMX_CHECK_BREAK((u32GuestCr4 & fSetCr4) == fSetCr4, VMX_IGS_CR4_FIXED1);
9282	HMVMX_CHECK_BREAK(!(u32GuestCr4 & ~fZapCr4), VMX_IGS_CR4_FIXED0);
9283
9284	/*
9285	* IA32_DEBUGCTL MSR.
9286	*/
9287	uint64_t u64Val;
9288	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, &u64Val);
9289	AssertRCBreak(rc);
9290	if ( (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_DEBUG)
9291	&& (u64Val & 0xfffffe3c)) /* Bits 31:9, bits 5:2 MBZ. */
9292	{
9293	HMVMX_ERROR_BREAK(VMX_IGS_DEBUGCTL_MSR_RESERVED);
9294	}
9295	uint64_t u64DebugCtlMsr = u64Val;
9296
9297	#ifdef VBOX_STRICT
9298	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
9299	AssertRCBreak(rc);
9300	Assert(u32Val == pVmcsInfo->u32EntryCtls);
9301	#endif
9302	bool const fLongModeGuest = RT_BOOL(pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_IA32E_MODE_GUEST);
9303
9304	/*
9305	* RIP and RFLAGS.
9306	*/
9307	uint32_t u32Eflags;
9308	#if HC_ARCH_BITS == 64
9309	rc = VMXReadVmcs64(VMX_VMCS_GUEST_RIP, &u64Val);
9310	AssertRCBreak(rc);
9311	/* pCtx->rip can be different than the one in the VMCS (e.g. run guest code and VM-exits that don't update it). */
9312	if ( !fLongModeGuest
9313	\|\| !pCtx->cs.Attr.n.u1Long)
9314	{
9315	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffff00000000)), VMX_IGS_LONGMODE_RIP_INVALID);
9316	}
9317	/** @todo If the processor supports N < 64 linear-address bits, bits 63:N
9318	* must be identical if the "IA-32e mode guest" VM-entry
9319	* control is 1 and CS.L is 1. No check applies if the
9320	* CPU supports 64 linear-address bits. */
9321
9322	/* Flags in pCtx can be different (real-on-v86 for instance). We are only concerned about the VMCS contents here. */
9323	rc = VMXReadVmcs64(VMX_VMCS_GUEST_RFLAGS, &u64Val);
9324	AssertRCBreak(rc);
9325	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffc08028)), /* Bit 63:22, Bit 15, 5, 3 MBZ. */
9326	VMX_IGS_RFLAGS_RESERVED);
9327	HMVMX_CHECK_BREAK((u64Val & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
9328	u32Eflags = u64Val;
9329	#else
9330	rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Eflags);
9331	AssertRCBreak(rc);
9332	HMVMX_CHECK_BREAK(!(u32Eflags & 0xffc08028), VMX_IGS_RFLAGS_RESERVED); /* Bit 31:22, Bit 15, 5, 3 MBZ. */
9333	HMVMX_CHECK_BREAK((u32Eflags & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
9334	#endif
9335
9336	if ( fLongModeGuest
9337	\|\| ( fUnrestrictedGuest
9338	&& !(u32GuestCr0 & X86_CR0_PE)))
9339	{
9340	HMVMX_CHECK_BREAK(!(u32Eflags & X86_EFL_VM), VMX_IGS_RFLAGS_VM_INVALID);
9341	}
9342
9343	uint32_t u32EntryInfo;
9344	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
9345	AssertRCBreak(rc);
9346	if ( VMX_ENTRY_INT_INFO_IS_VALID(u32EntryInfo)
9347	&& VMX_ENTRY_INT_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INT_INFO_TYPE_EXT_INT)
9348	{
9349	HMVMX_CHECK_BREAK(u32Eflags & X86_EFL_IF, VMX_IGS_RFLAGS_IF_INVALID);
9350	}
9351
9352	/*
9353	* 64-bit checks.
9354	*/
9355	#if HC_ARCH_BITS == 64
9356	if (fLongModeGuest)
9357	{
9358	HMVMX_CHECK_BREAK(u32GuestCr0 & X86_CR0_PG, VMX_IGS_CR0_PG_LONGMODE);
9359	HMVMX_CHECK_BREAK(u32GuestCr4 & X86_CR4_PAE, VMX_IGS_CR4_PAE_LONGMODE);
9360	}
9361
9362	if ( !fLongModeGuest
9363	&& (u32GuestCr4 & X86_CR4_PCIDE))
9364	{
9365	HMVMX_ERROR_BREAK(VMX_IGS_CR4_PCIDE);
9366	}
9367
9368	/** @todo CR3 field must be such that bits 63:52 and bits in the range
9369	* 51:32 beyond the processor's physical-address width are 0. */
9370
9371	if ( (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_DEBUG)
9372	&& (pCtx->dr[7] & X86_DR7_MBZ_MASK))
9373	{
9374	HMVMX_ERROR_BREAK(VMX_IGS_DR7_RESERVED);
9375	}
9376
9377	rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, &u64Val);
9378	AssertRCBreak(rc);
9379	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_ESP_NOT_CANONICAL);
9380
9381	rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, &u64Val);
9382	AssertRCBreak(rc);
9383	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_EIP_NOT_CANONICAL);
9384	#endif
9385
9386	/*
9387	* PERF_GLOBAL MSR.
9388	*/
9389	if (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_PERF_MSR)
9390	{
9391	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL, &u64Val);
9392	AssertRCBreak(rc);
9393	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffff8fffffffc)),
9394	VMX_IGS_PERF_GLOBAL_MSR_RESERVED); /* Bits 63:35, bits 31:2 MBZ. */
9395	}
9396
9397	/*
9398	* PAT MSR.
9399	*/
9400	if (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_PAT_MSR)
9401	{
9402	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PAT_FULL, &u64Val);
9403	AssertRCBreak(rc);
9404	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0x707070707070707)), VMX_IGS_PAT_MSR_RESERVED);
9405	for (unsigned i = 0; i < 8; i++)
9406	{
9407	uint8_t u8Val = (u64Val & 0xff);
9408	if ( u8Val != 0 /* UC */
9409	&& u8Val != 1 /* WC */
9410	&& u8Val != 4 /* WT */
9411	&& u8Val != 5 /* WP */
9412	&& u8Val != 6 /* WB */
9413	&& u8Val != 7 /* UC- */)
9414	{
9415	HMVMX_ERROR_BREAK(VMX_IGS_PAT_MSR_INVALID);
9416	}
9417	u64Val >>= 8;
9418	}
9419	}
9420
9421	/*
9422	* EFER MSR.
9423	*/
9424	if (pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_LOAD_EFER_MSR)
9425	{
9426	Assert(pVM->hm.s.vmx.fSupportsVmcsEfer);
9427	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_EFER_FULL, &u64Val);
9428	AssertRCBreak(rc);
9429	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffffffffff2fe)),
9430	VMX_IGS_EFER_MSR_RESERVED); /* Bits 63:12, bit 9, bits 7:1 MBZ. */
9431	HMVMX_CHECK_BREAK(RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL( pVmcsInfo->u32EntryCtls
9432	& VMX_ENTRY_CTLS_IA32E_MODE_GUEST),
9433	VMX_IGS_EFER_LMA_GUEST_MODE_MISMATCH);
9434	/** @todo r=ramshankar: Unrestricted check here is probably wrong, see
9435	* iemVmxVmentryCheckGuestState(). */
9436	HMVMX_CHECK_BREAK( fUnrestrictedGuest
9437	\|\| !(u32GuestCr0 & X86_CR0_PG)
9438	\|\| RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL(u64Val & MSR_K6_EFER_LME),
9439	VMX_IGS_EFER_LMA_LME_MISMATCH);
9440	}
9441
9442	/*
9443	* Segment registers.
9444	*/
9445	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
9446	\|\| !(pCtx->ldtr.Sel & X86_SEL_LDT), VMX_IGS_LDTR_TI_INVALID);
9447	if (!(u32Eflags & X86_EFL_VM))
9448	{
9449	/* CS */
9450	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1Present, VMX_IGS_CS_ATTR_P_INVALID);
9451	HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xf00), VMX_IGS_CS_ATTR_RESERVED);
9452	HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xfffe0000), VMX_IGS_CS_ATTR_RESERVED);
9453	HMVMX_CHECK_BREAK( (pCtx->cs.u32Limit & 0xfff) == 0xfff
9454	\|\| !(pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
9455	HMVMX_CHECK_BREAK( !(pCtx->cs.u32Limit & 0xfff00000)
9456	\|\| (pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
9457	/* CS cannot be loaded with NULL in protected mode. */
9458	HMVMX_CHECK_BREAK(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_CS_ATTR_UNUSABLE);
9459	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1DescType, VMX_IGS_CS_ATTR_S_INVALID);
9460	if (pCtx->cs.Attr.n.u4Type == 9 \|\| pCtx->cs.Attr.n.u4Type == 11)
9461	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_UNEQUAL);
9462	else if (pCtx->cs.Attr.n.u4Type == 13 \|\| pCtx->cs.Attr.n.u4Type == 15)
9463	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_MISMATCH);
9464	else if (pVM->hm.s.vmx.fUnrestrictedGuest && pCtx->cs.Attr.n.u4Type == 3)
9465	HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == 0, VMX_IGS_CS_ATTR_DPL_INVALID);
9466	else
9467	HMVMX_ERROR_BREAK(VMX_IGS_CS_ATTR_TYPE_INVALID);
9468
9469	/* SS */
9470	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9471	\|\| (pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL), VMX_IGS_SS_CS_RPL_UNEQUAL);
9472	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL), VMX_IGS_SS_ATTR_DPL_RPL_UNEQUAL);
9473	if ( !(pCtx->cr0 & X86_CR0_PE)
9474	\|\| pCtx->cs.Attr.n.u4Type == 3)
9475	{
9476	HMVMX_CHECK_BREAK(!pCtx->ss.Attr.n.u2Dpl, VMX_IGS_SS_ATTR_DPL_INVALID);
9477	}
9478	if (!(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
9479	{
9480	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u4Type == 3 \|\| pCtx->ss.Attr.n.u4Type == 7, VMX_IGS_SS_ATTR_TYPE_INVALID);
9481	HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u1Present, VMX_IGS_SS_ATTR_P_INVALID);
9482	HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xf00), VMX_IGS_SS_ATTR_RESERVED);
9483	HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xfffe0000), VMX_IGS_SS_ATTR_RESERVED);
9484	HMVMX_CHECK_BREAK( (pCtx->ss.u32Limit & 0xfff) == 0xfff
9485	\|\| !(pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
9486	HMVMX_CHECK_BREAK( !(pCtx->ss.u32Limit & 0xfff00000)
9487	\|\| (pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
9488	}
9489
9490	/* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxExportGuestSReg(). */
9491	if (!(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
9492	{
9493	HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_DS_ATTR_A_INVALID);
9494	HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u1Present, VMX_IGS_DS_ATTR_P_INVALID);
9495	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9496	\|\| pCtx->ds.Attr.n.u4Type > 11
9497	\|\| pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
9498	HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xf00), VMX_IGS_DS_ATTR_RESERVED);
9499	HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xfffe0000), VMX_IGS_DS_ATTR_RESERVED);
9500	HMVMX_CHECK_BREAK( (pCtx->ds.u32Limit & 0xfff) == 0xfff
9501	\|\| !(pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
9502	HMVMX_CHECK_BREAK( !(pCtx->ds.u32Limit & 0xfff00000)
9503	\|\| (pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
9504	HMVMX_CHECK_BREAK( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9505	\|\| (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_DS_ATTR_TYPE_INVALID);
9506	}
9507	if (!(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
9508	{
9509	HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_ES_ATTR_A_INVALID);
9510	HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u1Present, VMX_IGS_ES_ATTR_P_INVALID);
9511	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9512	\|\| pCtx->es.Attr.n.u4Type > 11
9513	\|\| pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
9514	HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xf00), VMX_IGS_ES_ATTR_RESERVED);
9515	HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xfffe0000), VMX_IGS_ES_ATTR_RESERVED);
9516	HMVMX_CHECK_BREAK( (pCtx->es.u32Limit & 0xfff) == 0xfff
9517	\|\| !(pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
9518	HMVMX_CHECK_BREAK( !(pCtx->es.u32Limit & 0xfff00000)
9519	\|\| (pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
9520	HMVMX_CHECK_BREAK( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9521	\|\| (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_ES_ATTR_TYPE_INVALID);
9522	}
9523	if (!(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
9524	{
9525	HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_FS_ATTR_A_INVALID);
9526	HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u1Present, VMX_IGS_FS_ATTR_P_INVALID);
9527	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9528	\|\| pCtx->fs.Attr.n.u4Type > 11
9529	\|\| pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL), VMX_IGS_FS_ATTR_DPL_RPL_UNEQUAL);
9530	HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xf00), VMX_IGS_FS_ATTR_RESERVED);
9531	HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xfffe0000), VMX_IGS_FS_ATTR_RESERVED);
9532	HMVMX_CHECK_BREAK( (pCtx->fs.u32Limit & 0xfff) == 0xfff
9533	\|\| !(pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
9534	HMVMX_CHECK_BREAK( !(pCtx->fs.u32Limit & 0xfff00000)
9535	\|\| (pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
9536	HMVMX_CHECK_BREAK( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9537	\|\| (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_FS_ATTR_TYPE_INVALID);
9538	}
9539	if (!(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
9540	{
9541	HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_GS_ATTR_A_INVALID);
9542	HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u1Present, VMX_IGS_GS_ATTR_P_INVALID);
9543	HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9544	\|\| pCtx->gs.Attr.n.u4Type > 11
9545	\|\| pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL), VMX_IGS_GS_ATTR_DPL_RPL_UNEQUAL);
9546	HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xf00), VMX_IGS_GS_ATTR_RESERVED);
9547	HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xfffe0000), VMX_IGS_GS_ATTR_RESERVED);
9548	HMVMX_CHECK_BREAK( (pCtx->gs.u32Limit & 0xfff) == 0xfff
9549	\|\| !(pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
9550	HMVMX_CHECK_BREAK( !(pCtx->gs.u32Limit & 0xfff00000)
9551	\|\| (pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
9552	HMVMX_CHECK_BREAK( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9553	\|\| (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_GS_ATTR_TYPE_INVALID);
9554	}
9555	/* 64-bit capable CPUs. */
9556	#if HC_ARCH_BITS == 64
9557	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
9558	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
9559	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
9560	\|\| X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
9561	HMVMX_CHECK_BREAK(!RT_HI_U32(pCtx->cs.u64Base), VMX_IGS_LONGMODE_CS_BASE_INVALID);
9562	HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->ss.u64Base),
9563	VMX_IGS_LONGMODE_SS_BASE_INVALID);
9564	HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->ds.u64Base),
9565	VMX_IGS_LONGMODE_DS_BASE_INVALID);
9566	HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->es.u64Base),
9567	VMX_IGS_LONGMODE_ES_BASE_INVALID);
9568	#endif
9569	}
9570	else
9571	{
9572	/* V86 mode checks. */
9573	uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
9574	if (pVmcsInfo->RealMode.fRealOnV86Active)
9575	{
9576	u32CSAttr = 0xf3; u32SSAttr = 0xf3;
9577	u32DSAttr = 0xf3; u32ESAttr = 0xf3;
9578	u32FSAttr = 0xf3; u32GSAttr = 0xf3;
9579	}
9580	else
9581	{
9582	u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u;
9583	u32DSAttr = pCtx->ds.Attr.u; u32ESAttr = pCtx->es.Attr.u;
9584	u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
9585	}
9586
9587	/* CS */
9588	HMVMX_CHECK_BREAK((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), VMX_IGS_V86_CS_BASE_INVALID);
9589	HMVMX_CHECK_BREAK(pCtx->cs.u32Limit == 0xffff, VMX_IGS_V86_CS_LIMIT_INVALID);
9590	HMVMX_CHECK_BREAK(u32CSAttr == 0xf3, VMX_IGS_V86_CS_ATTR_INVALID);
9591	/* SS */
9592	HMVMX_CHECK_BREAK((pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4), VMX_IGS_V86_SS_BASE_INVALID);
9593	HMVMX_CHECK_BREAK(pCtx->ss.u32Limit == 0xffff, VMX_IGS_V86_SS_LIMIT_INVALID);
9594	HMVMX_CHECK_BREAK(u32SSAttr == 0xf3, VMX_IGS_V86_SS_ATTR_INVALID);
9595	/* DS */
9596	HMVMX_CHECK_BREAK((pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4), VMX_IGS_V86_DS_BASE_INVALID);
9597	HMVMX_CHECK_BREAK(pCtx->ds.u32Limit == 0xffff, VMX_IGS_V86_DS_LIMIT_INVALID);
9598	HMVMX_CHECK_BREAK(u32DSAttr == 0xf3, VMX_IGS_V86_DS_ATTR_INVALID);
9599	/* ES */
9600	HMVMX_CHECK_BREAK((pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4), VMX_IGS_V86_ES_BASE_INVALID);
9601	HMVMX_CHECK_BREAK(pCtx->es.u32Limit == 0xffff, VMX_IGS_V86_ES_LIMIT_INVALID);
9602	HMVMX_CHECK_BREAK(u32ESAttr == 0xf3, VMX_IGS_V86_ES_ATTR_INVALID);
9603	/* FS */
9604	HMVMX_CHECK_BREAK((pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4), VMX_IGS_V86_FS_BASE_INVALID);
9605	HMVMX_CHECK_BREAK(pCtx->fs.u32Limit == 0xffff, VMX_IGS_V86_FS_LIMIT_INVALID);
9606	HMVMX_CHECK_BREAK(u32FSAttr == 0xf3, VMX_IGS_V86_FS_ATTR_INVALID);
9607	/* GS */
9608	HMVMX_CHECK_BREAK((pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4), VMX_IGS_V86_GS_BASE_INVALID);
9609	HMVMX_CHECK_BREAK(pCtx->gs.u32Limit == 0xffff, VMX_IGS_V86_GS_LIMIT_INVALID);
9610	HMVMX_CHECK_BREAK(u32GSAttr == 0xf3, VMX_IGS_V86_GS_ATTR_INVALID);
9611	/* 64-bit capable CPUs. */
9612	#if HC_ARCH_BITS == 64
9613	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
9614	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
9615	HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
9616	\|\| X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
9617	HMVMX_CHECK_BREAK(!RT_HI_U32(pCtx->cs.u64Base), VMX_IGS_LONGMODE_CS_BASE_INVALID);
9618	HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->ss.u64Base),
9619	VMX_IGS_LONGMODE_SS_BASE_INVALID);
9620	HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->ds.u64Base),
9621	VMX_IGS_LONGMODE_DS_BASE_INVALID);
9622	HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) \|\| !RT_HI_U32(pCtx->es.u64Base),
9623	VMX_IGS_LONGMODE_ES_BASE_INVALID);
9624	#endif
9625	}
9626
9627	/*
9628	* TR.
9629	*/
9630	HMVMX_CHECK_BREAK(!(pCtx->tr.Sel & X86_SEL_LDT), VMX_IGS_TR_TI_INVALID);
9631	/* 64-bit capable CPUs. */
9632	#if HC_ARCH_BITS == 64
9633	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->tr.u64Base), VMX_IGS_TR_BASE_NOT_CANONICAL);
9634	#endif
9635	if (fLongModeGuest)
9636	{
9637	HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u4Type == 11, /* 64-bit busy TSS. */
9638	VMX_IGS_LONGMODE_TR_ATTR_TYPE_INVALID);
9639	}
9640	else
9641	{
9642	HMVMX_CHECK_BREAK( pCtx->tr.Attr.n.u4Type == 3 /* 16-bit busy TSS. */
9643	\|\| pCtx->tr.Attr.n.u4Type == 11, /* 32-bit busy TSS.*/
9644	VMX_IGS_TR_ATTR_TYPE_INVALID);
9645	}
9646	HMVMX_CHECK_BREAK(!pCtx->tr.Attr.n.u1DescType, VMX_IGS_TR_ATTR_S_INVALID);
9647	HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u1Present, VMX_IGS_TR_ATTR_P_INVALID);
9648	HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & 0xf00), VMX_IGS_TR_ATTR_RESERVED); /* Bits 11:8 MBZ. */
9649	HMVMX_CHECK_BREAK( (pCtx->tr.u32Limit & 0xfff) == 0xfff
9650	\|\| !(pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
9651	HMVMX_CHECK_BREAK( !(pCtx->tr.u32Limit & 0xfff00000)
9652	\|\| (pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
9653	HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_TR_ATTR_UNUSABLE);
9654
9655	/*
9656	* GDTR and IDTR.
9657	*/
9658	#if HC_ARCH_BITS == 64
9659	rc = VMXReadVmcs64(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
9660	AssertRCBreak(rc);
9661	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_GDTR_BASE_NOT_CANONICAL);
9662
9663	rc = VMXReadVmcs64(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
9664	AssertRCBreak(rc);
9665	HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_IDTR_BASE_NOT_CANONICAL);
9666	#endif
9667
9668	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
9669	AssertRCBreak(rc);
9670	HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_GDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
9671
9672	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
9673	AssertRCBreak(rc);
9674	HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_IDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
9675
9676	/*
9677	* Guest Non-Register State.
9678	*/
9679	/* Activity State. */
9680	uint32_t u32ActivityState;
9681	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, &u32ActivityState);
9682	AssertRCBreak(rc);
9683	HMVMX_CHECK_BREAK( !u32ActivityState
9684	\|\| (u32ActivityState & RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Misc, VMX_BF_MISC_ACTIVITY_STATES)),
9685	VMX_IGS_ACTIVITY_STATE_INVALID);
9686	HMVMX_CHECK_BREAK( !(pCtx->ss.Attr.n.u2Dpl)
9687	\|\| u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_HLT, VMX_IGS_ACTIVITY_STATE_HLT_INVALID);
9688	uint32_t u32IntrState;
9689	rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &u32IntrState);
9690	AssertRCBreak(rc);
9691	if ( u32IntrState == VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS
9692	\|\| u32IntrState == VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
9693	{
9694	HMVMX_CHECK_BREAK(u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_ACTIVE, VMX_IGS_ACTIVITY_STATE_ACTIVE_INVALID);
9695	}
9696
9697	/** @todo Activity state and injecting interrupts. Left as a todo since we
9698	* currently don't use activity states but ACTIVE. */
9699
9700	HMVMX_CHECK_BREAK( !(pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_ENTRY_TO_SMM)
9701	\|\| u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_SIPI_WAIT, VMX_IGS_ACTIVITY_STATE_SIPI_WAIT_INVALID);
9702
9703	/* Guest interruptibility-state. */
9704	HMVMX_CHECK_BREAK(!(u32IntrState & 0xffffffe0), VMX_IGS_INTERRUPTIBILITY_STATE_RESERVED);
9705	HMVMX_CHECK_BREAK((u32IntrState & (VMX_VMCS_GUEST_INT_STATE_BLOCK_STI \| VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS))
9706	!= (VMX_VMCS_GUEST_INT_STATE_BLOCK_STI \| VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS),
9707	VMX_IGS_INTERRUPTIBILITY_STATE_STI_MOVSS_INVALID);
9708	HMVMX_CHECK_BREAK( (u32Eflags & X86_EFL_IF)
9709	\|\| !(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI),
9710	VMX_IGS_INTERRUPTIBILITY_STATE_STI_EFL_INVALID);
9711	if (VMX_ENTRY_INT_INFO_IS_VALID(u32EntryInfo))
9712	{
9713	if (VMX_ENTRY_INT_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INT_INFO_TYPE_EXT_INT)
9714	{
9715	HMVMX_CHECK_BREAK( !(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
9716	&& !(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS),
9717	VMX_IGS_INTERRUPTIBILITY_STATE_EXT_INT_INVALID);
9718	}
9719	else if (VMX_ENTRY_INT_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INT_INFO_TYPE_NMI)
9720	{
9721	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS),
9722	VMX_IGS_INTERRUPTIBILITY_STATE_MOVSS_INVALID);
9723	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI),
9724	VMX_IGS_INTERRUPTIBILITY_STATE_STI_INVALID);
9725	}
9726	}
9727	/** @todo Assumes the processor is not in SMM. */
9728	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI),
9729	VMX_IGS_INTERRUPTIBILITY_STATE_SMI_INVALID);
9730	HMVMX_CHECK_BREAK( !(pVmcsInfo->u32EntryCtls & VMX_ENTRY_CTLS_ENTRY_TO_SMM)
9731	\|\| (u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI),
9732	VMX_IGS_INTERRUPTIBILITY_STATE_SMI_SMM_INVALID);
9733	if ( (pVmcsInfo->u32PinCtls & VMX_PIN_CTLS_VIRT_NMI)
9734	&& VMX_ENTRY_INT_INFO_IS_VALID(u32EntryInfo)
9735	&& VMX_ENTRY_INT_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INT_INFO_TYPE_NMI)
9736	{
9737	HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI),
9738	VMX_IGS_INTERRUPTIBILITY_STATE_NMI_INVALID);
9739	}
9740
9741	/* Pending debug exceptions. */
9742	#if HC_ARCH_BITS == 64
9743	rc = VMXReadVmcs64(VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, &u64Val);
9744	AssertRCBreak(rc);
9745	/* Bits 63:15, Bit 13, Bits 11:4 MBZ. */
9746	HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffffaff0)), VMX_IGS_LONGMODE_PENDING_DEBUG_RESERVED);
9747	u32Val = u64Val; /* For pending debug exceptions checks below. */
9748	#else
9749	rc = VMXReadVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, &u32Val);
9750	AssertRCBreak(rc);
9751	/* Bits 31:15, Bit 13, Bits 11:4 MBZ. */
9752	HMVMX_CHECK_BREAK(!(u32Val & 0xffffaff0), VMX_IGS_PENDING_DEBUG_RESERVED);
9753	#endif
9754
9755	if ( (u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
9756	\|\| (u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS)
9757	\|\| u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_HLT)
9758	{
9759	if ( (u32Eflags & X86_EFL_TF)
9760	&& !(u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
9761	{
9762	/* Bit 14 is PendingDebug.BS. */
9763	HMVMX_CHECK_BREAK(u32Val & RT_BIT(14), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_SET);
9764	}
9765	if ( !(u32Eflags & X86_EFL_TF)
9766	\|\| (u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
9767	{
9768	/* Bit 14 is PendingDebug.BS. */
9769	HMVMX_CHECK_BREAK(!(u32Val & RT_BIT(14)), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_CLEAR);
9770	}
9771	}
9772
9773	/* VMCS link pointer. */
9774	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, &u64Val);
9775	AssertRCBreak(rc);
9776	if (u64Val != UINT64_C(0xffffffffffffffff))
9777	{
9778	HMVMX_CHECK_BREAK(!(u64Val & 0xfff), VMX_IGS_VMCS_LINK_PTR_RESERVED);
9779	/** @todo Bits beyond the processor's physical-address width MBZ. */
9780	/** @todo 32-bit located in memory referenced by value of this field (as a
9781	* physical address) must contain the processor's VMCS revision ID. */
9782	/** @todo SMM checks. */
9783	}
9784
9785	/** @todo Checks on Guest Page-Directory-Pointer-Table Entries when guest is
9786	* not using nested paging? */
9787	if ( pVM->hm.s.fNestedPaging
9788	&& !fLongModeGuest
9789	&& CPUMIsGuestInPAEModeEx(pCtx))
9790	{
9791	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &u64Val);
9792	AssertRCBreak(rc);
9793	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9794
9795	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &u64Val);
9796	AssertRCBreak(rc);
9797	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9798
9799	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &u64Val);
9800	AssertRCBreak(rc);
9801	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9802
9803	rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &u64Val);
9804	AssertRCBreak(rc);
9805	HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9806	}
9807
9808	/* Shouldn't happen but distinguish it from AssertRCBreak() errors. */
9809	if (uError == VMX_IGS_ERROR)
9810	uError = VMX_IGS_REASON_NOT_FOUND;
9811	} while (0);
9812
9813	pVCpu->hm.s.u32HMError = uError;
9814	return uError;
9815
9816	#undef HMVMX_ERROR_BREAK
9817	#undef HMVMX_CHECK_BREAK
9818	}
9819
9820
9821	/**
9822	* Setup the APIC-access page for virtualizing APIC access.
9823	*
9824	* This can cause a longjumps to R3 due to the acquisition of the PGM lock, hence
9825	* this not done as part of exporting guest state, see @bugref{8721}.
9826	*
9827	* @returns VBox status code.
9828	* @param pVCpu The cross context virtual CPU structure.
9829	*/
9830	static int hmR0VmxMapHCApicAccessPage(PVMCPU pVCpu)
9831	{
9832	PVM pVM = pVCpu->CTX_SUFF(pVM);
9833	uint64_t const u64MsrApicBase = APICGetBaseMsrNoCheck(pVCpu);
9834
9835	Assert(PDMHasApic(pVM));
9836	Assert(u64MsrApicBase);
9837
9838	RTGCPHYS const GCPhysApicBase = u64MsrApicBase & PAGE_BASE_GC_MASK;
9839	Log4Func(("Mappping HC APIC-access page at %#RGp\n", GCPhysApicBase));
9840
9841	/* Unalias any existing mapping. */
9842	int rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
9843	AssertRCReturn(rc, rc);
9844
9845	/* Map the HC APIC-access page in place of the MMIO page, also updates the shadow page tables if necessary. */
9846	Assert(pVM->hm.s.vmx.HCPhysApicAccess != NIL_RTHCPHYS);
9847	rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW \| X86_PTE_P);
9848	AssertRCReturn(rc, rc);
9849
9850	/* Update the per-VCPU cache of the APIC base MSR. */
9851	pVCpu->hm.s.vmx.u64GstMsrApicBase = u64MsrApicBase;
9852	return VINF_SUCCESS;
9853	}
9854
9855
9856	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
9857	/**
9858	* Merges the guest with the nested-guest MSR bitmap in preparation of executing the
9859	* nested-guest using hardware-assisted VMX.
9860	*
9861	* @param pVCpu The cross context virtual CPU structure.
9862	* @param pVmcsInfoNstGst The nested-guest VMCS info. object.
9863	* @param pVmcsInfoGst The guest VMCS info. object.
9864	*/
9865	static void hmR0VmxMergeMsrBitmapNested(PVMCPU pVCpu, PVMXVMCSINFO pVmcsInfoNstGst, PCVMXVMCSINFO pVmcsInfoGst)
9866	{
9867	uint64_t const pu64MsrBitmapNstGst = (uint64_t const )pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pvMsrBitmap);
9868	uint64_t const pu64MsrBitmapGst = (uint64_t const )pVmcsInfoGst->pvMsrBitmap;
9869	uint64_t pu64MsrBitmap = (uint64_t )pVmcsInfoNstGst->pvMsrBitmap;
9870	Assert(pu64MsrBitmapNstGst);
9871	Assert(pu64MsrBitmapGst);
9872	Assert(pu64MsrBitmap);
9873
9874	/*
9875	* We merge the guest MSR bitmap with the nested-guest MSR bitmap such that any
9876	* MSR that is intercepted by the guest is also intercepted while executing the
9877	* nested-guest using hardware-assisted VMX.
9878	*/
9879	uint32_t const cbFrag = sizeof(uint64_t);
9880	uint32_t const cFrags = X86_PAGE_4K_SIZE / cbFrag;
9881	for (uint32_t i = 0; i <= cFrags; i++)
9882	pu64MsrBitmap[i] = pu64MsrBitmapNstGst[i] \| pu64MsrBitmapGst[i];
9883	}
9884
9885
9886	/**
9887	* Merges the guest VMCS in to the nested-guest VMCS controls in preparation of
9888	* hardware-assisted VMX execution of the nested-guest.
9889	*
9890	* For a guest, we don't modify these controls once we set up the VMCS.
9891	*
9892	* For nested-guests since the guest hypervisor provides these controls on every
9893	* nested-guest VM-entry and could potentially change them everytime we need to
9894	* merge them before every nested-guest VM-entry.
9895	*
9896	* @returns VBox status code.
9897	* @param pVCpu The cross context virtual CPU structure.
9898	*/
9899	static int hmR0VmxMergeVmcsNested(PVMCPU pVCpu)
9900	{
9901	PVM pVM = pVCpu->CTX_SUFF(pVM);
9902	PCVMXVMCSINFO pVmcsInfoGst = &pVCpu->hm.s.vmx.VmcsInfo;
9903	PCVMXVVMCS pVmcsNstGst = pVCpu->cpum.GstCtx.hwvirt.vmx.CTX_SUFF(pVmcs);
9904	Assert(pVmcsNstGst);
9905
9906	/*
9907	* Merge the controls with the requirements of the guest VMCS.
9908	*
9909	* We do not need to validate the nested-guest VMX features specified in the
9910	* nested-guest VMCS with the features supported by the physical CPU as it's
9911	* already done by the VMLAUNCH/VMRESUME instruction emulation.
9912	*
9913	* This is because the VMX features exposed by CPUM (through CPUID/MSRs) to the
9914	* guest are derived from the VMX features supported by the physical CPU.
9915	*/
9916
9917	/* Pin-based VM-execution controls. */
9918	uint32_t const u32PinCtls = pVmcsNstGst->u32PinCtls \| pVmcsInfoGst->u32PinCtls;
9919
9920	/* Processor-based VM-execution controls. */
9921	uint32_t u32ProcCtls = (pVmcsNstGst->u32ProcCtls & ~VMX_PROC_CTLS_USE_IO_BITMAPS)
9922	\| (pVmcsInfoGst->u32ProcCtls & ~( VMX_PROC_CTLS_INT_WINDOW_EXIT
9923	\| VMX_PROC_CTLS_NMI_WINDOW_EXIT
9924	\| VMX_PROC_CTLS_USE_TPR_SHADOW
9925	\| VMX_PROC_CTLS_MONITOR_TRAP_FLAG));
9926
9927	/* Secondary processor-based VM-execution controls. */
9928	uint32_t const u32ProcCtls2 = (pVmcsNstGst->u32ProcCtls2 & ~VMX_PROC_CTLS2_VPID)
9929	\| (pVmcsInfoGst->u32ProcCtls2 & ~( VMX_PROC_CTLS2_VIRT_APIC_ACCESS
9930	\| VMX_PROC_CTLS2_INVPCID
9931	\| VMX_PROC_CTLS2_RDTSCP
9932	\| VMX_PROC_CTLS2_XSAVES_XRSTORS
9933	\| VMX_PROC_CTLS2_APIC_REG_VIRT
9934	\| VMX_PROC_CTLS2_VIRT_INT_DELIVERY
9935	\| VMX_PROC_CTLS2_VMFUNC));
9936
9937	/*
9938	* VM-entry controls:
9939	* These controls contains state that depends on the nested-guest state (primarily
9940	* EFER MSR) and is thus not constant through VMLAUNCH/VMRESUME and the nested-guest
9941	* VM-exit. Although the nested-hypervisor cannot change it, we need to in order to
9942	* properly continue executing the nested-guest if the EFER MSR changes but does not
9943	* cause a nested-guest VM-exits.
9944	*
9945	* VM-exit controls:
9946	* These controls specify the host state on return. We cannot use the controls from
9947	* the nested-hypervisor state as is as it would contain the guest state rather than
9948	* the host state. Since the host state is subject to change (e.g. preemption, trips
9949	* to ring-3, longjmp and rescheduling to a different host CPU) they are not constant
9950	* through VMLAUNCH/VMRESUME and the nested-guest VM-exit.
9951	*
9952	* VM-entry MSR-load:
9953	* The guest MSRs from the VM-entry MSR-load area are already loaded into the
9954	* guest-CPU context by the VMLAUNCH/VMRESUME instruction emulation.
9955	*
9956	* VM-exit MSR-store:
9957	* The VM-exit emulation will take care of populating the MSRs from the guest-CPU
9958	* context back into the VM-exit MSR-store area.
9959	*
9960	* VM-exit MSR-load areas:
9961	* This must contain the real host MSRs with hardware-assisted VMX execution. Hence,
9962	* we can entirely ignore what the nested-hypervisor wants to load here.
9963	*/
9964
9965	/*
9966	* Exception bitmap.
9967	*
9968	* We could remove #UD from the guest bitmap and merge it with the nested-guest
9969	* bitmap here (and avoid doing anything while exporting nested-guest state), but to
9970	* keep the code more flexible if intercepting exceptions become more dynamic in
9971	* the future we do it as part of exporting the nested-guest state.
9972	*/
9973	uint32_t const u32XcptBitmap = pVmcsNstGst->u32XcptBitmap \| pVmcsInfoGst->u32XcptBitmap;
9974
9975	/*
9976	* CR0/CR4 guest/host mask.
9977	*
9978	* Modifications by the nested-guest to CR0/CR4 bits owned by the host and the guest
9979	* must cause VM-exits, so we need to merge them here.
9980	*/
9981	uint64_t const u64Cr0Mask = pVmcsNstGst->u64Cr0Mask.u \| pVmcsInfoGst->u64Cr0Mask;
9982	uint64_t const u64Cr4Mask = pVmcsNstGst->u64Cr4Mask.u \| pVmcsInfoGst->u64Cr4Mask;
9983
9984	/*
9985	* Page-fault error-code mask and match.
9986	*
9987	* Although we require unrestricted guest execution (and thereby nested-paging) for
9988	* hardware-assisted VMX execution of nested-guests and thus the outer guest doesn't
9989	* normally intercept #PFs, it might intercept them for debugging purposes.
9990	*
9991	* If the outer guest is not intercepting #PFs, we can use the nested-guest #PF
9992	* filters. If the outer guest is intercepting #PFs we must intercept all #PFs.
9993	*/
9994	uint32_t u32XcptPFMask;
9995	uint32_t u32XcptPFMatch;
9996	if (!(pVmcsInfoGst->u32XcptBitmap & RT_BIT(X86_XCPT_PF)))
9997	{
9998	u32XcptPFMask = pVmcsNstGst->u32XcptPFMask;
9999	u32XcptPFMatch = pVmcsNstGst->u32XcptPFMatch;
10000	}
10001	else
10002	{
10003	u32XcptPFMask = 0;
10004	u32XcptPFMatch = 0;
10005	}
10006
10007	/*
10008	* Pause-Loop exiting.
10009	*/
10010	uint32_t const cPleGapTicks = RT_MIN(pVM->hm.s.vmx.cPleGapTicks, pVmcsNstGst->u32PleGap);
10011	uint32_t const cPleWindowTicks = RT_MIN(pVM->hm.s.vmx.cPleWindowTicks, pVmcsNstGst->u32PleWindow);
10012
10013	/*
10014	* I/O Bitmap.
10015	*
10016	* We do not use the I/O bitmap that may be provided by the guest hypervisor as we
10017	* always intercept all I/O port accesses.
10018	*/
10019	Assert(u32ProcCtls & VMX_PROC_CTLS_UNCOND_IO_EXIT);
10020
10021	/*
10022	* APIC-access page.
10023	*
10024	* The APIC-access page address has already been initialized while setting up the
10025	* nested-guest VMCS. In theory, even if the guest-physical address is invalid, it
10026	* should not be on any consequence to the host or to the guest for that matter, but
10027	* we only accept valid addresses verified by the VMLAUNCH/VMRESUME instruction
10028	* emulation to keep it simple.
10029	*/
10030
10031	/*
10032	* Virtual-APIC page and TPR threshold.
10033	*
10034	* We shall use the host-physical address of the virtual-APIC page in guest memory directly.
10035	* For this reason, we can access the virtual-APIC page of the nested-guest only using
10036	* PGM physical handlers as we must not assume a kernel virtual-address mapping exists and
10037	* requesting PGM for a mapping could be expensive/resource intensive (PGM mapping cache).
10038	*/
10039	RTHCPHYS HCPhysVirtApic = NIL_RTHCPHYS;
10040	uint32_t const u32TprThreshold = pVmcsNstGst->u32TprThreshold;
10041	if (u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
10042	{
10043	int rc = PGMPhysGCPhys2HCPhys(pVM, pVmcsNstGst->u64AddrVirtApic.u, &HCPhysVirtApic);
10044
10045	/*
10046	* If the guest hypervisor has loaded crap into the virtual-APIC page field
10047	* we would fail to obtain a valid host-physical address for its guest-physical
10048	* address.
10049	*
10050	* We currently do not support this scenario. Maybe in the future if there is a
10051	* pressing need we can explore making this particular set of conditions work.
10052	* Right now we just cause a VM-entry failure.
10053	*
10054	* This has already been checked by VMLAUNCH/VMRESUME instruction emulation,
10055	* so should not really failure at the moment.
10056	*/
10057	AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
10058	}
10059	else
10060	{
10061	/*
10062	* We must make sure CR8 reads/write must cause VM-exits when TPR shadowing is not
10063	* used by the guest hypervisor. Preventing MMIO accesses to the physical APIC will
10064	* be taken care of by EPT/shadow paging.
10065	*/
10066	if (pVM->hm.s.fAllow64BitGuests)
10067	{
10068	u32ProcCtls \|= VMX_PROC_CTLS_CR8_STORE_EXIT
10069	\| VMX_PROC_CTLS_CR8_LOAD_EXIT;
10070	}
10071	}
10072
10073	/*
10074	* Validate basic assumptions.
10075	*/
10076	PVMXVMCSINFO pVmcsInfoNstGst = &pVCpu->hm.s.vmx.VmcsInfoNstGst;
10077	Assert(pVM->hm.s.vmx.fAllowUnrestricted);
10078	Assert(pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_SECONDARY_CTLS);
10079	Assert(hmGetVmxActiveVmcsInfo(pVCpu) == pVmcsInfoNstGst);
10080
10081	/*
10082	* Commit it to the nested-guest VMCS.
10083	*/
10084	int rc = VINF_SUCCESS;
10085	if (pVmcsInfoNstGst->u32PinCtls != u32PinCtls)
10086	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, u32PinCtls);
10087	if (pVmcsInfoNstGst->u32ProcCtls != u32ProcCtls)
10088	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, u32ProcCtls);
10089	if (pVmcsInfoNstGst->u32ProcCtls2 != u32ProcCtls2)
10090	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, u32ProcCtls2);
10091	if (pVmcsInfoNstGst->u32XcptBitmap != u32XcptBitmap)
10092	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
10093	if (pVmcsInfoNstGst->u64Cr0Mask != u64Cr0Mask)
10094	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, u64Cr0Mask);
10095	if (pVmcsInfoNstGst->u64Cr4Mask != u64Cr4Mask)
10096	rc \|= VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, u64Cr4Mask);
10097	if (pVmcsInfoNstGst->u32XcptPFMask != u32XcptPFMask)
10098	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, u32XcptPFMask);
10099	if (pVmcsInfoNstGst->u32XcptPFMatch != u32XcptPFMatch)
10100	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, u32XcptPFMatch);
10101	if ( !(u32ProcCtls & VMX_PROC_CTLS_PAUSE_EXIT)
10102	&& (u32ProcCtls2 & VMX_PROC_CTLS2_PAUSE_LOOP_EXIT))
10103	{
10104	Assert(pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_PAUSE_LOOP_EXIT);
10105	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_GAP, cPleGapTicks);
10106	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_WINDOW, cPleWindowTicks);
10107	}
10108	if (u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
10109	{
10110	rc \|= VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
10111	rc \|= VMXWriteVmcs64(VMX_VMCS64_CTRL_VIRT_APIC_PAGEADDR_FULL, HCPhysVirtApic);
10112	}
10113	AssertRCReturn(rc, rc);
10114
10115	/*
10116	* Update the nested-guest VMCS cache.
10117	*/
10118	pVmcsInfoNstGst->u32PinCtls = u32PinCtls;
10119	pVmcsInfoNstGst->u32ProcCtls = u32ProcCtls;
10120	pVmcsInfoNstGst->u32ProcCtls2 = u32ProcCtls2;
10121	pVmcsInfoNstGst->u32XcptBitmap = u32XcptBitmap;
10122	pVmcsInfoNstGst->u64Cr0Mask = u64Cr0Mask;
10123	pVmcsInfoNstGst->u64Cr4Mask = u64Cr4Mask;
10124	pVmcsInfoNstGst->u32XcptPFMask = u32XcptPFMask;
10125	pVmcsInfoNstGst->u32XcptPFMatch = u32XcptPFMatch;
10126	pVmcsInfoNstGst->HCPhysVirtApic = HCPhysVirtApic;
10127
10128	/*
10129	* MSR bitmap.
10130	*
10131	* The MSR bitmap address has already been initialized while setting up the
10132	* nested-guest VMCS, here we need to merge the MSR bitmaps.
10133	*/
10134	if (u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
10135	hmR0VmxMergeMsrBitmapNested(pVCpu, pVmcsInfoNstGst, pVmcsInfoGst);
10136
10137	return VINF_SUCCESS;
10138	}
10139	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
10140
10141
10142	/**
10143	* Does the preparations before executing guest code in VT-x.
10144	*
10145	* This may cause longjmps to ring-3 and may even result in rescheduling to the
10146	* recompiler/IEM. We must be cautious what we do here regarding committing
10147	* guest-state information into the VMCS assuming we assuredly execute the
10148	* guest in VT-x mode.
10149	*
10150	* If we fall back to the recompiler/IEM after updating the VMCS and clearing
10151	* the common-state (TRPM/forceflags), we must undo those changes so that the
10152	* recompiler/IEM can (and should) use them when it resumes guest execution.
10153	* Otherwise such operations must be done when we can no longer exit to ring-3.
10154	*
10155	* @returns Strict VBox status code (i.e. informational status codes too).
10156	* @retval VINF_SUCCESS if we can proceed with running the guest, interrupts
10157	* have been disabled.
10158	* @retval VINF_EM_RESET if a triple-fault occurs while injecting a
10159	* double-fault into the guest.
10160	* @retval VINF_EM_DBG_STEPPED if @a fStepping is true and an event was
10161	* dispatched directly.
10162	* @retval VINF_* scheduling changes, we have to go back to ring-3.
10163	*
10164	* @param pVCpu The cross context virtual CPU structure.
10165	* @param pVmxTransient The VMX-transient structure.
10166	* @param fStepping Whether we are single-stepping the guest in the
10167	* hypervisor debugger. Makes us ignore some of the reasons
10168	* for returning to ring-3, and return VINF_EM_DBG_STEPPED
10169	* if event dispatching took place.
10170	*/
10171	static VBOXSTRICTRC hmR0VmxPreRunGuest(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, bool fStepping)
10172	{
10173	Assert(VMMRZCallRing3IsEnabled(pVCpu));
10174
10175	#ifdef VBOX_WITH_NESTED_HWVIRT_ONLY_IN_IEM
10176	if (pVmxTransient->fIsNestedGuest)
10177	{
10178	RT_NOREF2(pVCpu, fStepping);
10179	Log2Func(("Rescheduling to IEM due to nested-hwvirt or forced IEM exec -> VINF_EM_RESCHEDULE_REM\n"));
10180	return VINF_EM_RESCHEDULE_REM;
10181	}
10182	#endif
10183
10184	#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
10185	PGMRZDynMapFlushAutoSet(pVCpu);
10186	#endif
10187
10188	/*
10189	* Check and process force flag actions, some of which might require us to go back to ring-3.
10190	*/
10191	VBOXSTRICTRC rcStrict = hmR0VmxCheckForceFlags(pVCpu, fStepping);
10192	if (rcStrict == VINF_SUCCESS)
10193	{ /* FFs don't get set all the time. */ }
10194	else
10195	return rcStrict;
10196
10197	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
10198	/*
10199	* Switch to the nested-guest VMCS as we may have transitioned into executing
10200	* the nested-guest without leaving ring-0. Otherwise, if we came from ring-3
10201	* we would load the nested-guest VMCS while entering the VMX ring-0 session.
10202	*
10203	* We do this as late as possible to minimize (though not completely remove)
10204	* clearing/loading VMCS again due to premature trips to ring-3 above.
10205	*/
10206	if (pVmxTransient->fIsNestedGuest)
10207	{
10208	if (!pVCpu->hm.s.vmx.fSwitchedToNstGstVmcs)
10209	{
10210	/*
10211	* Ensure we have synced everything from the guest VMCS and also flag that
10212	* that we need to export the full (nested) guest-CPU context to the
10213	* nested-guest VMCS.
10214	*/
10215	HMVMX_CPUMCTX_ASSERT(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
10216	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_HOST_CONTEXT \| HM_CHANGED_ALL_GUEST);
10217
10218	RTCCUINTREG const fEFlags = ASMIntDisableFlags();
10219	int rc = hmR0VmxSwitchVmcs(&pVCpu->hm.s.vmx.VmcsInfo, &pVCpu->hm.s.vmx.VmcsInfoNstGst);
10220	if (RT_LIKELY(rc == VINF_SUCCESS))
10221	{
10222	pVCpu->hm.s.vmx.fSwitchedToNstGstVmcs = true;
10223	ASMSetFlags(fEFlags);
10224	pVmxTransient->pVmcsInfo = &pVCpu->hm.s.vmx.VmcsInfoNstGst;
10225
10226	/*
10227	* We use a different VM-exit MSR-store area for the nested-guest. Hence,
10228	* flag that we need to update the host MSR values there. Even if we decide
10229	* in the future to share the VM-exit MSR-store area page with the guest,
10230	* if its content differs, we would have to update the host MSRs anyway.
10231	*/
10232	pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs = false;
10233	Assert(!pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer); /** @todo NSTVMX: Paranoia remove later. */
10234	}
10235	else
10236	{
10237	ASMSetFlags(fEFlags);
10238	return rc;
10239	}
10240	}
10241
10242	/*
10243	* Merge guest VMCS controls with the nested-guest VMCS controls.
10244	*
10245	* Even if we have not executed the guest prior to this (e.g. when resuming
10246	* from a saved state), we should be okay with merging controls as we
10247	* initialize the guest VMCS controls as part of VM setup phase.
10248	*/
10249	if (!pVCpu->hm.s.vmx.fMergedNstGstCtls)
10250	{
10251	int rc = hmR0VmxMergeVmcsNested(pVCpu);
10252	AssertRCReturn(rc, rc);
10253	pVCpu->hm.s.vmx.fMergedNstGstCtls = true;
10254	}
10255	}
10256	#endif
10257
10258	/*
10259	* Virtualize memory-mapped accesses to the physical APIC (may take locks).
10260	* We look at the guest VMCS control here as we always set it when supported by
10261	* the physical CPU. Looking at the nested-guest control here would not be
10262	* possible because they are not merged yet.
10263	*/
10264	PVM pVM = pVCpu->CTX_SUFF(pVM);
10265	PVMXVMCSINFO pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
10266	if ( !pVCpu->hm.s.vmx.u64GstMsrApicBase
10267	&& (pVmcsInfo->u32ProcCtls2 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
10268	&& PDMHasApic(pVM))
10269	{
10270	int rc = hmR0VmxMapHCApicAccessPage(pVCpu);
10271	AssertRCReturn(rc, rc);
10272	}
10273
10274	/*
10275	* Evaluate events to be injected into the guest.
10276	*
10277	* Events in TRPM can be injected without inspecting the guest state.
10278	* If any new events (interrupts/NMI) are pending currently, we try to set up the
10279	* guest to cause a VM-exit the next time they are ready to receive the event.
10280	*/
10281	if (TRPMHasTrap(pVCpu))
10282	hmR0VmxTrpmTrapToPendingEvent(pVCpu);
10283
10284	uint32_t fIntrState;
10285	rcStrict = hmR0VmxEvaluatePendingEvent(pVCpu, pVmxTransient, &fIntrState);
10286
10287	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
10288	/*
10289	* While evaluating pending events if something failed (unlikely) or if we were
10290	* preparing to run a nested-guest but performed a nested-guest VM-exit, we should bail.
10291	*/
10292	if ( rcStrict != VINF_SUCCESS
10293	\|\| ( pVmxTransient->fIsNestedGuest
10294	&& !CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx)))
10295	return rcStrict;
10296	#endif
10297
10298	/*
10299	* Event injection may take locks (currently the PGM lock for real-on-v86 case) and thus
10300	* needs to be done with longjmps or interrupts + preemption enabled. Event injection might
10301	* also result in triple-faulting the VM.
10302	*
10303	* The above does not apply when executing a nested-guest (since unrestricted guest execution
10304	* is a requirement) regardless doing it avoid duplicating code elsewhere.
10305	*/
10306	rcStrict = hmR0VmxInjectPendingEvent(pVCpu, pVmxTransient, fIntrState, fStepping);
10307	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
10308	{ /* likely */ }
10309	else
10310	{
10311	AssertMsg(rcStrict == VINF_EM_RESET \|\| (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
10312	("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
10313	return rcStrict;
10314	}
10315
10316	/*
10317	* A longjump might result in importing CR3 even for VM-exits that don't necessarily
10318	* import CR3 themselves. We will need to update them here, as even as late as the above
10319	* hmR0VmxInjectPendingEvent() call may lazily import guest-CPU state on demand causing
10320	* the below force flags to be set.
10321	*/
10322	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
10323	{
10324	Assert(!(ASMAtomicUoReadU64(&pVCpu->cpum.GstCtx.fExtrn) & CPUMCTX_EXTRN_CR3));
10325	int rc2 = PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
10326	AssertMsgReturn(rc2 == VINF_SUCCESS \|\| rc2 == VINF_PGM_SYNC_CR3,
10327	("%Rrc\n", rc2), RT_FAILURE_NP(rc2) ? rc2 : VERR_IPE_UNEXPECTED_INFO_STATUS);
10328	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
10329	}
10330	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
10331	{
10332	PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
10333	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
10334	}
10335
10336	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
10337	/* Paranoia. */
10338	Assert(!pVmxTransient->fIsNestedGuest \|\| CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx));
10339	#endif
10340
10341	/*
10342	* No longjmps to ring-3 from this point on!!!
10343	* Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
10344	* This also disables flushing of the R0-logger instance (if any).
10345	*/
10346	VMMRZCallRing3Disable(pVCpu);
10347
10348	/*
10349	* Export the guest state bits.
10350	*
10351	* We cannot perform longjmps while loading the guest state because we do not preserve the
10352	* host/guest state (although the VMCS will be preserved) across longjmps which can cause
10353	* CPU migration.
10354	*
10355	* If we are injecting events to a real-on-v86 mode guest, we would have updated RIP and some segment
10356	* registers. Hence, loading of the guest state needs to be done -after- injection of events.
10357	*/
10358	rcStrict = hmR0VmxExportGuestStateOptimal(pVCpu, pVmxTransient);
10359	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
10360	{ /* likely */ }
10361	else
10362	{
10363	VMMRZCallRing3Enable(pVCpu);
10364	return rcStrict;
10365	}
10366
10367	/*
10368	* We disable interrupts so that we don't miss any interrupts that would flag preemption
10369	* (IPI/timers etc.) when thread-context hooks aren't used and we've been running with
10370	* preemption disabled for a while. Since this is purely to aid the
10371	* RTThreadPreemptIsPending() code, it doesn't matter that it may temporarily reenable and
10372	* disable interrupt on NT.
10373	*
10374	* We need to check for force-flags that could've possible been altered since we last
10375	* checked them (e.g. by PDMGetInterrupt() leaving the PDM critical section,
10376	* see @bugref{6398}).
10377	*
10378	* We also check a couple of other force-flags as a last opportunity to get the EMT back
10379	* to ring-3 before executing guest code.
10380	*/
10381	pVmxTransient->fEFlags = ASMIntDisableFlags();
10382
10383	if ( ( !VM_FF_IS_ANY_SET(pVM, VM_FF_EMT_RENDEZVOUS \| VM_FF_TM_VIRTUAL_SYNC)
10384	&& !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
10385	\|\| ( fStepping /* Optimized for the non-stepping case, so a bit of unnecessary work when stepping. */
10386	&& !VMCPU_FF_IS_ANY_SET(pVCpu, VMCPU_FF_HM_TO_R3_MASK & ~(VMCPU_FF_TIMER \| VMCPU_FF_PDM_CRITSECT))) )
10387	{
10388	if (!RTThreadPreemptIsPending(NIL_RTTHREAD))
10389	{
10390	pVCpu->hm.s.Event.fPending = false;
10391
10392	/*
10393	* We've injected any pending events. This is really the point of no return (to ring-3).
10394	*
10395	* Note! The caller expects to continue with interrupts & longjmps disabled on successful
10396	* returns from this function, so don't enable them here.
10397	*/
10398	return VINF_SUCCESS;
10399	}
10400
10401	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPendingHostIrq);
10402	rcStrict = VINF_EM_RAW_INTERRUPT;
10403	}
10404	else
10405	{
10406	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
10407	rcStrict = VINF_EM_RAW_TO_R3;
10408	}
10409
10410	ASMSetFlags(pVmxTransient->fEFlags);
10411	VMMRZCallRing3Enable(pVCpu);
10412
10413	return rcStrict;
10414	}
10415
10416
10417	/**
10418	* Final preparations before executing guest code using hardware-assisted VMX.
10419	*
10420	* We can no longer get preempted to a different host CPU and there are no returns
10421	* to ring-3. We ignore any errors that may happen from this point (e.g. VMWRITE
10422	* failures), this function is not intended to fail sans unrecoverable hardware
10423	* errors.
10424	*
10425	* @param pVCpu The cross context virtual CPU structure.
10426	* @param pVmxTransient The VMX-transient structure.
10427	*
10428	* @remarks Called with preemption disabled.
10429	* @remarks No-long-jump zone!!!
10430	*/
10431	static void hmR0VmxPreRunGuestCommitted(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
10432	{
10433	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
10434	Assert(VMMR0IsLogFlushDisabled(pVCpu));
10435	Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
10436	Assert(!pVCpu->hm.s.Event.fPending);
10437
10438	/*
10439	* Indicate start of guest execution and where poking EMT out of guest-context is recognized.
10440	*/
10441	VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
10442	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
10443
10444	PVM pVM = pVCpu->CTX_SUFF(pVM);
10445	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
10446
10447	if (!CPUMIsGuestFPUStateActive(pVCpu))
10448	{
10449	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestFpuState, x);
10450	if (CPUMR0LoadGuestFPU(pVM, pVCpu) == VINF_CPUM_HOST_CR0_MODIFIED)
10451	pVCpu->hm.s.fCtxChanged \|= HM_CHANGED_HOST_CONTEXT;
10452	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestFpuState, x);
10453	STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadGuestFpu);
10454	}
10455
10456	/*
10457	* Re-save the host state bits as we may've been preempted (only happens when
10458	* thread-context hooks are used or when the VM start function changes).
10459	* The 64-on-32 switcher saves the (64-bit) host state into the VMCS and if we
10460	* changed the switcher back to 32-bit, we must save the 32-bit host state here,
10461	* see @bugref{8432}.
10462	*
10463	* This may also happen when switching to/from a nested-guest VMCS without leaving
10464	* ring-0.
10465	*/
10466	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT)
10467	{
10468	int rc = hmR0VmxExportHostState(pVCpu);
10469	AssertRC(rc);
10470	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreemptExportHostState);
10471	}
10472	Assert(!(pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT));
10473
10474	/*
10475	* Export the state shared between host and guest (FPU, debug, lazy MSRs).
10476	*/
10477	if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)
10478	hmR0VmxExportSharedState(pVCpu, pVmxTransient);
10479	AssertMsg(!pVCpu->hm.s.fCtxChanged, ("fCtxChanged=%#RX64\n", pVCpu->hm.s.fCtxChanged));
10480
10481	/*
10482	* Store status of the shared guest/host debug state at the time of VM-entry.
10483	*/
10484	#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
10485	if (CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx))
10486	{
10487	pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActivePending(pVCpu);
10488	pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActivePending(pVCpu);
10489	}
10490	else
10491	#endif
10492	{
10493	pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActive(pVCpu);
10494	pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActive(pVCpu);
10495	}
10496
10497	/*
10498	* Always cache the TPR-shadow if the virtual-APIC page exists, thereby skipping
10499	* more than one conditional check. The post-run side of our code shall determine
10500	* if it needs to sync. the virtual APIC TPR with the TPR-shadow.
10501	*/
10502	if (pVmcsInfo->pbVirtApic)
10503	pVmxTransient->u8GuestTpr = pVmcsInfo->pbVirtApic[XAPIC_OFF_TPR];
10504
10505	/*
10506	* Update the host MSRs values in the VM-exit MSR-load area.
10507	*/
10508	if (!pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs)
10509	{
10510	if (pVmcsInfo->cExitMsrLoad > 0)
10511	hmR0VmxUpdateAutoLoadHostMsrs(pVCpu, pVmcsInfo);
10512	pVCpu->hm.s.vmx.fUpdatedHostAutoMsrs = true;
10513	}
10514
10515	/*
10516	* Evaluate if we need to intercept guest RDTSC/P accesses. Set up the
10517	* VMX-preemption timer based on the next virtual sync clock deadline.
10518	*/
10519	PHMPHYSCPU pHostCpu = hmR0GetCurrentCpu();
10520	RTCPUID const idCurrentCpu = pHostCpu->idCpu;
10521	if ( !pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer
10522	\|\| idCurrentCpu != pVCpu->hm.s.idLastCpu)
10523	{
10524	hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVCpu, pVmxTransient);
10525	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = true;
10526	}
10527
10528	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB flushing, set this across the world switch. */
10529	hmR0VmxFlushTaggedTlb(pHostCpu, pVCpu, pVmcsInfo); /* Invalidate the appropriate guest entries from the TLB. */
10530	Assert(idCurrentCpu == pVCpu->hm.s.idLastCpu);
10531	pVCpu->hm.s.vmx.LastError.idCurrentCpu = idCurrentCpu; /* Update the error reporting info. with the current host CPU. */
10532
10533	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
10534
10535	TMNotifyStartOfExecution(pVCpu); /* Notify TM to resume its clocks when TSC is tied to execution,
10536	as we're about to start executing the guest . */
10537
10538	/*
10539	* Load the guest TSC_AUX MSR when we are not intercepting RDTSCP.
10540	*
10541	* This is done this late as updating the TSC offsetting/preemption timer above
10542	* figures out if we can skip intercepting RDTSCP by calculating the number of
10543	* host CPU ticks till the next virtual sync deadline (for the dynamic case).
10544	*/
10545	if (pVmcsInfo->u32ProcCtls2 & VMX_PROC_CTLS2_RDTSCP)
10546	{
10547	if (!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_RDTSC_EXIT))
10548	{
10549	hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_TSC_AUX);
10550	int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K8_TSC_AUX, CPUMGetGuestTscAux(pVCpu),
10551	true /* fSetReadWrite /, true / fUpdateHostMsr */);
10552	AssertRC(rc);
10553	}
10554	else
10555	hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, pVmxTransient, MSR_K8_TSC_AUX);
10556	}
10557
10558	#ifdef VBOX_STRICT
10559	hmR0VmxCheckAutoLoadStoreMsrs(pVCpu, pVmcsInfo);
10560	hmR0VmxCheckHostEferMsr(pVCpu, pVmcsInfo);
10561	AssertRC(hmR0VmxCheckVmcsCtls(pVCpu, pVmcsInfo));
10562	#endif
10563
10564	#ifdef HMVMX_ALWAYS_CHECK_GUEST_STATE
10565	/** @todo r=ramshankar: We can now probably use iemVmxVmentryCheckGuestState here.
10566	* Add a PVMXMSRS parameter to it, so that IEM can look at the host MSRs. */
10567	uint32_t const uInvalidReason = hmR0VmxCheckGuestState(pVCpu, pVmcsInfo);
10568	if (uInvalidReason != VMX_IGS_REASON_NOT_FOUND)
10569	Log4(("hmR0VmxCheckGuestState returned %#x\n", uInvalidReason));
10570	#endif
10571	}
10572
10573
10574	/**
10575	* First C routine invoked after running guest code using hardware-assisted VMX.
10576	*
10577	* @param pVCpu The cross context virtual CPU structure.
10578	* @param pVmxTransient The VMX-transient structure.
10579	* @param rcVMRun Return code of VMLAUNCH/VMRESUME.
10580	*
10581	* @remarks Called with interrupts disabled, and returns with interrupts enabled!
10582	*
10583	* @remarks No-long-jump zone!!! This function will however re-enable longjmps
10584	* unconditionally when it is safe to do so.
10585	*/
10586	static void hmR0VmxPostRunGuest(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, int rcVMRun)
10587	{
10588	uint64_t const uHostTsc = ASMReadTSC(); /** @todo We can do a lot better here, see @bugref{9180#c38}. */
10589
10590	ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB flushing. */
10591	ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for EMT poking. */
10592	pVCpu->hm.s.fCtxChanged = 0; /* Exits/longjmps to ring-3 requires saving the guest state. */
10593	pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
10594	pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
10595	pVmxTransient->fVectoringDoublePF = false; /* Vectoring double page-fault needs to be determined later. */
10596
10597	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
10598	if (!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_RDTSC_EXIT))
10599	{
10600	uint64_t uGstTsc;
10601	if (!pVmxTransient->fIsNestedGuest)
10602	uGstTsc = uHostTsc + pVmcsInfo->u64TscOffset;
10603	else
10604	{
10605	uint64_t const uNstGstTsc = uHostTsc + pVmcsInfo->u64TscOffset;
10606	uGstTsc = CPUMRemoveNestedGuestTscOffset(pVCpu, uNstGstTsc);
10607	}
10608	TMCpuTickSetLastSeen(pVCpu, uGstTsc); /* Update TM with the guest TSC. */
10609	}
10610
10611	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatPreExit, x);
10612	TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
10613	VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
10614
10615	#if HC_ARCH_BITS == 64
10616	pVCpu->hm.s.vmx.fRestoreHostFlags \|= VMX_RESTORE_HOST_REQUIRED; /* Some host state messed up by VMX needs restoring. */
10617	#endif
10618	#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
10619	/* The 64-on-32 switcher maintains VMCS-launch state on its own
10620	and we need to leave it alone here. */
10621	if (pVmcsInfo->pfnStartVM != VMXR0SwitcherStartVM64)
10622	pVmcsInfo->fVmcsState \|= VMX_V_VMCS_LAUNCH_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
10623	#else
10624	pVmcsInfo->fVmcsState \|= VMX_V_VMCS_LAUNCH_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
10625	#endif
10626	#ifdef VBOX_STRICT
10627	hmR0VmxCheckHostEferMsr(pVCpu, pVmcsInfo); /* Verify that the host EFER MSR wasn't modified. */
10628	#endif
10629	Assert(!ASMIntAreEnabled());
10630	ASMSetFlags(pVmxTransient->fEFlags); /* Enable interrupts. */
10631	Assert(!VMMRZCallRing3IsEnabled(pVCpu));
10632
10633	/*
10634	* Save the basic VM-exit reason and check if the VM-entry failed.
10635	* See Intel spec. 24.9.1 "Basic VM-exit Information".
10636	*/
10637	uint32_t uExitReason;
10638	int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
10639	AssertRC(rc);
10640	pVmxTransient->uExitReason = VMX_EXIT_REASON_BASIC(uExitReason);
10641	pVmxTransient->fVMEntryFailed = VMX_EXIT_REASON_HAS_ENTRY_FAILED(uExitReason);
10642
10643	/*
10644	* Check if VMLAUNCH/VMRESUME succeeded.
10645	* If this failed, we cause a guru meditation and cease further execution.
10646	*/
10647	if (RT_LIKELY(rcVMRun == VINF_SUCCESS))
10648	{
10649	/*
10650	* Update the VM-exit history array here even if the VM-entry failed due to:
10651	* - Invalid guest state.
10652	* - MSR loading.
10653	* - Machine-check event.
10654	*
10655	* In any of the above cases we will still have a "valid" VM-exit reason
10656	* despite @a fVMEntryFailed being false.
10657	*
10658	* See Intel spec. 26.7 "VM-Entry failures during or after loading guest state".
10659	*
10660	* Note! We don't have CS or RIP at this point. Will probably address that later
10661	* by amending the history entry added here.
10662	*/
10663	EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_VMX, pVmxTransient->uExitReason & EMEXIT_F_TYPE_MASK),
10664	UINT64_MAX, uHostTsc);
10665
10666	if (RT_LIKELY(!pVmxTransient->fVMEntryFailed))
10667	{
10668	VMMRZCallRing3Enable(pVCpu);
10669
10670	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
10671	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
10672
10673	#if defined(HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE) \|\| defined(HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE)
10674	rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
10675	AssertRC(rc);
10676	#elif defined(HMVMX_ALWAYS_SAVE_GUEST_RFLAGS)
10677	rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_RFLAGS);
10678	AssertRC(rc);
10679	#else
10680	/*
10681	* Import the guest-interruptibility state always as we need it while evaluating
10682	* injecting events on re-entry.
10683	*
10684	* We don't import CR0 (when unrestricted guest execution is unavailable) despite
10685	* checking for real-mode while exporting the state because all bits that cause
10686	* mode changes wrt CR0 are intercepted.
10687	*/
10688	rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_HM_VMX_INT_STATE);
10689	AssertRC(rc);
10690	#endif
10691
10692	/*
10693	* Sync the TPR shadow with our APIC state.
10694	*/
10695	if ( !pVmxTransient->fIsNestedGuest
10696	&& (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW))
10697	{
10698	Assert(pVmcsInfo->pbVirtApic);
10699	if (pVmxTransient->u8GuestTpr != pVmcsInfo->pbVirtApic[XAPIC_OFF_TPR])
10700	{
10701	rc = APICSetTpr(pVCpu, pVmcsInfo->pbVirtApic[XAPIC_OFF_TPR]);
10702	AssertRC(rc);
10703	ASMAtomicOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
10704	}
10705	}
10706
10707	Assert(VMMRZCallRing3IsEnabled(pVCpu));
10708	return;
10709	}
10710	}
10711	else
10712	Log4Func(("VM-entry failure: rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", rcVMRun, pVmxTransient->fVMEntryFailed));
10713
10714	VMMRZCallRing3Enable(pVCpu);
10715	}
10716
10717
10718	/**
10719	* Runs the guest code using hardware-assisted VMX the normal way.
10720	*
10721	* @returns VBox status code.
10722	* @param pVCpu The cross context virtual CPU structure.
10723	* @param pcLoops Pointer to the number of executed loops.
10724	*/
10725	static VBOXSTRICTRC hmR0VmxRunGuestCodeNormal(PVMCPU pVCpu, uint32_t *pcLoops)
10726	{
10727	uint32_t const cMaxResumeLoops = pVCpu->CTX_SUFF(pVM)->hm.s.cMaxResumeLoops;
10728	Assert(pcLoops);
10729	Assert(*pcLoops <= cMaxResumeLoops);
10730
10731	VMXTRANSIENT VmxTransient;
10732	RT_ZERO(VmxTransient);
10733	VmxTransient.pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
10734
10735	/* Paranoia. */
10736	Assert(VmxTransient.pVmcsInfo == &pVCpu->hm.s.vmx.VmcsInfo);
10737	Assert(!CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx));
10738
10739	VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
10740	for (;;)
10741	{
10742	Assert(!HMR0SuspendPending());
10743	HMVMX_ASSERT_CPU_SAFE(pVCpu);
10744	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
10745
10746	/*
10747	* Preparatory work for running nested-guest code, this may force us to
10748	* return to ring-3.
10749	*
10750	* Warning! This bugger disables interrupts on VINF_SUCCESS!
10751	*/
10752	rcStrict = hmR0VmxPreRunGuest(pVCpu, &VmxTransient, false /* fStepping */);
10753	if (rcStrict != VINF_SUCCESS)
10754	break;
10755
10756	/* Interrupts are disabled at this point! */
10757	hmR0VmxPreRunGuestCommitted(pVCpu, &VmxTransient);
10758	int rcRun = hmR0VmxRunGuest(pVCpu, &VmxTransient);
10759	hmR0VmxPostRunGuest(pVCpu, &VmxTransient, rcRun);
10760	/* Interrupts are re-enabled at this point! */
10761
10762	/*
10763	* Check for errors with running the VM (VMLAUNCH/VMRESUME).
10764	*/
10765	if (RT_SUCCESS(rcRun))
10766	{ /* very likely */ }
10767	else
10768	{
10769	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
10770	hmR0VmxReportWorldSwitchError(pVCpu, rcRun, &VmxTransient);
10771	return rcRun;
10772	}
10773
10774	/*
10775	* Profile the VM-exit.
10776	*/
10777	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
10778	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
10779	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
10780	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
10781	HMVMX_START_EXIT_DISPATCH_PROF();
10782
10783	VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, &pVCpu->cpum.GstCtx, VmxTransient.uExitReason);
10784
10785	/*
10786	* Handle the VM-exit.
10787	*/
10788	#ifdef HMVMX_USE_FUNCTION_TABLE
10789	rcStrict = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, &VmxTransient);
10790	#else
10791	rcStrict = hmR0VmxHandleExit(pVCpu, &VmxTransient);
10792	#endif
10793	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
10794	if (rcStrict == VINF_SUCCESS)
10795	{
10796	if (++(*pcLoops) <= cMaxResumeLoops)
10797	continue;
10798	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
10799	rcStrict = VINF_EM_RAW_INTERRUPT;
10800	}
10801	break;
10802	}
10803
10804	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
10805	return rcStrict;
10806	}
10807
10808	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
10809	/**
10810	* Runs the nested-guest code using hardware-assisted VMX.
10811	*
10812	* @returns VBox status code.
10813	* @param pVCpu The cross context virtual CPU structure.
10814	* @param pcLoops Pointer to the number of executed loops.
10815	*
10816	* @sa hmR0VmxRunGuestCodeNormal().
10817	*/
10818	static VBOXSTRICTRC hmR0VmxRunGuestCodeNested(PVMCPU pVCpu, uint32_t *pcLoops)
10819	{
10820	uint32_t const cMaxResumeLoops = pVCpu->CTX_SUFF(pVM)->hm.s.cMaxResumeLoops;
10821	Assert(pcLoops);
10822	Assert(*pcLoops <= cMaxResumeLoops);
10823
10824	VMXTRANSIENT VmxTransient;
10825	RT_ZERO(VmxTransient);
10826	VmxTransient.pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
10827	VmxTransient.fIsNestedGuest = true;
10828
10829	VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
10830	for (;;)
10831	{
10832	Assert(!HMR0SuspendPending());
10833	HMVMX_ASSERT_CPU_SAFE(pVCpu);
10834	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
10835
10836	/*
10837	* Preparatory work for running guest code, this may force us to
10838	* return to ring-3.
10839	*
10840	* Warning! This bugger disables interrupts on VINF_SUCCESS!
10841	*/
10842	rcStrict = hmR0VmxPreRunGuest(pVCpu, &VmxTransient, false /* fStepping */);
10843	if (rcStrict != VINF_SUCCESS)
10844	break;
10845
10846	/* Interrupts are disabled at this point! */
10847	hmR0VmxPreRunGuestCommitted(pVCpu, &VmxTransient);
10848	int rcRun = hmR0VmxRunGuest(pVCpu, &VmxTransient);
10849	hmR0VmxPostRunGuest(pVCpu, &VmxTransient, rcRun);
10850	/* Interrupts are re-enabled at this point! */
10851
10852	/*
10853	* Check for errors with running the VM (VMLAUNCH/VMRESUME).
10854	*/
10855	if (RT_SUCCESS(rcRun))
10856	{ /* very likely */ }
10857	else
10858	{
10859	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
10860	hmR0VmxReportWorldSwitchError(pVCpu, rcRun, &VmxTransient);
10861	return rcRun;
10862	}
10863
10864	/*
10865	* Profile the VM-exit.
10866	*/
10867	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
10868	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
10869	STAM_COUNTER_INC(&pVCpu->hm.s.paStatNestedExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
10870	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
10871	HMVMX_START_EXIT_DISPATCH_PROF();
10872
10873	VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, &pVCpu->cpum.GstCtx, VmxTransient.uExitReason);
10874
10875	/*
10876	* Handle the VM-exit.
10877	*/
10878	rcStrict = hmR0VmxHandleExitNested(pVCpu, &VmxTransient);
10879	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
10880	if ( rcStrict == VINF_SUCCESS
10881	&& CPUMIsGuestInVmxNonRootMode(&pVCpu->cpum.GstCtx))
10882	{
10883	if (++(*pcLoops) <= cMaxResumeLoops)
10884	continue;
10885	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
10886	rcStrict = VINF_EM_RAW_INTERRUPT;
10887	}
10888	break;
10889	}
10890
10891	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
10892	return rcStrict;
10893	}
10894	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
10895
10896
10897	/** @name Execution loop for single stepping, DBGF events and expensive Dtrace
10898	* probes.
10899	*
10900	* The following few functions and associated structure contains the bloat
10901	* necessary for providing detailed debug events and dtrace probes as well as
10902	* reliable host side single stepping. This works on the principle of
10903	* "subclassing" the normal execution loop and workers. We replace the loop
10904	* method completely and override selected helpers to add necessary adjustments
10905	* to their core operation.
10906	*
10907	* The goal is to keep the "parent" code lean and mean, so as not to sacrifice
10908	* any performance for debug and analysis features.
10909	*
10910	* @{
10911	*/
10912
10913	/**
10914	* Transient per-VCPU debug state of VMCS and related info. we save/restore in
10915	* the debug run loop.
10916	*/
10917	typedef struct VMXRUNDBGSTATE
10918	{
10919	/** The RIP we started executing at. This is for detecting that we stepped. */
10920	uint64_t uRipStart;
10921	/** The CS we started executing with. */
10922	uint16_t uCsStart;
10923
10924	/** Whether we've actually modified the 1st execution control field. */
10925	bool fModifiedProcCtls : 1;
10926	/** Whether we've actually modified the 2nd execution control field. */
10927	bool fModifiedProcCtls2 : 1;
10928	/** Whether we've actually modified the exception bitmap. */
10929	bool fModifiedXcptBitmap : 1;
10930
10931	/** We desire the modified the CR0 mask to be cleared. */
10932	bool fClearCr0Mask : 1;
10933	/** We desire the modified the CR4 mask to be cleared. */
10934	bool fClearCr4Mask : 1;
10935	/** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC. */
10936	uint32_t fCpe1Extra;
10937	/** Stuff we do not want in VMX_VMCS32_CTRL_PROC_EXEC. */
10938	uint32_t fCpe1Unwanted;
10939	/** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC2. */
10940	uint32_t fCpe2Extra;
10941	/** Extra stuff we need in VMX_VMCS32_CTRL_EXCEPTION_BITMAP. */
10942	uint32_t bmXcptExtra;
10943	/** The sequence number of the Dtrace provider settings the state was
10944	* configured against. */
10945	uint32_t uDtraceSettingsSeqNo;
10946	/** VM-exits to check (one bit per VM-exit). */
10947	uint32_t bmExitsToCheck[3];
10948
10949	/** The initial VMX_VMCS32_CTRL_PROC_EXEC value (helps with restore). */
10950	uint32_t fProcCtlsInitial;
10951	/** The initial VMX_VMCS32_CTRL_PROC_EXEC2 value (helps with restore). */
10952	uint32_t fProcCtls2Initial;
10953	/** The initial VMX_VMCS32_CTRL_EXCEPTION_BITMAP value (helps with restore). */
10954	uint32_t bmXcptInitial;
10955	} VMXRUNDBGSTATE;
10956	AssertCompileMemberSize(VMXRUNDBGSTATE, bmExitsToCheck, (VMX_EXIT_MAX + 1 + 31) / 32 * 4);
10957	typedef VMXRUNDBGSTATE *PVMXRUNDBGSTATE;
10958
10959
10960	/**
10961	* Initializes the VMXRUNDBGSTATE structure.
10962	*
10963	* @param pVCpu The cross context virtual CPU structure of the
10964	* calling EMT.
10965	* @param pVmxTransient The VMX-transient structure.
10966	* @param pDbgState The debug state to initialize.
10967	*/
10968	static void hmR0VmxRunDebugStateInit(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState)
10969	{
10970	pDbgState->uRipStart = pVCpu->cpum.GstCtx.rip;
10971	pDbgState->uCsStart = pVCpu->cpum.GstCtx.cs.Sel;
10972
10973	pDbgState->fModifiedProcCtls = false;
10974	pDbgState->fModifiedProcCtls2 = false;
10975	pDbgState->fModifiedXcptBitmap = false;
10976	pDbgState->fClearCr0Mask = false;
10977	pDbgState->fClearCr4Mask = false;
10978	pDbgState->fCpe1Extra = 0;
10979	pDbgState->fCpe1Unwanted = 0;
10980	pDbgState->fCpe2Extra = 0;
10981	pDbgState->bmXcptExtra = 0;
10982	pDbgState->fProcCtlsInitial = pVmxTransient->pVmcsInfo->u32ProcCtls;
10983	pDbgState->fProcCtls2Initial = pVmxTransient->pVmcsInfo->u32ProcCtls2;
10984	pDbgState->bmXcptInitial = pVmxTransient->pVmcsInfo->u32XcptBitmap;
10985	}
10986
10987
10988	/**
10989	* Updates the VMSC fields with changes requested by @a pDbgState.
10990	*
10991	* This is performed after hmR0VmxPreRunGuestDebugStateUpdate as well
10992	* immediately before executing guest code, i.e. when interrupts are disabled.
10993	* We don't check status codes here as we cannot easily assert or return in the
10994	* latter case.
10995	*
10996	* @param pVCpu The cross context virtual CPU structure.
10997	* @param pVmxTransient The VMX-transient structure.
10998	* @param pDbgState The debug state.
10999	*/
11000	static void hmR0VmxPreRunGuestDebugStateApply(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState)
11001	{
11002	/*
11003	* Ensure desired flags in VMCS control fields are set.
11004	* (Ignoring write failure here, as we're committed and it's just debug extras.)
11005	*
11006	* Note! We load the shadow CR0 & CR4 bits when we flag the clearing, so
11007	* there should be no stale data in pCtx at this point.
11008	*/
11009	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
11010	if ( (pVmcsInfo->u32ProcCtls & pDbgState->fCpe1Extra) != pDbgState->fCpe1Extra
11011	\|\| (pVmcsInfo->u32ProcCtls & pDbgState->fCpe1Unwanted))
11012	{
11013	pVmcsInfo->u32ProcCtls \|= pDbgState->fCpe1Extra;
11014	pVmcsInfo->u32ProcCtls &= ~pDbgState->fCpe1Unwanted;
11015	VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
11016	Log6Func(("VMX_VMCS32_CTRL_PROC_EXEC: %#RX32\n", pVmcsInfo->u32ProcCtls));
11017	pDbgState->fModifiedProcCtls = true;
11018	}
11019
11020	if ((pVmcsInfo->u32ProcCtls2 & pDbgState->fCpe2Extra) != pDbgState->fCpe2Extra)
11021	{
11022	pVmcsInfo->u32ProcCtls2 \|= pDbgState->fCpe2Extra;
11023	VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pVmcsInfo->u32ProcCtls2);
11024	Log6Func(("VMX_VMCS32_CTRL_PROC_EXEC2: %#RX32\n", pVmcsInfo->u32ProcCtls2));
11025	pDbgState->fModifiedProcCtls2 = true;
11026	}
11027
11028	if ((pVmcsInfo->u32XcptBitmap & pDbgState->bmXcptExtra) != pDbgState->bmXcptExtra)
11029	{
11030	pVmcsInfo->u32XcptBitmap \|= pDbgState->bmXcptExtra;
11031	VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVmcsInfo->u32XcptBitmap);
11032	Log6Func(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP: %#RX32\n", pVmcsInfo->u32XcptBitmap));
11033	pDbgState->fModifiedXcptBitmap = true;
11034	}
11035
11036	if (pDbgState->fClearCr0Mask && pVmcsInfo->u64Cr0Mask != 0)
11037	{
11038	pVmcsInfo->u64Cr0Mask = 0;
11039	VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, 0);
11040	Log6Func(("VMX_VMCS_CTRL_CR0_MASK: 0\n"));
11041	}
11042
11043	if (pDbgState->fClearCr4Mask && pVmcsInfo->u64Cr4Mask != 0)
11044	{
11045	pVmcsInfo->u64Cr4Mask = 0;
11046	VMXWriteVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, 0);
11047	Log6Func(("VMX_VMCS_CTRL_CR4_MASK: 0\n"));
11048	}
11049
11050	NOREF(pVCpu);
11051	}
11052
11053
11054	/**
11055	* Restores VMCS fields that were changed by hmR0VmxPreRunGuestDebugStateApply for
11056	* re-entry next time around.
11057	*
11058	* @returns Strict VBox status code (i.e. informational status codes too).
11059	* @param pVCpu The cross context virtual CPU structure.
11060	* @param pVmxTransient The VMX-transient structure.
11061	* @param pDbgState The debug state.
11062	* @param rcStrict The return code from executing the guest using single
11063	* stepping.
11064	*/
11065	static VBOXSTRICTRC hmR0VmxRunDebugStateRevert(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState,
11066	VBOXSTRICTRC rcStrict)
11067	{
11068	/*
11069	* Restore VM-exit control settings as we may not reenter this function the
11070	* next time around.
11071	*/
11072	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
11073
11074	/* We reload the initial value, trigger what we can of recalculations the
11075	next time around. From the looks of things, that's all that's required atm. */
11076	if (pDbgState->fModifiedProcCtls)
11077	{
11078	if (!(pDbgState->fProcCtlsInitial & VMX_PROC_CTLS_MOV_DR_EXIT) && CPUMIsHyperDebugStateActive(pVCpu))
11079	pDbgState->fProcCtlsInitial \|= VMX_PROC_CTLS_MOV_DR_EXIT; /* Avoid assertion in hmR0VmxLeave */
11080	int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pDbgState->fProcCtlsInitial);
11081	AssertRCReturn(rc2, rc2);
11082	pVmcsInfo->u32ProcCtls = pDbgState->fProcCtlsInitial;
11083	}
11084
11085	/* We're currently the only ones messing with this one, so just restore the
11086	cached value and reload the field. */
11087	if ( pDbgState->fModifiedProcCtls2
11088	&& pVmcsInfo->u32ProcCtls2 != pDbgState->fProcCtls2Initial)
11089	{
11090	int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pDbgState->fProcCtls2Initial);
11091	AssertRCReturn(rc2, rc2);
11092	pVmcsInfo->u32ProcCtls2 = pDbgState->fProcCtls2Initial;
11093	}
11094
11095	/* If we've modified the exception bitmap, we restore it and trigger
11096	reloading and partial recalculation the next time around. */
11097	if (pDbgState->fModifiedXcptBitmap)
11098	pVmcsInfo->u32XcptBitmap = pDbgState->bmXcptInitial;
11099
11100	return rcStrict;
11101	}
11102
11103
11104	/**
11105	* Configures VM-exit controls for current DBGF and DTrace settings.
11106	*
11107	* This updates @a pDbgState and the VMCS execution control fields to reflect
11108	* the necessary VM-exits demanded by DBGF and DTrace.
11109	*
11110	* @param pVCpu The cross context virtual CPU structure.
11111	* @param pVmxTransient The VMX-transient structure. May update
11112	* fUpdatedTscOffsettingAndPreemptTimer.
11113	* @param pDbgState The debug state.
11114	*/
11115	static void hmR0VmxPreRunGuestDebugStateUpdate(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState)
11116	{
11117	/*
11118	* Take down the dtrace serial number so we can spot changes.
11119	*/
11120	pDbgState->uDtraceSettingsSeqNo = VBOXVMM_GET_SETTINGS_SEQ_NO();
11121	ASMCompilerBarrier();
11122
11123	/*
11124	* We'll rebuild most of the middle block of data members (holding the
11125	* current settings) as we go along here, so start by clearing it all.
11126	*/
11127	pDbgState->bmXcptExtra = 0;
11128	pDbgState->fCpe1Extra = 0;
11129	pDbgState->fCpe1Unwanted = 0;
11130	pDbgState->fCpe2Extra = 0;
11131	for (unsigned i = 0; i < RT_ELEMENTS(pDbgState->bmExitsToCheck); i++)
11132	pDbgState->bmExitsToCheck[i] = 0;
11133
11134	/*
11135	* Software interrupts (INT XXh) - no idea how to trigger these...
11136	*/
11137	PVM pVM = pVCpu->CTX_SUFF(pVM);
11138	if ( DBGF_IS_EVENT_ENABLED(pVM, DBGFEVENT_INTERRUPT_SOFTWARE)
11139	\|\| VBOXVMM_INT_SOFTWARE_ENABLED())
11140	{
11141	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
11142	}
11143
11144	/*
11145	* INT3 breakpoints - triggered by #BP exceptions.
11146	*/
11147	if (pVM->dbgf.ro.cEnabledInt3Breakpoints > 0)
11148	pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_BP);
11149
11150	/*
11151	* Exception bitmap and XCPT events+probes.
11152	*/
11153	for (int iXcpt = 0; iXcpt < (DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST + 1); iXcpt++)
11154	if (DBGF_IS_EVENT_ENABLED(pVM, (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + iXcpt)))
11155	pDbgState->bmXcptExtra \|= RT_BIT_32(iXcpt);
11156
11157	if (VBOXVMM_XCPT_DE_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_DE);
11158	if (VBOXVMM_XCPT_DB_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_DB);
11159	if (VBOXVMM_XCPT_BP_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_BP);
11160	if (VBOXVMM_XCPT_OF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_OF);
11161	if (VBOXVMM_XCPT_BR_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_BR);
11162	if (VBOXVMM_XCPT_UD_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_UD);
11163	if (VBOXVMM_XCPT_NM_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_NM);
11164	if (VBOXVMM_XCPT_DF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_DF);
11165	if (VBOXVMM_XCPT_TS_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_TS);
11166	if (VBOXVMM_XCPT_NP_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_NP);
11167	if (VBOXVMM_XCPT_SS_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_SS);
11168	if (VBOXVMM_XCPT_GP_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_GP);
11169	if (VBOXVMM_XCPT_PF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_PF);
11170	if (VBOXVMM_XCPT_MF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_MF);
11171	if (VBOXVMM_XCPT_AC_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_AC);
11172	if (VBOXVMM_XCPT_XF_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_XF);
11173	if (VBOXVMM_XCPT_VE_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_VE);
11174	if (VBOXVMM_XCPT_SX_ENABLED()) pDbgState->bmXcptExtra \|= RT_BIT_32(X86_XCPT_SX);
11175
11176	if (pDbgState->bmXcptExtra)
11177	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
11178
11179	/*
11180	* Process events and probes for VM-exits, making sure we get the wanted VM-exits.
11181	*
11182	* Note! This is the reverse of what hmR0VmxHandleExitDtraceEvents does.
11183	* So, when adding/changing/removing please don't forget to update it.
11184	*
11185	* Some of the macros are picking up local variables to save horizontal space,
11186	* (being able to see it in a table is the lesser evil here).
11187	*/
11188	#define IS_EITHER_ENABLED(a_pVM, a_EventSubName) \
11189	( DBGF_IS_EVENT_ENABLED(a_pVM, RT_CONCAT(DBGFEVENT_, a_EventSubName)) \
11190	\|\| RT_CONCAT3(VBOXVMM_, a_EventSubName, _ENABLED)() )
11191	#define SET_ONLY_XBM_IF_EITHER_EN(a_EventSubName, a_uExit) \
11192	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
11193	{ AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
11194	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
11195	} else do { } while (0)
11196	#define SET_CPE1_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec) \
11197	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
11198	{ \
11199	(pDbgState)->fCpe1Extra \|= (a_fCtrlProcExec); \
11200	AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
11201	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
11202	} else do { } while (0)
11203	#define SET_CPEU_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fUnwantedCtrlProcExec) \
11204	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
11205	{ \
11206	(pDbgState)->fCpe1Unwanted \|= (a_fUnwantedCtrlProcExec); \
11207	AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
11208	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
11209	} else do { } while (0)
11210	#define SET_CPE2_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec2) \
11211	if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
11212	{ \
11213	(pDbgState)->fCpe2Extra \|= (a_fCtrlProcExec2); \
11214	AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
11215	ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
11216	} else do { } while (0)
11217
11218	SET_ONLY_XBM_IF_EITHER_EN(EXIT_TASK_SWITCH, VMX_EXIT_TASK_SWITCH); /* unconditional */
11219	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_VIOLATION, VMX_EXIT_EPT_VIOLATION); /* unconditional */
11220	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_MISCONFIG, VMX_EXIT_EPT_MISCONFIG); /* unconditional (unless #VE) */
11221	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_ACCESS, VMX_EXIT_APIC_ACCESS); /* feature dependent, nothing to enable here */
11222	SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_WRITE, VMX_EXIT_APIC_WRITE); /* feature dependent, nothing to enable here */
11223
11224	SET_ONLY_XBM_IF_EITHER_EN(INSTR_CPUID, VMX_EXIT_CPUID); /* unconditional */
11225	SET_ONLY_XBM_IF_EITHER_EN( EXIT_CPUID, VMX_EXIT_CPUID);
11226	SET_ONLY_XBM_IF_EITHER_EN(INSTR_GETSEC, VMX_EXIT_GETSEC); /* unconditional */
11227	SET_ONLY_XBM_IF_EITHER_EN( EXIT_GETSEC, VMX_EXIT_GETSEC);
11228	SET_CPE1_XBM_IF_EITHER_EN(INSTR_HALT, VMX_EXIT_HLT, VMX_PROC_CTLS_HLT_EXIT); /* paranoia */
11229	SET_ONLY_XBM_IF_EITHER_EN( EXIT_HALT, VMX_EXIT_HLT);
11230	SET_ONLY_XBM_IF_EITHER_EN(INSTR_INVD, VMX_EXIT_INVD); /* unconditional */
11231	SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVD, VMX_EXIT_INVD);
11232	SET_CPE1_XBM_IF_EITHER_EN(INSTR_INVLPG, VMX_EXIT_INVLPG, VMX_PROC_CTLS_INVLPG_EXIT);
11233	SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVLPG, VMX_EXIT_INVLPG);
11234	SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDPMC, VMX_EXIT_RDPMC, VMX_PROC_CTLS_RDPMC_EXIT);
11235	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDPMC, VMX_EXIT_RDPMC);
11236	SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSC, VMX_EXIT_RDTSC, VMX_PROC_CTLS_RDTSC_EXIT);
11237	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSC, VMX_EXIT_RDTSC);
11238	SET_ONLY_XBM_IF_EITHER_EN(INSTR_RSM, VMX_EXIT_RSM); /* unconditional */
11239	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RSM, VMX_EXIT_RSM);
11240	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMM_CALL, VMX_EXIT_VMCALL); /* unconditional */
11241	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMM_CALL, VMX_EXIT_VMCALL);
11242	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMCLEAR, VMX_EXIT_VMCLEAR); /* unconditional */
11243	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMCLEAR, VMX_EXIT_VMCLEAR);
11244	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH); /* unconditional */
11245	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH);
11246	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRLD, VMX_EXIT_VMPTRLD); /* unconditional */
11247	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRLD, VMX_EXIT_VMPTRLD);
11248	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRST, VMX_EXIT_VMPTRST); /* unconditional */
11249	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRST, VMX_EXIT_VMPTRST);
11250	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMREAD, VMX_EXIT_VMREAD); /* unconditional */
11251	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMREAD, VMX_EXIT_VMREAD);
11252	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMRESUME, VMX_EXIT_VMRESUME); /* unconditional */
11253	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMRESUME, VMX_EXIT_VMRESUME);
11254	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMWRITE, VMX_EXIT_VMWRITE); /* unconditional */
11255	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMWRITE, VMX_EXIT_VMWRITE);
11256	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXOFF, VMX_EXIT_VMXOFF); /* unconditional */
11257	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXOFF, VMX_EXIT_VMXOFF);
11258	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXON, VMX_EXIT_VMXON); /* unconditional */
11259	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXON, VMX_EXIT_VMXON);
11260
11261	if ( IS_EITHER_ENABLED(pVM, INSTR_CRX_READ)
11262	\|\| IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
11263	{
11264	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_CR4
11265	\| CPUMCTX_EXTRN_APIC_TPR);
11266	AssertRC(rc);
11267
11268	#if 0 /** @todo fix me */
11269	pDbgState->fClearCr0Mask = true;
11270	pDbgState->fClearCr4Mask = true;
11271	#endif
11272	if (IS_EITHER_ENABLED(pVM, INSTR_CRX_READ))
11273	pDbgState->fCpe1Extra \|= VMX_PROC_CTLS_CR3_STORE_EXIT \| VMX_PROC_CTLS_CR8_STORE_EXIT;
11274	if (IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
11275	pDbgState->fCpe1Extra \|= VMX_PROC_CTLS_CR3_LOAD_EXIT \| VMX_PROC_CTLS_CR8_LOAD_EXIT;
11276	pDbgState->fCpe1Unwanted \|= VMX_PROC_CTLS_USE_TPR_SHADOW; /* risky? */
11277	/* Note! We currently don't use VMX_VMCS32_CTRL_CR3_TARGET_COUNT. It would
11278	require clearing here and in the loop if we start using it. */
11279	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_CRX);
11280	}
11281	else
11282	{
11283	if (pDbgState->fClearCr0Mask)
11284	{
11285	pDbgState->fClearCr0Mask = false;
11286	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR0);
11287	}
11288	if (pDbgState->fClearCr4Mask)
11289	{
11290	pDbgState->fClearCr4Mask = false;
11291	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR4);
11292	}
11293	}
11294	SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_READ, VMX_EXIT_MOV_CRX);
11295	SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_WRITE, VMX_EXIT_MOV_CRX);
11296
11297	if ( IS_EITHER_ENABLED(pVM, INSTR_DRX_READ)
11298	\|\| IS_EITHER_ENABLED(pVM, INSTR_DRX_WRITE))
11299	{
11300	/** @todo later, need to fix handler as it assumes this won't usually happen. */
11301	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_DRX);
11302	}
11303	SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_READ, VMX_EXIT_MOV_DRX);
11304	SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_WRITE, VMX_EXIT_MOV_DRX);
11305
11306	SET_CPEU_XBM_IF_EITHER_EN(INSTR_RDMSR, VMX_EXIT_RDMSR, VMX_PROC_CTLS_USE_MSR_BITMAPS); /* risky clearing this? */
11307	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDMSR, VMX_EXIT_RDMSR);
11308	SET_CPEU_XBM_IF_EITHER_EN(INSTR_WRMSR, VMX_EXIT_WRMSR, VMX_PROC_CTLS_USE_MSR_BITMAPS);
11309	SET_ONLY_XBM_IF_EITHER_EN( EXIT_WRMSR, VMX_EXIT_WRMSR);
11310	SET_CPE1_XBM_IF_EITHER_EN(INSTR_MWAIT, VMX_EXIT_MWAIT, VMX_PROC_CTLS_MWAIT_EXIT); /* paranoia */
11311	SET_ONLY_XBM_IF_EITHER_EN( EXIT_MWAIT, VMX_EXIT_MWAIT);
11312	SET_CPE1_XBM_IF_EITHER_EN(INSTR_MONITOR, VMX_EXIT_MONITOR, VMX_PROC_CTLS_MONITOR_EXIT); /* paranoia */
11313	SET_ONLY_XBM_IF_EITHER_EN( EXIT_MONITOR, VMX_EXIT_MONITOR);
11314	#if 0 /** @todo too slow, fix handler. */
11315	SET_CPE1_XBM_IF_EITHER_EN(INSTR_PAUSE, VMX_EXIT_PAUSE, VMX_PROC_CTLS_PAUSE_EXIT);
11316	#endif
11317	SET_ONLY_XBM_IF_EITHER_EN( EXIT_PAUSE, VMX_EXIT_PAUSE);
11318
11319	if ( IS_EITHER_ENABLED(pVM, INSTR_SGDT)
11320	\|\| IS_EITHER_ENABLED(pVM, INSTR_SIDT)
11321	\|\| IS_EITHER_ENABLED(pVM, INSTR_LGDT)
11322	\|\| IS_EITHER_ENABLED(pVM, INSTR_LIDT))
11323	{
11324	pDbgState->fCpe2Extra \|= VMX_PROC_CTLS2_DESC_TABLE_EXIT;
11325	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_GDTR_IDTR_ACCESS);
11326	}
11327	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SGDT, VMX_EXIT_GDTR_IDTR_ACCESS);
11328	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SIDT, VMX_EXIT_GDTR_IDTR_ACCESS);
11329	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LGDT, VMX_EXIT_GDTR_IDTR_ACCESS);
11330	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LIDT, VMX_EXIT_GDTR_IDTR_ACCESS);
11331
11332	if ( IS_EITHER_ENABLED(pVM, INSTR_SLDT)
11333	\|\| IS_EITHER_ENABLED(pVM, INSTR_STR)
11334	\|\| IS_EITHER_ENABLED(pVM, INSTR_LLDT)
11335	\|\| IS_EITHER_ENABLED(pVM, INSTR_LTR))
11336	{
11337	pDbgState->fCpe2Extra \|= VMX_PROC_CTLS2_DESC_TABLE_EXIT;
11338	ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_LDTR_TR_ACCESS);
11339	}
11340	SET_ONLY_XBM_IF_EITHER_EN( EXIT_SLDT, VMX_EXIT_LDTR_TR_ACCESS);
11341	SET_ONLY_XBM_IF_EITHER_EN( EXIT_STR, VMX_EXIT_LDTR_TR_ACCESS);
11342	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LLDT, VMX_EXIT_LDTR_TR_ACCESS);
11343	SET_ONLY_XBM_IF_EITHER_EN( EXIT_LTR, VMX_EXIT_LDTR_TR_ACCESS);
11344
11345	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVEPT, VMX_EXIT_INVEPT); /* unconditional */
11346	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVEPT, VMX_EXIT_INVEPT);
11347	SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSCP, VMX_EXIT_RDTSCP, VMX_PROC_CTLS_RDTSC_EXIT);
11348	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSCP, VMX_EXIT_RDTSCP);
11349	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVVPID, VMX_EXIT_INVVPID); /* unconditional */
11350	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVVPID, VMX_EXIT_INVVPID);
11351	SET_CPE2_XBM_IF_EITHER_EN(INSTR_WBINVD, VMX_EXIT_WBINVD, VMX_PROC_CTLS2_WBINVD_EXIT);
11352	SET_ONLY_XBM_IF_EITHER_EN( EXIT_WBINVD, VMX_EXIT_WBINVD);
11353	SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSETBV, VMX_EXIT_XSETBV); /* unconditional */
11354	SET_ONLY_XBM_IF_EITHER_EN( EXIT_XSETBV, VMX_EXIT_XSETBV);
11355	SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDRAND, VMX_EXIT_RDRAND, VMX_PROC_CTLS2_RDRAND_EXIT);
11356	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDRAND, VMX_EXIT_RDRAND);
11357	SET_CPE1_XBM_IF_EITHER_EN(INSTR_VMX_INVPCID, VMX_EXIT_INVPCID, VMX_PROC_CTLS_INVLPG_EXIT);
11358	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVPCID, VMX_EXIT_INVPCID);
11359	SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMFUNC, VMX_EXIT_VMFUNC); /* unconditional for the current setup */
11360	SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMFUNC, VMX_EXIT_VMFUNC);
11361	SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDSEED, VMX_EXIT_RDSEED, VMX_PROC_CTLS2_RDSEED_EXIT);
11362	SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDSEED, VMX_EXIT_RDSEED);
11363	SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSAVES, VMX_EXIT_XSAVES); /* unconditional (enabled by host, guest cfg) */
11364	SET_ONLY_XBM_IF_EITHER_EN(EXIT_XSAVES, VMX_EXIT_XSAVES);
11365	SET_ONLY_XBM_IF_EITHER_EN(INSTR_XRSTORS, VMX_EXIT_XRSTORS); /* unconditional (enabled by host, guest cfg) */
11366	SET_ONLY_XBM_IF_EITHER_EN( EXIT_XRSTORS, VMX_EXIT_XRSTORS);
11367
11368	#undef IS_EITHER_ENABLED
11369	#undef SET_ONLY_XBM_IF_EITHER_EN
11370	#undef SET_CPE1_XBM_IF_EITHER_EN
11371	#undef SET_CPEU_XBM_IF_EITHER_EN
11372	#undef SET_CPE2_XBM_IF_EITHER_EN
11373
11374	/*
11375	* Sanitize the control stuff.
11376	*/
11377	pDbgState->fCpe2Extra &= pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1;
11378	if (pDbgState->fCpe2Extra)
11379	pDbgState->fCpe1Extra \|= VMX_PROC_CTLS_USE_SECONDARY_CTLS;
11380	pDbgState->fCpe1Extra &= pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1;
11381	pDbgState->fCpe1Unwanted &= ~pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed0;
11382	if (pVCpu->hm.s.fDebugWantRdTscExit != RT_BOOL(pDbgState->fCpe1Extra & VMX_PROC_CTLS_RDTSC_EXIT))
11383	{
11384	pVCpu->hm.s.fDebugWantRdTscExit ^= true;
11385	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
11386	}
11387
11388	Log6(("HM: debug state: cpe1=%#RX32 cpeu=%#RX32 cpe2=%#RX32%s%s\n",
11389	pDbgState->fCpe1Extra, pDbgState->fCpe1Unwanted, pDbgState->fCpe2Extra,
11390	pDbgState->fClearCr0Mask ? " clr-cr0" : "",
11391	pDbgState->fClearCr4Mask ? " clr-cr4" : ""));
11392	}
11393
11394
11395	/**
11396	* Fires off DBGF events and dtrace probes for a VM-exit, when it's
11397	* appropriate.
11398	*
11399	* The caller has checked the VM-exit against the
11400	* VMXRUNDBGSTATE::bmExitsToCheck bitmap. The caller has checked for NMIs
11401	* already, so we don't have to do that either.
11402	*
11403	* @returns Strict VBox status code (i.e. informational status codes too).
11404	* @param pVCpu The cross context virtual CPU structure.
11405	* @param pVmxTransient The VMX-transient structure.
11406	* @param uExitReason The VM-exit reason.
11407	*
11408	* @remarks The name of this function is displayed by dtrace, so keep it short
11409	* and to the point. No longer than 33 chars long, please.
11410	*/
11411	static VBOXSTRICTRC hmR0VmxHandleExitDtraceEvents(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t uExitReason)
11412	{
11413	/*
11414	* Translate the event into a DBGF event (enmEvent + uEventArg) and at the
11415	* same time check whether any corresponding Dtrace event is enabled (fDtrace).
11416	*
11417	* Note! This is the reverse operation of what hmR0VmxPreRunGuestDebugStateUpdate
11418	* does. Must add/change/remove both places. Same ordering, please.
11419	*
11420	* Added/removed events must also be reflected in the next section
11421	* where we dispatch dtrace events.
11422	*/
11423	bool fDtrace1 = false;
11424	bool fDtrace2 = false;
11425	DBGFEVENTTYPE enmEvent1 = DBGFEVENT_END;
11426	DBGFEVENTTYPE enmEvent2 = DBGFEVENT_END;
11427	uint32_t uEventArg = 0;
11428	#define SET_EXIT(a_EventSubName) \
11429	do { \
11430	enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
11431	fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
11432	} while (0)
11433	#define SET_BOTH(a_EventSubName) \
11434	do { \
11435	enmEvent1 = RT_CONCAT(DBGFEVENT_INSTR_, a_EventSubName); \
11436	enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
11437	fDtrace1 = RT_CONCAT3(VBOXVMM_INSTR_, a_EventSubName, _ENABLED)(); \
11438	fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
11439	} while (0)
11440	switch (uExitReason)
11441	{
11442	case VMX_EXIT_MTF:
11443	return hmR0VmxExitMtf(pVCpu, pVmxTransient);
11444
11445	case VMX_EXIT_XCPT_OR_NMI:
11446	{
11447	uint8_t const idxVector = VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo);
11448	switch (VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo))
11449	{
11450	case VMX_EXIT_INT_INFO_TYPE_HW_XCPT:
11451	case VMX_EXIT_INT_INFO_TYPE_SW_XCPT:
11452	case VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT:
11453	if (idxVector <= (unsigned)(DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST))
11454	{
11455	if (VMX_EXIT_INT_INFO_IS_ERROR_CODE_VALID(pVmxTransient->uExitIntInfo))
11456	{
11457	hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11458	uEventArg = pVmxTransient->uExitIntErrorCode;
11459	}
11460	enmEvent1 = (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + idxVector);
11461	switch (enmEvent1)
11462	{
11463	case DBGFEVENT_XCPT_DE: fDtrace1 = VBOXVMM_XCPT_DE_ENABLED(); break;
11464	case DBGFEVENT_XCPT_DB: fDtrace1 = VBOXVMM_XCPT_DB_ENABLED(); break;
11465	case DBGFEVENT_XCPT_BP: fDtrace1 = VBOXVMM_XCPT_BP_ENABLED(); break;
11466	case DBGFEVENT_XCPT_OF: fDtrace1 = VBOXVMM_XCPT_OF_ENABLED(); break;
11467	case DBGFEVENT_XCPT_BR: fDtrace1 = VBOXVMM_XCPT_BR_ENABLED(); break;
11468	case DBGFEVENT_XCPT_UD: fDtrace1 = VBOXVMM_XCPT_UD_ENABLED(); break;
11469	case DBGFEVENT_XCPT_NM: fDtrace1 = VBOXVMM_XCPT_NM_ENABLED(); break;
11470	case DBGFEVENT_XCPT_DF: fDtrace1 = VBOXVMM_XCPT_DF_ENABLED(); break;
11471	case DBGFEVENT_XCPT_TS: fDtrace1 = VBOXVMM_XCPT_TS_ENABLED(); break;
11472	case DBGFEVENT_XCPT_NP: fDtrace1 = VBOXVMM_XCPT_NP_ENABLED(); break;
11473	case DBGFEVENT_XCPT_SS: fDtrace1 = VBOXVMM_XCPT_SS_ENABLED(); break;
11474	case DBGFEVENT_XCPT_GP: fDtrace1 = VBOXVMM_XCPT_GP_ENABLED(); break;
11475	case DBGFEVENT_XCPT_PF: fDtrace1 = VBOXVMM_XCPT_PF_ENABLED(); break;
11476	case DBGFEVENT_XCPT_MF: fDtrace1 = VBOXVMM_XCPT_MF_ENABLED(); break;
11477	case DBGFEVENT_XCPT_AC: fDtrace1 = VBOXVMM_XCPT_AC_ENABLED(); break;
11478	case DBGFEVENT_XCPT_XF: fDtrace1 = VBOXVMM_XCPT_XF_ENABLED(); break;
11479	case DBGFEVENT_XCPT_VE: fDtrace1 = VBOXVMM_XCPT_VE_ENABLED(); break;
11480	case DBGFEVENT_XCPT_SX: fDtrace1 = VBOXVMM_XCPT_SX_ENABLED(); break;
11481	default: break;
11482	}
11483	}
11484	else
11485	AssertFailed();
11486	break;
11487
11488	case VMX_EXIT_INT_INFO_TYPE_SW_INT:
11489	uEventArg = idxVector;
11490	enmEvent1 = DBGFEVENT_INTERRUPT_SOFTWARE;
11491	fDtrace1 = VBOXVMM_INT_SOFTWARE_ENABLED();
11492	break;
11493	}
11494	break;
11495	}
11496
11497	case VMX_EXIT_TRIPLE_FAULT:
11498	enmEvent1 = DBGFEVENT_TRIPLE_FAULT;
11499	//fDtrace1 = VBOXVMM_EXIT_TRIPLE_FAULT_ENABLED();
11500	break;
11501	case VMX_EXIT_TASK_SWITCH: SET_EXIT(TASK_SWITCH); break;
11502	case VMX_EXIT_EPT_VIOLATION: SET_EXIT(VMX_EPT_VIOLATION); break;
11503	case VMX_EXIT_EPT_MISCONFIG: SET_EXIT(VMX_EPT_MISCONFIG); break;
11504	case VMX_EXIT_APIC_ACCESS: SET_EXIT(VMX_VAPIC_ACCESS); break;
11505	case VMX_EXIT_APIC_WRITE: SET_EXIT(VMX_VAPIC_WRITE); break;
11506
11507	/* Instruction specific VM-exits: */
11508	case VMX_EXIT_CPUID: SET_BOTH(CPUID); break;
11509	case VMX_EXIT_GETSEC: SET_BOTH(GETSEC); break;
11510	case VMX_EXIT_HLT: SET_BOTH(HALT); break;
11511	case VMX_EXIT_INVD: SET_BOTH(INVD); break;
11512	case VMX_EXIT_INVLPG: SET_BOTH(INVLPG); break;
11513	case VMX_EXIT_RDPMC: SET_BOTH(RDPMC); break;
11514	case VMX_EXIT_RDTSC: SET_BOTH(RDTSC); break;
11515	case VMX_EXIT_RSM: SET_BOTH(RSM); break;
11516	case VMX_EXIT_VMCALL: SET_BOTH(VMM_CALL); break;
11517	case VMX_EXIT_VMCLEAR: SET_BOTH(VMX_VMCLEAR); break;
11518	case VMX_EXIT_VMLAUNCH: SET_BOTH(VMX_VMLAUNCH); break;
11519	case VMX_EXIT_VMPTRLD: SET_BOTH(VMX_VMPTRLD); break;
11520	case VMX_EXIT_VMPTRST: SET_BOTH(VMX_VMPTRST); break;
11521	case VMX_EXIT_VMREAD: SET_BOTH(VMX_VMREAD); break;
11522	case VMX_EXIT_VMRESUME: SET_BOTH(VMX_VMRESUME); break;
11523	case VMX_EXIT_VMWRITE: SET_BOTH(VMX_VMWRITE); break;
11524	case VMX_EXIT_VMXOFF: SET_BOTH(VMX_VMXOFF); break;
11525	case VMX_EXIT_VMXON: SET_BOTH(VMX_VMXON); break;
11526	case VMX_EXIT_MOV_CRX:
11527	hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
11528	if (VMX_EXIT_QUAL_CRX_ACCESS(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_CRX_ACCESS_READ)
11529	SET_BOTH(CRX_READ);
11530	else
11531	SET_BOTH(CRX_WRITE);
11532	uEventArg = VMX_EXIT_QUAL_CRX_REGISTER(pVmxTransient->uExitQual);
11533	break;
11534	case VMX_EXIT_MOV_DRX:
11535	hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
11536	if ( VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQual)
11537	== VMX_EXIT_QUAL_DRX_DIRECTION_READ)
11538	SET_BOTH(DRX_READ);
11539	else
11540	SET_BOTH(DRX_WRITE);
11541	uEventArg = VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQual);
11542	break;
11543	case VMX_EXIT_RDMSR: SET_BOTH(RDMSR); break;
11544	case VMX_EXIT_WRMSR: SET_BOTH(WRMSR); break;
11545	case VMX_EXIT_MWAIT: SET_BOTH(MWAIT); break;
11546	case VMX_EXIT_MONITOR: SET_BOTH(MONITOR); break;
11547	case VMX_EXIT_PAUSE: SET_BOTH(PAUSE); break;
11548	case VMX_EXIT_GDTR_IDTR_ACCESS:
11549	hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
11550	switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_BF_XDTR_INSINFO_INSTR_ID))
11551	{
11552	case VMX_XDTR_INSINFO_II_SGDT: SET_BOTH(SGDT); break;
11553	case VMX_XDTR_INSINFO_II_SIDT: SET_BOTH(SIDT); break;
11554	case VMX_XDTR_INSINFO_II_LGDT: SET_BOTH(LGDT); break;
11555	case VMX_XDTR_INSINFO_II_LIDT: SET_BOTH(LIDT); break;
11556	}
11557	break;
11558
11559	case VMX_EXIT_LDTR_TR_ACCESS:
11560	hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
11561	switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_BF_YYTR_INSINFO_INSTR_ID))
11562	{
11563	case VMX_YYTR_INSINFO_II_SLDT: SET_BOTH(SLDT); break;
11564	case VMX_YYTR_INSINFO_II_STR: SET_BOTH(STR); break;
11565	case VMX_YYTR_INSINFO_II_LLDT: SET_BOTH(LLDT); break;
11566	case VMX_YYTR_INSINFO_II_LTR: SET_BOTH(LTR); break;
11567	}
11568	break;
11569
11570	case VMX_EXIT_INVEPT: SET_BOTH(VMX_INVEPT); break;
11571	case VMX_EXIT_RDTSCP: SET_BOTH(RDTSCP); break;
11572	case VMX_EXIT_INVVPID: SET_BOTH(VMX_INVVPID); break;
11573	case VMX_EXIT_WBINVD: SET_BOTH(WBINVD); break;
11574	case VMX_EXIT_XSETBV: SET_BOTH(XSETBV); break;
11575	case VMX_EXIT_RDRAND: SET_BOTH(RDRAND); break;
11576	case VMX_EXIT_INVPCID: SET_BOTH(VMX_INVPCID); break;
11577	case VMX_EXIT_VMFUNC: SET_BOTH(VMX_VMFUNC); break;
11578	case VMX_EXIT_RDSEED: SET_BOTH(RDSEED); break;
11579	case VMX_EXIT_XSAVES: SET_BOTH(XSAVES); break;
11580	case VMX_EXIT_XRSTORS: SET_BOTH(XRSTORS); break;
11581
11582	/* Events that aren't relevant at this point. */
11583	case VMX_EXIT_EXT_INT:
11584	case VMX_EXIT_INT_WINDOW:
11585	case VMX_EXIT_NMI_WINDOW:
11586	case VMX_EXIT_TPR_BELOW_THRESHOLD:
11587	case VMX_EXIT_PREEMPT_TIMER:
11588	case VMX_EXIT_IO_INSTR:
11589	break;
11590
11591	/* Errors and unexpected events. */
11592	case VMX_EXIT_INIT_SIGNAL:
11593	case VMX_EXIT_SIPI:
11594	case VMX_EXIT_IO_SMI:
11595	case VMX_EXIT_SMI:
11596	case VMX_EXIT_ERR_INVALID_GUEST_STATE:
11597	case VMX_EXIT_ERR_MSR_LOAD:
11598	case VMX_EXIT_ERR_MACHINE_CHECK:
11599	break;
11600
11601	default:
11602	AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
11603	break;
11604	}
11605	#undef SET_BOTH
11606	#undef SET_EXIT
11607
11608	/*
11609	* Dtrace tracepoints go first. We do them here at once so we don't
11610	* have to copy the guest state saving and stuff a few dozen times.
11611	* Down side is that we've got to repeat the switch, though this time
11612	* we use enmEvent since the probes are a subset of what DBGF does.
11613	*/
11614	if (fDtrace1 \|\| fDtrace2)
11615	{
11616	hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
11617	hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
11618	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
11619	switch (enmEvent1)
11620	{
11621	/** @todo consider which extra parameters would be helpful for each probe. */
11622	case DBGFEVENT_END: break;
11623	case DBGFEVENT_XCPT_DE: VBOXVMM_XCPT_DE(pVCpu, pCtx); break;
11624	case DBGFEVENT_XCPT_DB: VBOXVMM_XCPT_DB(pVCpu, pCtx, pCtx->dr[6]); break;
11625	case DBGFEVENT_XCPT_BP: VBOXVMM_XCPT_BP(pVCpu, pCtx); break;
11626	case DBGFEVENT_XCPT_OF: VBOXVMM_XCPT_OF(pVCpu, pCtx); break;
11627	case DBGFEVENT_XCPT_BR: VBOXVMM_XCPT_BR(pVCpu, pCtx); break;
11628	case DBGFEVENT_XCPT_UD: VBOXVMM_XCPT_UD(pVCpu, pCtx); break;
11629	case DBGFEVENT_XCPT_NM: VBOXVMM_XCPT_NM(pVCpu, pCtx); break;
11630	case DBGFEVENT_XCPT_DF: VBOXVMM_XCPT_DF(pVCpu, pCtx); break;
11631	case DBGFEVENT_XCPT_TS: VBOXVMM_XCPT_TS(pVCpu, pCtx, uEventArg); break;
11632	case DBGFEVENT_XCPT_NP: VBOXVMM_XCPT_NP(pVCpu, pCtx, uEventArg); break;
11633	case DBGFEVENT_XCPT_SS: VBOXVMM_XCPT_SS(pVCpu, pCtx, uEventArg); break;
11634	case DBGFEVENT_XCPT_GP: VBOXVMM_XCPT_GP(pVCpu, pCtx, uEventArg); break;
11635	case DBGFEVENT_XCPT_PF: VBOXVMM_XCPT_PF(pVCpu, pCtx, uEventArg, pCtx->cr2); break;
11636	case DBGFEVENT_XCPT_MF: VBOXVMM_XCPT_MF(pVCpu, pCtx); break;
11637	case DBGFEVENT_XCPT_AC: VBOXVMM_XCPT_AC(pVCpu, pCtx); break;
11638	case DBGFEVENT_XCPT_XF: VBOXVMM_XCPT_XF(pVCpu, pCtx); break;
11639	case DBGFEVENT_XCPT_VE: VBOXVMM_XCPT_VE(pVCpu, pCtx); break;
11640	case DBGFEVENT_XCPT_SX: VBOXVMM_XCPT_SX(pVCpu, pCtx, uEventArg); break;
11641	case DBGFEVENT_INTERRUPT_SOFTWARE: VBOXVMM_INT_SOFTWARE(pVCpu, pCtx, (uint8_t)uEventArg); break;
11642	case DBGFEVENT_INSTR_CPUID: VBOXVMM_INSTR_CPUID(pVCpu, pCtx, pCtx->eax, pCtx->ecx); break;
11643	case DBGFEVENT_INSTR_GETSEC: VBOXVMM_INSTR_GETSEC(pVCpu, pCtx); break;
11644	case DBGFEVENT_INSTR_HALT: VBOXVMM_INSTR_HALT(pVCpu, pCtx); break;
11645	case DBGFEVENT_INSTR_INVD: VBOXVMM_INSTR_INVD(pVCpu, pCtx); break;
11646	case DBGFEVENT_INSTR_INVLPG: VBOXVMM_INSTR_INVLPG(pVCpu, pCtx); break;
11647	case DBGFEVENT_INSTR_RDPMC: VBOXVMM_INSTR_RDPMC(pVCpu, pCtx); break;
11648	case DBGFEVENT_INSTR_RDTSC: VBOXVMM_INSTR_RDTSC(pVCpu, pCtx); break;
11649	case DBGFEVENT_INSTR_RSM: VBOXVMM_INSTR_RSM(pVCpu, pCtx); break;
11650	case DBGFEVENT_INSTR_CRX_READ: VBOXVMM_INSTR_CRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
11651	case DBGFEVENT_INSTR_CRX_WRITE: VBOXVMM_INSTR_CRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
11652	case DBGFEVENT_INSTR_DRX_READ: VBOXVMM_INSTR_DRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
11653	case DBGFEVENT_INSTR_DRX_WRITE: VBOXVMM_INSTR_DRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
11654	case DBGFEVENT_INSTR_RDMSR: VBOXVMM_INSTR_RDMSR(pVCpu, pCtx, pCtx->ecx); break;
11655	case DBGFEVENT_INSTR_WRMSR: VBOXVMM_INSTR_WRMSR(pVCpu, pCtx, pCtx->ecx,
11656	RT_MAKE_U64(pCtx->eax, pCtx->edx)); break;
11657	case DBGFEVENT_INSTR_MWAIT: VBOXVMM_INSTR_MWAIT(pVCpu, pCtx); break;
11658	case DBGFEVENT_INSTR_MONITOR: VBOXVMM_INSTR_MONITOR(pVCpu, pCtx); break;
11659	case DBGFEVENT_INSTR_PAUSE: VBOXVMM_INSTR_PAUSE(pVCpu, pCtx); break;
11660	case DBGFEVENT_INSTR_SGDT: VBOXVMM_INSTR_SGDT(pVCpu, pCtx); break;
11661	case DBGFEVENT_INSTR_SIDT: VBOXVMM_INSTR_SIDT(pVCpu, pCtx); break;
11662	case DBGFEVENT_INSTR_LGDT: VBOXVMM_INSTR_LGDT(pVCpu, pCtx); break;
11663	case DBGFEVENT_INSTR_LIDT: VBOXVMM_INSTR_LIDT(pVCpu, pCtx); break;
11664	case DBGFEVENT_INSTR_SLDT: VBOXVMM_INSTR_SLDT(pVCpu, pCtx); break;
11665	case DBGFEVENT_INSTR_STR: VBOXVMM_INSTR_STR(pVCpu, pCtx); break;
11666	case DBGFEVENT_INSTR_LLDT: VBOXVMM_INSTR_LLDT(pVCpu, pCtx); break;
11667	case DBGFEVENT_INSTR_LTR: VBOXVMM_INSTR_LTR(pVCpu, pCtx); break;
11668	case DBGFEVENT_INSTR_RDTSCP: VBOXVMM_INSTR_RDTSCP(pVCpu, pCtx); break;
11669	case DBGFEVENT_INSTR_WBINVD: VBOXVMM_INSTR_WBINVD(pVCpu, pCtx); break;
11670	case DBGFEVENT_INSTR_XSETBV: VBOXVMM_INSTR_XSETBV(pVCpu, pCtx); break;
11671	case DBGFEVENT_INSTR_RDRAND: VBOXVMM_INSTR_RDRAND(pVCpu, pCtx); break;
11672	case DBGFEVENT_INSTR_RDSEED: VBOXVMM_INSTR_RDSEED(pVCpu, pCtx); break;
11673	case DBGFEVENT_INSTR_XSAVES: VBOXVMM_INSTR_XSAVES(pVCpu, pCtx); break;
11674	case DBGFEVENT_INSTR_XRSTORS: VBOXVMM_INSTR_XRSTORS(pVCpu, pCtx); break;
11675	case DBGFEVENT_INSTR_VMM_CALL: VBOXVMM_INSTR_VMM_CALL(pVCpu, pCtx); break;
11676	case DBGFEVENT_INSTR_VMX_VMCLEAR: VBOXVMM_INSTR_VMX_VMCLEAR(pVCpu, pCtx); break;
11677	case DBGFEVENT_INSTR_VMX_VMLAUNCH: VBOXVMM_INSTR_VMX_VMLAUNCH(pVCpu, pCtx); break;
11678	case DBGFEVENT_INSTR_VMX_VMPTRLD: VBOXVMM_INSTR_VMX_VMPTRLD(pVCpu, pCtx); break;
11679	case DBGFEVENT_INSTR_VMX_VMPTRST: VBOXVMM_INSTR_VMX_VMPTRST(pVCpu, pCtx); break;
11680	case DBGFEVENT_INSTR_VMX_VMREAD: VBOXVMM_INSTR_VMX_VMREAD(pVCpu, pCtx); break;
11681	case DBGFEVENT_INSTR_VMX_VMRESUME: VBOXVMM_INSTR_VMX_VMRESUME(pVCpu, pCtx); break;
11682	case DBGFEVENT_INSTR_VMX_VMWRITE: VBOXVMM_INSTR_VMX_VMWRITE(pVCpu, pCtx); break;
11683	case DBGFEVENT_INSTR_VMX_VMXOFF: VBOXVMM_INSTR_VMX_VMXOFF(pVCpu, pCtx); break;
11684	case DBGFEVENT_INSTR_VMX_VMXON: VBOXVMM_INSTR_VMX_VMXON(pVCpu, pCtx); break;
11685	case DBGFEVENT_INSTR_VMX_INVEPT: VBOXVMM_INSTR_VMX_INVEPT(pVCpu, pCtx); break;
11686	case DBGFEVENT_INSTR_VMX_INVVPID: VBOXVMM_INSTR_VMX_INVVPID(pVCpu, pCtx); break;
11687	case DBGFEVENT_INSTR_VMX_INVPCID: VBOXVMM_INSTR_VMX_INVPCID(pVCpu, pCtx); break;
11688	case DBGFEVENT_INSTR_VMX_VMFUNC: VBOXVMM_INSTR_VMX_VMFUNC(pVCpu, pCtx); break;
11689	default: AssertMsgFailed(("enmEvent1=%d uExitReason=%d\n", enmEvent1, uExitReason)); break;
11690	}
11691	switch (enmEvent2)
11692	{
11693	/** @todo consider which extra parameters would be helpful for each probe. */
11694	case DBGFEVENT_END: break;
11695	case DBGFEVENT_EXIT_TASK_SWITCH: VBOXVMM_EXIT_TASK_SWITCH(pVCpu, pCtx); break;
11696	case DBGFEVENT_EXIT_CPUID: VBOXVMM_EXIT_CPUID(pVCpu, pCtx, pCtx->eax, pCtx->ecx); break;
11697	case DBGFEVENT_EXIT_GETSEC: VBOXVMM_EXIT_GETSEC(pVCpu, pCtx); break;
11698	case DBGFEVENT_EXIT_HALT: VBOXVMM_EXIT_HALT(pVCpu, pCtx); break;
11699	case DBGFEVENT_EXIT_INVD: VBOXVMM_EXIT_INVD(pVCpu, pCtx); break;
11700	case DBGFEVENT_EXIT_INVLPG: VBOXVMM_EXIT_INVLPG(pVCpu, pCtx); break;
11701	case DBGFEVENT_EXIT_RDPMC: VBOXVMM_EXIT_RDPMC(pVCpu, pCtx); break;
11702	case DBGFEVENT_EXIT_RDTSC: VBOXVMM_EXIT_RDTSC(pVCpu, pCtx); break;
11703	case DBGFEVENT_EXIT_RSM: VBOXVMM_EXIT_RSM(pVCpu, pCtx); break;
11704	case DBGFEVENT_EXIT_CRX_READ: VBOXVMM_EXIT_CRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
11705	case DBGFEVENT_EXIT_CRX_WRITE: VBOXVMM_EXIT_CRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
11706	case DBGFEVENT_EXIT_DRX_READ: VBOXVMM_EXIT_DRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
11707	case DBGFEVENT_EXIT_DRX_WRITE: VBOXVMM_EXIT_DRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
11708	case DBGFEVENT_EXIT_RDMSR: VBOXVMM_EXIT_RDMSR(pVCpu, pCtx, pCtx->ecx); break;
11709	case DBGFEVENT_EXIT_WRMSR: VBOXVMM_EXIT_WRMSR(pVCpu, pCtx, pCtx->ecx,
11710	RT_MAKE_U64(pCtx->eax, pCtx->edx)); break;
11711	case DBGFEVENT_EXIT_MWAIT: VBOXVMM_EXIT_MWAIT(pVCpu, pCtx); break;
11712	case DBGFEVENT_EXIT_MONITOR: VBOXVMM_EXIT_MONITOR(pVCpu, pCtx); break;
11713	case DBGFEVENT_EXIT_PAUSE: VBOXVMM_EXIT_PAUSE(pVCpu, pCtx); break;
11714	case DBGFEVENT_EXIT_SGDT: VBOXVMM_EXIT_SGDT(pVCpu, pCtx); break;
11715	case DBGFEVENT_EXIT_SIDT: VBOXVMM_EXIT_SIDT(pVCpu, pCtx); break;
11716	case DBGFEVENT_EXIT_LGDT: VBOXVMM_EXIT_LGDT(pVCpu, pCtx); break;
11717	case DBGFEVENT_EXIT_LIDT: VBOXVMM_EXIT_LIDT(pVCpu, pCtx); break;
11718	case DBGFEVENT_EXIT_SLDT: VBOXVMM_EXIT_SLDT(pVCpu, pCtx); break;
11719	case DBGFEVENT_EXIT_STR: VBOXVMM_EXIT_STR(pVCpu, pCtx); break;
11720	case DBGFEVENT_EXIT_LLDT: VBOXVMM_EXIT_LLDT(pVCpu, pCtx); break;
11721	case DBGFEVENT_EXIT_LTR: VBOXVMM_EXIT_LTR(pVCpu, pCtx); break;
11722	case DBGFEVENT_EXIT_RDTSCP: VBOXVMM_EXIT_RDTSCP(pVCpu, pCtx); break;
11723	case DBGFEVENT_EXIT_WBINVD: VBOXVMM_EXIT_WBINVD(pVCpu, pCtx); break;
11724	case DBGFEVENT_EXIT_XSETBV: VBOXVMM_EXIT_XSETBV(pVCpu, pCtx); break;
11725	case DBGFEVENT_EXIT_RDRAND: VBOXVMM_EXIT_RDRAND(pVCpu, pCtx); break;
11726	case DBGFEVENT_EXIT_RDSEED: VBOXVMM_EXIT_RDSEED(pVCpu, pCtx); break;
11727	case DBGFEVENT_EXIT_XSAVES: VBOXVMM_EXIT_XSAVES(pVCpu, pCtx); break;
11728	case DBGFEVENT_EXIT_XRSTORS: VBOXVMM_EXIT_XRSTORS(pVCpu, pCtx); break;
11729	case DBGFEVENT_EXIT_VMM_CALL: VBOXVMM_EXIT_VMM_CALL(pVCpu, pCtx); break;
11730	case DBGFEVENT_EXIT_VMX_VMCLEAR: VBOXVMM_EXIT_VMX_VMCLEAR(pVCpu, pCtx); break;
11731	case DBGFEVENT_EXIT_VMX_VMLAUNCH: VBOXVMM_EXIT_VMX_VMLAUNCH(pVCpu, pCtx); break;
11732	case DBGFEVENT_EXIT_VMX_VMPTRLD: VBOXVMM_EXIT_VMX_VMPTRLD(pVCpu, pCtx); break;
11733	case DBGFEVENT_EXIT_VMX_VMPTRST: VBOXVMM_EXIT_VMX_VMPTRST(pVCpu, pCtx); break;
11734	case DBGFEVENT_EXIT_VMX_VMREAD: VBOXVMM_EXIT_VMX_VMREAD(pVCpu, pCtx); break;
11735	case DBGFEVENT_EXIT_VMX_VMRESUME: VBOXVMM_EXIT_VMX_VMRESUME(pVCpu, pCtx); break;
11736	case DBGFEVENT_EXIT_VMX_VMWRITE: VBOXVMM_EXIT_VMX_VMWRITE(pVCpu, pCtx); break;
11737	case DBGFEVENT_EXIT_VMX_VMXOFF: VBOXVMM_EXIT_VMX_VMXOFF(pVCpu, pCtx); break;
11738	case DBGFEVENT_EXIT_VMX_VMXON: VBOXVMM_EXIT_VMX_VMXON(pVCpu, pCtx); break;
11739	case DBGFEVENT_EXIT_VMX_INVEPT: VBOXVMM_EXIT_VMX_INVEPT(pVCpu, pCtx); break;
11740	case DBGFEVENT_EXIT_VMX_INVVPID: VBOXVMM_EXIT_VMX_INVVPID(pVCpu, pCtx); break;
11741	case DBGFEVENT_EXIT_VMX_INVPCID: VBOXVMM_EXIT_VMX_INVPCID(pVCpu, pCtx); break;
11742	case DBGFEVENT_EXIT_VMX_VMFUNC: VBOXVMM_EXIT_VMX_VMFUNC(pVCpu, pCtx); break;
11743	case DBGFEVENT_EXIT_VMX_EPT_MISCONFIG: VBOXVMM_EXIT_VMX_EPT_MISCONFIG(pVCpu, pCtx); break;
11744	case DBGFEVENT_EXIT_VMX_EPT_VIOLATION: VBOXVMM_EXIT_VMX_EPT_VIOLATION(pVCpu, pCtx); break;
11745	case DBGFEVENT_EXIT_VMX_VAPIC_ACCESS: VBOXVMM_EXIT_VMX_VAPIC_ACCESS(pVCpu, pCtx); break;
11746	case DBGFEVENT_EXIT_VMX_VAPIC_WRITE: VBOXVMM_EXIT_VMX_VAPIC_WRITE(pVCpu, pCtx); break;
11747	default: AssertMsgFailed(("enmEvent2=%d uExitReason=%d\n", enmEvent2, uExitReason)); break;
11748	}
11749	}
11750
11751	/*
11752	* Fire of the DBGF event, if enabled (our check here is just a quick one,
11753	* the DBGF call will do a full check).
11754	*
11755	* Note! DBGF sets DBGFEVENT_INTERRUPT_SOFTWARE in the bitmap.
11756	* Note! If we have to events, we prioritize the first, i.e. the instruction
11757	* one, in order to avoid event nesting.
11758	*/
11759	PVM pVM = pVCpu->CTX_SUFF(pVM);
11760	if ( enmEvent1 != DBGFEVENT_END
11761	&& DBGF_IS_EVENT_ENABLED(pVM, enmEvent1))
11762	{
11763	hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
11764	VBOXSTRICTRC rcStrict = DBGFEventGenericWithArgs(pVM, pVCpu, enmEvent1, DBGFEVENTCTX_HM, 1, uEventArg);
11765	if (rcStrict != VINF_SUCCESS)
11766	return rcStrict;
11767	}
11768	else if ( enmEvent2 != DBGFEVENT_END
11769	&& DBGF_IS_EVENT_ENABLED(pVM, enmEvent2))
11770	{
11771	hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
11772	VBOXSTRICTRC rcStrict = DBGFEventGenericWithArgs(pVM, pVCpu, enmEvent2, DBGFEVENTCTX_HM, 1, uEventArg);
11773	if (rcStrict != VINF_SUCCESS)
11774	return rcStrict;
11775	}
11776
11777	return VINF_SUCCESS;
11778	}
11779
11780
11781	/**
11782	* Single-stepping VM-exit filtering.
11783	*
11784	* This is preprocessing the VM-exits and deciding whether we've gotten far
11785	* enough to return VINF_EM_DBG_STEPPED already. If not, normal VM-exit
11786	* handling is performed.
11787	*
11788	* @returns Strict VBox status code (i.e. informational status codes too).
11789	* @param pVCpu The cross context virtual CPU structure of the calling EMT.
11790	* @param pVmxTransient The VMX-transient structure.
11791	* @param pDbgState The debug state.
11792	*/
11793	DECLINLINE(VBOXSTRICTRC) hmR0VmxRunDebugHandleExit(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState)
11794	{
11795	/*
11796	* Expensive (saves context) generic dtrace VM-exit probe.
11797	*/
11798	uint32_t const uExitReason = pVmxTransient->uExitReason;
11799	if (!VBOXVMM_R0_HMVMX_VMEXIT_ENABLED())
11800	{ /* more likely */ }
11801	else
11802	{
11803	hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
11804	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
11805	AssertRC(rc);
11806	VBOXVMM_R0_HMVMX_VMEXIT(pVCpu, &pVCpu->cpum.GstCtx, pVmxTransient->uExitReason, pVmxTransient->uExitQual);
11807	}
11808
11809	/*
11810	* Check for host NMI, just to get that out of the way.
11811	*/
11812	if (uExitReason != VMX_EXIT_XCPT_OR_NMI)
11813	{ /* normally likely */ }
11814	else
11815	{
11816	int rc2 = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11817	AssertRCReturn(rc2, rc2);
11818	uint32_t uIntType = VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo);
11819	if (uIntType == VMX_EXIT_INT_INFO_TYPE_NMI)
11820	return hmR0VmxExitXcptOrNmi(pVCpu, pVmxTransient);
11821	}
11822
11823	/*
11824	* Check for single stepping event if we're stepping.
11825	*/
11826	if (pVCpu->hm.s.fSingleInstruction)
11827	{
11828	switch (uExitReason)
11829	{
11830	case VMX_EXIT_MTF:
11831	return hmR0VmxExitMtf(pVCpu, pVmxTransient);
11832
11833	/* Various events: */
11834	case VMX_EXIT_XCPT_OR_NMI:
11835	case VMX_EXIT_EXT_INT:
11836	case VMX_EXIT_TRIPLE_FAULT:
11837	case VMX_EXIT_INT_WINDOW:
11838	case VMX_EXIT_NMI_WINDOW:
11839	case VMX_EXIT_TASK_SWITCH:
11840	case VMX_EXIT_TPR_BELOW_THRESHOLD:
11841	case VMX_EXIT_APIC_ACCESS:
11842	case VMX_EXIT_EPT_VIOLATION:
11843	case VMX_EXIT_EPT_MISCONFIG:
11844	case VMX_EXIT_PREEMPT_TIMER:
11845
11846	/* Instruction specific VM-exits: */
11847	case VMX_EXIT_CPUID:
11848	case VMX_EXIT_GETSEC:
11849	case VMX_EXIT_HLT:
11850	case VMX_EXIT_INVD:
11851	case VMX_EXIT_INVLPG:
11852	case VMX_EXIT_RDPMC:
11853	case VMX_EXIT_RDTSC:
11854	case VMX_EXIT_RSM:
11855	case VMX_EXIT_VMCALL:
11856	case VMX_EXIT_VMCLEAR:
11857	case VMX_EXIT_VMLAUNCH:
11858	case VMX_EXIT_VMPTRLD:
11859	case VMX_EXIT_VMPTRST:
11860	case VMX_EXIT_VMREAD:
11861	case VMX_EXIT_VMRESUME:
11862	case VMX_EXIT_VMWRITE:
11863	case VMX_EXIT_VMXOFF:
11864	case VMX_EXIT_VMXON:
11865	case VMX_EXIT_MOV_CRX:
11866	case VMX_EXIT_MOV_DRX:
11867	case VMX_EXIT_IO_INSTR:
11868	case VMX_EXIT_RDMSR:
11869	case VMX_EXIT_WRMSR:
11870	case VMX_EXIT_MWAIT:
11871	case VMX_EXIT_MONITOR:
11872	case VMX_EXIT_PAUSE:
11873	case VMX_EXIT_GDTR_IDTR_ACCESS:
11874	case VMX_EXIT_LDTR_TR_ACCESS:
11875	case VMX_EXIT_INVEPT:
11876	case VMX_EXIT_RDTSCP:
11877	case VMX_EXIT_INVVPID:
11878	case VMX_EXIT_WBINVD:
11879	case VMX_EXIT_XSETBV:
11880	case VMX_EXIT_RDRAND:
11881	case VMX_EXIT_INVPCID:
11882	case VMX_EXIT_VMFUNC:
11883	case VMX_EXIT_RDSEED:
11884	case VMX_EXIT_XSAVES:
11885	case VMX_EXIT_XRSTORS:
11886	{
11887	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
11888	AssertRCReturn(rc, rc);
11889	if ( pVCpu->cpum.GstCtx.rip != pDbgState->uRipStart
11890	\|\| pVCpu->cpum.GstCtx.cs.Sel != pDbgState->uCsStart)
11891	return VINF_EM_DBG_STEPPED;
11892	break;
11893	}
11894
11895	/* Errors and unexpected events: */
11896	case VMX_EXIT_INIT_SIGNAL:
11897	case VMX_EXIT_SIPI:
11898	case VMX_EXIT_IO_SMI:
11899	case VMX_EXIT_SMI:
11900	case VMX_EXIT_ERR_INVALID_GUEST_STATE:
11901	case VMX_EXIT_ERR_MSR_LOAD:
11902	case VMX_EXIT_ERR_MACHINE_CHECK:
11903	case VMX_EXIT_APIC_WRITE: /* Some talk about this being fault like, so I guess we must process it? */
11904	break;
11905
11906	default:
11907	AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
11908	break;
11909	}
11910	}
11911
11912	/*
11913	* Check for debugger event breakpoints and dtrace probes.
11914	*/
11915	if ( uExitReason < RT_ELEMENTS(pDbgState->bmExitsToCheck) * 32U
11916	&& ASMBitTest(pDbgState->bmExitsToCheck, uExitReason) )
11917	{
11918	VBOXSTRICTRC rcStrict = hmR0VmxHandleExitDtraceEvents(pVCpu, pVmxTransient, uExitReason);
11919	if (rcStrict != VINF_SUCCESS)
11920	return rcStrict;
11921	}
11922
11923	/*
11924	* Normal processing.
11925	*/
11926	#ifdef HMVMX_USE_FUNCTION_TABLE
11927	return g_apfnVMExitHandlers[uExitReason](pVCpu, pVmxTransient);
11928	#else
11929	return hmR0VmxHandleExit(pVCpu, pVmxTransient, uExitReason);
11930	#endif
11931	}
11932
11933
11934	/**
11935	* Single steps guest code using hardware-assisted VMX.
11936	*
11937	* This is -not- the same as the guest single-stepping itself (say using EFLAGS.TF)
11938	* but single-stepping through the hypervisor debugger.
11939	*
11940	* @returns Strict VBox status code (i.e. informational status codes too).
11941	* @param pVCpu The cross context virtual CPU structure.
11942	* @param pcLoops Pointer to the number of executed loops.
11943	*
11944	* @note Mostly the same as hmR0VmxRunGuestCodeNormal().
11945	*/
11946	static VBOXSTRICTRC hmR0VmxRunGuestCodeDebug(PVMCPU pVCpu, uint32_t *pcLoops)
11947	{
11948	uint32_t const cMaxResumeLoops = pVCpu->CTX_SUFF(pVM)->hm.s.cMaxResumeLoops;
11949	Assert(pcLoops);
11950	Assert(*pcLoops <= cMaxResumeLoops);
11951
11952	VMXTRANSIENT VmxTransient;
11953	RT_ZERO(VmxTransient);
11954	VmxTransient.pVmcsInfo = hmGetVmxActiveVmcsInfo(pVCpu);
11955
11956	/* Set HMCPU indicators. */
11957	bool const fSavedSingleInstruction = pVCpu->hm.s.fSingleInstruction;
11958	pVCpu->hm.s.fSingleInstruction = pVCpu->hm.s.fSingleInstruction \|\| DBGFIsStepping(pVCpu);
11959	pVCpu->hm.s.fDebugWantRdTscExit = false;
11960	pVCpu->hm.s.fUsingDebugLoop = true;
11961
11962	/* State we keep to help modify and later restore the VMCS fields we alter, and for detecting steps. */
11963	VMXRUNDBGSTATE DbgState;
11964	hmR0VmxRunDebugStateInit(pVCpu, &VmxTransient, &DbgState);
11965	hmR0VmxPreRunGuestDebugStateUpdate(pVCpu, &VmxTransient, &DbgState);
11966
11967	/*
11968	* The loop.
11969	*/
11970	VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
11971	for (;;)
11972	{
11973	Assert(!HMR0SuspendPending());
11974	HMVMX_ASSERT_CPU_SAFE(pVCpu);
11975	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
11976	bool fStepping = pVCpu->hm.s.fSingleInstruction;
11977
11978	/* Set up VM-execution controls the next two can respond to. */
11979	hmR0VmxPreRunGuestDebugStateApply(pVCpu, &VmxTransient, &DbgState);
11980
11981	/*
11982	* Preparatory work for running guest code, this may force us to
11983	* return to ring-3.
11984	*
11985	* Warning! This bugger disables interrupts on VINF_SUCCESS!
11986	*/
11987	rcStrict = hmR0VmxPreRunGuest(pVCpu, &VmxTransient, fStepping);
11988	if (rcStrict != VINF_SUCCESS)
11989	break;
11990
11991	/* Interrupts are disabled at this point! */
11992	hmR0VmxPreRunGuestCommitted(pVCpu, &VmxTransient);
11993
11994	/* Override any obnoxious code in the above two calls. */
11995	hmR0VmxPreRunGuestDebugStateApply(pVCpu, &VmxTransient, &DbgState);
11996
11997	/*
11998	* Finally execute the guest.
11999	*/
12000	int rcRun = hmR0VmxRunGuest(pVCpu, &VmxTransient);
12001
12002	hmR0VmxPostRunGuest(pVCpu, &VmxTransient, rcRun);
12003	/* Interrupts are re-enabled at this point! */
12004
12005	/* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
12006	if (RT_SUCCESS(rcRun))
12007	{ /* very likely */ }
12008	else
12009	{
12010	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
12011	hmR0VmxReportWorldSwitchError(pVCpu, rcRun, &VmxTransient);
12012	return rcRun;
12013	}
12014
12015	/* Profile the VM-exit. */
12016	AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
12017	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
12018	STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
12019	STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
12020	HMVMX_START_EXIT_DISPATCH_PROF();
12021
12022	VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, &pVCpu->cpum.GstCtx, VmxTransient.uExitReason);
12023
12024	/*
12025	* Handle the VM-exit - we quit earlier on certain VM-exits, see hmR0VmxHandleExitDebug().
12026	*/
12027	rcStrict = hmR0VmxRunDebugHandleExit(pVCpu, &VmxTransient, &DbgState);
12028	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
12029	if (rcStrict != VINF_SUCCESS)
12030	break;
12031	if (++(*pcLoops) > cMaxResumeLoops)
12032	{
12033	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
12034	rcStrict = VINF_EM_RAW_INTERRUPT;
12035	break;
12036	}
12037
12038	/*
12039	* Stepping: Did the RIP change, if so, consider it a single step.
12040	* Otherwise, make sure one of the TFs gets set.
12041	*/
12042	if (fStepping)
12043	{
12044	int rc = hmR0VmxImportGuestState(pVCpu, VmxTransient.pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
12045	AssertRC(rc);
12046	if ( pVCpu->cpum.GstCtx.rip != DbgState.uRipStart
12047	\|\| pVCpu->cpum.GstCtx.cs.Sel != DbgState.uCsStart)
12048	{
12049	rcStrict = VINF_EM_DBG_STEPPED;
12050	break;
12051	}
12052	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_DR7);
12053	}
12054
12055	/*
12056	* Update when dtrace settings changes (DBGF kicks us, so no need to check).
12057	*/
12058	if (VBOXVMM_GET_SETTINGS_SEQ_NO() != DbgState.uDtraceSettingsSeqNo)
12059	hmR0VmxPreRunGuestDebugStateUpdate(pVCpu, &VmxTransient, &DbgState);
12060	}
12061
12062	/*
12063	* Clear the X86_EFL_TF if necessary.
12064	*/
12065	if (pVCpu->hm.s.fClearTrapFlag)
12066	{
12067	int rc = hmR0VmxImportGuestState(pVCpu, VmxTransient.pVmcsInfo, CPUMCTX_EXTRN_RFLAGS);
12068	AssertRC(rc);
12069	pVCpu->hm.s.fClearTrapFlag = false;
12070	pVCpu->cpum.GstCtx.eflags.Bits.u1TF = 0;
12071	}
12072	/** @todo there seems to be issues with the resume flag when the monitor trap
12073	* flag is pending without being used. Seen early in bios init when
12074	* accessing APIC page in protected mode. */
12075
12076	/*
12077	* Restore VM-exit control settings as we may not re-enter this function the
12078	* next time around.
12079	*/
12080	rcStrict = hmR0VmxRunDebugStateRevert(pVCpu, &VmxTransient, &DbgState, rcStrict);
12081
12082	/* Restore HMCPU indicators. */
12083	pVCpu->hm.s.fUsingDebugLoop = false;
12084	pVCpu->hm.s.fDebugWantRdTscExit = false;
12085	pVCpu->hm.s.fSingleInstruction = fSavedSingleInstruction;
12086
12087	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
12088	return rcStrict;
12089	}
12090
12091
12092	/** @} */
12093
12094
12095	/**
12096	* Checks if any expensive dtrace probes are enabled and we should go to the
12097	* debug loop.
12098	*
12099	* @returns true if we should use debug loop, false if not.
12100	*/
12101	static bool hmR0VmxAnyExpensiveProbesEnabled(void)
12102	{
12103	/* It's probably faster to OR the raw 32-bit counter variables together.
12104	Since the variables are in an array and the probes are next to one
12105	another (more or less), we have good locality. So, better read
12106	eight-nine cache lines ever time and only have one conditional, than
12107	128+ conditionals, right? */
12108	return ( VBOXVMM_R0_HMVMX_VMEXIT_ENABLED_RAW() /* expensive too due to context */
12109	\| VBOXVMM_XCPT_DE_ENABLED_RAW()
12110	\| VBOXVMM_XCPT_DB_ENABLED_RAW()
12111	\| VBOXVMM_XCPT_BP_ENABLED_RAW()
12112	\| VBOXVMM_XCPT_OF_ENABLED_RAW()
12113	\| VBOXVMM_XCPT_BR_ENABLED_RAW()
12114	\| VBOXVMM_XCPT_UD_ENABLED_RAW()
12115	\| VBOXVMM_XCPT_NM_ENABLED_RAW()
12116	\| VBOXVMM_XCPT_DF_ENABLED_RAW()
12117	\| VBOXVMM_XCPT_TS_ENABLED_RAW()
12118	\| VBOXVMM_XCPT_NP_ENABLED_RAW()
12119	\| VBOXVMM_XCPT_SS_ENABLED_RAW()
12120	\| VBOXVMM_XCPT_GP_ENABLED_RAW()
12121	\| VBOXVMM_XCPT_PF_ENABLED_RAW()
12122	\| VBOXVMM_XCPT_MF_ENABLED_RAW()
12123	\| VBOXVMM_XCPT_AC_ENABLED_RAW()
12124	\| VBOXVMM_XCPT_XF_ENABLED_RAW()
12125	\| VBOXVMM_XCPT_VE_ENABLED_RAW()
12126	\| VBOXVMM_XCPT_SX_ENABLED_RAW()
12127	\| VBOXVMM_INT_SOFTWARE_ENABLED_RAW()
12128	\| VBOXVMM_INT_HARDWARE_ENABLED_RAW()
12129	) != 0
12130	\|\| ( VBOXVMM_INSTR_HALT_ENABLED_RAW()
12131	\| VBOXVMM_INSTR_MWAIT_ENABLED_RAW()
12132	\| VBOXVMM_INSTR_MONITOR_ENABLED_RAW()
12133	\| VBOXVMM_INSTR_CPUID_ENABLED_RAW()
12134	\| VBOXVMM_INSTR_INVD_ENABLED_RAW()
12135	\| VBOXVMM_INSTR_WBINVD_ENABLED_RAW()
12136	\| VBOXVMM_INSTR_INVLPG_ENABLED_RAW()
12137	\| VBOXVMM_INSTR_RDTSC_ENABLED_RAW()
12138	\| VBOXVMM_INSTR_RDTSCP_ENABLED_RAW()
12139	\| VBOXVMM_INSTR_RDPMC_ENABLED_RAW()
12140	\| VBOXVMM_INSTR_RDMSR_ENABLED_RAW()
12141	\| VBOXVMM_INSTR_WRMSR_ENABLED_RAW()
12142	\| VBOXVMM_INSTR_CRX_READ_ENABLED_RAW()
12143	\| VBOXVMM_INSTR_CRX_WRITE_ENABLED_RAW()
12144	\| VBOXVMM_INSTR_DRX_READ_ENABLED_RAW()
12145	\| VBOXVMM_INSTR_DRX_WRITE_ENABLED_RAW()
12146	\| VBOXVMM_INSTR_PAUSE_ENABLED_RAW()
12147	\| VBOXVMM_INSTR_XSETBV_ENABLED_RAW()
12148	\| VBOXVMM_INSTR_SIDT_ENABLED_RAW()
12149	\| VBOXVMM_INSTR_LIDT_ENABLED_RAW()
12150	\| VBOXVMM_INSTR_SGDT_ENABLED_RAW()
12151	\| VBOXVMM_INSTR_LGDT_ENABLED_RAW()
12152	\| VBOXVMM_INSTR_SLDT_ENABLED_RAW()
12153	\| VBOXVMM_INSTR_LLDT_ENABLED_RAW()
12154	\| VBOXVMM_INSTR_STR_ENABLED_RAW()
12155	\| VBOXVMM_INSTR_LTR_ENABLED_RAW()
12156	\| VBOXVMM_INSTR_GETSEC_ENABLED_RAW()
12157	\| VBOXVMM_INSTR_RSM_ENABLED_RAW()
12158	\| VBOXVMM_INSTR_RDRAND_ENABLED_RAW()
12159	\| VBOXVMM_INSTR_RDSEED_ENABLED_RAW()
12160	\| VBOXVMM_INSTR_XSAVES_ENABLED_RAW()
12161	\| VBOXVMM_INSTR_XRSTORS_ENABLED_RAW()
12162	\| VBOXVMM_INSTR_VMM_CALL_ENABLED_RAW()
12163	\| VBOXVMM_INSTR_VMX_VMCLEAR_ENABLED_RAW()
12164	\| VBOXVMM_INSTR_VMX_VMLAUNCH_ENABLED_RAW()
12165	\| VBOXVMM_INSTR_VMX_VMPTRLD_ENABLED_RAW()
12166	\| VBOXVMM_INSTR_VMX_VMPTRST_ENABLED_RAW()
12167	\| VBOXVMM_INSTR_VMX_VMREAD_ENABLED_RAW()
12168	\| VBOXVMM_INSTR_VMX_VMRESUME_ENABLED_RAW()
12169	\| VBOXVMM_INSTR_VMX_VMWRITE_ENABLED_RAW()
12170	\| VBOXVMM_INSTR_VMX_VMXOFF_ENABLED_RAW()
12171	\| VBOXVMM_INSTR_VMX_VMXON_ENABLED_RAW()
12172	\| VBOXVMM_INSTR_VMX_VMFUNC_ENABLED_RAW()
12173	\| VBOXVMM_INSTR_VMX_INVEPT_ENABLED_RAW()
12174	\| VBOXVMM_INSTR_VMX_INVVPID_ENABLED_RAW()
12175	\| VBOXVMM_INSTR_VMX_INVPCID_ENABLED_RAW()
12176	) != 0
12177	\|\| ( VBOXVMM_EXIT_TASK_SWITCH_ENABLED_RAW()
12178	\| VBOXVMM_EXIT_HALT_ENABLED_RAW()
12179	\| VBOXVMM_EXIT_MWAIT_ENABLED_RAW()
12180	\| VBOXVMM_EXIT_MONITOR_ENABLED_RAW()
12181	\| VBOXVMM_EXIT_CPUID_ENABLED_RAW()
12182	\| VBOXVMM_EXIT_INVD_ENABLED_RAW()
12183	\| VBOXVMM_EXIT_WBINVD_ENABLED_RAW()
12184	\| VBOXVMM_EXIT_INVLPG_ENABLED_RAW()
12185	\| VBOXVMM_EXIT_RDTSC_ENABLED_RAW()
12186	\| VBOXVMM_EXIT_RDTSCP_ENABLED_RAW()
12187	\| VBOXVMM_EXIT_RDPMC_ENABLED_RAW()
12188	\| VBOXVMM_EXIT_RDMSR_ENABLED_RAW()
12189	\| VBOXVMM_EXIT_WRMSR_ENABLED_RAW()
12190	\| VBOXVMM_EXIT_CRX_READ_ENABLED_RAW()
12191	\| VBOXVMM_EXIT_CRX_WRITE_ENABLED_RAW()
12192	\| VBOXVMM_EXIT_DRX_READ_ENABLED_RAW()
12193	\| VBOXVMM_EXIT_DRX_WRITE_ENABLED_RAW()
12194	\| VBOXVMM_EXIT_PAUSE_ENABLED_RAW()
12195	\| VBOXVMM_EXIT_XSETBV_ENABLED_RAW()
12196	\| VBOXVMM_EXIT_SIDT_ENABLED_RAW()
12197	\| VBOXVMM_EXIT_LIDT_ENABLED_RAW()
12198	\| VBOXVMM_EXIT_SGDT_ENABLED_RAW()
12199	\| VBOXVMM_EXIT_LGDT_ENABLED_RAW()
12200	\| VBOXVMM_EXIT_SLDT_ENABLED_RAW()
12201	\| VBOXVMM_EXIT_LLDT_ENABLED_RAW()
12202	\| VBOXVMM_EXIT_STR_ENABLED_RAW()
12203	\| VBOXVMM_EXIT_LTR_ENABLED_RAW()
12204	\| VBOXVMM_EXIT_GETSEC_ENABLED_RAW()
12205	\| VBOXVMM_EXIT_RSM_ENABLED_RAW()
12206	\| VBOXVMM_EXIT_RDRAND_ENABLED_RAW()
12207	\| VBOXVMM_EXIT_RDSEED_ENABLED_RAW()
12208	\| VBOXVMM_EXIT_XSAVES_ENABLED_RAW()
12209	\| VBOXVMM_EXIT_XRSTORS_ENABLED_RAW()
12210	\| VBOXVMM_EXIT_VMM_CALL_ENABLED_RAW()
12211	\| VBOXVMM_EXIT_VMX_VMCLEAR_ENABLED_RAW()
12212	\| VBOXVMM_EXIT_VMX_VMLAUNCH_ENABLED_RAW()
12213	\| VBOXVMM_EXIT_VMX_VMPTRLD_ENABLED_RAW()
12214	\| VBOXVMM_EXIT_VMX_VMPTRST_ENABLED_RAW()
12215	\| VBOXVMM_EXIT_VMX_VMREAD_ENABLED_RAW()
12216	\| VBOXVMM_EXIT_VMX_VMRESUME_ENABLED_RAW()
12217	\| VBOXVMM_EXIT_VMX_VMWRITE_ENABLED_RAW()
12218	\| VBOXVMM_EXIT_VMX_VMXOFF_ENABLED_RAW()
12219	\| VBOXVMM_EXIT_VMX_VMXON_ENABLED_RAW()
12220	\| VBOXVMM_EXIT_VMX_VMFUNC_ENABLED_RAW()
12221	\| VBOXVMM_EXIT_VMX_INVEPT_ENABLED_RAW()
12222	\| VBOXVMM_EXIT_VMX_INVVPID_ENABLED_RAW()
12223	\| VBOXVMM_EXIT_VMX_INVPCID_ENABLED_RAW()
12224	\| VBOXVMM_EXIT_VMX_EPT_VIOLATION_ENABLED_RAW()
12225	\| VBOXVMM_EXIT_VMX_EPT_MISCONFIG_ENABLED_RAW()
12226	\| VBOXVMM_EXIT_VMX_VAPIC_ACCESS_ENABLED_RAW()
12227	\| VBOXVMM_EXIT_VMX_VAPIC_WRITE_ENABLED_RAW()
12228	) != 0;
12229	}
12230
12231
12232	/**
12233	* Runs the guest using hardware-assisted VMX.
12234	*
12235	* @returns Strict VBox status code (i.e. informational status codes too).
12236	* @param pVCpu The cross context virtual CPU structure.
12237	*/
12238	VMMR0DECL(VBOXSTRICTRC) VMXR0RunGuestCode(PVMCPU pVCpu)
12239	{
12240	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
12241	Assert(VMMRZCallRing3IsEnabled(pVCpu));
12242	Assert(!ASMAtomicUoReadU64(&pCtx->fExtrn));
12243	HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
12244
12245	VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pCtx);
12246
12247	VBOXSTRICTRC rcStrict;
12248	uint32_t cLoops = 0;
12249	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
12250	bool const fInNestedGuestMode = CPUMIsGuestInVmxNonRootMode(pCtx);
12251	#else
12252	bool const fInNestedGuestMode = false;
12253	#endif
12254	if (!fInNestedGuestMode)
12255	{
12256	if ( !pVCpu->hm.s.fUseDebugLoop
12257	&& (!VBOXVMM_ANY_PROBES_ENABLED() \|\| !hmR0VmxAnyExpensiveProbesEnabled())
12258	&& !DBGFIsStepping(pVCpu)
12259	&& !pVCpu->CTX_SUFF(pVM)->dbgf.ro.cEnabledInt3Breakpoints)
12260	rcStrict = hmR0VmxRunGuestCodeNormal(pVCpu, &cLoops);
12261	else
12262	rcStrict = hmR0VmxRunGuestCodeDebug(pVCpu, &cLoops);
12263	}
12264	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
12265	else
12266	rcStrict = VINF_VMX_VMLAUNCH_VMRESUME;
12267
12268	if (rcStrict == VINF_VMX_VMLAUNCH_VMRESUME)
12269	rcStrict = hmR0VmxRunGuestCodeNested(pVCpu, &cLoops);
12270	#endif
12271
12272	if (rcStrict == VERR_EM_INTERPRETER)
12273	rcStrict = VINF_EM_RAW_EMULATE_INSTR;
12274	else if (rcStrict == VINF_EM_RESET)
12275	rcStrict = VINF_EM_TRIPLE_FAULT;
12276
12277	int rc2 = hmR0VmxExitToRing3(pVCpu, rcStrict);
12278	if (RT_FAILURE(rc2))
12279	{
12280	pVCpu->hm.s.u32HMError = (uint32_t)VBOXSTRICTRC_VAL(rcStrict);
12281	rcStrict = rc2;
12282	}
12283	Assert(!ASMAtomicUoReadU64(&pCtx->fExtrn));
12284	Assert(!VMMRZCallRing3IsNotificationSet(pVCpu));
12285	return rcStrict;
12286	}
12287
12288
12289	#ifndef HMVMX_USE_FUNCTION_TABLE
12290	/**
12291	* Handles a guest VM-exit from hardware-assisted VMX execution.
12292	*
12293	* @returns Strict VBox status code (i.e. informational status codes too).
12294	* @param pVCpu The cross context virtual CPU structure.
12295	* @param pVmxTransient The VMX-transient structure.
12296	*/
12297	DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12298	{
12299	#ifdef DEBUG_ramshankar
12300	#define VMEXIT_CALL_RET(a_fSave, a_CallExpr) \
12301	do { \
12302	if (a_fSave != 0) \
12303	hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL); \
12304	VBOXSTRICTRC rcStrict = a_CallExpr; \
12305	if (a_fSave != 0) \
12306	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST); \
12307	return rcStrict; \
12308	} while (0)
12309	#else
12310	# define VMEXIT_CALL_RET(a_fSave, a_CallExpr) return a_CallExpr
12311	#endif
12312	uint32_t const rcReason = pVmxTransient->uExitReason;
12313	switch (rcReason)
12314	{
12315	case VMX_EXIT_EPT_MISCONFIG: VMEXIT_CALL_RET(0, hmR0VmxExitEptMisconfig(pVCpu, pVmxTransient));
12316	case VMX_EXIT_EPT_VIOLATION: VMEXIT_CALL_RET(0, hmR0VmxExitEptViolation(pVCpu, pVmxTransient));
12317	case VMX_EXIT_IO_INSTR: VMEXIT_CALL_RET(0, hmR0VmxExitIoInstr(pVCpu, pVmxTransient));
12318	case VMX_EXIT_CPUID: VMEXIT_CALL_RET(0, hmR0VmxExitCpuid(pVCpu, pVmxTransient));
12319	case VMX_EXIT_RDTSC: VMEXIT_CALL_RET(0, hmR0VmxExitRdtsc(pVCpu, pVmxTransient));
12320	case VMX_EXIT_RDTSCP: VMEXIT_CALL_RET(0, hmR0VmxExitRdtscp(pVCpu, pVmxTransient));
12321	case VMX_EXIT_APIC_ACCESS: VMEXIT_CALL_RET(0, hmR0VmxExitApicAccess(pVCpu, pVmxTransient));
12322	case VMX_EXIT_XCPT_OR_NMI: VMEXIT_CALL_RET(0, hmR0VmxExitXcptOrNmi(pVCpu, pVmxTransient));
12323	case VMX_EXIT_MOV_CRX: VMEXIT_CALL_RET(0, hmR0VmxExitMovCRx(pVCpu, pVmxTransient));
12324	case VMX_EXIT_EXT_INT: VMEXIT_CALL_RET(0, hmR0VmxExitExtInt(pVCpu, pVmxTransient));
12325	case VMX_EXIT_INT_WINDOW: VMEXIT_CALL_RET(0, hmR0VmxExitIntWindow(pVCpu, pVmxTransient));
12326	case VMX_EXIT_TPR_BELOW_THRESHOLD: VMEXIT_CALL_RET(0, hmR0VmxExitTprBelowThreshold(pVCpu, pVmxTransient));
12327	case VMX_EXIT_MWAIT: VMEXIT_CALL_RET(0, hmR0VmxExitMwait(pVCpu, pVmxTransient));
12328	case VMX_EXIT_MONITOR: VMEXIT_CALL_RET(0, hmR0VmxExitMonitor(pVCpu, pVmxTransient));
12329	case VMX_EXIT_TASK_SWITCH: VMEXIT_CALL_RET(0, hmR0VmxExitTaskSwitch(pVCpu, pVmxTransient));
12330	case VMX_EXIT_PREEMPT_TIMER: VMEXIT_CALL_RET(0, hmR0VmxExitPreemptTimer(pVCpu, pVmxTransient));
12331	case VMX_EXIT_RDMSR: VMEXIT_CALL_RET(0, hmR0VmxExitRdmsr(pVCpu, pVmxTransient));
12332	case VMX_EXIT_WRMSR: VMEXIT_CALL_RET(0, hmR0VmxExitWrmsr(pVCpu, pVmxTransient));
12333	case VMX_EXIT_VMCALL: VMEXIT_CALL_RET(0, hmR0VmxExitVmcall(pVCpu, pVmxTransient));
12334	case VMX_EXIT_MOV_DRX: VMEXIT_CALL_RET(0, hmR0VmxExitMovDRx(pVCpu, pVmxTransient));
12335	case VMX_EXIT_HLT: VMEXIT_CALL_RET(0, hmR0VmxExitHlt(pVCpu, pVmxTransient));
12336	case VMX_EXIT_INVD: VMEXIT_CALL_RET(0, hmR0VmxExitInvd(pVCpu, pVmxTransient));
12337	case VMX_EXIT_INVLPG: VMEXIT_CALL_RET(0, hmR0VmxExitInvlpg(pVCpu, pVmxTransient));
12338	case VMX_EXIT_RSM: VMEXIT_CALL_RET(0, hmR0VmxExitRsm(pVCpu, pVmxTransient));
12339	case VMX_EXIT_MTF: VMEXIT_CALL_RET(0, hmR0VmxExitMtf(pVCpu, pVmxTransient));
12340	case VMX_EXIT_PAUSE: VMEXIT_CALL_RET(0, hmR0VmxExitPause(pVCpu, pVmxTransient));
12341	case VMX_EXIT_GDTR_IDTR_ACCESS: VMEXIT_CALL_RET(0, hmR0VmxExitXdtrAccess(pVCpu, pVmxTransient));
12342	case VMX_EXIT_LDTR_TR_ACCESS: VMEXIT_CALL_RET(0, hmR0VmxExitXdtrAccess(pVCpu, pVmxTransient));
12343	case VMX_EXIT_WBINVD: VMEXIT_CALL_RET(0, hmR0VmxExitWbinvd(pVCpu, pVmxTransient));
12344	case VMX_EXIT_XSETBV: VMEXIT_CALL_RET(0, hmR0VmxExitXsetbv(pVCpu, pVmxTransient));
12345	case VMX_EXIT_RDRAND: VMEXIT_CALL_RET(0, hmR0VmxExitRdrand(pVCpu, pVmxTransient));
12346	case VMX_EXIT_INVPCID: VMEXIT_CALL_RET(0, hmR0VmxExitInvpcid(pVCpu, pVmxTransient));
12347	case VMX_EXIT_GETSEC: VMEXIT_CALL_RET(0, hmR0VmxExitGetsec(pVCpu, pVmxTransient));
12348	case VMX_EXIT_RDPMC: VMEXIT_CALL_RET(0, hmR0VmxExitRdpmc(pVCpu, pVmxTransient));
12349	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
12350	case VMX_EXIT_VMCLEAR: VMEXIT_CALL_RET(0, hmR0VmxExitVmclear(pVCpu, pVmxTransient));
12351	case VMX_EXIT_VMLAUNCH: VMEXIT_CALL_RET(0, hmR0VmxExitVmlaunch(pVCpu, pVmxTransient));
12352	case VMX_EXIT_VMPTRLD: VMEXIT_CALL_RET(0, hmR0VmxExitVmptrld(pVCpu, pVmxTransient));
12353	case VMX_EXIT_VMPTRST: VMEXIT_CALL_RET(0, hmR0VmxExitVmptrst(pVCpu, pVmxTransient));
12354	case VMX_EXIT_VMREAD: VMEXIT_CALL_RET(0, hmR0VmxExitVmread(pVCpu, pVmxTransient));
12355	case VMX_EXIT_VMRESUME: VMEXIT_CALL_RET(0, hmR0VmxExitVmwrite(pVCpu, pVmxTransient));
12356	case VMX_EXIT_VMWRITE: VMEXIT_CALL_RET(0, hmR0VmxExitVmresume(pVCpu, pVmxTransient));
12357	case VMX_EXIT_VMXOFF: VMEXIT_CALL_RET(0, hmR0VmxExitVmxoff(pVCpu, pVmxTransient));
12358	case VMX_EXIT_VMXON: VMEXIT_CALL_RET(0, hmR0VmxExitVmxon(pVCpu, pVmxTransient));
12359	#else
12360	case VMX_EXIT_VMCLEAR:
12361	case VMX_EXIT_VMLAUNCH:
12362	case VMX_EXIT_VMPTRLD:
12363	case VMX_EXIT_VMPTRST:
12364	case VMX_EXIT_VMREAD:
12365	case VMX_EXIT_VMRESUME:
12366	case VMX_EXIT_VMWRITE:
12367	case VMX_EXIT_VMXOFF:
12368	case VMX_EXIT_VMXON:
12369	return hmR0VmxExitSetPendingXcptUD(pVCpu, pVmxTransient);
12370	#endif
12371
12372	case VMX_EXIT_TRIPLE_FAULT: return hmR0VmxExitTripleFault(pVCpu, pVmxTransient);
12373	case VMX_EXIT_NMI_WINDOW: return hmR0VmxExitNmiWindow(pVCpu, pVmxTransient);
12374	case VMX_EXIT_INIT_SIGNAL: return hmR0VmxExitInitSignal(pVCpu, pVmxTransient);
12375	case VMX_EXIT_SIPI: return hmR0VmxExitSipi(pVCpu, pVmxTransient);
12376	case VMX_EXIT_IO_SMI: return hmR0VmxExitIoSmi(pVCpu, pVmxTransient);
12377	case VMX_EXIT_SMI: return hmR0VmxExitSmi(pVCpu, pVmxTransient);
12378	case VMX_EXIT_ERR_MSR_LOAD: return hmR0VmxExitErrMsrLoad(pVCpu, pVmxTransient);
12379	case VMX_EXIT_ERR_INVALID_GUEST_STATE: return hmR0VmxExitErrInvalidGuestState(pVCpu, pVmxTransient);
12380	case VMX_EXIT_ERR_MACHINE_CHECK: return hmR0VmxExitErrMachineCheck(pVCpu, pVmxTransient);
12381
12382	case VMX_EXIT_INVEPT:
12383	case VMX_EXIT_INVVPID:
12384	case VMX_EXIT_VMFUNC:
12385	case VMX_EXIT_XSAVES:
12386	case VMX_EXIT_XRSTORS:
12387	return hmR0VmxExitSetPendingXcptUD(pVCpu, pVmxTransient);
12388
12389	case VMX_EXIT_ENCLS:
12390	case VMX_EXIT_RDSEED:
12391	case VMX_EXIT_PML_FULL:
12392	default:
12393	return hmR0VmxExitErrUndefined(pVCpu, pVmxTransient);
12394	}
12395	#undef VMEXIT_CALL_RET
12396	}
12397	#endif /* !HMVMX_USE_FUNCTION_TABLE */
12398
12399
12400	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
12401	/**
12402	* Handles a nested-guest VM-exit from hardware-assisted VMX execution.
12403	*
12404	* @returns Strict VBox status code (i.e. informational status codes too).
12405	* @param pVCpu The cross context virtual CPU structure.
12406	* @param pVmxTransient The VMX-transient structure.
12407	*/
12408	DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExitNested(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12409	{
12410	uint32_t const rcReason = pVmxTransient->uExitReason;
12411	switch (rcReason)
12412	{
12413	case VMX_EXIT_EPT_MISCONFIG:
12414	case VMX_EXIT_EPT_VIOLATION:
12415	case VMX_EXIT_IO_INSTR:
12416	case VMX_EXIT_CPUID:
12417	case VMX_EXIT_RDTSC:
12418	case VMX_EXIT_RDTSCP:
12419	case VMX_EXIT_APIC_ACCESS:
12420	case VMX_EXIT_XCPT_OR_NMI:
12421	case VMX_EXIT_MOV_CRX:
12422	case VMX_EXIT_EXT_INT:
12423	case VMX_EXIT_INT_WINDOW:
12424	case VMX_EXIT_TPR_BELOW_THRESHOLD:
12425	case VMX_EXIT_MWAIT:
12426	case VMX_EXIT_MONITOR:
12427	case VMX_EXIT_TASK_SWITCH:
12428	case VMX_EXIT_PREEMPT_TIMER:
12429	case VMX_EXIT_RDMSR:
12430	case VMX_EXIT_WRMSR:
12431	case VMX_EXIT_VMCALL:
12432	case VMX_EXIT_MOV_DRX:
12433	case VMX_EXIT_HLT:
12434	case VMX_EXIT_INVD:
12435	case VMX_EXIT_INVLPG:
12436	case VMX_EXIT_RSM:
12437	case VMX_EXIT_MTF:
12438	case VMX_EXIT_PAUSE:
12439	case VMX_EXIT_GDTR_IDTR_ACCESS:
12440	case VMX_EXIT_LDTR_TR_ACCESS:
12441	case VMX_EXIT_WBINVD:
12442	case VMX_EXIT_XSETBV:
12443	case VMX_EXIT_RDRAND:
12444	case VMX_EXIT_INVPCID:
12445	case VMX_EXIT_GETSEC:
12446	case VMX_EXIT_RDPMC:
12447	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
12448	case VMX_EXIT_VMCLEAR:
12449	case VMX_EXIT_VMLAUNCH:
12450	case VMX_EXIT_VMPTRLD:
12451	case VMX_EXIT_VMPTRST:
12452	case VMX_EXIT_VMREAD:
12453	case VMX_EXIT_VMRESUME:
12454	case VMX_EXIT_VMWRITE:
12455	case VMX_EXIT_VMXOFF:
12456	case VMX_EXIT_VMXON:
12457	#endif
12458	case VMX_EXIT_TRIPLE_FAULT:
12459	case VMX_EXIT_NMI_WINDOW:
12460	case VMX_EXIT_INIT_SIGNAL:
12461	case VMX_EXIT_SIPI:
12462	case VMX_EXIT_IO_SMI:
12463	case VMX_EXIT_SMI:
12464	case VMX_EXIT_ERR_MSR_LOAD:
12465	case VMX_EXIT_ERR_INVALID_GUEST_STATE:
12466	case VMX_EXIT_ERR_MACHINE_CHECK:
12467
12468	case VMX_EXIT_INVEPT:
12469	case VMX_EXIT_INVVPID:
12470	case VMX_EXIT_VMFUNC:
12471	case VMX_EXIT_XSAVES:
12472	case VMX_EXIT_XRSTORS:
12473
12474	case VMX_EXIT_ENCLS:
12475	case VMX_EXIT_RDSEED:
12476	case VMX_EXIT_PML_FULL:
12477	default:
12478	return hmR0VmxExitErrUndefined(pVCpu, pVmxTransient);
12479	}
12480	#undef VMEXIT_CALL_RET
12481	}
12482	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
12483
12484
12485	#ifdef VBOX_STRICT
12486	/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
12487	# define HMVMX_ASSERT_PREEMPT_CPUID_VAR() \
12488	RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
12489
12490	# define HMVMX_ASSERT_PREEMPT_CPUID() \
12491	do { \
12492	RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
12493	AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
12494	} while (0)
12495
12496	# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
12497	do { \
12498	AssertPtr((a_pVCpu)); \
12499	AssertPtr((a_pVmxTransient)); \
12500	Assert((a_pVmxTransient)->fVMEntryFailed == false); \
12501	Assert((a_pVmxTransient)->pVmcsInfo); \
12502	Assert(ASMIntAreEnabled()); \
12503	HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu); \
12504	HMVMX_ASSERT_PREEMPT_CPUID_VAR(); \
12505	Log4Func(("vcpu[%RU32] -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v\n", (a_pVCpu)->idCpu)); \
12506	HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu); \
12507	if (VMMR0IsLogFlushDisabled((a_pVCpu))) \
12508	HMVMX_ASSERT_PREEMPT_CPUID(); \
12509	HMVMX_STOP_EXIT_DISPATCH_PROF(); \
12510	} while (0)
12511
12512	# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
12513	do { \
12514	Log4Func(("\n")); \
12515	} while (0)
12516	#else
12517	# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
12518	do { \
12519	HMVMX_STOP_EXIT_DISPATCH_PROF(); \
12520	NOREF((a_pVCpu)); NOREF((a_pVmxTransient)); \
12521	} while (0)
12522	# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) do { } while (0)
12523	#endif
12524
12525
12526	/**
12527	* Advances the guest RIP by the specified number of bytes.
12528	*
12529	* @param pVCpu The cross context virtual CPU structure.
12530	* @param cbInstr Number of bytes to advance the RIP by.
12531	*
12532	* @remarks No-long-jump zone!!!
12533	*/
12534	DECLINLINE(void) hmR0VmxAdvanceGuestRipBy(PVMCPU pVCpu, uint32_t cbInstr)
12535	{
12536	/* Advance the RIP. */
12537	pVCpu->cpum.GstCtx.rip += cbInstr;
12538	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
12539
12540	/* Update interrupt inhibition. */
12541	if ( VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS)
12542	&& pVCpu->cpum.GstCtx.rip != EMGetInhibitInterruptsPC(pVCpu))
12543	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
12544	}
12545
12546
12547	/**
12548	* Advances the guest RIP after reading it from the VMCS.
12549	*
12550	* @returns VBox status code, no informational status codes.
12551	* @param pVCpu The cross context virtual CPU structure.
12552	* @param pVmxTransient The VMX-transient structure.
12553	*
12554	* @remarks No-long-jump zone!!!
12555	*/
12556	static int hmR0VmxAdvanceGuestRip(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12557	{
12558	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12559	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS);
12560	AssertRCReturn(rc, rc);
12561
12562	hmR0VmxAdvanceGuestRipBy(pVCpu, pVmxTransient->cbInstr);
12563	return VINF_SUCCESS;
12564	}
12565
12566
12567	/**
12568	* Handle a condition that occurred while delivering an event through the guest
12569	* IDT.
12570	*
12571	* @returns Strict VBox status code (i.e. informational status codes too).
12572	* @retval VINF_SUCCESS if we should continue handling the VM-exit.
12573	* @retval VINF_HM_DOUBLE_FAULT if a \#DF condition was detected and we ought
12574	* to continue execution of the guest which will delivery the \#DF.
12575	* @retval VINF_EM_RESET if we detected a triple-fault condition.
12576	* @retval VERR_EM_GUEST_CPU_HANG if we detected a guest CPU hang.
12577	*
12578	* @param pVCpu The cross context virtual CPU structure.
12579	* @param pVmxTransient The VMX-transient structure.
12580	*
12581	* @remarks No-long-jump zone!!!
12582	*/
12583	static VBOXSTRICTRC hmR0VmxCheckExitDueToEventDelivery(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12584	{
12585	uint32_t const uExitVector = VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo);
12586
12587	int rc2 = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
12588	rc2 \|= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
12589	AssertRCReturn(rc2, rc2);
12590
12591	VBOXSTRICTRC rcStrict = VINF_SUCCESS;
12592	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
12593	if (VMX_IDT_VECTORING_INFO_IS_VALID(pVmxTransient->uIdtVectoringInfo))
12594	{
12595	uint32_t const uIdtVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
12596	uint32_t const uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
12597
12598	/*
12599	* If the event was a software interrupt (generated with INT n) or a software exception
12600	* (generated by INT3/INTO) or a privileged software exception (generated by INT1), we
12601	* can handle the VM-exit and continue guest execution which will re-execute the
12602	* instruction rather than re-injecting the exception, as that can cause premature
12603	* trips to ring-3 before injection and involve TRPM which currently has no way of
12604	* storing that these exceptions were caused by these instructions (ICEBP's #DB poses
12605	* the problem).
12606	*/
12607	IEMXCPTRAISE enmRaise;
12608	IEMXCPTRAISEINFO fRaiseInfo;
12609	if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
12610	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
12611	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
12612	{
12613	enmRaise = IEMXCPTRAISE_REEXEC_INSTR;
12614	fRaiseInfo = IEMXCPTRAISEINFO_NONE;
12615	}
12616	else if (VMX_EXIT_INT_INFO_IS_VALID(pVmxTransient->uExitIntInfo))
12617	{
12618	uint32_t const uExitVectorType = VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo);
12619	uint32_t const fIdtVectorFlags = hmR0VmxGetIemXcptFlags(uIdtVector, uIdtVectorType);
12620	uint32_t const fExitVectorFlags = hmR0VmxGetIemXcptFlags(uExitVector, uExitVectorType);
12621	/** @todo Make AssertMsgReturn as just AssertMsg later. */
12622	AssertMsgReturn(uExitVectorType == VMX_EXIT_INT_INFO_TYPE_HW_XCPT,
12623	("Unexpected VM-exit interruption vector type %#x!\n", uExitVectorType), VERR_VMX_IPE_5);
12624
12625	enmRaise = IEMEvaluateRecursiveXcpt(pVCpu, fIdtVectorFlags, uIdtVector, fExitVectorFlags, uExitVector, &fRaiseInfo);
12626
12627	/* Determine a vectoring #PF condition, see comment in hmR0VmxExitXcptPF(). */
12628	if (fRaiseInfo & (IEMXCPTRAISEINFO_EXT_INT_PF \| IEMXCPTRAISEINFO_NMI_PF))
12629	{
12630	pVmxTransient->fVectoringPF = true;
12631	enmRaise = IEMXCPTRAISE_PREV_EVENT;
12632	}
12633	}
12634	else
12635	{
12636	/*
12637	* If an exception or hardware interrupt delivery caused an EPT violation/misconfig or APIC access
12638	* VM-exit, then the VM-exit interruption-information will not be valid and we end up here.
12639	* It is sufficient to reflect the original event to the guest after handling the VM-exit.
12640	*/
12641	Assert( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
12642	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
12643	\|\| uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT);
12644	enmRaise = IEMXCPTRAISE_PREV_EVENT;
12645	fRaiseInfo = IEMXCPTRAISEINFO_NONE;
12646	}
12647
12648	/*
12649	* On CPUs that support Virtual NMIs, if this VM-exit (be it an exception or EPT violation/misconfig
12650	* etc.) occurred while delivering the NMI, we need to clear the block-by-NMI field in the guest
12651	* interruptibility-state before re-delivering the NMI after handling the VM-exit. Otherwise the
12652	* subsequent VM-entry would fail.
12653	*
12654	* See Intel spec. 30.7.1.2 "Resuming Guest Software after Handling an Exception". See @bugref{7445}.
12655	*/
12656	if ( VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_BLOCK_NMIS)
12657	&& uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
12658	&& ( enmRaise == IEMXCPTRAISE_PREV_EVENT
12659	\|\| (fRaiseInfo & IEMXCPTRAISEINFO_NMI_PF))
12660	&& (pVmcsInfo->u32PinCtls & VMX_PIN_CTLS_VIRT_NMI))
12661	{
12662	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
12663	}
12664
12665	switch (enmRaise)
12666	{
12667	case IEMXCPTRAISE_CURRENT_XCPT:
12668	{
12669	Log4Func(("IDT: Pending secondary Xcpt: uIdtVectoringInfo=%#RX64 uExitIntInfo=%#RX64\n",
12670	pVmxTransient->uIdtVectoringInfo, pVmxTransient->uExitIntInfo));
12671	Assert(rcStrict == VINF_SUCCESS);
12672	break;
12673	}
12674
12675	case IEMXCPTRAISE_PREV_EVENT:
12676	{
12677	uint32_t u32ErrCode;
12678	if (VMX_IDT_VECTORING_INFO_IS_ERROR_CODE_VALID(pVmxTransient->uIdtVectoringInfo))
12679	{
12680	rc2 = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
12681	AssertRCReturn(rc2, rc2);
12682	u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
12683	}
12684	else
12685	u32ErrCode = 0;
12686
12687	/* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF, see hmR0VmxExitXcptPF(). */
12688	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
12689	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
12690	0 /* cbInstr */, u32ErrCode, pVCpu->cpum.GstCtx.cr2);
12691
12692	Log4Func(("IDT: Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->hm.s.Event.u64IntInfo,
12693	pVCpu->hm.s.Event.u32ErrCode));
12694	Assert(rcStrict == VINF_SUCCESS);
12695	break;
12696	}
12697
12698	case IEMXCPTRAISE_REEXEC_INSTR:
12699	Assert(rcStrict == VINF_SUCCESS);
12700	break;
12701
12702	case IEMXCPTRAISE_DOUBLE_FAULT:
12703	{
12704	/*
12705	* Determing a vectoring double #PF condition. Used later, when PGM evaluates the
12706	* second #PF as a guest #PF (and not a shadow #PF) and needs to be converted into a #DF.
12707	*/
12708	if (fRaiseInfo & IEMXCPTRAISEINFO_PF_PF)
12709	{
12710	pVmxTransient->fVectoringDoublePF = true;
12711	Log4Func(("IDT: Vectoring double #PF %#RX64 cr2=%#RX64\n", pVCpu->hm.s.Event.u64IntInfo,
12712	pVCpu->cpum.GstCtx.cr2));
12713	rcStrict = VINF_SUCCESS;
12714	}
12715	else
12716	{
12717	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
12718	hmR0VmxSetPendingXcptDF(pVCpu);
12719	Log4Func(("IDT: Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->hm.s.Event.u64IntInfo,
12720	uIdtVector, uExitVector));
12721	rcStrict = VINF_HM_DOUBLE_FAULT;
12722	}
12723	break;
12724	}
12725
12726	case IEMXCPTRAISE_TRIPLE_FAULT:
12727	{
12728	Log4Func(("IDT: Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", uIdtVector, uExitVector));
12729	rcStrict = VINF_EM_RESET;
12730	break;
12731	}
12732
12733	case IEMXCPTRAISE_CPU_HANG:
12734	{
12735	Log4Func(("IDT: Bad guest! Entering CPU hang. fRaiseInfo=%#x\n", fRaiseInfo));
12736	rcStrict = VERR_EM_GUEST_CPU_HANG;
12737	break;
12738	}
12739
12740	default:
12741	{
12742	AssertMsgFailed(("IDT: vcpu[%RU32] Unexpected/invalid value! enmRaise=%#x\n", pVCpu->idCpu, enmRaise));
12743	rcStrict = VERR_VMX_IPE_2;
12744	break;
12745	}
12746	}
12747	}
12748	else if ( VMX_EXIT_INT_INFO_IS_VALID(pVmxTransient->uExitIntInfo)
12749	&& VMX_EXIT_INT_INFO_IS_NMI_UNBLOCK_IRET(pVmxTransient->uExitIntInfo)
12750	&& uExitVector != X86_XCPT_DF
12751	&& (pVmcsInfo->u32PinCtls & VMX_PIN_CTLS_VIRT_NMI))
12752	{
12753	/*
12754	* Execution of IRET caused this fault when NMI blocking was in effect (i.e we're in the guest NMI handler).
12755	* We need to set the block-by-NMI field so that NMIs remain blocked until the IRET execution is restarted.
12756	* See Intel spec. 30.7.1.2 "Resuming guest software after handling an exception".
12757	*/
12758	if (!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_BLOCK_NMIS))
12759	{
12760	Log4Func(("Setting VMCPU_FF_BLOCK_NMIS. fValid=%RTbool uExitReason=%u\n",
12761	VMX_EXIT_INT_INFO_IS_VALID(pVmxTransient->uExitIntInfo), pVmxTransient->uExitReason));
12762	VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
12763	}
12764	}
12765
12766	Assert( rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_HM_DOUBLE_FAULT
12767	\|\| rcStrict == VINF_EM_RESET \|\| rcStrict == VERR_EM_GUEST_CPU_HANG);
12768	return rcStrict;
12769	}
12770
12771
12772	/** @name VM-exit handlers.
12773	* @{
12774	*/
12775	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
12776	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
12777	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
12778
12779	/**
12780	* VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
12781	*/
12782	HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12783	{
12784	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12785	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
12786	/* Windows hosts (32-bit and 64-bit) have DPC latency issues. See @bugref{6853}. */
12787	if (VMMR0ThreadCtxHookIsEnabled(pVCpu))
12788	return VINF_SUCCESS;
12789	return VINF_EM_RAW_INTERRUPT;
12790	}
12791
12792
12793	/**
12794	* VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI).
12795	*/
12796	HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmi(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12797	{
12798	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12799	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
12800
12801	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
12802	int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
12803	AssertRCReturn(rc, rc);
12804
12805	uint32_t uIntType = VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo);
12806	Assert( !(pVmcsInfo->u32ExitCtls & VMX_EXIT_CTLS_ACK_EXT_INT)
12807	&& uIntType != VMX_EXIT_INT_INFO_TYPE_EXT_INT);
12808	Assert(VMX_EXIT_INT_INFO_IS_VALID(pVmxTransient->uExitIntInfo));
12809
12810	if (uIntType == VMX_EXIT_INT_INFO_TYPE_NMI)
12811	{
12812	/*
12813	* This cannot be a guest NMI as the only way for the guest to receive an NMI is if we
12814	* injected it ourselves and anything we inject is not going to cause a VM-exit directly
12815	* for the event being injected[1]. Go ahead and dispatch the NMI to the host[2].
12816	*
12817	* [1] -- See Intel spec. 27.2.3 "Information for VM Exits During Event Delivery".
12818	* [2] -- See Intel spec. 27.5.5 "Updating Non-Register State".
12819	*/
12820	VMXDispatchHostNmi();
12821	STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGC);
12822	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
12823	return VINF_SUCCESS;
12824	}
12825
12826	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
12827	VBOXSTRICTRC rcStrictRc1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
12828	if (RT_UNLIKELY(rcStrictRc1 == VINF_SUCCESS))
12829	{ /* likely */ }
12830	else
12831	{
12832	if (rcStrictRc1 == VINF_HM_DOUBLE_FAULT)
12833	rcStrictRc1 = VINF_SUCCESS;
12834	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
12835	return rcStrictRc1;
12836	}
12837
12838	uint32_t uExitIntInfo = pVmxTransient->uExitIntInfo;
12839	uint32_t uVector = VMX_EXIT_INT_INFO_VECTOR(uExitIntInfo);
12840	switch (uIntType)
12841	{
12842	case VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT: /* Privileged software exception. (#DB from ICEBP) */
12843	Assert(uVector == X86_XCPT_DB);
12844	RT_FALL_THRU();
12845	case VMX_EXIT_INT_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
12846	Assert(uVector == X86_XCPT_BP \|\| uVector == X86_XCPT_OF \|\| uIntType == VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT);
12847	RT_FALL_THRU();
12848	case VMX_EXIT_INT_INFO_TYPE_HW_XCPT:
12849	{
12850	/*
12851	* If there's any exception caused as a result of event injection, the resulting
12852	* secondary/final execption will be pending, we shall continue guest execution
12853	* after injecting the event. The page-fault case is complicated and we manually
12854	* handle any currently pending event in hmR0VmxExitXcptPF.
12855	*/
12856	if (!pVCpu->hm.s.Event.fPending)
12857	{ /* likely */ }
12858	else if (uVector != X86_XCPT_PF)
12859	{
12860	rc = VINF_SUCCESS;
12861	break;
12862	}
12863
12864	switch (uVector)
12865	{
12866	case X86_XCPT_PF: rc = hmR0VmxExitXcptPF(pVCpu, pVmxTransient); break;
12867	case X86_XCPT_GP: rc = hmR0VmxExitXcptGP(pVCpu, pVmxTransient); break;
12868	case X86_XCPT_MF: rc = hmR0VmxExitXcptMF(pVCpu, pVmxTransient); break;
12869	case X86_XCPT_DB: rc = hmR0VmxExitXcptDB(pVCpu, pVmxTransient); break;
12870	case X86_XCPT_BP: rc = hmR0VmxExitXcptBP(pVCpu, pVmxTransient); break;
12871	case X86_XCPT_AC: rc = hmR0VmxExitXcptAC(pVCpu, pVmxTransient); break;
12872
12873	case X86_XCPT_NM: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
12874	rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
12875	case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF);
12876	rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
12877	case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
12878	rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
12879	case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
12880	rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
12881	case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
12882	rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
12883	case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
12884	rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
12885	case X86_XCPT_TS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestTS);
12886	rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
12887	default:
12888	{
12889	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
12890	if (pVmcsInfo->RealMode.fRealOnV86Active)
12891	{
12892	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
12893	Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
12894	Assert(CPUMIsGuestInRealModeEx(&pVCpu->cpum.GstCtx));
12895
12896	rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_CR0);
12897	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12898	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
12899	AssertRCReturn(rc, rc);
12900	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(uExitIntInfo),
12901	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode,
12902	0 /* GCPtrFaultAddress */);
12903	}
12904	else
12905	{
12906	AssertMsgFailed(("Unexpected VM-exit caused by exception %#x\n", uVector));
12907	pVCpu->hm.s.u32HMError = uVector;
12908	rc = VERR_VMX_UNEXPECTED_EXCEPTION;
12909	}
12910	break;
12911	}
12912	}
12913	break;
12914	}
12915
12916	default:
12917	{
12918	pVCpu->hm.s.u32HMError = uExitIntInfo;
12919	rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE;
12920	AssertMsgFailed(("Unexpected interruption info %#x\n", VMX_EXIT_INT_INFO_TYPE(uExitIntInfo)));
12921	break;
12922	}
12923	}
12924	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
12925	return rc;
12926	}
12927
12928
12929	/**
12930	* VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
12931	*/
12932	HMVMX_EXIT_NSRC_DECL hmR0VmxExitIntWindow(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12933	{
12934	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12935
12936	/* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
12937	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
12938	int rc = hmR0VmxClearIntWindowExitVmcs(pVmcsInfo);
12939	AssertRCReturn(rc, rc);
12940
12941	/* Evaluate and deliver pending events and resume guest execution. */
12942	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
12943	return VINF_SUCCESS;
12944	}
12945
12946
12947	/**
12948	* VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
12949	*/
12950	HMVMX_EXIT_NSRC_DECL hmR0VmxExitNmiWindow(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12951	{
12952	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12953
12954	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
12955	if (RT_UNLIKELY(!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT))) /** @todo NSTVMX: Turn this into an assertion. */
12956	{
12957	AssertMsgFailed(("Unexpected NMI-window exit.\n"));
12958	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
12959	}
12960
12961	Assert(!VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_BLOCK_NMIS));
12962
12963	/*
12964	* If block-by-STI is set when we get this VM-exit, it means the CPU doesn't block NMIs following STI.
12965	* It is therefore safe to unblock STI and deliver the NMI ourselves. See @bugref{7445}.
12966	*/
12967	uint32_t fIntrState;
12968	int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &fIntrState);
12969	AssertRCReturn(rc, rc);
12970	Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS));
12971	if (fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
12972	{
12973	if (VMCPU_FF_IS_SET(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
12974	VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
12975
12976	fIntrState &= ~VMX_VMCS_GUEST_INT_STATE_BLOCK_STI;
12977	rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INT_STATE, fIntrState);
12978	AssertRCReturn(rc, rc);
12979	}
12980
12981	/* Indicate that we no longer need to VM-exit when the guest is ready to receive NMIs, it is now ready */
12982	rc = hmR0VmxClearNmiWindowExitVmcs(pVmcsInfo);
12983	AssertRCReturn(rc, rc);
12984
12985	/* Evaluate and deliver pending events and resume guest execution. */
12986	return VINF_SUCCESS;
12987	}
12988
12989
12990	/**
12991	* VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
12992	*/
12993	HMVMX_EXIT_NSRC_DECL hmR0VmxExitWbinvd(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12994	{
12995	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12996	return hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
12997	}
12998
12999
13000	/**
13001	* VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
13002	*/
13003	HMVMX_EXIT_NSRC_DECL hmR0VmxExitInvd(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13004	{
13005	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13006	return hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
13007	}
13008
13009
13010	/**
13011	* VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
13012	*/
13013	HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13014	{
13015	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13016
13017	/*
13018	* Get the state we need and update the exit history entry.
13019	*/
13020	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13021	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13022	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
13023	AssertRCReturn(rc, rc);
13024
13025	VBOXSTRICTRC rcStrict;
13026	PCEMEXITREC pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
13027	EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_CPUID),
13028	pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
13029	if (!pExitRec)
13030	{
13031	/*
13032	* Regular CPUID instruction execution.
13033	*/
13034	rcStrict = IEMExecDecodedCpuid(pVCpu, pVmxTransient->cbInstr);
13035	if (rcStrict == VINF_SUCCESS)
13036	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
13037	else if (rcStrict == VINF_IEM_RAISED_XCPT)
13038	{
13039	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13040	rcStrict = VINF_SUCCESS;
13041	}
13042	}
13043	else
13044	{
13045	/*
13046	* Frequent exit or something needing probing. Get state and call EMHistoryExec.
13047	*/
13048	int rc2 = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
13049	AssertRCReturn(rc2, rc2);
13050
13051	Log4(("CpuIdExit/%u: %04x:%08RX64: %#x/%#x -> EMHistoryExec\n",
13052	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.eax, pVCpu->cpum.GstCtx.ecx));
13053
13054	rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
13055	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
13056
13057	Log4(("CpuIdExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
13058	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
13059	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
13060	}
13061	return rcStrict;
13062	}
13063
13064
13065	/**
13066	* VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
13067	*/
13068	HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13069	{
13070	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13071
13072	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13073	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_CR4);
13074	AssertRCReturn(rc, rc);
13075
13076	if (pVCpu->cpum.GstCtx.cr4 & X86_CR4_SMXE)
13077	return VINF_EM_RAW_EMULATE_INSTR;
13078
13079	AssertMsgFailed(("hmR0VmxExitGetsec: unexpected VM-exit when CR4.SMXE is 0.\n"));
13080	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
13081	}
13082
13083
13084	/**
13085	* VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
13086	*/
13087	HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13088	{
13089	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13090
13091	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13092	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
13093	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13094	AssertRCReturn(rc, rc);
13095
13096	VBOXSTRICTRC rcStrict = IEMExecDecodedRdtsc(pVCpu, pVmxTransient->cbInstr);
13097	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
13098	{
13099	/* If we get a spurious VM-exit when TSC offsetting is enabled,
13100	we must reset offsetting on VM-entry. See @bugref{6634}. */
13101	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TSC_OFFSETTING)
13102	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
13103	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
13104	}
13105	else if (rcStrict == VINF_IEM_RAISED_XCPT)
13106	{
13107	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13108	rcStrict = VINF_SUCCESS;
13109	}
13110	return rcStrict;
13111	}
13112
13113
13114	/**
13115	* VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
13116	*/
13117	HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13118	{
13119	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13120
13121	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13122	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK \| CPUMCTX_EXTRN_TSC_AUX);
13123	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13124	AssertRCReturn(rc, rc);
13125
13126	VBOXSTRICTRC rcStrict = IEMExecDecodedRdtscp(pVCpu, pVmxTransient->cbInstr);
13127	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
13128	{
13129	/* If we get a spurious VM-exit when TSC offsetting is enabled,
13130	we must reset offsetting on VM-reentry. See @bugref{6634}. */
13131	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TSC_OFFSETTING)
13132	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
13133	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
13134	}
13135	else if (rcStrict == VINF_IEM_RAISED_XCPT)
13136	{
13137	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13138	rcStrict = VINF_SUCCESS;
13139	}
13140	return rcStrict;
13141	}
13142
13143
13144	/**
13145	* VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
13146	*/
13147	HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13148	{
13149	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13150
13151	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13152	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_CR4 \| CPUMCTX_EXTRN_CR0
13153	\| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_SS);
13154	AssertRCReturn(rc, rc);
13155
13156	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
13157	rc = EMInterpretRdpmc(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx));
13158	if (RT_LIKELY(rc == VINF_SUCCESS))
13159	{
13160	rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
13161	Assert(pVmxTransient->cbInstr == 2);
13162	}
13163	else
13164	{
13165	AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
13166	rc = VERR_EM_INTERPRETER;
13167	}
13168	return rc;
13169	}
13170
13171
13172	/**
13173	* VM-exit handler for VMCALL (VMX_EXIT_VMCALL). Unconditional VM-exit.
13174	*/
13175	HMVMX_EXIT_DECL hmR0VmxExitVmcall(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13176	{
13177	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13178
13179	VBOXSTRICTRC rcStrict = VERR_VMX_IPE_3;
13180	if (EMAreHypercallInstructionsEnabled(pVCpu))
13181	{
13182	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13183	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_RIP \| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_CR0
13184	\| CPUMCTX_EXTRN_SS \| CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_EFER);
13185	AssertRCReturn(rc, rc);
13186
13187	/* Perform the hypercall. */
13188	rcStrict = GIMHypercall(pVCpu, &pVCpu->cpum.GstCtx);
13189	if (rcStrict == VINF_SUCCESS)
13190	{
13191	rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
13192	AssertRCReturn(rc, rc);
13193	}
13194	else
13195	Assert( rcStrict == VINF_GIM_R3_HYPERCALL
13196	\|\| rcStrict == VINF_GIM_HYPERCALL_CONTINUING
13197	\|\| RT_FAILURE(rcStrict));
13198
13199	/* If the hypercall changes anything other than guest's general-purpose registers,
13200	we would need to reload the guest changed bits here before VM-entry. */
13201	}
13202	else
13203	Log4Func(("Hypercalls not enabled\n"));
13204
13205	/* If hypercalls are disabled or the hypercall failed for some reason, raise #UD and continue. */
13206	if (RT_FAILURE(rcStrict))
13207	{
13208	hmR0VmxSetPendingXcptUD(pVCpu);
13209	rcStrict = VINF_SUCCESS;
13210	}
13211
13212	return rcStrict;
13213	}
13214
13215
13216	/**
13217	* VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
13218	*/
13219	HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13220	{
13221	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13222	Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging \|\| pVCpu->hm.s.fUsingDebugLoop);
13223
13224	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13225	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
13226	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13227	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
13228	AssertRCReturn(rc, rc);
13229
13230	VBOXSTRICTRC rcStrict = IEMExecDecodedInvlpg(pVCpu, pVmxTransient->cbInstr, pVmxTransient->uExitQual);
13231
13232	if (rcStrict == VINF_SUCCESS \|\| rcStrict == VINF_PGM_SYNC_CR3)
13233	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
13234	else if (rcStrict == VINF_IEM_RAISED_XCPT)
13235	{
13236	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13237	rcStrict = VINF_SUCCESS;
13238	}
13239	else
13240	AssertMsgFailed(("Unexpected IEMExecDecodedInvlpg(%#RX64) sttus: %Rrc\n", pVmxTransient->uExitQual,
13241	VBOXSTRICTRC_VAL(rcStrict)));
13242	return rcStrict;
13243	}
13244
13245
13246	/**
13247	* VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
13248	*/
13249	HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13250	{
13251	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13252
13253	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13254	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_SS);
13255	AssertRCReturn(rc, rc);
13256
13257	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
13258	rc = EMInterpretMonitor(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx));
13259	if (RT_LIKELY(rc == VINF_SUCCESS))
13260	rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
13261	else
13262	{
13263	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMonitor: EMInterpretMonitor failed with %Rrc\n", rc));
13264	rc = VERR_EM_INTERPRETER;
13265	}
13266	return rc;
13267	}
13268
13269
13270	/**
13271	* VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
13272	*/
13273	HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13274	{
13275	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13276
13277	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13278	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_CR0 \| CPUMCTX_EXTRN_RFLAGS \| CPUMCTX_EXTRN_SS);
13279	AssertRCReturn(rc, rc);
13280
13281	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
13282	VBOXSTRICTRC rc2 = EMInterpretMWait(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx));
13283	rc = VBOXSTRICTRC_VAL(rc2);
13284	if (RT_LIKELY( rc == VINF_SUCCESS
13285	\|\| rc == VINF_EM_HALT))
13286	{
13287	int rc3 = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
13288	AssertRCReturn(rc3, rc3);
13289
13290	if ( rc == VINF_EM_HALT
13291	&& EMMonitorWaitShouldContinue(pVCpu, pCtx))
13292	rc = VINF_SUCCESS;
13293	}
13294	else
13295	{
13296	AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMwait: EMInterpretMWait failed with %Rrc\n", rc));
13297	rc = VERR_EM_INTERPRETER;
13298	}
13299	AssertMsg(rc == VINF_SUCCESS \|\| rc == VINF_EM_HALT \|\| rc == VERR_EM_INTERPRETER,
13300	("hmR0VmxExitMwait: failed, invalid error code %Rrc\n", rc));
13301	return rc;
13302	}
13303
13304
13305	/**
13306	* VM-exit handler for RSM (VMX_EXIT_RSM). Unconditional VM-exit.
13307	*/
13308	HMVMX_EXIT_NSRC_DECL hmR0VmxExitRsm(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13309	{
13310	/*
13311	* Execution of RSM outside of SMM mode causes #UD regardless of VMX root or VMX non-root
13312	* mode. In theory, we should never get this VM-exit. This can happen only if dual-monitor
13313	* treatment of SMI and VMX is enabled, which can (only?) be done by executing VMCALL in
13314	* VMX root operation. If we get here, something funny is going on.
13315	*
13316	* See Intel spec. 33.15.5 "Enabling the Dual-Monitor Treatment".
13317	*/
13318	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13319	AssertMsgFailed(("Unexpected RSM VM-exit\n"));
13320	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
13321	}
13322
13323
13324	/**
13325	* VM-exit handler for SMI (VMX_EXIT_SMI). Unconditional VM-exit.
13326	*/
13327	HMVMX_EXIT_NSRC_DECL hmR0VmxExitSmi(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13328	{
13329	/*
13330	* This can only happen if we support dual-monitor treatment of SMI, which can be activated
13331	* by executing VMCALL in VMX root operation. Only an STM (SMM transfer monitor) would get
13332	* this VM-exit when we (the executive monitor) execute a VMCALL in VMX root mode or receive
13333	* an SMI. If we get here, something funny is going on.
13334	*
13335	* See Intel spec. 33.15.6 "Activating the Dual-Monitor Treatment"
13336	* See Intel spec. 25.3 "Other Causes of VM-Exits"
13337	*/
13338	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13339	AssertMsgFailed(("Unexpected SMI VM-exit\n"));
13340	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
13341	}
13342
13343
13344	/**
13345	* VM-exit handler for IO SMI (VMX_EXIT_IO_SMI). Unconditional VM-exit.
13346	*/
13347	HMVMX_EXIT_NSRC_DECL hmR0VmxExitIoSmi(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13348	{
13349	/* Same treatment as VMX_EXIT_SMI. See comment in hmR0VmxExitSmi(). */
13350	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13351	AssertMsgFailed(("Unexpected IO SMI VM-exit\n"));
13352	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
13353	}
13354
13355
13356	/**
13357	* VM-exit handler for SIPI (VMX_EXIT_SIPI). Conditional VM-exit.
13358	*/
13359	HMVMX_EXIT_NSRC_DECL hmR0VmxExitSipi(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13360	{
13361	/*
13362	* SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest activity state is used.
13363	* We don't make use of it as our guests don't have direct access to the host LAPIC.
13364	* See Intel spec. 25.3 "Other Causes of VM-exits".
13365	*/
13366	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13367	AssertMsgFailed(("Unexpected SIPI VM-exit\n"));
13368	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
13369	}
13370
13371
13372	/**
13373	* VM-exit handler for INIT signal (VMX_EXIT_INIT_SIGNAL). Unconditional
13374	* VM-exit.
13375	*/
13376	HMVMX_EXIT_NSRC_DECL hmR0VmxExitInitSignal(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13377	{
13378	/*
13379	* INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM.
13380	* See Intel spec. 33.14.1 Default Treatment of SMI Delivery" and Intel spec. 29.3 "VMX Instructions" for "VMXON".
13381	*
13382	* It is -NOT- blocked in VMX non-root operation so we can, in theory, still get these VM-exits.
13383	* See Intel spec. "23.8 Restrictions on VMX operation".
13384	*/
13385	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13386	return VINF_SUCCESS;
13387	}
13388
13389
13390	/**
13391	* VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
13392	* VM-exit.
13393	*/
13394	HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13395	{
13396	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13397	return VINF_EM_RESET;
13398	}
13399
13400
13401	/**
13402	* VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
13403	*/
13404	HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13405	{
13406	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13407
13408	int rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
13409	AssertRCReturn(rc, rc);
13410
13411	HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RFLAGS); /* Advancing the RIP above should've imported eflags. */
13412	if (EMShouldContinueAfterHalt(pVCpu, &pVCpu->cpum.GstCtx)) /* Requires eflags. */
13413	rc = VINF_SUCCESS;
13414	else
13415	rc = VINF_EM_HALT;
13416
13417	if (rc != VINF_SUCCESS)
13418	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHltToR3);
13419	return rc;
13420	}
13421
13422
13423	/**
13424	* VM-exit handler for instructions that result in a \#UD exception delivered to
13425	* the guest.
13426	*/
13427	HMVMX_EXIT_NSRC_DECL hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13428	{
13429	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13430	hmR0VmxSetPendingXcptUD(pVCpu);
13431	return VINF_SUCCESS;
13432	}
13433
13434
13435	/**
13436	* VM-exit handler for expiry of the VMX-preemption timer.
13437	*/
13438	HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13439	{
13440	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13441
13442	/* If the VMX-preemption timer has expired, reinitialize the preemption timer on next VM-entry. */
13443	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
13444
13445	/* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
13446	PVM pVM = pVCpu->CTX_SUFF(pVM);
13447	bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
13448	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
13449	return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
13450	}
13451
13452
13453	/**
13454	* VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
13455	*/
13456	HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13457	{
13458	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13459
13460	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13461	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13462	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK \| CPUMCTX_EXTRN_CR4);
13463	AssertRCReturn(rc, rc);
13464
13465	VBOXSTRICTRC rcStrict = IEMExecDecodedXsetbv(pVCpu, pVmxTransient->cbInstr);
13466	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, rcStrict != VINF_IEM_RAISED_XCPT ? HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS
13467	: HM_CHANGED_RAISED_XCPT_MASK);
13468
13469	PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
13470	pVCpu->hm.s.fLoadSaveGuestXcr0 = (pCtx->cr4 & X86_CR4_OSXSAVE) && pCtx->aXcr[0] != ASMGetXcr0();
13471
13472	return rcStrict;
13473	}
13474
13475
13476	/**
13477	* VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
13478	*/
13479	HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13480	{
13481	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13482	/** @todo Use VM-exit instruction information. */
13483	return VERR_EM_INTERPRETER;
13484	}
13485
13486
13487	/**
13488	* VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE).
13489	* Error VM-exit.
13490	*/
13491	HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13492	{
13493	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13494	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
13495	AssertRCReturn(rc, rc);
13496
13497	rc = hmR0VmxCheckVmcsCtls(pVCpu, pVmcsInfo);
13498	if (RT_FAILURE(rc))
13499	return rc;
13500
13501	uint32_t const uInvalidReason = hmR0VmxCheckGuestState(pVCpu, pVmcsInfo);
13502	NOREF(uInvalidReason);
13503
13504	#ifdef VBOX_STRICT
13505	uint32_t fIntrState;
13506	RTHCUINTREG uHCReg;
13507	uint64_t u64Val;
13508	uint32_t u32Val;
13509	rc = hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
13510	rc \|= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
13511	rc \|= hmR0VmxReadEntryInstrLenVmcs(pVmxTransient);
13512	rc \|= VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &fIntrState);
13513	AssertRCReturn(rc, rc);
13514
13515	Log4(("uInvalidReason %u\n", uInvalidReason));
13516	Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntInfo));
13517	Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
13518	Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
13519	Log4(("VMX_VMCS32_GUEST_INT_STATE %#RX32\n", fIntrState));
13520
13521	rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val); AssertRC(rc);
13522	Log4(("VMX_VMCS_GUEST_CR0 %#RX32\n", u32Val));
13523	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
13524	Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
13525	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
13526	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
13527	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
13528	Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
13529	rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
13530	Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
13531	rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
13532	Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
13533
13534	hmR0DumpRegs(pVCpu);
13535	#endif
13536
13537	return VERR_VMX_INVALID_GUEST_STATE;
13538	}
13539
13540
13541	/**
13542	* VM-exit handler for VM-entry failure due to an MSR-load
13543	* (VMX_EXIT_ERR_MSR_LOAD). Error VM-exit.
13544	*/
13545	HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13546	{
13547	AssertMsgFailed(("Unexpected MSR-load exit\n"));
13548	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
13549	}
13550
13551
13552	/**
13553	* VM-exit handler for VM-entry failure due to a machine-check event
13554	* (VMX_EXIT_ERR_MACHINE_CHECK). Error VM-exit.
13555	*/
13556	HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13557	{
13558	AssertMsgFailed(("Unexpected machine-check event exit\n"));
13559	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
13560	}
13561
13562
13563	/**
13564	* VM-exit handler for all undefined reasons. Should never ever happen.. in
13565	* theory.
13566	*/
13567	HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrUndefined(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13568	{
13569	RT_NOREF2(pVCpu, pVmxTransient);
13570	AssertMsgFailed(("Huh!? Undefined VM-exit reason %d\n", pVmxTransient->uExitReason));
13571	return VERR_VMX_UNDEFINED_EXIT_CODE;
13572	}
13573
13574
13575	/**
13576	* VM-exit handler for XDTR (LGDT, SGDT, LIDT, SIDT) accesses
13577	* (VMX_EXIT_GDTR_IDTR_ACCESS) and LDT and TR access (LLDT, LTR, SLDT, STR).
13578	* Conditional VM-exit.
13579	*/
13580	HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13581	{
13582	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13583
13584	/* By default, we don't enable VMX_PROC_CTLS2_DESCRIPTOR_TABLE_EXIT. */
13585	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitXdtrAccess);
13586	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13587	if (pVmcsInfo->u32ProcCtls2 & VMX_PROC_CTLS2_DESC_TABLE_EXIT)
13588	return VERR_EM_INTERPRETER;
13589	AssertMsgFailed(("Unexpected XDTR access\n"));
13590	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
13591	}
13592
13593
13594	/**
13595	* VM-exit handler for RDRAND (VMX_EXIT_RDRAND). Conditional VM-exit.
13596	*/
13597	HMVMX_EXIT_DECL hmR0VmxExitRdrand(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13598	{
13599	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13600
13601	/* By default, we don't enable VMX_PROC_CTLS2_RDRAND_EXIT. */
13602	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13603	if (pVmcsInfo->u32ProcCtls2 & VMX_PROC_CTLS2_RDRAND_EXIT)
13604	return VERR_EM_INTERPRETER;
13605	AssertMsgFailed(("Unexpected RDRAND exit\n"));
13606	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
13607	}
13608
13609
13610	/**
13611	* VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
13612	*/
13613	HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13614	{
13615	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13616
13617	/** @todo Optimize this: We currently drag in in the whole MSR state
13618	* (CPUMCTX_EXTRN_ALL_MSRS) here. We should optimize this to only get
13619	* MSRs required. That would require changes to IEM and possibly CPUM too.
13620	* (Should probably do it lazy fashion from CPUMAllMsrs.cpp). */
13621	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13622	uint32_t const idMsr = pVCpu->cpum.GstCtx.ecx;
13623	uint64_t fImport = IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| CPUMCTX_EXTRN_ALL_MSRS;
13624	switch (idMsr)
13625	{
13626	case MSR_K8_FS_BASE: fImport \|= CPUMCTX_EXTRN_FS; break;
13627	case MSR_K8_GS_BASE: fImport \|= CPUMCTX_EXTRN_GS; break;
13628	}
13629
13630	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13631	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, fImport);
13632	AssertRCReturn(rc, rc);
13633
13634	Log4Func(("ecx=%#RX32\n", idMsr));
13635
13636	#ifdef VBOX_STRICT
13637	if (pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
13638	{
13639	if ( hmR0VmxIsAutoLoadGuestMsr(pVmcsInfo, idMsr)
13640	&& idMsr != MSR_K6_EFER)
13641	{
13642	AssertMsgFailed(("Unexpected RDMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n", idMsr));
13643	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
13644	}
13645	if (hmR0VmxIsLazyGuestMsr(pVCpu, idMsr))
13646	{
13647	Assert(pVmcsInfo->pvMsrBitmap);
13648	uint32_t fMsrpm = HMGetVmxMsrPermission(pVmcsInfo->pvMsrBitmap, idMsr);
13649	if (fMsrpm & VMXMSRPM_ALLOW_RD)
13650	{
13651	AssertMsgFailed(("Unexpected RDMSR for a passthru lazy-restore MSR. ecx=%#RX32\n", idMsr));
13652	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
13653	}
13654	}
13655	}
13656	#endif
13657
13658	VBOXSTRICTRC rcStrict = IEMExecDecodedRdmsr(pVCpu, pVmxTransient->cbInstr);
13659	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
13660	if (rcStrict == VINF_SUCCESS)
13661	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS
13662	\| HM_CHANGED_GUEST_RAX \| HM_CHANGED_GUEST_RDX);
13663	else if (rcStrict == VINF_IEM_RAISED_XCPT)
13664	{
13665	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13666	rcStrict = VINF_SUCCESS;
13667	}
13668	else
13669	AssertMsg(rcStrict == VINF_CPUM_R3_MSR_READ, ("Unexpected IEMExecDecodedRdmsr rc (%Rrc)\n", VBOXSTRICTRC_VAL(rcStrict)));
13670
13671	return rcStrict;
13672	}
13673
13674
13675	/**
13676	* VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
13677	*/
13678	HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13679	{
13680	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13681
13682	/** @todo Optimize this: We currently drag in in the whole MSR state
13683	* (CPUMCTX_EXTRN_ALL_MSRS) here. We should optimize this to only get
13684	* MSRs required. That would require changes to IEM and possibly CPUM too.
13685	* (Should probably do it lazy fashion from CPUMAllMsrs.cpp). */
13686	uint32_t const idMsr = pVCpu->cpum.GstCtx.ecx;
13687	uint64_t fImport = IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK \| CPUMCTX_EXTRN_ALL_MSRS;
13688
13689	/*
13690	* The FS and GS base MSRs are not part of the above all-MSRs mask.
13691	* Although we don't need to fetch the base as it will be overwritten shortly, while
13692	* loading guest-state we would also load the entire segment register including limit
13693	* and attributes and thus we need to load them here.
13694	*/
13695	switch (idMsr)
13696	{
13697	case MSR_K8_FS_BASE: fImport \|= CPUMCTX_EXTRN_FS; break;
13698	case MSR_K8_GS_BASE: fImport \|= CPUMCTX_EXTRN_GS; break;
13699	}
13700
13701	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13702	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13703	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, fImport);
13704	AssertRCReturn(rc, rc);
13705
13706	Log4Func(("ecx=%#RX32 edx:eax=%#RX32:%#RX32\n", idMsr, pVCpu->cpum.GstCtx.edx, pVCpu->cpum.GstCtx.eax));
13707
13708	VBOXSTRICTRC rcStrict = IEMExecDecodedWrmsr(pVCpu, pVmxTransient->cbInstr);
13709	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
13710
13711	if (rcStrict == VINF_SUCCESS)
13712	{
13713	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
13714
13715	/* If this is an X2APIC WRMSR access, update the APIC state as well. */
13716	if ( idMsr == MSR_IA32_APICBASE
13717	\|\| ( idMsr >= MSR_IA32_X2APIC_START
13718	&& idMsr <= MSR_IA32_X2APIC_END))
13719	{
13720	/*
13721	* We've already saved the APIC related guest-state (TPR) in post-run phase.
13722	* When full APIC register virtualization is implemented we'll have to make
13723	* sure APIC state is saved from the VMCS before IEM changes it.
13724	*/
13725	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
13726	}
13727	else if (idMsr == MSR_IA32_TSC) /* Windows 7 does this during bootup. See @bugref{6398}. */
13728	pVmxTransient->fUpdatedTscOffsettingAndPreemptTimer = false;
13729	else if (idMsr == MSR_K6_EFER)
13730	{
13731	/*
13732	* If the guest touches the EFER MSR we need to update the VM-Entry and VM-Exit controls
13733	* as well, even if it is -not- touching bits that cause paging mode changes (LMA/LME).
13734	* We care about the other bits as well, SCE and NXE. See @bugref{7368}.
13735	*/
13736	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_EFER_MSR \| HM_CHANGED_VMX_ENTRY_EXIT_CTLS);
13737	}
13738
13739	/* Update MSRs that are part of the VMCS and auto-load/store area when MSR-bitmaps are not supported. */
13740	if (!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS))
13741	{
13742	switch (idMsr)
13743	{
13744	case MSR_IA32_SYSENTER_CS: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_CS_MSR); break;
13745	case MSR_IA32_SYSENTER_EIP: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_EIP_MSR); break;
13746	case MSR_IA32_SYSENTER_ESP: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_ESP_MSR); break;
13747	case MSR_K8_FS_BASE: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_FS); break;
13748	case MSR_K8_GS_BASE: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_GS); break;
13749	case MSR_K6_EFER: /* Nothing to do, already handled above. */ break;
13750	default:
13751	{
13752	if (hmR0VmxIsAutoLoadGuestMsr(pVmcsInfo, idMsr))
13753	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
13754	else if (hmR0VmxIsLazyGuestMsr(pVCpu, idMsr))
13755	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_VMX_GUEST_LAZY_MSRS);
13756	break;
13757	}
13758	}
13759	}
13760	#ifdef VBOX_STRICT
13761	else
13762	{
13763	/* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
13764	switch (idMsr)
13765	{
13766	case MSR_IA32_SYSENTER_CS:
13767	case MSR_IA32_SYSENTER_EIP:
13768	case MSR_IA32_SYSENTER_ESP:
13769	case MSR_K8_FS_BASE:
13770	case MSR_K8_GS_BASE:
13771	{
13772	uint32_t fMsrpm = HMGetVmxMsrPermission(pVmcsInfo->pvMsrBitmap, idMsr);
13773	Assert(fMsrpm == (VMXMSRPM_ALLOW_RD \| VMXMSRPM_ALLOW_WR));
13774
13775	uint32_t u32Proc;
13776	rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Proc);
13777	AssertRC(rc);
13778	Assert(u32Proc == pVmcsInfo->u32ProcCtls);
13779	Assert(u32Proc & VMX_PROC_CTLS_USE_MSR_BITMAPS);
13780
13781	AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32 fMsrpm=%#RX32\n", idMsr, fMsrpm));
13782	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
13783	}
13784
13785	/* Writes to MSRs in auto-load/store area/swapped MSRs, shouldn't cause VM-exits with MSR-bitmaps. */
13786	default:
13787	{
13788	if (hmR0VmxIsAutoLoadGuestMsr(pVmcsInfo, idMsr))
13789	{
13790	/* EFER MSR writes are always intercepted. */
13791	if (idMsr != MSR_K6_EFER)
13792	{
13793	AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
13794	idMsr));
13795	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
13796	}
13797	}
13798
13799	if (hmR0VmxIsLazyGuestMsr(pVCpu, idMsr))
13800	{
13801	Assert(pVmcsInfo->pvMsrBitmap);
13802	uint32_t fMsrpm = HMGetVmxMsrPermission(pVmcsInfo->pvMsrBitmap, idMsr);
13803	if (fMsrpm & VMXMSRPM_ALLOW_WR)
13804	{
13805	AssertMsgFailed(("Unexpected WRMSR for passthru, lazy-restore MSR. ecx=%#RX32\n", idMsr));
13806	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
13807	}
13808	}
13809	break;
13810	}
13811	}
13812	}
13813	#endif /* VBOX_STRICT */
13814	}
13815	else if (rcStrict == VINF_IEM_RAISED_XCPT)
13816	{
13817	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
13818	rcStrict = VINF_SUCCESS;
13819	}
13820	else
13821	AssertMsg(rcStrict == VINF_CPUM_R3_MSR_WRITE, ("Unexpected IEMExecDecodedWrmsr rc (%Rrc)\n", VBOXSTRICTRC_VAL(rcStrict)));
13822
13823	return rcStrict;
13824	}
13825
13826
13827	/**
13828	* VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
13829	*/
13830	HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13831	{
13832	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13833	/** @todo The guest has likely hit a contended spinlock. We might want to
13834	* poke a schedule different guest VCPU. */
13835	return VINF_EM_RAW_INTERRUPT;
13836	}
13837
13838
13839	/**
13840	* VM-exit handler for when the TPR value is lowered below the specified
13841	* threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
13842	*/
13843	HMVMX_EXIT_NSRC_DECL hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13844	{
13845	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13846	Assert(pVmxTransient->pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW);
13847
13848	/*
13849	* The TPR shadow would've been synced with the APIC TPR in the post-run phase.
13850	* We'll re-evaluate pending interrupts and inject them before the next VM
13851	* entry so we can just continue execution here.
13852	*/
13853	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
13854	return VINF_SUCCESS;
13855	}
13856
13857
13858	/**
13859	* VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
13860	* VM-exit.
13861	*
13862	* @retval VINF_SUCCESS when guest execution can continue.
13863	* @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
13864	* @retval VERR_EM_INTERPRETER when something unexpected happened, fallback to
13865	* interpreter.
13866	*/
13867	HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13868	{
13869	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13870	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
13871
13872	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
13873	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
13874	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13875	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
13876	AssertRCReturn(rc, rc);
13877
13878	VBOXSTRICTRC rcStrict;
13879	PVM pVM = pVCpu->CTX_SUFF(pVM);
13880	RTGCUINTPTR const uExitQual = pVmxTransient->uExitQual;
13881	uint32_t const uAccessType = VMX_EXIT_QUAL_CRX_ACCESS(uExitQual);
13882	switch (uAccessType)
13883	{
13884	case VMX_EXIT_QUAL_CRX_ACCESS_WRITE: /* MOV to CRx */
13885	{
13886	uint32_t const uOldCr0 = pVCpu->cpum.GstCtx.cr0;
13887	rcStrict = IEMExecDecodedMovCRxWrite(pVCpu, pVmxTransient->cbInstr, VMX_EXIT_QUAL_CRX_REGISTER(uExitQual),
13888	VMX_EXIT_QUAL_CRX_GENREG(uExitQual));
13889	AssertMsg( rcStrict == VINF_SUCCESS
13890	\|\| rcStrict == VINF_IEM_RAISED_XCPT
13891	\|\| rcStrict == VINF_PGM_SYNC_CR3, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
13892
13893	switch (VMX_EXIT_QUAL_CRX_REGISTER(uExitQual))
13894	{
13895	case 0:
13896	{
13897	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged,
13898	HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR0);
13899	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR0Write);
13900	Log4Func(("CR0 write rcStrict=%Rrc CR0=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cr0));
13901
13902	/*
13903	* This is a kludge for handling switches back to real mode when we try to use
13904	* V86 mode to run real mode code directly. Problem is that V86 mode cannot
13905	* deal with special selector values, so we have to return to ring-3 and run
13906	* there till the selector values are V86 mode compatible.
13907	*
13908	* Note! Using VINF_EM_RESCHEDULE_REM here rather than VINF_EM_RESCHEDULE since the
13909	* latter is an alias for VINF_IEM_RAISED_XCPT which is converted to VINF_SUCCESs
13910	* at the end of this function.
13911	*/
13912	if ( rc == VINF_SUCCESS
13913	&& !pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest
13914	&& CPUMIsGuestInRealModeEx(&pVCpu->cpum.GstCtx)
13915	&& (uOldCr0 & X86_CR0_PE)
13916	&& !(pVCpu->cpum.GstCtx.cr0 & X86_CR0_PE) )
13917	{
13918	/** @todo check selectors rather than returning all the time. */
13919	Log4Func(("CR0 write, back to real mode -> VINF_EM_RESCHEDULE_REM\n"));
13920	rcStrict = VINF_EM_RESCHEDULE_REM;
13921	}
13922	break;
13923	}
13924
13925	case 2:
13926	{
13927	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR2Write);
13928	/* Nothing to do here, CR2 it's not part of the VMCS. */
13929	break;
13930	}
13931
13932	case 3:
13933	{
13934	Assert( !pVM->hm.s.fNestedPaging
13935	\|\| !CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx)
13936	\|\| pVCpu->hm.s.fUsingDebugLoop);
13937	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR3Write);
13938	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged,
13939	HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR3);
13940	Log4Func(("CR3 write rcStrict=%Rrc CR3=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cr3));
13941	break;
13942	}
13943
13944	case 4:
13945	{
13946	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR4Write);
13947	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged,
13948	HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR4);
13949	Log4Func(("CR4 write rc=%Rrc CR4=%#RX64 fLoadSaveGuestXcr0=%u\n", VBOXSTRICTRC_VAL(rcStrict),
13950	pVCpu->cpum.GstCtx.cr4, pVCpu->hm.s.fLoadSaveGuestXcr0));
13951	break;
13952	}
13953
13954	case 8:
13955	{
13956	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR8Write);
13957	Assert(!(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW));
13958	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged,
13959	HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_APIC_TPR);
13960	break;
13961	}
13962	default:
13963	AssertMsgFailed(("Invalid CRx register %#x\n", VMX_EXIT_QUAL_CRX_REGISTER(uExitQual)));
13964	break;
13965	}
13966	break;
13967	}
13968
13969	case VMX_EXIT_QUAL_CRX_ACCESS_READ: /* MOV from CRx */
13970	{
13971	Assert( !pVM->hm.s.fNestedPaging
13972	\|\| !CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx)
13973	\|\| pVCpu->hm.s.fUsingDebugLoop
13974	\|\| VMX_EXIT_QUAL_CRX_REGISTER(uExitQual) != 3);
13975	/* CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
13976	Assert( VMX_EXIT_QUAL_CRX_REGISTER(uExitQual) != 8
13977	\|\| !(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW));
13978
13979	rcStrict = IEMExecDecodedMovCRxRead(pVCpu, pVmxTransient->cbInstr, VMX_EXIT_QUAL_CRX_GENREG(uExitQual),
13980	VMX_EXIT_QUAL_CRX_REGISTER(uExitQual));
13981	AssertMsg( rcStrict == VINF_SUCCESS
13982	\|\| rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
13983	#ifdef VBOX_WITH_STATISTICS
13984	switch (VMX_EXIT_QUAL_CRX_REGISTER(uExitQual))
13985	{
13986	case 0: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR0Read); break;
13987	case 2: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR2Read); break;
13988	case 3: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR3Read); break;
13989	case 4: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR4Read); break;
13990	case 8: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR8Read); break;
13991	}
13992	#endif
13993	Log4Func(("CR%d Read access rcStrict=%Rrc\n", VMX_EXIT_QUAL_CRX_REGISTER(uExitQual),
13994	VBOXSTRICTRC_VAL(rcStrict)));
13995	if (VMX_EXIT_QUAL_CRX_GENREG(uExitQual) == X86_GREG_xSP)
13996	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_RSP);
13997	else
13998	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
13999	break;
14000	}
14001
14002	case VMX_EXIT_QUAL_CRX_ACCESS_CLTS: /* CLTS (Clear Task-Switch Flag in CR0) */
14003	{
14004	rcStrict = IEMExecDecodedClts(pVCpu, pVmxTransient->cbInstr);
14005	AssertMsg( rcStrict == VINF_SUCCESS
14006	\|\| rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
14007
14008	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR0);
14009	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
14010	Log4Func(("CLTS rcStrict=%d\n", VBOXSTRICTRC_VAL(rcStrict)));
14011	break;
14012	}
14013
14014	case VMX_EXIT_QUAL_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
14015	{
14016	/* Note! LMSW cannot clear CR0.PE, so no fRealOnV86Active kludge needed here. */
14017	rc = hmR0VmxReadGuestLinearAddrVmcs(pVCpu, pVmxTransient);
14018	AssertRCReturn(rc, rc);
14019	rcStrict = IEMExecDecodedLmsw(pVCpu, pVmxTransient->cbInstr, VMX_EXIT_QUAL_CRX_LMSW_DATA(uExitQual),
14020	pVmxTransient->uGuestLinearAddr);
14021	AssertMsg( rcStrict == VINF_SUCCESS
14022	\|\| rcStrict == VINF_IEM_RAISED_XCPT
14023	, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
14024
14025	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_CR0);
14026	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
14027	Log4Func(("LMSW rcStrict=%d\n", VBOXSTRICTRC_VAL(rcStrict)));
14028	break;
14029	}
14030
14031	default:
14032	AssertMsgFailedReturn(("Invalid access-type in Mov CRx VM-exit qualification %#x\n", uAccessType),
14033	VERR_VMX_UNEXPECTED_EXCEPTION);
14034	}
14035
14036	Assert( (pVCpu->hm.s.fCtxChanged & (HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS))
14037	== (HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS));
14038	if (rcStrict == VINF_IEM_RAISED_XCPT)
14039	{
14040	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
14041	rcStrict = VINF_SUCCESS;
14042	}
14043
14044	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
14045	NOREF(pVM);
14046	return rcStrict;
14047	}
14048
14049
14050	/**
14051	* VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
14052	* VM-exit.
14053	*/
14054	HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14055	{
14056	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14057	STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
14058
14059	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
14060	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14061	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
14062	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14063	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK \| CPUMCTX_EXTRN_SREG_MASK
14064	\| CPUMCTX_EXTRN_EFER);
14065	/* EFER MSR also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
14066	AssertRCReturn(rc, rc);
14067
14068	/* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
14069	uint32_t uIOPort = VMX_EXIT_QUAL_IO_PORT(pVmxTransient->uExitQual);
14070	uint8_t uIOWidth = VMX_EXIT_QUAL_IO_WIDTH(pVmxTransient->uExitQual);
14071	bool fIOWrite = (VMX_EXIT_QUAL_IO_DIRECTION(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_IO_DIRECTION_OUT);
14072	bool fIOString = VMX_EXIT_QUAL_IO_IS_STRING(pVmxTransient->uExitQual);
14073	bool fGstStepping = RT_BOOL(pCtx->eflags.Bits.u1TF);
14074	bool fDbgStepping = pVCpu->hm.s.fSingleInstruction;
14075	AssertReturn(uIOWidth <= 3 && uIOWidth != 2, VERR_VMX_IPE_1);
14076
14077	/*
14078	* Update exit history to see if this exit can be optimized.
14079	*/
14080	VBOXSTRICTRC rcStrict;
14081	PCEMEXITREC pExitRec = NULL;
14082	if ( !fGstStepping
14083	&& !fDbgStepping)
14084	pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
14085	!fIOString
14086	? !fIOWrite
14087	? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_READ)
14088	: EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_WRITE)
14089	: !fIOWrite
14090	? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_STR_READ)
14091	: EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_IO_PORT_STR_WRITE),
14092	pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
14093	if (!pExitRec)
14094	{
14095	/* I/O operation lookup arrays. */
14096	static uint32_t const s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses. */
14097	static uint32_t const s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving result in AL/AX/EAX. */
14098	uint32_t const cbValue = s_aIOSizes[uIOWidth];
14099	uint32_t const cbInstr = pVmxTransient->cbInstr;
14100	bool fUpdateRipAlready = false; /* ugly hack, should be temporary. */
14101	PVM pVM = pVCpu->CTX_SUFF(pVM);
14102	if (fIOString)
14103	{
14104	/*
14105	* INS/OUTS - I/O String instruction.
14106	*
14107	* Use instruction-information if available, otherwise fall back on
14108	* interpreting the instruction.
14109	*/
14110	Log4Func(("cs:rip=%#04x:%#RX64 %#06x/%u %c str\n", pCtx->cs.Sel, pCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
14111	AssertReturn(pCtx->dx == uIOPort, VERR_VMX_IPE_2);
14112	bool const fInsOutsInfo = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_INS_OUTS);
14113	if (fInsOutsInfo)
14114	{
14115	int rc2 = hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
14116	AssertRCReturn(rc2, rc2);
14117	AssertReturn(pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize <= 2, VERR_VMX_IPE_3);
14118	AssertCompile(IEMMODE_16BIT == 0 && IEMMODE_32BIT == 1 && IEMMODE_64BIT == 2);
14119	IEMMODE const enmAddrMode = (IEMMODE)pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize;
14120	bool const fRep = VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQual);
14121	if (fIOWrite)
14122	rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, fRep, cbInstr,
14123	pVmxTransient->ExitInstrInfo.StrIo.iSegReg, true /fIoChecked/);
14124	else
14125	{
14126	/*
14127	* The segment prefix for INS cannot be overridden and is always ES. We can safely assume X86_SREG_ES.
14128	* Hence "iSegReg" field is undefined in the instruction-information field in VT-x for INS.
14129	* See Intel Instruction spec. for "INS".
14130	* See Intel spec. Table 27-8 "Format of the VM-Exit Instruction-Information Field as Used for INS and OUTS".
14131	*/
14132	rcStrict = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, fRep, cbInstr, true /fIoChecked/);
14133	}
14134	}
14135	else
14136	rcStrict = IEMExecOne(pVCpu);
14137
14138	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
14139	fUpdateRipAlready = true;
14140	}
14141	else
14142	{
14143	/*
14144	* IN/OUT - I/O instruction.
14145	*/
14146	Log4Func(("cs:rip=%04x:%08RX64 %#06x/%u %c\n", pCtx->cs.Sel, pCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
14147	uint32_t const uAndVal = s_aIOOpAnd[uIOWidth];
14148	Assert(!VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQual));
14149	if (fIOWrite)
14150	{
14151	rcStrict = IOMIOPortWrite(pVM, pVCpu, uIOPort, pCtx->eax & uAndVal, cbValue);
14152	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
14153	if ( rcStrict == VINF_IOM_R3_IOPORT_WRITE
14154	&& !pCtx->eflags.Bits.u1TF)
14155	rcStrict = EMRZSetPendingIoPortWrite(pVCpu, uIOPort, cbInstr, cbValue, pCtx->eax & uAndVal);
14156	}
14157	else
14158	{
14159	uint32_t u32Result = 0;
14160	rcStrict = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbValue);
14161	if (IOM_SUCCESS(rcStrict))
14162	{
14163	/* Save result of I/O IN instr. in AL/AX/EAX. */
14164	pCtx->eax = (pCtx->eax & ~uAndVal) \| (u32Result & uAndVal);
14165	}
14166	if ( rcStrict == VINF_IOM_R3_IOPORT_READ
14167	&& !pCtx->eflags.Bits.u1TF)
14168	rcStrict = EMRZSetPendingIoPortRead(pVCpu, uIOPort, cbInstr, cbValue);
14169	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
14170	}
14171	}
14172
14173	if (IOM_SUCCESS(rcStrict))
14174	{
14175	if (!fUpdateRipAlready)
14176	{
14177	hmR0VmxAdvanceGuestRipBy(pVCpu, cbInstr);
14178	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
14179	}
14180
14181	/*
14182	* INS/OUTS with REP prefix updates RFLAGS, can be observed with triple-fault guru
14183	* while booting Fedora 17 64-bit guest.
14184	*
14185	* See Intel Instruction reference for REP/REPE/REPZ/REPNE/REPNZ.
14186	*/
14187	if (fIOString)
14188	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RFLAGS);
14189
14190	/*
14191	* If any I/O breakpoints are armed, we need to check if one triggered
14192	* and take appropriate action.
14193	* Note that the I/O breakpoint type is undefined if CR4.DE is 0.
14194	*/
14195	rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_DR7);
14196	AssertRCReturn(rc, rc);
14197
14198	/** @todo Optimize away the DBGFBpIsHwIoArmed call by having DBGF tell the
14199	* execution engines about whether hyper BPs and such are pending. */
14200	uint32_t const uDr7 = pCtx->dr[7];
14201	if (RT_UNLIKELY( ( (uDr7 & X86_DR7_ENABLED_MASK)
14202	&& X86_DR7_ANY_RW_IO(uDr7)
14203	&& (pCtx->cr4 & X86_CR4_DE))
14204	\|\| DBGFBpIsHwIoArmed(pVM)))
14205	{
14206	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
14207
14208	/* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
14209	VMMRZCallRing3Disable(pVCpu);
14210	HM_DISABLE_PREEMPT(pVCpu);
14211
14212	bool fIsGuestDbgActive = CPUMR0DebugStateMaybeSaveGuest(pVCpu, true /* fDr6 */);
14213
14214	VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, pCtx, uIOPort, cbValue);
14215	if (rcStrict2 == VINF_EM_RAW_GUEST_TRAP)
14216	{
14217	/* Raise #DB. */
14218	if (fIsGuestDbgActive)
14219	ASMSetDR6(pCtx->dr[6]);
14220	if (pCtx->dr[7] != uDr7)
14221	pVCpu->hm.s.fCtxChanged \|= HM_CHANGED_GUEST_DR7;
14222
14223	hmR0VmxSetPendingXcptDB(pVCpu);
14224	}
14225	/* rcStrict is VINF_SUCCESS, VINF_IOM_R3_IOPORT_COMMIT_WRITE, or in [VINF_EM_FIRST..VINF_EM_LAST],
14226	however we can ditch VINF_IOM_R3_IOPORT_COMMIT_WRITE as it has VMCPU_FF_IOM as backup. */
14227	else if ( rcStrict2 != VINF_SUCCESS
14228	&& (rcStrict == VINF_SUCCESS \|\| rcStrict2 < rcStrict))
14229	rcStrict = rcStrict2;
14230	AssertCompile(VINF_EM_LAST < VINF_IOM_R3_IOPORT_COMMIT_WRITE);
14231
14232	HM_RESTORE_PREEMPT();
14233	VMMRZCallRing3Enable(pVCpu);
14234	}
14235	}
14236
14237	#ifdef VBOX_STRICT
14238	if ( rcStrict == VINF_IOM_R3_IOPORT_READ
14239	\|\| rcStrict == VINF_EM_PENDING_R3_IOPORT_READ)
14240	Assert(!fIOWrite);
14241	else if ( rcStrict == VINF_IOM_R3_IOPORT_WRITE
14242	\|\| rcStrict == VINF_IOM_R3_IOPORT_COMMIT_WRITE
14243	\|\| rcStrict == VINF_EM_PENDING_R3_IOPORT_WRITE)
14244	Assert(fIOWrite);
14245	else
14246	{
14247	# if 0 /** @todo r=bird: This is missing a bunch of VINF_EM_FIRST..VINF_EM_LAST
14248	* statuses, that the VMM device and some others may return. See
14249	* IOM_SUCCESS() for guidance. */
14250	AssertMsg( RT_FAILURE(rcStrict)
14251	\|\| rcStrict == VINF_SUCCESS
14252	\|\| rcStrict == VINF_EM_RAW_EMULATE_INSTR
14253	\|\| rcStrict == VINF_EM_DBG_BREAKPOINT
14254	\|\| rcStrict == VINF_EM_RAW_GUEST_TRAP
14255	\|\| rcStrict == VINF_EM_RAW_TO_R3
14256	\|\| rcStrict == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
14257	# endif
14258	}
14259	#endif
14260	STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
14261	}
14262	else
14263	{
14264	/*
14265	* Frequent exit or something needing probing. Get state and call EMHistoryExec.
14266	*/
14267	int rc2 = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
14268	AssertRCReturn(rc2, rc2);
14269	STAM_COUNTER_INC(!fIOString ? fIOWrite ? &pVCpu->hm.s.StatExitIOWrite : &pVCpu->hm.s.StatExitIORead
14270	: fIOWrite ? &pVCpu->hm.s.StatExitIOStringWrite : &pVCpu->hm.s.StatExitIOStringRead);
14271	Log4(("IOExit/%u: %04x:%08RX64: %s%s%s %#x LB %u -> EMHistoryExec\n",
14272	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
14273	VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQual) ? "REP " : "",
14274	fIOWrite ? "OUT" : "IN", fIOString ? "S" : "", uIOPort, uIOWidth));
14275
14276	rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
14277	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
14278
14279	Log4(("IOExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
14280	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
14281	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
14282	}
14283	return rcStrict;
14284	}
14285
14286
14287	/**
14288	* VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
14289	* VM-exit.
14290	*/
14291	HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14292	{
14293	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14294
14295	/* Check if this task-switch occurred while delivery an event through the guest IDT. */
14296	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
14297	AssertRCReturn(rc, rc);
14298	if (VMX_EXIT_QUAL_TASK_SWITCH_TYPE(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_TASK_SWITCH_TYPE_IDT)
14299	{
14300	rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
14301	AssertRCReturn(rc, rc);
14302	if (VMX_IDT_VECTORING_INFO_IS_VALID(pVmxTransient->uIdtVectoringInfo))
14303	{
14304	uint32_t uErrCode;
14305	RTGCUINTPTR GCPtrFaultAddress;
14306	uint32_t const uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
14307	uint32_t const uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
14308	bool const fErrorCodeValid = VMX_IDT_VECTORING_INFO_IS_ERROR_CODE_VALID(pVmxTransient->uIdtVectoringInfo);
14309	if (fErrorCodeValid)
14310	{
14311	rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
14312	AssertRCReturn(rc, rc);
14313	uErrCode = pVmxTransient->uIdtVectoringErrorCode;
14314	}
14315	else
14316	uErrCode = 0;
14317
14318	if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
14319	&& uVector == X86_XCPT_PF)
14320	GCPtrFaultAddress = pVCpu->cpum.GstCtx.cr2;
14321	else
14322	GCPtrFaultAddress = 0;
14323
14324	rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14325	AssertRCReturn(rc, rc);
14326
14327	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
14328	pVmxTransient->cbInstr, uErrCode, GCPtrFaultAddress);
14329
14330	Log4Func(("Pending event. uIntType=%#x uVector=%#x\n", uIntType, uVector));
14331	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
14332	return VINF_EM_RAW_INJECT_TRPM_EVENT;
14333	}
14334	}
14335
14336	/* Fall back to the interpreter to emulate the task-switch. */
14337	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
14338	return VERR_EM_INTERPRETER;
14339	}
14340
14341
14342	/**
14343	* VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
14344	*/
14345	HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14346	{
14347	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14348
14349	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14350	pVmcsInfo->u32ProcCtls &= ~VMX_PROC_CTLS_MONITOR_TRAP_FLAG;
14351	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
14352	AssertRCReturn(rc, rc);
14353	return VINF_EM_DBG_STEPPED;
14354	}
14355
14356
14357	/**
14358	* VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
14359	*/
14360	HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14361	{
14362	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14363	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
14364
14365	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
14366	VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
14367	if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
14368	{
14369	/* For some crazy guest, if an event delivery causes an APIC-access VM-exit, go to instruction emulation. */
14370	if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
14371	{
14372	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
14373	return VINF_EM_RAW_INJECT_TRPM_EVENT;
14374	}
14375	}
14376	else
14377	{
14378	if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
14379	rcStrict1 = VINF_SUCCESS;
14380	return rcStrict1;
14381	}
14382
14383	/* IOMMIOPhysHandler() below may call into IEM, save the necessary state. */
14384	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14385	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
14386	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
14387	AssertRCReturn(rc, rc);
14388
14389	/* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
14390	uint32_t uAccessType = VMX_EXIT_QUAL_APIC_ACCESS_TYPE(pVmxTransient->uExitQual);
14391	VBOXSTRICTRC rcStrict2;
14392	switch (uAccessType)
14393	{
14394	case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
14395	case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
14396	{
14397	AssertMsg( !(pVmcsInfo->u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
14398	\|\| VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQual) != XAPIC_OFF_TPR,
14399	("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
14400
14401	RTGCPHYS GCPhys = pVCpu->hm.s.vmx.u64GstMsrApicBase; /* Always up-to-date, as it is not part of the VMCS. */
14402	GCPhys &= PAGE_BASE_GC_MASK;
14403	GCPhys += VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQual);
14404	PVM pVM = pVCpu->CTX_SUFF(pVM);
14405	Log4Func(("Linear access uAccessType=%#x GCPhys=%#RGp Off=%#x\n", uAccessType, GCPhys,
14406	VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQual)));
14407
14408	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
14409	rcStrict2 = IOMMMIOPhysHandler(pVM, pVCpu,
14410	uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ ? 0 : X86_TRAP_PF_RW,
14411	CPUMCTX2CORE(pCtx), GCPhys);
14412	Log4Func(("IOMMMIOPhysHandler returned %Rrc\n", VBOXSTRICTRC_VAL(rcStrict2)));
14413	if ( rcStrict2 == VINF_SUCCESS
14414	\|\| rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
14415	\|\| rcStrict2 == VERR_PAGE_NOT_PRESENT)
14416	{
14417	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
14418	\| HM_CHANGED_GUEST_APIC_TPR);
14419	rcStrict2 = VINF_SUCCESS;
14420	}
14421	break;
14422	}
14423
14424	default:
14425	Log4Func(("uAccessType=%#x\n", uAccessType));
14426	rcStrict2 = VINF_EM_RAW_EMULATE_INSTR;
14427	break;
14428	}
14429
14430	if (rcStrict2 != VINF_SUCCESS)
14431	STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchApicAccessToR3);
14432	return rcStrict2;
14433	}
14434
14435
14436	/**
14437	* VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
14438	* VM-exit.
14439	*/
14440	HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14441	{
14442	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14443
14444	/* We should -not- get this VM-exit if the guest's debug registers were active. */
14445	if (pVmxTransient->fWasGuestDebugStateActive)
14446	{
14447	AssertMsgFailed(("Unexpected MOV DRx exit\n"));
14448	HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
14449	}
14450
14451	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14452	if ( !pVCpu->hm.s.fSingleInstruction
14453	&& !pVmxTransient->fWasHyperDebugStateActive)
14454	{
14455	Assert(!DBGFIsStepping(pVCpu));
14456	Assert(pVmcsInfo->u32XcptBitmap & RT_BIT(X86_XCPT_DB));
14457
14458	/* Don't intercept MOV DRx any more. */
14459	pVmcsInfo->u32ProcCtls &= ~VMX_PROC_CTLS_MOV_DR_EXIT;
14460	int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVmcsInfo->u32ProcCtls);
14461	AssertRCReturn(rc, rc);
14462
14463	/* We're playing with the host CPU state here, make sure we can't preempt or longjmp. */
14464	VMMRZCallRing3Disable(pVCpu);
14465	HM_DISABLE_PREEMPT(pVCpu);
14466
14467	/* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
14468	CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
14469	Assert(CPUMIsGuestDebugStateActive(pVCpu) \|\| HC_ARCH_BITS == 32);
14470
14471	HM_RESTORE_PREEMPT();
14472	VMMRZCallRing3Enable(pVCpu);
14473
14474	#ifdef VBOX_WITH_STATISTICS
14475	rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
14476	AssertRCReturn(rc, rc);
14477	if (VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_DRX_DIRECTION_WRITE)
14478	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
14479	else
14480	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
14481	#endif
14482	STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
14483	return VINF_SUCCESS;
14484	}
14485
14486	/*
14487	* EMInterpretDRx[Write\|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER MSR, CS.
14488	* The EFER MSR is always up-to-date.
14489	* Update the segment registers and DR7 from the CPU.
14490	*/
14491	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
14492	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
14493	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, CPUMCTX_EXTRN_SREG_MASK \| CPUMCTX_EXTRN_DR7);
14494	AssertRCReturn(rc, rc);
14495	Log4Func(("cs:rip=%#04x:%#RX64\n", pCtx->cs.Sel, pCtx->rip));
14496
14497	PVM pVM = pVCpu->CTX_SUFF(pVM);
14498	if (VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQual) == VMX_EXIT_QUAL_DRX_DIRECTION_WRITE)
14499	{
14500	rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pCtx),
14501	VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQual),
14502	VMX_EXIT_QUAL_DRX_GENREG(pVmxTransient->uExitQual));
14503	if (RT_SUCCESS(rc))
14504	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_DR7);
14505	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
14506	}
14507	else
14508	{
14509	rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pCtx),
14510	VMX_EXIT_QUAL_DRX_GENREG(pVmxTransient->uExitQual),
14511	VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQual));
14512	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
14513	}
14514
14515	Assert(rc == VINF_SUCCESS \|\| rc == VERR_EM_INTERPRETER);
14516	if (RT_SUCCESS(rc))
14517	{
14518	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
14519	AssertRCReturn(rc2, rc2);
14520	return VINF_SUCCESS;
14521	}
14522	return rc;
14523	}
14524
14525
14526	/**
14527	* VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
14528	* Conditional VM-exit.
14529	*/
14530	HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14531	{
14532	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14533	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
14534
14535	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
14536	VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
14537	if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
14538	{
14539	/* If event delivery causes an EPT misconfig (MMIO), go back to instruction emulation as otherwise
14540	injecting the original pending event would most likely cause the same EPT misconfig VM-exit. */
14541	if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
14542	{
14543	STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
14544	return VINF_EM_RAW_INJECT_TRPM_EVENT;
14545	}
14546	}
14547	else
14548	{
14549	if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
14550	rcStrict1 = VINF_SUCCESS;
14551	return rcStrict1;
14552	}
14553
14554	/*
14555	* Get sufficent state and update the exit history entry.
14556	*/
14557	RTGCPHYS GCPhys;
14558	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14559	int rc = VMXReadVmcs64(VMX_VMCS64_RO_GUEST_PHYS_ADDR_FULL, &GCPhys);
14560	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
14561	AssertRCReturn(rc, rc);
14562
14563	VBOXSTRICTRC rcStrict;
14564	PCEMEXITREC pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
14565	EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM \| EMEXIT_F_HM, EMEXITTYPE_MMIO),
14566	pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
14567	if (!pExitRec)
14568	{
14569	/*
14570	* If we succeed, resume guest execution.
14571	* If we fail in interpreting the instruction because we couldn't get the guest physical address
14572	* of the page containing the instruction via the guest's page tables (we would invalidate the guest page
14573	* in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
14574	* weird case. See @bugref{6043}.
14575	*/
14576	PVM pVM = pVCpu->CTX_SUFF(pVM);
14577	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
14578	rcStrict = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pCtx), GCPhys, UINT32_MAX);
14579	Log4Func(("At %#RGp RIP=%#RX64 rc=%Rrc\n", GCPhys, pCtx->rip, VBOXSTRICTRC_VAL(rcStrict)));
14580	if ( rcStrict == VINF_SUCCESS
14581	\|\| rcStrict == VERR_PAGE_TABLE_NOT_PRESENT
14582	\|\| rcStrict == VERR_PAGE_NOT_PRESENT)
14583	{
14584	/* Successfully handled MMIO operation. */
14585	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS
14586	\| HM_CHANGED_GUEST_APIC_TPR);
14587	rcStrict = VINF_SUCCESS;
14588	}
14589	}
14590	else
14591	{
14592	/*
14593	* Frequent exit or something needing probing. Get state and call EMHistoryExec.
14594	*/
14595	int rc2 = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
14596	AssertRCReturn(rc2, rc2);
14597
14598	Log4(("EptMisscfgExit/%u: %04x:%08RX64: %RGp -> EMHistoryExec\n",
14599	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, GCPhys));
14600
14601	rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
14602	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
14603
14604	Log4(("EptMisscfgExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
14605	pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
14606	VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
14607	}
14608	return VBOXSTRICTRC_TODO(rcStrict);
14609	}
14610
14611
14612	/**
14613	* VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
14614	* VM-exit.
14615	*/
14616	HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14617	{
14618	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14619	Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
14620
14621	/* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
14622	VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
14623	if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
14624	{
14625	/* In the unlikely case that the EPT violation happened as a result of delivering an event, log it. */
14626	if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
14627	Log4Func(("EPT violation with an event pending u64IntInfo=%#RX64\n", pVCpu->hm.s.Event.u64IntInfo));
14628	}
14629	else
14630	{
14631	if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
14632	rcStrict1 = VINF_SUCCESS;
14633	return rcStrict1;
14634	}
14635
14636	RTGCPHYS GCPhys;
14637	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14638	int rc = VMXReadVmcs64(VMX_VMCS64_RO_GUEST_PHYS_ADDR_FULL, &GCPhys);
14639	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
14640	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, IEM_CPUMCTX_EXTRN_MUST_MASK);
14641	AssertRCReturn(rc, rc);
14642
14643	/* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
14644	AssertMsg(((pVmxTransient->uExitQual >> 7) & 3) != 2, ("%#RX64", pVmxTransient->uExitQual));
14645
14646	RTGCUINT uErrorCode = 0;
14647	if (pVmxTransient->uExitQual & VMX_EXIT_QUAL_EPT_INSTR_FETCH)
14648	uErrorCode \|= X86_TRAP_PF_ID;
14649	if (pVmxTransient->uExitQual & VMX_EXIT_QUAL_EPT_DATA_WRITE)
14650	uErrorCode \|= X86_TRAP_PF_RW;
14651	if (pVmxTransient->uExitQual & VMX_EXIT_QUAL_EPT_ENTRY_PRESENT)
14652	uErrorCode \|= X86_TRAP_PF_P;
14653
14654	TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
14655
14656
14657	/* Handle the pagefault trap for the nested shadow table. */
14658	PVM pVM = pVCpu->CTX_SUFF(pVM);
14659	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
14660
14661	Log4Func(("EPT violation %#x at %#RX64 ErrorCode %#x cs:rip=%#04x:%#RX64\n", pVmxTransient->uExitQual, GCPhys, uErrorCode,
14662	pCtx->cs.Sel, pCtx->rip));
14663
14664	VBOXSTRICTRC rcStrict2 = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pCtx), GCPhys);
14665	TRPMResetTrap(pVCpu);
14666
14667	/* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
14668	if ( rcStrict2 == VINF_SUCCESS
14669	\|\| rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
14670	\|\| rcStrict2 == VERR_PAGE_NOT_PRESENT)
14671	{
14672	/* Successfully synced our nested page tables. */
14673	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
14674	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RSP \| HM_CHANGED_GUEST_RFLAGS);
14675	return VINF_SUCCESS;
14676	}
14677
14678	Log4Func(("EPT return to ring-3 rcStrict2=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict2)));
14679	return rcStrict2;
14680	}
14681
14682	/** @} */
14683
14684	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
14685	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= VM-exit exception handlers =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
14686	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
14687
14688	/**
14689	* VM-exit exception handler for \#MF (Math Fault: floating point exception).
14690	*/
14691	static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14692	{
14693	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14694	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
14695
14696	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CR0);
14697	AssertRCReturn(rc, rc);
14698
14699	if (!(pVCpu->cpum.GstCtx.cr0 & X86_CR0_NE))
14700	{
14701	/* Convert a #MF into a FERR -> IRQ 13. See @bugref{6117}. */
14702	rc = PDMIsaSetIrq(pVCpu->CTX_SUFF(pVM), 13, 1, 0 /* uTagSrc */);
14703
14704	/** @todo r=ramshankar: The Intel spec. does -not- specify that this VM-exit
14705	* provides VM-exit instruction length. If this causes problem later,
14706	* disassemble the instruction like it's done on AMD-V. */
14707	int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
14708	AssertRCReturn(rc2, rc2);
14709	return rc;
14710	}
14711
14712	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
14713	pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
14714	return rc;
14715	}
14716
14717
14718	/**
14719	* VM-exit exception handler for \#BP (Breakpoint exception).
14720	*/
14721	static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14722	{
14723	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14724	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
14725
14726	int rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
14727	AssertRCReturn(rc, rc);
14728
14729	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
14730	rc = DBGFRZTrap03Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx));
14731	if (rc == VINF_EM_RAW_GUEST_TRAP)
14732	{
14733	rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
14734	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14735	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
14736	AssertRCReturn(rc, rc);
14737
14738	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
14739	pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
14740	}
14741
14742	Assert(rc == VINF_SUCCESS \|\| rc == VINF_EM_RAW_GUEST_TRAP \|\| rc == VINF_EM_DBG_BREAKPOINT);
14743	return rc;
14744	}
14745
14746
14747	/**
14748	* VM-exit exception handler for \#AC (alignment check exception).
14749	*/
14750	static int hmR0VmxExitXcptAC(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14751	{
14752	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14753
14754	/*
14755	* Re-inject it. We'll detect any nesting before getting here.
14756	*/
14757	int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
14758	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14759	AssertRCReturn(rc, rc);
14760	Assert(ASMAtomicUoReadU32(&pVmxTransient->fVmcsFieldsRead) & HMVMX_READ_EXIT_INTERRUPTION_INFO);
14761
14762	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
14763	pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
14764	return VINF_SUCCESS;
14765	}
14766
14767
14768	/**
14769	* VM-exit exception handler for \#DB (Debug exception).
14770	*/
14771	static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14772	{
14773	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14774	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
14775
14776	/*
14777	* Get the DR6-like values from the VM-exit qualification and pass it to DBGF
14778	* for processing.
14779	*/
14780	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
14781
14782	/* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
14783	uint64_t uDR6 = X86_DR6_INIT_VAL;
14784	uDR6 \|= (pVmxTransient->uExitQual & (X86_DR6_B0 \| X86_DR6_B1 \| X86_DR6_B2 \| X86_DR6_B3 \| X86_DR6_BD \| X86_DR6_BS));
14785
14786	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
14787	rc = DBGFRZTrap01Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx), uDR6, pVCpu->hm.s.fSingleInstruction);
14788	Log6Func(("rc=%Rrc\n", rc));
14789	if (rc == VINF_EM_RAW_GUEST_TRAP)
14790	{
14791	/*
14792	* The exception was for the guest. Update DR6, DR7.GD and
14793	* IA32_DEBUGCTL.LBR before forwarding it.
14794	* (See Intel spec. 27.1 "Architectural State before a VM-Exit".)
14795	*/
14796	VMMRZCallRing3Disable(pVCpu);
14797	HM_DISABLE_PREEMPT(pVCpu);
14798
14799	pCtx->dr[6] &= ~X86_DR6_B_MASK;
14800	pCtx->dr[6] \|= uDR6;
14801	if (CPUMIsGuestDebugStateActive(pVCpu))
14802	ASMSetDR6(pCtx->dr[6]);
14803
14804	HM_RESTORE_PREEMPT();
14805	VMMRZCallRing3Enable(pVCpu);
14806
14807	rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_DR7);
14808	AssertRCReturn(rc, rc);
14809
14810	/* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
14811	pCtx->dr[7] &= ~X86_DR7_GD;
14812
14813	/* Paranoia. */
14814	pCtx->dr[7] &= ~X86_DR7_RAZ_MASK;
14815	pCtx->dr[7] \|= X86_DR7_RA1_MASK;
14816
14817	rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)pCtx->dr[7]);
14818	AssertRCReturn(rc, rc);
14819
14820	/*
14821	* Raise #DB in the guest.
14822	*
14823	* It is important to reflect exactly what the VM-exit gave us (preserving the
14824	* interruption-type) rather than use hmR0VmxSetPendingXcptDB() as the #DB could've
14825	* been raised while executing ICEBP (INT1) and not the regular #DB. Thus it may
14826	* trigger different handling in the CPU (like skipping DPL checks), see @bugref{6398}.
14827	*
14828	* Intel re-documented ICEBP/INT1 on May 2018 previously documented as part of
14829	* Intel 386, see Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
14830	*/
14831	rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
14832	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14833	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
14834	AssertRCReturn(rc, rc);
14835	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
14836	pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
14837	return VINF_SUCCESS;
14838	}
14839
14840	/*
14841	* Not a guest trap, must be a hypervisor related debug event then.
14842	* Update DR6 in case someone is interested in it.
14843	*/
14844	AssertMsg(rc == VINF_EM_DBG_STEPPED \|\| rc == VINF_EM_DBG_BREAKPOINT, ("%Rrc\n", rc));
14845	AssertReturn(pVmxTransient->fWasHyperDebugStateActive, VERR_HM_IPE_5);
14846	CPUMSetHyperDR6(pVCpu, uDR6);
14847
14848	return rc;
14849	}
14850
14851
14852	/**
14853	* Hacks its way around the lovely mesa driver's backdoor accesses.
14854	*
14855	* @sa hmR0SvmHandleMesaDrvGp
14856	*/
14857	static int hmR0VmxHandleMesaDrvGp(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PCPUMCTX pCtx)
14858	{
14859	LogFunc(("cs:rip=%#04x:%#RX64 rcx=%#RX64 rbx=%#RX64\n", pCtx->cs.Sel, pCtx->rip, pCtx->rcx, pCtx->rbx));
14860	RT_NOREF(pCtx);
14861
14862	/* For now we'll just skip the instruction. */
14863	return hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
14864	}
14865
14866
14867	/**
14868	* Checks if the \#GP'ing instruction is the mesa driver doing it's lovely
14869	* backdoor logging w/o checking what it is running inside.
14870	*
14871	* This recognizes an "IN EAX,DX" instruction executed in flat ring-3, with the
14872	* backdoor port and magic numbers loaded in registers.
14873	*
14874	* @returns true if it is, false if it isn't.
14875	* @sa hmR0SvmIsMesaDrvGp
14876	*/
14877	DECLINLINE(bool) hmR0VmxIsMesaDrvGp(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PCPUMCTX pCtx)
14878	{
14879	/* 0xed: IN eAX,dx */
14880	uint8_t abInstr[1];
14881	if (pVmxTransient->cbInstr != sizeof(abInstr))
14882	return false;
14883
14884	/* Check that it is #GP(0). */
14885	if (pVmxTransient->uExitIntErrorCode != 0)
14886	return false;
14887
14888	/* Check magic and port. */
14889	Assert(!(pCtx->fExtrn & (CPUMCTX_EXTRN_RAX \| CPUMCTX_EXTRN_RDX \| CPUMCTX_EXTRN_RCX)));
14890	/Log(("hmR0VmxIsMesaDrvGp: rax=%RX64 rdx=%RX64\n", pCtx->rax, pCtx->rdx));/
14891	if (pCtx->rax != UINT32_C(0x564d5868))
14892	return false;
14893	if (pCtx->dx != UINT32_C(0x5658))
14894	return false;
14895
14896	/* Flat ring-3 CS. */
14897	AssertCompile(HMVMX_CPUMCTX_EXTRN_ALL & CPUMCTX_EXTRN_CS);
14898	Assert(!(pCtx->fExtrn & CPUMCTX_EXTRN_CS));
14899	/Log(("hmR0VmxIsMesaDrvGp: cs.Attr.n.u2Dpl=%d base=%Rx64\n", pCtx->cs.Attr.n.u2Dpl, pCtx->cs.u64Base));/
14900	if (pCtx->cs.Attr.n.u2Dpl != 3)
14901	return false;
14902	if (pCtx->cs.u64Base != 0)
14903	return false;
14904
14905	/* Check opcode. */
14906	AssertCompile(HMVMX_CPUMCTX_EXTRN_ALL & CPUMCTX_EXTRN_RIP);
14907	Assert(!(pCtx->fExtrn & CPUMCTX_EXTRN_RIP));
14908	int rc = PGMPhysSimpleReadGCPtr(pVCpu, abInstr, pCtx->rip, sizeof(abInstr));
14909	/Log(("hmR0VmxIsMesaDrvGp: PGMPhysSimpleReadGCPtr -> %Rrc %#x\n", rc, abInstr[0]));/
14910	if (RT_FAILURE(rc))
14911	return false;
14912	if (abInstr[0] != 0xed)
14913	return false;
14914
14915	return true;
14916	}
14917
14918
14919	/**
14920	* VM-exit exception handler for \#GP (General-protection exception).
14921	*
14922	* @remarks Requires pVmxTransient->uExitIntInfo to be up-to-date.
14923	*/
14924	static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
14925	{
14926	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
14927	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
14928
14929	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
14930	PVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
14931	if (pVmcsInfo->RealMode.fRealOnV86Active)
14932	{ /* likely */ }
14933	else
14934	{
14935	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
14936	Assert(pVCpu->hm.s.fUsingDebugLoop \|\| pVCpu->hm.s.fTrapXcptGpForLovelyMesaDrv);
14937	#endif
14938	/* If the guest is not in real-mode or we have unrestricted guest execution support, reflect #GP to the guest. */
14939	int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
14940	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
14941	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
14942	rc \|= hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
14943	AssertRCReturn(rc, rc);
14944	Log4Func(("Gst: cs:rip=%#04x:%#RX64 ErrorCode=%#x cr0=%#RX64 cpl=%u tr=%#04x\n", pCtx->cs.Sel, pCtx->rip,
14945	pVmxTransient->uExitIntErrorCode, pCtx->cr0, CPUMGetGuestCPL(pVCpu), pCtx->tr.Sel));
14946
14947	if ( !pVCpu->hm.s.fTrapXcptGpForLovelyMesaDrv
14948	\|\| !hmR0VmxIsMesaDrvGp(pVCpu, pVmxTransient, pCtx))
14949	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
14950	pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
14951	else
14952	rc = hmR0VmxHandleMesaDrvGp(pVCpu, pVmxTransient, pCtx);
14953	return rc;
14954	}
14955
14956	Assert(CPUMIsGuestInRealModeEx(pCtx));
14957	Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
14958
14959	int rc = hmR0VmxImportGuestState(pVCpu, pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
14960	AssertRCReturn(rc, rc);
14961
14962	VBOXSTRICTRC rcStrict = IEMExecOne(pVCpu);
14963	if (rcStrict == VINF_SUCCESS)
14964	{
14965	if (!CPUMIsGuestInRealModeEx(pCtx))
14966	{
14967	/*
14968	* The guest is no longer in real-mode, check if we can continue executing the
14969	* guest using hardware-assisted VMX. Otherwise, fall back to emulation.
14970	*/
14971	pVmcsInfo->RealMode.fRealOnV86Active = false;
14972	if (HMCanExecuteVmxGuest(pVCpu, pCtx))
14973	{
14974	Log4Func(("Mode changed but guest still suitable for executing using hardware-assisted VMX\n"));
14975	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
14976	}
14977	else
14978	{
14979	Log4Func(("Mode changed -> VINF_EM_RESCHEDULE\n"));
14980	rcStrict = VINF_EM_RESCHEDULE;
14981	}
14982	}
14983	else
14984	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
14985	}
14986	else if (rcStrict == VINF_IEM_RAISED_XCPT)
14987	{
14988	rcStrict = VINF_SUCCESS;
14989	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
14990	}
14991	return VBOXSTRICTRC_VAL(rcStrict);
14992	}
14993
14994
14995	/**
14996	* VM-exit exception handler wrapper for generic exceptions. Simply re-injects
14997	* the exception reported in the VMX transient structure back into the VM.
14998	*
14999	* @remarks Requires uExitIntInfo in the VMX transient structure to be
15000	* up-to-date.
15001	*/
15002	static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15003	{
15004	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15005	#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
15006	PCVMXVMCSINFO pVmcsInfo = pVmxTransient->pVmcsInfo;
15007	AssertMsg(pVCpu->hm.s.fUsingDebugLoop \|\| pVmcsInfo->RealMode.fRealOnV86Active,
15008	("uVector=%#x u32XcptBitmap=%#X32\n",
15009	VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo), pVmcsInfo->u32XcptBitmap));
15010	NOREF(pVmcsInfo);
15011	#endif
15012
15013	/* Re-inject the exception into the guest. This cannot be a double-fault condition which would have been handled in
15014	hmR0VmxCheckExitDueToEventDelivery(). */
15015	int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
15016	rc \|= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15017	AssertRCReturn(rc, rc);
15018	Assert(ASMAtomicUoReadU32(&pVmxTransient->fVmcsFieldsRead) & HMVMX_READ_EXIT_INTERRUPTION_INFO);
15019
15020	#ifdef DEBUG_ramshankar
15021	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CS \| CPUMCTX_EXTRN_RIP);
15022	Log(("hmR0VmxExitXcptGeneric: Reinjecting Xcpt. uVector=%#x cs:rip=%#04x:%#RX64\n",
15023	VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo), pCtx->cs.Sel, pCtx->rip));
15024	#endif
15025
15026	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), pVmxTransient->cbInstr,
15027	pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
15028	return VINF_SUCCESS;
15029	}
15030
15031
15032	/**
15033	* VM-exit exception handler for \#PF (Page-fault exception).
15034	*/
15035	static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15036	{
15037	HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15038	PVM pVM = pVCpu->CTX_SUFF(pVM);
15039	int rc = hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15040	rc \|= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
15041	rc \|= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
15042	AssertRCReturn(rc, rc);
15043
15044	if (!pVM->hm.s.fNestedPaging)
15045	{ /* likely */ }
15046	else
15047	{
15048	#if !defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) && !defined(HMVMX_ALWAYS_TRAP_PF)
15049	Assert(pVCpu->hm.s.fUsingDebugLoop);
15050	#endif
15051	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
15052	if (RT_LIKELY(!pVmxTransient->fVectoringDoublePF))
15053	{
15054	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), 0 /* cbInstr */,
15055	pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQual);
15056	}
15057	else
15058	{
15059	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
15060	hmR0VmxSetPendingXcptDF(pVCpu);
15061	Log4Func(("Pending #DF due to vectoring #PF w/ NestedPaging\n"));
15062	}
15063	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
15064	return rc;
15065	}
15066
15067	/* If it's a vectoring #PF, emulate injecting the original event injection as PGMTrap0eHandler() is incapable
15068	of differentiating between instruction emulation and event injection that caused a #PF. See @bugref{6607}. */
15069	if (pVmxTransient->fVectoringPF)
15070	{
15071	Assert(pVCpu->hm.s.Event.fPending);
15072	return VINF_EM_RAW_INJECT_TRPM_EVENT;
15073	}
15074
15075	PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
15076	rc = hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
15077	AssertRCReturn(rc, rc);
15078
15079	Log4Func(("#PF: cr2=%#RX64 cs:rip=%#04x:%#RX64 uErrCode %#RX32 cr3=%#RX64\n", pVmxTransient->uExitQual, pCtx->cs.Sel,
15080	pCtx->rip, pVmxTransient->uExitIntErrorCode, pCtx->cr3));
15081
15082	TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQual, (RTGCUINT)pVmxTransient->uExitIntErrorCode);
15083	rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntErrorCode, CPUMCTX2CORE(pCtx), (RTGCPTR)pVmxTransient->uExitQual);
15084
15085	Log4Func(("#PF: rc=%Rrc\n", rc));
15086	if (rc == VINF_SUCCESS)
15087	{
15088	/*
15089	* This is typically a shadow page table sync or a MMIO instruction. But we may have
15090	* emulated something like LTR or a far jump. Any part of the CPU context may have changed.
15091	*/
15092	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
15093	TRPMResetTrap(pVCpu);
15094	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
15095	return rc;
15096	}
15097
15098	if (rc == VINF_EM_RAW_GUEST_TRAP)
15099	{
15100	if (!pVmxTransient->fVectoringDoublePF)
15101	{
15102	/* It's a guest page fault and needs to be reflected to the guest. */
15103	uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
15104	TRPMResetTrap(pVCpu);
15105	pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
15106	hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo), 0 /* cbInstr */,
15107	uGstErrorCode, pVmxTransient->uExitQual);
15108	}
15109	else
15110	{
15111	/* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
15112	TRPMResetTrap(pVCpu);
15113	pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF to replace it with #DF. */
15114	hmR0VmxSetPendingXcptDF(pVCpu);
15115	Log4Func(("#PF: Pending #DF due to vectoring #PF\n"));
15116	}
15117
15118	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
15119	return VINF_SUCCESS;
15120	}
15121
15122	TRPMResetTrap(pVCpu);
15123	STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
15124	return rc;
15125	}
15126
15127	#ifdef VBOX_WITH_NESTED_HWVIRT_VMX
15128	/** @name VMX instruction handlers.
15129	* @{
15130	*/
15131	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
15132	/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VMX instructions VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
15133	/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
15134
15135	/**
15136	* VM-exit handler for VMCLEAR (VMX_EXIT_VMCLEAR). Unconditional VM-exit.
15137	*/
15138	HMVMX_EXIT_DECL hmR0VmxExitVmclear(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15139	{
15140	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15141
15142	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15143	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
15144	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
15145	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
15146	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15147	AssertRCReturn(rc, rc);
15148
15149	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15150
15151	VMXVEXITINFO ExitInfo;
15152	RT_ZERO(ExitInfo);
15153	ExitInfo.uReason = pVmxTransient->uExitReason;
15154	ExitInfo.u64Qual = pVmxTransient->uExitQual;
15155	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
15156	ExitInfo.cbInstr = pVmxTransient->cbInstr;
15157	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
15158
15159	VBOXSTRICTRC rcStrict = IEMExecDecodedVmclear(pVCpu, &ExitInfo);
15160	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15161	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_HWVIRT);
15162	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15163	{
15164	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15165	rcStrict = VINF_SUCCESS;
15166	}
15167	return rcStrict;
15168	}
15169
15170
15171	/**
15172	* VM-exit handler for VMLAUNCH (VMX_EXIT_VMLAUNCH). Unconditional VM-exit.
15173	*/
15174	HMVMX_EXIT_DECL hmR0VmxExitVmlaunch(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15175	{
15176	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15177
15178	/* Import the entire VMCS state for now as we would be switching VMCS on successful VMLAUNCH,
15179	otherwise we could import just IEM_CPUMCTX_EXTRN_VMX_VMENTRY_MASK. */
15180	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15181	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
15182	AssertRCReturn(rc, rc);
15183
15184	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15185
15186	VBOXSTRICTRC rcStrict = IEMExecDecodedVmlaunchVmresume(pVCpu, pVmxTransient->cbInstr, VMXINSTRID_VMLAUNCH);
15187	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15188	{
15189	rcStrict = VINF_VMX_VMLAUNCH_VMRESUME;
15190	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
15191	}
15192	Assert(rcStrict != VINF_IEM_RAISED_XCPT);
15193	return rcStrict;
15194	}
15195
15196
15197	/**
15198	* VM-exit handler for VMPTRLD (VMX_EXIT_VMPTRLD). Unconditional VM-exit.
15199	*/
15200	HMVMX_EXIT_DECL hmR0VmxExitVmptrld(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15201	{
15202	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15203
15204	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15205	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
15206	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
15207	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
15208	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15209	AssertRCReturn(rc, rc);
15210
15211	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15212
15213	VMXVEXITINFO ExitInfo;
15214	RT_ZERO(ExitInfo);
15215	ExitInfo.uReason = pVmxTransient->uExitReason;
15216	ExitInfo.u64Qual = pVmxTransient->uExitQual;
15217	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
15218	ExitInfo.cbInstr = pVmxTransient->cbInstr;
15219	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
15220
15221	VBOXSTRICTRC rcStrict = IEMExecDecodedVmptrld(pVCpu, &ExitInfo);
15222	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15223	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_HWVIRT);
15224	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15225	{
15226	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15227	rcStrict = VINF_SUCCESS;
15228	}
15229	return rcStrict;
15230	}
15231
15232
15233	/**
15234	* VM-exit handler for VMPTRST (VMX_EXIT_VMPTRST). Unconditional VM-exit.
15235	*/
15236	HMVMX_EXIT_DECL hmR0VmxExitVmptrst(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15237	{
15238	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15239
15240	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15241	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
15242	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
15243	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
15244	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15245	AssertRCReturn(rc, rc);
15246
15247	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15248
15249	VMXVEXITINFO ExitInfo;
15250	RT_ZERO(ExitInfo);
15251	ExitInfo.uReason = pVmxTransient->uExitReason;
15252	ExitInfo.u64Qual = pVmxTransient->uExitQual;
15253	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
15254	ExitInfo.cbInstr = pVmxTransient->cbInstr;
15255	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_WRITE, &ExitInfo.GCPtrEffAddr);
15256
15257	VBOXSTRICTRC rcStrict = IEMExecDecodedVmptrst(pVCpu, &ExitInfo);
15258	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15259	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
15260	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15261	{
15262	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15263	rcStrict = VINF_SUCCESS;
15264	}
15265	return rcStrict;
15266	}
15267
15268
15269	/**
15270	* VM-exit handler for VMREAD (VMX_EXIT_VMREAD). Unconditional VM-exit.
15271	*/
15272	HMVMX_EXIT_DECL hmR0VmxExitVmread(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15273	{
15274	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15275
15276	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15277	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
15278	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
15279	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
15280	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15281	AssertRCReturn(rc, rc);
15282
15283	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15284
15285	VMXVEXITINFO ExitInfo;
15286	RT_ZERO(ExitInfo);
15287	ExitInfo.uReason = pVmxTransient->uExitReason;
15288	ExitInfo.u64Qual = pVmxTransient->uExitQual;
15289	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
15290	ExitInfo.cbInstr = pVmxTransient->cbInstr;
15291	if (!ExitInfo.InstrInfo.VmreadVmwrite.fIsRegOperand)
15292	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_WRITE, &ExitInfo.GCPtrEffAddr);
15293
15294	VBOXSTRICTRC rcStrict = IEMExecDecodedVmread(pVCpu, &ExitInfo);
15295	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15296	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS);
15297	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15298	{
15299	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15300	rcStrict = VINF_SUCCESS;
15301	}
15302	return rcStrict;
15303	}
15304
15305
15306	/**
15307	* VM-exit handler for VMRESUME (VMX_EXIT_VMRESUME). Unconditional VM-exit.
15308	*/
15309	HMVMX_EXIT_DECL hmR0VmxExitVmresume(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15310	{
15311	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15312
15313	/* Import the entire VMCS state for now as we would be switching VMCS on successful VMRESUME,
15314	otherwise we could import just IEM_CPUMCTX_EXTRN_VMX_VMENTRY_MASK. */
15315	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15316	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, HMVMX_CPUMCTX_EXTRN_ALL);
15317	AssertRCReturn(rc, rc);
15318
15319	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15320
15321	VBOXSTRICTRC rcStrict = IEMExecDecodedVmlaunchVmresume(pVCpu, pVmxTransient->cbInstr, VMXINSTRID_VMRESUME);
15322	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15323	{
15324	rcStrict = VINF_VMX_VMLAUNCH_VMRESUME;
15325	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
15326	}
15327	Assert(rcStrict != VINF_IEM_RAISED_XCPT);
15328	return rcStrict;
15329	}
15330
15331
15332	/**
15333	* VM-exit handler for VMWRITE (VMX_EXIT_VMWRITE). Unconditional VM-exit.
15334	*/
15335	HMVMX_EXIT_DECL hmR0VmxExitVmwrite(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15336	{
15337	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15338
15339	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15340	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
15341	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
15342	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
15343	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15344	AssertRCReturn(rc, rc);
15345
15346	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15347
15348	VMXVEXITINFO ExitInfo;
15349	RT_ZERO(ExitInfo);
15350	ExitInfo.uReason = pVmxTransient->uExitReason;
15351	ExitInfo.u64Qual = pVmxTransient->uExitQual;
15352	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
15353	ExitInfo.cbInstr = pVmxTransient->cbInstr;
15354	if (!ExitInfo.InstrInfo.VmreadVmwrite.fIsRegOperand)
15355	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
15356
15357	VBOXSTRICTRC rcStrict = IEMExecDecodedVmwrite(pVCpu, &ExitInfo);
15358	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15359	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_HWVIRT);
15360	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15361	{
15362	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15363	rcStrict = VINF_SUCCESS;
15364	}
15365	return rcStrict;
15366	}
15367
15368
15369	/**
15370	* VM-exit handler for VMXOFF (VMX_EXIT_VMXOFF). Unconditional VM-exit.
15371	*/
15372	HMVMX_EXIT_DECL hmR0VmxExitVmxoff(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15373	{
15374	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15375
15376	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15377	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_CR4
15378	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK);
15379	AssertRCReturn(rc, rc);
15380
15381	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15382
15383	VBOXSTRICTRC rcStrict = IEMExecDecodedVmxoff(pVCpu, pVmxTransient->cbInstr);
15384	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15385	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_HWVIRT);
15386	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15387	{
15388	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15389	rcStrict = VINF_SUCCESS;
15390	}
15391	return rcStrict;
15392	}
15393
15394
15395	/**
15396	* VM-exit handler for VMXON (VMX_EXIT_VMXON). Unconditional VM-exit.
15397	*/
15398	HMVMX_EXIT_DECL hmR0VmxExitVmxon(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
15399	{
15400	HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
15401
15402	int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
15403	rc \|= hmR0VmxImportGuestState(pVCpu, pVmxTransient->pVmcsInfo, CPUMCTX_EXTRN_RSP \| CPUMCTX_EXTRN_SREG_MASK
15404	\| IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK);
15405	rc \|= hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
15406	rc \|= hmR0VmxReadExitQualVmcs(pVCpu, pVmxTransient);
15407	AssertRCReturn(rc, rc);
15408
15409	HMVMX_CHECK_EXIT_DUE_TO_VMX_INSTR(pVCpu, pVmxTransient->uExitReason);
15410
15411	VMXVEXITINFO ExitInfo;
15412	RT_ZERO(ExitInfo);
15413	ExitInfo.uReason = pVmxTransient->uExitReason;
15414	ExitInfo.u64Qual = pVmxTransient->uExitQual;
15415	ExitInfo.InstrInfo.u = pVmxTransient->ExitInstrInfo.u;
15416	ExitInfo.cbInstr = pVmxTransient->cbInstr;
15417	HMVMX_DECODE_MEM_OPERAND(pVCpu, ExitInfo.InstrInfo.u, ExitInfo.u64Qual, VMXMEMACCESS_READ, &ExitInfo.GCPtrEffAddr);
15418
15419	VBOXSTRICTRC rcStrict = IEMExecDecodedVmxon(pVCpu, &ExitInfo);
15420	if (RT_LIKELY(rcStrict == VINF_SUCCESS))
15421	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP \| HM_CHANGED_GUEST_RFLAGS \| HM_CHANGED_GUEST_HWVIRT);
15422	else if (rcStrict == VINF_IEM_RAISED_XCPT)
15423	{
15424	ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
15425	rcStrict = VINF_SUCCESS;
15426	}
15427	return rcStrict;
15428	}
15429
15430	/** @} */
15431	#endif /* VBOX_WITH_NESTED_HWVIRT_VMX */
15432

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 78232

Download in other formats: